<commit_before>#!/usr/bin/env python3
import json
import urllib
import urllib.request

from html.parser import HTMLParser

# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.strict = False
        self.convert_charrefs = True
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

class Api(object):

    def parse_comment(self, html):
        """Strip most HTML tags."""
        # Indent comment string
        html = html.replace("<br>", '\n    ')
        html = html.replace("&gt;", '>')
        html = html.replace("&quot;", '"')

        # After preserving some tags, strip all of them
        s = MLStripper()
        s.feed(html)

        return "    " + s.get_data() # Indent first line of comment
    
    def get_threads(self, board, page=1):
        """Return first threads by their first post, board and page."""
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/" + str(page) + ".json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread list\n" + str(error)

        if data:
            data = json.loads(data)
            for post in data["threads"]:
                result += str(post["posts"][0]["no"]) + " " + post["posts"][0]["now"] + "\n"
                result += self.parse_comment(post["posts"][0]["com"] + "\n\n")

        return result

    def get_thread(self, id):
        """Get particular thread by id."""
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/thread/" + str(id) + ".json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread\n" + str(error)

        return result

    def get_archive(self, board):
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/archive.json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread list\n" + str(error)

        return result<commit_msg>Fix wrong display of thread comments<commit_after>#!/usr/bin/env python3
import json
import urllib
import urllib.request

from html.parser import HTMLParser

# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python
class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.strict = False
        self.convert_charrefs = True
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

class Api(object):

    def parse_comment(self, html):
        """Strip most HTML tags."""
        # Indent comment string
        html = html.replace("<br>", '\n    ')
        html = html.replace("&gt;", '>')
        html = html.replace("&quot;", '"')

        # After preserving some tags, strip all of them
        s = MLStripper()
        s.feed(html)

        return "    " + s.get_data() # Indent first line of comment
    
    def get_threads(self, board, page=1):
        """Return first threads by their first post, board and page."""
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/" + str(page) + ".json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread list\n" + str(error)

        if data:
            data = json.loads(data)
            for post in data["threads"]:
                result += "\n\nNo. " + str(post["posts"][0]["no"]) + " " + post["posts"][0]["now"] + "\n"
                result += self.parse_comment(post["posts"][0]["com"])

        return result

    def get_thread(self, id):
        """Get particular thread by id."""
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/thread/" + str(id) + ".json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread\n" + str(error)

        return result

    def get_archive(self, board):
        data = None
        result = ""

        try:
            data = urllib.request.urlopen("https://a.4cdn.org/" + board + "/archive.json").read().decode("utf-8")
        except urllib.error.HTTPError as error:
            return "Could not generate thread list\n" + str(error)

        return result
{'fraction_non_alphanumeric': 0.14969299174253653, 'fraction_numerical': 0.007198814312936693, 'mean_word_length': 2.111989459815547, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4561975', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1431, 'n_words': 350}