<commit_before>import json
import sys


def convert(source, texttype):
    if texttype == "html":
        return html2text(source)
    if texttype in ["md", "markdown"]:
        return html2text(markdown2html(source))
    if texttype in ["rst"]:
        return html2text(rst2html(source))
    if texttype == "ipynb":
        return html2text(markdown2html(ipynb2markdown(source)))
    if texttype != "txt":
        print("filetype not detected, assuming plaintext")
    return source


def html2text(html):
    try:
        from bs4 import BeautifulSoup
    except ImportError:
        notinstalled("beautifulsoup", "HTML", "text")
        sys.exit(4)
    soup = BeautifulSoup(html, "html.parser")
    # remove scripts from html
    for script in soup(["script", "style"]):
        script.extract()
    return soup.get_text()


def markdown2html(markdown):
    try:
        import markdown2
    except ImportError:
        notinstalled("markdown2", "markdown", "HTML")
        sys.exit(4)

    return markdown2.markdown(markdown)


def ipynb2markdown(ipynb):
    j = json.loads(ipynb)
    markdown = ""
    for cell in j["cells"]:
        markdown += "".join(cell["source"]) + "\n"
    return markdown


def rst2html(rst):
    try:
        from docutils.core import publish_string
    except ImportError:
        notinstalled("markdown2", "markdown", "HTML")
        sys.exit(4)
    return publish_string(rst, writer_name="html5")


def notinstalled(package, convert_from, convert_to):
    print(
        """{package} is needed to convert {source} to {target}
you can install it with pip:
pip install {package}""".format(package=package, source=convert_from, target=convert_to)
    )<commit_msg>support checking transifex json files<commit_after>import json
import sys


def convert(source, texttype):
    if texttype == "html":
        return html2text(source)
    if texttype in ["md", "markdown"]:
        return html2text(markdown2html(source))
    if texttype in ["rst"]:
        return html2text(rst2html(source))
    if texttype == "ipynb":
        return html2text(markdown2html(ipynb2markdown(source)))
    if texttype == "json":
        return transifexjson2txt(source)
    if texttype != "txt":
        print("filetype not detected, assuming plaintext")
    return source


def html2text(html):
    try:
        from bs4 import BeautifulSoup
    except ImportError:
        notinstalled("beautifulsoup", "HTML", "text")
        sys.exit(4)
    soup = BeautifulSoup(html, "html.parser")
    # remove scripts from html
    for script in soup(["script", "style"]):
        script.extract()
    return soup.get_text()


def markdown2html(markdown):
    try:
        import markdown2
    except ImportError:
        notinstalled("markdown2", "markdown", "HTML")
        sys.exit(4)

    return markdown2.markdown(markdown)


def ipynb2markdown(ipynb):
    j = json.loads(ipynb)
    markdown = ""
    for cell in j["cells"]:
        markdown += "".join(cell["source"]) + "\n"
    return markdown


def rst2html(rst):
    try:
        from docutils.core import publish_string
    except ImportError:
        notinstalled("markdown2", "markdown", "HTML")
        sys.exit(4)
    return publish_string(rst, writer_name="html5")


def transifexjson2txt(jsondata):
    data = json.loads(jsondata)
    text = ""
    for category, content in data.items():
        for key, value in content.items():
            print(value)
            text += value + "\n"
    return text

def notinstalled(package, convert_from, convert_to):
    print(
        """{package} is needed to convert {source} to {target}
you can install it with pip:
pip install {package}""".format(package=package, source=convert_from, target=convert_to)
    )
{'fraction_non_alphanumeric': 0.11902203116603977, 'fraction_numerical': 0.011821601289629231, 'mean_word_length': 3.109271523178808, 'pattern_counts': {'":': 7, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7292762', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1127, 'n_words': 300}