<commit_before>"""
Quick & dirt script to convert a Word Docx file to HTML.

Requires pypandoc pytidylib  (see requirements.txt)

"""
import os
import sys
try:
    import pypandoc
    from tidylib import tidy_document
except ImportError:
    print("\n\nRequires pypandoc and pytidylib. See requirements.txt\n\n")


def convert_to_html(filename):
    # Do the conversion with pandoc
    output = pypandoc.convert(filename, 'html')

    # Clean up with tidy...
    output, errors = tidy_document(output,  options={
        'numeric-entities': 1,
        'wrap': 80,
    })
    print(errors)

    # replace smart quotes.
    output = output.replace(u"\u2018", '&lsquo;').replace(u"\u2019", '&rsquo;')
    output = output.replace(u"\u201c", "&ldquo;").replace(u"\u201d", "&rdquo;")

    # write the output
    filename, ext = os.path.splitext(filename)
    filename = "{0}.html".format(filename)
    with open(filename, 'w') as f:
        f.write(output.encode("UTF-8"))

    print("Done! Output written to: {}\n".format(filename))


def main():
    if len(sys.argv) == 2:
        convert_to_html(sys.argv[1])
    else:
        print("\nUSAGE: word2html <filename>\n")


if __name__ == "__main__":
    main()<commit_msg>experimentally work w/ python2 and python3<commit_after>"""
Quick & dirt script to convert a Word Docx file to HTML.

Requires pypandoc pytidylib  (see requirements.txt)

"""
import os
import sys
try:
    import pypandoc
    from tidylib import tidy_document
except ImportError:
    print("\n\nRequires pypandoc and pytidylib. See requirements.txt\n\n")


def convert_to_html(filename):
    # Do the conversion with pandoc
    output = pypandoc.convert(filename, 'html')

    # Clean up with tidy...
    output, errors = tidy_document(output,  options={
        'numeric-entities': 1,
        'wrap': 80,
    })
    print(errors)

    # replace smart quotes.
    output = output.replace(u"\u2018", '&lsquo;').replace(u"\u2019", '&rsquo;')
    output = output.replace(u"\u201c", "&ldquo;").replace(u"\u201d", "&rdquo;")

    # write the output
    filename, ext = os.path.splitext(filename)
    filename = "{0}.html".format(filename)
    with open(filename, 'w') as f:
        # Python 2 "fix". If this isn't a string, encode it.
        if type(output) is not str:
            output = output.encode('utf-8')
        f.write(output)

    print("Done! Output written to: {}\n".format(filename))


def main():
    if len(sys.argv) == 2:
        convert_to_html(sys.argv[1])
    else:
        print("\nUSAGE: word2html <filename>\n")


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.1453982985305491, 'fraction_numerical': 0.01817478731631864, 'mean_word_length': 3.1194267515923566, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4778696', 'n_tokens_mistral': 924, 'n_tokens_neox': 863, 'n_words': 227}