<commit_before>import base64
import gzip
from collections import namedtuple

Page = namedtuple(
    "Page", "url, html, text, mime_type, encoding")


def read_lett(f, slang, tlang, source_tokenizer=None, target_tokenizer=None,
              no_html=False):
    s, t = {}, {}
    fh = f
    if f.name.endswith('.gz'):
        fh = gzip.open(fh, 'r')
    for line in fh:
        lang, mine, enc, url, html, text = line.strip().split("\t")

        if no_html:
            html = ''
        else:
            html = base64.b64decode(html)

        text = base64.b64decode(text).decode("utf-8")
        # assert lang in [slang, tlang]

        if lang == slang and source_tokenizer is not None:
            text = source_tokenizer.process(text)
        elif lang == tlang and target_tokenizer is not None:
            text = target_tokenizer.process(text)
        else:
            # ignore other languages
            continue

        p = Page(url, html, text, mine, enc)
        if lang == slang:
            s[url] = p
        elif lang == tlang:
            t[url] = p
    return s, t


def write_tokenized_lett(f, pages):
    pass<commit_msg>default call to lett.py prints stats<commit_after>import base64
import gzip
import sys
from collections import namedtuple

Page = namedtuple(
    "Page", "url, html, text, mime_type, encoding")


def read_lett(f, slang, tlang, source_tokenizer=None, target_tokenizer=None,
              no_html=False):
    s, t = {}, {}
    fh = f
    if f.name.endswith('.gz'):
        fh = gzip.GzipFile(fileobj=fh, mode='r')
    for line in fh:
        lang, mine, enc, url, html, text = line[:-1].split("\t")

        if no_html:
            html = ''
        else:
            html = base64.b64decode(html)

        text = base64.b64decode(text).decode("utf-8")
        # assert lang in [slang, tlang]

        if lang == slang and source_tokenizer is not None:
            text = source_tokenizer.process(text)
        elif lang == tlang and target_tokenizer is not None:
            text = target_tokenizer.process(text)

        p = Page(url, html, text, mine, enc)
        if lang == slang:
            s[url] = p
        elif lang == tlang:
            t[url] = p
    return s, t


def write_tokenized_lett(f, pages):
    pass

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        'lettfile', help='input lett file', type=argparse.FileType('r'))
    parser.add_argument('-slang', help='source language', default='en')
    parser.add_argument('-tlang', help='target language', default='fr')
    args = parser.parse_args()

    # read source and target corpus
    s, t = read_lett(args.lettfile, args.slang, args.tlang)

    sys.stderr.write("Read %d %s docs and %d %s docs from %s\n" %
                     (len(s), args.slang,
                      len(t), args.tlang, args.lettfile.name))
    sys.stderr.write("%d possible pairs for %s\n" %
                     (len(s) * len(t), args.lettfile.name))
{'fraction_non_alphanumeric': 0.1237936772046589, 'fraction_numerical': 0.007653910149750416, 'mean_word_length': 2.2149732620320854, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '721404', 'n_tokens_mistral': 990, 'n_tokens_neox': 951, 'n_words': 265}