<commit_before>from wordfreq import simple_tokenize, tokenize
from collections import defaultdict
from operator import itemgetter
from ftfy import fix_text
import math
import csv
import msgpack
import gzip
import regex


# Match common cases of URLs: the schema http:// or https:// followed by
# non-whitespace characters.
URL_RE = regex.compile(r'https?://(?:\S)+')
HAN_RE = regex.compile(r'[\p{Script=Han}]+')


def count_tokens(filename):
    """
    Count tokens that appear in a file, running each line through our
    simple tokenizer.

    URLs will be skipped, and Unicode errors will become separate tokens
    containing '�'.
    """
    counts = defaultdict(int)
    with open(filename, encoding='utf-8', errors='replace') as infile:
        for line in infile:
            line = URL_RE.sub('', line.strip())
            for token in simple_tokenize(line):
                counts[token] += 1

    return counts


def read_values(filename, cutoff=0, lang=None):
    """
    Read words and their frequency or count values from a CSV file. Returns
    a dictionary of values and the total of all values.

    Only words with a value greater than or equal to `cutoff` are returned.

    If `cutoff` is greater than 0, the csv file must be sorted by value
    in descending order.
    """
    values = defaultdict(float)
    total = 0.
    with open(filename, encoding='utf-8', newline='') as infile:
        for key, strval in csv.reader(infile):
            val = float(strval)
            key = fix_text(key)
            if val < cutoff:
                break
            tokens = tokenize(key, lang) if lang is not None else simple_tokenize(key)
            for token in tokens:
                # Use += so that, if we give the reader concatenated files with
                # duplicates, it does the right thing
                values[token] += val
                total += val
    return values, total


def read_freqs(filename, cutoff=0, lang=None):
    """
    Read words and their frequencies from a CSV file, normalizing the
    frequencies to add up to 1.

    Only words with a frequency greater than or equal to `cutoff` are returned.

    If `cutoff` is greater than 0, the csv file must be sorted by frequency
    in descending order.

    If lang is given, read_freqs will apply language specific preprocessing
    operations.
    """
    values, total = read_values(filename, cutoff, lang)
    for word in values:
        values[word] /= total

    if lang == 'en':
        return correct_apostrophe_trimming(values)


def freqs_to_cBpack(in_filename, out_filename, cutoff=-600, lang=None):
    """
    Convert a csv file of words and their frequencies to a file in the
    idiosyncratic 'cBpack' format.

    Only words with a frequency greater than `cutoff` centibels will be
    written to the new file.

    This cutoff should not be stacked with a cutoff in `read_freqs`; doing
    so would skew the resulting frequencies.
    """
    freqs = read_freqs(in_filename, cutoff=0, lang=lang)
    cBpack = []
    for token, freq in freqs.items():
        cB = round(math.log10(freq) * 100)
        if cB <= cutoff:
            continue
        neg_cB = -cB
        while neg_cB >= len(cBpack):
            cBpack.append([])
        cBpack[neg_cB].append(token)

    for sublist in cBpack:
        sublist.sort()<commit_msg>actually fix logic of apostrophe-fixing<commit_after>from wordfreq import simple_tokenize, tokenize
from collections import defaultdict
from operator import itemgetter
from ftfy import fix_text
import math
import csv
import msgpack
import gzip
import regex


# Match common cases of URLs: the schema http:// or https:// followed by
# non-whitespace characters.
URL_RE = regex.compile(r'https?://(?:\S)+')
HAN_RE = regex.compile(r'[\p{Script=Han}]+')


def count_tokens(filename):
    """
    Count tokens that appear in a file, running each line through our
    simple tokenizer.

    URLs will be skipped, and Unicode errors will become separate tokens
    containing '�'.
    """
    counts = defaultdict(int)
    with open(filename, encoding='utf-8', errors='replace') as infile:
        for line in infile:
            line = URL_RE.sub('', line.strip())
            for token in simple_tokenize(line):
                counts[token] += 1

    return counts


def read_values(filename, cutoff=0, lang=None):
    """
    Read words and their frequency or count values from a CSV file. Returns
    a dictionary of values and the total of all values.

    Only words with a value greater than or equal to `cutoff` are returned.

    If `cutoff` is greater than 0, the csv file must be sorted by value
    in descending order.
    """
    values = defaultdict(float)
    total = 0.
    with open(filename, encoding='utf-8', newline='') as infile:
        for key, strval in csv.reader(infile):
            val = float(strval)
            key = fix_text(key)
            if val < cutoff:
                break
            tokens = tokenize(key, lang) if lang is not None else simple_tokenize(key)
            for token in tokens:
                # Use += so that, if we give the reader concatenated files with
                # duplicates, it does the right thing
                values[token] += val
                total += val
    return values, total


def read_freqs(filename, cutoff=0, lang=None):
    """
    Read words and their frequencies from a CSV file, normalizing the
    frequencies to add up to 1.

    Only words with a frequency greater than or equal to `cutoff` are returned.

    If `cutoff` is greater than 0, the csv file must be sorted by frequency
    in descending order.

    If lang is given, read_freqs will apply language specific preprocessing
    operations.
    """
    values, total = read_values(filename, cutoff, lang)
    for word in values:
        values[word] /= total

    if lang == 'en':
        values = correct_apostrophe_trimming(values)

    return values


def freqs_to_cBpack(in_filename, out_filename, cutoff=-600, lang=None):
    """
    Convert a csv file of words and their frequencies to a file in the
    idiosyncratic 'cBpack' format.

    Only words with a frequency greater than `cutoff` centibels will be
    written to the new file.

    This cutoff should not be stacked with a cutoff in `read_freqs`; doing
    so would skew the resulting frequencies.
    """
    freqs = read_freqs(in_filename, cutoff=0, lang=lang)
    cBpack = []
    for token, freq in freqs.items():
        cB = round(math.log10(freq) * 100)
        if cB <= cutoff:
            continue
        neg_cB = -cB
        while neg_cB >= len(cBpack):
            cBpack.append([])
        cBpack[neg_cB].append(token)

    for sublist in cBpack:
        sublist.sort()
{'fraction_non_alphanumeric': 0.08277703604806408, 'fraction_numerical': 0.0053404539385847796, 'mean_word_length': 2.5823591923485654, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1783288', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1891, 'n_words': 761}