<commit_before>import csv
import hashlib
import os


def generate_score(column_name, column_value):
    """
    Return an int from a couple name/value.

    To be a valid score for Redis, it has to be between
    -9007199254740992 and 9007199254740992.
    See: http://redis.io/commands/zadd
    """
    # TODO: check for collisions? potential performances bottleneck?
    key = (column_name + column_value).encode('utf-8')
    base = ''.join(str(ord(c)) for c in hashlib.md5(key).hexdigest())
    return int(base[:15])


def extract_csv_filepaths(folder):
    """Walk over a `folder` to yield every CSV filepaths in it."""
    for dirpath, dirnames, filenames in os.walk(folder):
        for filename in filenames:
            if filename.endswith('.csv'):
                yield dirpath + filename


def iter_over_csv_file(csv_filepath):
    """Open and enumerate over a CSV file located at `csv_filepath`."""
    with open(csv_filepath, encoding='cp1252') as csv_file:
        for i, data in enumerate(csv.DictReader(csv_file, delimiter=';')):
            yield i, data<commit_msg>Do not require the user to enter a folder with a final slash<commit_after>import csv
import hashlib
import os


def generate_score(column_name, column_value):
    """
    Return an int from a couple name/value.

    To be a valid score for Redis, it has to be between
    -9007199254740992 and 9007199254740992.
    See: http://redis.io/commands/zadd
    """
    # TODO: check for collisions? potential performances bottleneck?
    key = (column_name + column_value).encode('utf-8')
    base = ''.join(str(ord(c)) for c in hashlib.md5(key).hexdigest())
    return int(base[:15])


def extract_csv_filepaths(folder):
    """Walk over a `folder` to yield every CSV filepaths in it."""
    for dirpath, dirnames, filenames in os.walk(folder):
        for filename in filenames:
            if filename.endswith('.csv'):
                yield os.path.join(dirpath, filename)


def iter_over_csv_file(csv_filepath):
    """Open and enumerate over a CSV file located at `csv_filepath`."""
    with open(csv_filepath, encoding='cp1252') as csv_file:
        for i, data in enumerate(csv.DictReader(csv_file, delimiter=';')):
            yield i, data
{'fraction_non_alphanumeric': 0.1087151841868823, 'fraction_numerical': 0.03593890386343217, 'mean_word_length': 3.332684824902724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7271576', 'n_tokens_mistral': 757, 'n_tokens_neox': 681, 'n_words': 221}