<commit_before>"""
data_importer.py

Manages the location of the saved .csv files. 
"""

from toolz import unique, partial
import csv
import os

# force python lazy functions to act
force = list


def open_csv(file_path, skip_head=True):
    """
    TODO: write this docstring 
    """
    f = csv.reader(open(file_path, 'r'))

    if skip_head:
        next(f)

    return f


def open_dict_csv(file_path):
    """
    TODO: write this docstring 
    """
    return csv.DictReader(open(file_path, 'r'))


def minus_key(d, k):
    """
    TODO: write this docstring
    """
    d.pop(k)
    return d


def unique_from_column(n, columns):
    """
    TODO: write this docstring 
    """
    return unique([row[n] for row in columns])

zeroth = partial(unique_from_column, 0)
firsts = partial(unique_from_column, 1)
seconds = partial(unique_from_column, 2)
thirds = partial(unique_from_column, 3)
fourths = partial(unique_from_column, 4)
fifths = partial(unique_from_column, 5)
sixths = partial(unique_from_column, 6)
sevenths = partial(unique_from_column, 7)
eighths = partial(unique_from_column, 8)
ninths = partial(unique_from_column, 9)

# This is the location of your local copy
# of our Drop box folder with the data on it
# for example on my computer it is: /home/wil/Data/Trans
# export RAD_DATA_BASE="/home/wil/Data/Trans"
BASE_DATA_DIR = os.environ['RAD_DATA_BASE']
data_dir = partial(os.path.join, BASE_DATA_DIR)
<commit_msg>Use env.get instead<commit_after>"""
data_importer.py

Manages the location of the saved .csv files. 
"""

from toolz import unique, partial
import csv
import os

# force python lazy functions to act
force = list


def open_csv(file_path, skip_head=True):
    """
    TODO: write this docstring 
    """
    f = csv.reader(open(file_path, 'r'))

    if skip_head:
        next(f)

    return f


def open_dict_csv(file_path):
    """
    TODO: write this docstring 
    """
    return csv.DictReader(open(file_path, 'r'))


def minus_key(d, k):
    """
    TODO: write this docstring
    """
    d.pop(k)
    return d


def unique_from_column(n, columns):
    """
    TODO: write this docstring 
    """
    return unique([row[n] for row in columns])

zeroth = partial(unique_from_column, 0)
firsts = partial(unique_from_column, 1)
seconds = partial(unique_from_column, 2)
thirds = partial(unique_from_column, 3)
fourths = partial(unique_from_column, 4)
fifths = partial(unique_from_column, 5)
sixths = partial(unique_from_column, 6)
sevenths = partial(unique_from_column, 7)
eighths = partial(unique_from_column, 8)
ninths = partial(unique_from_column, 9)

# This is the location of your local copy
# of our Drop box folder with the data on it
# for example on my computer it is: /home/wil/Data/Trans
# export RAD_DATA_BASE="/home/wil/Data/Trans"
BASE_DATA_DIR = os.environ.get('RAD_DATA_BASE')
data_dir = partial(os.path.join, BASE_DATA_DIR)

{'fraction_non_alphanumeric': 0.10910354412786658, 'fraction_numerical': 0.006949270326615705, 'mean_word_length': 4.006956521739131, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6328981', 'n_tokens_mistral': 1039, 'n_tokens_neox': 984, 'n_words': 276}