<commit_before>'''
Randomly sample from a month of the taxi data and save the resulting CSV

Run this on the server or some place where the full data is located.
'''

import os

import numpy as np
import pandas as pd


def unique_columns(dframe):
    '''
    Return the dataframe with only the unique column names
    '''

    # Find the integer indices for unique values by taking the one that
    # appears first
    seen = set()
    indices = []
    for index, name in enumerate(dframe.columns):
        if name not in seen:
            seen.add(name)
            indices.append(index)

    return dframe.iloc[:, indices]


def samplerows(dframe, size_MB, replace=False):
    '''
    Randomly sample a dataframe to be no larger than size_MB

    Note- Assumes all data is 64 bit numeric arrays
    '''

    bytesize = size_MB * 2 ** 20
    n, p = dframe.shape
    nrows = int(min(n, bytesize / (8 * p)))
    rows = np.random.choice(dframe.index.values, nrows, replace=replace)

    return dframe.loc[rows]

 
def main(size_MB, datadir):

    # Change this line to sample from other files
    filenames = ['trip_data_1.csv', 'trip_fare_1.csv']
    filenames = [os.sep.join((datadir, x)) for x in filenames]

    # Join the two dataframes
    taxi = pd.concat([pd.read_csv(x) for x in filenames], axis=1)

    # Strip out whitespace from column names
    taxi = taxi.rename(columns=lambda x: x.strip())

    taxi = unique_columns(taxi)

    taxisample = samplerows(taxi, size_MB)

    taxisample.to_csv('taxi.csv', index=False)


if __name__ == '__main__':

    datadir = '/home/data/NYCTaxis'
    datadir = '/Users/clark/projects/taxi/python/data'

    main(size_MB=500, datadir=datadir)
<commit_msg>running on macro.ucdavis.edu now<commit_after>'''
Randomly sample from a month of the taxi data and save the resulting CSV

Run this on the server or some place where the full data is located.
'''

import os

import numpy as np
import pandas as pd


def unique_columns(dframe):
    '''
    Return the dataframe with only the unique column names
    '''

    # Find the integer indices for unique values by taking the one that
    # appears first
    seen = set()
    indices = []
    for index, name in enumerate(dframe.columns):
        if name not in seen:
            seen.add(name)
            indices.append(index)

    return dframe.iloc[:, indices]


def samplerows(dframe, size_MB, replace=False):
    '''
    Randomly sample a dataframe to be no larger than size_MB

    Note- Assumes all data is 64 bit numeric arrays
    '''

    bytesize = size_MB * 2 ** 20
    n, p = dframe.shape
    nrows = int(min(n, bytesize / (8 * p)))
    rows = np.random.choice(dframe.index.values, nrows, replace=replace)

    return dframe.loc[rows]

 
def main(size_MB, taxidir, outfile):

    # Change this line to sample from other files
    filenames = ['trip_data_1.csv', 'trip_fare_1.csv']
    filenames = [os.sep.join((taxidir, x)) for x in filenames]

    # Join the two dataframes
    taxi = pd.concat([pd.read_csv(x) for x in filenames], axis=1)

    # Strip out whitespace from column names
    taxi = taxi.rename(columns=lambda x: x.strip())

    taxi = unique_columns(taxi)

    taxisample = samplerows(taxi, size_MB)

    taxisample.to_csv(outfile, index=False)


if __name__ == '__main__':

    main(size_MB=200, taxidir='data', outfile='data/taxi.csv')

{'fraction_non_alphanumeric': 0.10145789943469205, 'fraction_numerical': 0.007140731925022315, 'mean_word_length': 3.036014405762305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2678135', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1082, 'n_words': 363}