<jupyter_start><jupyter_text>Process Files and Convert to Parquet<jupyter_code>import pandas as pd
from pathlib import Path
import os
from tqdm.notebook import tqdm
from p_tqdm import p_map, p_umap, p_imap, p_uimap<jupyter_output><empty_output><jupyter_text>Set the base directory of the files.<jupyter_code>base_dir = Path('/media/robmulla/moardata/reddit_place2/')
def process_csv(fn):
    # fn = '2022_place_canvas_history-000000000000.csv.gzip'
    df = pd.read_csv(base_dir / fn, compression='gzip')

    df['timestamp'] = pd.to_datetime(df['timestamp'])

    df['x'] = df['coordinate'].str.split(',', expand=True)[0].astype('int')
    df['y'] = df['coordinate'].str.split(',', expand=True)[1].astype('int')
    try:
        df['x2'] = df['coordinate'].str.split(',', expand=True)[2].astype('int')
        df['y2'] = df['coordinate'].str.split(',', expand=True)[3].astype('int')
        print(f'{fn} has additional x,y points')
    except:
        pass
    df = df.drop('coordinate', axis=1)
    out_fn = base_dir / (fn.strip('csv.gzip') + '.parquet')
    df.to_parquet(out_fn)
csvs = [f for f in os.listdir(base_dir) if f.endswith('csv.gzip')]
p_map(process_csv, csvs, num_cpus=26)
print('test this 1,2,3')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.177938808373591, 'fraction_numerical': 0.023349436392914653, 'mean_word_length': 4.975961538461538, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '345970', 'license': 'Apache-2.0', 'n_tokens_mistral': 498, 'n_tokens_neox': 473, 'n_words': 85, 'path': '026-reddit-place-data/notebooks/002-parse_official.ipynb', 'repo_name': 'RobMulla/twitch-stream-projects', 'size': '1242'}