<jupyter_start><jupyter_text>[Original Challange]()<jupyter_code>!git clone https://github.com/abhishekkrthakur/csv_test.git
# shameless advertising ;)
CONNECT_WORKER = False
if CONNECT_WORKER:
    !pip install clearml-agent
    from google.colab import drive
    drive.mount("/content/drive")
    !cp /content/drive/My\ Drive/clearml.conf ~
    !clearml-agent daemon --queue colab
else:
    # will use the demo server unless configured
    !pip install clearml
    from clearml import Task
    task = Task.init('the csv challange','twitter challanges')

%cd /content/csv_test/
from pathlib import Path
files = [p for p in Path('data/').glob('*.csv')]
files.sort(key = lambda p : int(p.stem))

!pip install pandas
import pandas as pd

USE_PANDAS = False
# The pandas way - don't do it, it will crash the runtime
if USE_PANDAS:
    def pure_pandas(file_list):
        return pd.concat([pd.read_csv(f) for f in file_list])
else:
    def pure_pandas(*args):
        return None
        


%timeit pure_pandas(files[:5000])
import csv

# copy pasted form stackoverflow otherwise pip install more_itertools
def chunked(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]

def pure_csv(flist, chunksize=30000):
    with open('answer.csv',mode='w') as output:
        writer = csv.writer(output)
        # first file
        with open(flist[0],'r',newline='') as f1:
            reader = csv.reader(f1)
            rows = [r for r in reader]
        for r in rows:
            writer.writerow(r)
        # rest is chunked
        if len(flist) > chunksize:
            chunks = chunked(flist[1:], chunksize)
        else:
            chunks = [flist[1:]]
            
        for this_chunk in chunks:
            rows = []
            for f in this_chunk:
                with open(f,'r',newline='') as input:
                    reader = csv.reader(input)
                    next(reader)
                    rows.extend([r for r in reader])
            for r in rows:
                writer.writerow(r)


%timeit pure_csv(files)
ans.info()
ans = pd.read_csv('answer.csv')
del ans<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11041292639138241, 'fraction_numerical': 0.006732495511669659, 'mean_word_length': 2.2635431918008786, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '151083', 'license': 'MIT', 'n_tokens_mistral': 751, 'n_tokens_neox': 712, 'n_words': 199, 'path': 'Read_CSVs_from_Github_Pandas_Challange.ipynb', 'repo_name': 'elvinaqa/Hands-MediumON-ML', 'size': '2228'}