<jupyter_start><jupyter_text>Feature Enrichment main loop<jupyter_code>featureStats = open('TwoYearsDataEnrichment_10P_15P_20P_Pump.txt','w')
fileIndex = 0

startTime = time.time()
s3 = boto3.client('s3') 

for file_name in tqdm(csvFileList):
    print('Processing file [%s]' %file_name)
    coin_name = file_name.split('/')[-1].split('_')[0]
    
    fileIndex +=1
    
    obj = s3.get_object(Bucket = BUCKET_NAME, Key = file_name)
    df = pd.read_csv(obj['Body'], index_col='0', parse_dates=True)
    
    df.drop(['Close Time', 'Quote Asset Volume', 'Taker buy base asset volume', 'Taker buy quote asset volume', 'Ignore'], axis=1, inplace=True)
    
    insertNewFeaturesToDf(file_name, df, featureStats)
    
    df.to_csv(destS3Url + "Enriched_" + file_name.split('/')[-1] , index=True)    

featureStats.write('\nTotal elapsed time to insert all features={}\n'.format(time.time()-startTime))
featureStats.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15352260778128285, 'fraction_numerical': 0.01892744479495268, 'mean_word_length': 4.840490797546012, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '167351', 'license': 'Apache-2.0', 'n_tokens_mistral': 357, 'n_tokens_neox': 347, 'n_words': 65, 'path': 'src/EnrichTwoYearsData-10P_15P_20P_PND.ipynb', 'repo_name': 'aaivu/aaivu-pump-and-dump-detection-crypto-currencies', 'size': '951'}