<jupyter_start><jupyter_text>**Utility File**<jupyter_code>%%writefile testutility.py

import logging  
import os
import subprocess
import yaml
import pandas as pd
import gc
import datetime
import re
from os.path import getsize
import gzip

################
# FILE READING #
################

def read_config_file(filepath):
  with open(filepath , 'r') as stream:
    try:
      return yaml.safe_load(stream)
    except yaml.YAMLError as exc:
      logging.error(exc)


def replacer(string , char):
  pattern = char + '{2,}'
  string = re.sub(pattern , char , string)
  return string

# This function is for removing white spaces , special characters,
# leading and trailing underscores , and replacing double characters 
# with one character

#############################################
# VALIDATION OF DATA COLUMNS WITH YAML FILE #
#############################################

def col_header_val(df , table_config):
  df.columns = df.columns.str.lower()
  df.columns = df.columns.str.replace("[^\w]" , '_' ,regex = True)
  df.columns = list(map(lambda x: x.strip('_') , list(df.columns)))
  df.columns = list(map(lambda x: replacer(x , '_') , list(df.columns)))
  expected_col = list(map(lambda x: x.lower() , table_config['columns']))
  expected_col.sort()
  df.columns = list(map(lambda x: x.lower() , list(df.columns)))
  df = df.reindex(sorted(df.columns) , axis = 1)

  if len(df.columns)== len(expected_col) and list(expected_col)== list(df.columns):
    print("column name and column length validation passed")
    return 1
  else:
    print("column name and column length validation failed")
    mismatched_columns_file = list(set(df.columns).difference(expected_col))
    print("following file columns are not in the YAML file" , mismatched_columns_file)
    missing_YAML_file = list(set(expected_col).difference(df.columns))
    print("following YAML columns are not in the file uploaded" , missing_YAML_file)
    logging.info(f"df columns:{df.columns}")
    logging.info(f"expected columns:{expected_col}")
    return 0 

def file_summary(filepath , df):
  print("size of file is {0} GB".format(round(getsize(filepath)/(1024*1024*1024) , 2)))
  print("the csv file has {0} rows".format(df.shape[0]))
  print("the csv file has {0} columns".format(len(df.columns)))

def write_file(df , root_folder , outfile , outbound_delimiter):
  outfile_name = root_folder + outfile + '.txt'
  df.to_csv(outfile_name, header=None, index=None, sep=outbound_delimiter, mode='a')
  f_in = open(outfile_name , 'rb')
  f_out = gzip.open(f'{outfile}.txt.gz', 'wb')
  f_out.writelines(f_in)
  f_out.close()
  f_in.close()<jupyter_output>Writing testutility.py<jupyter_text>**YAML Configuration File**<jupyter_code>%%writefile file.yaml
file_type: csv
dataset_name: testfile
file_name: test_data
table_name: edsurv
inbound_delimiter: ","
outbound_delimiter: "|"
skip_leading_rows: 1
root_folder: "/content/drive/MyDrive/"
columns: 
    - antiNucleus
    - event_File
    - event_Number
    - event_Time
    - hist_File
    - multiplicity
    - NaboveLb
    - NbelowLb
    - NLb
    - primary_Tracks
    - prod_Time
    - Pt 
    - run_Number
    - vertexx
    - vertexy
    - vertexz

# I saved the large file on my Google Drive
from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>**Dataset Source**<jupyter_code>#https://sdm.lbl.gov/fastbit/data/samples.html<jupyter_output><empty_output><jupyter_text>**File Columns**<jupyter_code>#the uploaded dataset does not have headers, so i had to add them manually
column_names = ['__antiNucleus' , 'event__File' , 'event__Number' , 'event__Time' , '__hist__File' , 'multiplicity' , 
                'NaboveLb' , 'NbelowLb__' , 'NLb' , 'primary__Tracks' , 'prod__Time' , '__Pt' ,
                'run__Number' , 'vertexX$$' , 'vertexY' , 'vertexZ']

import testutility as util
import time
import pandas as pd
config_data = util.read_config_file("file.yaml")
config_data<jupyter_output><empty_output><jupyter_text>**File Reading Using Parameters**<jupyter_code>file_type = config_data['file_type']
source_file = config_data['root_folder'] + config_data['file_name'] + f".{file_type}"<jupyter_output><empty_output><jupyter_text>**File Reading Methods**  **Dask Method**<jupyter_code>pip install fsspec
from dask import dataframe as dd


start = time.time()
dask_df = dd.read_csv(source_file , delimiter = config_data['inbound_delimiter']
                      , names = column_names)
end = time.time()
print("Read csv with dask took: ",(end-start),"sec")
dask_df.head(5)<jupyter_output><empty_output><jupyter_text>**Pandas Method**<jupyter_code>start = time.time()
pandas_df = pd.read_csv(source_file , delimiter = config_data['inbound_delimiter']
                      , names = column_names)
end = time.time()
print("Read csv with pandas took: ",(end-start),"sec")<jupyter_output>Read csv with pandas took:  37.546066999435425 sec<jupyter_text>**Using Pandas Chunksize**<jupyter_code>start = time.time()
chunk = pd.read_csv(source_file , delimiter = config_data['inbound_delimiter']
                      , names = column_names , chunksize = 100000)
end = time.time()
print("Read csv with pandas chunk took: ",(end-start),"sec")
pd_chunk_df = pd.concat(chunk)<jupyter_output>Read csv with pandas chunk took:  0.13499212265014648 sec<jupyter_text>**Validation Process**<jupyter_code>util.col_header_val(pandas_df , config_data)
print("columns of our data" , list(pandas_df.columns))
print("columns of the YAML file" , config_data['columns'])
if util.col_header_val(pandas_df , config_data) == 0:
  print("Validation failed")
else :
  print("Validation passed")
  util.write_file(pandas_df , config_data['root_folder'] , 'out' , config_data['outbound_delimiter'])
  print("file writing is successful")<jupyter_output>column name and column length validation passed
Validation passed
file writing is successful<jupyter_text>**File Information Summary**<jupyter_code>util.file_summary(source_file , pandas_df)<jupyter_output>size of file is 1.99 GB
the csv file has 15857625 rows
the csv file has 16 columns
{'fraction_non_alphanumeric': 0.1425294888597641, 'fraction_numerical': 0.012778505897771953, 'mean_word_length': 5.086739780658026, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '396490', 'license': 'MIT', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2046, 'n_words': 461, 'path': 'data_ingestion.ipynb', 'repo_name': 'Motamensalih/LISUM03_motamen_week6', 'size': '6104'}