<jupyter_start><jupyter_text>Cleaning Pipeline  Importing Packages, defining AC list and Folders<jupyter_code># Importing Packages

import pandas as pd
import numpy as np
import re
import glob
import os
import concurrent # for parallel instances
import warnings

# importing helper functions
from cleaning_functions import cleaning
from cleaning_functions import connect_party
from cleaning_functions import connect_party2

##### For when constituencies come from book.xlsx #####

# Defining Selection of Constituencies

df = pd.read_excel('/home/hennes/Downloads/Book.xlsx')

# get relevant pdf numbers

worklist = df[df['Ready for Cleaning and Merging?'] == 'y']['Constituency number'].tolist()

# give appropriate filename endings to items

for idx, item in enumerate(worklist):
    if len(str(item)) == 1:
        worklist[idx] = f'AC00{item}.csv'
    if len(str(item)) == 2:
        worklist[idx] = f'AC0{item}.csv'
    if len(str(item)) == 3:
        worklist[idx] = f'AC{item}.csv'
        
worklist = tuple(worklist)

# Defining Folders
folder = '/home/hennes/Internship/constituencies/'
save_folder = '/home/hennes/Internship/constituencies_edit/'
old = '/home/hennes/Internship/old_files/'
candidates = pd.read_excel('/home/hennes/Internship/Party_Data_2019.xlsx')
PC_AC = set(sorted([folder.split('-')[0]+'-'+ folder.split('-')[1] for folder in next(os.walk(old))[1]]))
PC_AC_dict = {e.split('-')[1]: e.split('-')[0] for e in PC_AC}
constituencies = sorted([os.path.split(file)[-1] for file in glob.glob(folder+'*') if file.endswith(".csv")]) # list with all files
constituencies = [file for file in constituencies if file.endswith(worklist)]

# list of ACs belonging to individual PCs
PClist = []
unique_pc = set([e.split('-')[0] for e in PC_AC])
edited_ac = [os.path.split(e)[1] for e in glob.glob(save_folder+'*')]

for x in unique_pc:
    PClist.append([e.split('-')[1] for e in PC_AC
                   if e.split('-')[0] == x and e.split('-')[1]+'.csv' in edited_ac])

# defining function options so that it can be executed with concurrent futures    
def pipeline(c):
    try:
        df = cleaning(c, candidates, A_serial= False, year=2019, max_digits = 5, max_value = 1600)
        df.to_csv(save_folder+c, index=False)
    except Exception as e:
        print(e)
        print(f'Problem with {c}. Jump to next one. \n')
        return
# for when constituencies are just all csv files that are in folder (and not PC)
constituencies = sorted([os.path.split(file)[-1] for file in glob.glob(folder+'*') if file.endswith(".csv")]) # list with all files<jupyter_output><empty_output><jupyter_text>Running Program<jupyter_code># for running with multiple cores

# ignore pandas userwarnings
warnings.simplefilter(action='ignore', category=UserWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)

with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
        executor.map(pipeline, constituencies)
# for running with single core (better troubleshooting)

# ignore pandas userwarnings
warnings.simplefilter(action='ignore', category=UserWarning)
warnings.simplefilter(action='ignore', category=RuntimeWarning)

for c in constituencies:
    pipeline(c)
# to connect party for PC constituencies when constituencies in save folder still in PC format
for element in constituencies:
    df = pd.read_csv(save_folder+element)
    connect_party(df, element)

# to connect party to AC constituencies when constituencies in save folder already in AC format
for element in constituencies:
    df = pd.read_csv(save_folder+element) 
    connect_party2(df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12248275862068965, 'fraction_numerical': 0.009379310344827587, 'mean_word_length': 4.792332268370607, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '46511', 'license': 'MIT', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1155, 'n_words': 330, 'path': 'Cleaning Pipeline.ipynb', 'repo_name': 'hbarnehl/adapted-image-table-ocr', 'size': '3625'}