<jupyter_start><jupyter_text>Cleaning Company Data[Data Source] (Generated for testing purposes)<jupyter_code>import pandas as pd
import numpy as np
# Change the path to the correct location of your source data
company = pd.read_csv('./real/Company_data.csv', sep=';');
initial_rows = len(company);
initial_columns = len(company.columns);
# Show initial amount of rows and columns 
print(initial_rows)
print(initial_columns)
# Display the first lines of the dataset
company.head(10)
# remove any unnecessary columns (depends on original dataset)
company_reduced = company[['Date','Time','Global_active_power']]
company_reduced.head(10)
# remove any 
company_reduced.replace(to_replace=r'^\D+[^\.:/-]?\D*[^\.:/-]?\D*$', value=np.NaN, regex=True, inplace=True)
company_reduced.dropna(inplace=True)
company_reduced = company_reduced[company_reduced['Global_active_power'] > '0']
company_reduced.head(10)
# add the necessary columns that will be filled later
company_reduced.insert(2,'Latitude', '0')
company_reduced.insert(3,'Longitude', '0')

company_reduced.insert(4,'Size', '0')
company_reduced.insert(5,'Office_or_factory', '0')
company_reduced.insert(6,'Solarpanels', '0')
company_reduced.insert(7,'Heating_type', '0')
company_reduced.insert(8,'Operating_hours', '0')

company_reduced.head(10)
# change path to the location for the desired output
company_reduced.to_csv('./real/company_filtered.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14106145251396648, 'fraction_numerical': 0.01606145251396648, 'mean_word_length': 8.0125786163522, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '579713', 'license': 'MIT', 'n_tokens_mistral': 485, 'n_tokens_neox': 456, 'n_words': 106, 'path': 'real/cleansing_company.ipynb', 'repo_name': 'tm-pe2/ai-data', 'size': '1432'}