<jupyter_start><jupyter_text>Data Exploration v0<jupyter_code>import os
import sys
import clean_dataset as cl
import pandas as pd
import spacy

'''import and process taxonomy from excel'''
taxonomy = pd.read_excel (os.path.abspath(os.path.join('..', 'data/raw'))+'/tagging_table.xlsx')
# get column names:
columnNames = taxonomy.iloc[0] 
taxonomy = taxonomy[1:] 
taxonomy.columns = columnNames
print('raw data shape:', taxonomy.shape)
# delete entries without PIMS ID:
taxonomy = taxonomy[taxonomy['PIMS #'].notna()]      
print('only entries with PIMS ID:', taxonomy.shape)                
# delete columns without names:
taxonomy = taxonomy.loc[:, taxonomy.columns.notnull()]   
print('only columns with entries:', taxonomy.shape)                
# remove white spaces in column names and lowercase names:
taxonomy.columns = taxonomy.columns.str.replace(' ', '_').str.lower()
#%%
'''extract relevant data fields''' 
#remove all projects without titles:
taxonomy = taxonomy[taxonomy['title'].notna()]     
print('with title:', taxonomy.shape)

#keep only projects with description:
description = taxonomy[taxonomy['project_description'].notna()]     
description = description[['pims_#', 'project_description']]
print('with description:', description.shape)

cl.clean(description, description['project_description'])
cl.segment(description, description['project_description'])
description.columns
description.new_cleaned_column.head
description.list_sentences.head
taxonomy.shape
import make_dataset as mk
mk.labeling_encoding(taxonomy, ['technical_team', 'technical_sub-sub_area_1', 'technical_sub-area_1'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12995729103111653, 'fraction_numerical': 0.003050640634533252, 'mean_word_length': 6.03862660944206, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '462692', 'license': 'MIT', 'n_tokens_mistral': 491, 'n_tokens_neox': 492, 'n_words': 120, 'path': 'oumaima/data_cleaning_test.ipynb', 'repo_name': 'OumaimaHourrane/SmallDataLongDocuments', 'size': '1639'}