<jupyter_start><jupyter_text>Imports<jupyter_code>%load_ext autoreload
%autoreload 2
import glob
import re

import pandas as pd
import spotify_mexico.data.load as load<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>data_raw = load.data_raw
data_processed = load.data_processed<jupyter_output><empty_output><jupyter_text>Read files as dataframes<jupyter_code># All files ending with .csv
paths = glob.glob(f'{data_raw}/*.csv')
dfs = []
# Read files as dataframes
for i in paths:
    df = load.load_file(file=f'{i}')
    dfs.append(df)<jupyter_output><empty_output><jupyter_text>Process data<jupyter_code>for i in range(len(paths)):
    date = re.findall(r'weekly-(\d*-\d*-\d*)', paths[i])
    year = re.findall(r'weekly-(\d*)', paths[i])
    month = re.findall(r'weekly-\d*-(\d*)', paths[i])
    days = re.findall(r'weekly-\d*-\d*-(\d*)', paths[i])
    day = int(days[0])
    if day <= 7:
        week = 1
    if day > 7 and day <= 14:
        week = 2
    if day > 14 and day <= 21:
        week = 3
    if day > 21 and day <= 28:
        week = 4
    if day > 28:
        week = 5
    dfs[i]['date'] = date[0]
    dfs[i]['year'] = year[0]
    dfs[i]['month'] = month[0]
    dfs[i]['week'] = week<jupyter_output><empty_output><jupyter_text>Concatenate dataframes<jupyter_code>df = pd.concat(dfs)
df<jupyter_output><empty_output><jupyter_text>Save dataframe<jupyter_code>df.to_csv(f'{data_processed}/spotify_mexico.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16205266711681296, 'fraction_numerical': 0.016205266711681297, 'mean_word_length': 4.330935251798561, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '457960', 'license': 'MIT', 'n_tokens_mistral': 624, 'n_tokens_neox': 597, 'n_words': 107, 'path': 'notebooks/0.0-process.ipynb', 'repo_name': 'fer-aguirre/spotify-mexico', 'size': '1481'}