<jupyter_start><jupyter_text>Cleaning<jupyter_code>import sqlite3
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import bs4
import requests
import re
import time
from sqlalchemy import create_engine
from tqdm import tqdm
valid_dates = list([str(x)[:10] for x in pd.date_range(start="01/02/2021", end='30/04/2021')])
from glob import glob
filepaths = glob('datasets/weather/*')
for file in filepaths:
    df = pd.read_csv(file)
    df['Date'] = df['Year'].astype(str) + "-" + df['Month'].astype(str) + "-" + df['Day'].astype(str)
    df['Date'] = df['Date'].astype(str)
    df = df[df['Date'].isin(valid_dates)]
    df.to_csv(file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15439093484419264, 'fraction_numerical': 0.028328611898016998, 'mean_word_length': 5.36936936936937, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '586797', 'license': 'MIT', 'n_tokens_mistral': 277, 'n_tokens_neox': 261, 'n_words': 62, 'path': 'Weather Cleaning.ipynb', 'repo_name': 'ralphcaubalejo/Geospatial-Analysis-Project', 'size': '706'}