<jupyter_start><jupyter_text>Reads `.json` Files and Combines them into `.csv`<jupyter_code>df = pd.DataFrame()
for num in np.arange(15, 995, 10):
    lower_bound = str(num)
    upper_bound = str(num + 10)
    filename = lower_bound + "to" + upper_bound + ".json"
    # Check if file is empty
    if os.stat(filename).st_size != 0:    
        some_df = pd.read_json(filename)
        df = pd.concat([df, some_df])
len(df)
df.drop_duplicates(subset='url', keep='first', inplace=True)

len(df)
df.shape
df.to_csv('cancun_scrape_5.csv', index=False)
df[df['url'] == 'https://www.airbnb.com/rooms/10823831']<jupyter_output><empty_output><jupyter_text>Reads multiple `.csv` Files and Combines them into one `.csv`<jupyter_code>cancun_df = pd.DataFrame()
growth_list = []
for num in np.arange(1, 6):
    filename = "cancun_scrape_{0}.csv".format(num)
    some_df = pd.read_csv(filename)
    
    # Calculate dataframe size before
    prev_size = len(cancun_df)
    
    # Combine dataframes and drop the duplicates
    cancun_df = pd.concat([cancun_df, some_df])
    cancun_df = cancun_df.drop_duplicates(subset='url', keep='first')

    # Calculate dataframe size after
    after_size = len(cancun_df)
    
    # Calculate growth
    growth = after_size - prev_size
    growth_list.append(growth)
    
    print("On iteration {0}, we added {1} new rows".format(num, growth))
cancun_df.shape
cancun_df.to_csv('cancun_final.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13437926330150068, 'fraction_numerical': 0.01637107776261937, 'mean_word_length': 4.129370629370629, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '413971', 'license': 'MIT', 'n_tokens_mistral': 578, 'n_tokens_neox': 551, 'n_words': 110, 'path': 'notebooks/kl-cancun-data-cleaning.ipynb', 'repo_name': 'kailu3/airbnb_optimization', 'size': '1466'}