<jupyter_start><jupyter_text>Import packages<jupyter_code>import pandas as pd
import datetime
import os
import xmltodict
import urllib.request
import gzip
import numpy as np
import copy<jupyter_output><empty_output><jupyter_text>Sitemaps<jupyter_code>#read in active sitemaps
basepath = '../data/sitemaps/sitemaps_raw/'

sitemaps_list = []

for filename in os.listdir(basepath):
    if filename.endswith(".gz"): 
        with gzip.open(basepath + filename, 'r') as fd:
            gz = fd.read()
            sitemaps_list.append(xmltodict.parse(gz))
#define function to extract info from sitemap
def getSitemapInfo(sitemap):
    sitemapinfo = []
    for i in range(len(sitemap['urlset']['url'])):
        url = sitemap['urlset']['url'][i]['loc']
        lastmod = sitemap['urlset']['url'][i]['lastmod']
        sitemapinfo.append([url, lastmod])
    return sitemapinfo
#store in dataframe
sitemaps = pd.DataFrame()
for sitemap in sitemaps_list:
    sitemaps = sitemaps.append(getSitemapInfo(sitemap))
#rename columns
sitemaps = sitemaps.rename(columns={0: "url", 1: "last-mod"})<jupyter_output><empty_output><jupyter_text>Convert urls to include /f/<jupyter_code>sitemaps_no_fix = sitemaps[sitemaps['url'].str.contains('https://www.gofundme.com/f/')]
sitemaps_to_fix = sitemaps[~sitemaps['url'].str.contains('https://www.gofundme.com/f/')]
new_urls = copy.deepcopy(sitemaps_to_fix['url'].str.replace('https://www.gofundme.com/', 'https://www.gofundme.com/f/'))
sitemaps_to_fix.loc[:,'url'] = new_urls.loc[::]
sitemaps = sitemaps_no_fix.append(sitemaps_to_fix)
urls_to_fix = []
for url in sitemaps['url']:
    if 'https://www.gofundme.com/f/' not in url:
        urls_to_fix.append(url)
#urls_to_fix
sitemaps = sitemaps[~sitemaps['url'].isin(urls_to_fix)]
urls_to_fix = []
for url in sitemaps['url']:
    if 'https://www.gofundme.com/f/' not in url:
        urls_to_fix.append(url)
urls_to_fix<jupyter_output><empty_output><jupyter_text>Spit combined into chunks of 1000 for scraping and export<jupyter_code>#randomize order of urls
sitemaps = sitemaps.sample(frac=1).reset_index(drop=True)
n = 1000  #chunk row size
list_df = [sitemaps[i:i+n] for i in range(0,sitemaps.shape[0],n)]
for i in range(len(list_df)):
    df = list_df[i]
    filename = "../data/sitemaps/sitemaps_csv/sitemaps_" + str(i) + ".csv"
    df.to_csv(filename)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15169491525423728, 'fraction_numerical': 0.005508474576271186, 'mean_word_length': 5.164490861618799, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '213073', 'license': 'MIT', 'n_tokens_mistral': 931, 'n_tokens_neox': 924, 'n_words': 158, 'path': 'code/01-prepare_sitemaps.ipynb', 'repo_name': 'sdoerstling/medical_crowdfunding_methods', 'size': '2360'}