<jupyter_start><jupyter_text>Load all Links<jupyter_code>with open('./all_battles.json', 'r') as f:
    campaigns = json.load(f)
campaigns['African Front']['Syria–Lebanon Campaign'].keys()<jupyter_output><empty_output><jupyter_text>Parse all battles<jupyter_code>def _parse_in_depth(element, name):
    '''attempts to scrape data for every
    element with url attribute - and all the children
    if there are any'''
    
    if 'children' in element:
        for k, child in element['children'].items():
            parsed = _parse_in_depth(child, k)
            element['children'][k].update(parsed)
            
    if element.get('url', None):
        try:
            element.update(parse_battle_page(element['url']))    
        except Exception as e:
            raise Exception(name, e)
            
    time.sleep(.1) # let's be good citizens!
    return element
campaigns_parsed = {}

for fr_name, front in campaigns.items():
    print(fr_name)
    campaigns_parsed[fr_name] = {}
    for cp_name, campaign in front.items():
        print(f'    {cp_name}')
        campaigns_parsed[fr_name][cp_name] = _parse_in_depth(campaign, cp_name)
campaigns_parsed['African Front']['North African Campaign'].keys()
campaigns_parsed['Eastern Front']['Battle of Someri']<jupyter_output><empty_output><jupyter_text>Store all data<jupyter_code>with open('all_battles_parsed.json', 'w') as f:
    json.dump(campaigns_parsed, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12611991729841487, 'fraction_numerical': 0.0006891798759476223, 'mean_word_length': 3.823920265780731, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '46291', 'license': 'MIT', 'n_tokens_mistral': 478, 'n_tokens_neox': 475, 'n_words': 100, 'path': 'Chapter07/7_Scraping_part_3.ipynb', 'repo_name': 'urantialife/Python-Programming-Projects-Learn-Python-3.7-by-building-applications', 'size': '1451'}