<commit_before>Old Url,New Url,Status
http://www.mod.uk,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/Home/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/gone.css,,200
http://www.mod.uk/defenceinternet/home/index.htm,,410
http://www.mod.uk/DefenceInternet/DefenceNews,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/defenceinternet/defencenews/indepth/operationsinafghanistan.htm,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceFor/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/NR/rdonlyres/584CAAE8-8B39-4271-86E8-86320C7804B7/0/Carrier3_761x437.jpg,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/SofS/,https://www.gov.uk/government/ministers/secretary-of-state-for-defence#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinISD/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state-international-security-strategy#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinAF/Speeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinVet/Speeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinUSSLSD/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state--20#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/SeniorOfficials/SeniorOfficialsSpeeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinDES/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state-defence-equipment-support-and-technology#announcements,301
http://www.mod.uk/NR/rdonlyres/2A4B43F3-3D65-4379-97D7-DF62BFF30CED/0/des_logo_77x45.gif,,410
http://www.mod.uk/NR/rdonlyres/FCB4697E-C5FA-47B0-A68D-A99E30BFF2D2/0/Astute2.JPG,,410
http://www.mod.uk/NR/rdonlyres/CDB61790-0863-4A4D-96BA-EDBFB721900B/0/ukfafghan300x110.jpg,,410
http://www.mod.uk/DefenceInternet/DefenceNews/EstateAndEnvironment/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/NR/rdonlyres/2E8E470F-B782-4FAA-A258-F76ABE8D7CC8/0/SniperRifle2.jpg,,410
http://www.mod.uk/DefenceInternet/AZIndex/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/SecurityandIntelligence/DVA/,https://www.gov.uk/security-vetting-and-clearance,301
http://www.mod.uk/DefenceInternet/AboutDefence/,,410
http://www.mod.uk/DefenceInternet/DefenceNews/EquipmentAndLogistics/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/DefencePolicyAndBusiness/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/MilitaryOperations/,https://www.gov.uk/government/announcements?departments[]=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/DefenceEstateandEnvironment/AccessRecreation/,https://www.gov.uk/public-access-to-military-areas,301
http://www.mod.uk/DefenceInternet/DefenceNews/TrainingAndAdventure/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/HistoryAndHonour/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/Personnel/Pensions/ArmedForcesPensionsAndRedundancy.htm,https://www.gov.uk/pensions-and-compensation-for-veterans,301
http://www.mod.uk/NR/rdonlyres/301F5D15-42E3-4E15-9E4B-E44448A279DA/0/BB110106004.jpg,,410
http://www.mod.uk/DefenceInternet/DefenceNews/RN/,https://www.gov.uk/government/announcements?departments[]=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/DefenceEstateandEnvironment/MODFilmLocations/,https://www.gov.uk/ministry-of-defence-film-locations,301
http://www.mod.uk/NR/rdonlyres/29A237C3-6A9E-4EDA-8C48-02C026653055/0/RoyalNavyPoliceRankBadges.JPG,,410
http://www.mod.uk/NR/rdonlyres/BD9ADA1B-0F5D-473B-A825-10CF0328AB77/0/SniperRifle1.JPG,,410
http://www.mod.uk/NR/rdonlyres/32740CC1-A05A-4E72-87B8-30F3799B5E2B/0/NewSniperSystem2.jpg,,410
http://www.mod.uk/DefenceInternet/FactSheets/OperationsFactsheets/OperationsInAfghanistanBritishFatalities.htm,https://www.gov.uk/uk-forces-operations-in-afghanistan,301
http://www.mod.uk/DefenceInternet/DefenceNews/MilitaryOperations/RoyalNavySavesSpanishVesselFromPirateAttack.htm,https://www.gov.uk/government/news/royal-navy-saves-spanish-vessel-from-pirate-attack,301
http://www.mod.uk/NR/rdonlyres/AC309B8D-0D71-41D3-8B5E-344098FA16AF/0/EurofighterTyphoons.JPG,,410
<commit_msg>removed subset test to large mod page<commit_after>Old Url,New Url,Status
http://www.mod.uk,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/Home/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/gone.css,,200
http://www.mod.uk/defenceinternet/home/index.htm,,410
http://www.mod.uk/DefenceInternet/DefenceNews,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/defenceinternet/defencenews/indepth/operationsinafghanistan.htm,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceFor/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/NR/rdonlyres/584CAAE8-8B39-4271-86E8-86320C7804B7/0/Carrier3_761x437.jpg,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/SofS/,https://www.gov.uk/government/ministers/secretary-of-state-for-defence#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinISD/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state-international-security-strategy#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinAF/Speeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinVet/Speeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinUSSLSD/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state--20#announcements,301
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/SeniorOfficials/SeniorOfficialsSpeeches.htm,,410
http://www.mod.uk/DefenceInternet/AboutDefence/People/Speeches/MinDES/,https://www.gov.uk/government/ministers/parliamentary-under-secretary-of-state-defence-equipment-support-and-technology#announcements,301
http://www.mod.uk/NR/rdonlyres/2A4B43F3-3D65-4379-97D7-DF62BFF30CED/0/des_logo_77x45.gif,,410
http://www.mod.uk/NR/rdonlyres/FCB4697E-C5FA-47B0-A68D-A99E30BFF2D2/0/Astute2.JPG,,410
http://www.mod.uk/NR/rdonlyres/CDB61790-0863-4A4D-96BA-EDBFB721900B/0/ukfafghan300x110.jpg,,410
http://www.mod.uk/DefenceInternet/DefenceNews/EstateAndEnvironment/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/NR/rdonlyres/2E8E470F-B782-4FAA-A258-F76ABE8D7CC8/0/SniperRifle2.jpg,,410
http://www.mod.uk/DefenceInternet/AZIndex/,https://www.gov.uk/government/organisations/ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/SecurityandIntelligence/DVA/,https://www.gov.uk/security-vetting-and-clearance,301
http://www.mod.uk/DefenceInternet/AboutDefence/,,410
http://www.mod.uk/DefenceInternet/DefenceNews/EquipmentAndLogistics/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/DefencePolicyAndBusiness/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/MilitaryOperations/,https://www.gov.uk/government/announcements?departments[]=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/DefenceEstateandEnvironment/AccessRecreation/,https://www.gov.uk/public-access-to-military-areas,301
http://www.mod.uk/DefenceInternet/DefenceNews/TrainingAndAdventure/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/DefenceNews/HistoryAndHonour/,https://www.gov.uk/government/announcements?departments%5B%5D=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/Personnel/Pensions/ArmedForcesPensionsAndRedundancy.htm,https://www.gov.uk/pensions-and-compensation-for-veterans,301
http://www.mod.uk/NR/rdonlyres/301F5D15-42E3-4E15-9E4B-E44448A279DA/0/BB110106004.jpg,,410
http://www.mod.uk/DefenceInternet/DefenceNews/RN/,https://www.gov.uk/government/announcements?departments[]=ministry-of-defence,301
http://www.mod.uk/DefenceInternet/AboutDefence/WhatWeDo/DefenceEstateandEnvironment/MODFilmLocations/,https://www.gov.uk/ministry-of-defence-film-locations,301
http://www.mod.uk/NR/rdonlyres/29A237C3-6A9E-4EDA-8C48-02C026653055/0/RoyalNavyPoliceRankBadges.JPG,,410
http://www.mod.uk/NR/rdonlyres/BD9ADA1B-0F5D-473B-A825-10CF0328AB77/0/SniperRifle1.JPG,,410
http://www.mod.uk/NR/rdonlyres/32740CC1-A05A-4E72-87B8-30F3799B5E2B/0/NewSniperSystem2.jpg,,410
http://www.mod.uk/DefenceInternet/DefenceNews/MilitaryOperations/RoyalNavySavesSpanishVesselFromPirateAttack.htm,https://www.gov.uk/government/news/royal-navy-saves-spanish-vessel-from-pirate-attack,301
http://www.mod.uk/NR/rdonlyres/AC309B8D-0D71-41D3-8B5E-344098FA16AF/0/EurofighterTyphoons.JPG,,410

{'fraction_non_alphanumeric': 0.16208365608055772, 'fraction_numerical': 0.07329589465530596, 'mean_word_length': 6.44164265129683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 51, 'lorem ipsum': 0, 'www.': 134, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1535975', 'n_tokens_mistral': 4595, 'n_tokens_neox': 3997, 'n_words': 95}