<commit_before>Old Url,New Url,Status
http://og.decc.gov.uk/,https://www.gov.uk/government/organisations/department-of-energy-climate-change,301
http://og.decc.gov.uk/gone.css,,200
http://og.decc.gov.uk/favicon.ico,,410
http://og.decc.gov.uk/en/olgs/cms/data_maps/data_maps.aspx,https://www.gov.uk/oil-and-gas-offshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/ukcs_inc_exp/ukcs_inc_exp.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fdocs%2f3190-appendix-1-field-unit-version-uk-reserves.pdf&minwidth=true,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licences.aspx,https://www.gov.uk/browse/business/licences/oil-and-gas-licensing,301
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fep%2fwells%2fAppendix4_tab3.xls&minwidth=true,,410
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fappendices%2fappendix9.xls&minwidth=true,,410
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fappendices%2fappendix10.xls&minwidth=true,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licence_data/licence_data.aspx,https://www.gov.uk/oil-and-gas-licence-data,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/offshore_maps/quad_maps/quad_maps.aspx,http://tools.decc.gov.uk/en/content/cms/tools/quad_maps/quad_maps.aspx,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/offshore_maps/offshore_maps.aspx,https://www.gov.uk/oil-and-gas-offshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/explorationpro.aspx,https://www.gov.uk/browse/business/generating-energy/oil-and-gas-exploration-and-production,301
http://og.decc.gov.uk/en/olgs/cms/pons_and_cop/pons/pons.aspx,https://www.gov.uk/oil-and-gas-petroleum-operations-notices,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/bus_opps.aspx,https://www.gov.uk/browse/business/generating-energy/oil-and-gas-exploration-and-production,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/uk_production/uk_production.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/pathfinder/pathfinder.aspx,https://www.gov.uk/oil-and-gas-project-pathfinder,301
http://og.decc.gov.uk/en/olgs/cms/licences/licence_data/recent_licence/recent_licence.aspx,http://tools.decc.gov.uk/en/content/cms/tools/oglicences/oglicences.aspx,301
http://og.decc.gov.uk/en/olgs/cms/home/whatsnew/whatsnew.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/pons_and_codes/pons_and_codes.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/types_of_licen/types_of_licen.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/licensing_guid.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/well_data/well_data.aspx,https://www.gov.uk/oil-and-gas-wells,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/onshore_maps/onshore_maps.aspx,https://www.gov.uk/oil-and-gas-onshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/field_data.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/sitemap.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/explorationpro/onshore/onshore.aspx,https://www.gov.uk/oil-and-gas-onshore-exploration-and-production#resumption-of-shale-gas-exploration,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/onshore/onshore.aspx%2C,,410
http://og.decc.gov.uk/en/olgs/cms/explorationpro/onshore/onshore.aspx%2C%202010.,,410
http://og.decc.gov.uk/en/olgs/cms/licences/lic_rounds/lic_rounds.aspx,https://www.gov.uk/oil-and-gas-licensing-rounds,301
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/pears/pears.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/field_app_2012/field_app_2012.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/olgs/cms/home/portal_help/p_contact/p_contact.aspx?template=lite,,410
http://og.decc.gov.uk/en/olgs/cms/environment/leg_guidance/ocr/ocr.aspx,https://www.gov.uk/oil-and-gas-offshore-environmental-legislation,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/oil_spills/oil_historic/oil_historic.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/data_maps/data_release/data_release.aspx,http://tools.decc.gov.uk/en/content/cms/data_release/%20data_release.aspx,301
http://og.decc.gov.uk/en/olgs/cms/licences/licence_relinq/relinquishment/relinquishment/relinquishment.aspx,http://tools.decc.gov.uk/en/content/cms/tools/relinquishment/relinquishment.aspx,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/pipelines/gd_pwa/gd_pwa.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/explorationpro/operatorship/onshore_ops/onshore_ops.aspx,https://www.gov.uk/oil-and-gas-operatorship,301<commit_msg>Removing URL with dot at end from smoke tests<commit_after>Old Url,New Url,Status
http://og.decc.gov.uk/,https://www.gov.uk/government/organisations/department-of-energy-climate-change,301
http://og.decc.gov.uk/gone.css,,200
http://og.decc.gov.uk/favicon.ico,,410
http://og.decc.gov.uk/en/olgs/cms/data_maps/data_maps.aspx,https://www.gov.uk/oil-and-gas-offshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/ukcs_inc_exp/ukcs_inc_exp.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fdocs%2f3190-appendix-1-field-unit-version-uk-reserves.pdf&minwidth=true,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licences.aspx,https://www.gov.uk/browse/business/licences/oil-and-gas-licensing,301
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fep%2fwells%2fAppendix4_tab3.xls&minwidth=true,,410
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fappendices%2fappendix9.xls&minwidth=true,,410
http://og.decc.gov.uk/publications/basket.aspx?filetype=4&filepath=og%2fdata-maps%2fappendices%2fappendix10.xls&minwidth=true,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licence_data/licence_data.aspx,https://www.gov.uk/oil-and-gas-licence-data,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/offshore_maps/quad_maps/quad_maps.aspx,http://tools.decc.gov.uk/en/content/cms/tools/quad_maps/quad_maps.aspx,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/offshore_maps/offshore_maps.aspx,https://www.gov.uk/oil-and-gas-offshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/explorationpro.aspx,https://www.gov.uk/browse/business/generating-energy/oil-and-gas-exploration-and-production,301
http://og.decc.gov.uk/en/olgs/cms/pons_and_cop/pons/pons.aspx,https://www.gov.uk/oil-and-gas-petroleum-operations-notices,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/bus_opps.aspx,https://www.gov.uk/browse/business/generating-energy/oil-and-gas-exploration-and-production,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/uk_production/uk_production.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/pathfinder/pathfinder.aspx,https://www.gov.uk/oil-and-gas-project-pathfinder,301
http://og.decc.gov.uk/en/olgs/cms/licences/licence_data/recent_licence/recent_licence.aspx,http://tools.decc.gov.uk/en/content/cms/tools/oglicences/oglicences.aspx,301
http://og.decc.gov.uk/en/olgs/cms/home/whatsnew/whatsnew.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/pons_and_codes/pons_and_codes.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/types_of_licen/types_of_licen.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/licensing_guid.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/well_data/well_data.aspx,https://www.gov.uk/oil-and-gas-wells,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/onshore_maps/onshore_maps.aspx,https://www.gov.uk/oil-and-gas-onshore-maps-and-gis-shapefiles,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/field_data.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/sitemap.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/explorationpro/onshore/onshore.aspx,https://www.gov.uk/oil-and-gas-onshore-exploration-and-production#resumption-of-shale-gas-exploration,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/onshore/onshore.aspx%2C,,410
http://og.decc.gov.uk/en/olgs/cms/licences/lic_rounds/lic_rounds.aspx,https://www.gov.uk/oil-and-gas-licensing-rounds,301
http://og.decc.gov.uk/en/olgs/cms/licences/licensing_guid/pears/pears.aspx,https://www.gov.uk/oil-and-gas-petroleum-licensing-guidance,301
http://og.decc.gov.uk/en/olgs/cms/bus_opps/field_app_2012/field_app_2012.aspx,https://www.gov.uk/oil-and-gas-uk-field-data,301
http://og.decc.gov.uk/en/olgs/cms/home/portal_help/p_contact/p_contact.aspx?template=lite,,410
http://og.decc.gov.uk/en/olgs/cms/environment/leg_guidance/ocr/ocr.aspx,https://www.gov.uk/oil-and-gas-offshore-environmental-legislation,301
http://og.decc.gov.uk/en/olgs/cms/data_maps/field_data/oil_spills/oil_historic/oil_historic.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/data_maps/data_release/data_release.aspx,http://tools.decc.gov.uk/en/content/cms/data_release/%20data_release.aspx,301
http://og.decc.gov.uk/en/olgs/cms/licences/licence_relinq/relinquishment/relinquishment/relinquishment.aspx,http://tools.decc.gov.uk/en/content/cms/tools/relinquishment/relinquishment.aspx,301
http://og.decc.gov.uk/en/olgs/cms/explorationpro/pipelines/gd_pwa/gd_pwa.aspx,,410
http://og.decc.gov.uk/en/olgs/cms/explorationpro/operatorship/onshore_ops/onshore_ops.aspx,https://www.gov.uk/oil-and-gas-operatorship,301
{'fraction_non_alphanumeric': 0.18960617133576937, 'fraction_numerical': 0.03227771010962241, 'mean_word_length': 5.247939124920736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 44, 'lorem ipsum': 0, 'www.': 44, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2348926', 'n_tokens_mistral': 4637, 'n_tokens_neox': 4214, 'n_words': 92}