<jupyter_start><jupyter_text>Preprocess [global wildfire dataset](https://www.nature.com/articles/s41597-019-0312-2)- Data source: data is available in [this PANGAEA repository](https://doi.pangaea.de/10.1594/PANGAEA.895835)<jupyter_code>import numpy as np
import pandas as pd
import geopandas as gpd
import xarray as xr
from tqdm import tqdm
from shapely.ops import cascaded_union
import requests
import zipfile
import tarfile
import io
import os
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Download data<jupyter_code>base_url = 'http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/'
base_filename = 'MODIS_BA_GLOBAL_1_'
local_path = '../data/MODIS_GWIS_Active_FireEvents'

years = np.arange(2017, 2018, 1).astype(np.str)
months = np.arange(1, 13, 1).astype(np.str)

if not os.path.isdir(local_path):
    os.mkdir(local_path)

for year in years:
    for month in months:
        filename = base_filename + month + '_' + year
        url = base_url + filename + '.zip'
        print(url)
        print('Downloading zip file')
        r = requests.get(url)
        z = zipfile.ZipFile(io.BytesIO(r.content))
        print('Extract zip file')
        z.extractall(path=local_path) # extract to folder
        print('Untar tar file')
        tar = tarfile.open(f"{local_path}/{filename+'.tar'}")
        tar.extractall(path=f'{local_path}/{filename}/')
        tar.close()
        print('Remove tar file')
        os.remove(f"{local_path}/{filename}.tar")<jupyter_output>http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_1_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_2_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_3_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_4_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_5_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_6_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove tar file
http://hs.pangaea.de/Maps/MCD64A1_burnt-areas/MODIS_BA_GLOBAL_1_7_2017.zip
Downloading zip file
Extract zip file
Untar tar file
Remove t[...]<jupyter_text>**Concatenate data**<jupyter_code>base_filename = 'MODIS_BA_GLOBAL_1_'
local_path = '../data/MODIS_GWIS_Active_FireEvents'

years = np.arange(2001, 2018, 1).astype(np.str)
months = np.arange(1, 13, 1).astype(np.str)

n = 0
for year in years:
    print(f'{year}:')
    for month in months:
        filename = base_filename + month + '_' + year
        gdf = gpd.read_file(f'{local_path}/{filename}/{filename}.shp', bbox = [-180, 18, -54, 72])
        gdf = gdf[gdf['Type'] == 'FinalArea']
        if n == 0:
            gdf_all = gdf
        else:
            gdf_all = pd.concat([gdf_all, gdf])
        n += 1
gdf_all.plot()<jupyter_output><empty_output><jupyter_text>**Save wildfire GeoDataFrame**<jupyter_code>gdf_all.iloc[256:].to_file('../data/MODIS_GWIS_Active_FireEvents/wildfires/wildfires.shp')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11961722488038277, 'fraction_numerical': 0.04007177033492823, 'mean_word_length': 4.2265625, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '641226', 'license': 'MIT', 'n_tokens_mistral': 1385, 'n_tokens_neox': 1288, 'n_words': 250, 'path': 'processing/preprocess_wildfire_dataset.ipynb', 'repo_name': 'Vizzuality/usa-resilience-opportunity-map-data', 'size': '3344'}