<jupyter_start><jupyter_text>Calculate BaselineCreates baseline mean, standard deviation, variance, and median GeoTiffs for specified each month and specified date ranges.Baseline years:* 2012-2019 (Jan-Dec)Comparison Year:* 2020 (Jan-Dec)  Environment Setup<jupyter_code># Load Notebook formatter
%load_ext nb_black
# %reload_ext nb_black
# Import packages
import os
import glob
import re
import warnings
import viirs
# Set Options
warnings.filterwarnings("ignore")
# sns.set(font_scale=1.5, style="whitegrid")
# sns.set(font_scale=1.5)
# pd.set_option("display.max_columns", None)
# pd.set_option("display.max_rows", None)
# pd.set_option("precision", 15)
# Set working directory
os.chdir("..")
print(f"Working directory: {os.getcwd()}")<jupyter_output><empty_output><jupyter_text>User-Defined Variables<jupyter_code># Set paths
geotiff_input_folder = os.path.join(
    "03-processed-data", "raster", "south-korea", "vnp46a2-clipped"
)

statistics_output_folder = os.path.join(
    "03-processed-data", "raster", "south-korea", "statistics", "vnp46a2"
)<jupyter_output><empty_output><jupyter_text>Data Acquisition and Preprocessing<jupyter_code># Get list of radiance rasters
# radiance_geotiffs = glob.glob(os.path.join(geotiff_input_folder, "*.tif"))
# print(f"Found {len(radiance_geotiffs)} files")
# # Get export metadata (for exporting statistics, same for all files)
# metadata = viirs.extract_geotiff_metadata(
#     glob.glob(os.path.join(geotiff_input_folder, "*.tif"))[0]
# )
# metadata<jupyter_output><empty_output><jupyter_text>Data Processing  Setup<jupyter_code># Get export metadata (for exporting statistics, same for all files)
metadata = viirs.extract_geotiff_metadata(
    glob.glob(os.path.join(geotiff_input_folder, "*.tif"))[0]
)
metadata
# Set month numbers (for filtering data) and abbreviations (for output name)
month_numbers = [
    "01",
    "02",
    "03",
    "04",
    "05",
    "06",
    "07",
    "08",
    "09",
    "10",
    "11",
    "12",
]

month_abbreviations = [
    "jan",
    "feb",
    "mar",
    "apr",
    "may",
    "jun",
    "jul",
    "aug",
    "sep",
    "oct",
    "nov",
    "dec",
]
# Set date ranges of interest (use leap year to include 2/29)
date_ranges = [
    [
        date[4:]
        for date in viirs.create_date_range(
            start_date="2020-03-15", end_date="2020-04-14"
        )
    ],
    [
        date[4:]
        for date in viirs.create_date_range(
            start_date="2020-04-15", end_date="2020-05-14"
        )
    ],
    [
        date[4:]
        for date in viirs.create_date_range(
            start_date="2020-05-15", end_date="2020-06-14"
        )
    ],
]
date_ranges<jupyter_output><empty_output><jupyter_text>Baseline Data (2012-2019)  Monthly<jupyter_code># Calculate statistics for all months over the baseline years
for index, baseline_month in enumerate(month_numbers):
    print(f"Started month: {month_abbreviations[index].capitalize()}")
    print("Gathering GeoTiffs within range...")
    # Get list of arrays for acqusition dates that match the month
    array_list = [
        viirs.read_geotiff_into_array(geotiff)
        for geotiff in glob.glob(os.path.join(geotiff_input_folder, "*.tif"))
        if re.compile(f"^{baseline_month}$").match(
            # Acquisition month
            os.path.basename(geotiff)[12:14]
        )
        # Acquisition year
        and os.path.basename(geotiff)[8:12] != "2020"
    ]

    print(f"Number of arrays: {len(array_list)}")
    print("Calculating statistics...")
    # Calculate mean, variance, standard deviation, and median
    statistics = {
        "mean": {
            "array": viirs.calculate_statistic(array_list, statistic="mean"),
            "file": (
                "vnp46a2-south-korea-2012-2019-mean-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "variance": {
            "array": viirs.calculate_statistic(
                array_list, statistic="variance"
            ),
            "file": (
                "vnp46a2-south-korea-2012-2019-variance-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "deviation": {
            "array": viirs.calculate_statistic(
                array_list, statistic="deviation"
            ),
            "file": (
                "vnp46a2-south-korea-2012-2019-deviation-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "median": {
            "array": viirs.calculate_statistic(array_list, statistic="median"),
            "file": (
                "vnp46a2-south-korea-2012-2019-median-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
    }

    print("Exporting statistics to GeoTiffs...")
    # Export stastistic arrays to GeoTiff
    for statistic in statistics.keys():
        try:
            viirs.export_array(
                array=statistics.get(statistic).get("array"),
                output_path=os.path.join(
                    statistics_output_folder,
                    statistics.get(statistic).get("file"),
                ),
                metadata=metadata,
            )
        except Exception as error:
            print(error)

    # Ouput completion message
    print(f"Completed month: {month_abbreviations[index].capitalize()}\n")<jupyter_output><empty_output><jupyter_text>Inter-Month Ranges<jupyter_code># Complete for all defined date ranges
for date_range in date_ranges:
    print(f"Started date range: {date_range[0]}-{date_range[-1]}")
    # Initialize list for storing arrays
    array_list = []

    print("Gathering GeoTiffs within range...")
    # Get data within date range over the baseline years
    for geotiff in glob.glob(os.path.join(geotiff_input_folder, "*.tif")):
        for month_day in date_range:
            month = month_day[:2]
            day = month_day[2:]

            # Get arrays from GeoTiffs within the month-day range
            if (
                # Acquisition month
                re.compile(f"^{month}$").match(
                    os.path.basename(geotiff)[12:14]
                )
                # Acquisition day
                and re.compile(f"^{day}$").match(
                    os.path.basename(geotiff)[14:16]
                )
                # Acquisition year
                and os.path.basename(geotiff)[8:12] != "2020"
            ):
                array_list.append(viirs.read_geotiff_into_array(geotiff))

    print(f"Number of arrays: {len(array_list)}")
    print("Calculating statistics...")
    # Calculate mean, variance, standard deviation, and median
    statistics = {
        "mean": {
            "array": viirs.calculate_statistic(array_list, statistic="mean"),
            "file": (
                "vnp46a2-south-korea-2012-2019-mean-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "variance": {
            "array": viirs.calculate_statistic(
                array_list, statistic="variance"
            ),
            "file": (
                "vnp46a2-south-korea-2012-2019-variance-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "deviation": {
            "array": viirs.calculate_statistic(
                array_list, statistic="deviation"
            ),
            "file": (
                "vnp46a2-south-korea-2012-2019-deviation-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "median": {
            "array": viirs.calculate_statistic(array_list, statistic="median"),
            "file": (
                "vnp46a2-south-korea-2012-2019-median-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
    }

    print("Exporting statistics to GeoTiffs...")
    # Export stastistic arrays to GeoTiff
    for statistic in statistics.keys():
        try:
            viirs.export_array(
                array=statistics.get(statistic).get("array"),
                output_path=os.path.join(
                    statistics_output_folder,
                    statistics.get(statistic).get("file"),
                ),
                metadata=metadata,
            )
        except Exception as error:
            print(error)

    print(f"Completed date range: {date_range[0]}-{date_range[-1]}\n")<jupyter_output><empty_output><jupyter_text>Comparison Data (2020)  Monthly<jupyter_code># Calculate statistics for all months over the baseline years
for index, baseline_month in enumerate(month_numbers):
    print(f"Started month: {month_abbreviations[index].capitalize()}")
    print("Gathering GeoTiffs within range...")
    # Get list of arrays for acqusition dates that match the month
    array_list = [
        viirs.read_geotiff_into_array(geotiff)
        for geotiff in glob.glob(os.path.join(geotiff_input_folder, "*.tif"))
        if re.compile(f"^{baseline_month}$").match(
            # Acquisition month
            os.path.basename(geotiff)[12:14]
        )
        # Acquisition year
        and os.path.basename(geotiff)[8:12] == "2020"
    ]

    print(f"Number of arrays: {len(array_list)}")
    print("Calculating statistics...")
    # Calculate mean, variance, standard deviation, and median
    statistics = {
        "mean": {
            "array": viirs.calculate_statistic(array_list, statistic="mean"),
            "file": (
                "vnp46a2-south-korea-2020-mean-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "variance": {
            "array": viirs.calculate_statistic(
                array_list, statistic="variance"
            ),
            "file": (
                "vnp46a2-south-korea-2020-variance-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "deviation": {
            "array": viirs.calculate_statistic(
                array_list, statistic="deviation"
            ),
            "file": (
                "vnp46a2-south-korea-2020-deviation-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
        "median": {
            "array": viirs.calculate_statistic(array_list, statistic="median"),
            "file": (
                "vnp46a2-south-korea-2020-median-"
                f"{baseline_month}-{month_abbreviations[index]}.tif"
            ),
        },
    }

    print("Exporting statistics to GeoTiffs...")
    # Export stastistic arrays to GeoTiff
    for statistic in statistics.keys():
        try:
            viirs.export_array(
                array=statistics.get(statistic).get("array"),
                output_path=os.path.join(
                    statistics_output_folder,
                    statistics.get(statistic).get("file"),
                ),
                metadata=metadata,
            )
        except Exception as error:
            print(error)

    # Ouput completion message
    print(f"Completed month: {month_abbreviations[index].capitalize()}\n")<jupyter_output><empty_output><jupyter_text>Inter-Month Ranges<jupyter_code># Complete for all defined date ranges
for date_range in date_ranges:
    print(f"Started date range: {date_range[0]}-{date_range[-1]}")
    # Initialize list for storing arrays
    array_list = []

    print("Gathering GeoTiffs within range...")
    # Get data within date range over the baseline years
    for geotiff in glob.glob(os.path.join(geotiff_input_folder, "*.tif")):
        for month_day in date_range:
            month = month_day[:2]
            day = month_day[2:]

            # Get arrays from GeoTiffs within the month-day range
            if (
                # Acquisition month
                re.compile(f"^{month}$").match(
                    os.path.basename(geotiff)[12:14]
                )
                # Acquisition day
                and re.compile(f"^{day}$").match(
                    os.path.basename(geotiff)[14:16]
                )
                # Acquisition year
                and os.path.basename(geotiff)[8:12] == "2020"
            ):
                array_list.append(viirs.read_geotiff_into_array(geotiff))

    print(f"Number of arrays: {len(array_list)}")
    print("Calculating statistics...")
    # Calculate mean, variance, standard deviation, and median
    statistics = {
        "mean": {
            "array": viirs.calculate_statistic(array_list, statistic="mean"),
            "file": (
                "vnp46a2-south-korea-2020-mean-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "variance": {
            "array": viirs.calculate_statistic(
                array_list, statistic="variance"
            ),
            "file": (
                "vnp46a2-south-korea-2020-variance-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "deviation": {
            "array": viirs.calculate_statistic(
                array_list, statistic="deviation"
            ),
            "file": (
                "vnp46a2-south-korea-2020-deviation-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
        "median": {
            "array": viirs.calculate_statistic(array_list, statistic="median"),
            "file": (
                "vnp46a2-south-korea-2020-median-"
                f"{date_range[0]}-{date_range[-1]}.tif"
            ),
        },
    }

    print("Exporting statistics to GeoTiffs...")
    # Export stastistic arrays to GeoTiff
    for statistic in statistics.keys():
        try:
            viirs.export_array(
                array=statistics.get(statistic).get("array"),
                output_path=os.path.join(
                    statistics_output_folder,
                    statistics.get(statistic).get("file"),
                ),
                metadata=metadata,
            )
        except Exception as error:
            print(error)

    print(f"Completed date range: {date_range[0]}-{date_range[-1]}\n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12842671955797974, 'fraction_numerical': 0.024367783523411488, 'mean_word_length': 2.0518806744487676, 'pattern_counts': {'":': 48, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '642968', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4767, 'n_tokens_neox': 4268, 'n_words': 740, 'path': '01-code-scripts/calculate_baseline.ipynb', 'repo_name': 'calekochenour/nighttime-radiance', 'size': '14117'}