<jupyter_start><jupyter_text>IntroThis notebook presents the workflow to retrieve land cover transitions between two dates from GEE. - It requires three datasets: landcover, mountains (bioclimatic ranges) and a countries dataset- The input paraemeters are the desired dates, landcover transition matrix and ESA CCI translation matrix.<jupyter_code>import ee
import pandas as pd
from functools import partial
import sepal_ui.aoi as aoi
aoi_tile = aoi.AoiTile()
aoi_tile
ee.Initialize()
landcover = ee.ImageCollection("users/amitghosh/sdg_module/esa/cci_landcover")
mountains = ee.Image("users/xavidelamo/SDG1542_Mntn_BioclimaticBelts")
countries = ee.FeatureCollection("projects/ee-xavidelamo/assets/M49_Countries");
start_year = 2005
end_year = 2010
from_, target = (
    pd.read_csv("data/cci_map_matrix.csv")[["from_code", "target_code"]]
    .transpose()
    .values.tolist()
)
lc_f_class = [f"lc_f{c}" for c in range(1, 10 + 1)]
def export_map(image, name, aoi, folder="RSA"):
    """Export images"""

    if not isinstance(image, ee.Image):
        raise Exception("The source has to be an image")

    test_task = ee.batch.Export.image.toDrive(
        **{
            "region": aoi.geometry(),
            "image": image,
            "folder": "RSA",
            "description": name,
        }
    )

    test_task.start()
def reduce_regions(iso_country, map_=False):
    """Reduce regions by bioclimatic belt, lcyear1 and lcyear2

    iso_country  (str, FeatureCollection): If str, it will be the isocode, otherwise,
        it will be a feature collection use to clip with.

    map_ (bool, optional): Whether we are calling this function from a ee.map function or not

    """

    aoi = (
        countries.filter(ee.Filter.eq("ISOCode", iso_country)) if map_ else iso_country
    )

    clip_mountains = mountains.clip(aoi)

    landcover_start = (
        landcover.filter(ee.Filter.calendarRange(start_year, start_year, "year"))
        .first()
        .clip(aoi)
        .remap(from_, target)
    )

    landcover_end = (
        landcover.filter(ee.Filter.calendarRange(end_year, end_year, "year"))
        .first()
        .clip(aoi)
        .remap(from_, target)
    )

    scale = (
        landcover_start.projection()
        .nominalScale()
        .min(clip_mountains.projection().nominalScale())
    )

    zonal_change = (
        ee.Image.pixelArea()
        .divide(1e6)
        .updateMask(clip_mountains.mask())
        .addBands(landcover_end)
        .addBands(landcover_start)
        .addBands(clip_mountains)
        .reduceRegion(
            **{
                "scale": scale,
                "reducer": ee.Reducer.sum().group(1).group(2).group(3),
                "geometry": aoi,
                "maxPixels": 1e19,
                "bestEffort": True,
                # "tileScale": 4,
            }
        )
    )

    return (
        [
            clip_mountains,
            landcover_start,
            landcover_end,
            ee.List([iso_country if iso_country else "Test", zonal_change]),
        ]
        if not map_
        else ee.List([iso_country, zonal_change])
    )<jupyter_output><empty_output><jupyter_text>Loop over all countries<jupyter_code>result = countries.aggregate_array("ISOCode").map(partial(reduce_regions, map_=True))

all_areas = ee.FeatureCollection(ee.Feature(None, {"results": result}))

task = ee.batch.Export.table.toDrive(
    **{
        "collection": all_areas,
        "description": f"grouped_data_{start_year}_{end_year}",
        "fileFormat": "CSV",
    }
)
task.start()<jupyter_output><empty_output><jupyter_text>Use test data with only one feature collection<jupyter_code># Use test
test_aoi = aoi_tile.view.model.feature_collection
test_result = reduce_regions(test_aoi)
test_area = ee.FeatureCollection(ee.Feature(None, {"results": test_result[3]}))
test_task = ee.batch.Export.table.toDrive(
    **{
        "folder": "RSA",
        "collection": test_area,
        "description": "test_grouped_2000_2015_in_mountain3",
        "fileFormat": "CSV",
    }
)
# test_task.start()

# Also export layers

export_map(test_result[0], "test_mountains", test_aoi)
export_map(test_result[1], "test_lcstart", test_aoi)
export_map(test_result[2], "test_lcend", test_aoi)<jupyter_output><empty_output><jupyter_text>Read resultsThe following worflow will read the GEE output and will un-nest the data for each country, bioclimatic belt, and land cover change classes.<jupyter_code>import pandas as pd
import re
df = pd.read_csv("data/global_group_1_2_3/grouped_data_2015_2018.csv")
df;
line = (
    re.sub(r"([a-zA-Z]+)", r"'\1'", df.results.loc[0])
    .replace("=", ":")
    .replace("'E'", "E")
    .replace("<'FeatureCollection'>,", "")
)
area = eval(line)
df = pd.DataFrame(area, columns=["iso_code", "groups"])
# df = df[df.iso_code.isin(["COL", "ARG"])].reset_index(drop=True).reset_index() # Test with a small df
df = df.reset_index()
df.rename(columns={"index": "iso_index"}, inplace=True)
df;
base_df = pd.json_normalize(df["groups"])
base_df = pd.json_normalize(base_df["groups"]).reset_index()
base_df.rename(columns={"index": "iso_index"}, inplace=True)
base_df;
belts_df_melted = (
    pd.melt(base_df, id_vars=["iso_index"])
    .reset_index()[["iso_index", "value"]]
    .reset_index()
)
belts_df_melted.rename(columns={"index": "belt_index"}, inplace=True)
belts_df_melted;
belt_df = pd.json_normalize(belts_df_melted["value"]).reset_index()
belt_df.rename(columns={"index": "belt_index", "group": "belt_class"}, inplace=True)
belt_df;
from_df = pd.json_normalize(belt_df["groups"]).reset_index()
from_df.rename(columns={"index": "belt_index"}, inplace=True)
from_df_melted = pd.melt(from_df, id_vars=["belt_index"]).reset_index()[
    ["index", "belt_index", "value"]
]
from_df_melted.rename(columns={"index": "from_index"}, inplace=True)
from_df_melted;
from_df = pd.json_normalize(from_df_melted["value"]).reset_index()
from_df.rename(columns={"index": "from_index", "group": "from_class"}, inplace=True)
from_df;
to_df = pd.json_normalize(from_df["groups"]).reset_index()
to_df.rename(columns={"index": "from_index"}, inplace=True)
to_df_melted = pd.melt(to_df, id_vars=["from_index"]).reset_index()[
    ["index", "from_index", "value"]
]
to_df_melted.rename(columns={"index": "to_index"}, inplace=True)
to_df_melted;
to_df = pd.json_normalize(to_df_melted["value"]).reset_index()
to_df.rename(columns={"index": "to_index", "group": "to_class"}, inplace=True)
to_df;<jupyter_output><empty_output><jupyter_text>Merge all the dataframesGo back and merge every dataframe with their indexes.<jupyter_code>merge_to_df = pd.merge(
    to_df_melted[["to_index", "from_index"]],
    to_df,
    left_on="to_index",
    right_on="to_index",
)
merge_to_df;
merge_from_df = pd.merge(
    from_df[["from_index", "from_class"]],
    merge_to_df,
    left_on="from_index",
    right_on="from_index",
)
merge_from_df = pd.merge(
    from_df_melted[["from_index", "belt_index"]],
    merge_from_df,
    left_on="from_index",
    right_on="from_index",
)
merge_from_df;
merge_belt_df = pd.merge(
    belt_df[["belt_index", "belt_class"]],
    merge_from_df,
    left_on="belt_index",
    right_on="belt_index",
)
merge_belt_df = pd.merge(
    belts_df_melted[["belt_index", "iso_index"]],
    merge_belt_df,
    left_on="belt_index",
    right_on="belt_index",
)
merge_belt_df;
merged_df = pd.merge(
    df[["iso_index", "iso_code"]],
    merge_belt_df,
    left_on="iso_index",
    right_on="iso_index",
)
merged_df;
merged_df.to_csv("data/global_group_1_2_3/global_grouped_data_2015_2018.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1453732912723449, 'fraction_numerical': 0.008675078864353312, 'mean_word_length': 4.1412162162162165, 'pattern_counts': {'":': 33, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '525438', 'license': 'MIT', 'n_tokens_mistral': 2832, 'n_tokens_neox': 2740, 'n_words': 472, 'path': 'component/scripts/new_indicator/1_reduce_results.ipynb', 'repo_name': 'sepal-contrib/sepal_mgci', 'size': '7608'}