<jupyter_start><jupyter_text>XtractomaticpyA Notebook for loading in environmental data, transforming it into points and into a `pandas` df, and combining it with the tag data.<jupyter_code>import datetime as dt
import xarray as xr
import numpy as np
import pandas as pd
from typing import Dict, Union
import fsspec
import matplotlib.pyplot as plt
from datetime import datetime 
shark_dir = "track_shark144020.csv"
track_ex = pd.read_csv(shark_dir, parse_dates=['datetime']) # in pandas, read_csv

# track_ex["lon"] = np.where(
#     track_ex["lon"] < 180,
#     track_ex["lon"] + 360,
#     track_ex["lon"])

lat_min = track_ex["lat"].min() - 2.0
lat_max = track_ex["lat"].max() + 2.0
lon_min = track_ex["lon"].min() - 2.0
lon_max = track_ex["lon"].max() + 2.0

xy_bbox = dict(latitude=slice(lat_min,lat_max), longitude=slice(lon_min,lon_max))

plt.plot(track_ex.lon,track_ex.lat)

xy_bbox
track_ex
# grab track data for a few tag datapoints
track_2014 = track_ex.iloc[0:100]
track_2014<jupyter_output><empty_output><jupyter_text>load in environmental datahere is the SST from MUR<jupyter_code># bring in data for SST
file_location = 's3://mur-sst/zarr'
ikey = fsspec.get_mapper(file_location, anon=True)
ds_sst = xr.open_zarr(ikey,consolidated=True)
ds_sst

# we should put the SST through the subsetting workflow that Felippe worked up. Will run a lot faster<jupyter_output><empty_output><jupyter_text>here is the SSH and velocity from CMEMS<jupyter_code>ds_ssh = xr.open_dataset("data/ssh_data.nc")
# Converting the lat lon to -180 180
with xr.set_options(keep_attrs=True):
    ds_ssh = ds_ssh.assign({'longitude':(((ds_ssh.longitude + 180) % 360) - 180)})

ds_ssh_renamed = ds_ssh.rename({'latitude':'lat', 'longitude':'lon', 'time':'time'}) # "old name" : "new name"
ds_ssh_renamed<jupyter_output><empty_output><jupyter_text>here is the Ocean Color data from MODISA, accessed via OpenDap<jupyter_code># calculate year day for time range of tagged data
day_list = []
year_list = []
for index, row in track_2014.iterrows():
    row_time = pd.to_datetime(row["datetime"])
    day_of_year = datetime(row_time.year, row_time.month, row_time.day).timetuple().tm_yday
    year_list.append(row_time.year)
    day_list.append(day_of_year)
day_string = [str(x) for x in day_list]
year_string = [str(x) for x in year_list]

# create access url for ocean color on opendap, merge datafiles to xarray
url = []
base_dir = 'https://oceandata.sci.gsfc.nasa.gov/opendap/hyrax/MODISA/L3SMI/'
suffix = '.L3m_DAY_CHL_chlor_a_4km.nc'

k = 0
for day in day_string:
    url.append('https://oceandata.sci.gsfc.nasa.gov:443/opendap/MODISA/L3SMI/' + year_string[k] +'/' + day + '/A'+year_string[k] + day + '.L3m_DAY_CHL_chlor_a_4km.nc')
    k = k+1

def add_id(ds):
    ds.coords['time_coverage_start'] = pd.to_datetime(ds.attrs['time_coverage_start'])
    return ds

chl = xr.open_mfdataset(url, combine = 'nested', concat_dim='time_coverage_start', preprocess=add_id)
chl = chl.sel( lat=slice(lat_max, lat_min), lon=slice(lon_min,lon_max)) # these data have the lat indexed backwards... no idea why, but it works this way
chl
# extraction and combining here
# for each environmental data set, change the object name in the envdata_point() function


def fuction_dataset_point(**kwargs) -> Dict[str, Union[float, int]]:
    pass




def extract(function_dataset_point, 
            df: pd.DataFrame, 
            map_coordinates: Dict[str, str], 
            rename_variables: Dict[str, str]) -> pd.DataFrame:
    """
    function_dataset_point: environmental data in a point format, to be transformed
    df tag data in a pandas format
    map_coordinates: key is name of column in dataframe, value is the name of the coordinate in dataset
    rename_variables: TBD
    """
    
    def get_row(row) -> Dict[str, Union[float, int]]:
        extract_coordinates = {}
        
        for key, val in map_coordinates.items():
            extract_coordinates[val] = row[key]
        
        result = function_dataset_point(**extract_coordinates)
        
        # rename variables here and transform result
        return result
    
    return df.apply(
        lambda row: get_row(row), axis=1, result_type="expand"
    )



# CHANGE YOUR ENVIRONMENTAL DATA SOURCE HERE
def ssh_point(lat, lon, time) -> Dict[str, Union[float, int]]: # change variable here
    ds = ds_ssh_renamed.sel(lat=lat, lon=lon, time=time, method="nearest") # change variable here

    results = {}
    
    for var in ds.variables:
        if var not in ds.coords:
            results[var] = ds[var].values
    
    return results

result = pd.concat([track_2014, 
                    extract(ssh_point, # change variable here
                            track_2014, 
                            {"lat": "lat", "lon": "lon", "datetime": "time"}, # change variable here, df : ds
                            {}
                           )
                   ], axis=1)
result # sst in Kelvin?

result.to_csv('ssh_matched.csv')
result['mag'] = (result['ugos']**2 + result[vgos**2] ** 0.5)
result<jupyter_output><empty_output><jupyter_text>Generalized function `xtractopy()` **Before running:** you must rename the coordinates of the environmental xArray, otherwise the code may not run. I want to integrate this into the function itself so the user only has to pass a string of characters, but I have no idea how to implement this. Template code for renaming coordinates:<jupyter_code>ds_envdata_renamed = ds_envdata.rename({"latitude":"lat", "longitude":"lon","time":"time"}) # "old name":"new name"
ds_envdata_renamed # check

def xtractopy(envdata,
              tagdata: pd.DataFrame,
             filename: [str]):
    """
    envdata: environmental data in an DataArray format
    tagdata: tag data in a pandas format
    filename: the name of the file .csv output file, as a "string"
    """
    def fuction_dataset_point(**kwargs) -> Dict[str, Union[float, int]]:
        pass

    def extract(function_dataset_point, 
                df: tagdata, 
                map_coordinates: Dict[str, str], 
                rename_variables: Dict[str, str]
               ) -> pd.DataFrame:
        """
        function_dataset_point: environmental data in a point format, to be transformed
        map_coordinates: key is name of column in dataframe, value is the name of the coordinate in dataset
        rename_variables: TBD
        """
    
        def get_row(row) -> Dict[str, Union[float, int]]:
            extract_coordinates = {}
        
            for key, val in map_coordinates.items():
                extract_coordinates[val] = row[key]
        
            result = function_dataset_point(**extract_coordinates)
        
            # rename variables here and transform result TBD
            return result
    
        return df.apply(
            lambda row: get_row(row), axis=1, result_type="expand"
        )


    def envdata_point(lat, lon, time) -> Dict[str, Union[float, int]]:
        ds = envdata.sel(lat=lat, lon=lon, time=time, method="nearest")

        results = {}
    
        for var in ds.variables:
            if var not in ds.coords:
                results[var] = ds[var].values
    
        return results

    combined_dat = pd.concat([tagdata, 
                        extract(envdata_point,
                                tagdata, 
                                {"lat": "lat", "lon": "lon", "datetime": "time"}, 
                                {}
                               )
                       ], axis=1)
    combined_dat.to_csv("".join([filename, ".csv"])) # need to figure out how to paste the title into the csv file
    return combined_dat

# test
# xtractopy(ds_sst, track_2014, "testdata")
# test with new shark data
shark_dir = "track_shark137736.csv"
track_2 = pd.read_csv(shark_dir, parse_dates=['datetime']) # in pandas, read_csv
xtractopy(ds_sst, track_2, "sst_matched_3years_mag_shark2")
# test with SSH data
xtractopy(ds_ssh_renamed, track_2, "ssh_matched_3years_mag_shark2")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11044701780600175, 'fraction_numerical': 0.011953679491968622, 'mean_word_length': 3.070957932083122, 'pattern_counts': {'":': 10, '<': 22, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '642260', 'license': 'MIT', 'n_tokens_mistral': 2729, 'n_tokens_neox': 2585, 'n_words': 694, 'path': 'OHW_development/developmental_materials/xtractomaticpy-Copy2_AC.ipynb', 'repo_name': 'atchin/OHW21_proj_tag_data', 'size': '8031'}