<jupyter_start><jupyter_text>Extract data by geography<jupyter_code># Generate two queries to select the Eastern and Western halves of the data around the international date line
adm0 = gpd.read_file('/home/jupyter-wb411133/data/regional/pacific_observatory/admin/Adm0_Pacific.shp')
adm0 = adm0.to_crs('epsg:4326')
adm0['min_lon'] = adm0['geometry'].apply(lambda x: x.bounds[0])
adm0['min_lat'] = adm0['geometry'].apply(lambda x: x.bounds[1])
adm0['max_lon'] = adm0['geometry'].apply(lambda x: x.bounds[2])
adm0['max_lat'] = adm0['geometry'].apply(lambda x: x.bounds[3])

# As the set of countries cross the international date line, we need to do some math to 
#    determine the actual longitude bounds
westing = adm0.loc[adm0['min_lon'] > 0]['min_lon'].min()
easting = adm0.loc[adm0['max_lon'] < 0]['max_lon'].max()
southing = adm0.total_bounds[1]
northing = adm0.total_bounds[3]

print(f'{westing} : {easting}, {southing} : {northing}')
inD = dd.read_csv('/home/jupyter-wb411133/data/global/AIS/WorldBank_SAIS_globalAOI_20190101_20201231_0000000031**.csv', dtype=col_types)
inD.head()
# Subset to just fishing boats
inD = inD.loc[inD['ship_and_cargo_type'] == '30']

#Now, do two select by locations 
tPrint("Starting")
eastern_hemi = inD.loc[(inD['longitude'] > westing) & (inD['latitude'] > southing) & (inD['latitude'] < northing)].compute()
tPrint("Eastern Done")
western_hemi = inD.loc[(inD['longitude'] < easting) & (inD['latitude'] > southing) & (inD['latitude'] < northing)].compute()
tPrint("Western Done")<jupyter_output>14:51:40	Starting<jupyter_text>Generate routes per boat<jupyter_code>importlib.reload(ais)
out_folder = '/home/jupyter-wb411133/data/global/AIS_output'
    
if not os.path.exists(out_folder):
    os.makedirs(out_folder)

def create_ship_tracks(curD):
    try:
        mmsi = curD['mmsi'][0]
        out_folder = '/home/jupyter-wb411133/data/global/AIS_output'
        curAIS = ais.ais_helper(inD = curD)
        curShp = curAIS.read_simple_geom()
        lines = ais.generate_linear_features(curShp)
        lines['mmsi'] = mmsi
        out_file = os.path.join(out_folder, f"{mmsi}.geojson")
        try:
            lines.to_file(out_file, driver="GeoJSON")
        except:
            pass
        return(lines)
    except:
        pass
    
res = inD.groupby("mmsi").apply(create_ship_tracks).compute()
inD.groupby("mmsi").count().compute()
importlib.reload(ais)
curD = inD.loc[inD['mmsi'] == unique_boats[0]]
curD.head()
importlib.reload(ais)

importlib.reload(ais)

in_file = 's3://wbgdecinternal-ntl/AIS_TMP/MMSI_247298800.csv'
inD = ais.ais_file(in_file)
tempD = inD.read_simple_geom()
tempD['DAY'] = tempD['timestamp'].apply(lambda x: x[:10])
tempD.to_file(f'/home/wb411133/temp/AIS/{os.path.basename(in_file).replace(".csv","")}.geojson', driver="GeoJSON")
# Generate lines file for individual ship data
importlib.reload(ais)
line_features = ais.generate_linear_features(tempD)
line_features.to_file(f'/home/wb411133/temp/AIS/{os.path.basename(in_file).replace(".csv","_lines_simple")}.geojson', driver="GeoJSON")<jupyter_output><empty_output><jupyter_text>Processing daily files<jupyter_code>daily_file = 's3://wbgdecinternal-ntl/AIS_TMP/MAY2.csv'
inD = pd.read_csv(daily_file)
inD = inD.loc[~inD['latitude'].isna()]
inD['DAY'] = inD['timestamp'].apply(lambda x: x[:10])
inD['DAY'].value_counts()
inD_geom = [Point(x['longitude'], x['latitude']) for idx, x in inD.iterrows()]
inGPD = gpd.GeoDataFrame(inD, geometry=inD_geom, crs='epsg:4326')
rMisc.rasterizeDataFrame?
out_file = "/home/wb411133/temp/AIS/may2.tif"
rMisc.rasterizeDataFrame(inGPD, out_file, res=0.1, mergeAlg="ADD")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16666666666666666, 'fraction_numerical': 0.03585112205801861, 'mean_word_length': 5.66970802919708, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '606241', 'license': 'MIT', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1431, 'n_words': 217, 'path': 'notebooks/AIS_Data_exploration.ipynb', 'repo_name': 'worldbank/GOST_AIS', 'size': '3654'}