<jupyter_start><jupyter_text>Store alaska inventory as parquet format<jupyter_code># seisbench planetary computer environment
!mamba env create --quiet -f ../seisbench-testing/environment.yml<jupyter_output>Preparing transaction: ...working... done
Verifying transaction: ...working... done
Executing transaction: ...working... Enabling notebook extension jupyter-js-widgets/extension...
      - Validating: [32mOK[0m

done
Installing pip dependencies: ...working... done<jupyter_text>QuakeMLReading quakeML is slow. We can extract relevant data from this file into a tablular format and save to a more cloud-friendly performant format like Parquet https://parquet.apache.org/documentation/latest/<jupyter_code>import obspy
import obsplus
import pandas as pd
from obspy.clients.fdsn import Client
from datetime import timedelta
%%time

cat = obspy.core.event.read_events('../alaska_catalog/data_acquisition/catalog_css/2018_10/XO_2018_10.quakeml') #obspy/io/quakeml/core.py:184: UserWarning: Could not convert 2020-11-19T01:12:60.000Z to type <class 'obspy.core.utcdatetime.UTCDateTime'>. Returning None.
print(cat)
type(cat)<jupyter_output><empty_output><jupyter_text>NOTE: use obsplus package to facilitate dataframe conversions  Events<jupyter_code>%%time

dfE = obsplus.events_to_df(cat)
dfE.head()
dfE.iloc[0]
# ArrowInvalid: Casting from timestamp[ns] to timestamp[us] would lose data: 1610761489008999936
# https://github.com/pandas-dev/pandas/issues/31572 
# df.to_parquet('XO_2018_10.parquet')
dfE.to_parquet('XO_2018_10_events.parquet', version='2.6')
# Round trip

df = pd.read_parquet('XO_2018_10_events.parquet')
df.head()<jupyter_output><empty_output><jupyter_text>Arrivals<jupyter_code>%%time 

dfA = obsplus.arrivals_to_df(cat)
dfA
dfA.iloc[0]
dfA.to_parquet('XO_2018_10_arrivals.parquet', version='2.6')
# round trip
df = pd.read_parquet('XO_2018_10_arrivals.parquet')
df.head()<jupyter_output><empty_output><jupyter_text>Picks<jupyter_code>%%time
dfP = obsplus.picks_to_df(cat)
dfP
dfP.iloc[0]
dfP.to_parquet('XO_2018_10_picks.parquet', version='2.6')
df = pd.read_parquet('XO_2018_10_picks.parquet')
df.head()
!ls -ltrh XO*<jupyter_output>-rw-rw-r-- 1 jovyan users   30M Jan 19 00:46 XO_2018_10.quakeml
-rw-rw-r-- 1 jovyan users   85K Jan 19 16:46 XO_2018_10.parquet
-rw-r--r-- 1 jovyan users   85K Jan 19 18:34 XO_2018_10_events.parquet
-rw-r--r-- 1 jovyan users 1016K Jan 19 18:40 XO_2018_10_arrivals.parquet
-rw-r--r-- 1 jovyan users  772K Jan 19 18:41 XO_2018_10_picks.parquet<jupyter_text>Stations and response<jupyter_code># obspy.core.event.catalog.Catalog doesn't include lat,lon, etc.
# better to pass obspy.core.inventory.inventory.Inventory from IRIS search...
dfS = obsplus.stations_to_df(cat)
dfS
# NOTE: not sorted in chronological order... how sorted?
print(dfE.time.min())
print(dfE.iloc[0].time)
network = "XO"
channel = "BH*,BN*,EH*,HH*,HN*,SH*"
#station = None
station = ','.join(dfS.query(' network == "XO" ').station.unique().tolist())
starttime = dfE.time.min()


inventory = Client("iris").get_stations(starttime = starttime, 
                                           endtime  = starttime + timedelta(weeks=4),
                                           network = network,
                                           channel = channel,
                                           station = station,
                                           level = 'response',
                                           location = '')
    
print(type(inventory))
print(inventory)
dfS = obsplus.stations_to_df(inventory)
dfS
s = inventory[0][0]
s
c = inventory[0][0][0] #[network][station][channel]
c
r = c.response
r
r.instrument_sensitivity.input_units.lower()
r.instrument_sensitivity.value
starttime
inventory.get_channel_metadata('XO.EP14..HHE')
def get_response_value(inventory, seed_id, start_date):
    ''' given obsplus station dataframe and obspy inventory, get instrument response string '''
    response = inventory.get_response(seed_id, start_date)
    return response.instrument_sensitivity.value
dfS['response'] = dfS.apply(lambda row: get_response_value(inventory, row.seed_id, row.start_date), axis=1)
def get_response_units(inventory, seed_id, start_date):
    ''' given obsplus station dataframe and obspy inventory, get instrument response string '''
    response = inventory.get_response(seed_id, start_date)
    return response.instrument_sensitivity.input_units.lower()
dfS['units'] = dfS.apply(lambda row: get_response_units(inventory, row.seed_id, row.start_date), axis=1)
dfS.head()
columns = ['longitude','latitude','elevation','network','sample_rate']
dfS.groupby('station')[columns].first()
list_agg = dfS.groupby(by='name').agg({'channel': lambda x: list(x), 
                                          'response': lambda x: list(x)
                                         },
                                        )
list_agg<jupyter_output><empty_output><jupyter_text>Remote Read<jupyter_code>%%time

dfP = pd.read_parquet('https://github.com/scottyhq/quakeflow-examples/raw/main/XO_2018_10_picks.parquet')
dfE = pd.read_parquet('https://github.com/scottyhq/quakeflow-examples/raw/main/XO_2018_10_events.parquet')
dfA = pd.read_parquet('https://github.com/scottyhq/quakeflow-examples/raw/main/XO_2018_10_arrivals.parquet')<jupyter_output>CPU times: user 215 ms, sys: 28.4 ms, total: 244 ms
Wall time: 1.91 s
{'fraction_non_alphanumeric': 0.12539593814048816, 'fraction_numerical': 0.04266815725731321, 'mean_word_length': 3.9703703703703703, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '402148', 'license': 'MIT', 'n_tokens_mistral': 2039, 'n_tokens_neox': 1860, 'n_words': 357, 'path': 'inventory-parquet.ipynb', 'repo_name': 'scottyhq/quakeflow-examples', 'size': '5367'}