<jupyter_start><jupyter_text>Read irradiances in zarr format<jupyter_code>CERES_GOES_VERSION = '2022_04_25_1650910589'
coin = xarray.open_zarr(f'gs://upwelling_irradiance/ceres_goes/{CERES_GOES_VERSION}/2018/021/16/s20180211630395_olr', consolidated=True)
coin
plt.figure(figsize=(15,15))
plt.imshow(coin.predicted_olr, cmap='magma')<jupyter_output><empty_output><jupyter_text>Read tfrecord training/validation data<jupyter_code>import tensorflow as tf
tf.io.FixedLenFeature([], tf.string).dtype == tf.string
spec = {
    'surface_type_index': tf.io.FixedLenFeature([], tf.string),
    'surface_type_percent_coverage': tf.io.FixedLenFeature([], tf.string),
    'psf_weights': tf.io.FixedLenFeature([], tf.string),
    'goes_lats': tf.io.FixedLenFeature([], tf.string),
    'goes_lngs': tf.io.FixedLenFeature([], tf.string),
    'goes_timestamp': tf.io.FixedLenFeature([], tf.int64),
    'ceres_timestamp': tf.io.FixedLenFeature([], tf.int64),
    'ceres_olr': tf.io.FixedLenFeature([], tf.float32),
    'ceres_rsr': tf.io.FixedLenFeature([], tf.float32),
}
for channel in range(1, 17):
  spec[f'goes_channel_{channel}'] = tf.io.FixedLenFeature([], tf.string)

def _parse_examples(serialized):
  return tf.io.parse_example(serialized, spec)

validation_dataset = tf.data.TFRecordDataset(
    f'gs://upwelling_irradiance/ceres_goes/{CERES_GOES_VERSION}/training_and_validation_data/before_20190402/validation.tfrecords-00000-of-00100')
validation_dataset = validation_dataset.map(_parse_examples, num_parallel_calls=tf.data.AUTOTUNE)

# Meta features are useful for analysis of model bias on subsets of the data.
META_FEATURE_NAMES = ('ceres_latitude', 'ceres_longitude',
                      'ceres_viewing_zenith', 'ceres_relative_azimuth',
                      'ceres_solar_zenith', 'calculated_solar_zenith')

for feature_name in META_FEATURE_NAMES:
  spec[feature_name] = tf.io.FixedLenFeature([], tf.float32)


def _parse_tensors(features):
  for feature_name in features:
    if spec[feature_name].dtype == tf.string:
      features[feature_name] = tf.io.parse_tensor(
          features[feature_name], out_type=tf.float32)
          
  return features

validation_dataset = validation_dataset.map(_parse_tensors, num_parallel_calls=tf.data.AUTOTUNE)

for features in validation_dataset.take(1):
  break

features<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12936170212765957, 'fraction_numerical': 0.03361702127659574, 'mean_word_length': 5.698005698005698, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '646487', 'license': 'Apache-2.0', 'n_tokens_mistral': 896, 'n_tokens_neox': 842, 'n_words': 121, 'path': 'collocated_irradiance_network/Collocated_Irradiance_Network_data_loading.ipynb', 'repo_name': 'santakd/google-research', 'size': '2350'}