<jupyter_start><jupyter_text>Single NC verification<jupyter_code>df['file'].nunique()
nc = df['file'][0]
df_nc = df[df['file'] == nc]
margin = 0.2
n_points = 50
df_nc_fil = df_nc[((df_nc['lon']>=-margin) &
          (df_nc['lon']<=margin) &
          (df_nc['lat']>=-margin) &
          (df_nc['lat']<=margin))]
df_nc_fil.pivot('lon', 'lat', 'NO2').values
from scipy import interpolate
x = df_nc_fil['lon'].values
y = df_nc_fil['lat'].values
z = df_nc_fil['NO2'].values
f = interpolate.interp2d(x, y, z, kind='linear',bounds_error=False, fill_value=np.nan)
xnew = np.linspace(-margin,margin,n_points)
ynew = np.linspace(-margin,margin,n_points)
znew = f(xnew, ynew)
znew
plt.figure(figsize=(10,10))
plt.imshow(df_nc_fil.pivot('lon', 'lat', 'NO2').values, cmap='hot', interpolation='none')<jupyter_output><empty_output><jupyter_text>Lat and long are not equally separated, so the above representation is distorted.Current interpolation:<jupyter_code>plt.figure(figsize=(10,10))
plt.imshow(znew, cmap='hot', interpolation='nearest')
# Gets time of the satellite over the closest latitude to the sensor and rounds to next hour
pd.to_datetime(df_nc_fil[df_nc_fil.index == df_nc_fil['lon'].abs().idxmin()]['date']).dt.round('h')
df_nc_fil = df_nc[((df_nc['lon']>=low_lon) &
          (df_nc['lon']<=up_lon) &
          (df_nc['lat']>=low_lat) &
          (df_nc['lat']<=up_lat))]
df_nc_fil
df_nc[df_nc['lon']<=0].sort_values('lon').iloc[-5]
df_nc[df_nc['lon']<=0].sort_values('lon').iloc[-5]
res = df_nc.pivot('lon', 'lat', 'NO2')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20257234726688103, 'fraction_numerical': 0.01414790996784566, 'mean_word_length': 6.2372093023255815, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '563251', 'license': 'MIT', 'n_tokens_mistral': 641, 'n_tokens_neox': 625, 'n_words': 97, 'path': 'code/sentinel_data_import.ipynb', 'repo_name': 'whereisthepollution/WHEREISTHEPOLLUTION', 'size': '1555'}