<jupyter_start><jupyter_text>Start in on rotations<jupyter_code># Read in parquet data
fm_10hz_df = pd.read_parquet(os.path.join(base_dir, '10hz_versions', 'fm_0621_10hz_v01.parq'), engine='pyarrow')<jupyter_output><empty_output><jupyter_text>Process EC data<jupyter_code>importlib.reload(coord_rotate)
importlib.reload(conversions)
time_freq = '5T'

# Convert SOS to temperature

fm_10hz_df['t_sonic'] = conversions.sound_to_ts_K(fm_10hz_df['C'], eq_type='gill_manual')

# Rotate sonics to get bear-naive wind directions and yaw/pitch angles from upcoming DR

def rotate_2d(x0, y0, theta):
    """
    Rotate point or vector by theta (+ anticlockwise)
    
    Args:
        x0 (float) : x-component of wind vector (u) [m/s]
        y0 (float) : y-component of wind vector (v) [m/s]
        theta (float) : angle of rotation (+ anticlockwise) [deg]
        
    Returns:
        x1 (float) : rotated x-component of wind vector
        y1 (float) : rotated y-component of wind vector
    """
    
    cos_theta = np.cos(np.radians(theta))
    sin_theta = np.sin(np.radians(theta))
    
    x1 = x0 * cos_theta - y0 * sin_theta
    y1 = x0 * sin_theta + y0 * cos_theta
    
    return x1, y1
    
# https://www.eol.ucar.edu/content/wind-direction-quick-reference
# NOTE: Might move all rotations to the wd_mean assignment, prolly doesn't matter?
fm_10hz_df['Ux_rot_met'], fm_10hz_df['Uy_rot_met'] = rotate_2d(fm_10hz_df['Ux'], fm_10hz_df['Uy'], 42)

fm_10hz_5min_avg = fm_10hz_df.resample(time_freq, closed='right', label='right').mean()

fm_10hz_5min_avg['wd_instrument'] = mpcalc.wind_direction(fm_10hz_5min_avg['Ux_rot_met'].values * units('m/s'), fm_10hz_5min_avg['Uy_rot_met'].values * units('m/s'))
fm_10hz_5min_avg['wd_mean'] = (fm_10hz_5min_avg['wd_instrument'] + 270) % 360
fm_10hz_5min_avg['ws_mean'] = np.sqrt(fm_10hz_5min_avg['Ux']**2 + fm_10hz_5min_avg['Uy']**2 + fm_10hz_5min_avg['Uz']**2)
fm_10hz_5min_avg['yaw_angle'] = coord_rotate.yaw_angle(fm_10hz_5min_avg['Ux'], fm_10hz_5min_avg['Uy'])
fm_10hz_5min_avg['pitch_angle'] = coord_rotate.pitch_angle(fm_10hz_5min_avg['Ux'], fm_10hz_5min_avg['Uy'], fm_10hz_5min_avg['Uz'])

# Calculate DR
fm_10hz_dr = coord_rotate.double_rotate_pd(fm_10hz_df.loc[:,['Ux','Uy','Uz']], '5T')
fm_10hz_all = pd.concat([fm_10hz_df, fm_10hz_dr], axis=1)

# Covariances
cov_5min = fm_10hz_all.groupby(pd.Grouper(freq=time_freq, closed='right', label='right')).cov()

cov_pairs = {
    'Ux_Ux' : ['Ux', 'Ux'],
    'Ux_Uy' : ['Ux', 'Uy'],
    'Ux_Uz' : ['Ux', 'Uz'],
    'Uy_Uy' : ['Uy', 'Uy'],
    'Uy_Uz' : ['Uy', 'Uz'],
    'Uz_Uz' : ['Uz', 'Uz'],
    'Ux_t_sonic' : ['Ux', 't_sonic'],
    'Uy_t_sonic' : ['Uy', 't_sonic'],
    'Uz_t_sonic' : ['Uz', 't_sonic']
}

# Assign covariances to running 5 minute dataframe
for k, pair in cov_pairs.items():
    fm_10hz_5min_avg[f'{k}_cov'] = cov_5min.loc[(slice(None),pair[0]),pair[1]].droplevel(1)

# Calculate variables needed for fluxes, then merge everything into one happy df

pres = met_utils.pressure_fao(1985.16)
fm_5min_df['rho_air'] = met_utils.rho_air(fm_5min_df['e_a_Avg']*1.e3, pres*1.e2, fm_5min_df['AirTC_Avg']+273.15)
fm_5min_df['cp_air'] = met_utils.c_p(met_utils.specific_humidity(fm_5min_df['e_a_Avg']*1e3, pres*1e2))
fm_5min_full = pd.concat([fm_5min_df, fm_10hz_5min_avg], axis=1).sort_index().asfreq('5T')

# Calculate sensible heat fluxes and momentum fluxes using slow data temp/rh

for cov in list(cov_pairs.keys())[-3:]:
    fm_5min_full[f'Qh_{cov}'] = fm_5min_full['rho_air'] * fm_5min_full['cp_air'] * fm_5min_full[f'{cov}_cov']

for cov in list(cov_pairs.keys())[:-3]:
    fm_5min_full[f'Qm_{cov}'] = fm_5min_full['rho_air'] * fm_5min_full[f'{cov}_cov']

fm_5min_full['tke'] = .5 * (fm_5min_full['Ux_Ux_cov'] + fm_5min_full['Uy_Uy_cov'] + fm_5min_full['Uz_Uz_cov'])
# Rename vars and drop as needed
drop_cols = ['RECORD', 'Ux_rot_met', 'Uy_rot_met', 'wd_instrument', 'Ux', 'Uy', 'Uz', 'C']

rename_dict = {
	'PTemp_Avg' : 'p_temp',
	'Batt_volt_Min' : 'batt_volt_min',
	'Ux_Avg' : 'Ux',
	'Uy_Avg' : 'Uy',
	'Uz_Avg' : 'Uz',
	'C_Avg' : 'C',
	'T_sonic_Avg' : 't_sonic_f0_C',
	'AirTC_Avg' : 't_air',
	'RH_Avg' : 'rh',
	'e_s_Avg' : 'es',
	'e_a_Avg' : 'ea',
	't_sonic' : 't_sonic_f1_K',
	'wd_mean' : 'wd',
	'ws_mean' : 'ws',
}

fm_5min_full = fm_5min_full.drop(columns=drop_cols)
fm_5min_full = fm_5min_full.rename(columns=rename_dict)
# Save out data
fm_5min_full.to_csv(os.path.join(base_dir, '5min_versions', 'fm_prelim_dr_fluxes.csv'), float_format='%g') 
# QA/QC (if needed)

# Code up vickers and mahrt 1999

# Notes for next one
# Email about t sonic conversions/corrections
# Add QA/QC
# Min. obs filter<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16712213579987387, 'fraction_numerical': 0.03889005675846122, 'mean_word_length': 4.79537149817296, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '333171', 'license': 'MIT', 'n_tokens_mistral': 2322, 'n_tokens_neox': 2166, 'n_words': 337, 'path': 'fluxtopo/fm_preprelim_proc.ipynb', 'repo_name': 'miksch/topo-flux', 'size': '4757'}