<jupyter_start><jupyter_text>Rudics Prawler Parser -> CSV for erddap<jupyter_code>!rm *.csv
import os
import glob #python >= 3.5

import datetime

import pandas as pd

import seawater as sw
import calc.aanderaa_corrO2_sal as optode_O2_corr
#list id's in download path
root_path = '/Users/bell/ecoraid/2019/Moorings/19bsitaepr2a/raw/rudics/eclipse.pmel.noaa.gov/rudics/FOCI/M002'

id_dir = os.listdir(root_path)
id_dir = sorted(id_dir)

profile_counter = 4211 
# 19bsitaepr-2a 
#0,147,184,426,697,972,1025,1272,1554,1825,1854,2070,2424,2770,2832,3084,3358,3633,3687,3935,4211
#   : 4:2, 4:3, 5:0, 5:1, 5:2, 5:3, 6:0, 6:1, 6:2, 6:3, 7:0, 7:1, 7:2, 7:3, 8:0, 8:1, 8:2, 8:3, 9:0, 9:1, 9:2
month='9'
day='2'

for indfile in sorted(glob.glob(root_path+'/CM002_0['+month+']_['+day+']*')):
    print(indfile)
    with open(indfile) as f:
        for line in f:
            if ('%%' in line):
                flag = line.split()[0]
                if ('praw002' in line) or ('praw03' in line):
                    profile_counter += 1
                next(f)
                continue
            if not (line.strip()):
                continue
            if 'eng' in flag:
                with open("eng.csv", "a") as myfile:
                    myfile.write(line.strip()+'\n')
            if 'GPS' in flag:
                with open("gps.csv", "a") as myfile:
                    myfile.write(line.strip()+'\n')
            if 'BARO' in flag:
                with open("baro.csv", "a") as myfile:
                    myfile.write(line.strip()+'\n')
            if 'ATRH' in flag:
                with open("atrh.csv", "a") as myfile:
                    myfile.write(line.strip()+'\n')
            if 'WIND' in flag:
                with open("wind.csv", "a") as myfile:
                    myfile.write(line.strip()+'\n')
            if 'praw002' in flag:
                with open("praw002.csv", "a") as myfile:
                    myfile.write(str(profile_counter) + ','+line.strip()+'\n')        
            if 'praw03' in flag:
                with open("praw03.csv", "a") as myfile:
                    myfile.write(str(profile_counter) + ','+line.strip()+'\n')  
                    
print("Total Profile number:{profile_counter}".format(profile_counter=profile_counter))
line
### apply corrections to raw prawler data, merge met data
# a gps line gets put into the atrh file?

atrh = pd.read_csv('atrh.csv',header=None,names=['datetime','RH','AT'],
                   parse_dates=True,index_col='datetime')

wind = pd.read_csv('wind.csv',header=None,names=['datetime','U','USD','V','VSD'],
                   parse_dates=True,index_col='datetime')

baro = pd.read_csv('baro.csv',header=None,names=['datetime','BP'],
                   parse_dates=True,index_col='datetime')
#concat and save
metdata = pd.concat([baro, wind, atrh], axis=1, sort=False)
metdata['id']='19BSITAEPR2A'
metdata['dep_lat']=56.873
metdata['dep_lon']=-164.065

## simple qc
metdata=metdata[(metdata['AT']!='U')&(metdata['AT']!='D')]
"""p03 = pd.read_csv('praw03.csv',header=None,
                  names=['profilenumber','datetime','CD','CT','CC','OT','02','CH','TB'],
                   parse_dates=True,index_col='datetime')

## simple qc
p03=p03[(p03.index>='2000') & (p03.index<=datetime.datetime.now().strftime('%Y-%m-%d'))]
p03.index = pd.to_datetime(p03.index)

"""
def date_parser(date_string):
    try:
        date = datetime.datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
    except:
        date = 0
        print(date, date_string)
        pass
    return date

p02 = pd.read_csv('praw002.csv',header=None,
                  names=['profilenumber','datetime','CD','CT','CC','OT','02','CH','TB'],
                   parse_dates=True,index_col='datetime')

## simple qc of times, all zeros, 1309/1310/48??/0812 years
# watch for  odd times that keep units from being red properly
#p02 = pd.read_csv('praw002.csv',header=None,
#                  names=['profilenumber','datetime','CD','CT','CC','OT','02','CH','TB'])
"""#initial deployment of instrument
p03.CD = p03.CD/100
p03.CT = p03.CT/1000
p03.CC = p03.CC/1000
p03.OT = p03.OT/1000
p03['02'] = p03['02']/100
### blocked sensor
p03.CH = (p03.CH-50)*0.0
p03.TB = (p03.TB-50)*0.0
# convert cond to salinity
p03['S'] = sw.salt(r=p03.CC/42.914,t=p03.CT,p=p03.CD)

### convert cond to salinity
p03.S = sw.salt(r=p03.CC/42.914,t=p03.CT,p=p03.CD)
### sal correct and convert oxygen
p03['O2_corr'] = optode_O2_corr.O2_sal_comp(oxygen_conc=p03['02'],
                                     salinity=p03.S,
                                     temperature=p03.CT)
p03['O2_corr_umkg'] = optode_O2_corr.O2_molar2umkg(oxygen_conc=p03.O2_corr,
                                     salinity=p03.S,
                                     temperature=p03.CT,
                                     pressure=p03.CD)

p03['O2PercentSat'] = optode_O2_corr.O2PercentSat(oxygen_conc=p03.O2_corr_umkg,
                                     salinity=p03.S,
                                     temperature=p03.CT,
                                     pressure=p03.CD)

p03['dep_lat']=56.873
p03['dep_lon']=-164.065
                        """
#redeployment of instrument
p02.CD = p02.CD/100
p02.CT = p02.CT/1000
p02.CC = p02.CC/1000
p02.OT = p02.OT/1000
p02['02'] = p02['02']/100
p02.CH = (p02.CH-50)*0.0122
p02.TB = (p02.TB-50)*0.0247
# convert cond to salinity
p02['S'] = sw.salt(r=p02.CC/42.914,t=p02.CT,p=p02.CD)

### sal correct and convert oxygen
p02['O2_corr'] = optode_O2_corr.O2_sal_comp(oxygen_conc=p02['02'],
                                     salinity=p02.S,
                                     temperature=p02.CT)
p02['O2_corr_umkg'] = optode_O2_corr.O2_molar2umkg(oxygen_conc=p02.O2_corr,
                                     salinity=p02.S,
                                     temperature=p02.CT,
                                     pressure=p02.CD)

p02['O2PercentSat'] = optode_O2_corr.O2PercentSat(oxygen_conc=p02.O2_corr_umkg,
                                     salinity=p02.S,
                                     temperature=p02.CT,
                                     pressure=p02.CD)

p02['dep_lat']=56.873
p02['dep_lon']=-164.065
current_date = datetime.date.today().strftime('%Y%m%d')
"""p03.to_csv('19BSITAEPR2A_p03_20190'+month+'.'+day+'.csv', 
            date_format='%Y-%m-%dT%H:%M:%SZ')"""
p02.to_csv('19BSITAEPR2A_p02_20190'+month+'.'+day+'.csv', 
           date_format='%Y-%m-%d %H:%M:%S')
metdata.to_csv('19BSITAEPR2A_met_20190'+month+'.'+day+'.csv', 
               date_format='%Y-%m-%d %H:%M:%S')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17063913769546077, 'fraction_numerical': 0.08122058600273266, 'mean_word_length': 2.266236985622211, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '449473', 'license': 'MIT', 'n_tokens_mistral': 2791, 'n_tokens_neox': 2423, 'n_words': 374, 'path': 'PrawlerDeployments/PrawlerRudicstoCSV.ipynb', 'repo_name': 'shaunwbell/ipythonnb', 'size': '6587'}