<commit_before>#!/usr/local/sci/bin/python

#***************************************
# 06 June 2014 KMW - v1
# 
# Reads in clean world ISTI data file and outputs a file for each station
# in GHCN format
# Additionally reads in the ISTI station list file and outputs in 
# GHCN format
# Additionally outputs clean world ISTI into ISTI format

#************************************************************************
#                                 START
#************************************************************************
# USE python2.7
# python2.7 ConvertISTItoGHCN_NOV2014.py
#
# JUL 2015
# This has been adapted to output my single ASCII format (stations in rows, months in columns) to ISTI and GHCN format
# Could later be adapted to output netCDF too
#
# AUG 2015
# Added the missing data mask from the real data - so we now output only the missing data versions in ISTI and GHCN format
# BUT also output a missing data version in my singla ASCII format
#
# REQUIRES
#************************************************************************
# Set up python imports
import datetime as dt
import numpy as np
import sys, os
from scipy.optimize import curve_fit,fsolve,leastsq
from scipy import pi,sqrt,exp
from scipy.special import erf
import scipy.stats
from math import sqrt,pi
import struct
import pdb	# for stopping and restarting with editability (stop is pdb.set_trace(),restart is c)

# RESTART VALUE
Restarter='------'	#'------'		#'    01010000' station ID

# Set up file locations
STATLIST='/data/local/hadkw/ISTI/LISTS/v101_JUL2015/ISTILONGINVENTORY_stage3proxyelevs_JUL2015.dat'
INFILEE='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/FIXCLEAN1_ISTI_stage3proxyelevs_loess03MDA60CLSfix_JUL2015.txt'	#_stage3
INREALS='/data/local/hadkw/ISTI/DATA/ISTIv101_JUL2015/results_merged/merge_'
OUTSTATLIST='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/ISTILONGINVENTORYghcn_stage3proxyelevs_JUL2015.dat'
OUTFILEEGHCN='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/GHCN_TYPE/' # add ID number here
OUTFILEEISTI='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/ISTI_TYPE/' # add ID number here
OUTFILEEMASK='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/MASKEDFIXCLEAN1_ISTI_stage3proxyelevs_loess03MDA60CLSfix_JUL2015.txt'

# Set up variables and holders
MDI=-9999
RStYr=1850
REdYr=2015
RYrCount=(REdYr-RStYr)+1
RMonCount=RYrCount*12
RYrArr=np.array(range(RYrCount))+RStYr
StYr=1860
EdYr=2018
YrCount=(EdYr-StYr)+1
MonCount=YrCount*12
YrArr=np.array(range(YrCount))+StYr
nstations=0	# defined after reading in station list
StatIDs=[]	# nstations list filled after reading in station list
StatLats=[]	# nstations list filled after reading in station list
StatLons=[]
StatElevs=[]
StatCOs=[]
StatNames=[]
Benchmark_ID='BCHMCAAA'	# Benchmark Clean World AAA

#************************************************************************
# Subroutines
#************************************************************************
# READDATA
def ReadData(FileName,typee,delimee,ASTruth,ColumnChoice):
    ''' Use numpy genfromtxt reading to read in all rows from a complex array '''<commit_msg>Updated file to create AUG2015 and work with full time array 1800-2018<commit_after>#!/usr/local/sci/bin/python

#***************************************
# 06 June 2014 KMW - v1
# 
# Reads in clean world ISTI data file and outputs a file for each station
# in GHCN format
# Additionally reads in the ISTI station list file and outputs in 
# GHCN format
# Additionally outputs clean world ISTI into ISTI format

#************************************************************************
#                                 START
#************************************************************************
# USE python2.7
# python2.7 ConvertISTItoGHCN_NOV2014.py
#
# JUL 2015
# This has been adapted to output my single ASCII format (stations in rows, months in columns) to ISTI and GHCN format
# Could later be adapted to output netCDF too
#
# AUG 2015
# Added the missing data mask from the real data - so we now output only the missing data versions in ISTI and GHCN format
# BUT also output a missing data version in my singla ASCII format
#
# REQUIRES
#************************************************************************
# Set up python imports
import datetime as dt
import numpy as np
import sys, os
from scipy.optimize import curve_fit,fsolve,leastsq
from scipy import pi,sqrt,exp
from scipy.special import erf
import scipy.stats
from math import sqrt,pi
import struct
import pdb	# for stopping and restarting with editability (stop is pdb.set_trace(),restart is c)

# RESTART VALUE
Restarter='------'	#'------'		#'    01010000' station ID

# Set up file locations
STATLIST='/data/local/hadkw/ISTI/LISTS/v101_JUL2015/ISTILONGINVENTORY_stage3proxyelevs_JUL2015.dat'
INFILEE='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/FIXCLEAN1_ISTI_stage3proxyelevs_loess03MDA60CLSfix_AUG2015.txt'	#_stage3
INREALS='/data/local/hadkw/ISTI/DATA/ISTIv101_JUL2015/results_merged/merge_'
OUTSTATLIST='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/ISTILONGINVENTORYghcn_stage3proxyelevs_AUG2015.dat'
OUTFILEEGHCN='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/GHCN_TYPE/' # add ID number here
OUTFILEEISTI='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/ISTI_TYPE/' # add ID number here
OUTFILEEMASK='/data/local/hadkw/ISTI/DATA/CLEANWORLDS/v101_JUL2015/MASKEDFIXCLEAN1_ISTI_stage3proxyelevs_loess03MDA60CLSfix_AUG2015.txt'

# Set up variables and holders
MDI=-9999
RStYr=1850
REdYr=2015
RYrCount=(REdYr-RStYr)+1
RMonCount=RYrCount*12
RYrArr=np.array(range(RYrCount))+RStYr
StYr=1800
EdYr=2018
YrCount=(EdYr-StYr)+1
MonCount=YrCount*12
YrArr=np.array(range(YrCount))+StYr
nstations=0	# defined after reading in station list
StatIDs=[]	# nstations list filled after reading in station list
StatLats=[]	# nstations list filled after reading in station list
StatLons=[]
StatElevs=[]
StatCOs=[]
StatNames=[]
Benchmark_ID='BCHMCAAA'	# Benchmark Clean World AAA

#************************************************************************
# Subroutines
#************************************************************************
# READDATA
def ReadData(FileName,typee,delimee,ASTruth,ColumnChoice):
    ''' Use numpy genfromtxt reading to read in all rows from a complex array '''
{'fraction_non_alphanumeric': 0.20431130131558092, 'fraction_numerical': 0.0456490727532097, 'mean_word_length': 6.018909899888765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '332053', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2042, 'n_words': 529}