<commit_before>"""
Filename:     create_zw3_table.py
Author:       <NAME>, <EMAIL>
Description:  Takes all the ZW3 climatology results and puts
              them in a single table

"""

# Import general Python modules #

import sys, os, pdb
import argparse
import numpy, pandas
import cdms2

# Import my modules #

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'phd':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import netcdf_io as nio
except ImportError:
    raise ImportError('Must run this script from anywhere within the phd git repo')


# Define functions #



def get_fourier(infile):
    """Extract Fourier coefficient data and output to a pandas DataFrame"""
    
    fin = cdms2.open(infile)
    var_list = fin.listvariables()
    time_axis = fin.getAxis('time').asComponentTime()
    fin.close()
    
    output = pandas.DataFrame(index=map(gio.standard_datetime, time_index))
    
    for var in var_list:
        indata = nio.InputData(infile, var)
	output[var] = indata.data

    return output


def get_zw3(infile):
    """Extract ZW3 index and output to a pandas DataFrame"""

    indata = nio.InputData(infile, 'zw3')
    time_axis = indata.data.getTime().asComponentTime()
    
    output = pandas.DataFrame(indata.data, index=map(gio.standard_datetime, time_index))

    return output
    
    
def get_env(infile, normalised=False):
    """Extract wave envelope stats and output to pandas DataFrame"""
    
    data = pandas.read_csv(infile, header=1, index_col=0)
    
    if normalised:
        # rename columns or else join won't work later



def main(inargs):
    """Run the program."""

    # Read data and check inputs #
    
    fourier_DataFrame = get_fourier(inargs.fourier_file)
    zw3_DataFrame = get_zw3(inargs.zw3_file)
    env_DataFrame = get_env(inargs.env_file, normalised=False)
    nenv_DataFrame = get_env(inargs.nenv_file, normalised=True)
    
    output = fourier_DataFrame.join([zw3_DataFrame, env_DataFrame, nenv_DataFrame])
    metadata = gio.get_timestamp()
    output.to_csv(inargs.outfile)  # At the moment you can't write metadata headers with to_csv, hence metadata hasn't been used<commit_msg>Figured out how to change column labels<commit_after>"""
Filename:     create_zw3_table.py
Author:       <NAME>, <EMAIL>
Description:  Takes all the ZW3 climatology results and puts
              them in a single table

"""

# Import general Python modules #

import sys, os, pdb
import argparse
import numpy, pandas
import cdms2

# Import my modules #

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'phd':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import netcdf_io as nio
except ImportError:
    raise ImportError('Must run this script from anywhere within the phd git repo')


# Define functions #



def get_fourier(infile):
    """Extract Fourier coefficient data and output to a pandas DataFrame"""
    
    fin = cdms2.open(infile)
    var_list = fin.listvariables()
    time_axis = fin.getAxis('time').asComponentTime()
    fin.close()
    
    output = pandas.DataFrame(index=map(gio.standard_datetime, time_index))
    
    for var in var_list:
        indata = nio.InputData(infile, var)
	output[var] = indata.data

    return output


def get_zw3(infile):
    """Extract ZW3 index and output to a pandas DataFrame"""

    indata = nio.InputData(infile, 'zw3')
    time_axis = indata.data.getTime().asComponentTime()
    
    output = pandas.DataFrame(indata.data, index=map(gio.standard_datetime, time_index))

    return output
    
    
def get_env(infile, normalised=False):
    """Extract wave envelope stats and output to pandas DataFrame"""
    
    df = pandas.read_csv(infile, header=1, index_col=0)
    tag = 'nenv' if normalised else 'env' 
    df.rename(columns=lambda x: tag+'_'+x, inplace=True)

    # change index values in place using gio.standard_datetime()


def main(inargs):
    """Run the program."""

    # Read data and check inputs #
    
    fourier_DataFrame = get_fourier(inargs.fourier_file)
    zw3_DataFrame = get_zw3(inargs.zw3_file)
    env_DataFrame = get_env(inargs.env_file, normalised=False)
    nenv_DataFrame = get_env(inargs.nenv_file, normalised=True)
    
    output = fourier_DataFrame.join([zw3_DataFrame, env_DataFrame, nenv_DataFrame])
    metadata = gio.get_timestamp()
    output.to_csv(inargs.outfile)  # At the moment you can't write metadata headers with to_csv, hence metadata hasn't been used
{'fraction_non_alphanumeric': 0.1005879882402352, 'fraction_numerical': 0.005879882402351953, 'mean_word_length': 3.566634707574305, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '347238', 'n_tokens_mistral': 1596, 'n_tokens_neox': 1553, 'n_words': 420}