<commit_before># Import general Python modules

import os, sys, pdb
import argparse
import numpy, pandas
import xray
import operator
from itertools import groupby


# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import convenient_universal as uconv
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')


# Define functions

#
#def add_duration(DataFrame, metric, metric_threshold):
#    """Add a duration column to the input DataFrame. 
#
#    An event is defined by the number of consecutive timesteps greater than 
#    the metric_threshold.
#    
#    Note that every timestep is assigned a duration value equal to the number of
#    days in the entire event.
#    
#    """
#
#    events = DataFrame[metric].map(lambda x: x > metric_threshold)
#    event_list = events.tolist()
#    grouped_events = [(k, sum(1 for i in g)) for k,g in groupby(event_list)]
#
#    duration = []
#    for event in grouped_events:
#        if event[0]:
#            data = [event[1]] * event[1]
#        else:
#            data = [0] * event[1]
#        duration.append(data)
#
#    DataFrame['duration'] = reduce(operator.add, duration)
#    DataFrame['event'] = events
#
#    return DataFrame  


def main(inargs):
    """Run the program"""

    # Read data
    dset_in = xray.open_dataset(inargs.fourier_file)
    df = dset_in.to_dataframe()

    # Select the amplitude columns
    amp_df = df.loc[:, df.columns.map(lambda x: 'amp' in x)]

    # Rank the amplitude columns
    arank = amp_df.apply(numpy.argsort, axis=1)

    # Select the ones where wave 5 and 6 are in the top 3 amplitudes
    wave5_top3 = arank['wave5_amp'] <= 2
    wave6_top3 = arank['wave6_amp'] <= 2
    top3 = wave5_top3.tolist() and wave6_top3.tolist()
    # df.iloc[top3]

    # Generate duration information<commit_msg>Fixed PSA ranking bug<commit_after># Import general Python modules

import os, sys, pdb
import argparse
import numpy, pandas
import xray
import operator
from itertools import groupby
from scipy.stats import rankdata


# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'climate-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
    import convenient_universal as uconv
except ImportError:
    raise ImportError('Must run this script from anywhere within the climate-analysis git repo')


# Define functions

#
#def add_duration(DataFrame, metric, metric_threshold):
#    """Add a duration column to the input DataFrame. 
#
#    An event is defined by the number of consecutive timesteps greater than 
#    the metric_threshold.
#    
#    Note that every timestep is assigned a duration value equal to the number of
#    days in the entire event.
#    
#    """
#
#    events = DataFrame[metric].map(lambda x: x > metric_threshold)
#    event_list = events.tolist()
#    grouped_events = [(k, sum(1 for i in g)) for k,g in groupby(event_list)]
#
#    duration = []
#    for event in grouped_events:
#        if event[0]:
#            data = [event[1]] * event[1]
#        else:
#            data = [0] * event[1]
#        duration.append(data)
#
#    DataFrame['duration'] = reduce(operator.add, duration)
#    DataFrame['event'] = events
#
#    return DataFrame  


def main(inargs):
    """Run the program"""

    # Read data
    dset_in = xray.open_dataset(inargs.fourier_file)
    df = dset_in.to_dataframe()

    # Select the amplitude columns
    amp_df = df.loc[:, df.columns.map(lambda x: 'amp' in x)]

    # Rank the amplitude columns
    arank = amp_df.apply(rankdata, axis=1)

    # Select the ones where wave 5 and 6 are in the top 3 amplitudes
    wave5_top3 = arank['wave5_amp'] >= 8
    wave6_top3 = arank['wave6_amp'] >= 8
    top3 = wave5_top3.tolist() and wave6_top3.tolist()
    # df.iloc[top3]

    # Generate duration information
{'fraction_non_alphanumeric': 0.10803193987787693, 'fraction_numerical': 0.011742602160638797, 'mean_word_length': 3.2335984095427435, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '223880', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1372, 'n_words': 426}