<commit_before>import glob
import icecap as icp
import numpy as np
import os
#import string
import fnmatch
import subradar as sr
import rsr
import pandas as pd


def params():
    """get various parameters defining the season
    """
    out = {'code_path':os.getcwd()}
    out['season'] = out['code_path'].split('/')[-3]
    out['process'] = out['code_path'].split('/')[-1]
    out['root_path'] = '/'.join(out['code_path'].split('/')[0:-5])
    out['norm_path'] = out['root_path'] + '/targ/norm'
    out['rsr_path'] = out['code_path'].replace('code', 'targ')
    out['cmp_path'] = out['rsr_path'].replace('RSR', 'CMP')
    out['pik_path'] = out['root_path'] + '/orig/xtra/'+out['season']+'/PIK/' + out['process']
    out['foc_path'] = out['root_path'] + '/targ/xtra/' + out['season']+ '/FOC/Best_Versions/S1_POS'
    out['sweep_path'] = out['root_path'] + '/targ/xtra/' + out['season']+ '/FOC/Best_Versions/S5_VEW'
    out['tpro_path'] = out['root_path'] + '/targ/tpro'
    out['season_flight_pst'] = out['root_path'] + '/syst/linux/lib/dbase/season_flight_pst'
    return out


def pik(pst, process=None, **kwargs):
    """Get available PIK files for a PST
    """
    p = icp.get.params()
    if process is None:
        process = p['process']
    folder = '/'.join([p['pik_path'].replace('/'+p['process'],''), process, pst])
    files = glob.glob(folder + '/*.*')
    names = [i.split('/')[-1] for i in files]
    products = [i.split('.')[0] for i in names]
    pik = [i.split('.')[1] for i in names]

    return products, pik


def cmp(pst, process=None, **kwargs):
    """Get available radar data in CMP for a PST
    """
    p = icp.get.params()
    if process is None:
        process = p['process']
    folder = '/'.join([p['cmp_path'].replace('/'+p['process'],''), process, pst])
    files = glob.glob(folder + '/*[!.meta]')
    products = [i.split('/')[-1] for i in files]
    return products


def pst(pattern, **kwargs):
    """Get PSTs for the current season that match a given pattern (regex)
    """
    p = icp.get.params()
    data = np.genfromtxt(p['season_flight_pst'], delimiter=' ', dtype=np.str)
    i = np.where(data[:,2] == p['season'])
    pst = data[i,0]
    return fnmatch.filter(pst.flatten(), pattern)


def sweep(pst, **kwargs):
    """Get available sweeps files for a PST
    """
    p = icp.get.params()
    folder = '/'.join([p['sweep_path'], pst])
    files = glob.glob(folder + '/*sweeps*')
    products = [i.split('/')[-1] for i in files]
    return products


def rsr_data(pst, **kwargs):
    """Display data avaialble to launch RSR
    """
    psts = icp.get.pst(pst)
    cmps = [ icp.get.cmp(i, process='pik1') for i in psts ]
    cmps_1m = [ icp.get.cmp(i, process='pik1.1m') for i in psts ]
    piks = [ icp.get.pik(i, process='pik1')[1] for i in psts]
    piks_1m = [ icp.get.pik(i, process='pik1.1m')[1] for i in psts]
    sweeps = [ icp.get.sweep(i) for i in psts ]
    d = {'PST':psts}
    df = pd.DataFrame(d)
    #df['CMP_pik1'] = cmps
    df['sweeps'] = sweeps
    df['CMP_pik1.1m'] = cmps_1m
    df['PIK_pik1'] = piks
    df['PIK1_pik1.1m'] = piks_1m
    pd.set_option('display.max_rows', 500)
    pd.set_option('display.max_columns', 500)
    pd.set_option('display.width', 1000)
    return df

<commit_msg>List RSR products achieved<commit_after>import glob
import icecap as icp
import numpy as np
import os
#import string
import fnmatch
import subradar as sr
import rsr
import pandas as pd


def params():
    """get various parameters defining the season
    """
    out = {'code_path':os.getcwd()}
    out['season'] = out['code_path'].split('/')[-3]
    out['process'] = out['code_path'].split('/')[-1]
    out['root_path'] = '/'.join(out['code_path'].split('/')[0:-5])
    out['norm_path'] = out['root_path'] + '/targ/norm'
    out['rsr_path'] = out['code_path'].replace('code', 'targ')
    out['cmp_path'] = out['rsr_path'].replace('RSR', 'CMP')
    out['pik_path'] = out['root_path'] + '/orig/xtra/'+out['season']+'/PIK/' + out['process']
    out['foc_path'] = out['root_path'] + '/targ/xtra/' + out['season']+ '/FOC/Best_Versions/S1_POS'
    out['sweep_path'] = out['root_path'] + '/targ/xtra/' + out['season']+ '/FOC/Best_Versions/S5_VEW'
    out['tpro_path'] = out['root_path'] + '/targ/tpro'
    out['season_flight_pst'] = out['root_path'] + '/syst/linux/lib/dbase/season_flight_pst'
    return out


def pik(pst, process=None, **kwargs):
    """Get available PIK files for a PST
    """
    p = icp.get.params()
    if process is None:
        process = p['process']
    folder = '/'.join([p['pik_path'].replace('/'+p['process'],''), process, pst])
    files = glob.glob(folder + '/*.*')
    names = [i.split('/')[-1] for i in files]
    products = [i.split('.')[0] for i in names]
    pik = [i.split('.')[1] for i in names]

    return products, pik


def cmp(pst, process=None, **kwargs):
    """Get available radar data in CMP for a PST
    """
    p = icp.get.params()
    if process is None:
        process = p['process']
    folder = '/'.join([p['cmp_path'].replace('/'+p['process'],''), process, pst])
    files = glob.glob(folder + '/*[!.meta]')
    products = [i.split('/')[-1] for i in files]
    return products


def pst(pattern, **kwargs):
    """Get PSTs for the current season that match a given pattern (regex)
    """
    p = icp.get.params()
    data = np.genfromtxt(p['season_flight_pst'], delimiter=' ', dtype=np.str)
    i = np.where(data[:,2] == p['season'])
    pst = data[i,0]
    return fnmatch.filter(pst.flatten(), pattern)


def sweep(pst, **kwargs):
    """Get available sweeps files for a PST
    """
    p = icp.get.params()
    folder = '/'.join([p['sweep_path'], pst])
    files = glob.glob(folder + '/*sweeps*')
    products = [i.split('/')[-1] for i in files]
    return products


def rsr(pst, process=None, **kwargs):
    """Get available rsr files
    """
    p = icp.get.params()
    if process is None:
        process = p['process']
    folder = '/'.join([p['rsr_path'].replace('/'+p['process'],''), process, pst])
    files = glob.glob(folder + '/*[!.meta]')
    products = [i.split('/')[-1] for i in files]
    pik = [i.split('.')[1] for i in products]
    return pik


def rsr_data(pst, **kwargs):
    """Display data avaialble to launch RSR
    """
    psts = icp.get.pst(pst)
    cmps = [ icp.get.cmp(i, process='pik1') for i in psts ]
    cmps_1m = [ icp.get.cmp(i, process='pik1.1m') for i in psts ]
    piks = [ icp.get.pik(i, process='pik1')[1] for i in psts]
    piks_1m = [ icp.get.pik(i, process='pik1.1m')[1] for i in psts]
    sweeps = [ icp.get.sweep(i) for i in psts ]
    rsr_1m = [ icp.get.rsr(i, process='pik1.1m') for i in psts  ]
    d = {'PST':psts}
    df = pd.DataFrame(d)
    #df['CMP_pik1'] = cmps
    df['sweeps'] = sweeps
    df['CMP_pik1.1m'] = cmps_1m
    df['PIK_pik1'] = piks
    df['PIK1_pik1.1m'] = piks_1m
    df['RSR_1m'] = rsr_1m
    pd.set_option('display.max_rows', 500)
    pd.set_option('display.max_columns', 500)
    pd.set_option('display.width', 1000)
    return df


{'fraction_non_alphanumeric': 0.2056434373663959, 'fraction_numerical': 0.012968505059142084, 'mean_word_length': 3.2844932844932844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5223698', 'n_tokens_mistral': 2777, 'n_tokens_neox': 2705, 'n_words': 603}