<commit_before>import os
import numpy as np
import pandas as pd
from pyemu.pst.pst_utils import SFMT,IFMT,FFMT

def pilot_points_to_tpl(pp_file,tpl_file=None,name_prefix=None):
    assert os.path.exists(pp_file)
    if tpl_file is None:
        tpl_file = pp_file+".tpl"

    pp_df = pd.read_csv(pp_file,delim_whitespace=True,header=None,
                        names=["name","x","y","zone","value"])

    if name_prefix is not None:
        digits = str(len(str(pp_df.shape[0])))
        fmt = "{0:0"+digits+"d}"
        names = [name_prefix+fmt.format(i) for i in range(pp_df.shape[0])]
    else:
        names = pp_df.name.copy()

    too_long = []
    for name in names:
        if len(name) > 12:
            too_long.append(name)
    if len(too_long) > 0:
        raise Exception("the following parameter names are too long:" +\
                        ",".join(too_long))

    tpl_entries = ["~    {0}    ~".format(name) for name in names]
    pp_df.loc[:,"tpl"] = tpl_entries
    fmt = {"name":SFMT,"x":FFMT,"y":FFMT,"zone":IFMT,"tpl":SFMT}

    f_tpl = open(tpl_file,'w')
    f_tpl.write("ptf ~\n")
    f_tpl.write(pp_df.to_string(col_space=0,
                              columns=["name","x","y","zone","tpl"],
                              formatters=fmt,
                              justify="left",
                              header=False,
                              index=False) + '\n')

    return pp_df

def fac2real(pp_file,factors_file,out_file="test.ref",
             upper_lim=1.0e+30,lower_lim=-1.0e+30):
    assert os.path.exists(pp_file)
    assert os.path.exists(factors_file)
    pp_data = pd.read_csv(pp_file,delim_whitespace=True,header=None,
                          names=["name","value"],usecols=[0,4])
    pp_data.loc[:,"name"] = pp_data.name.apply(lambda x: x.lower())

    f_fac = open(factors_file,'r')
    fpp_file = f_fac.readline()
    fzone_file = f_fac.readline()
    ncol,nrow = [int(i) for i in f_fac.readline().strip().split()]
    npp = int(f_fac.readline().strip())
    pp_names = [f_fac.readline().strip().lower() for _ in range(npp)]<commit_msg>few more tweaks to pp to tpl<commit_after>import os
import numpy as np
import pandas as pd
from pyemu.pst.pst_utils import SFMT,IFMT,FFMT

def pilot_points_to_tpl(pp_file,tpl_file=None,name_prefix=None):
    assert os.path.exists(pp_file)
    if tpl_file is None:
        tpl_file = pp_file+".tpl"

    pp_df = pd.read_csv(pp_file,delim_whitespace=True,header=None,
                        names=["name","x","y","zone","parval1"])

    if name_prefix is not None:
        digits = str(len(str(pp_df.shape[0])))
        fmt = "{0:0"+digits+"d}"
        names = [name_prefix+fmt.format(i) for i in range(pp_df.shape[0])]
    else:
        names = pp_df.name.copy()

    too_long = []
    for name in names:
        if len(name) > 12:
            too_long.append(name)
    if len(too_long) > 0:
        raise Exception("the following parameter names are too long:" +\
                        ",".join(too_long))

    tpl_entries = ["~    {0}    ~".format(name) for name in names]
    pp_df.loc[:,"tpl"] = tpl_entries
    pp_df.loc[:,"parnme"] = names
    fmt = {"name":SFMT,"x":FFMT,"y":FFMT,"zone":IFMT,"tpl":SFMT}

    f_tpl = open(tpl_file,'w')
    f_tpl.write("ptf ~\n")
    f_tpl.write(pp_df.to_string(col_space=0,
                              columns=["name","x","y","zone","tpl"],
                              formatters=fmt,
                              justify="left",
                              header=False,
                              index=False) + '\n')

    return pp_df

def fac2real(pp_file,factors_file,out_file="test.ref",
             upper_lim=1.0e+30,lower_lim=-1.0e+30):
    assert os.path.exists(pp_file)
    assert os.path.exists(factors_file)
    pp_data = pd.read_csv(pp_file,delim_whitespace=True,header=None,
                          names=["name","value"],usecols=[0,4])
    pp_data.loc[:,"name"] = pp_data.name.apply(lambda x: x.lower())

    f_fac = open(factors_file,'r')
    fpp_file = f_fac.readline()
    fzone_file = f_fac.readline()
    ncol,nrow = [int(i) for i in f_fac.readline().strip().split()]
    npp = int(f_fac.readline().strip())
    pp_names = [f_fac.readline().strip().lower() for _ in range(npp)]
{'fraction_non_alphanumeric': 0.15558166862514688, 'fraction_numerical': 0.00963572267920094, 'mean_word_length': 2.4350282485875705, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '886318', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1529, 'n_words': 253}