<commit_before>import argparse
from datetime import datetime

import numpy as np
import pandas as pd
from gurobipy import *


def read_input_csv(filename, typ=None):
    sldf = pd.read_csv(filename, header=0, dtype=typ)
    sldf[sldf.columns[0]] = sldf[sldf.columns[0]].astype(str)
    sldf.set_index(sldf.columns[0], inplace=True)
    return sldf.to_dict('index'), sorted(sldf.columns.values), list(sldf.index.values)


def read_slots_interviews(filename):
    sidict = pd.read_csv(filename, dtype=object).to_dict('list')
    return dict((key, int(v[0])) for key, v in sidict.items())


def read_shortlists(filename):
    sldf = pd.read_csv(filename, dtype=object)
    comps = list(sldf.columns.values)
    comtupl = [(c, n) for c in comps for n in list(sldf[c].dropna().values)]
    return dict((x, 1) for x in comtupl), comps, sorted(set([x[1] for x in comtupl]))


def read_lp(filename):
    exnames = []
    with open(filename) as f:
        for csvline in f:
            exnames = exnames + [x for x in csvline.strip().split(',') if len(str(x).strip()) > 0]

    return sorted(set(exnames))
<commit_msg>strip spaces in lp list<commit_after>import argparse
from datetime import datetime

import numpy as np
import pandas as pd
from gurobipy import *


def read_input_csv(filename, typ=None):
    sldf = pd.read_csv(filename, header=0, dtype=typ)
    sldf[sldf.columns[0]] = sldf[sldf.columns[0]].astype(str)
    sldf.set_index(sldf.columns[0], inplace=True)
    return sldf.to_dict('index'), sorted(sldf.columns.values), list(sldf.index.values)


def read_slots_interviews(filename):
    sidict = pd.read_csv(filename, dtype=object).to_dict('list')
    return dict((key, int(v[0])) for key, v in sidict.items())


def read_shortlists(filename):
    sldf = pd.read_csv(filename, dtype=object)
    comps = list(sldf.columns.values)
    comtupl = [(c, n) for c in comps for n in list(sldf[c].dropna().values)]
    return dict((x, 1) for x in comtupl), comps, sorted(set([x[1] for x in comtupl]))


def read_lp(filename):
    exnames = []
    with open(filename) as f:
        for csvline in f:
            exnames = exnames + [str(x).strip() for x in csvline.strip().split(',') if len(str(x).strip()) > 0]

    return sorted(set(exnames))

{'fraction_non_alphanumeric': 0.14413208389112003, 'fraction_numerical': 0.0071396697902722, 'mean_word_length': 4.1422018348623855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6234311', 'n_tokens_mistral': 846, 'n_tokens_neox': 823, 'n_words': 202}