<jupyter_start><jupyter_text>Reading IOB files from data2/<jupyter_code>def get_data(filename):
    df = pd.read_csv(filename,delim_whitespace=True,names=['word','label'])
    beg_indices = list(df[df['word'] == 'BOS'].index)+[df.shape[0]]
    sents,labels,intents = [],[],[]
    for i in range(len(beg_indices[:-1])):
        sents.append(df[beg_indices[i]+1:beg_indices[i+1]-1]['word'].values)
        labels.append(df[beg_indices[i]+1:beg_indices[i+1]-1]['label'].values)
        intents.append(df.loc[beg_indices[i+1]-1]['label'])    
    return np.array(sents),np.array(labels),np.array(intents)

def get_data2(filename):
    with open(filename) as f:
        contents = f.read()
    sents,labels,intents = [],[],[]
    for line in contents.strip().split('\n'):
        words,labs = [i.split(' ') for i in line.split('\t')]
        sents.append(words[1:-1])
        labels.append(labs[1:-1])
        intents.append(labs[-1])
    return np.array(sents),np.array(labels),np.array(intents)

read_method = {'data2/atis-2.dev.w-intent.iob':get_data,
               'data2/atis.train.w-intent.iob':get_data2,
               'data2/atis.test.w-intent.iob':get_data,
              'data2/atis-2.train.w-intent.iob':get_data2}

def fetch_data(fname):
    func = read_method[fname]
    return func(fname)
# Example
sents,labels,intents = fetch_data('data2/atis.train.w-intent.iob')
def display(n,intents):
    sense = []
    print ("INTENT : ",intents[n])
    for i in range(len(sents[n])):
    #     sense.append({"word_index":word_indices[0][i],"word":words2idx[word_indices[0][i]],"entity_index":name_entities[0][i],"entity":tables2idx[name_entities[0][i]],"label_index":labels[0][i],"label":labels2idx[labels[0][i]]})
        sense.append({"word":sents[n][i],"label":labels[n][i]})
    return pd.DataFrame(sense)

display(1)<jupyter_output>INTENT :  atis_flight
{'fraction_non_alphanumeric': 0.20053763440860214, 'fraction_numerical': 0.01935483870967742, 'mean_word_length': 4.317142857142857, 'pattern_counts': {'":': 8, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '115579', 'license': 'MIT', 'n_tokens_mistral': 725, 'n_tokens_neox': 721, 'n_words': 75, 'path': 'atis_experiments.ipynb', 'repo_name': 'py-ranoid/IntEnt', 'size': '1860'}