<jupyter_start><jupyter_text>Logistic Regression for 10 vs. 37 BPs Several ideas from the code below come from <NAME> & <NAME><jupyter_code>#unzip dataset
if os.path.isdir('decisions') == False:
    ZipFile("decisions.zip").extractall("decisions")
df = pd.read_csv('metadata1037.csv') 
df
ids = list(df['doc_id'])
bp_label = list(df['bp'])
bp_set = list(set(df['bp']))
bp_set
if os.path.isdir('bp1037') == False:
    os.mkdir('bp1037') 
    
if os.path.isdir('bp1037/10') == False:
    os.mkdir('bp1037/10') 
    
    
if os.path.isdir('bp1037/37') == False:
    os.mkdir('bp1037/37') 
for index in range(len(ids)):
    if bp_label[index] == '[10]':
        string = 'decisions/decisions/'+ids[index]+'.txt'
        shutil.copyfile(string, 'bp1037/10/'+ids[index]+'.txt')
    elif bp_label[index] == '[37]':
        string = 'decisions/decisions/'+ids[index]+'.txt'
        shutil.copyfile(string, 'bp1037/37/'+ids[index]+'.txt')
bp_10_37 = load_files("bp1037")
bp_train, label_train = bp_10_37.data, bp_10_37.target
print("type of text_train: {}".format(type(bp_train)))
print("length of text_train: {}".format(len(bp_train)))
print("text_train[1]:\n{}".format(bp_train[1].decode("utf-8")))
def numeric_masker(dataset):
    '''Substitues all integers in a text dataset by an empty string
    Input: dataset made of texts
    Output: the same dataset made of texts, only with masked integers'''
    masked_set = []
    for index in range(len(dataset)):
        new_string = ''
        for word in range(len(dataset[index])):
            if dataset[index][word].isnumeric()==False:
                new_string += dataset[index][word]
            else:
                new_string += ''
        masked_set.append(new_string)
    return masked_set
#balances dataset to have same number of bps 10 and 37
bp_10 = [bp_train[index] for index in range(len(label_train)) if label_train[index]==0]
print(len(bp_10),'    ', list(label_train).count(0))

#choose randomly from bp 10 same number elements as bp 37
random_indexes = np.random.permutation(len(bp_10))
bp_10_bal = [bp_10[index] for index in random_indexes[:list(label_train).count(1)]]
print(len(bp_10_bal),'    ', list(label_train).count(1))

#produces the balanced set
bp_37 = [bp_train[index] for index in range(len(label_train)) if label_train[index]==1]
bp_train_bal = bp_10_bal+bp_37
bp_train_bal = np.array(bp_train_bal)
label_train_bal = list(np.zeros(len(bp_10_bal)))+list(np.ones(len(bp_37)))
label_train_bal = np.array(label_train_bal)
print(len(bp_train_bal),'    ', len(label_train_bal))
uft_dataset = [bp_train_bal[index].decode("utf-8") for index in range(len(bp_train_bal))]

masked_dataset = numeric_masker(uft_dataset)
masked_dataset[0]
pipe = make_pipeline(TfidfVectorizer(min_df=5, norm=None),
LogisticRegression())
param_grid = {'logisticregression__C': [0.001, 0.01, 0.1, 1, 10]}
grid = GridSearchCV(pipe, param_grid, cv=5)
grid.fit(masked_dataset, label_train_bal)
print("Best cross-validation score: {:.2f}".format(grid.best_score_))
vectorizer = grid.best_estimator_.named_steps["tfidfvectorizer"]
# transform the training dataset
X_train = vectorizer.transform(masked_dataset)
# find maximum value for each of the features over the dataset
max_value = X_train.max(axis=0).toarray().ravel()
sorted_by_tfidf = max_value.argsort()
# get feature names
feature_names = np.array(vectorizer.get_feature_names())
mglearn.tools.visualize_coefficients(
grid.best_estimator_.named_steps["logisticregression"].coef_,
feature_names, n_top_features=40)<jupyter_output><empty_output><jupyter_text>Súmula Vinculante 10: Viola a cláusula de reserva de plenário (CF, artigo 97) a decisão de órgão fracionário de tribunal que, embora não declare expressamente a inconstitucionalidade de lei ou ato normativo do Poder Público, afasta sua incidência, no todo ou em parte.Súmula Vinculante 37: Não cabe ao Poder Judiciário, que não tem função legislativa, aumentar vencimentos de servidores públicos sob o fundamento de isonomia<jupyter_code>#deletes extra folder for splitting the dataset
shutil.rmtree('bp1037')<jupyter_output><empty_output><jupyter_text>One vs. All Regression for BPs<jupyter_code>#unzip dataset
if os.path.isdir('decisions_allbps') == False:
    ZipFile("decisions_allbps.zip").extractall("decisions_allbps")
df = pd.read_csv('metadata.csv') 
df
ids = list(df['doc_id'])
minister = list(df['minister'])
bp_label = list(df['bp'])
bp_set = list(set(df['bp']))
bp_set_single_bp = [element for element in bp_set if len(element)<5]
bp_set_single_bp
if os.path.isdir('allbp') == False:
    os.mkdir('allbp') 
    
if os.path.isdir('allbp/3') == False:
    os.mkdir('allbp/3') 
    
if os.path.isdir('allbp/33') == False:
    os.mkdir('allbp/33') 

if os.path.isdir('allbp/37') == False:
    os.mkdir('allbp/37')

if os.path.isdir('allbp/4') == False:
    os.mkdir('allbp/4') 
    
if os.path.isdir('allbp/14') == False:
    os.mkdir('allbp/14') 
    
if os.path.isdir('allbp/17') == False:
    os.mkdir('allbp/17') 
    
if os.path.isdir('allbp/26') == False:
    os.mkdir('allbp/26') 

if os.path.isdir('allbp/20') == False:
    os.mkdir('allbp/20') 
    
if os.path.isdir('allbp/10') == False:
    os.mkdir('allbp/10') 

if os.path.isdir('allbp/11') == False:
    os.mkdir('allbp/11') 
for index in range(len(ids)):
    string = 'decisions_allbps/decisions/'+ids[index]+'.txt'
    if bp_label[index] == '[3]':
        shutil.copyfile(string, 'allbp/3/'+ids[index]+'.txt')
    elif bp_label[index] == '[33]':
        shutil.copyfile(string, 'allbp/33/'+ids[index]+'.txt')
    elif bp_label[index] == '[37]':
        shutil.copyfile(string, 'allbp/37/'+ids[index]+'.txt')
    elif bp_label[index] == '[4]':
        shutil.copyfile(string, 'allbp/4/'+ids[index]+'.txt')
    elif bp_label[index] == '[14]':
        shutil.copyfile(string, 'allbp/14/'+ids[index]+'.txt')
    elif bp_label[index] == '[17]':
        shutil.copyfile(string, 'allbp/17/'+ids[index]+'.txt')
    elif bp_label[index] == '[26]':
        shutil.copyfile(string, 'allbp/26/'+ids[index]+'.txt')
    elif bp_label[index] == '[20]':
        shutil.copyfile(string, 'allbp/20/'+ids[index]+'.txt')
    elif bp_label[index] == '[10]':
        shutil.copyfile(string, 'allbp/10/'+ids[index]+'.txt')
    elif bp_label[index] == '[11]':
        shutil.copyfile(string, 'allbp/11/'+ids[index]+'.txt')
bp = load_files("allbp")
bp_train, label_train = bp.data, bp.target
print("type of text_train: {}".format(type(bp_train)))
print("length of text_train: {}".format(len(bp_train)))
print("text_train[1]:\n{}".format(bp_train[1].decode("utf-8")))
label_set = list(set(label_train))
for element in label_set:
    print(element, '---->', list(label_train).count(element))
#take only 760 elements of each bp
bp_train_bal = []
label_train_bal = []

for bp in label_set:
    bp_only = [bp_train[index] for index in range(len(label_train)) if label_train[index]==bp]
    random_indexes = np.random.permutation(len(bp_only))
    bp_only = [bp_only[index] for index in random_indexes[:760]]
    bp_train_bal += bp_only
    label_train_bal += [bp for i in range(len(bp_only))]
uft_dataset = [bp_train_bal[index].decode("utf-8") for index in range(len(bp_train_bal))]
masked_dataset = numeric_masker(uft_dataset)
masked_dataset[0]
pipe = make_pipeline(TfidfVectorizer(min_df=5, norm=None),
LogisticRegression())
param_grid = {'logisticregression__C': [0.001, 0.01, 0.1, 1, 10]}
grid = GridSearchCV(pipe, param_grid, cv=5)
grid.fit(masked_dataset, label_train_bal)
print("Best cross-validation score: {:.2f}".format(grid.best_score_))
vectorizer = grid.best_estimator_.named_steps["tfidfvectorizer"]
# transform the training dataset
X_train = vectorizer.transform(masked_dataset)
# find maximum value for each of the features over the dataset
max_value = X_train.max(axis=0).toarray().ravel()
sorted_by_tfidf = max_value.argsort()
# get feature names
feature_names = np.array(vectorizer.get_feature_names())
for bp in range(len(grid.best_estimator_.named_steps["logisticregression"].coef_)):
    test_coef = grid.best_estimator_.named_steps["logisticregression"].coef_[bp]
    idx = np.argpartition(test_coef, -40)[-40:]
    indices = idx[np.argsort((-test_coef)[idx])]
    
    most_important_feature = feature_names[indices]
    print(bp, '---->', most_important_feature)<jupyter_output>0 ----> ['adc' 'reserva' 'inconstitucionalidade' 'regional' 'fracionário'
 'plenário' 'cláusula' 'órgão' 'norma' 'art' 'subsidiária' 'região'
 'trabalhistas' 'tribunal' 'justiça' 'ofensa' 'trabalho' 'constituição'
 'município' 'sem' 'normativo' 'afastado' 'afasta' 'epp' 'cf' 'julgamento'
 'responsabilidade' 'declare' 'transportes' 'declarou' 'sul' 'gonçalves'
 'do' 'declarar' 'dispositivo' 'contrariedade' 'ltda' 'incompatibilidade'
 'acórdão' 'tst']
1 ----> ['algemas' 'júri' 'uso' 'audiência' 'comarca' 'nulidade' 'prisão' 'vara'
 'juíza' 'aranalde' 'algemado' 'manuel' 'requisitório' 'criminal'
 'circunscrição' 'solicitem' 'venceslau' 'da' 'idelos' 'lojistas'
 'shopping' 'luciana' 'transgressão' 'regente' 'marinho' 'arquive' 'leme'
 'avulsa' 'requerido' 'feito' 'carneiro' 'se' 'despacho' 'regularize'
 'geovani' 'juízo' 'petição' 'flagrante' 'escusar' 'damacena']
2 ----> ['acesso' 'criminal' 'inquérito' 'se' 'negado' 'polícia' 'informações'
 'petição' 'defesa' 'autos' 'ministério' 'deleg[...]<jupyter_text>One vs. All Ministers<jupyter_code>#remove nan minister entry
ministers_set = list(set(df['minister']))
ministers_set.pop(0)
ministers_set
for element in ministers_set:
    print(element, '---->',minister.count(element))
#use only ministers that  have more than 700 citations
ministers_set_copy = ministers_set[:]
for element in ministers_set_copy:
    if minister.count(element)<700:
        ministers_set.remove(element)
ministers_set
#make a folder to store the data
if os.path.isdir('minister') == False:
    os.mkdir('minister') 
    
#make a folder for each limit, allowing then to use sklearn.datasets.load_files
for minis in ministers_set:
    if os.path.isdir('minister/'+str(minis)) == False:
        os.mkdir('minister/'+str(minis)) 
for index in range(len(ids)):
    if minister[index] in ministers_set:
        string = 'decisions_allbps/decisions/'+ids[index]+'.txt'
        shutil.copyfile(string, 'minister/'+str(minister[index])+'/'+ ids[index]+'.txt')
ministers = load_files("minister")
#creates train and label sets
minister_train, label_train = ministers.data, ministers.target
print("type of text_train: {}".format(type(minister_train)))
print("length of text_train: {}".format(len(minister_train)))
print("text_train[1]:\n{}".format(minister_train[1].decode("utf-8")))
label_set = list(set(label_train))
for element in label_set:
    print(element, '---->', list(label_train).count(element))
#take only 700 elements of each bp
minister_train_bal = []
label_train_bal = []

#balances the dataset so every minister have same representativity
for minis in label_set:
    minister_only = [minister_train[index] for index in range(len(label_train)) if label_train[index]==minis]
    random_indexes = np.random.permutation(len(minister_only)) 
    minister_only = [minister_only[index] for index in random_indexes[:700]] #picks 700 random documents for each minister
    minister_train_bal += minister_only
    label_train_bal += [minis for i in range(len(minister_only))]
#make a list of words to mask
minister_names = [minis.split() for minis in ministers_set]
flat_names = [item for sublist in minister_names for item in sublist]
flat_names += ['carmen', 'luis', 'lucia']
#substitute gender specific words that can be used to relate to some ministers
flat_names += ['relatora', 'relator', 'ministro','ministra','minis', 'turma', 'primeira', 'segunda', 'min']
cap_names = [minis.capitalize() for minis in flat_names]
up_names = [minis.upper() for minis in flat_names]
mask_list = flat_names+cap_names+up_names
#add puntuation to everything
comma_list = [word+',' for word in mask_list]
dot_list = [word+'.' for word in mask_list]
mask_list += comma_list+dot_list
mask_list
def masker(dataset, list_of_masked):
    '''Substitues words that are in the masked list by an empty string
    Input: dataset made of texts
    Output: the same dataset made of texts, only with masked words'''
    masked_set = []
    for index in range(len(dataset)):
        new_string = ''
        splitted_doc = dataset[index].split()
        for word in range(len(splitted_doc)):
            if splitted_doc[word] in list_of_masked:
                new_string += ''
            else:
                new_string += ' '+ splitted_doc[word]
        masked_set.append(new_string)
    return masked_set
uft_dataset = [minister_train_bal[index].decode("utf-8") for index in range(len(minister_train_bal))]
masked_dataset = masker(uft_dataset, mask_list)
masked_dataset[0]
pipe = make_pipeline(TfidfVectorizer(min_df=5, norm=None),
LogisticRegression())
param_grid = {'logisticregression__C': [0.001, 0.01, 0.1, 1, 10]}
grid = GridSearchCV(pipe, param_grid, cv=5)
grid.fit(masked_dataset, label_train_bal)
print("Best cross-validation score: {:.2f}".format(grid.best_score_))
vectorizer = grid.best_estimator_.named_steps["tfidfvectorizer"]
# transform the training dataset
X_train = vectorizer.transform(masked_dataset)
# find maximum value for each of the features over the dataset
max_value = X_train.max(axis=0).toarray().ravel()
sorted_by_tfidf = max_value.argsort()
# get feature names
feature_names = np.array(vectorizer.get_feature_names())
for minis in range(len(grid.best_estimator_.named_steps["logisticregression"].coef_)):
    test_coef = grid.best_estimator_.named_steps["logisticregression"].coef_[minis]
    idx = np.argpartition(test_coef, -40)[-40:]
    indices = idx[np.argsort((-test_coef)[idx])]
    
    #shows 40 most important words for deciding belonging to each class
    most_important_feature = feature_names[indices]
    print(minis, '---->', most_important_feature)
#deletes extra folder for splitting the dataset
shutil.rmtree('minister')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15139186295503212, 'fraction_numerical': 0.02069950035688794, 'mean_word_length': 5.210549645390071, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '581755', 'license': 'MIT', 'n_tokens_mistral': 5278, 'n_tokens_neox': 5038, 'n_words': 1066, 'path': 'LogisticRegressionBPsSTF.ipynb', 'repo_name': 'HLovisiEnnes/TDAResearch', 'size': '14026'}