<jupyter_start><jupyter_text>Normalize Form Fields ClassifierThoughts:- **Make this work with new package**- fill training data with forms selected based on connections based on random sampeling in jur- make the training only take palce if source and labeled are different (filter on old_fields = new fields)- don't lable auto-fields label with adjacent text from another field use `_ignore_`- consider training only within jurisdiction- should I norm vectors in reormating function? Load Libraries<jupyter_code>jurs = [
        "MI",
        "UT",
        "ME"
        ]
import pandas as pd
import re
from os import walk
from sklearn.metrics import accuracy_score

if 1==2:
    import PyPDF2
    from sklearn.preprocessing import normalize
    import urllib.request, json 
    from urllib.request import urlopen
    from bs4 import BeautifulSoup
    import os.path
    from os import path
    import numpy as np
    #!pip install py-readability-metrics
    from readability import Readability
    #!python -m nltk.downloader punkt

    import networkx as nx

    from matplotlib.pyplot import figure

    from pyvis.network import Network

    import time
    from datetime import date

    import spacy


    from sklearn.metrics import accuracy_score

    from sklearn.model_selection import KFold

    from sklearn.linear_model import LogisticRegression
    from sklearn import tree
    from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, VotingClassifier
    from sklearn.naive_bayes import GaussianNB
    from sklearn.neural_network import MLPClassifier
    from sklearn.ensemble import GradientBoostingClassifier

    from joblib import dump, load

    import math

    from sklearn.metrics.pairwise import cosine_similarity# define matrix with all zero values

    #import nltk
    #nltk.download('stopwords')

    from nltk.corpus import stopwords
    from nltk.tokenize import word_tokenize
import sys
sys.path.insert(0, '../../formfyxer')
import formfyxer.lit_explorer as ff<jupyter_output><empty_output><jupyter_text>Get training and test form data from each of our jurisdictionsFirst we'll start by grabing a set of forms with automaticly-generated form field names.<jupyter_code>included_fields = [
                    "users1_name",
                    "users1_birthdate",
                    "users1_address_line_one",
                    "users1_address_line_two",
                    "users1_address_city",
                    "users1_address_state",
                    "users1_address_zip",
                    "users1_phone_number",
                    "users1_email",
                    "plantiffs1_name",
                    "defendants1_name",
                    "petitioners1_name",
                    "respondents1_name",
                    "docket_number",
                    "trial_court_county",
                    "users1_signature",
                    "signature_date"
                    ]
path = "ML_training/auto/"
f = []
for (dirpath, dirnames, filenames) in walk(path):
    f.extend(filenames)
    break
f1 = [file for file in f]

auto_df = pd.DataFrame([],columns=["form_id","jurisdiction","group","n","per","this_field"])

for file in f1:
    if re.search(".*\.pdf",file):
        form_id = re.search("(.*)\.pdf",file).groups()[0]
        print(path+file)
        stats = ff.parse_form(path+file,"title","group")
        i = 0 
        n_fields = len(stats["fields"])
        last_field = "null"
        for stat in stats["fields"]:
            #print(form_id,df[df["id"]==form_id]["jurisdiction"].values[0])
            #auto_df = auto_df.append(pd.DataFrame([[form_id,df[df["id"]==form_id]["jurisdiction"].values[0],df[df["id"]==form_id]["group"].values[0],i,(i/n_fields),lit.reCase(stat)]],columns=["form_id","jurisdiction","group","n","per","this_field"]), ignore_index=True,sort=False)
            auto_df = auto_df.append(pd.DataFrame([[form_id,"state","group",i,(i/n_fields),ff.reCase(stat)]],columns=["form_id","jurisdiction","group","n","per","this_field"]), ignore_index=True,sort=False)
            i += 1

auto_df
auto_df["this_field"].unique()<jupyter_output><empty_output><jupyter_text>Now we'll grab the same set of forms with ground truth label names.<jupyter_code>path = "ML_training/ground/"
f = []
for (dirpath, dirnames, filenames) in walk(path):
    f.extend(filenames)
    break
f1 = [file for file in f]

ground_df = pd.DataFrame([],columns=["form_id","n"]+included_fields+["this_field_ground"])

for file in f1:
    if re.search(".*\.pdf",file):
        form_id = re.search("(.*)\.pdf",file).groups()[0]
        print(path+file)
        stats = ff.parse_form(path+file,"title","group")
        i = 0 
        n_fields = len(stats["fields"])
        last_field = "null"
        for stat in stats["fields"]:
            #print("\t"+str(i)+","+str(i/n_fields)+","+last_field+","+stat[:5])
            if "_ignore_" not in stat:
                #
                # this * is a hold over from form explorer that should be removed
                if re.sub("^\*","",stat) not in included_fields:
                    stat = "_nonstandard_"
                field_matches = []
                for item in included_fields:
                    if last_field==item:
                        field_matches.append(1)
                    else:
                        field_matches.append(0)
                ground_df = ground_df.append(pd.DataFrame([[form_id,i]+field_matches+[stat]],columns=["form_id","n"]+included_fields+["this_field_ground"]), ignore_index=True,sort=False)
            last_field = stat
            i += 1

ground_df
ground_df["this_field_ground"].unique()<jupyter_output><empty_output><jupyter_text>All together nowLet's merge the form data and create some features.<jupyter_code>df_all = auto_df.merge(ground_df, how="inner")
print(len(df_all))
display(df_all.head())

# take care of some categorical variables
parts = pd.get_dummies(df_all['jurisdiction'])
parts = pd.concat([parts, pd.get_dummies(df_all['group'])], axis=1)
df_all = pd.concat([df_all, parts], axis=1)
jurisdictions = df_all['jurisdiction'].unique().tolist()

#groups = df_all['group'].unique().tolist()
groups = ["state"]

df_all = df_all.drop(['jurisdiction','group'], 1, errors='ignore').copy()
display(df_all.head())
df_all["this_field_vec"] = df_all["this_field"].apply(ff.vectorize)
#df_all["last_field_ground_vec"] = df_all["last_field_ground"].apply(vectorize)
df_all = df_all.drop(['this_field','last_field_ground'], 1, errors='ignore').copy()
display(df_all.head())
#df_all["last_field_ground_vec"] = df_all["last_field_ground_vec"].apply(norm)
display(df_all.head())
df_all = pd.concat([df_all, pd.DataFrame(df_all.this_field_vec.values.tolist(), index= df_all.index).add_prefix("vec_this")], axis=1)
#df_all = pd.concat([df_all, pd.DataFrame(df_all.last_field_ground_vec.values.tolist(), index= df_all.index).add_prefix("vec_last")], axis=1)
#df_all = df_all.drop(['this_field_vec','last_field_ground_vec'], 1, errors='ignore').copy()
df_all = df_all.drop(['this_field_vec'], 1, errors='ignore').copy()
display(df_all.head())
df_class = pd.concat([df_all[["form_id"]], df_all[jurisdictions]], axis=1)
df_class = pd.concat([df_class, df_all[["n","per"]]], axis=1)
df_class = pd.concat([df_class, df_all[groups], df_all[included_fields]], axis=1)
df_class = pd.concat([df_class, df_all[[x for x in df_all.columns if "vec_" in x]], df_all[["this_field_ground"]]], axis=1)
df_class
#down sample
if 1==1:
    df0 = df_class[df_class["this_field_ground"]!="_nonstandard_"]
    df1 = df_class[df_class["this_field_ground"]=="_nonstandard_"][:len(df_class[df_class["this_field_ground"]!="_nonstandard_"])]
    df_class = pd.concat([df0,df1],ignore_index=True)

    display(df_class)
1-len(df_class["this_field_ground"].unique())/len(df_class["this_field_ground"])
form_list = pd.DataFrame([],columns=["form_id"])
form_list["form_id"] = df_class["form_id"].unique()
holdout = form_list.sample(frac=0.2)

# create a dataframe containing a random sample of rows
class_holdout = df_class.loc[df_class.form_id.isin(holdout["form_id"])] 

# create a dataframe that conatins the rows from except for those in holdout
class_training = df_class.loc[~df_all.form_id.isin(holdout["form_id"])]

# make a training dataframe containing just features
features_train = class_training.drop(["form_id","this_field_ground"], axis=1).values

# make a training dataframe containing only the target
labels_train = class_training["this_field_ground"].values

# make a testing/holdout dataframe containing just features
features_test = class_holdout.drop(["form_id","this_field_ground"], axis=1).values

# make a testing/holdout dataframe containing only the target
labels_test = class_holdout["this_field_ground"].values

print("size of training",len(class_training))
print("size of testing",len(class_holdout))
from sklearn.linear_model import LogisticRegression
model = LogisticRegression(fit_intercept = False, C = 1e9)
clf_1 = model.fit(features_train, labels_train)
pred = clf_1.predict(features_test)
prob = clf_1.predict_proba(features_test)[:,1] 
print("Logistic Regression")

print(accuracy_score(labels_test,pred))
#evaluate(prob, pred, labels_test)  

print("")
print(labels_test,"\n")
print(pred)
from sklearn.ensemble import AdaBoostClassifier
clf_2 = AdaBoostClassifier()
clf_2 = clf_2.fit(features_train, labels_train)
pred = clf_2.predict(features_test)
prob = clf_2.predict_proba(features_test)#[:,1] 
print("AdaBoost Forest")

print(accuracy_score(labels_test,pred))
#evaluate(prob,pred, labels_test)  

print("")
print(labels_test,"\n")
print(pred)
from sklearn.ensemble import GradientBoostingClassifier
clf_3 = GradientBoostingClassifier()
clf_3 = clf_3.fit(features_train, labels_train)
pred = clf_3.predict(features_test)
prob = clf_3.predict_proba(features_test)#[:,1] 
print("Gradient Boost")

print(accuracy_score(labels_test,pred))
#evaluate(prob,pred, labels_test)  

print("")
print(labels_test,"\n")
print(pred)
from sklearn.ensemble import RandomForestClassifier
clf_4 = RandomForestClassifier()
clf_4 = clf_4.fit(features_train, labels_train)
pred = clf_4.predict(features_test)
prob = clf_4.predict_proba(features_test)#[:,1] 
print("Random Forest")

print(accuracy_score(labels_test,pred))
#evaluate(prob,pred, labels_test)  

print("")
print(labels_test,"\n")
print(pred)
overwrite = input("Overwrite old data? (y/n)\n")
if "y" in overwrite.lower():    
    dump(included_fields, '../formfyxer/data/included_fields.joblib') 
    dump(clf_4, '../formfyxer/data/clf_field_names.joblib') 
    dump(jurisdictions, '../formfyxer/data/jurisdictions.joblib') 
    dump(groups, '../formfyxer/data/groups.joblib')<jupyter_output>Overwrite old data? (y/n)
y
{'fraction_non_alphanumeric': 0.1336564451486625, 'fraction_numerical': 0.007736042501631094, 'mean_word_length': 4.218871595330739, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '567359', 'license': 'MIT', 'n_tokens_mistral': 3631, 'n_tokens_neox': 3501, 'n_words': 701, 'path': 'dev-testing/train classifier.ipynb', 'repo_name': 'SuffolkLITLab/FormFyxer', 'size': '10729'}