<commit_before>from time import gmtime, strftime
import pandas as pd
from classifier import Classifier
from sklearn import svm
from tools import random_sample
from sklearn import metrics

from preprocessing.FeatureExporter import FeatureExporter
from preprocessing.FeatureImporter import FeatureImporter
from sklearn.linear_model.logistic import LogisticRegression

time_sub = strftime("%Y-%m-%d %H:%M:%S", gmtime()).replace(' ', '__')

train_df = pd.read_csv("data/training_set.txt", sep=" ", header=None)
train_df.columns = ["source", "target", "label"]

test_df = pd.read_csv("data/testing_set.txt", sep=" ", header=None)
test_df.columns = ["source", "target"]

node_information_df = pd.read_csv("data/node_information.csv", sep=",", header=None)
node_information_df.columns = ["ID", "year", "title", "authors", "journalName", "abstract"]
node_information_df = node_information_df.reset_index().set_index("ID")
node_information_df["authors"].fillna("", inplace=True)

df_dict = dict()
training_set_percentage = 0.05

df_dict["train"] = {
    "filename": 'training_set.txt',
    "df": random_sample(train_df, p=training_set_percentage)
}

testing_on_train = True
# features = ["commonNeighbours","original","inOutDegree","similarity"]
features = ["authors"]
verbose = True
freq = 5000

# By uncommenting you can tune in the parameters
parameters = {}
# parameters = {"percentile":95,"metric":"degrees"}

if testing_on_train:
    df_dict["test"] = {
        "filename": 'testing_training_set.txt',<commit_msg>correction time<commit_after>from time import localtime, strftime
import pandas as pd
from classifier import Classifier
from sklearn import svm
from tools import random_sample
from sklearn import metrics

from preprocessing.FeatureExporter import FeatureExporter
from preprocessing.FeatureImporter import FeatureImporter
from sklearn.linear_model.logistic import LogisticRegression

time_sub = strftime("%Y-%m-%d %H:%M:%S", localtime()).replace(' ', '__')

train_df = pd.read_csv("data/training_set.txt", sep=" ", header=None)
train_df.columns = ["source", "target", "label"]

test_df = pd.read_csv("data/testing_set.txt", sep=" ", header=None)
test_df.columns = ["source", "target"]

node_information_df = pd.read_csv("data/node_information.csv", sep=",", header=None)
node_information_df.columns = ["ID", "year", "title", "authors", "journalName", "abstract"]
node_information_df = node_information_df.reset_index().set_index("ID")
node_information_df["authors"].fillna("", inplace=True)

df_dict = dict()
training_set_percentage = 0.05

df_dict["train"] = {
    "filename": 'training_set.txt',
    "df": random_sample(train_df, p=training_set_percentage)
}


testing_on_train = False
features = ["commonNeighbours", "original", "inOutDegree", "similarity", "authors"]
# features = ["authors"]
verbose = True
freq = 5000

# By uncommenting you can tune in the parameters
parameters = {}
# parameters = {"percentile":95,"metric":"degrees"}

if testing_on_train:
    df_dict["test"] = {
        "filename": 'testing_training_set.txt',
{'fraction_non_alphanumeric': 0.14487516425755584, 'fraction_numerical': 0.005913272010512484, 'mean_word_length': 6.827763496143959, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '687643', 'n_tokens_mistral': 993, 'n_tokens_neox': 956, 'n_words': 218}