<jupyter_start><jupyter_text>ABE0 with Hyperopt - Within Repo<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsRegressor
import numpy as np
import hyperopt
from sklearn.model_selection import cross_val_score
from hyperopt import fmin, tpe, hp, SparkTrials, STATUS_OK, Trials
import mlflow


def objective(params):
    reg = KNeighborsRegressor(**params)
    mae = cross_val_score(reg, X_train, y_train, scoring='neg_mean_absolute_error').mean()
    # fmin() tries to minimize the objective, so we need to redirect neg_mean_absolute_error with "-"
    return {'loss': -mae, 'status': STATUS_OK}

SEARCH_SPACE = hp.choice('classifier_type', [
    {
        'n_neighbors': hp.choice('n_neighbors', list(np.arange(2, 10+1, 1, dtype=int))),
        'weights': hp.choice('weights', ['uniform', 'distance']),
        'algorithm': hp.choice('algorithm', ['auto', 'ball_tree', 'kd_tree', 'brute']),
        'leaf_size': hp.choice('leaf_size', list(np.arange(10, 50+1, 10, dtype=int))),
        'p': hp.choice('p', list(np.arange(1, 2+1, 1, dtype=int))),
        'n_jobs': hp.choice('n_jobs', [16])
    }
])

PATH = "../sp_dataset/marked_data/"
# cross project - within repo
WITHIN_REPO = [
                {'train': ['mesos'], 'test': ['usergrid']},
                {'train': ['usergrid'], 'test': ['mesos']},
                {'train': ['appceleratorstudio'], 'test': ['aptanastudio']},
                {'train': ['appceleratorstudio'], 'test': ['titanium']},
                {'train': ['titanium'], 'test': ['appceleratorstudio']},
                {'train': ['aptanastudio'], 'test': ['titanium']},
                {'train': ['mule'], 'test': ['mulestudio']},
                {'train': ['mulestudio'], 'test': ['mule']}
              ]

report = []
for pair in WITHIN_REPO:
    # split to 60% training and 40% validation
    data = pd.read_csv(PATH + pair["train"][0] + ".csv")
    
    train_data = data[: int(len(data)*0.6)]
    train_data = data
    
    X_train = train_data["title"].tolist()
    y_train = train_data["storypoint"].tolist()
    val_data = data[int(len(data)*0.6):]

    test_data = pd.read_csv(PATH + pair["test"][0] + ".csv")
    X_test = test_data["title"].tolist()
    y_test = np.array(test_data["storypoint"].tolist())
    
    # apply BoW feature extraction
    vectorizer = TfidfVectorizer(norm='l2')
    vectorizer = vectorizer.fit(X_train)
    vectorizer = vectorizer.fit(X_test)
    
    X_train = vectorizer.transform(X_train).todense()
    X_test = vectorizer.transform(X_test).todense()

    algo = tpe.suggest
    with mlflow.start_run():
        best_result = fmin(fn=objective, 
                           space=SEARCH_SPACE,
                           algo=algo,
                           max_evals=32)
    best_model = hyperopt.space_eval(SEARCH_SPACE, best_result)    
    neigh = KNeighborsRegressor(n_neighbors=best_model['n_neighbors'],
                                weights=best_model['weights'],
                                algorithm=best_model['algorithm'],
                                leaf_size=best_model['leaf_size'],
                                p=best_model['p'])
    neigh.fit(X_train, y_train)
    preds = neigh.predict(X_test)
    mae = round(sum(abs(preds - y_test)) / len(preds), 2)
    report.append(f"""Train {pair["train"][0]} | Test {pair["test"][0]}: {mae}""")
maes = []
train_file = []
test_file = []

for record in report:
    print(record)
    train_file.append(record.split(" ")[1])
    test_file.append(record.split(" ")[4].strip(":"))
    mae_val = record.split(":")[-1]
    mae_val = float(mae_val.strip())
    maes.append(mae_val)
pd.DataFrame(data={"train_file": train_file,
                   "test_file": test_file,
                   "mae": maes}).to_csv("within_repo_abe0_hyperopt.csv", index=False)<jupyter_output><empty_output><jupyter_text>ABE0 with Hyperopt - Cross Repo<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsRegressor
import numpy as np
import hyperopt
from sklearn.model_selection import cross_val_score
from hyperopt import fmin, tpe, hp, SparkTrials, STATUS_OK, Trials
import mlflow


def objective(params):
    reg = KNeighborsRegressor(**params)
    mae = cross_val_score(reg, X_train, y_train, scoring='neg_mean_absolute_error').mean()
    # fmin() tries to minimize the objective, so we need to redirect neg_mean_absolute_error with "-"
    return {'loss': -mae, 'status': STATUS_OK}

SEARCH_SPACE = hp.choice('classifier_type', [
    {
        'n_neighbors': hp.choice('n_neighbors', list(np.arange(2, 10+1, 1, dtype=int))),
        'weights': hp.choice('weights', ['uniform', 'distance']),
        'algorithm': hp.choice('algorithm', ['auto', 'ball_tree', 'kd_tree', 'brute']),
        'leaf_size': hp.choice('leaf_size', list(np.arange(10, 50+1, 10, dtype=int))),
        'p': hp.choice('p', list(np.arange(1, 2+1, 1, dtype=int))),
        'n_jobs': hp.choice('n_jobs', [16])
    }
])

PATH = "../sp_dataset/marked_data/"

# cross project - cross repo
CROSS_REPO = [
                {'train': ['clover'], 'test': ['usergrid']},
                {'train': ['talendesb'], 'test': ['mesos']},
                {'train': ['talenddataquality'], 'test': ['aptanastudio']},
                {'train': ['mule'], 'test': ['titanium']},
                {'train': ['talenddataquality'], 'test': ['appceleratorstudio']},
                {'train': ['mulestudio'], 'test': ['titanium']},
                {'train': ['appceleratorstudio'], 'test': ['mulestudio']},
                {'train': ['appceleratorstudio'], 'test': ['mule']}
             ]
report = []
for pair in CROSS_REPO:
    # split to 60% training and 40% validation
    data = pd.read_csv(PATH + pair["train"][0] + ".csv")
    
    train_data = data[: int(len(data)*0.6)]
    train_data = data
    
    X_train = train_data["title"].tolist()
    y_train = train_data["storypoint"].tolist()
    val_data = data[int(len(data)*0.6):]

    test_data = pd.read_csv(PATH + pair["test"][0] + ".csv")
    X_test = test_data["title"].tolist()
    y_test = np.array(test_data["storypoint"].tolist())
    
    # apply BoW feature extraction
    vectorizer = TfidfVectorizer(norm='l2')
    vectorizer = vectorizer.fit(X_train)
    vectorizer = vectorizer.fit(X_test)
    
    X_train = vectorizer.transform(X_train).todense()
    X_test = vectorizer.transform(X_test).todense()

    algo = tpe.suggest
    with mlflow.start_run():
        best_result = fmin(fn=objective, 
                           space=SEARCH_SPACE,
                           algo=algo,
                           max_evals=32)
    best_model = hyperopt.space_eval(SEARCH_SPACE, best_result)    
    neigh = KNeighborsRegressor(n_neighbors=best_model['n_neighbors'],
                                weights=best_model['weights'],
                                algorithm=best_model['algorithm'],
                                leaf_size=best_model['leaf_size'],
                                p=best_model['p'])
    neigh.fit(X_train, y_train)
    preds = neigh.predict(X_test)
    mae = round(sum(abs(preds - y_test)) / len(preds), 2)
    report.append(f"""Train {pair["train"][0]} | Test {pair["test"][0]}: {mae}""")
maes = []
train_file = []
test_file = []

for record in report:
    print(record)
    train_file.append(record.split(" ")[1])
    test_file.append(record.split(" ")[4].strip(":"))
    mae_val = record.split(":")[-1]
    mae_val = float(mae_val.strip())
    maes.append(mae_val)
pd.DataFrame(data={"train_file": train_file,
                   "test_file": test_file,
                   "mae": maes}).to_csv("cross_repo_abe0_hyperopt.csv", index=False)<jupyter_output><empty_output><jupyter_text>ABE0 without Hyperopt - Within Repo<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsRegressor
import numpy as np


PATH = "../sp_dataset/marked_data/"
# cross project - within repo
WITHIN_REPO = [
                {'train': ['mesos'], 'test': ['usergrid']},
                {'train': ['usergrid'], 'test': ['mesos']},
                {'train': ['appceleratorstudio'], 'test': ['aptanastudio']},
                {'train': ['appceleratorstudio'], 'test': ['titanium']},
                {'train': ['titanium'], 'test': ['appceleratorstudio']},
                {'train': ['aptanastudio'], 'test': ['titanium']},
                {'train': ['mule'], 'test': ['mulestudio']},
                {'train': ['mulestudio'], 'test': ['mule']}
              ]

report = []
for pair in WITHIN_REPO:
    # split to 60% training and 40% validation
    data = pd.read_csv(PATH + pair["train"][0] + ".csv")
    
    train_data = data[: int(len(data)*0.6)]
    train_data = data
    
    X_train = train_data["title"].tolist()
    y_train = train_data["storypoint"].tolist()
    val_data = data[int(len(data)*0.6):]

    test_data = pd.read_csv(PATH + pair["test"][0] + ".csv")
    X_test = test_data["title"].tolist()
    y_test = np.array(test_data["storypoint"].tolist())
    
    # apply BoW feature extraction
    vectorizer = TfidfVectorizer(norm='l2')
    vectorizer = vectorizer.fit(X_train)
    vectorizer = vectorizer.fit(X_test)
    
    X_train = vectorizer.transform(X_train).todense()
    X_test = vectorizer.transform(X_test).todense()
    
    neigh = KNeighborsRegressor(n_neighbors=3)
    neigh.fit(X_train, y_train)
    preds = neigh.predict(X_test)
    mae = round(sum(abs(preds - y_test)) / len(preds), 2)
    report.append(f"""Train {pair["train"][0]} | Test {pair["test"][0]}: {mae}""")
maes = []
train_file = []
test_file = []

for record in report:
    print(record)
    train_file.append(record.split(" ")[1])
    test_file.append(record.split(" ")[4].strip(":"))
    mae_val = record.split(":")[-1]
    mae_val = float(mae_val.strip())
    maes.append(mae_val)
pd.DataFrame(data={"train_file": train_file,
                   "test_file": test_file,
                   "mae": maes}).to_csv("within_repo_abe0_no_hyperopt.csv", index=False)<jupyter_output><empty_output><jupyter_text>ABE0 without Hyperopt - Cross Repo<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsRegressor
import numpy as np


PATH = "../sp_dataset/marked_data/"

# cross project - cross repo
CROSS_REPO = [
                {'train': ['clover'], 'test': ['usergrid']},
                {'train': ['talendesb'], 'test': ['mesos']},
                {'train': ['talenddataquality'], 'test': ['aptanastudio']},
                {'train': ['mule'], 'test': ['titanium']},
                {'train': ['talenddataquality'], 'test': ['appceleratorstudio']},
                {'train': ['mulestudio'], 'test': ['titanium']},
                {'train': ['appceleratorstudio'], 'test': ['mulestudio']},
                {'train': ['appceleratorstudio'], 'test': ['mule']}
             ]

report = []
for pair in CROSS_REPO:
    # split to 60% training and 40% validation
    data = pd.read_csv(PATH + pair["train"][0] + ".csv")
    
    train_data = data[: int(len(data)*0.6)]
    train_data = data
    
    X_train = train_data["title"].tolist()
    y_train = train_data["storypoint"].tolist()
    val_data = data[int(len(data)*0.6):]

    test_data = pd.read_csv(PATH + pair["test"][0] + ".csv")
    X_test = test_data["title"].tolist()
    y_test = np.array(test_data["storypoint"].tolist())
    
    # apply BoW feature extraction
    vectorizer = TfidfVectorizer(norm='l2')
    vectorizer = vectorizer.fit(X_train)
    vectorizer = vectorizer.fit(X_test)
    
    X_train = vectorizer.transform(X_train).todense()
    X_test = vectorizer.transform(X_test).todense()
    
    neigh = KNeighborsRegressor(n_neighbors=3)
    neigh.fit(X_train, y_train)
    preds = neigh.predict(X_test)
    mae = round(sum(abs(preds - y_test)) / len(preds), 2)
    report.append(f"""Train {pair["train"][0]} | Test {pair["test"][0]}: {mae}""")
maes = []
train_file = []
test_file = []

for record in report:
    print(record)
    train_file.append(record.split(" ")[1])
    test_file.append(record.split(" ")[4].strip(":"))
    mae_val = record.split(":")[-1]
    mae_val = float(mae_val.strip())
    maes.append(mae_val)
pd.DataFrame(data={"train_file": train_file,
                   "test_file": test_file,
                   "mae": maes}).to_csv("cross_repo_abe0_no_hyperopt.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16993776926759216, 'fraction_numerical': 0.009414392851444072, 'mean_word_length': 3.09640522875817, 'pattern_counts': {'":': 20, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '648856', 'license': 'MIT', 'n_tokens_mistral': 4468, 'n_tokens_neox': 4319, 'n_words': 775, 'path': 'abe0/reproduce_abe0.ipynb', 'repo_name': 'awsm-research/gpt2sp', 'size': '12534'}