<commit_before>#!/usr/bin/python
from os.path import join,exists,dirname
import numpy as np
import pickle
from sklearn.datasets import load_svmlight_file, dump_svmlight_file
from sklearn.metrics import f1_score, precision_recall_fscore_support
from uda_common import zero_pivot_columns, zero_nonpivot_columns, read_pivots, evaluate_and_print_scores, align_test_X_train, get_f1, find_best_c, read_feature_groups, read_feature_lookup
import os
import scipy.sparse
import sys
from sklearn import svm

def main(args):
    if len(args) < 1:
        sys.stderr.write("Required argument(s): <combined & reduced labeled data>\n\n")
        sys.exit(-1)

    goal_ind = 2

    sys.stderr.write("Reading source data from %s\n" % (args[0]))
    all_X, all_y = load_svmlight_file(args[0])
    num_instances, num_feats = all_X.shape

    domain_map = read_feature_groups(join(dirname(args[0]), 'reduced-feature-groups.txt'))
    domain_inds = domain_map['Domain']

    feature_map = read_feature_lookup(join(dirname(args[0]), 'reduced-features-lookup.txt'))

    for direction in [0,1]:
        score_ave = 0
        sys.stderr.write("using domain %s as source, %s as target\n"  %
            (feature_map[domain_inds[direction]],feature_map[domain_inds[1-direction]]))

        train_instance_inds = np.where(all_X[:,domain_inds[direction]].toarray() > 0)[0]
        X_train = all_X[train_instance_inds,:]
        y_train = all_y[train_instance_inds]
        num_train_instances = X_train.shape[0]

        test_instance_inds = np.where(all_X[:,domain_inds[1-direction]].toarray() > 0)[0]
        X_test = all_X[test_instance_inds,:]
        y_test = all_y[test_instance_inds]
        num_test_instances = X_test.shape[0]

        (l2_c, l2_f1) = find_best_c(X_train, y_train, scorer=f1_score, pos_label=goal_ind)
        print("Tuning on source selected c value %f" % (l2_c))

        clf = svm.LinearSVC(C=l2_c)
        clf.fit(X_train, y_train)
        y_predicted = clf.predict(X_test)
        # f1 = f1_score(y_test, y_predicted, pos_label=goal_ind)
        p,r,f1,_ = precision_recall_fscore_support(y_test, y_predicted, average='binary', pos_label=goal_ind)
        print("Precision,recall,F-score on test data is %f\t%f\t%f" % (p,r,f1))

        best_c = best_f = best_r = best_p = 0
        for c_exp in range(-3, 4):
            c = 10. ** c_exp
            #print("Testing with c=%f" % (c))
            clf = svm.LinearSVC(C=c)
            clf.fit(X_train, y_train)
            y_predicted = clf.predict(X_test)
            # f1 = f1_score(y_test, y_predicted, pos_label=goal_ind)
            p,r,f1,_ = precision_recall_fscore_support(y_test, y_predicted, average='binary', pos_label=goal_ind)
            if f1 > best_f:
                best_f = f1
                best_r = r
                best_p = p
                best_c = c

        print("P/R/F if we tune c=%f to optimize test set F score is %f\t%f\t%f"%  (best_c, best_p, best_r, best_f))



if __name__ == "__main__":
    main(sys.argv[1:])<commit_msg>Add prevalence info to no adaptation, and search a wider gamut of c values.<commit_after>#!/usr/bin/python
from os.path import join,exists,dirname
import numpy as np
import pickle
from sklearn.datasets import load_svmlight_file, dump_svmlight_file
from sklearn.metrics import f1_score, precision_recall_fscore_support
from uda_common import zero_pivot_columns, zero_nonpivot_columns, read_pivots, evaluate_and_print_scores, align_test_X_train, get_f1, find_best_c, read_feature_groups, read_feature_lookup
import os
import scipy.sparse
import sys
from sklearn import svm

def main(args):
    if len(args) < 1:
        sys.stderr.write("Required argument(s): <combined & reduced labeled data>\n\n")
        sys.exit(-1)

    goal_ind = 2

    sys.stderr.write("Reading source data from %s\n" % (args[0]))
    all_X, all_y = load_svmlight_file(args[0])
    num_instances, num_feats = all_X.shape

    domain_map = read_feature_groups(join(dirname(args[0]), 'reduced-feature-groups.txt'))
    domain_inds = domain_map['Domain']

    feature_map = read_feature_lookup(join(dirname(args[0]), 'reduced-features-lookup.txt'))

    for direction in [0,1]:
        score_ave = 0
        sys.stderr.write("using domain %s as source, %s as target\n"  %
            (feature_map[domain_inds[direction]],feature_map[domain_inds[1-direction]]))

        train_instance_inds = np.where(all_X[:,domain_inds[direction]].toarray() > 0)[0]
        X_train = all_X[train_instance_inds,:]
        y_train = all_y[train_instance_inds]
        num_train_instances = X_train.shape[0]

        test_instance_inds = np.where(all_X[:,domain_inds[1-direction]].toarray() > 0)[0]
        X_test = all_X[test_instance_inds,:]
        y_test = all_y[test_instance_inds]
        num_test_instances = X_test.shape[0]

        source_prevalence = len(np.where(y_train == goal_ind)[0]) / float(len(y_train))
        sys.stderr.write("Source prevalence is: %f\n" % source_prevalence)
        target_prevalence = len(np.where(y_test == goal_ind)[0]) / float(len(y_test))
        sys.stderr.write("Target prevalence is: %f\n" % target_prevalence)
        

        (l2_c, l2_f1) = find_best_c(X_train, y_train, scorer=f1_score, pos_label=goal_ind)
        # p,r,f,_ = cross_val_score(svm.LinearSVC(C=C, penalty=penalty, dual=dual), X_train, y_train, scoring=scorer, n_jobs=1))
        print("Tuning on source selected c value %f" % (l2_c))

        clf = svm.LinearSVC(C=l2_c)
        clf.fit(X_train, y_train)
        y_predicted = clf.predict(X_test)
        predicted_prevalence = len(np.where(y_predicted == goal_ind)[0]) / float(len(y_test))
        sys.stderr.write("Predicted (target) prevalence is %f\n" % (predicted_prevalence))

        # f1 = f1_score(y_test, y_predicted, pos_label=goal_ind)
        p,r,f1,_ = precision_recall_fscore_support(y_test, y_predicted, average='binary', pos_label=goal_ind)
        print("Precision,recall,F-score on test data is %f\t%f\t%f" % (p,r,f1))

        best_c = best_f = best_r = best_p = predicted_prevalence = 0
        for c_exp in range(-10, 0):
            c = 2. ** c_exp
            #print("Testing with c=%f" % (c))
            clf = svm.LinearSVC(C=c)
            clf.fit(X_train, y_train)
            y_predicted = clf.predict(X_test)
            # f1 = f1_score(y_test, y_predicted, pos_label=goal_ind)
            p,r,f1,_ = precision_recall_fscore_support(y_test, y_predicted, average='binary', pos_label=goal_ind)
            if f1 > best_f:
                best_f = f1
                best_r = r
                best_p = p
                best_c = c
                predicted_prevalence = len(np.where(y_predicted == goal_ind)[0]) / float(len(y_test))

        print("P/R/F if we tune c=%f to optimize test set F score is %f\t%f\t%f with prevalence %f"%  (best_c, best_p, best_r, best_f, predicted_prevalence))



if __name__ == "__main__":
    main(sys.argv[1:])
{'fraction_non_alphanumeric': 0.11965811965811966, 'fraction_numerical': 0.012603215993046502, 'mean_word_length': 3.2696351267779837, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6587050', 'n_tokens_mistral': 2599, 'n_tokens_neox': 2508, 'n_words': 477}