<jupyter_start><jupyter_text>Train on C3 train and test on C3 test<jupyter_code>C3_train_df = pd.read_csv(os.environ['C3_TRAIN'])
C3_train_df['pp_comment_text'] = C3_train_df['pp_comment_text'].astype(str)

C3_test_df = pd.read_csv(os.environ['C3_TEST'])
C3_test_df['pp_comment_text'] = C3_test_df['pp_comment_text'].astype(str)
y_C3_train = C3_train_df.constructive_binary     
X_C3_train = C3_train_df.drop(['constructive_binary'], axis = 1)

y_C3_test = C3_test_df.constructive_binary     
X_C3_test = C3_test_df.drop(['constructive_binary'], axis = 1)
classifier = FBConstructivenessClassifier(X_C3_train, y_C3_train, X_C3_test, y_C3_test)
models = {'logistic regression': LogisticRegression, 
          'SVM' : SGDClassifier, 
          'random forest' : RandomForestClassifier, 
          'xgboost' : xgb.XGBClassifier
         }

for model_name, model_class in models.items():
    t = time.time()
    print(model_name, ":")
    m = model_class()
    pipeline = classifier.train_pipeline(classifier = model_class())    
    classifier.show_scores(pipeline)
    elapsed_time = time.time() - t
    print("Elapsed time: %.1f s" % elapsed_time)
    print()
pipeline = classifier.train_pipeline(classifier = xgb.XGBClassifier())    
classifier.show_scores(pipeline)
classifier.write_model_scores_df(pipeline, results_csv_path = os.environ['HOME'] + 'models/feature_based_model_predictions.csv')
def run_experiments_on_feature_sets(fb_classifier,
                                    classifier = xgb.XGBClassifier(),
                                    feature_sets = [['ngram_feats'], 
                                                    ['tfidf_feats'], 
                                                    ['length_feats'], 
                                                    ['argumentation_feats'], 
                                                    ['text_quality_feats'], 
                                                    ['named_entity_feats'], 
                                                    ['perspective_content_value_feats'], 
                                                    ['perspective_aggressiveness_feats'], 
                                                    ['perspecitive_toxicity_feats']]):
    
    all_feats = []
    for feature_set in feature_sets: 
        print('\n\nFeature set:', feature_set)
        all_feats.extend(feature_set)
        pipeline = fb_classifier.train_pipeline(classifier = classifier, 
                                             feature_set = feature_set)    
        fb_classifier.show_scores(pipeline)

    print('\n\nResults with all features: ', all_feats)    
    pipeline = fb_classifier.train_pipeline(classifier = classifier, 
                                         feature_set = all_feats)    
    fb_classifier.show_scores(pipeline)<jupyter_output><empty_output><jupyter_text>Train on C3 train and test on NYT<jupyter_code>C3_train_df = pd.read_csv(os.environ['C3_TRAIN'])
C3_train_df['pp_comment_text'] = C3_train_df['pp_comment_text'].astype(str)

#NYT_df = pd.read_csv(os.environ['NYT_YNACC'])
NYT_df = pd.read_csv(os.environ['HOME'] + 'data/processed/NYT_YNACC_feats_preprocessed.csv')
NYT_df['pp_comment_text'] = NYT_df['pp_comment_text'].astype(str)

C3_test_df = pd.read_csv(os.environ['C3_TEST'])
C3_test_df['pp_comment_text'] = C3_test_df['pp_comment_text'].astype(str)
y_C3_train = C3_train_df.constructive_binary     
X_C3_train = C3_train_df.drop(['constructive_binary'], axis = 1)

y_NYT = NYT_df.constructive_binary
X_NYT = NYT_df.drop(['constructive_binary'], axis = 1)

y_C3_test = C3_test_df.constructive_binary     
X_C3_test = C3_test_df.drop(['constructive_binary'], axis = 1)
fb_classifier = FBConstructivenessClassifier(X_C3_train, y_C3_train, X_NYT, y_NYT)
run_experiments_on_feature_sets(fb_classifier)
run_experiments_on_feature_sets(fb_classifier, classifier=SGDClassifier())
run_dl_experiment(C3_train_df, NYT_df , model = 'cnn')
run_dl_experiment(C3_train_df, NYT_df , model = 'lstm')<jupyter_output>len of encoded docs:  9600
Pad sequences (samples x time)
Padded data shape: (9600, 100)
Number of words not found in glove embeddings:  655
Percentage non-zero elements:  0.9757530955461098
Building model...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_2 (Embedding)      (None, 100, 300)          8116500   
_________________________________________________________________
bidirectional_1 (Bidirection (None, 256)               439296    
_________________________________________________________________
dropout_2 (Dropout)          (None, 256)               0         
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 257       
=================================================================
Total params: 8,556,053
Trainable params: 439,553
Non-trainable p[...]<jupyter_text>Train on C3 train and test on C3 test<jupyter_code>fb_classifier = FBConstructivenessClassifier(X_C3_train, y_C3_train, X_C3_test, y_C3_test)
run_experiments_on_feature_sets(fb_classifier, classifier=SGDClassifier())
run_experiments_on_feature_sets(fb_classifier)<jupyter_output>Feature set: ['ngram_feats']
Classifier:  XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,
       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,
       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,
       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,
       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,
       silent=True, subsample=1)
Feature set:  ['ngram_feats']
COMMENTS COL:  pp_comment_text
Model trained and pickled in file:  /home/vkolhatk/dev/constructiveness/models/saved_model.h5<jupyter_text>Train on NYT and test on C3 test<jupyter_code>fb_classifier = FBConstructivenessClassifier(X_NYT, y_NYT, X_C3_test, y_C3_test)

run_experiments_on_feature_sets(fb_classifier, classifier=SGDClassifier())
run_experiments_on_feature_sets(fb_classifier)
run_dl_experiment(NYT_df, C3_test_df, model = 'cnn')
run_dl_experiment(NYT_df, C3_test_df, model = 'lstm')<jupyter_output>len of encoded docs:  30325
Pad sequences (samples x time)
Padded data shape: (30325, 100)
Number of words not found in glove embeddings:  1620
Percentage non-zero elements:  0.9656240059378645
Building model...
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_4 (Embedding)      (None, 100, 300)          14146500  
_________________________________________________________________
bidirectional_2 (Bidirection (None, 256)               439296    
_________________________________________________________________
dropout_4 (Dropout)          (None, 256)               0         
_________________________________________________________________
dense_6 (Dense)              (None, 1)                 257       
=================================================================
Total params: 14,586,053
Trainable params: 439,553
Non-trainab[...]<jupyter_text>Train on C3 train and test on NYT<jupyter_code>print clf.coef
def f_importances(coef, names):
    imp = coef
    imp,names = zip(*sorted(zip(imp,names)))
    plt.barh(range(len(names)), imp, align='center')
    plt.yticks(range(len(names)), names)
    plt.show()

features_names = ['input1', 'input2']
svm = svm.SVC(kernel='linear')
svm.fit(X, Y)
f_importances(svm.coef_, features_names)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12556751848488779, 'fraction_numerical': 0.03165131664288494, 'mean_word_length': 3.614003590664273, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '195008', 'license': 'MIT', 'n_tokens_mistral': 2543, 'n_tokens_neox': 2365, 'n_words': 399, 'path': 'notebooks/2.0-kvarada-domain-adaptation_experiments.ipynb', 'repo_name': 'kvarada/constructiveness', 'size': '7709'}