<jupyter_start><jupyter_text>Amazon sentiment analysis: Spectral Feature Alignment SimplifiedData downloaded from: processed_acl.tar.gz, processed for <NAME>, <NAME>, <NAME>. Biographies, Bollywood, Boom-boxes and Blenders: Domain Adaptation for Sentiment Classification. Association of Computational Linguistics (ACL), 2007<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import Binarizer
from sklearn.metrics import mutual_info_score
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from read_funcs import organise_data, vectorise_data, select_high_freq_data
%matplotlib inline
src = 'dvd'
tgt = 'kitchen'
XB, Y_src, XD, Y_tgt = organise_data(src, tgt)
# Vectorise the raw data
X_src, X_tgt, features = vectorise_data(XB, XD)
# Reduce the no. of features
N = 10000
X_src, X_tgt, features = select_high_freq_data(X_src, X_tgt, features, N)
# Base case examined in Sent_InstanceWeighting<jupyter_output><empty_output><jupyter_text>SFA find domain independent features using mutual information<jupyter_code>def compute_mutual_info(X, Y):
    N = X.shape[1]
    mutual_info = []
    for i in range(N):
        mutual_info.append(mutual_info_score(X[:,i], Y))
    return mutual_info
X = np.r_[X_src, X_tgt]
Y = np.r_[np.ones(X_src.shape[0]), np.zeros(X_tgt.shape[0])]
print(X.shape, Y.shape)
# Features descriminant for which domain it comes from.
mutual_info_src_tgt = compute_mutual_info(X, Y)
# Features descriminant for the source domain label
mutual_info_src = compute_mutual_info(X_src, Y_src)<jupyter_output><empty_output><jupyter_text>To select features that are non-descriminant for which domain and are descriminant for the source domain. We subtract the two mutual information socre.<jupyter_code>mutual_info_src_tgt = [mutual_info_src_tgt[i] - mutual_info_src[i] \
                       for i in range(len(mutual_info_src))]
features = np.asarray(features)
sort_idx = np.argsort(mutual_info_src_tgt)
print(features[sort_idx[:20]])
src_waste_idx = (X_src[:, sort_idx[0]] > 0)*(X_src[:, sort_idx[1]] == 0)
X_src_waste = X_src[src_waste_idx, :]
Y_src_waste = Y_src[src_waste_idx]
tgt_waste_idx = (X_tgt[:, sort_idx[0]] > 0)*(X_tgt[:, sort_idx[1]] == 0)
X_tgt_waste = X_tgt[tgt_waste_idx, :]
Y_tgt_waste = Y_tgt[tgt_waste_idx]
src_great_idx = (X_src[:, sort_idx[1]] > 0)*(X_src[:, sort_idx[0]] == 0)
X_src_great = X_src[src_great_idx, :]
Y_src_great = Y_src[src_great_idx]
tgt_great_idx = (X_tgt[:, sort_idx[1]] > 0)*(X_tgt[:, sort_idx[0]] == 0)
X_tgt_great = X_tgt[tgt_great_idx, :]
Y_tgt_great = Y_tgt[tgt_great_idx]
Y_src = np.asarray(Y_src)
Y_src_sub = Y_src[X_src[:, sort_idx[0]] > 0]
Y_src_sub = np.r_[Y_src_sub,Y_src[(X_src[:, sort_idx[1]] > 0)*(X_src[:, sort_idx[0]] == 0)]]
Y_tgt = np.asarray(Y_tgt)
Y_tgt_sub = Y_tgt[X_tgt[:, sort_idx[0]] > 0]
Y_tgt_sub = np.r_[Y_tgt_sub,Y_tgt[(X_tgt[:, sort_idx[1]] > 0)*(X_tgt[:, sort_idx[0]] == 0)]]
X = np.r_[X_src_waste, X_tgt_waste, X_src_great, X_tgt_great]
X = X[:, sort_idx]
features = features[sort_idx]
l = 2
m = 100
X = X[:,:m]
features = features[:m]
Phi_DI = X[:, :l]
Phi_DS = X[:, l:]
print(Phi_DI.shape, Phi_DS.shape)
# Binarise the data
binarizer = Binarizer().fit(Phi_DI)
Phi_DI = binarizer.transform(Phi_DI)
Phi_DS = binarizer.transform(Phi_DS)
plt.plot(Phi_DI[:,0])
plt.plot(Phi_DI[:,1])
print(X_src_waste.shape[0],X_tgt_waste.shape[0],X_src_great.shape[0],X_tgt_great.shape[0])
M = np.dot(Phi_DS.T, Phi_DI)
non_zero_entries = np.sum(M, axis = 1) > 0
print('nonzero entries', np.sum(np.sum(M, axis = 1) == 0))
Phi_DS = Phi_DS[:,non_zero_entries]
X = X[:, np.r_[np.array([True, True]), non_zero_entries]]
features = features[np.r_[np.array([True, True]), non_zero_entries]]<jupyter_output>nonzero entries 0<jupyter_text>SFA: find low-dimensional mapping<jupyter_code>M = np.dot(Phi_DS.T, Phi_DI)
m = X.shape[1]
A = np.c_[np.zeros((m-l, m-l)), M]
A = np.r_[A, np.c_[M.T, np.zeros((l,l))]]
print(M.shape, A.shape)
# Build Diagonal matrix D
sum_col = np.sum(A, axis = 1)
D_inv_sqrt = np.diag(1/np.sqrt(sum_col))
k = 100 # clusters
L = np.dot(D_inv_sqrt,A).dot(D_inv_sqrt)
W, V = np.linalg.eig(L) # really slow
idx = abs(W).argsort()[::-1]
W = W[idx]
V = V[:, idx]
plt.plot(abs(W))
U = V[:m-l, :k]
U = np.real(U)
U.shape
u1 = U[:,2]
u1_sorted = sorted(range(len(u1)), key=lambda i: np.abs(u1[i]))
u1_pos_subspace = [features[i+2] for i in u1_sorted[-1:-21:-1]]
u1_neg_subspace = [features[i+2] for i in u1_sorted[:20]]
print(np.asarray(u1_pos_subspace))
print(np.asarray(u1_neg_subspace))
k = 4
U = V[:m-l, :k]
gamma = 0.6
X_src_sub = np.r_[X_src_waste[:, sort_idx[:m]], X_src_great[:, sort_idx[:m]]]
X_tgt_sub = np.r_[X_tgt_waste[:, sort_idx[:m]], X_tgt_great[:, sort_idx[:m]]]
Y_src_sub = np.r_[Y_src_waste, Y_src_great]
Y_tgt_sub = np.r_[Y_tgt_waste, Y_tgt_great]
X_src_aug = np.c_[X_src_sub, gamma*np.dot(X_src_sub[:,l:], U)]
X_tgt_aug = np.c_[X_tgt_sub, gamma*np.dot(X_tgt_sub[:,l:], U)]
X_src_aug.shape
def tune_reg_param_unsupervised(C_test, X_src_SCL, Y_src, X_tgt, Y_tgt, dev_size):
    X_train, X_dev, Y_train, Y_dev = train_test_split(X_src_SCL, Y_src, test_size = dev_size, random_state = 3)
    acc_train = []
    acc_dev = []
    for C in C_test:
        model_SCL = LogisticRegression(C = C) 
        model_SCL.fit(X_train, Y_train)
        acc_train.append(model_SCL.score(X_train, Y_train))
        acc_dev.append(model_SCL.score(X_dev, Y_dev))
    C_opt = C_test[np.argmax(acc_dev)]
    model_SCL = LogisticRegression(C = C_opt) 
    model_SCL.fit(X_train, Y_train)
    print('optimal alpha', C_opt, 'max acc', max(acc_dev),'test acc', model_SCL.score(X_tgt, Y_tgt))
    plt.plot(C_test, acc_train)
    plt.plot(C_test, acc_dev)
C_test = np.linspace(0.01,1,20)
tune_reg_param_unsupervised(C_test, X_src_sub, Y_src_sub, X_tgt_sub, Y_tgt_sub, 0.2)
tune_reg_param_unsupervised(C_test, X_src_aug, Y_src_sub, X_tgt_aug, Y_tgt_sub, 0.2)
model = LogisticRegression(C = 0.3) 
model.fit(X_src_sub, Y_src_sub)
print('gamma: {}, train {:s} acc: {:.3f}, test {:s} acc: {:.3f}'\
      .format(gamma, src, model.score(X_src_sub, Y_src_sub), \
              tgt, model.score(X_tgt_sub, Y_tgt_sub)))
gamma_list = np.logspace(-1,4,6)
for gamma in gamma_list:
    X_src_aug = np.c_[X_src_sub, gamma*np.dot(X_src_sub[:,l:], U)]
    X_tgt_aug = np.c_[X_tgt_sub, gamma*np.dot(X_tgt_sub[:,l:], U)]
    model = LogisticRegression(C = 0.3) 
    model.fit(X_src_aug, Y_src_sub)
    print('gamma: {}, train {:s} acc: {:.3f}, test {:s} acc: {:.3f}'\
          .format(gamma, src, model.score(X_src_aug, Y_src_sub), \
                  tgt, model.score(X_tgt_aug, Y_tgt_sub)))<jupyter_output>gamma: 0.1, train dvd acc: 0.853, test kitchen acc: 0.824
gamma: 1.0, train dvd acc: 0.853, test kitchen acc: 0.829
gamma: 10.0, train dvd acc: 0.857, test kitchen acc: 0.832
gamma: 100.0, train dvd acc: 0.858, test kitchen acc: 0.834
gamma: 1000.0, train dvd acc: 0.858, test kitchen acc: 0.830
gamma: 10000.0, train dvd acc: 0.804, test kitchen acc: 0.809
{'fraction_non_alphanumeric': 0.15550478366414394, 'fraction_numerical': 0.025417678137940884, 'mean_word_length': 5.49721706864564, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '295403', 'license': 'MIT', 'n_tokens_mistral': 3224, 'n_tokens_neox': 3098, 'n_words': 596, 'path': 'ML_transferLearning/Sent_SFA_Simplified.ipynb', 'repo_name': 'meichen91/MachineLearning-Snippets', 'size': '7024'}