<jupyter_start><jupyter_text>Dealing with dimensions in classification Running example                    DocID        Tokens        E-devices                Training        0        apple ios mac book fruit        TRUE                Training        1        apple mac book apple store fruit        TRUE                Training        2        microsoft ibm apple oracle        TRUE                Training        3        apple banana mango fruit        FALSE                Training        4        apple fruit        FALSE                Test        5        apple mac book fruit        ?                Test        6        mac fruit color        ?<jupyter_code>from collections import Counter
import numpy as np
from IPython.core.display import display, HTML


def to_table(tokens, M, rnd=3):
    row = "".join(['<th>'+x+'</th>' for x in tokens])
    table = "<table><tr>"+row+"</tr>"
    for doc in M:
        row = "<tr>"+"".join(['<td>'+str(round(x, rnd))+'</td>' for x in doc])+"</tr>"
        table += row
    table += '</table>'
    display(HTML(table)) 

def model(tokens, docs):
    dictionary, tf = {}, {}
    for i, d in enumerate(docs):
        tf[i] = dict(Counter(d).most_common())
        for k in tf[i].keys():
            try:
                dictionary[k] += 1
            except KeyError:
                dictionary[k] = 1
    M = np.zeros((len(docs), len(tokens)))
    for i in range(0, len(docs)):
        for j, k in enumerate(tokens):
            try:
                M[i,j] = float(tf[i][k]) / (np.log((float(len(docs)) / dictionary[k])) + 1)
            except KeyError:
                pass
        M[i] /= np.linalg.norm(M[i])
    return M

training = [
    ['apple', 'ios', 'mac', 'book', 'fruit'],
    ['apple', 'mac', 'book', 'apple', 'store', 'fruit'],
    ['microsoft', 'ibm', 'apple', 'oracle'],
    ['apple', 'banana', 'mango', 'fruit'],
    ['apple', 'fruit']
]
testing = [
    ['apple', 'mac', 'book', 'fruit'],
    ['mac', 'fruit', 'color']
]
tokens = set()
for d in training + testing:
    for t in d:
        tokens.add(t)
tokens = sorted(list(tokens))

T = model(tokens, training)
X = model(tokens, testing)
to_table(tokens, T)
to_table(tokens, X)<jupyter_output><empty_output><jupyter_text>Rocchio<jupyter_code>from scipy.spatial import distance

def rocchio(c, nc, doc, df=None):
    c_m, nc_m = np.mean(c, axis=0), np.mean(nc, axis=0)
    if df is None:
        df = distance.euclidean
    s, b = df(doc, c_m), df(doc, nc_m)
    if  s < b:
        return True, s-b
    else:
        return False, s-b
for i in range(0, len(testing)):
    R = rocchio(T[:3,:], T[3:,:], X[i], df=distance.euclidean)
    print " ".join(testing[i]), R
display(HTML('<h2>class</h2>'))
to_table(tokens, T[:3,:])
display(HTML('<h3>centroid</h3>'))
to_table(tokens, np.array([np.mean(T[:3,:], axis=0)]))
display(HTML('<h2>no class</h2>'))
to_table(tokens, T[3:,:])
display(HTML('<h3>centroid</h3>'))
to_table(tokens, np.array([np.mean(T[3:,:], axis=0)]))<jupyter_output><empty_output><jupyter_text>Understand terminology<jupyter_code>import matplotlib.pyplot as plt


def visualize(ax, D, labels, d1=0, d2=1, marker='o', d1l='d1', d2l='d2'):
    for d in range(0, D.shape[0]):
        ax.scatter(D[d, d1], D[d, d2], label=labels[d], marker=marker)
        ax.set_xlabel(d1l)
        ax.set_ylabel(d2l)

cmean, ncmean = np.mean(T[:3,:], axis=0), np.mean(T[3:,:], axis=0)
means = np.vstack((np.array([cmean]), np.array([ncmean])))
cols = 6
c_tokens = []
for ki, kk in enumerate(tokens):
    for kj in tokens[ki+1:]:
        c_tokens.append((kk, kj))
k = len(c_tokens)
fig, axes = plt.subplots(nrows=int(np.ceil(float(k)/cols)), ncols=cols, figsize=(14, 20))
for cl in range(0, k):
    visualize(axes[cl/cols, cl%cols], T, [str(x) for x in range(0, T.shape[0])], 
              d1=tokens.index(c_tokens[cl][0]), d2=tokens.index(c_tokens[cl][1]), 
              d1l=c_tokens[cl][0], d2l=c_tokens[cl][1])
    visualize(axes[cl/cols, cl%cols], means, ['c', 'nc'], 
              d1=tokens.index(c_tokens[cl][0]), d2=tokens.index(c_tokens[cl][1]), marker='x',
              d1l=c_tokens[cl][0], d2l=c_tokens[cl][1])
    visualize(axes[cl/cols, cl%cols], X, [str(x) for x in range(0, X.shape[0])], 
              d1=tokens.index(c_tokens[cl][0]), d2=tokens.index(c_tokens[cl][1]), marker='^',
              d1l=c_tokens[cl][0], d2l=c_tokens[cl][1])
plt.tight_layout()
axes[cl/cols, cl%cols].legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Measuring utility<jupyter_code>filtered, th = [], 0.01
for i, t in enumerate(tokens):
    v = np.var(T.T[i])
    print t, v
    if v >= th:
        filtered.append(t)
print filtered
Tf = model(filtered, training)
Xf = model(filtered, testing)
for i in range(0, len(testing)):
    R = rocchio(Tf[:3,:], Tf[3:,:], Xf[i], df=distance.euclidean)
    print " ".join(testing[i]), R<jupyter_output>apple mac book fruit (True, -0.34468579233846175)
mac fruit color (True, -0.14804488328266008)<jupyter_text>Mutual utility<jupyter_code>C = np.cov(T.T, ddof=0)
test = 'mac'
for i, t in enumerate(tokens):
    print test, t, C[tokens.index(test), i]
clusters, th = {}, 0.01
for i, t in enumerate(tokens):
    clusters[t] = []
    for k in tokens[i:]:
        cv = C[i, tokens.index(k)]
        if cv > th:
            clusters[t].append(k)
sets = []
for o in [x for x in clusters.values() if len(x) > 0]:
    sets.append(set(o))
sets = sorted(sets, key=lambda x: -len(x))
for s in sets:
    print s
refined = []
for s in sets:
    ins = True
    for i, k in enumerate(refined):
        if len(s.intersection(k)) > 0:
            refined[i] = s.union(k)
            ins = False
            break
    if ins:
        refined.append(s)
for s in refined:
    print s<jupyter_output>set(['oracle', 'microsoft', 'ibm'])
set(['mac', 'book', 'ios'])
set(['mango', 'banana'])
set(['fruit'])<jupyter_text>Use new terminology<jupyter_code>training_r = []
testing_r = []
for doc in training:
    n_doc = []
    for token in doc:
        for i, s in enumerate(refined):
            if token in s:
                n_doc.append(str(i))
                break
    training_r.append(n_doc)
for doc in testing:
    n_doc = []
    for token in doc:
        for i, s in enumerate(refined):
            if token in s:
                n_doc.append(str(i))
                break
    testing_r.append(n_doc)

tokens_r = set()
for d in training_r + testing_r:
    for t in d:
        tokens_r.add(t)
tokens_r = sorted(list(tokens_r))

Tr = model(tokens_r, training_r)
Xr = model(tokens_r, testing_r)
to_table(tokens_r, Tr)
to_table(tokens_r, Xr)
for i in range(0, len(testing_r)):
    R = rocchio(Tr[:3,:], Tr[3:,:], Xr[i], df=distance.euclidean)
    print " ".join(testing_r[i]), R
means = np.vstack((np.array([np.mean(Tr[:3,:], axis=0)]), np.array([Tr[3,:]])))
cols = 3
cr_tokens = []
for ki, kk in enumerate(tokens_r):
    for kj in tokens_r[ki+1:]:
        cr_tokens.append((kk, kj))
k = len(cr_tokens)
fig, axes = plt.subplots(nrows=int(np.ceil(float(k)/cols)), ncols=cols, figsize=(14, 8))
for cl in range(0, k):
    visualize(axes[cl/cols, cl%cols], Tr, [str(x) for x in range(0, Tr.shape[0])], 
              d1=tokens_r.index(cr_tokens[cl][0]), d2=tokens_r.index(cr_tokens[cl][1]), 
              d1l=cr_tokens[cl][0], d2l=cr_tokens[cl][1])
    visualize(axes[cl/cols, cl%cols], means, ['c', 'nc'], 
              d1=tokens_r.index(cr_tokens[cl][0]), d2=tokens_r.index(cr_tokens[cl][1]), 
              marker='x',
              d1l=cr_tokens[cl][0], d2l=cr_tokens[cl][1])
    visualize(axes[cl/cols, cl%cols], Xr, [str(x) for x in range(0, Xr.shape[0])], 
              d1=tokens_r.index(cr_tokens[cl][0]), d2=tokens_r.index(cr_tokens[cl][1]), 
              marker='^',
              d1l=cr_tokens[cl][0], d2l=cr_tokens[cl][1])
plt.tight_layout()
axes[cl/cols, cl%cols].legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Better dimensionality reduction<jupyter_code>w, v = np.linalg.eig(C)<jupyter_output><empty_output><jupyter_text>v : (..., M, M) arrayThe normalized (unit “length”) eigenvectors, such that the column v[:,i] is the eigenvector corresponding to the eigenvalue w[i].<jupyter_code>print w[4]
print v[:,4]<jupyter_output>6.30543614712e-18
[ 0.00332364 -0.00300661 -0.11327894  0.         -0.45831447 -0.30305569
  0.55608868 -0.42982335 -0.07884044 -0.30305569 -0.30305569 -0.03699773]<jupyter_text>Sort the eigenvalues<jupyter_code>sw = sorted([(x, i) for i, x in enumerate(w)], key=lambda k: -k[0])
dimensions = 4
Ct = np.zeros((C.shape[0], dimensions))
features = [v[:, x[1]] for x in sw[:dimensions]]  
Fm = np.array(features)
Tt = np.dot(Fm, T.T).T
Xt = np.dot(Fm, X.T).T
cmean, ncmean = np.mean(Tt[:3,:], axis=0), np.mean(Tt[3:,:], axis=0)
means = np.vstack((np.array([cmean]), np.array([ncmean])))
print Tt.shape
print Tt
for i in range(0, len(testing)):
    R = rocchio(Tt[:3,:], Tt[3:,:], Xt[i], df=distance.euclidean)
    print " ".join(testing[i]), R
for test in Xt:
    print distance.euclidean(test, cmean), distance.euclidean(test, ncmean)
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(8, 8))
visualize(axes, Tt, [str(x) for x in range(0, Tt.shape[0])])
visualize(axes, Xt, [str(x) for x in range(0, Xt.shape[0])], marker='^')
visualize(axes, means, [str(x) for x in range(0, means.shape[0])], marker='x')
plt.tight_layout()
axes.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18283701569004163, 'fraction_numerical': 0.0329811079090618, 'mean_word_length': 3.1204925241864556, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '236992', 'license': 'MIT', 'n_tokens_mistral': 3818, 'n_tokens_neox': 3608, 'n_words': 807, 'path': 'transformation/dimensionality.ipynb', 'repo_name': 'afflint/pygen', 'size': '9369'}