<jupyter_start><jupyter_text>Load sentences<jupyter_code># Load sentences
sentences = []
indi_sent_rev = []
with open('text_0_AutomotiveProd7.txt') as f:
    i = 1
    rev_num = -1
    for line in f:
        if i == 1:
            rev_num += 1
            l = line.strip()
            indi_sent = l.split('.')
            indi_sent = [a.strip() for a in indi_sent]
            indi_sent = list(filter(None, indi_sent))
            for k in range(len(indi_sent)):
                indi_sent_rev.append(rev_num)
            sentences.extend(indi_sent)
            i *= -1
        else:
            i *= -1
print(len(sentences))<jupyter_output>382<jupyter_text>Encode sentences<jupyter_code>embeddings = model.encode(sentences, bsize=128, tokenize=False, verbose=True)
print('nb sentences encoded : {0}'.format(len(embeddings)))
print(embeddings.shape)<jupyter_output>(382, 4096)<jupyter_text>Load ratings<jupyter_code>ratings = []
f = open('rating_0_AutomotiveProd7.txt', 'r')
for l in f:
    ratings.append(float(l))
print(ratings[:10])<jupyter_output>[5.0, 4.0, 4.0, 5.0, 4.0, 5.0, 5.0, 1.0, 5.0, 5.0]<jupyter_text>Load review ids<jupyter_code>ids = []
f = open('id_0_AutomotiveProd7.txt', 'r')
for l in f:
    ids.append(l)<jupyter_output><empty_output><jupyter_text>Append ratings to embeddings<jupyter_code>print(indi_sent_rev[:10])
embeddings = list(embeddings)
for i in range(len(sentences)):
    embeddings[i] = np.append(embeddings[i], np.array([ratings[indi_sent_rev[i]]]))
embeddings = np.array(embeddings)
print(embeddings.shape)<jupyter_output>(382, 4097)<jupyter_text>Spectral Clustering<jupyter_code>from sklearn.cluster import SpectralClustering
from sklearn import metrics

# Spectral Clustering
sc = SpectralClustering(40, affinity='nearest_neighbors', n_init=1000)
sc.fit(embeddings)
print('spectral clustering')
print(sc.labels_)
labels = sc.labels_.flatten()
print(labels.shape)
labels = list(labels)
for i in set(sc.labels_):
    print(str(i)+': '+str(labels.count(i)))
for i in range(40):
    print('Cluster '+str(i)+':')
    cnt = 1
    for k, l in enumerate(sentences):
        if sc.labels_[k] == i:
            print(str(cnt)+': '+l)
            cnt += 1
    print()<jupyter_output>Cluster 0:
1: This was an early Fathers Day gift for my husband, and he couldn't be happier! Easy to use, love the ratchet instead of a twist and lock,held the load in the truck without shifting when we went camping (literally 40 miles of bad road), good pads on ends so truck bed is protected
2: I wish I had bought one of these for my truck years ago!  No more cargo rolling around in the truck's bed!
3: It's worth the money paid and I love it because it hold the item loaded in the rear in my truck bed
4: I bought two of these and use them to support my slide-out on my Lance truck camper when extended
5: I bought this for a trip this past summer to keep things from moving in the back of the pickup and worked great
6: If you are like me, I just through items in the back of my truck and hope they are still there wherever I'm going
7: But with this Ratchet Cargo Bar, I can still through things, but I can now through them in the right place
8: Even though I havent used this on my truckbed i[...]<jupyter_text>Test for Original embeddings<jupyter_code>embeddings_orig = model.encode(sentences, bsize=128, tokenize=False, verbose=True)
print('nb sentences encoded : {0}'.format(len(embeddings_orig)))
print(embeddings_orig.shape)
from sklearn.cluster import SpectralClustering
from sklearn import metrics

# Spectral Clustering
sc = SpectralClustering(40, affinity='nearest_neighbors', n_init=1000)
sc.fit(embeddings_orig)
print('spectral clustering')
print(sc.labels_)
labels = sc.labels_.flatten()
print(labels.shape)
labels = list(labels)
for i in set(sc.labels_):
    print(str(i)+': '+str(labels.count(i)))
for i in range(40):
    print('Cluster '+str(i)+':')
    cnt = 1
    for k, l in enumerate(sentences):
        if sc.labels_[k] == i:
            print(str(cnt)+': '+l)
            cnt += 1
    print()<jupyter_output>Cluster 0:
1: I was looking for a sturdy, quality made cargo restraint bar and this is not it
2: I thought this cargo bar would be easier to place in a lot let time
3: I would recommend trying a different cargo bar
4: I do not recommend this cargo bar
5: If you want a quality, heavy duty cargo bar at a good price, this is it
6: Someone recommended a cargo bar and I bought this one
7: I'm fairly pleased with this cargo bar
8: I would recommend you keep looking for a sturdier cargo bar

Cluster 1:
1: You have to make sure you don't ratchet this thing down too hard, if you do, you will regret that last click
2: Just a wee bit tricky to learn to operate at first, blame instructions on one word, instead of saying "rotate" should say "pull over" for the lever, engage last black notch (so easy to miss), then adjust length to short or long quickly
3: When it is tightened it just pops off the gear and will not create enough tension to hold in place
4: Used to have one of the ones that used thre[...]<jupyter_text>Try splitting clusters<jupyter_code>embeddings_split = model.encode(sentences, bsize=128, tokenize=False, verbose=True)
print('nb sentences encoded : {0}'.format(len(embeddings_split)))
print(embeddings_split.shape)
from sklearn.cluster import SpectralClustering
from sklearn import metrics

# Spectral Clustering
sc = SpectralClustering(30, affinity='nearest_neighbors', n_init=1000)
sc.fit(embeddings_split)
print('spectral clustering')
print(sc.labels_)
labels = sc.labels_.flatten()
print(labels.shape)
labels = list(labels)
for i in set(sc.labels_):
    print(str(i)+': '+str(labels.count(i)))
cluster_new = []
for i in range(30):
    temp_sentences = []
    pos_sent = []
    neg_sent = []
    #print('Cluster '+str(i)+':')
    for k, l in enumerate(sentences):
        if sc.labels_[k] == i:
            temp_sentences.append(k)
    for sent in temp_sentences:
        if ratings[indi_sent_rev[sent]] >= 3.0:
            pos_sent.append(sent)
        else:
            neg_sent.append(sent)
    cluster_new.append(pos_sent)
    cluster_new.append(neg_sent)
print(len(cluster_new))
print(cluster_new[:30])
for i in range(30):
    print('Cluster '+str(i)+'_pos:')
    cnt = 1
    for j in cluster_new[2*i]:
        print(str(cnt)+'. '+sentences[j])
        cnt += 1
    print()
    cnt = 1
    print('Cluster '+str(i)+'_neg:')
    for j in cluster_new[2*i+1]:
        print(str(cnt)+'. '+sentences[j])
        cnt += 1
    print()
print(cluster_new)
print(indi_sent_rev)
print(len(sentences), len(indi_sent_rev))
matrix = []
for i in range(rev_num+1):
    matrix_row = []
    for clusters in cluster_new:
        flag = 0
        for sent_ind in clusters:
            if indi_sent_rev[sent_ind] == i:
                matrix_row.append('1')
                flag = 1
                break
        if flag == 0:
            matrix_row.append('0')
    matrix.append(matrix_row)
print(rev_num)
print(len(matrix))
for i in matrix:
    print(len(i))
print(matrix[5])
f = open('AutomotiveProd7_matrix.csv', 'w')
for i in range(len(matrix)):
    f.write(ids[i].strip()+',')
    f.write(','.join(matrix[i]))
    f.write('\n')
f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11589587371919136, 'fraction_numerical': 0.01924674605372473, 'mean_word_length': 3.8476510067114096, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '289904', 'license': 'Apache-2.0', 'n_tokens_mistral': 2497, 'n_tokens_neox': 2355, 'n_words': 773, 'path': 'FB_Infersent/encoder/Models/include_ratings-spectral.ipynb', 'repo_name': 'shreyansh26/RevOpiD-IJCNLP-17-', 'size': '7222'}