<jupyter_start><jupyter_text>--- Синонимы<jupyter_code>%%time
tags_synonyms = []
with open('../data/syn.txt') as f:
    lines = f.readlines()
    lines = map(lambda l: l.decode('utf-8'), lines)
    lines.sort()
    for tag in tags_norm:
        tag_syns = []
        for word in tag.split(' '):
            idx = bisect_left(lines, word+'|')
            line = lines[idx]
            if line.startswith(word+'|'):
                tag_syns += get_normalized_tokens(line.split('|')[1])
        tags_synonyms.append(tag_syns)
cPickle.dump(tags_synonyms, open('../data/tags_synonyms.p', 'wb'))
tags_synonyms = cPickle.load(open('../data/tags_synonyms.p', 'rb'))<jupyter_output><empty_output><jupyter_text>--- Неудачные веса<jupyter_code>%%time
for tag_idx in xrange(len(tags_norm)):
    tag = tags_norm[tag_idx]
    for word_idx in xrange(len(tag)):
        w1 = np.array(map(lambda text: w in text, X_train[y_train == t])).sum() / float((y_train == t).sum())
        w2 = np.array(map(lambda text: w in text, X_train[y_train != t])).sum() / float((y_train != t).sum())
        tag[word_idx] = (tag[word_idx], w1/w2)
def predict_n_tags(text, n):
    predicted_n = [(-1, -1)] * n
    heapq.heapify(predicted_n)
    for tag in tags_norm:
        trust_level = 0.
        for word, weight in tag:
            if word in text:
                trust_level += weight
        heapq.heappushpop(predicted_n, (trust_level, ' '.join(zip(*tag)[0])))
    return predicted_n
idx = 0
print y_train[idx]
print X_train[idx]
predicted_3 = predict_n_tags(X_train[idx], 3)
for pair in predicted_3:
    print u'{0} - {1}'.format(pair[1], pair[0])<jupyter_output><empty_output><jupyter_text>---<jupyter_code># Counts the number of words from <words>, that appear in <text>
def count_entries(words, text):
    count = 0
    for word in words:
        if word in text:
            count += 0.5
    return count

PAIR_WEIGHT = 2
def count_entries2(words, text):
    count = 0
    words = words.split(' ')
    for i in range(len(words)-1):
        if words[i] in text:
            count += 1
            pair = ' '.join((words[i], words[i+1]))
            if pair in text:
                count += PAIR_WEIGHT
    if words[-1] in text:
        count += 1
    return count
def count_part_entries2(words, text, part_size, weight, pair_weight):
    count = 0
    words = words.split(' ')
    words = map(lambda w: w[:part_size], words)
    text = ' '.join(map(lambda w: w[:part_size], text.split(' ')))
    for i in range(len(words)-1):
        if words[i] in text:
            count += weight
            pair = ' '.join((words[i], words[i+1]))
            if pair in text:
                count += pair_weight * weight
    if words[-1] in text:
        count += 1
    return count
# Returns <num_tags> the most suitable tags for <text>
def get_top_tags(text, tags, num_tags, part_size, two_counts, part_weight=1, pair_weight=0):
    similarities = []
    for tag in tags:
        if two_counts:
            similarities.append(count_entries2(tag, text) / float(len(tag.split(' '))))
            similarities[-1] += count_part_entries2(tag, text, part_size, part_weight, pair_weight) / float(len(tag.split(' ')))
        else:
            similarities.append(count_part_entries2(tag, text, part_size, part_weight, pair_weight) / float(len(tag.split(' '))))
#         similarities[-1] += count_part_entries2(tag, text, 5) / float(len(tag.split(PAIR_WEIGHT ' ')))
#     for tag, syns in zip(tags, tags_synonyms):
#         similarities.append(count_entries2(tag, text) / float(len(tag)))
#         similarities[-1] += count_entries(syns, text) / float(len(tag))
    result = []
    for _ in range(num_tags):
        idx = np.argmax(similarities)
        result.append((tags[idx], similarities[idx]))
        similarities[idx] = 0.
    return result
%%time

_, X_small, _, y_small = train_test_split(X, y, test_size=0.2)
    
num_tested = len(X_small)
for part_size in (4,5,6):
    for two_counts in (False,):
        if two_counts:
            for part_weight in (0.5, 1., 1.5):
                num_guessed = 0
                for i in xrange(num_tested):
                    top3 = get_top_tags(X_small[i], tags_norm, 3, part_size, two_counts, part_weight)
                    if y_small[i] in zip(*top3)[0]:
                        num_guessed += 1
                acc = num_guessed / float(num_tested)
                print part_size, two_counts, part_weight
                print acc
        else:
            for pair_weight in (0,1,2):
                num_guessed = 0
                for i in xrange(num_tested):
                    top3 = get_top_tags(X_small[i], tags_norm, 3, part_size, two_counts)
                    if y_small[i] in zip(*top3)[0]:
                        num_guessed += 1
                acc = num_guessed / float(num_tested)
                print part_size, two_counts, pair_weight
                print acc
%%time
                
# X_small = X
# y_small = y
accuracies = []
for j in range(3):
    _, X_small, _, y_small = train_test_split(X, y, test_size=0.01)
    num_guessed = 0
    num_tested = len(X_small)
    for i in xrange(num_tested):
        top3 = get_top_tags(X_small[i], tags_norm, 3, 6, False, 1, 1)
        if y_small[i] in zip(*top3)[0]:
            num_guessed += 1
    acc = num_guessed / float(num_tested)
    print acc
    accuracies.append(acc)

print 'Avg. accuracy: {0}'.format(np.mean(accuracies))<jupyter_output>0.481467329003
0.489491784486
0.510126098586
Avg. accuracy: 0.493695070692
CPU times: user 4min 27s, sys: 456 ms, total: 4min 27s
Wall time: 4min 26s<jupyter_text>---<jupyter_code>tags_sorted = data_tagged.tag.value_counts().index.values
tag_encoder = dict(zip(tags_sorted, range(len(tags_sorted))))
y_tag_encoded = np.array(map(lambda c: tag_encoder[c], y_tag))
y_tag_encoded[:5]<jupyter_output><empty_output><jupyter_text>--- Генерим фичи из тегов<jupyter_code>key_words = []
for tag in tags_norm:
    key_words += tag
key_words = set(key_words)
%%time

X_small = X
y_small = y
# _, X_small, _, y_small = train_test_split(X, y, test_size=0.1)
pipeline = Pipeline([
    ('vect', CountVectorizer(binary=True, lowercase=False, vocabulary=key_words)),
    ('clf', SGDClassifier())
#     ('clf', LogisticRegression(solver='lbfgs', multi_class='multinomial'))
])
cross_val_score(pipeline, X_small, y_small, verbose=4, scoring='accuracy').mean()
# %%time

from sklearn.neighbors import LSHForest
from sklearn.metrics import accuracy_score


key_words = []
for tag in tags_norm:
    key_words += tag
key_words = set(key_words)    

# _, X_small, _, y_small = train_test_split(X, y, test_size=0.1)
# pipeline = Pipeline([
#     ('vect', CountVectorizer(binary=True, lowercase=False, vocabulary=key_words)),
#     ('clf', LSHForest())
# #     ('clf', LogisticRegression(solver='lbfgs', multi_class='multinomial'))
# ])
# cross_val_score(pipeline, X_small, y_small, verbose=4, scoring='accuracy').mean()


accuracies = []
for j in range(3):
    _, X_small, _, y_small = train_test_split(X, y, test_size=0.5)
    X_small = CountVectorizer(binary=True, lowercase=False, vocabulary=key_words).fit_transform(X_small)
    lshf = LSHForest().fit(X_small, y_small)
    num_guessed = 0
    num_tested = X_small.shape[0]
    for i in xrange(num_tested):
        if i == lshf.kneighbors(X_small[i], 1)[1][0]:
            num_guessed += 1
    acc = num_guessed / float(num_tested)
    print acc
    accuracies.append(acc)

print 'Avg. accuracy: {0}'.format(np.mean(accuracies))
import re

class Porter:
	PERFECTIVEGROUND =  re.compile(u"((ив|ивши|ившись|ыв|ывши|ывшись)|((?<=[ая])(в|вши|вшись)))$")
	REFLEXIVE = re.compile(u"(с[яь])$")
	ADJECTIVE = re.compile(u"(ее|ие|ые|ое|ими|ыми|ей|ий|ый|ой|ем|им|ым|ом|его|ого|ему|ому|их|ых|ую|юю|ая|яя|ою|ею)$")
	PARTICIPLE = re.compile(u"((ивш|ывш|ующ)|((?<=[ая])(ем|нн|вш|ющ|щ)))$")
	VERB = re.compile(u"((ила|ыла|ена|ейте|уйте|ите|или|ыли|ей|уй|ил|ыл|им|ым|ен|ило|ыло|ено|ят|ует|уют|ит|ыт|ены|ить|ыть|ишь|ую|ю)|((?<=[ая])(ла|на|ете|йте|ли|й|л|ем|н|ло|но|ет|ют|ны|ть|ешь|нно)))$")
	NOUN = re.compile(u"(а|ев|ов|ие|ье|е|иями|ями|ами|еи|ии|и|ией|ей|ой|ий|й|иям|ям|ием|ем|ам|ом|о|у|ах|иях|ях|ы|ь|ию|ью|ю|ия|ья|я)$")
	RVRE = re.compile(u"^(.*?[аеиоуыэюя])(.*)$")
	DERIVATIONAL = re.compile(u".*[^аеиоуыэюя]+[аеиоуыэюя].*ость?$")
	DER = re.compile(u"ость?$")
	SUPERLATIVE = re.compile(u"(ейше|ейш)$")
	I = re.compile(u"и$")
	P = re.compile(u"ь$")
	NN = re.compile(u"нн$")

	def stem(word):
		word = word.lower()
		word = word.replace(u'ё', u'е')
		m = re.match(Porter.RVRE, word)
		if m.groups():
			pre = m.group(1)
			rv = m.group(2)
			temp = Porter.PERFECTIVEGROUND.sub('', rv, 1)
			if temp == rv:
				rv = Porter.REFLEXIVE.sub('', rv, 1)
				temp = Porter.ADJECTIVE.sub('', rv, 1)
				if temp != rv:
					rv = temp
					rv = Porter.PARTICIPLE.sub('', rv, 1)
				else:
					temp = Porter.VERB.sub('', rv, 1)
					if temp == rv:
						rv = Porter.NOUN.sub('', rv, 1)
					else:
						rv = temp
			else:
				rv = temp
			
			rv = Porter.I.sub('', rv, 1)

			if re.match(Porter.DERIVATIONAL, rv):
				rv = Porter.DER.sub('', rv, 1)

			temp = Porter.P.sub('', rv, 1)
			if temp == rv:
				rv = Porter.SUPERLATIVE.sub('', rv, 1)
				rv = Porter.NN.sub(u'н', rv, 1)
			else:
				rv = temp
			word = pre+rv
		return word
	stem=staticmethod(stem)
	
print Porter.stem(u'покупки')<jupyter_output>кредитов
{'fraction_non_alphanumeric': 0.16060735671514115, 'fraction_numerical': 0.01924721984602224, 'mean_word_length': 3.1293598233995583, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '612305', 'license': 'MIT', 'n_tokens_mistral': 3909, 'n_tokens_neox': 3638, 'n_words': 707, 'path': 'tags_classifier/modeling.ipynb', 'repo_name': 'ilivans/law-themes-classifier', 'size': '9352'}