<jupyter_start><jupyter_text>Scraping<jupyter_code>df = pd.read_excel('keywords and logos.xlsx', sheet_name='20190215')[['shortname','kw_supporter','kw_supported','keywords']]; df
_or = lambda kw: ' OR '.join(['#' + k.strip() for k in kw.split(',')])

qry_base = {row.shortname:_or(row.keywords) for row in df.itertuples()}
supporters = df.shortname[df.kw_supporter==True]
qry_sup = ' OR '.join([qry_base[sup] for sup in supporters]); qry_sup
qry = {row.shortname: '(' + qry_base[row.shortname] + ') AND (' + qry_sup + ')' if row.kw_supported==True else qry_base[row.shortname] for row in df.itertuples()}
%%time
what = ['id', 'timestamp', 'text']
texcrapy.scrap(qry, what=what, lang='ko', end='2019-01-31', download_to='scrapped/twitter')<jupyter_output><empty_output><jupyter_text>Making Corpus<jupyter_code>def preproc(text, remove_url=True, remove_mention=True, remove_hashtag=False):
    LINEBREAK = r'\n' # str.replace에서는 r'\n'으로 검색이 안된다
    RT = '((?: rt)|(?:^rt))[^ @]?'
    EMOJI = r'[\U00010000-\U0010ffff]'
    DOTS = '…'
    LONG_BLANK = r'[ ]+'
    SPECIALS = r'([^ a-zA-Z0-9_\u3131-\u3163\uac00-\ud7a3]+)|([ㄱ-ㅣ]+)'
    
    # \u3131-\u3163\uac00-\ud7a3 는 한글을 의미함
    # URL = r'(?P<url>(https?://)?(www[.])?[^ \u3131-\u3163\uac00-\ud7a3]+[.][a-z]{2,6}\b([^ \u3131-\u3163\uac00-\ud7a3]*))'
    URL1 = r'(?:https?:\/\/)?(?:www[.])?[^ :\u3131-\u3163\uac00-\ud7a3]+[.][a-z]{2,6}\b(?:[^ \u3131-\u3163\uac00-\ud7a3]*)'
    URL2 = r'pic.twitter.com/[a-zA-Z0-9_]+'
    URL = '|'.join((URL1, URL2))
    
    HASHTAG = r'#(?P<inner_hashtag>[^ #@]+)'
    MENTION = r'@(?P<inner_mention>[^ #@]+)' 
    
    text = text.lower()
    
    if remove_url:
        text = re.sub(URL, ' ', text)

    if remove_mention:
        text = re.sub(MENTION, ' ', text)        
    else:
        text = re.sub(MENTION, ' \g<inner_mention>', text)
        
    if remove_hashtag:
        text = re.sub(HASHTAG, ' ', text)
    else:
        text = re.sub(HASHTAG, ' \g<inner_hashtag>', text)
        
    text = re.sub('|'.join((LINEBREAK, RT, EMOJI, DOTS, SPECIALS)), ' ', text)
    return re.sub(LONG_BLANK, ' ', text).strip()
    

class JsonCorpus:
    def __init__(self, *fnames, textkey='text'):
        self.fnames = fnames
        self.textkey = textkey
        self.corpus = self._corpus()

    def _corpus2(self):
        corpus = {}
        nfiles = len(self.fnames)
        _preproc = lambda doc: preproc(doc[self.textkey])
        f_preproc = np.vectorize(_preproc)
        
        for i, fname in enumerate(self.fnames):
            with open(fname, encoding='UTF-8-sig') as f: 
                for item, docs in json.load(f).items():
                    corpus[item] = f_preproc(docs) if len(docs)>0 else []
                    
                pct = '%.2f' % (100 * (i+1) / nfiles)
                print('\r {pct}% completed'.format(pct=pct), end='')
                
        print('\n')
        return corpus
    
    
    def _corpus(self):
        corpus = {}
        nfiles = len(self.fnames)
        
        for i, fname in enumerate(self.fnames):
            with open(fname, encoding='UTF-8-sig') as f:                
                for item, docs in json.load(f).items():
                    corpus[item] = [preproc(doc[self.textkey]) for doc in docs]
                
                pct = '%.2f' % (100 * (i+1) / nfiles)
                print('\r {pct}% completed'.format(pct=pct), end='')
        
        print('\n')
        return corpus
    
                
    def __iter__(self):
        for sents in self.corpus.values():
            yield from sents

    def __len__(self):
        return sum([len(sents) for sents in self.corpus.values()])
    
    def tokenize(self, tagger):
        return Tokens(tagger, **self.corpus)


class Tokens:
    def __init__(self, tagger, **corpora):
        self.tagger = tagger
        self.tokensdict = self._get_tokens2(**corpora)
    
    
    def _get_tokens(self, **corpora):
        tokens = [] #{}
        ths = []
        n_items = len(corpora)
        
        for item, corpus in list(corpora.items())[:50]:
            th = Thread(target=do_concurrent_tagging, args=(item, corpus, n_items, tokens))
            ths.append(th)
            
        for th in ths: th.start()
        for th in ths: th.join()
        
        print('\n')
        return tokens
    
    
    def _get_tokens2(self, **corpora):
        tokens = {}
        for item, corpus in tqdm_notebook(list(corpora.items())[:]):
            tokens[item] = [[w[0] for w in self.tagger.tag(corp) if w[1] is not None] for corp in set(corpus)]
        return tokens
    
    
    def freq(self):
        return {
            item: Text(sum(toks, [])).vocab() 
            for item, toks in tqdm_notebook(self.tokensdict.items())
            if item not in ['ootd','fashion','category']
        }
                
    def __iter__(self):
        for toks in self.tokensdict.values():
            yield from toks
            
    def __len__(self):
        return sum([len(toks) for toks in self.tokensdict.values()])
%%time
fnames = ['scrapped/twitter/' + fname for fname in os.listdir('scrapped/twitter')]
jcorpus = JsonCorpus(*fnames)<jupyter_output>100.00% completed

Wall time: 26.6 s<jupyter_text>Making Dictionary  1. From Scraping keywords<jupyter_code>df_keywords = pd.read_excel('keywords and logos.xlsx', sheet_name='20190304')['keywords']
keywords = {w.strip() for w in ', '.join(df_keywords).split(',')}<jupyter_output><empty_output><jupyter_text>2. Soynlp nouns<jupyter_code>noun_extractor = LRNounExtractor_v2(verbose=True)
_soynouns = noun_extractor.train_extract(jcorpus, min_noun_score=0.3, min_noun_frequency=5)
soynouns = _soynouns.keys()
soyngrams = {v for k,v in noun_extractor._compounds_components.items() if k in soynouns}
word_extractor = WordExtractor()
word_extractor.train(jcorpus)
_soywords = word_extractor.extract()
def word_score(score):
    return score.cohesion_forward * math.exp(score.right_branching_entropy)

soywords = {word for word, score in _soywords.items() if word_score(score)>0.1}<jupyter_output><empty_output><jupyter_text>3. Korean words<jupyter_code>with open('dic_system.txt', encoding='UTF-8-sig') as f:
    lines = f.readlines()
    
kowords = {tok.split('\t')[0] for tok in lines}<jupyter_output><empty_output><jupyter_text>4. English words<jupyter_code>enwords = set(WORDS.words())<jupyter_output><empty_output><jupyter_text>5. Custom words<jupyter_code>cwords = '''
'''

cwords = set(re.findall(r'[^ ,]+', re.sub(r'\n', '', cwords)))
pos_dict = {
    'Adverb': {}, 
    'Noun': keywords | soynouns | soywords | kowords | enwords | cwords, 
    'Josa': {},
    'Verb': {},
    'Adjective': {},
    'Exclamation': {},
}

dictionary = Dict(pos_dict)
generator = LRTemplateMatcher(dictionary)    
evaluator = LREvaluator()
postprocessor = UnknowLRPostprocessor()
tagger = SimpleTagger(generator, evaluator, postprocessor)
tagger.tag(jcorpus.corpus['crocs'][250])
%%time
tokens = jcorpus.tokenize(tagger)
set(jcorpus.corpus['crocs']);
tokens.tokensdict['crocs'];<jupyter_output><empty_output><jupyter_text>Tagger setup: 현재 이부분은 안쓴다  1. Passtags<jupyter_code>stoptags = {'Foreign','Punctuation','KoreanParticle','Josa','Eomi','PreEomi','Exclamation','Determiner'}
passtags = set(twitter.tagset.keys()) - stoptags<jupyter_output><empty_output><jupyter_text>2. Stopwords<jupyter_code>stopwords_en = STOPWORDS.words('english')

with open('stopwords-ko.json', encoding='UTF-8-sig') as f:
    stopwords_ko = json.load(f)
    
stopwords_custom = '''
    엔, 줄, 내, 에는, 잖아요, 라는, 데, 는, 것은, 는, 인가요, 란, 말인가, 시, 은, 하네, 이나, 지, 있음, 수, 있는, 
    있으신가, 요, 그렇다면, 하고, 내는, 후, 생, 가거, 한, 하시겠어요, 만, 되네, 하려면, 하다, 랑, 하게, 근데, pic, 님의, 
    호, 하세요, 흑, 적인, 다욧, 니, 보니, 건, 다, 위해, 히, 께, 난, 해, 면서, 용, 보자, 가서, 하는, 중, 도, 
    잉, 인, 울, 하러가기, 인, 준, 든, 이셨으면, 와의, 
    드는, 되네요, 크흐, 이걸, 서, 에도, 됩니다, 하다고, 날, 
    하네요, 했습니다, 해지네요, 이였냐, 더, 
    되니, 하지않을, 아아, htm, 고, 냐고, 이야, 아니야, 너네, 된, 이냐, 해봐라, 해서, 
    vi, 
'''    

stopwords_custom = re.findall(r'[^ ,]+', re.sub(r'\n','',stopwords_custom))
stopwords = set(stopwords_en + stopwords_ko)# + stopwords_custom)<jupyter_output><empty_output><jupyter_text>3. N-grams<jupyter_code>cngrams = {
    ('캘빈','클라인'),('켈빈','클라인'),
}
#     ('the','boyz'),
#     ('어글리','슈즈'),
#     ('리미티드','에디션'),
#     ('키코','코스타디노브'), ('kiko','kostadinov'),
#     ('젤','카야노'),
#     ('젤','라이트'),('gel','lyte'),
#     ('스튜어트','와이츠먼'),
#     ('주니','블랙'),
#     ('jun','hyo','seong'),
#     ('gel','ptg'),
#     ('오니츠카','타이거'),
#     ('노박','조코비치'),
#     ('후디','티셔츠'),
#     ('최종병기','활'),
#     ('아이제아','토마스'),
#     ('gel','ace')
# ]

ngrams = list(cngrams | soyngrams)
tagger = Postprocessor(twitter)#, stopwords=stopwords, passtags=passtags, ngrams=ngrams)
%%time
tokens = jcorpus.tokenize(tagger)
def do_concurrent(tagger, item, corpus):
    return item, [untag(tagger.tag(corp)) for corp in corpus]
%%time
if __name__ == '__main__':
    values = [delayed(do_concurrent)(tagger, item, corpus) for item, corpus in list(jcorpus.corpus.items())[:5]]
    result = compute(*values, scheduler='processes')
%%time
test = [do_concurrent(tagger, item, corpus) for item, corpus in list(jcorpus.corpus.items())[:5]]
tokensfreq = tokens.freq()
with open('model/tokensfreq.json', 'w', encoding='UTF-8-sig') as f:
    json.dump(tokensfreq, f, ensure_ascii=False)
with open('model/tokensfreq.json', encoding='UTF-8-sig') as f:
    tokensfreq = json.load(f)
tokensfreq.keys();<jupyter_output><empty_output><jupyter_text>Word2vec<jupyter_code>w2v = Word2Vec(tokens, size=100, window=5, min_count=10, workers=4, sg=1)
w2v.save('model/word2vec.model')
w2v.init_sims(replace=True)
%%time
w2v = Word2Vec.load('model/word2vec.model')
# blist = pd.read_excel('keywords and logos.xlsx', sheet_name='20190304')[['shortname','keywords']][3:].to_dict('record')
# res = []
# for brand1 in blist:
#     for brand2 in blist:
#         b1 = [w.strip() for w in brand1['keywords'].split(',')]
#         b2 = [w.strip() for w in brand2['keywords'].split(',')]
        
#         try:
#             sim = w2v.wv.n_similarity(b1, b2)
#             res.append((brand1['shortname']+'-'+brand2['shortname'], sim))
            
#         except:
#             _b1 = [w for w in b1 if w in w2v.wv.vocab]
#             _b2 = [w for w in b2 if w in w2v.wv.vocab]
#             if len(_b1)*len(_b2)!=0:
#                 sim = w2v.wv.n_similarity(_b1, _b2)
#                 res.append((brand1['shortname']+'-'+brand2['shortname'], sim))
# sorted(res, key=lambda x:x[1])
# %%time
# dd = dict(w2v.wv.most_similar(positive=['사랑','가성비','스포티'], topn=1000))
# %%time
# out = {}
# for bname in tokensfreq.keys():
#     try:
#         out[bname] = w2v.wv.n_similarity([bname], ['편한','여성스러운'])
#     except:
#         pass

# out = sorted(out.items(), key=lambda x: x[1])
w2v.wv.most_similar(positive=['머래요'], topn=10)
id_pools = {
    '럭셔리': '럭셔리,고급,호화,과시,luxury,esteem', 
    '캐주얼': '캐주얼,캐쥬얼,casual', 
    
    '유니크': '유니크,독특,독창,unique,개성,only,참신,신선,특이,아이디어,철학',
    '대중성': '대중,popular,널리,흔한', 
    
    '정통성': '정통,traditional,트레디셔널,클래식,classic,품격,약속,신뢰,예측', 
    '트렌디': '트렌디,트랜디,트렌드,트랜드,유행,trend,trendy,변화,새로운',#,'민감'],
    
    '포멀': '포멀,노멀,normal,평범,일상,무난,기본',
    '액티브': '화제,인기,hot,튀는,신상,최신,액티브,active,앞서가는,실험', 
#     '화제성': '화제,인기,hot,튀는', 
    
    #'가성비': ['가성비','저렴','효율','성능','실용'], 
    #'신뢰성': ['신뢰','믿음','trust','견고','품질','안정','클래식','classic'],
    #'활동성': ['활동','활발','운동','액티브','스포츠','active','sport','sports','sporty'],
    #'과감함': ['과감','선도','선구','대담','강렬','선명','예술','art'],
    # 철학, 환상, 신상, 새로운, 
}
df = pd.read_excel('keywords and logos.xlsx', sheet_name='20190304')[['shortname','keywords']]
_kws = lambda kws: [kw.strip() for kw in kws.split(',') if kw.strip() in w2v.wv.vocab]
# _sim = lambda kws: {k:1 for k,v in id_pools.items()}
_sim = lambda kws: {k:w2v.wv.n_similarity(_kws(kws), v.split(',')) for k,v in id_pools.items()}

idty = {}
for row in list(df.itertuples())[3:]:
    try:
        idty[row.shortname] = _sim(row.keywords)
    except:
        pass
idty
dct = Dictionary(tokens)
# dct.filter_extremes(no_below=20, no_above=0.2)
dct.compactify()
corp = {item:[dct.doc2bow(tok) for tok in toks] for item, toks in tqdm_notebook(tokens.tokensdict.items())}
model_tfidf = TfidfModel(sum(corp.values(),[]), id2word=dct)
wordlist = list(dct.values())
identities = list(id_pools.keys())
brands = list(tokens.tokensdict.keys())
# sim_mat = np.zeros((len(identities),len(wordlist)))
# for i, idty in enumerate(tqdm_notebook(identities)):
#     for j, word in enumerate(wordlist):
#         sims = []
#         for w in id_pools[idty].split(','):
#             try:
#                 sims.append(w2v.wv.similarity(w, word))
#             except:
#                 pass

#         sim_mat[i,j] = np.mean(sims)
len(w2v.wv.vocab), len(wordlist)<jupyter_output><empty_output><jupyter_text>여기서 주의할점:여기에서는 각 브랜드의 token을 가지고 similarity를 계산하지만, 검색(Search)할때에는 빠른 검색을 위해 각 브랜드명 자체를 가지고 계산함--> 이제는 여기에서도 각 브랜드명 자체를 가지고 계산하자!<jupyter_code>sim_mat = np.zeros((len(identities),len(wordlist)))
for i, idty in enumerate(tqdm_notebook(identities)):
    for j, word in enumerate(wordlist):
        if word in w2v.wv.vocab:
            sim_mat[i,j] = w2v.wv.n_similarity([word], id_pools[idty].split(','))
        else:
            sim_mat[i,j] = 0
sim_mat.shape
def get_brand_vecs(bname):
    brand_tfidf  = model_tfidf[corp[bname]]
    return np.vstack([sparse2full(c, len(dct)) for c in brand_tfidf]).mean(axis=0)

def plot_id(bname):
    val = sim_mat.dot(get_brand_vecs(bname))
    #val /= val.sum()
    pd.Series(val, index=identities).plot.barh()
id_dict = {}
for bname in tqdm_notebook(brands):
    if bname not in ['ootd','fashion','category']:
        try:
            id_dict[bname] = sim_mat.dot(get_brand_vecs(bname))
        except:
            print(bname)
pd.DataFrame(id_dict, index=identities).to_pickle('model/id_dict.pkl')
# df = pd.DataFrame(id_dict, index=identities) #
df = pd.read_pickle('model/id_dict.pkl')
df[['adidas','chanel']].plot.barh(figsize=(5,7))
# df_id = pd.DataFrame(columns=df.columns)
# df_id.loc['럭셔리-캐주얼'] = df.loc['럭셔리'] / (df.loc['럭셔리'] + df.loc['캐주얼'])
# df_id.loc['유니크-대중성'] = df.loc['유니크'] / (df.loc['유니크'] + df.loc['대중성'])
# df_id.loc['정통성-트렌디'] = df.loc['정통성'] / (df.loc['정통성'] + df.loc['트렌디'])
# df_id.loc['포멀-화제성'] = df.loc['포멀'] / (df.loc['포멀'] + df.loc['화제성'])
# df_id.to_pickle('model/id_dict.pkl')
# df_id[['nike','crocs','chanel']].plot.barh(figsize=(8,5))<jupyter_output><empty_output><jupyter_text>여기서 normalize 축을 바꾸면 어떨까?<jupyter_code>#df = pd.read_pickle('model/id_dict.pkl')
min_max_scaler = preprocessing.MinMaxScaler(feature_range=(0.1, 1))
X_train_minmax = min_max_scaler.fit_transform(df_id)
df2 = df_id.copy()
df2[:] = X_train_minmax
df2[['nike','crocs']].plot.barh(figsize=(5,7))
pca = PCA(n_components=0.9)
X_reduced = pca.fit_transform(df2.T)
# cluster = KMeans(n_clusters=10, random_state=0).fit(X_reduced)
# cluster = DBSCAN().fit(X_reduced)
cluster = AffinityPropagation().fit(X_reduced)
clustered = pd.Series(cluster.labels_, index=df2.columns); clustered
clrd = pd.DataFrame(clustered.sort_values()).reset_index()
clrd.columns = ['bname', 'cluster']
clrd.to_excel('model/clustered.xlsx')
clrd[clrd.bname=='chanel']
clustered.index[clustered==10]
df2.loc['신뢰성']
df_kw = pd.read_excel('keywords and logos.xlsx', sheet_name='20190215')[['shortname','keywords']].set_index('shortname')
df_kw;
def plot_wc(bname):
    with open('model/tokensfreq.json', encoding='UTF-8-sig') as f:
        j = json.load(f)[bname]
        kws = [kw.strip() for kw in df_kw.keywords.loc[bname].split(',')]
        [j.pop(kw, None) for kw in kws]

    x,y = np.ogrid[:300, :300]
    mask = (x-150)**2 + (y-150)**2 > 130**2
    mask = 255 * mask.astype(int)
        
    font_path = r'c:\Windows\Fonts\NanumBarunGothic.ttf'
    wc = WordCloud(width=100, height=100, background_color='white', font_path=font_path, random_state=0, mask=mask)
    
    try:
        fig, ax = plt.subplots(figsize=(7,7))
        ax.imshow(wc.generate_from_frequencies(j), interpolation='bilinear')
        ax.axis('off')
        fig.savefig('model/wordcloud/wc_' + bname + '.png', format='png')
        
    except:
        print(bname)
        
    finally:
        plt.close(fig)
for row in tqdm_notebook(df_kw.itertuples()):
    bname = row.Index
    
    fname = 'model/wordcloud/wc_' + bname + '.png'
    if os.path.isfile(fname):
        continue
        
    if bname not in ['ootd','fashion','category']:
        plot_wc(bname)
df2.T['신뢰성'].sort_values()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18996068944662836, 'fraction_numerical': 0.019413365588146356, 'mean_word_length': 3.429681221537637, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '62007', 'license': 'MIT', 'n_tokens_mistral': 7419, 'n_tokens_neox': 7649, 'n_words': 1136, 'path': 'archive/texcrapy4 2019.ipynb', 'repo_name': 'gem763/texcrapy', 'size': '16535'}