# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip

for toks, twt in zip(text.txt.iloc[:6], text.text.iloc[:6]):
    print(toks)
    print(twt)
    print('-' * 10)
    


# What about a new tweet you are considering?  
# Notice how I changed the token spelling (BOW),  
# but not the *"semantics"* of the tweet.  

# In[169]:

tweet = 'I want to help build django with a job in Chicago'
tweet_bow = vocab.doc2bow(tweet.split())
tweet_tfidf = tfidf[tweet_bow]
tweet_topics = pd.Series(dict(lsi[tweet_tfidf]))
# Now that the math is done let's convert to a friendlier format with words as the keys/index
tweet_tfidf = pd.Series(dict([(vocab[i], x) for (i, x) in tweet_tfidf])) 
print('\nLSI Topic Vector')
tweet_topics


# Compare the topic vector above to the TFIDF vector below.  
# What's better about TFIDF compared to topic vectors?  
# What can we do about it?  

# In[170]:


# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


# Path: docs/notebooks/09 Features -- LSI with Gensim.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip


for toks, twt in zip(text.txt.iloc[:6], text.text.iloc[:6]):
    print(toks)
    print(twt)
    print('-' * 10)
    


# What about a new tweet you are considering?  
# Notice how I changed the token spelling (BOW),  
# but not the *"semantics"* of the tweet.  

# In[169]:

tweet = 'I want to help build django with a job in Chicago'
tweet_bow = vocab.doc2bow(tweet.split())
tweet_tfidf = tfidf[tweet_bow]
tweet_topics = pd.Series(dict(lsi[tweet_tfidf]))
# Now that the math is done let's convert to a friendlier format with words as the keys/index
tweet_tfidf = pd.Series(dict([(vocab[i], x) for (i, x) in tweet_tfidf])) 
print('\nLSI Topic Vector')
tweet_topics


# Compare the topic vector above to the TFIDF vector below.  
# What's better about TFIDF compared to topic vectors?  
# What can we do about it?  

# In[170]:


print('TFIDF Frequency Vector')