# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip


# In[122]:

print([round(np.dot(df.T[i], df.T[i+1]), 4) for i in range(5)])


# Better normalize these...

# In[123]:

print([round(np.dot(df.T[i], df.T[i+1]) / np.linalg.norm(df.T[i]) / np.linalg.norm(df.T[i+1]), 4) for i in range(5)])
# for comparison the TFIDF scores right below
print([round(np.dot(tfidf6[i], tfidf6[i+1]), 4) for i in range(5)])


# So the really chummy neighbors are 1 & 2 and 3 & 4  
# Surprisingly 2 & 3 didn't hit it off, and no pairing got a zero!   
# And the last 2 seem to share a "latent" similarity that TFIDF missed entirely!!!
# And LSI picked up on the python<->Python similarity (tweets 0 and 1)

# In[133]:

with gzip.open(os.path.join(DATA_PATH, 'text.csv.gz'), 'rb') as f:
    text = pd.DataFrame.from_csv(f, encoding='utf8')


# In[188]:

for toks, twt in zip(text.txt.iloc[:6], text.text.iloc[:6]):

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


# Path: docs/notebooks/09 Features -- LSI with Gensim.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip



# In[122]:

print([round(np.dot(df.T[i], df.T[i+1]), 4) for i in range(5)])


# Better normalize these...

# In[123]:

print([round(np.dot(df.T[i], df.T[i+1]) / np.linalg.norm(df.T[i]) / np.linalg.norm(df.T[i+1]), 4) for i in range(5)])
# for comparison the TFIDF scores right below
print([round(np.dot(tfidf6[i], tfidf6[i+1]), 4) for i in range(5)])


# So the really chummy neighbors are 1 & 2 and 3 & 4  
# Surprisingly 2 & 3 didn't hit it off, and no pairing got a zero!   
# And the last 2 seem to share a "latent" similarity that TFIDF missed entirely!!!
# And LSI picked up on the python<->Python similarity (tweets 0 and 1)

# In[133]:

with gzip.open(os.path.join(DATA_PATH, 'text.csv.gz'), 'rb') as f:
    text = pd.DataFrame.from_csv(f, encoding='utf8')


# In[188]:

for toks, twt in zip(text.txt.iloc[:6], text.text.iloc[:6]):

    print(toks)