# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip


# In[2]:

get_ipython().magic(u'matplotlib inline')
np = pd.np
display(HTML("<style>.container { width:100% !important; }</style>"))
pd.set_option('display.max_rows', 6)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 500)
pd.set_option('precision', 2)
get_ipython().magic(u'precision 4')
get_ipython().magic(u'pprint')


# So the new things are LsiModel and scatmat

# In[3]:



# Load cleaned tweet data  
# Don't forget to fix up the tokens!  
# Can you think of a better way to save a list of lists of strings?
# What about the raw, unprocessed unicode tweet text itself?

# In[6]:

with gzip.open(os.path.join(DATA_PATH, 'datetimes.csv.gz'), 'rb') as f:
    nums = pd.read_csv(f, engine='python', encoding='utf-8')

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))
# 
# Path: twip/plot.py
# def scatmat(df, category=None, colors='rgob',
#             num_plots=4, num_topics=100, num_columns=4,
#             show=False, block=False, data_path=DATA_PATH, save=False, verbose=1):
#     """FIXME: empty plots that dont go away, Plot and/save scatter matrix in groups of num_columns topics"""
# 
#     if category is None:
#         category = list(df.columns)[-1]
#     if category in df.columns:
#         category = df[category]
#     else:
#         category = pd.Series(category)
# 
#     suffix = compose_suffix(len(df), num_topics, save)
#     save = bool(save)
#     for i in range(min(num_plots * num_columns, num_topics) / num_plots):
#         scatter_matrix(df[df.columns[i * num_columns:(i + 1) * num_columns]],
#                        marker='+', c=[colors[int(x) % len(colors)] for x in category.values],
#                        figsize=(18, 12))
#         if save:
#             name = 'scatmat_topics_{}-{}.jpg'.format(i * num_columns, (i + 1) * num_columns) + suffix
#             plt.savefig(os.path.join(data_path, name + '.jpg'))
#         if show:
#             if block:
#                 plt.show()
#             else:
#                 plt.show(block=False)


# Path: docs/notebooks/09 Features -- LSI with Gensim.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from gensim.models import LsiModel
from twip.plot import scatmat
from gensim.models import LsiModel
import os
import gzip
import pandas as pd
import matplotlib
import gzip



# In[2]:

get_ipython().magic(u'matplotlib inline')
np = pd.np
display(HTML("<style>.container { width:100% !important; }</style>"))
pd.set_option('display.max_rows', 6)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 500)
pd.set_option('precision', 2)
get_ipython().magic(u'precision 4')
get_ipython().magic(u'pprint')


# So the new things are LsiModel and scatmat

# In[3]:



# Load cleaned tweet data  
# Don't forget to fix up the tokens!  
# Can you think of a better way to save a list of lists of strings?
# What about the raw, unprocessed unicode tweet text itself?

# In[6]:

with gzip.open(os.path.join(DATA_PATH, 'datetimes.csv.gz'), 'rb') as f:
    nums = pd.read_csv(f, engine='python', encoding='utf-8')

with gzip.open(os.path.join(DATA_PATH, 'text.csv.gz'), 'rb') as f: