# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
import os
import pandas as pd
import string
import matplotlib
import gzip

get_ipython().magic(u'pinfo TfidfModel')


# In[26]:

TfidfModel(df.txt)


# In[27]:

TfidfModel(df.tokens)


# In[28]:

TfidfModel((d.doc2bow(tokens) for tokens in df.tokens))


# But there's a simpler way.  
# We already have a vocabulary  
# with term and document frequencies in a matrix...  

# In[33]:

pd.Series(d.dfs)


# In[34]:


# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/08 Features -- TFIDF with Gensim.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from gensim.models import TfidfModel
from gensim.corpora import Dictionary
import os
import pandas as pd
import string
import matplotlib
import gzip


get_ipython().magic(u'pinfo TfidfModel')


# In[26]:

TfidfModel(df.txt)


# In[27]:

TfidfModel(df.tokens)


# In[28]:

TfidfModel((d.doc2bow(tokens) for tokens in df.tokens))


# But there's a simpler way.  
# We already have a vocabulary  
# with term and document frequencies in a matrix...  

# In[33]:

pd.Series(d.dfs)


# In[34]:


pd.Series(d.iteritems())