# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel, LsiModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from sklearn.linear_model import SGDRegressor
from sklearn.svm import SVR
from matplotlib import pyplot as plt
from pug.nlp.stats import Confusion
import os
import gzip
import pandas as pd
import matplotlib

# In[3]:



# In[6]:

lsi = LsiModel.load(os.path.join(DATA_PATH, 'lsi100'))
lsi2 = LsiModel.load(os.path.join(DATA_PATH, 'lsi2'))


# In[7]:

with gzip.open(os.path.join(DATA_PATH, 'tweet_topic_vectors.csv.gz'), 'rb') as f:
    topics = pd.DataFrame.from_csv(f, encoding='utf8')
topics = topics.fillna(0)


# In[8]:

dates = pd.read_csv(os.path.join(DATA_PATH, 'datetimes.csv.gz'), engine='python')
nums = pd.read_csv(os.path.join(DATA_PATH, 'numbers.csv.gz'), engine='python')


# In[9]:

nums.favorite_count.hist(bins=[0,1,2,3,4,5,7,10,15,25,40,100,1000])
plt.yscale('log', nonposy='clip')
plt.xscale('log', nonposy='clip')
plt.xlabel('Number of Favorites')

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/11 Learning -- Regression.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel, LsiModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from sklearn.linear_model import SGDRegressor
from sklearn.svm import SVR
from matplotlib import pyplot as plt
from pug.nlp.stats import Confusion
import os
import gzip
import pandas as pd
import matplotlib


# In[3]:



# In[6]:

lsi = LsiModel.load(os.path.join(DATA_PATH, 'lsi100'))
lsi2 = LsiModel.load(os.path.join(DATA_PATH, 'lsi2'))


# In[7]:

with gzip.open(os.path.join(DATA_PATH, 'tweet_topic_vectors.csv.gz'), 'rb') as f:
    topics = pd.DataFrame.from_csv(f, encoding='utf8')
topics = topics.fillna(0)


# In[8]:

dates = pd.read_csv(os.path.join(DATA_PATH, 'datetimes.csv.gz'), engine='python')
nums = pd.read_csv(os.path.join(DATA_PATH, 'numbers.csv.gz'), engine='python')


# In[9]:

nums.favorite_count.hist(bins=[0,1,2,3,4,5,7,10,15,25,40,100,1000])
plt.yscale('log', nonposy='clip')
plt.xscale('log', nonposy='clip')
plt.xlabel('Number of Favorites')

plt.ylabel('Number of Tweets')