# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel, LsiModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from pug.nlp.stats import Confusion
import os
import gzip
import pandas as pd
import matplotlib
results['predicted'] = disc.predict(topics.values)
results['truth'] = nums.favorite_count.values > 0
conf = Confusion(results)
conf


# At least the confusion matrix looks balanced now

# In[174]:

results.predicted.corr(results.truth)


# Should have known, imbalance doesn't help...

# In[179]:

portion_neg = 2 * portion_pos
mask = ((np.random.binomial(1, portion_neg, size=len(nums)).astype(bool) & neg) | pos)
disc = LinearDiscriminantAnalysis().fit(topics.values[mask], (nums.favorite_count.values > 0)[mask])
print(sum(mask))
print(sum(pos) * 2)


# In[180]:

results = pd.DataFrame()
results['predicted'] = disc.predict(topics.values)
results['truth'] = nums.favorite_count.values > 0
conf = Confusion(results)

# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/10 Learning -- Predicting Impact.py
from past.builtins import basestring
from twip.constant import DATA_PATH
from gensim.models import TfidfModel, LsiModel
from gensim.corpora import Dictionary
from IPython.display import display, HTML 
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from pug.nlp.stats import Confusion
import os
import gzip
import pandas as pd
import matplotlib

results['predicted'] = disc.predict(topics.values)
results['truth'] = nums.favorite_count.values > 0
conf = Confusion(results)
conf


# At least the confusion matrix looks balanced now

# In[174]:

results.predicted.corr(results.truth)


# Should have known, imbalance doesn't help...

# In[179]:

portion_neg = 2 * portion_pos
mask = ((np.random.binomial(1, portion_neg, size=len(nums)).astype(bool) & neg) | pos)
disc = LinearDiscriminantAnalysis().fit(topics.values[mask], (nums.favorite_count.values > 0)[mask])
print(sum(mask))
print(sum(pos) * 2)


# In[180]:

results = pd.DataFrame()
results['predicted'] = disc.predict(topics.values)
results['truth'] = nums.favorite_count.values > 0
conf = Confusion(results)

conf