# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


import os
import pandas as pd
import matplotlib
import seaborn
import re
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from pug.nlp import constant
# In[16]:

GB = 8 * (100 * 1000 * len(tfdf)) / 1.e9
GB


# In[6]:

tfdf


# Fortunately the odd words are at the top and bottom of an alphabetical index!  
# And it does look like the less useful tokens aren't used many times or in many documents.  
# What do you notice that might help distinguish "natural" words (zoom, zoos, zope, zynga) from URLs and machine-code (000, zzp, zsl107)?  

# In[11]:

tfdf = tfdf[tfdf.df > 9]
tfdf = tfdf[(tfdf.df > 9) & (((tfdf.df - tfdf.tf) / tfdf.tf) < 0.15)]
tfdf = tfdf[(tfdf.df > 20) & (((tfdf.df - tfdf.tf) / tfdf.tf) < 0.15)]
tfdf


# In[ ]:

Numpy arrays (guts of Pandas DataFrame) require 8 bytes for each double-precision value (int64)


# In[14]:


# Path: twip/constant.py
# DATA_PATH = os.path.abspath(os.path.join(BASE_PATH, '..', 'data'))


# Path: docs/notebooks/03 Data -- Getting Selective.py
import os
import pandas as pd
import matplotlib
import seaborn
import re
from twip.constant import DATA_PATH
from IPython.display import display, HTML 
from pug.nlp import constant

# In[16]:

GB = 8 * (100 * 1000 * len(tfdf)) / 1.e9
GB


# In[6]:

tfdf


# Fortunately the odd words are at the top and bottom of an alphabetical index!  
# And it does look like the less useful tokens aren't used many times or in many documents.  
# What do you notice that might help distinguish "natural" words (zoom, zoos, zope, zynga) from URLs and machine-code (000, zzp, zsl107)?  

# In[11]:

tfdf = tfdf[tfdf.df > 9]
tfdf = tfdf[(tfdf.df > 9) & (((tfdf.df - tfdf.tf) / tfdf.tf) < 0.15)]
tfdf = tfdf[(tfdf.df > 20) & (((tfdf.df - tfdf.tf) / tfdf.tf) < 0.15)]
tfdf


# In[ ]:

Numpy arrays (guts of Pandas DataFrame) require 8 bytes for each double-precision value (int64)


# In[14]:


GB = 8 * (100 * 1000 * len(tfdf)) / 1.e9