<jupyter_start><jupyter_text><jupyter_code># 20JUN21, SUN
# [Public] Location:  https://drive.google.com/drive/folders/1fFoFxY4i3YU-KNkz84_AG9Nftxr4Alnb
from google.colab import drive
drive.mount('/content/gdrive')

!pip install pyLDAvis<jupyter_output>Collecting pyLDAvis
[?25l  Downloading https://files.pythonhosted.org/packages/03/a5/15a0da6b0150b8b68610cc78af80364a80a9a4c8b6dd5ee549b8989d4b60/pyLDAvis-3.3.1.tar.gz (1.7MB)
[K     |████████████████████████████████| 1.7MB 4.0MB/s 
[?25h  Installing build dependencies ... [?25l[?25hdone
  Getting requirements to build wheel ... [?25l[?25hdone
  Installing backend dependencies ... [?25l[?25hdone
    Preparing wheel metadata ... [?25l[?25hdone
Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (1.0.1)
Collecting numpy>=1.20.0
[?25l  Downloading https://files.pythonhosted.org/packages/a5/42/560d269f604d3e186a57c21a363e77e199358d054884e61b73e405dd217c/numpy-1.20.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.3MB)
[K     |████████████████████████████████| 15.3MB 187kB/s 
[?25hRequirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from pyLDAvis) (2.11.3)
Requirement already satisfied: gensi[...]<jupyter_text>Click &nbsp; 'RESTART RUNTIME', &nbsp; and then &nbsp; 'Yes'<jupyter_code>cd "gdrive/MyDrive/Time Magazine"

import re, os, ast, time, requests, urllib.request
from datetime import datetime
from pytz import timezone


!sudo apt install  tesseract-ocr
!pip install pytesseract

import pytesseract
pytesseract.pytesseract.tesseract_cmd = (r'/usr/bin/tesseract')

try:
    from PIL import Image
except ImportError:
    import Image
import pytesseract, cv2, os
import cv2, random, itertools
from google.colab.patches import cv2_imshow
import numpy as np
from sklearn.cluster import DBSCAN


!python -m spacy download en_core_web_sm

import spacy
nlp = spacy.load('en_core_web_sm')

import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')

stopword_set = set(stopwords.words('english'))


from __future__ import print_function

import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation<jupyter_output><empty_output><jupyter_text>Scraping [Time Magazine](https://time.com/vault/)1923 - 2015<jupyter_code>def get_issue_dates(year):
    url = 'https://time.com/vault/year/{}/'.format(year)
    page = requests.get(url)
    issue_dates  =  re.findall('\"coverDate\":\"(\d\d\d\d-\d\d-\d\d)\"', page.text)
    return issue_dates

def download_page_image(url, path):
    try:
        urllib.request.urlretrieve(url, path)
        return True
    except Exception as e:
        print('\n\tURL:', url, '\n\tPath:', path)
        print('\t', e)
        return False

def download_issue(issue_date):
    year, month, day  =  issue_date.split('-')
    
    # Find Number of Pages
    url = 'https://time.com/vault/issue/{}/page/1/'.format(issue_date)
    page = requests.get(url)
    page_count = len(ast.literal_eval(re.findall('\"pagePDFs\":(\[[^\[\]]*\])', page.text)[0]))

    # Create Issue Directory
    if not os.path.exists(year+'/'+issue_date):    os.mkdir(year+'/'+issue_date)

    # Download Pages
    for  page_number  in range(1, page_count+1):
        page_url  =  'https://content.time.com/time/subscriber/vault/{}/{}/{}/{}/1550.jpg'.format(year, month, year+month+day, page_number)
        
        # urllib.request.urlretrieve(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))
        count = 0
        success  =  download_page_image(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))
        while not success and count < 3:
            success  =  download_page_image(page_url, '{}/{}/{}.jpg'.format(year, issue_date, page_number))
            time.sleep(1)
            count += 1


def get_timestamp():
    now_utc = datetime.now(timezone('UTC'))
    return now_utc.astimezone(timezone('Asia/Kolkata')).strftime("[%d-%m-%Y %H:%M:%S]")

%%time


for  year  in list(range(1923, 2021+1)):
    year = str(year)
    
    # Create year Directory
    if not os.path.exists(year):    os.mkdir(year)

    issue_dates  =  get_issue_dates(year)
    for  issue_date  in issue_dates:
        download_issue(issue_date)


    print('\n{}  Completed: {}.'.format(get_timestamp(), year))<jupyter_output><empty_output><jupyter_text>Text Extracation (2000-2015)<jupyter_code>for  year  in  range(2000, 2016):
    year = str(year)
    
    for  path,subdirs,files  in os.walk(year):
        dir_files = os.listdir(path)
        print(path)
        
        for name in files:
            if name.endswith('.jpg')  and  name[:-4]+'.txt' not in dir_files:
                img_path = os.path.join(path, name)

                image = cv2.imread(img_path)
                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)    # Convert to Black and White
                gray = cv2.resize(gray, None, fx = 3, fy = 3, interpolation = cv2.INTER_CUBIC)    # Zoom 3X

                text  =  pytesseract.image_to_string(gray)    # OCR

                # Save text
                f = open(img_path[:-4]+'.txt', 'w')
                f.write(text)
                f.close()<jupyter_output><empty_output><jupyter_text>What is on the cover page?The most Frequent Terms<jupyter_code>try:    from unidecode import unidecode
except:
    !pip install unidecode
    from unidecode import unidecode


def read_text_file(filename):
    with open(filename) as f:    
        content = ' '.join([unidecode(i.strip())   for i in f.readlines()   if i.strip()]).replace('- ', '')
    return content

def get_magazine_text(path):
    pages  =  sorted([os.path.join(path, file)   for file in os.listdir(path)   if file.endswith('.txt')], key=lambda x: int(x.split('/')[-1][:-4]))
    texts  =  [read_text_file(page)  for page in pages]
    return texts

release_dates  =  sum([[os.path.join(str(year), dir_)  for dir_ in os.listdir(str(year))  if '.' not in dir_]  for year in range(2000, 2016)], [])

texts = []
for  release_date  in release_dates:
    # print('\n\n\n', release_date)
    text = read_text_file(release_date+'/1.txt')
    texts.append(text)
    # print(text)

    # doc = nlp(text)
    # for entity in doc.ents:
    #     print(entity.label_, ' | ', entity.text)
from collections import Counter
l = ' '.join(texts).lower().split()
l = [i  for i in l  if len(i)>=3 and i not in stopword_set]
c = Counter(l)
c.most_common()<jupyter_output><empty_output><jupyter_text>Looks like it's mostly about the US and Geopolitics.<jupyter_code># Save all texts in a Pickle File (for quick retreval).
# texts = {}
# count = 0

# for  release_date  in release_dates:
#     print(release_date)
#     for  file_  in os.listdir(release_date):
#         count += 1
#         texts[os.path.join(release_date, file_)]  =  (count, get_magazine_text(release_date))


# import pickle
# with open('time_magazine_texts.pickle', 'wb') as f:
#     pickle.dump(texts, f, protocol=pickle.HIGHEST_PROTOCOL)

# with open('time_magazine_texts.pickle', 'rb') as f:
#     texts = pickle.load(f)<jupyter_output>2000/2000-01-01
2000/2000-01-17
2000/2000-01-24
2000/2000-01-31
2000/2000-02-07
2000/2000-02-14
2000/2000-02-21
2000/2000-02-28
2000/2000-03-06
2000/2000-03-13
2000/2000-03-20
2000/2000-03-27
2000/2000-04-03
2000/2000-04-10
2000/2000-04-17
2000/2000-04-24
2000/2000-05-01
2000/2000-05-08
2000/2000-05-15
2000/2000-05-22
2000/2000-05-29
2000/2000-06-05
2000/2000-06-12
2000/2000-06-19
2000/2000-06-26
2000/2000-07-03
2000/2000-07-10
2000/2000-07-17
2000/2000-07-24
2000/2000-07-31
2000/2000-08-07
2000/2000-08-14
2000/2000-08-21
2000/2000-08-28
2000/2000-09-04
2000/2000-09-11
2000/2000-09-18
2000/2000-09-25
2000/2000-10-02
2000/2000-10-09
2000/2000-10-16
2000/2000-10-23
2000/2000-10-30
2000/2000-11-06
2000/2000-11-13
2000/2000-11-20
2000/2000-11-27
2000/2000-12-04
2000/2000-12-11
2000/2000-12-18
2000/2000-12-25
2001/2001-01-08
2001/2001-01-15
2001/2001-01-22
2001/2001-01-29
2001/2001-02-05
2001/2001-02-12
2001/2001-02-19
2001/2001-02-26
2001/2001-03-05
2001/2001-03-12
2001/2001-03-19
2001/200[...]<jupyter_text>Let's try to find some topics and their dominant terms<jupyter_code>len(release_dates)

# Let's see for a year first.
%%time
l = [get_magazine_text(release_date)   for release_date in release_dates[:1]]
pages = [item  for sublist in l  for item in sublist]
n_components = 5    # Number of topics
tf_vectorizer = CountVectorizer(strip_accents = 'unicode',
                                stop_words = 'english',
                                lowercase = True,
                                token_pattern = r'\<KEY>',
                                max_df = 0.5, 
                                min_df = 10)
dtm_tf = tf_vectorizer.fit_transform(pages)
print(dtm_tf.shape)


# for TF DTM
lda_tf = LatentDirichletAllocation(n_components=n_components, random_state=0)
lda_tf.fit(dtm_tf)


pyLDAvis.sklearn.prepare(lda_tf, dtm_tf, tf_vectorizer)
tfidf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',
                                   stop_words = 'english',
                                   lowercase = True,
                                   token_pattern = r'\b[a-zA-Z]{3,}\b',
                                   max_df = 0.5, 
                                   min_df = 10)
dtm_tfidf = tfidf_vectorizer.fit_transform(pages)
print(dtm_tfidf.shape)


# for TFIDF DTM
lda_tfidf = LatentDirichletAllocation(n_components=n_components, random_state=0)
lda_tfidf.fit(dtm_tfidf)


pyLDAvis.sklearn.prepare(lda_tfidf, dtm_tfidf, tfidf_vectorizer)
# TODO: For all the issues, and interpret<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14228600717580728, 'fraction_numerical': 0.10425422860071758, 'mean_word_length': 3.199741713301765, 'pattern_counts': {'":': 2, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '500286', 'license': 'MIT', 'n_tokens_mistral': 4411, 'n_tokens_neox': 3673, 'n_words': 686, 'path': '01.ipynb', 'repo_name': 'The-Gupta/Time', 'size': '9765'}