<commit_before># -*- coding: utf-8 -*-
"""
Bot that creates word clouds from scientific articles and posts them to Twitter
"""
from datetime import datetime
from glob import glob
import feedparser as fp
import json
import logging
import numpy as np
import os
from PIL import Image
import pickle
import time
from tinydb import TinyDB, where
from wordcloud import WordCloud, STOPWORDS

from parse_article import extract_text
from settings import Settings
from shorten_url_api import UrlShortener
from twitter_api import TwitterApi


class AtmosSciBot(object):
    def __init__(self, curdir, settings, twitter_api, url_shortener, logger):
        self.BOT_NAME = settings.get_bot_name()
        self.curdir = curdir
        self.logger = logger
        self.j_list_path = os.path.join(self.curdir,
                                        settings.get_journal_list())
        self.db_file = settings.get_db_file()

        # Word Cloud settings
        self.minwords = settings.get_min_words()
        self.stopwords_dir = settings.get_stopwords_dir()
        self.dpi = settings.get_dpi()
        self.width = settings.get_width()
        self.height = settings.get_height()
        self.wordcloud_mask = settings.get_wordcloud_mask()
        self.temp_dir = settings.get_temp_dir()
        self.temp_file = settings.get_temp_file()
        self.mentions_file = settings.get_mentions_file()

        self.twitter_api = twitter_api
        self.url_shortener = url_shortener

    def check_new_entry(self, url):
        query_result = self.DB.search(where('url') == url)
        new_entry = False if len(query_result) > 0 else True
        return new_entry

    def write_entry(self, url, j_short_name):
        tstamp = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        new_entry = dict(journal_short_name=j_short_name,
                         url=url,
                         datetime=tstamp)
        self.DB.insert(new_entry)

    def make_title(self, url, journal, title):
        if 'discuss' in url:
            # EGU discussion journals
            journal_name = journal+'D'
        else:
            journal_name = journal
        return '{}: {}'.format(journal_name, title)

    def make_img_file(self):
        output_dir = os.path.join(self.curdir, self.temp_dir)
        if not os.path.isdir(output_dir):
            os.mkdir(output_dir)
        tstamp = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        self.img_file = os.path.join(output_dir,
                                     self.temp_file.format(datetime=tstamp))

    def get_exclude_words(self):
        """ list of words to exclude """
        exclude_words = list(STOPWORDS)
        stopword_files = glob(os.path.join(self.curdir,
                                           self.stopwords_dir,
                                           '*.txt'))
        for fname in stopword_files:
            with open(fname, 'r') as f:
                exclude_words += f.read().split('\n')
        self.exclude_words = set(exclude_words)

    def generate_wc(self, background_color='#ffffff'):
        """generate wordcloud and save to file"""
        # fig_kw = dict(figsize=(self.width/self.dpi, self.height/self.dpi),
        #               dpi=self.dpi)
        self.get_exclude_words()
        try:
            imgpath = os.path.join(self.curdir, self.wordcloud_mask)
            arr = np.array(Image.open(imgpath))
            # Other masks can be extracted from
            # Font-Awesome (http://minimaxir.com/2016/05/wordclouds/)

            wc = WordCloud(width=self.width, height=self.height,
                           stopwords=self.exclude_words,
                           background_color=background_color, mode='RGBA',
                           mask=arr).generate(self.text)

            self.make_img_file()
            wc.to_file(self.img_file)
            self.error_in_wordcloud_gen = None

        except Exception as e:
            self.error_in_wordcloud_gen = e

    def save_mentions(self, mentions):
        with open(self.mentions_file, 'wb') as f:
            pickle.dump(mentions, f)

    def load_mentions(self):
        with open(self.mentions_file, 'rb') as f:
            try:
                mentions = pickle.load(f)
            except:
                mentions = []<commit_msg>fix path to pickle dump<commit_after># -*- coding: utf-8 -*-
"""
Bot that creates word clouds from scientific articles and posts them to Twitter
"""
from datetime import datetime
from glob import glob
import feedparser as fp
import json
import logging
import numpy as np
import os
from PIL import Image
import pickle
import time
from tinydb import TinyDB, where
from wordcloud import WordCloud, STOPWORDS

from parse_article import extract_text
from settings import Settings
from shorten_url_api import UrlShortener
from twitter_api import TwitterApi


class AtmosSciBot(object):
    def __init__(self, curdir, settings, twitter_api, url_shortener, logger):
        self.BOT_NAME = settings.get_bot_name()
        self.curdir = curdir
        self.logger = logger
        self.j_list_path = os.path.join(self.curdir,
                                        settings.get_journal_list())
        self.db_file = settings.get_db_file()

        # Word Cloud settings
        self.minwords = settings.get_min_words()
        self.stopwords_dir = settings.get_stopwords_dir()
        self.dpi = settings.get_dpi()
        self.width = settings.get_width()
        self.height = settings.get_height()
        self.wordcloud_mask = settings.get_wordcloud_mask()
        self.temp_dir = settings.get_temp_dir()
        self.temp_file = settings.get_temp_file()
        self.mentions_file = settings.get_mentions_file()

        self.twitter_api = twitter_api
        self.url_shortener = url_shortener

    def check_new_entry(self, url):
        query_result = self.DB.search(where('url') == url)
        new_entry = False if len(query_result) > 0 else True
        return new_entry

    def write_entry(self, url, j_short_name):
        tstamp = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        new_entry = dict(journal_short_name=j_short_name,
                         url=url,
                         datetime=tstamp)
        self.DB.insert(new_entry)

    def make_title(self, url, journal, title):
        if 'discuss' in url:
            # EGU discussion journals
            journal_name = journal+'D'
        else:
            journal_name = journal
        return '{}: {}'.format(journal_name, title)

    def make_img_file(self):
        output_dir = os.path.join(self.curdir, self.temp_dir)
        if not os.path.isdir(output_dir):
            os.mkdir(output_dir)
        tstamp = datetime.utcnow().strftime('%Y%m%d%H%M%S')
        self.img_file = os.path.join(output_dir,
                                     self.temp_file.format(datetime=tstamp))

    def get_exclude_words(self):
        """ list of words to exclude """
        exclude_words = list(STOPWORDS)
        stopword_files = glob(os.path.join(self.curdir,
                                           self.stopwords_dir,
                                           '*.txt'))
        for fname in stopword_files:
            with open(fname, 'r') as f:
                exclude_words += f.read().split('\n')
        self.exclude_words = set(exclude_words)

    def generate_wc(self, background_color='#ffffff'):
        """generate wordcloud and save to file"""
        # fig_kw = dict(figsize=(self.width/self.dpi, self.height/self.dpi),
        #               dpi=self.dpi)
        self.get_exclude_words()
        try:
            imgpath = os.path.join(self.curdir, self.wordcloud_mask)
            arr = np.array(Image.open(imgpath))
            # Other masks can be extracted from
            # Font-Awesome (http://minimaxir.com/2016/05/wordclouds/)

            wc = WordCloud(width=self.width, height=self.height,
                           stopwords=self.exclude_words,
                           background_color=background_color, mode='RGBA',
                           mask=arr).generate(self.text)

            self.make_img_file()
            wc.to_file(self.img_file)
            self.error_in_wordcloud_gen = None

        except Exception as e:
            self.error_in_wordcloud_gen = e

    def save_mentions(self, mentions):
        with open(os.path.join(self.curdir, self.mentions_file), 'wb') as f:
            pickle.dump(mentions, f)

    def load_mentions(self):
        with open(os.path.join(self.curdir, self.mentions_file), 'rb') as f:
            try:
                mentions = pickle.load(f)
            except:
                mentions = []
{'fraction_non_alphanumeric': 0.09691629955947137, 'fraction_numerical': 0.0018548574078367725, 'mean_word_length': 2.111071042192571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1402202', 'n_tokens_mistral': 2600, 'n_tokens_neox': 2513, 'n_words': 546}