<commit_before>###########################################################################
# (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
# #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
"""Run scraper

Usage:
  scrape.py run [options] [<scraper>...]
  scrape.py list
  scrape.py report [--email] [--date=<date>]
  scrape.py log <uuid>
  scrape.py -h | --help

Options:
  -h --help        Show this screen.
  --from=<date>    Scrape articles from date (default: today)
  --to=<date>      Scrape articles up to and including date (default: today)
  --dry-run        Do not commit to database
  --batched        Regularly save to database. This prevents scraping from using too much memory,
                   but may leave articlesets half-filled in case of errors.
  --batch-size=<n> If running in batched mode, this determines the batch size. For continuous
                   scrapers a low value is suitable for "real-time" purposes (default: 1000).
  --update         Update comment threads of existing articles

"""
from __future__ import print_function

import collections
from StringIO import StringIO
from email.utils import formatdate
import glob
from itertools import imap
import json
import logging
import os.path
import sys
import jinja2
import datetime
import tabulate
import uuid

from django.core.mail import EmailMultiAlternatives, get_connection
import errno

import amcatscraping.tools
from amcatscraping.tools import read_date, get_boolean, to_date
import amcatscraping.scraper


JINJA_ENV = jinja2.Environment(loader=jinja2.PackageLoader('amcatscraping', 'templates'))
EMAIL_TEMPLATE = JINJA_ENV.get_template('log_email.html')

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

log = logging.getLogger(__name__)

MODULE_PATH = os.path.abspath(os.path.join(*amcatscraping.__path__))
ROOT_PATH = os.path.abspath(os.path.join(MODULE_PATH, ".."))
DEFAULT_CONFIG_FILE = os.path.join(MODULE_PATH, "default.conf")
USER_CONFIG_FILE = os.path.abspath(os.path.expanduser("~/.scrapers.conf"))
LOG_DIR = os.path.expanduser("~/.cache/scraperlogs/")
TODAY = datetime.date.today()

SECTIONS = {"*", "store", "mail"}

ScraperResult = collections.namedtuple("ScraperResult", ["name", "narticles", "failed", "log"])


def get_scraper_class(scraper, relative_path):
    scraper_module, scraper_class = relative_path.rsplit(".", 1)

    if not get_boolean(scraper["is_absolute_classpath"]):
        scraper_module = "amcatscraping.scrapers.%s" % scraper_module

    scraper_module = __import__(scraper_module, fromlist=["non-empty"])
    return getattr(scraper_module, scraper_class)


def run_single(config, args, scraper_config, scraper_class):
    # Scraper config
    articleset_id = int(scraper_config["articleset"])
    project_id = int(scraper_config["project"])
    username = scraper_config.get("username")
    password = scraper_config.get("password")

    # Store config
    protocol = "https" if config.getboolean("store", "ssl") else "http"
    host_url = "%s://%s:%s" % (protocol, config.get("store", "host"), config.get("store", "port"))
    host_username = config.get("store", "username")
    host_password = config.get("store", "password")

    min_date = max_date = datetime.date.today()

    if args["--from"]:
        min_date = to_date(read_date(args["--from"]))

    if args["--to"]:
        max_date = to_date(read_date(args["--to"]))

    opts = {
        "project_id": project_id,
        "articleset_id": articleset_id,
        "api_host": host_url,
        "api_user": host_username,
        "api_password": <PASSWORD>,
        "username": username,
        "password": password,
        "log_errors": True,
        "min_date": min_date,
        "max_date": max_date,
        "batched": args["--batched"],
        "dry_run": args["--dry-run"],
        "batch_size": int(args.get("--batch-size") or 1000)
    }

    scraper = scraper_class(**opts)
    method = "run_update" if args["--update"] else "run"

    try:
        return list(getattr(scraper, method)()), False
    except NotImplementedError:
        if args["--update"]:
            log.info("Updating not implemented for {scraper_class.__name__}".format(**locals()))
        else:
            log.exception("Running scraper {scraper_class.__name__} resulted in an exception:".format(**locals()))
    except Exception as e:
        log.exception("Running scraper {scraper_class.__name__} resulted in an exception:".format(**locals()))

    return [], True


def _run(config, args, scrapers):
    all_scrapers = dict(get_scrapers(config))
    scrapers = {label: all_scrapers[label] for label in scrapers} or all_scrapers

    if not scrapers:
        print("No scrapers found.")
        sys.exit(1)

    non_existing_scrapers = set(scrapers) - set(all_scrapers)
    if non_existing_scrapers:
        print("Scraper(s) not found: %s. Use:\n" % ", ".join(non_existing_scrapers))
        print("\tscrape.py list\n")
        print("to list existing scrapers")
        sys.exit(1)

    root_logger = logging.getLogger(amcatscraping.scraper.__name__)
    for label, scraper in scrapers.items():
        log_buffer = StringIO()
        log_handler = logging.StreamHandler(log_buffer)
        root_logger.addHandler(log_handler)

        scraper_class = get_scraper_class(scraper, scraper["class"])
        articles, failed = run_single(config, args, scraper, scraper_class)

        root_logger.removeHandler(log_handler)
        narticles = len(articles.pop()) if articles else 0
        yield ScraperResult(label, narticles, failed, log_buffer.getvalue())


def run(config, args, scrapers):
    """Run scrapers and write logs afterwards"""
    logs = collections.OrderedDict()
    for label, narticles, failed, log in _run(config, args, scrapers):
        logs[label] = (datetime.datetime.now(), narticles, failed, log)

    identifier = str(uuid.uuid4())
    log_dir = os.path.join(LOG_DIR, TODAY.strftime("%Y-%m-%d"))
    log_file = os.path.join(log_dir, identifier + ".json")

    try:
        os.makedirs(log_dir)<commit_msg>Fixed bug casing scrape to crash<commit_after>###########################################################################
# (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
# #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
"""Run scraper

Usage:
  scrape.py run [options] [<scraper>...]
  scrape.py list
  scrape.py report [--email] [--date=<date>]
  scrape.py log <uuid>
  scrape.py -h | --help

Options:
  -h --help        Show this screen.
  --from=<date>    Scrape articles from date (default: today)
  --to=<date>      Scrape articles up to and including date (default: today)
  --dry-run        Do not commit to database
  --batched        Regularly save to database. This prevents scraping from using too much memory,
                   but may leave articlesets half-filled in case of errors.
  --batch-size=<n> If running in batched mode, this determines the batch size. For continuous
                   scrapers a low value is suitable for "real-time" purposes (default: 1000).
  --update         Update comment threads of existing articles

"""
from __future__ import print_function

import collections
from StringIO import StringIO
from email.utils import formatdate
import glob
from itertools import imap
import json
import logging
import os.path
import sys
import jinja2
import datetime
import tabulate
import uuid

from django.core.mail import EmailMultiAlternatives, get_connection
import errno

import amcatscraping.tools
from amcatscraping.tools import read_date, get_boolean, to_date
import amcatscraping.scraper


JINJA_ENV = jinja2.Environment(loader=jinja2.PackageLoader('amcatscraping', 'templates'))
EMAIL_TEMPLATE = JINJA_ENV.get_template('log_email.html')

try:
    import configparser
except ImportError:
    import ConfigParser as configparser

log = logging.getLogger(__name__)

MODULE_PATH = os.path.abspath(os.path.join(*amcatscraping.__path__))
ROOT_PATH = os.path.abspath(os.path.join(MODULE_PATH, ".."))
DEFAULT_CONFIG_FILE = os.path.join(MODULE_PATH, "default.conf")
USER_CONFIG_FILE = os.path.abspath(os.path.expanduser("~/.scrapers.conf"))
LOG_DIR = os.path.expanduser("~/.cache/scraperlogs/")
TODAY = datetime.date.today()

SECTIONS = {"*", "store", "mail"}

ScraperResult = collections.namedtuple("ScraperResult", ["name", "narticles", "failed", "log"])


def get_scraper_class(scraper, relative_path):
    scraper_module, scraper_class = relative_path.rsplit(".", 1)

    if not get_boolean(scraper["is_absolute_classpath"]):
        scraper_module = "amcatscraping.scrapers.%s" % scraper_module

    scraper_module = __import__(scraper_module, fromlist=["non-empty"])
    return getattr(scraper_module, scraper_class)


def run_single(config, args, scraper_config, scraper_class):
    # Scraper config
    articleset_id = int(scraper_config["articleset"])
    project_id = int(scraper_config["project"])
    username = scraper_config.get("username")
    password = scraper_config.get("password")

    # Store config
    protocol = "https" if config.getboolean("store", "ssl") else "http"
    host_url = "%s://%s:%s" % (protocol, config.get("store", "host"), config.get("store", "port"))
    host_username = config.get("store", "username")
    host_password = config.get("store", "password")

    min_date = max_date = datetime.date.today()

    if args["--from"]:
        min_date = to_date(read_date(args["--from"]))

    if args["--to"]:
        max_date = to_date(read_date(args["--to"]))

    opts = {
        "project_id": project_id,
        "articleset_id": articleset_id,
        "api_host": host_url,
        "api_user": host_username,
        "api_password": host_password,
        "username": username,
        "password": password,
        "log_errors": True,
        "min_date": min_date,
        "max_date": max_date,
        "batched": args["--batched"],
        "dry_run": args["--dry-run"],
        "batch_size": int(args.get("--batch-size") or 1000)
    }

    scraper = scraper_class(**opts)
    method = "run_update" if args["--update"] else "run"

    try:
        return list(getattr(scraper, method)()), False
    except NotImplementedError:
        if args["--update"]:
            log.info("Updating not implemented for {scraper_class.__name__}".format(**locals()))
        else:
            log.exception("Running scraper {scraper_class.__name__} resulted in an exception:".format(**locals()))
    except Exception as e:
        log.exception("Running scraper {scraper_class.__name__} resulted in an exception:".format(**locals()))

    return [], True


def _run(config, args, scrapers):
    all_scrapers = dict(get_scrapers(config))
    scrapers = {label: all_scrapers[label] for label in scrapers} or all_scrapers

    if not scrapers:
        print("No scrapers found.")
        sys.exit(1)

    non_existing_scrapers = set(scrapers) - set(all_scrapers)
    if non_existing_scrapers:
        print("Scraper(s) not found: %s. Use:\n" % ", ".join(non_existing_scrapers))
        print("\tscrape.py list\n")
        print("to list existing scrapers")
        sys.exit(1)

    root_logger = logging.getLogger(amcatscraping.scraper.__name__)
    for label, scraper in scrapers.items():
        log_buffer = StringIO()
        log_handler = logging.StreamHandler(log_buffer)
        root_logger.addHandler(log_handler)

        scraper_class = get_scraper_class(scraper, scraper["class"])
        articles, failed = run_single(config, args, scraper, scraper_class)

        root_logger.removeHandler(log_handler)
        yield ScraperResult(label, len(articles), failed, log_buffer.getvalue())


def run(config, args, scrapers):
    """Run scrapers and write logs afterwards"""
    logs = collections.OrderedDict()
    for label, narticles, failed, log in _run(config, args, scrapers):
        logs[label] = (datetime.datetime.now(), narticles, failed, log)

    identifier = str(uuid.uuid4())
    log_dir = os.path.join(LOG_DIR, TODAY.strftime("%Y-%m-%d"))
    log_file = os.path.join(log_dir, identifier + ".json")

    try:
        os.makedirs(log_dir)
{'fraction_non_alphanumeric': 0.13431269674711438, 'fraction_numerical': 0.002308499475341028, 'mean_word_length': 3.033860045146727, 'pattern_counts': {'":': 26, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3589385', 'n_tokens_mistral': 4315, 'n_tokens_neox': 4170, 'n_words': 1114}