<commit_before>#!/usr/bin/env python
"""
Provides functionality to crawl and extract news articles from commoncrawl.org. Filter criteria, such as publish date
and host list, can be defined. Currently, all WARC files will be downloaded to the path WORKINGDIR/cc_download_warc, if
not otherwise specified.
"""
import logging
import os
import time
from functools import partial
from multiprocessing import Pool
import datetime
import gzip
from urllib.parse import urlparse

import boto3
import botocore
from dateutil import parser
import requests
from scrapy.utils.log import configure_logging

from ..crawler.commoncrawl_extractor import CommonCrawlExtractor

__author__ = "<NAME>"
__copyright__ = "Copyright 2017"
__credits__ = ["<NAME>"]

# commoncrawl.org
__cc_base_url = 'https://data.commoncrawl.org/'
__cc_bucket = 'commoncrawl'

# log file of fully extracted WARC files
__log_pathname_fully_extracted_warcs = None

# logging
logging.basicConfig(level=logging.INFO)
__logger = logging.getLogger(__name__)

__number_of_warc_files_on_cc = 0

__extern_callback_on_warc_completed = None
__counter_article_passed = 0
__counter_article_discarded = 0
__counter_article_error = 0
__counter_article_total = 0
__counter_warc_skipped = 0
__counter_warc_processed = 0
__start_time = time.time()

# When Common Crawl started.
__common_crawl_start_date = datetime.datetime(2016, 8, 26)

def __setup(local_download_dir_warc, log_level):
    """
    Setup
    :return:
    """
    os.makedirs(local_download_dir_warc, exist_ok=True)

    global __log_pathname_fully_extracted_warcs
    __log_pathname_fully_extracted_warcs = os.path.join(local_download_dir_warc, 'fullyextractedwarcs.list')

    # make loggers quiet
    configure_logging({"LOG_LEVEL": "ERROR"})
    logging.getLogger('requests').setLevel(logging.CRITICAL)
    logging.getLogger('readability').setLevel(logging.CRITICAL)
    logging.getLogger('PIL').setLevel(logging.CRITICAL)
    logging.getLogger('newspaper').setLevel(logging.CRITICAL)
    logging.getLogger('newsplease').setLevel(logging.CRITICAL)
    logging.getLogger('urllib3').setLevel(logging.CRITICAL)
    logging.getLogger('jieba').setLevel(logging.CRITICAL)

    boto3.set_stream_logger('botocore', log_level)
    boto3.set_stream_logger('boto3', log_level)

    # set own logger
    logging.basicConfig(level=log_level)
    __logger = logging.getLogger(__name__)
    __logger.setLevel(log_level)


def __get_publishing_date(warc_record, article):
    """
    Extracts the publishing date from the article
    :param warc_record:
    :return:
    """
    if article.publish_date:
        return parser.parse(article.publish_date)
    else:
        return None


def __get_download_url(name):
    """
    Creates a download url given the name
    :param name:
    :return:
    """
    return __cc_base_url + name


def __iterate_by_month(start_date=None, end_date=None, month_step=1):
    if start_date is None:
        # The starting month of Common Crawl.
        start_date = __common_crawl_start_date
    if end_date is None:
        # Until now.
        end_date = datetime.datetime.today()
    current_date = start_date
    while current_date < end_date:
        yield current_date
        carry, new_month = divmod(current_date.month - 1 + month_step, 12)
        new_month += 1
        current_date = current_date.replace(year=current_date.year + carry,
                                            month=new_month)


def __extract_date_from_warc_filename(path):
    fn = os.path.basename(path)
    # Assume the filename pattern is CC-NEWS-20160911145202-00018.warc.gz
    fn = fn.replace('CC-NEWS-', '')
    dt = fn.split('-')[0]

    try:
        return datetime.datetime.strptime(dt, '%Y%m%d%H%M%S')
    except:
        # return date clearly outside the range
        return datetime.datetime(1900, 1, 1)


def __date_within_period(date, start_date=None, end_date=None):
    if start_date is None:
        # The starting month of Common Crawl.
        start_date = __common_crawl_start_date
    if end_date is None:
        # Until now.
        end_date = datetime.datetime.today()
    return start_date <= date < end_date
<commit_msg>Common Crawl crawler: fix selection of WARC files by time range: - must always fetch WARC file listing for the month of the end date<commit_after>#!/usr/bin/env python
"""
Provides functionality to crawl and extract news articles from commoncrawl.org. Filter criteria, such as publish date
and host list, can be defined. Currently, all WARC files will be downloaded to the path WORKINGDIR/cc_download_warc, if
not otherwise specified.
"""
import logging
import os
import time
from functools import partial
from multiprocessing import Pool
import datetime
import gzip
from urllib.parse import urlparse

import boto3
import botocore
from dateutil import parser
import requests
from scrapy.utils.log import configure_logging

from ..crawler.commoncrawl_extractor import CommonCrawlExtractor

__author__ = "<NAME>"
__copyright__ = "Copyright 2017"
__credits__ = ["<NAME>"]

# commoncrawl.org
__cc_base_url = 'https://data.commoncrawl.org/'
__cc_bucket = 'commoncrawl'

# log file of fully extracted WARC files
__log_pathname_fully_extracted_warcs = None

# logging
logging.basicConfig(level=logging.INFO)
__logger = logging.getLogger(__name__)

__number_of_warc_files_on_cc = 0

__extern_callback_on_warc_completed = None
__counter_article_passed = 0
__counter_article_discarded = 0
__counter_article_error = 0
__counter_article_total = 0
__counter_warc_skipped = 0
__counter_warc_processed = 0
__start_time = time.time()

# When Common Crawl started.
__common_crawl_start_date = datetime.datetime(2016, 8, 26)

def __setup(local_download_dir_warc, log_level):
    """
    Setup
    :return:
    """
    os.makedirs(local_download_dir_warc, exist_ok=True)

    global __log_pathname_fully_extracted_warcs
    __log_pathname_fully_extracted_warcs = os.path.join(local_download_dir_warc, 'fullyextractedwarcs.list')

    # make loggers quiet
    configure_logging({"LOG_LEVEL": "ERROR"})
    logging.getLogger('requests').setLevel(logging.CRITICAL)
    logging.getLogger('readability').setLevel(logging.CRITICAL)
    logging.getLogger('PIL').setLevel(logging.CRITICAL)
    logging.getLogger('newspaper').setLevel(logging.CRITICAL)
    logging.getLogger('newsplease').setLevel(logging.CRITICAL)
    logging.getLogger('urllib3').setLevel(logging.CRITICAL)
    logging.getLogger('jieba').setLevel(logging.CRITICAL)

    boto3.set_stream_logger('botocore', log_level)
    boto3.set_stream_logger('boto3', log_level)

    # set own logger
    logging.basicConfig(level=log_level)
    __logger = logging.getLogger(__name__)
    __logger.setLevel(log_level)


def __get_publishing_date(warc_record, article):
    """
    Extracts the publishing date from the article
    :param warc_record:
    :return:
    """
    if article.publish_date:
        return parser.parse(article.publish_date)
    else:
        return None


def __get_download_url(name):
    """
    Creates a download url given the name
    :param name:
    :return:
    """
    return __cc_base_url + name


def __iterate_by_month(start_date=None, end_date=None, month_step=1):
    if start_date is None:
        # The starting month of Common Crawl.
        start_date = __common_crawl_start_date
    if end_date is None:
        # Until now.
        end_date = datetime.datetime.today()
    current_date = start_date
    yield current_date
    while True:
        carry, new_month = divmod(current_date.month - 1 + month_step, 12)
        new_month += 1
        current_date = current_date.replace(year=current_date.year + carry,
                                            month=new_month)
        yield current_date
        if current_date > end_date:
            break


def __extract_date_from_warc_filename(path):
    fn = os.path.basename(path)
    # Assume the filename pattern is CC-NEWS-20160911145202-00018.warc.gz
    fn = fn.replace('CC-NEWS-', '')
    dt = fn.split('-')[0]

    try:
        return datetime.datetime.strptime(dt, '%Y%m%d%H%M%S')
    except:
        # return date clearly outside the range
        return datetime.datetime(1900, 1, 1)


def __date_within_period(date, start_date=None, end_date=None):
    if start_date is None:
        # The starting month of Common Crawl.
        start_date = __common_crawl_start_date
    if end_date is None:
        # Until now.
        end_date = datetime.datetime.today()
    return start_date <= date < end_date

{'fraction_non_alphanumeric': 0.08683374514648783, 'fraction_numerical': 0.012707377338510413, 'mean_word_length': 3.8934945308002304, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2545658', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2700, 'n_words': 628}