<commit_before>###########################################################################
# (C) Vrije Universiteit, Amsterdam (the Netherlands)                     #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
from __future__ import print_function, unicode_literals

from collections import defaultdict
import datetime
import logging

from amcatscraping.scraper import UnitScraper, DateRangeScraper, PropertyCheckMixin
from amcatscraping.tools import html2text, read_date


ARCHIEF_URL = "http://www.geenstijl.nl/mt/archieven/maandelijks/%Y/%m/"


log = logging.getLogger(__name__)


def _parse_comment_footer(footer):
    author, date, time = footer.rsplit("|", 2)

    day, month, year = date.split("-")
    hour, minute = time.split(":")

    timestamp = datetime.datetime(int(year)+2000, int(month), int(day), int(hour), int(minute))

    return author, timestamp


class GeenstijlScraper(PropertyCheckMixin, UnitScraper, DateRangeScraper):
    medium = "Geenstijl"

    def __init__(self, **kwargs):
        super(GeenstijlScraper, self).__init__(**kwargs)
        self.articles = defaultdict(set)
        self.session.encoding = "iso-8859-1"

    def get_units(self):
        for date in self.dates:
            day_string = date.strftime("%d-%m-%y")

            if day_string not in self.articles:
                self._get_archive(date)

            for article_url in self.articles[day_string]:
                yield date, article_url

    def _get_archive(self, date):
        """Fill article-link cache with all articles written in the same month as 'date'"""
        doc = self.session.get_html(date.strftime(ARCHIEF_URL))

        for li in doc.cssselect("li"):
            if not li.text:
                continue

            link = li.cssselect("a")[0].get("href")
            self.articles[li.text.strip()].add(link.strip())
<commit_msg>Added IGNORE_URLS for broken pages (Geenstijl)<commit_after>###########################################################################
# (C) Vrije Universiteit, Amsterdam (the Netherlands)                     #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
from __future__ import print_function, unicode_literals

from collections import defaultdict
import datetime
import logging

from amcatscraping.scraper import UnitScraper, DateRangeScraper, PropertyCheckMixin
from amcatscraping.tools import html2text, read_date


ARCHIEF_URL = "http://www.geenstijl.nl/mt/archieven/maandelijks/%Y/%m/"

# These urls are deaud.
IGNORE_URLS = {
    "http://www.geenstijl.nl/mt/archieven/2012/10/man_flasht_mes_op_groest.html",
}


log = logging.getLogger(__name__)


def _parse_comment_footer(footer):
    author, date, time = footer.rsplit("|", 2)

    day, month, year = date.split("-")
    hour, minute = time.split(":")

    timestamp = datetime.datetime(int(year)+2000, int(month), int(day), int(hour), int(minute))

    return author, timestamp


class GeenstijlScraper(PropertyCheckMixin, UnitScraper, DateRangeScraper):
    medium = "Geenstijl"

    def __init__(self, **kwargs):
        super(GeenstijlScraper, self).__init__(**kwargs)
        self.articles = defaultdict(set)
        self.session.encoding = "iso-8859-1"

    def get_units(self):
        for date in self.dates:
            day_string = date.strftime("%d-%m-%y")

            if day_string not in self.articles:
                self._get_archive(date)

            for article_url in self.articles[day_string]:
                if article_url not in IGNORE_URLS:
                    yield date, article_url

    def _get_archive(self, date):
        """Fill article-link cache with all articles written in the same month as 'date'"""
        doc = self.session.get_html(date.strftime(ARCHIEF_URL))

        for li in doc.cssselect("li"):
            if not li.text:
                continue

            link = li.cssselect("a")[0].get("href")
            self.articles[li.text.strip()].add(link.strip())

{'fraction_non_alphanumeric': 0.1323975115648429, 'fraction_numerical': 0.005104482373584304, 'mean_word_length': 2.0720235178833906, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '399789', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1564, 'n_words': 482}