<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

import datetime
import base64
import urlparse
import uuid
from urllib import quote

import pyamf
from pyamf import remoting
from pyamf.flex import messaging
from amcatscraping.article import Article

from amcatscraping.scraper import LoginMixin, PropertyCheckMixin, UnitScraper, DateRangeScraper
from amcatscraping.tools import parse_form



# Login page
LOGINURL = "https://caps.{domain}/service/login?service=http%3A%2F%2Fkrant.{domain}%2F%3Fpaper%3D{paper_id}%26zone%3D{regio_code}"
AUTHURL = "https://caps.{domain}/service/validate?service="
SAVEURL = "http://krant.%(domain)s/ipaper-online/saveLoginHistory?zone=%(regio_code)s&pubId=%(main_id)s&method=CAPS&paperId=%(paper_id)s&login=%(username)s"
TICKET = "SSO_{self.paper_id}:{ticket_url.scheme}://{ticket_url.netloc}/@{ticket}"

# AMF API page
AMFURL = "http://krant.{domain}/ipaper-online/blaze/amf"

# Text displayed when login successfull
LOGIN_SUCCESS = "Log In Successful"

# Generate a valid uuid for an AMF object
uuid4 = lambda: str(uuid.uuid4()).upper()

# Convienience function
def get_pubdate(paper):
    pd = dict([(k, int(v)) for k,v in paper['pubDate'].items()])
    pd['month'] += 1
    return datetime.date(**pd)


class PCMScraper(LoginMixin, PropertyCheckMixin, UnitScraper, DateRangeScraper):
    paper_id = None
    context_id = None
    domain = None

    def __init__(self, *args, **kwargs):
        self.client_id = uuid4()
        self.headers = {
            'DSMessagingVersion':  1,
            'DSId': 'nil'
        }

        super(PCMScraper, self).__init__(*args, **kwargs)


    def login(self, username, password):
        """
        Parse login form and fill in wanted parts
        """
        # Get latest paper id
        latest = self._get_latest()
        latest = latest[(sorted(latest.keys())[-1])]
        paper_id = int(latest['paperId'])

        # Build url
        url = LOGINURL.format(paper_id=paper_id,
                              regio_code=self.context_id,
                              domain=self.domain)

        # Login
        login_page = self.session.get_html(url)
        form = parse_form(login_page)
        form['username'] = username
        form['password'] = password
        login_page = self.session.post(url, data=form)

        # Resolve ticket_url and save it
        ticket_url = urlparse.urlparse(login_page.url)
        ticket_url_params = dict(urlparse.parse_qsl(ticket_url.query))
        
        if 'ticket' not in ticket_url_params:
            return False

        # Handshake server
        com = self.create_message(messaging.CommandMessage, operation=5)
        req = self.create_request(com)
        env = self.create_envelope(req)
        res = self.apiget(env).bodies[0][1]

        self.headers.update(res.body.headers)

        # Send AMF Auth message to server
        ticket = ticket_url_params['ticket']
        ticket = TICKET.format(**locals())

        com = self.create_message(messaging.CommandMessage, operation=8)
        com.destination = 'auth'
        com.body = base64.b64encode(ticket)

        com.headers["DSEndpoint"] = "_IPaperOnlineServiceLocator_AMFChannel1"
        com.correlationId = ""

        env = self.create_envelope(self.create_request(com))
        res = self.apiget(env)

        return "ErrorMessage" not in str(res)

    def get_units(self):
        latest = self._get_latest()
        for date in self.dates:
            page = latest.get(date)
            if page is None:
                continue

            pid = int(page['paperId'])
            paper = self._get_paper(pid, date)
            for index in paper:<commit_msg>Retry PCM login if logging in fails<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

import datetime
import base64
import urlparse
import uuid
import time
from urllib import quote
import logging
log = logging.getLogger(__name__)

import pyamf
from pyamf import remoting
from pyamf.flex import messaging
from amcatscraping.article import Article

from amcatscraping.scraper import LoginMixin, PropertyCheckMixin, UnitScraper, DateRangeScraper
from amcatscraping.tools import parse_form



# Login page
LOGINURL = "https://caps.{domain}/service/login?service=http%3A%2F%2Fkrant.{domain}%2F%3Fpaper%3D{paper_id}%26zone%3D{regio_code}"
AUTHURL = "https://caps.{domain}/service/validate?service="
SAVEURL = "http://krant.%(domain)s/ipaper-online/saveLoginHistory?zone=%(regio_code)s&pubId=%(main_id)s&method=CAPS&paperId=%(paper_id)s&login=%(username)s"
TICKET = "SSO_{self.paper_id}:{ticket_url.scheme}://{ticket_url.netloc}/@{ticket}"

# AMF API page
AMFURL = "http://krant.{domain}/ipaper-online/blaze/amf"

# Text displayed when login successfull
LOGIN_SUCCESS = "Log In Successful"

# Generate a valid uuid for an AMF object
uuid4 = lambda: str(uuid.uuid4()).upper()

# Convienience function
def get_pubdate(paper):
    pd = dict([(k, int(v)) for k,v in paper['pubDate'].items()])
    pd['month'] += 1
    return datetime.date(**pd)


class PCMScraper(LoginMixin, PropertyCheckMixin, UnitScraper, DateRangeScraper):
    paper_id = None
    context_id = None
    domain = None

    def __init__(self, *args, **kwargs):
        self.client_id = uuid4()
        self.headers = {
            'DSMessagingVersion':  1,
            'DSId': 'nil'
        }

        super(PCMScraper, self).__init__(*args, **kwargs)


    def login(self, username, password):
        """
        Parse login form and fill in wanted parts
        """
        # Get latest paper id
        latest = self._get_latest()
        latest = latest[(sorted(latest.keys())[-1])]
        paper_id = int(latest['paperId'])

        # Build url
        url = LOGINURL.format(paper_id=paper_id,
                              regio_code=self.context_id,
                              domain=self.domain)

        # Login
        login_page = self.session.get_html(url)
        form = parse_form(login_page)
        form['username'] = username
        form['password'] = password

        
        def _get_ticket(form):
            # Get ticket from login form
            login_page = self.session.post(url, data=form)
            ticket_url = urlparse.urlparse(login_page.url)
            ticket_url_params = dict(urlparse.parse_qsl(ticket_url.query))
            return ticket_url, ticket_url_params.get('ticket')

        for i in range(20):
            ticket_url, ticket = _get_ticket(form)
            if ticket:
                break
            else:
                log.warn("Login attempt {i} failed, retrying in 5 seconds".format(**locals()))
                time.sleep(5)
                
        if not ticket:
            return False

        # Handshake server
        com = self.create_message(messaging.CommandMessage, operation=5)
        req = self.create_request(com)
        env = self.create_envelope(req)
        res = self.apiget(env).bodies[0][1]

        self.headers.update(res.body.headers)

        # Send AMF Auth message to server
        ticket = TICKET.format(**locals())

        com = self.create_message(messaging.CommandMessage, operation=8)
        com.destination = 'auth'
        com.body = base64.b64encode(ticket)

        com.headers["DSEndpoint"] = "_IPaperOnlineServiceLocator_AMFChannel1"
        com.correlationId = ""

        env = self.create_envelope(self.create_request(com))
        res = self.apiget(env)

        return "ErrorMessage" not in str(res)

    def get_units(self):
        latest = self._get_latest()
        for date in self.dates:
            page = latest.get(date)
            if page is None:
                continue

            pid = int(page['paperId'])
            paper = self._get_paper(pid, date)
            for index in paper:
{'fraction_non_alphanumeric': 0.1241107479330898, 'fraction_numerical': 0.005575850797923477, 'mean_word_length': 2.193063228974831, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5817861', 'n_tokens_mistral': 2934, 'n_tokens_neox': 2820, 'n_words': 732}