<commit_before>#!/usr/bin/env python
"""
Provides client interface to Plans.

"""

import sys
if sys.version_info >= (3,3):
    from urllib.parse import urlparse, parse_qsl
    from http.cookiejar import LWPCookieJar
    from html.parser import HTMLParser
elif sys.version_info < (3,):
    from urlparse import urlparse, parse_qsl
    from cookielib import LWPCookieJar
    from HTMLParser import HTMLParser
    str = unicode


import json
import re
import bs4
import requests

from .util import plans_md5, convert_endings

class PlansError(Exception):
    """Exception raised when there is an error talking to plans."""
    pass


class PlansPageParser(HTMLParser):
    """HTML parser for GrinnellPlans pages."""

    def handle_starttag(self, tag, attrs):
        if tag == 'body':
            # parse out id of body tag
            # (can use to identify page)
            self.page_id = dict(attrs).get('id', None)
            if self.page_id is None:
                # old interface <body> tag has no attrs
                raise PlansError('Postmodern interface required')
        if tag == 'a':
            # parse out username to see who we're logged in as.
            # amazingly, the only place this reliably appears
            # is in the bug report link at the bottom of every page.
            attrs = dict(attrs)
            href = urlparse(attrs.get('href'))
            if (href.netloc == 'code.google.com'
                    and href.path == '/p/grinnellplans/issues/entry'):
                # if this is the bug submission link
                query = parse_qsl(href.query)
                comment = dict(query)['comment']
                # find username in submission content using brackets
                start, stop = comment.index('['), comment.index(']')
                self.username = comment[start + 1:stop]

# -------------------------------------------
#              PLANS SCRAPEY-I
# get it? like "api" except... oh, never mind
# -------------------------------------------


class PlansConnection(object):
    """
    Encapsulates an active login to plans.

    """

    def __init__(self, cookiejar=None,
                 base_url="http://www.grinnellplans.com"):
        """
        Create a new plans connection.

        Optional keyword arguments:

        cookiejar -- an existing cookielib.CookieJar to store
                     credentials in.
        base_url --  URL at which to access plans, no trailing slash.

        """
        self.base_url = base_url
        if cookiejar is None:
            self.cookiejar = LWPCookieJar()
        else:
            self.cookiejar = cookiejar
        self.session = requests.Session()
        self.session.cookies = self.cookiejar
        self.parser = PlansPageParser()
        self.username = None

    def _get_page(self, name, get=None, post=None):
        """
        Retrieve an HTML page from plans.

        """
        method = 'GET' if post is None else 'POST'
        url = '/'.join((self.base_url, name))
        req = requests.Request(method, url, params=get, data=post)
        prepped = self.session.prepare_request(req)
        try:
            handle = self.session.send(prepped)
        except requests.exceptions.ConnectionError:
            err = "Check your internet connection. Plans could also be down."
            raise PlansError(err)
        return handle

    def _parse_message(self, soup):
        """
        Scrape details from an infomessage or alertmessage div.

        Returns a dictionary of the message parameters.

        """
        kind, = soup.attrs[u'class']
        title = soup.findChild().text
        body = ''.join(t.text for t in soup.findChildren()[1:])
        message = dict(kind=kind, title=title, body=body)
        for val in message.values():
            assert type(val) == str
        return message

    def _canonicalize_plantext(self, plan):
        """
        Modify reserialized plan text to match what was served.

        For consistency, we want to return plan text *exactly* how it
        is formatted when served. However, variants of certain tags
        (e.g. <br> vs <br/>) are syntactically equivalent in HTML, and
        may be interchanged when parsed and reserialized.
<commit_msg>Use https by default, and verify the certificate<commit_after>#!/usr/bin/env python
"""
Provides client interface to Plans.

"""

import sys
if sys.version_info >= (3,3):
    from urllib.parse import urlparse, parse_qsl
    from http.cookiejar import LWPCookieJar
    from html.parser import HTMLParser
elif sys.version_info < (3,):
    from urlparse import urlparse, parse_qsl
    from cookielib import LWPCookieJar
    from HTMLParser import HTMLParser
    str = unicode


import json
import re
import bs4
import requests

from .util import plans_md5, convert_endings

class PlansError(Exception):
    """Exception raised when there is an error talking to plans."""
    pass


class PlansPageParser(HTMLParser):
    """HTML parser for GrinnellPlans pages."""

    def handle_starttag(self, tag, attrs):
        if tag == 'body':
            # parse out id of body tag
            # (can use to identify page)
            self.page_id = dict(attrs).get('id', None)
            if self.page_id is None:
                # old interface <body> tag has no attrs
                raise PlansError('Postmodern interface required')
        if tag == 'a':
            # parse out username to see who we're logged in as.
            # amazingly, the only place this reliably appears
            # is in the bug report link at the bottom of every page.
            attrs = dict(attrs)
            href = urlparse(attrs.get('href'))
            if (href.netloc == 'code.google.com'
                    and href.path == '/p/grinnellplans/issues/entry'):
                # if this is the bug submission link
                query = parse_qsl(href.query)
                comment = dict(query)['comment']
                # find username in submission content using brackets
                start, stop = comment.index('['), comment.index(']')
                self.username = comment[start + 1:stop]

# -------------------------------------------
#              PLANS SCRAPEY-I
# get it? like "api" except... oh, never mind
# -------------------------------------------


class PlansConnection(object):
    """
    Encapsulates an active login to plans.

    """

    def __init__(self, cookiejar=None,
                 base_url="https://www.grinnellplans.com"):
        """
        Create a new plans connection.

        Optional keyword arguments:

        cookiejar -- an existing cookielib.CookieJar to store
                     credentials in.
        base_url --  URL at which to access plans, no trailing slash.

        """
        self.base_url = base_url
        if cookiejar is None:
            self.cookiejar = LWPCookieJar()
        else:
            self.cookiejar = cookiejar
        self.session = requests.Session()
        self.session.cookies = self.cookiejar
        self.parser = PlansPageParser()
        self.username = None

    def _get_page(self, name, get=None, post=None):
        """
        Retrieve an HTML page from plans.

        """
        method = 'GET' if post is None else 'POST'
        url = '/'.join((self.base_url, name))
        req = requests.Request(method, url, params=get, data=post)
        prepped = self.session.prepare_request(req)
        try:
            handle = self.session.send(prepped,
                                       verify=url.startswith('https'))
        except requests.exceptions.ConnectionError:
            err = "Check your internet connection. Plans could also be down."
            raise PlansError(err)
        return handle

    def _parse_message(self, soup):
        """
        Scrape details from an infomessage or alertmessage div.

        Returns a dictionary of the message parameters.

        """
        kind, = soup.attrs[u'class']
        title = soup.findChild().text
        body = ''.join(t.text for t in soup.findChildren()[1:])
        message = dict(kind=kind, title=title, body=body)
        for val in message.values():
            assert type(val) == str
        return message

    def _canonicalize_plantext(self, plan):
        """
        Modify reserialized plan text to match what was served.

        For consistency, we want to return plan text *exactly* how it
        is formatted when served. However, variants of certain tags
        (e.g. <br> vs <br/>) are syntactically equivalent in HTML, and
        may be interchanged when parsed and reserialized.

{'fraction_non_alphanumeric': 0.1092896174863388, 'fraction_numerical': 0.0016277177072433438, 'mean_word_length': 1.9549982823771899, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3659995', 'n_tokens_mistral': 2357, 'n_tokens_neox': 2243, 'n_words': 776}