<commit_before>#!/usr/bin/env python
"""
MODULE:    carookee

AUTHOR(S): <NAME> <<EMAIL>>

PURPOSE:   Script to dump the textual content of a carookee forum

COPYRIGHT: (C) 2013 <NAME>

           This Program is free software under the Apache
           License Version 2.0.
"""
from requests import Session
import lxml.html
import re


def _get_pagecount(html):
    """look if there is more then on page of topics or posts"""
    nums = 0
    base = ""
    for ele in html.iter('a'):
        href = ele.get('href', '')
        if href.endswith('#bot'):
            base, nums = href[:-4].split('?p=')
            base += "?p="
            break
    return base, int(nums)

def get_topics(html):
    return [(e.get('href'), e.text)
            for e in html.iter('a')
            if e.get('class', '') == 'topictitle']

class Carookee(Session):
    """requests Session class extended by login and dump
       methods for carookee's forum website"""

    DOMAIN = "http://www.carookee.net"

    RXP_DATE = re.compile(r'''
        ^[^:]+:\s+
        ([0-3][0-9]\.[01][0-9]\.\d{2},\s+[0-2][0-9]:[0-5][0-9])
        \s+[^:]+:\s+(\S.*)
        ''', re.X | re.U)

    def __init__(self, forum, *args, **kwargs):
        Session.__init__(self, *args, **kwargs)
        self.forum = forum


    def get_html(self, url):
        """get the url and parse it with lxml.html
           returns: lxml.html Elementtree"""
        res = self.get(url)
        html = lxml.html.fromstring(res.content)
        html.make_links_absolute(self.DOMAIN)
        return html


    def login(self, username, password):
        html = self.get_html ("%s/forum/%s/login" % (self.DOMAIN, self.forum))
        vals = {}
        for elem in html.iter('input'):
            vals[elem.name] = elem.value
        vals['username'] = username
        vals['password'] = password
        action = html.forms[0].action
        res = self.post(action, vals)
        return res.ok


    def list_subforums(self):
        html = self.get_html("%s/forum/%s" % (self.DOMAIN, self.forum))
        subf = []
        for elem in html.iter('a'):
            if elem.get('class', '') == 'forumlink':
                subf.append((elem.get('href'), elem.text))
        return subf


    def list_topics(self, sflink):


        html = self.get_html(sflink)
        topics = []
        topics.extend(get_topics(html))

        # look for more then on page of topics
        base, nums = _get_pagecount(html)
        for i in range (2, nums + 1):
            html = self.get_html("%s%i" %(base, i))
            topics.extend(get_topics(html))
        return topics


    def get_topic(self, tplink):
        html = self.get_html(tplink)
        pages = []
        posts = []
        pages.append(html)

        base, nums = _get_pagecount(html)
        for i in range (2, nums + 1):
            pages.append(self.get_html("%s%i" %(base, i)))

        for page in pages:
            # find table with the content
            for row in page.findall(".//table[@class='forumline']./tr"):
                try:
                    author = row.find(".//span[@class='name']").text_content()
                except:
                    author = ""

                try:
                    detail = row.find(".//span[@class='postdetails'][1]"
                            ).text_content()
                    date, subject = self.RXP_DATE.search(detail).groups()

                except:
                    date = ""
                    subject = ""

                try:
                    content = row.find(".//span[@class='postbody']"
                            ).text_content()
                except:
                    content = ""

                if content:
                    posts.append({
                        'author': author,
                        'date': date,
                        'subject': subject,
                        'content': content,
                        })
        return posts


if __name__ == '__main__':
    import sys
    import json

    forum, username, password = sys.argv[1:]

    c = Carookee(forum)
    c.login(username, password)

    sforums = c.list_subforums()

    print "["

    for link, fname in sforums:
        topics = c.list_topics(link)
        for link, tname in topics:
            t = c.get_topic(link)
            json.dump({
                'Forum': fname,
                'Topic': tname,
                'Thread': t,
                }, sys.stdout, indent=4)
            print ","
    print "{}]"
<commit_msg>Cleanups in code formatting (pep8)<commit_after>#!/usr/bin/env python
"""
MODULE:    carookee

AUTHOR(S): <NAME> <<EMAIL>>

PURPOSE:   Script to dump the textual content of a carookee forum

COPYRIGHT: (C) 2013 <NAME>

           This Program is free software under the Apache
           License Version 2.0.
"""
from requests import Session
import lxml.html
import re


def _get_pagecount(html):
    """look if there is more then on page of topics or posts"""
    nums = 0
    base = ""
    for ele in html.iter('a'):
        href = ele.get('href', '')
        if href.endswith('#bot'):
            base, nums = href[:-4].split('?p=')
            base += "?p="
            break
    return base, int(nums)


def get_topics(html):
    return [(e.get('href'), e.text)
            for e in html.iter('a')
            if e.get('class', '') == 'topictitle']


class Carookee(Session):
    """requests Session class extended by login and dump
       methods for carookee's forum website"""

    DOMAIN = "http://www.carookee.net"

    RXP_DATE = re.compile(r'''
        ^[^:]+:\s+
        ([0-3][0-9]\.[01][0-9]\.\d{2},\s+[0-2][0-9]:[0-5][0-9])
        \s+[^:]+:\s+(\S.*)
        ''', re.X | re.U)

    def __init__(self, forum, *args, **kwargs):
        Session.__init__(self, *args, **kwargs)
        self.forum = forum

    def get_html(self, url):
        """get the url and parse it with lxml.html
           returns: lxml.html Elementtree"""
        res = self.get(url)
        html = lxml.html.fromstring(res.content)
        html.make_links_absolute(self.DOMAIN)
        return html

    def login(self, username, password):
        html = self.get_html("%s/forum/%s/login" % (self.DOMAIN, self.forum))
        vals = {}
        for elem in html.iter('input'):
            vals[elem.name] = elem.value
        vals['username'] = username
        vals['password'] = password
        action = html.forms[0].action
        res = self.post(action, vals)
        return res.ok

    def list_subforums(self):
        html = self.get_html("%s/forum/%s" % (self.DOMAIN, self.forum))
        subf = []
        for elem in html.iter('a'):
            if elem.get('class', '') == 'forumlink':
                subf.append((elem.get('href'), elem.text))
        return subf

    def list_topics(self, sflink):

        html = self.get_html(sflink)
        topics = []
        topics.extend(get_topics(html))

        # look for more then on page of topics
        base, nums = _get_pagecount(html)
        for i in range(2, nums + 1):
            html = self.get_html("%s%i" % (base, i))
            topics.extend(get_topics(html))
        return topics

    def get_topic(self, tplink):
        html = self.get_html(tplink)
        pages = []
        posts = []
        pages.append(html)

        base, nums = _get_pagecount(html)
        for i in range(2, nums + 1):
            pages.append(self.get_html("%s%i" % (base, i)))

        for page in pages:
            # find table with the content
            for row in page.findall(".//table[@class='forumline']./tr"):
                try:
                    author = row.find(".//span[@class='name']").text_content()
                except:
                    author = ""

                try:
                    detail = row.find(".//span[@class='postdetails'][1]"
                                      ).text_content()
                    date, subject = self.RXP_DATE.search(detail).groups()

                except:
                    date = ""
                    subject = ""

                try:
                    content = row.find(".//span[@class='postbody']"
                                       ).text_content()
                except:
                    content = ""

                if content:
                    posts.append(
                        {
                            'author': author,
                            'date': date,
                            'subject': subject,
                            'content': content,
                        })
        return posts


if __name__ == '__main__':
    import sys
    import json

    forum, username, password = sys.argv[1:]

    c = Carookee(forum)
    c.login(username, password)

    sforums = c.list_subforums()

    print "["

    for link, fname in sforums:
        topics = c.list_topics(link)
        for link, tname in topics:
            t = c.get_topic(link)
            json.dump(
                {
                    'Forum': fname,
                    'Topic': tname,
                    'Thread': t,
                }, sys.stdout, indent=4)
            print ","
    print "{}]"

{'fraction_non_alphanumeric': 0.14068814855270345, 'fraction_numerical': 0.007318405243036592, 'mean_word_length': 1.5412156536219817, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2296685', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2713, 'n_words': 661}