<commit_before>from __future__ import unicode_literals

__author__ = 'lorenzo'

import os
from bs4 import BeautifulSoup
import feedparser

import webapp2

from datastore.models import WebResource
from config.config import _DEBUG

# Adapted from http://tuhrig.de/writing-an-online-scraper-on-google-app-engine-python/


class Scrawler(webapp2.RequestHandler):
    """
    A very basic crawler for RSS links
    """

    def get(self):
        """
        Handler for the cronjob: /cron/startcrawling
        :return:
        """
        self.store_feeds()

    def load_links(self):
        """
        Loads RSS links from a local file. They are in an XML file with tag <outline/>
        :return: A list of URLs of RSS-feeds
        """
        feeds_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts', 'files', 'newsfox.xml')
        with open(feeds_file) as f:
            markup = f.read()
        body = BeautifulSoup(markup, "lxml-xml").body
        outlines = body.find_all('outline')
        links = []
        for o in outlines:
            try:
                links.append(str(o['xmlUrl']))
            except KeyError as e:
                pass
            except Exception as e:
                raise e
        return links

    def read_feed(self, ln):
        """
        Parse a link with feedparser library.
        :param ln: a link to a RSS-feed
        :return: a list of dictionaries containing news from the feed
        """
        feed = feedparser.parse(ln)

        if feed and feed["entries"]:
                return feed["entries"]
        else:
            print ValueError('No links. Or cannot parse them in: ' + str(feed))
            return None

    def store_feeds(self):
        for l in self.load_links():
            entries = self.read_feed(l)
            if entries:
                for entry in entries:
                    query = WebResource.query().filter(WebResource.url == entry["link"])
                    if query.count() == 0:
                        print "STORING: " + entry["link"]
                        try:
                            if 'summary' in entry:
                                s, t = BeautifulSoup(entry['summary'], "lxml"), BeautifulSoup(entry['title'], "lxml")
                                entry['summary'], entry['title'] = s.get_text(), t.get_text()
                            i = WebResource.store_feed(entry)
                            print "STORED: " + str(i)
                        except Exception as e:
                            print "Cannot Store: " + str(e) + entry['link']


application = webapp2.WSGIApplication([
    webapp2.Route('/cron/startcrawling', Scrawler),
], debug=_DEBUG)<commit_msg>Scrawler: minor change<commit_after>from __future__ import unicode_literals

__author__ = 'lorenzo'

import os
from bs4 import BeautifulSoup
import feedparser

import webapp2

from datastore.models import WebResource
from config.config import _DEBUG

# Adapted from http://tuhrig.de/writing-an-online-scraper-on-google-app-engine-python/


class Scrawler(webapp2.RequestHandler):
    """
    A very basic crawler for RSS links
    """

    def get(self):
        """
        Handler for the cronjob: /cron/startcrawling
        :return:
        """
        self.store_feeds()

    def load_links(self):
        """
        Loads RSS links from a local file. They are in an XML file with tag <outline/>
        :return: A list of URLs of RSS-feeds
        """
        feeds_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts', 'files', 'newsfox.xml')
        with open(feeds_file) as f:
            markup = f.read()
        body = BeautifulSoup(markup, "lxml-xml").body
        outlines = body.find_all('outline')
        links = []
        for o in outlines:
            try:
                links.append(str(o['xmlUrl']))
            except KeyError as e:
                pass
            except Exception as e:
                raise e
        return links

    def read_feed(self, ln):
        """
        Parse a link with feedparser library.
        :param ln: a link to a RSS-feed
        :return: a list of dictionaries containing news from the feed
        """
        feed = feedparser.parse(ln)

        if feed and feed["entries"]:
                return feed["entries"]
        else:
            print ValueError('No links. Or cannot parse them in: ' + str(feed))
            return None

    def store_feeds(self):
        for l in self.load_links():
            entries = self.read_feed(l)
            if entries:
                for entry in entries:
                    query = WebResource.query().filter(WebResource.url == entry["link"])
                    if query.count() == 0:
                        print "STORING: " + entry["link"]
                        try:
                            if 'summary' in entry:
                                s, t = BeautifulSoup(entry['summary'], "lxml"), BeautifulSoup(entry['title'], "lxml")
                                entry['summary'], entry['title'] = s.get_text(), t.get_text()
                            else:
                                t = BeautifulSoup(entry['title'], "lxml")
                                entry['summary'], entry['title'] = None , t.get_text()
                            i = WebResource.store_feed(entry)
                            print "STORED: " + str(i)
                        except Exception as e:
                            print "Cannot Store: " + str(e) + entry['link']


application = webapp2.WSGIApplication([
    webapp2.Route('/cron/startcrawling', Scrawler),
], debug=_DEBUG)
{'fraction_non_alphanumeric': 0.10668563300142248, 'fraction_numerical': 0.0021337126600284497, 'mean_word_length': 1.6595744680851063, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471093', 'n_tokens_mistral': 1594, 'n_tokens_neox': 1530, 'n_words': 431}