<commit_before>import datetime
import hashlib
import os
import threading
try:
    import Queue
except ImportError:
    import queue as Queue
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from flask import Flask, g, jsonify, render_template, request, url_for
from libearth.codecs import Rfc3339
from libearth.compat import binary
from libearth.crawler import CrawlError, crawl
from libearth.parser.autodiscovery import autodiscovery, FeedUrlNotFoundError
from libearth.repository import FileSystemRepository
from libearth.session import Session
from libearth.stage import Stage
from libearth.subscribe import Category, Subscription, SubscriptionList
from libearth.tz import now
from werkzeug.exceptions import HTTPException

from .wsgi import MethodRewriteMiddleware

app = Flask(__name__)
app.wsgi_app = MethodRewriteMiddleware(app.wsgi_app)

crawling_queue = Queue.Queue()


app.config.update(
    ALLFEED='All Feeds',
    SESSION_ID=None,
    PAGE_SIZE=20,
)


def crawl_category():
    while True:
        category_id, feed_id = crawling_queue.get()
        ids = {}
        cursor = Cursor(category_id)

        if not feed_id:
            for sub in cursor.recursive_subscriptions:
                ids[sub.feed_uri] = sub.feed_id
        else:
            for sub in cursor.recursive_subscriptions:
                if sub.feed_id == feed_id:
                    ids[sub.feed_uri] = sub.feed_id
                    break

        iterator = iter(crawl(ids.keys(), app.config['CRAWLER_THREAD']))
        while True:
            try:
                feed_url, feed_data, crawler_hints = next(iterator)
                with get_stage() as stage:
                    stage.feeds[ids[feed_url]] = feed_data
            except CrawlError:
                continue
            except StopIteration:
                break
        crawling_queue.task_done()


threading.Thread(target=crawl_category)


class IteratorNotFound(ValueError):
    """Rise when the iterator does not exist"""


class JsonException(HTTPException):
    """Base exception to return json response when raised.
    Exceptions inherit this class must declare `error` and `message`.

    """
    def get_response(self, environ):
        r = jsonify(error=self.error, message=self.message)
        r.status_code = 404
        return r


class InvalidCategoryID(ValueError, JsonException):
    """Rise when the category ID is not valid."""

    error = 'category-id-invalid'<commit_msg>start thread, app.config<commit_after>import datetime
import hashlib
import os
import threading
try:
    import Queue
except ImportError:
    import queue as Queue
try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from flask import Flask, g, jsonify, render_template, request, url_for
from libearth.codecs import Rfc3339
from libearth.compat import binary
from libearth.crawler import CrawlError, crawl
from libearth.parser.autodiscovery import autodiscovery, FeedUrlNotFoundError
from libearth.repository import FileSystemRepository
from libearth.session import Session
from libearth.stage import Stage
from libearth.subscribe import Category, Subscription, SubscriptionList
from libearth.tz import now
from werkzeug.exceptions import HTTPException

from .wsgi import MethodRewriteMiddleware

app = Flask(__name__)
app.wsgi_app = MethodRewriteMiddleware(app.wsgi_app)

crawling_queue = Queue.Queue()


app.config.update(
    ALLFEED='All Feeds',
    SESSION_ID=None,
    PAGE_SIZE=20,
    CRAWLER_THREAD=4,
)


def crawl_category():
    while True:
        category_id, feed_id = crawling_queue.get()
        ids = {}
        cursor = Cursor(category_id)

        if not feed_id:
            for sub in cursor.recursive_subscriptions:
                ids[sub.feed_uri] = sub.feed_id
        else:
            for sub in cursor.recursive_subscriptions:
                if sub.feed_id == feed_id:
                    ids[sub.feed_uri] = sub.feed_id
                    break

        iterator = iter(crawl(ids.keys(), app.config['CRAWLER_THREAD']))
        while True:
            try:
                feed_url, feed_data, crawler_hints = next(iterator)
                with get_stage() as stage:
                    stage.feeds[ids[feed_url]] = feed_data
            except CrawlError:
                continue
            except StopIteration:
                break
        crawling_queue.task_done()


threading.Thread(target=crawl_category).start()


class IteratorNotFound(ValueError):
    """Rise when the iterator does not exist"""


class JsonException(HTTPException):
    """Base exception to return json response when raised.
    Exceptions inherit this class must declare `error` and `message`.

    """
    def get_response(self, environ):
        r = jsonify(error=self.error, message=self.message)
        r.status_code = 404
        return r


class InvalidCategoryID(ValueError, JsonException):
    """Rise when the category ID is not valid."""

    error = 'category-id-invalid'
{'fraction_non_alphanumeric': 0.07363799640790261, 'fraction_numerical': 0.004589902215126721, 'mean_word_length': 2.891304347826087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4410883', 'n_tokens_mistral': 1501, 'n_tokens_neox': 1405, 'n_words': 400}