<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from random import choice
from datetime import datetime, timedelta

from tornado import gen
from tornado.web import asynchronous

from holmes.handlers import BaseHandler
from holmes.models import Page, Settings


class NextJobHandler(BaseHandler):

    @gen.coroutine
    @asynchronous
    def get(self):
        last_week = datetime.now() - timedelta(seconds=self.application.config.REVIEW_EXPIRATION_IN_SECONDS)

        settings = Settings.instance(self.db)

        pages_in_need_of_review = self.db.query(Page.uuid, Page.url, Page.score, Page.last_review_date) \
            .filter(Page.last_review == None) \
            .order_by(Page.score.desc())[:200]

        if len(pages_in_need_of_review) == 0:
            pages_in_need_of_review = self.db.query(Page.uuid, Page.url, Page.score, Page.last_review_date) \
                .filter(Page.last_review != None) \
                .order_by(Page.score.desc())[:200]

            if len(pages_in_need_of_review) == 0:
                if settings.lambda_score > 0:
                    self.update_pages_score_by(settings, settings.lambda_score, self.handle_scores_updated)
                    return

                self.write('')
                self.finish()
                return

        if settings.lambda_score > pages_in_need_of_review[0].score:
            self.update_pages_score_by(settings, settings.lambda_score, self.handle_scores_updated)
            return

        page = choice(pages_in_need_of_review)

        if page.last_review_date is not None and page.last_review_date > last_week:
            self.write('')
            self.finish()
            return

        self.write_json({
            'page': str(page.uuid),
            'url': page.url,
            'score': page.score<commit_msg>Minor fix with last review date to stop from being stalled<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from random import choice
from datetime import datetime, timedelta

from tornado import gen
from tornado.web import asynchronous

from holmes.handlers import BaseHandler
from holmes.models import Page, Settings


class NextJobHandler(BaseHandler):

    @gen.coroutine
    @asynchronous
    def get(self):
        expired_time = datetime.now() - timedelta(seconds=self.application.config.REVIEW_EXPIRATION_IN_SECONDS)

        settings = Settings.instance(self.db)

        pages_in_need_of_review = self.db.query(Page.uuid, Page.url, Page.score, Page.last_review_date) \
            .filter(Page.last_review_date == None) \
            .order_by(Page.score.desc())[:200]

        if len(pages_in_need_of_review) == 0:
            pages_in_need_of_review = self.db.query(Page.uuid, Page.url, Page.score, Page.last_review_date) \
                .filter(Page.last_review_date <= expired_time) \
                .order_by(Page.score.desc())[:200]

            if len(pages_in_need_of_review) == 0:
                if settings.lambda_score > 0:
                    self.update_pages_score_by(settings, settings.lambda_score, self.handle_scores_updated)
                    return

                self.write('')
                self.finish()
                return

        if settings.lambda_score > pages_in_need_of_review[0].score:
            self.update_pages_score_by(settings, settings.lambda_score, self.handle_scores_updated)
            return

        page = choice(pages_in_need_of_review)

        if page.last_review_date is not None and page.last_review_date > expired_time:
            self.write('')
            self.finish()
            return

        self.write_json({
            'page': str(page.uuid),
            'url': page.url,
            'score': page.score
{'fraction_non_alphanumeric': 0.09777061509535322, 'fraction_numerical': 0.0059092130002686, 'mean_word_length': 2.333930170098478, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1524350', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1078, 'n_words': 207}