<commit_before>import math
import logging
import json
from collections import OrderedDict
import re
import copy

from pyelasticsearch import ElasticSearch
from pyelasticsearch.exceptions import ElasticHttpNotFoundError
import pytz

from bs4 import BeautifulSoup

from . import db, app
from pmg.models.resources import *  # noqa
from pmg.models.base import resource_slugs

PHRASE_RE = re.compile(r'"([^"]*)("|$)')


class Search:

    esserver = app.config['ES_SERVER']
    index_name = "pmg"
    search_fields = ["title^2", "description", "fulltext", "attachments"]
    exact_search_fields = ["title.exact^2", "description.exact", "fulltext.exact", "attachments_exact"]
    es = ElasticSearch(esserver)
    per_batch = 200
    logger = logging.getLogger(__name__)

    reindex_changes = app.config['SEARCH_REINDEX_CHANGES']
    """ Should updates to models be reindexed? """

    friendly_data_types = OrderedDict([
        ("committee", "Committees"),
        ("committee_meeting", "Committee Meetings"),
        ("bill", "Bills"),
        ("member", "MPs"),
        ("hansard", "Hansards"),
        ("briefing", "Media Briefings"),
        # this is both QuestionReply and CommitteeQuestion objects
        ("minister_question", "Questions & Replies"),
        ("tabled_committee_report", "Tabled Committee Reports"),
        ("call_for_comment", "Calls for Comments"),
        ("policy_document", "Policy Documents"),
        ("gazette", "Gazettes"),
        ("daily_schedule", "Daily Schedules"),
    ])

    def reindex_all(self, data_type):
        """ Index all content of a data_type """
        try:
            self.drop_index(data_type)
        except:
            self.logger.warn("Couldn't find %s index" % data_type)

        self.mapping(data_type)

        models = [m for m in resource_slugs.itervalues() if m.resource_content_type == data_type]
        for model in models:
            self.reindex_for_model(model)

    def reindex_for_model(self, model):
        """ Index all content of type +model+ """
        ids = [r[0] for r in db.session.query(model.id).all()]

        if model.__name__ == 'Bill':
            per_batch = 1
        else:
            per_batch = self.per_batch

        count = len(ids)
        pages = int(math.ceil(float(count) / float(per_batch)))
        self.logger.info("Reindexing for %s" % model.__name__)
        self.logger.info("Pages %s, %s per page, %s items" % (pages, per_batch, count))

        while len(ids):
            self.logger.info("Items left %s" % len(ids))
            id_subsection = []
            for x in xrange(0, per_batch):
                if ids:
                    id_subsection.append(ids.pop())

            rows = db.session.query(model).filter(model.id.in_(id_subsection)).all()
            items = [Transforms.serialise(r) for r in rows]
            self.add_many(model.resource_content_type, items)

    def drop_index(self, data_type):
        self.logger.info("Dropping %s index" % data_type)
        self.es.delete_all(self.index_name, data_type)
        self.logger.info("Dropped %s index" % data_type)

    def add_obj(self, obj):
        self.add(obj.resource_content_type, Transforms.serialise(obj))
<commit_msg>Restrict max size we index<commit_after>import math
import logging
import json
from collections import OrderedDict
import re
import copy
import cPickle

from pyelasticsearch import ElasticSearch
from pyelasticsearch.exceptions import ElasticHttpNotFoundError
import pytz

from bs4 import BeautifulSoup

from . import db, app
from pmg.models.resources import *  # noqa
from pmg.models.base import resource_slugs

PHRASE_RE = re.compile(r'"([^"]*)("|$)')
MAX_INDEXABLE_BYTES = 104857600 # Limit ElasticSearch/Netty has by default

class Search:

    esserver = app.config['ES_SERVER']
    index_name = "pmg"
    search_fields = ["title^2", "description", "fulltext", "attachments"]
    exact_search_fields = ["title.exact^2", "description.exact", "fulltext.exact", "attachments_exact"]
    es = ElasticSearch(esserver)
    per_batch = 200
    logger = logging.getLogger(__name__)

    reindex_changes = app.config['SEARCH_REINDEX_CHANGES']
    """ Should updates to models be reindexed? """

    friendly_data_types = OrderedDict([
        ("committee", "Committees"),
        ("committee_meeting", "Committee Meetings"),
        ("bill", "Bills"),
        ("member", "MPs"),
        ("hansard", "Hansards"),
        ("briefing", "Media Briefings"),
        # this is both QuestionReply and CommitteeQuestion objects
        ("minister_question", "Questions & Replies"),
        ("tabled_committee_report", "Tabled Committee Reports"),
        ("call_for_comment", "Calls for Comments"),
        ("policy_document", "Policy Documents"),
        ("gazette", "Gazettes"),
        ("daily_schedule", "Daily Schedules"),
    ])

    def reindex_all(self, data_type):
        """ Index all content of a data_type """
        try:
            self.drop_index(data_type)
        except:
            self.logger.warn("Couldn't find %s index" % data_type)

        self.mapping(data_type)

        models = [m for m in resource_slugs.itervalues() if m.resource_content_type == data_type]
        for model in models:
            self.reindex_for_model(model)

    def reindex_for_model(self, model):
        """ Index all content of type +model+ """
        ids = [r[0] for r in db.session.query(model.id).all()]

        if model.__name__ == 'Bill':
            per_batch = 1
        else:
            per_batch = self.per_batch

        count = len(ids)
        pages = int(math.ceil(float(count) / float(per_batch)))
        self.logger.info("Reindexing for %s" % model.__name__)
        self.logger.info("Pages %s, %s per page, %s items" % (pages, per_batch, count))

        while len(ids):
            self.logger.info("Items left %s" % len(ids))
            id_subsection = []
            for x in xrange(0, per_batch):
                if ids:
                    id_subsection.append(ids.pop())

            rows = db.session.query(model).filter(model.id.in_(id_subsection)).all()
            items = self.serialise(rows)
            self.add_many(model.resource_content_type, items)

    def serialise(self, rows):
        items = []
        for row in rows:
            item = Transforms.serialise(row)
            size = len(cPickle.dumps(item))
            if size < MAX_INDEXABLE_BYTES:
                items.append(item)
            else:
                self.logger.info("Not indexing object %r bigger than max (%d > %d)",
                                 row, size, MAX_INDEXABLE_BYTES)
        return items

    def drop_index(self, data_type):
        self.logger.info("Dropping %s index" % data_type)
        self.es.delete_all(self.index_name, data_type)
        self.logger.info("Dropped %s index" % data_type)

    def add_obj(self, obj):
        self.add(obj.resource_content_type, Transforms.serialise(obj))

{'fraction_non_alphanumeric': 0.12213079255088782, 'fraction_numerical': 0.0038977912516240795, 'mean_word_length': 2.724731182795699, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169076', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2082, 'n_words': 502}