<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Recall is a program for storing bookmarks of different things
# Copyright (C) 2012  <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json

import requests

import convenience

settings = convenience.settings

def index_root(root):
    url = "http://{hostname}:{port}/{index_root}/{type}/{id}".format(
        hostname = settings["RECALL_ELASTICSEARCH_HOST"],
        port = int(settings["RECALL_ELASTICSEARCH_PORT"]),
        index_root = settings["RECALL_ELASTICSEARCH_INDEX"],
        type = "mark",
        id = root["@"] + str(root["~"]))
    db = convenience.db()
    facts = db.marks.find({":": {"@": root["@"], "~": root["~"]}})
    for fact in facts:
        root.setdefault("about", []).append(fact["about"])
    requests.post(url, data=json.dumps(root))

def index_fact(fact):
    db = convenience.db()
    root = db.marks.find_one({"@": fact[":"]["@"], "~": fact[":"]["~"]})
    try:
        root.setdefault("about", []).append(fact["about"])
        del root["_id"]
        url = "http://{hostname}:{port}/{index_root}/{type}/{id}".format(
            hostname = settings["RECALL_ELASTICSEARCH_HOST"],
            port = int(settings["RECALL_ELASTICSEARCH_PORT"]),
            index_root = settings["RECALL_ELASTICSEARCH_INDEX"],
            type = "mark",
            id = root["@"] + str(root["~"]))
        requests.post(url, data=json.dumps(root))
    except:
        pass

def index_new_marks():
    def pop_mark():
        connection = convenience.redis_connection()
        entry = connection.blpop("marks")
        mark_as_string = entry[1]
        return json.loads(mark_as_string)

    def is_root(mark):
        return ":" not in mark

    mark = pop_mark()
    if is_root(mark):
        index_root(mark)
    else:
        index_fact(mark)

def main():
    pass

if __name__ == "__main__":
    convenience.load_settings()
    while(True):
        index_new_marks()<commit_msg>Simplify worker so that _records_ are processed in the queue, and marks are what is indexed by ES<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Recall is a program for storing bookmarks of different things
# Copyright (C) 2012  <NAME>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
import logging
import traceback

import requests

import convenience

settings = convenience.settings

def index(record):
    def is_root(record):
        return ":" not in record

    db = convenience.db()

    if is_root(record):
        mark = record
    else:
        mark = db.marks.find_one({"@": record[":"]["@"], "~": record[":"]["~"]})

    if "_id" in mark:
        del mark["_id"]

    url = "http://{hostname}:{port}/{index}/{type}/{id}".format(
        hostname = settings["RECALL_ELASTICSEARCH_HOST"],
        port = int(settings["RECALL_ELASTICSEARCH_PORT"]),
        index = settings["RECALL_ELASTICSEARCH_INDEX"],
        type = "mark",
        id = mark["@"] + str(mark["~"]))
    facts = db.marks.find({":": {"@": mark["@"], "~": mark["~"]}})
    for fact in facts:
        mark.setdefault("about", []).append(fact["about"])
    print "Reindexing: {mark}".format(
        mark=mark)
    requests.post(url, data=json.dumps(mark))


def index_new_mark():
    def pop_mark():
        connection = convenience.redis_connection()
        entry = connection.blpop("marks")
        mark_as_string = entry[1]
        return json.loads(mark_as_string)

    mark = pop_mark()
    index(mark)

def main():
    convenience.load_settings()
    while(True):
        try:
            index_new_mark()
        except Exception as e:
            traceback.print_exc(e)

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.13669650122050447, 'fraction_numerical': 0.0028478437754271765, 'mean_word_length': 3.2097602739726026, 'pattern_counts': {'":': 20, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6133132', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1478, 'n_words': 453}