<commit_before>import logging
import re

from django.apps import apps
from elasticsearch import helpers
from elasticsearch import Elasticsearch

from project import settings

from share.tasks import ProviderTask
from share.models import AbstractCreativeWork
from share.models import Entity
from share.models import Person
from share.models import Tag
from share.models import Subject
from share.models import Association
from share.models import Funder
from share.models import Publisher
from share.models import Institution
from share.models import Organization

logger = logging.getLogger(__name__)


def safe_substr(value, length=32000):
    if value:
        return str(value)[:length]
    return None


def score_text(text):
    return int(
        (len(re.findall('(?!\d)\w', text)) / (1 + len(re.findall('[\W\d]', text))))
        / (len(text) / 100)
    )


def add_suggest(obj):
    if obj['name']:
        obj['suggest'] = {
            'input': re.split('[\\s,]', obj['name']) + [obj['name']],
            'output': obj['name'],
            'payload': {
                'id': obj['id'],
                'name': obj['name'],
                'type': obj['type'],
            },
            'weight': score_text(obj['name'])
        }
    return obj


__shareuser_cache = {}
def sources(qs):
    sus = []
    for through in qs:
        if through.shareuser_id not in __shareuser_cache:
            __shareuser_cache[through.shareuser_id] = through.shareuser
        sus.append(__shareuser_cache[through.shareuser_id])
    return sus


class IndexModelTask(ProviderTask):

    def do_run(self, model_name, ids):
        es_client = Elasticsearch(settings.ELASTICSEARCH_URL, retry_on_timeout=True, timeout=30)
        model = apps.get_model('share', model_name)
        for resp in helpers.streaming_bulk(es_client, self.bulk_stream(model, ids)):
            logger.debug(resp)

    def bulk_stream(self, model, ids):
        opts = {'_index': settings.ELASTICSEARCH_INDEX, '_type': model.__name__.lower()}
        qs = model.objects.filter(id__in=ids)<commit_msg>Remove elasticsearch from the equation<commit_after>import logging
import re

from django.apps import apps
from elasticsearch import helpers
from elasticsearch import Elasticsearch

from project import settings

from share.tasks import ProviderTask
from share.models import AbstractCreativeWork
from share.models import Entity
from share.models import Person
from share.models import Tag
from share.models import Subject
from share.models import Association
from share.models import Funder
from share.models import Publisher
from share.models import Institution
from share.models import Organization

logger = logging.getLogger(__name__)


def safe_substr(value, length=32000):
    if value:
        return str(value)[:length]
    return None


def score_text(text):
    return int(
        (len(re.findall('(?!\d)\w', text)) / (1 + len(re.findall('[\W\d]', text))))
        / (len(text) / 100)
    )


def add_suggest(obj):
    if obj['name']:
        obj['suggest'] = {
            'input': re.split('[\\s,]', obj['name']) + [obj['name']],
            'output': obj['name'],
            'payload': {
                'id': obj['id'],
                'name': obj['name'],
                'type': obj['type'],
            },
            'weight': score_text(obj['name'])
        }
    return obj


__shareuser_cache = {}
def sources(qs):
    sus = []
    for through in qs:
        if through.shareuser_id not in __shareuser_cache:
            __shareuser_cache[through.shareuser_id] = through.shareuser
        sus.append(__shareuser_cache[through.shareuser_id])
    return sus


class IndexModelTask(ProviderTask):

    def do_run(self, model_name, ids):
        # es_client = Elasticsearch(settings.ELASTICSEARCH_URL, retry_on_timeout=True, timeout=30)
        model = apps.get_model('share', model_name)
        # for resp in helpers.streaming_bulk(es_client,
        for _ in self.bulk_stream(model, ids):
            pass
        # for resp in helpers.streaming_bulk(es_client, self.bulk_stream(model, ids)):
        #     logger.debug(resp)

    def bulk_stream(self, model, ids):
        opts = {'_index': settings.ELASTICSEARCH_INDEX, '_type': model.__name__.lower()}
        qs = model.objects.filter(id__in=ids)
{'fraction_non_alphanumeric': 0.11698904684222793, 'fraction_numerical': 0.005127010020974132, 'mean_word_length': 3.0528800755429653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7629864', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1256, 'n_words': 326}