<commit_before>package org.lilycms.indexer.engine;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.lilycms.indexer.model.indexerconf.DerefValue;
import org.lilycms.indexer.model.indexerconf.DerefValue.Follow;
import org.lilycms.indexer.model.indexerconf.DerefValue.FieldFollow;
import org.lilycms.indexer.model.indexerconf.DerefValue.VariantFollow;
import org.lilycms.indexer.model.indexerconf.DerefValue.MasterFollow;
import org.lilycms.indexer.model.indexerconf.FieldValue;
import org.lilycms.indexer.model.indexerconf.IndexerConf;
import org.lilycms.indexer.model.indexerconf.Value;
import org.lilycms.indexer.model.indexerconf.Formatter;
import org.lilycms.repository.api.*;
import org.lilycms.util.io.Closer;
import org.lilycms.util.repo.VersionTag;

import java.io.InputStream;
import java.util.*;

/**
 * Evaluates an index field value (a {@link Value}) to a value.
 */
public class ValueEvaluator {
    private Log log = LogFactory.getLog(getClass());

    private IndexerConf conf;

    public ValueEvaluator(IndexerConf conf) {
        this.conf = conf;
    }

    /**
     * Evaluates a value for a given record & vtag.
     *
     * @return null if there is no value
     */
    public List<String> eval(Value valueDef, IdRecord record, Repository repository, String vtag) {
        Object value = evalValue(valueDef, record, repository, vtag);
        if (value == null)
            return null;

        if (valueDef.extractContent()) {
            return extractContent(valueDef, value, record, repository);
        }

        ValueType valueType = valueDef.getValueType();
        Formatter formatter = valueDef.getFormatter() != null ? conf.getFormatters().getFormatter(valueDef.getFormatter()) : conf.getFormatters().getFormatter(valueType);

        return formatter.format(value, valueType);
    }

    private List<String> extractContent(Value valueDef, Object value, IdRecord record, Repository repository) {
        // At this point we can be sure the value will be a blob, this is validated during
        // the construction of the indexer conf.

        ValueType valueType = valueDef.getValueType();

        List<Blob> blobs = new ArrayList<Blob>();
        collectBlobs(value, valueType, blobs);

        if (blobs.size() == 0)
            return null;

        List<String> result = new ArrayList<String>(blobs.size());

        Parser parser = new AutoDetectParser();

        // TODO add some debug (or even info) logging to indicate what we are working on.
        for (Blob blob : blobs) {
            InputStream is = null;
            try {
                is = repository.getInputStream(blob);

                // TODO make write limit configurable
                BodyContentHandler ch = new BodyContentHandler();

                Metadata metadata = new Metadata();
                metadata.add(Metadata.CONTENT_TYPE, blob.getMimetype());
                if (blob.getName() != null)
                    metadata.add(Metadata.RESOURCE_NAME_KEY, blob.getName());

                ParseContext parseContext = new ParseContext();

                parser.parse(is, ch, metadata, parseContext);
<commit_msg>Indexer engine: the Tika parser is thread safe and takes quite some time to initialize, so reuse the same copy. Furthermore, inside Tika the loading of the tagsoup parser's schema structure is done over and over again (accounting for about a third of the total parsing time in a typical mbox-import), made a fix for this: https://issues.apache.org/jira/browse/TIKA-528<commit_after>package org.lilycms.indexer.engine;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.BodyContentHandler;
import org.lilycms.indexer.model.indexerconf.DerefValue;
import org.lilycms.indexer.model.indexerconf.DerefValue.Follow;
import org.lilycms.indexer.model.indexerconf.DerefValue.FieldFollow;
import org.lilycms.indexer.model.indexerconf.DerefValue.VariantFollow;
import org.lilycms.indexer.model.indexerconf.DerefValue.MasterFollow;
import org.lilycms.indexer.model.indexerconf.FieldValue;
import org.lilycms.indexer.model.indexerconf.IndexerConf;
import org.lilycms.indexer.model.indexerconf.Value;
import org.lilycms.indexer.model.indexerconf.Formatter;
import org.lilycms.repository.api.*;
import org.lilycms.util.io.Closer;
import org.lilycms.util.repo.VersionTag;

import java.io.InputStream;
import java.util.*;

/**
 * Evaluates an index field value (a {@link Value}) to a value.
 */
public class ValueEvaluator {
    private Log log = LogFactory.getLog(getClass());

    private IndexerConf conf;

    private Parser tikaParser = new AutoDetectParser();

    public ValueEvaluator(IndexerConf conf) {
        this.conf = conf;
    }

    /**
     * Evaluates a value for a given record & vtag.
     *
     * @return null if there is no value
     */
    public List<String> eval(Value valueDef, IdRecord record, Repository repository, String vtag) {
        Object value = evalValue(valueDef, record, repository, vtag);
        if (value == null)
            return null;

        if (valueDef.extractContent()) {
            return extractContent(valueDef, value, record, repository);
        }

        ValueType valueType = valueDef.getValueType();
        Formatter formatter = valueDef.getFormatter() != null ? conf.getFormatters().getFormatter(valueDef.getFormatter()) : conf.getFormatters().getFormatter(valueType);

        return formatter.format(value, valueType);
    }

    private List<String> extractContent(Value valueDef, Object value, IdRecord record, Repository repository) {
        // At this point we can be sure the value will be a blob, this is validated during
        // the construction of the indexer conf.

        ValueType valueType = valueDef.getValueType();

        List<Blob> blobs = new ArrayList<Blob>();
        collectBlobs(value, valueType, blobs);

        if (blobs.size() == 0)
            return null;

        List<String> result = new ArrayList<String>(blobs.size());

        // TODO add some debug (or even info) logging to indicate what we are working on.
        for (Blob blob : blobs) {
            InputStream is = null;
            try {
                is = repository.getInputStream(blob);

                // TODO make write limit configurable
                BodyContentHandler ch = new BodyContentHandler();

                Metadata metadata = new Metadata();
                metadata.add(Metadata.CONTENT_TYPE, blob.getMimetype());
                if (blob.getName() != null)
                    metadata.add(Metadata.RESOURCE_NAME_KEY, blob.getName());

                ParseContext parseContext = new ParseContext();

                tikaParser.parse(is, ch, metadata, parseContext);

{'fraction_non_alphanumeric': 0.0995405819295559, 'fraction_numerical': 0.0006960879855213699, 'mean_word_length': 3.123995407577497, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '681531', 'n_tokens_mistral': 2100, 'n_tokens_neox': 1987, 'n_words': 525}