<commit_before>package org.atlasapi.content;

import com.google.common.base.Objects;
import com.google.common.base.Throwables;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractIdleService;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.metabroadcast.common.query.Selection;
import com.metabroadcast.common.time.DateTimeZones;
import org.atlasapi.criteria.AttributeQuerySet;
import org.atlasapi.entity.Id;
import org.atlasapi.entity.ResourceRef;
import org.atlasapi.entity.util.Resolved;
import org.atlasapi.media.entity.Publisher;
import org.atlasapi.util.EsPersistenceException;
import org.atlasapi.util.EsQueryBuilder;
import org.atlasapi.util.EsSortQueryBuilder;
import org.atlasapi.util.FiltersBuilder;
import org.atlasapi.util.FutureSettingActionListener;
import org.atlasapi.util.ImmutableCollectors;
import org.atlasapi.util.Strings;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.NoShardAvailableActionException;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.exists.ExistsRequestBuilder;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Requests;
import org.elasticsearch.index.get.GetField;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermFilterBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.Node;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.atlasapi.EsSchema.CONTENT_INDEX;

public class EsContentIndex extends AbstractIdleService implements ContentIndex {

    public static final String CONTENT = "content";
    private final Logger log = LoggerFactory.getLogger(EsContentIndex.class);

    private static final int DEFAULT_LIMIT = 50;

    private final Node esClient;
    private final String index;
    private final long requestTimeout;

    private Set<String> existingIndexes;
    private final ContentResolver resolver;

    private final EsQueryBuilder queryBuilder = new EsQueryBuilder();
    private final EsSortQueryBuilder sortBuilder = new EsSortQueryBuilder();
    private final ContentGroupIndexUpdater contentGroupIndexUpdater = new ContentGroupIndexUpdater();

    public EsContentIndex(Node esClient, String indexName, long requestTimeout, ContentResolver resolver) {
        this.esClient = checkNotNull(esClient);
        this.requestTimeout = requestTimeout;
        this.index = checkNotNull(indexName);
        this.resolver = checkNotNull(resolver);
    }

    @Override
    protected void startUp() throws IOException {
        if (createIndex(index)) {
            putTypeMappings();
        }
        log.info("Found existing indices {}", existingIndexes);
    }

    @Override
    protected void shutDown() throws Exception {

    }
    private boolean createIndex(String name) {
        ActionFuture<IndicesExistsResponse> exists = esClient.client().admin().indices().exists(
            Requests.indicesExistsRequest(name)
        );
        if (!timeoutGet(exists).isExists()) {
            try {
                log.info("Creating index {}", name);
                timeoutGet(esClient.client().admin().indices().create(Requests.createIndexRequest(
                        name)));
            } catch (IndexAlreadyExistsException iaee) {
                log.info("Already exists: {}", name);
                return false;
            }
            return true;
        } else {
            log.info("Index {} exists", name);
            return false;
        }
    }

    private void putTypeMappings() throws IOException {
        log.info("Putting mapping for type {}", EsContent.TOP_LEVEL_CONTAINER);
        doMappingRequest(Requests.putMappingRequest(index)
            .type(EsContent.TOP_LEVEL_CONTAINER)
            .source(EsContent.getTopLevelMapping(EsContent.TOP_LEVEL_CONTAINER)));
        log.info("Putting mapping for type {}", EsContent.TOP_LEVEL_ITEM);
        doMappingRequest(Requests.putMappingRequest(index)
            .type(EsContent.TOP_LEVEL_ITEM)
            .source(EsContent.getTopLevelMapping(EsContent.TOP_LEVEL_ITEM)));
        log.info("Putting mapping for type {}", EsContent.CHILD_ITEM);
        doMappingRequest(Requests.putMappingRequest(index)
                .type(EsContent.CHILD_ITEM)
                .source(EsContent.getChildMapping()));
    }

    private void doMappingRequest(PutMappingRequest req) {
        try {
            timeoutGet(esClient.client().admin().indices().putMapping(req));
        } catch (MergeMappingException mme) {
            log.info("Merge Mapping Exception: {}/{}", req.indices(), req.type());
        }

    }

    private EsContent toEsContent(Item item) {
        return new EsContent()
                .id(item.getId().longValue())
                .type(ContentType.fromContent(item).get())
                .source(item.getSource() != null ? item.getSource().key() : null)
                .aliases(item.getAliases())
                .title(item.getTitle())
                .genre(item.getGenres())
                .age(item.getRestrictions().stream()
                                .map(Restriction::getMinimumAge)
                                .filter(a -> a != null)
                                .max(Integer::compare)
                                .orElse(null)
                )
                .price(makeEsPrices(item.getManifestedAs()))
                .flattenedTitle(flattenedOrNull(item.getTitle()))
                .parentTitle(item.getTitle())
                .parentFlattenedTitle(flattenedOrNull(item.getTitle()))
                .specialization(item.getSpecialization() != null ?
                                item.getSpecialization().name() :
                                null)
                .broadcasts(makeESBroadcasts(item))
                .locations(makeESLocations(item))
                .topics(makeESTopics(item));
    }

    private EsContent toEsContainer(Container container) {
        EsContent indexed = new EsContent()
            .id(container.getId().longValue())
            .type(ContentType.fromContent(container).get())
            .source(container.getSource() != null ? container.getSource().key() : null)
            .aliases(container.getAliases())
            .title(container.getTitle())
            .genre(container.getGenres())
            .age(ageRestrictionFromContainer(container))
            .flattenedTitle(flattenedOrNull(container.getTitle()))
            .parentTitle(container.getTitle())
            .parentFlattenedTitle(flattenedOrNull(container.getTitle()))
            .specialization(container.getSpecialization() != null ?
                    container.getSpecialization().name() :
                    null)
            .topics(makeESTopics(container));
        if (!container.getItemRefs().isEmpty()) {
            indexed.hasChildren(Boolean.TRUE);
            indexChildrenData(container);
        } else {
            indexed.hasChildren(Boolean.FALSE);<commit_msg>Index priority field from Item and Container<commit_after>package org.atlasapi.content;

import com.google.common.base.Objects;
import com.google.common.base.Throwables;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractIdleService;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.metabroadcast.common.query.Selection;
import com.metabroadcast.common.time.DateTimeZones;
import org.atlasapi.criteria.AttributeQuerySet;
import org.atlasapi.entity.Id;
import org.atlasapi.entity.ResourceRef;
import org.atlasapi.entity.util.Resolved;
import org.atlasapi.media.entity.Publisher;
import org.atlasapi.util.EsPersistenceException;
import org.atlasapi.util.EsQueryBuilder;
import org.atlasapi.util.EsSortQueryBuilder;
import org.atlasapi.util.FiltersBuilder;
import org.atlasapi.util.FutureSettingActionListener;
import org.atlasapi.util.ImmutableCollectors;
import org.atlasapi.util.Strings;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.NoShardAvailableActionException;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.exists.ExistsRequestBuilder;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Requests;
import org.elasticsearch.index.get.GetField;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermFilterBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.node.Node;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHitField;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import static com.google.common.base.Preconditions.checkNotNull;
import static org.atlasapi.EsSchema.CONTENT_INDEX;

public class EsContentIndex extends AbstractIdleService implements ContentIndex {

    public static final String CONTENT = "content";
    private final Logger log = LoggerFactory.getLogger(EsContentIndex.class);

    private static final int DEFAULT_LIMIT = 50;

    private final Node esClient;
    private final String index;
    private final long requestTimeout;

    private Set<String> existingIndexes;
    private final ContentResolver resolver;

    private final EsQueryBuilder queryBuilder = new EsQueryBuilder();
    private final EsSortQueryBuilder sortBuilder = new EsSortQueryBuilder();
    private final ContentGroupIndexUpdater contentGroupIndexUpdater = new ContentGroupIndexUpdater();

    public EsContentIndex(Node esClient, String indexName, long requestTimeout, ContentResolver resolver) {
        this.esClient = checkNotNull(esClient);
        this.requestTimeout = requestTimeout;
        this.index = checkNotNull(indexName);
        this.resolver = checkNotNull(resolver);
    }

    @Override
    protected void startUp() throws IOException {
        if (createIndex(index)) {
            putTypeMappings();
        }
        log.info("Found existing indices {}", existingIndexes);
    }

    @Override
    protected void shutDown() throws Exception {

    }
    private boolean createIndex(String name) {
        ActionFuture<IndicesExistsResponse> exists = esClient.client().admin().indices().exists(
            Requests.indicesExistsRequest(name)
        );
        if (!timeoutGet(exists).isExists()) {
            try {
                log.info("Creating index {}", name);
                timeoutGet(esClient.client().admin().indices().create(Requests.createIndexRequest(
                        name)));
            } catch (IndexAlreadyExistsException iaee) {
                log.info("Already exists: {}", name);
                return false;
            }
            return true;
        } else {
            log.info("Index {} exists", name);
            return false;
        }
    }

    private void putTypeMappings() throws IOException {
        log.info("Putting mapping for type {}", EsContent.TOP_LEVEL_CONTAINER);
        doMappingRequest(Requests.putMappingRequest(index)
            .type(EsContent.TOP_LEVEL_CONTAINER)
            .source(EsContent.getTopLevelMapping(EsContent.TOP_LEVEL_CONTAINER)));
        log.info("Putting mapping for type {}", EsContent.TOP_LEVEL_ITEM);
        doMappingRequest(Requests.putMappingRequest(index)
            .type(EsContent.TOP_LEVEL_ITEM)
            .source(EsContent.getTopLevelMapping(EsContent.TOP_LEVEL_ITEM)));
        log.info("Putting mapping for type {}", EsContent.CHILD_ITEM);
        doMappingRequest(Requests.putMappingRequest(index)
                .type(EsContent.CHILD_ITEM)
                .source(EsContent.getChildMapping()));
    }

    private void doMappingRequest(PutMappingRequest req) {
        try {
            timeoutGet(esClient.client().admin().indices().putMapping(req));
        } catch (MergeMappingException mme) {
            log.info("Merge Mapping Exception: {}/{}", req.indices(), req.type());
        }

    }

    private EsContent toEsContent(Item item) {
        return new EsContent()
                .id(item.getId().longValue())
                .type(ContentType.fromContent(item).get())
                .source(item.getSource() != null ? item.getSource().key() : null)
                .aliases(item.getAliases())
                .title(item.getTitle())
                .genre(item.getGenres())
                .age(item.getRestrictions().stream()
                                .map(Restriction::getMinimumAge)
                                .filter(a -> a != null)
                                .max(Integer::compare)
                                .orElse(null)
                )
                .price(makeEsPrices(item.getManifestedAs()))
                .flattenedTitle(flattenedOrNull(item.getTitle()))
                .parentTitle(item.getTitle())
                .parentFlattenedTitle(flattenedOrNull(item.getTitle()))
                .specialization(item.getSpecialization() != null ?
                                item.getSpecialization().name() :
                                null)
                .priority(item.getPriority())
                .broadcasts(makeESBroadcasts(item))
                .locations(makeESLocations(item))
                .topics(makeESTopics(item));
    }

    private EsContent toEsContainer(Container container) {
        EsContent indexed = new EsContent()
            .id(container.getId().longValue())
            .type(ContentType.fromContent(container).get())
            .source(container.getSource() != null ? container.getSource().key() : null)
            .aliases(container.getAliases())
            .title(container.getTitle())
            .genre(container.getGenres())
            .age(ageRestrictionFromContainer(container))
            .flattenedTitle(flattenedOrNull(container.getTitle()))
            .parentTitle(container.getTitle())
            .parentFlattenedTitle(flattenedOrNull(container.getTitle()))
            .specialization(container.getSpecialization() != null ?
                    container.getSpecialization().name() :
                    null)
            .priority(container.getPriority())
            .topics(makeESTopics(container));
        if (!container.getItemRefs().isEmpty()) {
            indexed.hasChildren(Boolean.TRUE);
            indexChildrenData(container);
        } else {
            indexed.hasChildren(Boolean.FALSE);
{'fraction_non_alphanumeric': 0.1031055900621118, 'fraction_numerical': 0.00045172219085262564, 'mean_word_length': 3.1604416255579046, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3635535', 'n_tokens_mistral': 4702, 'n_tokens_neox': 4676, 'n_words': 767}