<commit_before>package org.recap.camel.datadump.consumer;

import org.apache.camel.Exchange;
import org.apache.camel.FluentProducerTemplate;
import org.apache.camel.builder.DefaultFluentProducerTemplate;
import org.apache.commons.collections.CollectionUtils;
import org.recap.RecapConstants;
import org.recap.camel.datadump.callable.BibEntityPreparerCallable;
import org.recap.model.jpa.BibliographicEntity;
import org.recap.repository.BibliographicDetailsRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.concurrent.*;

/**
 * Created by peris on 11/1/16.
 */
public class BibEntityGeneratorActiveMQConsumer {

    private static final Logger logger = LoggerFactory.getLogger(BibEntityGeneratorActiveMQConsumer.class);

    private BibliographicDetailsRepository bibliographicDetailsRepository;
    private ExecutorService executorService;

    /**
     * Instantiates a new Bib entity generator active mq consumer.
     *
     * @param bibliographicDetailsRepository the bibliographic details repository
     */
    public BibEntityGeneratorActiveMQConsumer(BibliographicDetailsRepository bibliographicDetailsRepository) {
        this.bibliographicDetailsRepository = bibliographicDetailsRepository;
    }

    /**
     * This method is invoked by the route to build bibliographic entities using the data dump solr search results.
     * The bibliographic entities are then passed to another route for data dump export.
     *
     * @param exchange the exchange
     * @throws Exception the exception
     */
    public void processBibEntities(Exchange exchange) throws Exception {

        long startTime = System.currentTimeMillis();

        Map results = (Map) exchange.getIn().getBody();
        List<HashMap> dataDumpSearchResults = (List<HashMap>) results.get("dataDumpSearchResults");



        List<BibliographicEntity> bibliographicEntities = new ArrayList<>();

        List<Integer> bibIdList = new ArrayList<>();
        Map<Integer, List<Integer>> bibItemMap = new HashMap<>();
        for (Iterator<HashMap> iterator = dataDumpSearchResults.iterator(); iterator.hasNext(); ) {
            HashMap hashMap = iterator.next();
            Integer bibId = (Integer) hashMap.get("bibId");
            List<Integer> itemIds = (List<Integer>) hashMap.get("itemIds");
            if(CollectionUtils.isNotEmpty(itemIds)) {
                bibIdList.add(bibId);
                bibItemMap.put(bibId, itemIds);
            }
        }

        if(CollectionUtils.isNotEmpty(bibIdList)) {
            List<BibliographicEntity> bibliographicEntityList = bibliographicDetailsRepository.getBibliographicEntityList(bibIdList);

            List<Callable<BibliographicEntity>> callables = new ArrayList<>();

            for (Iterator<BibliographicEntity> iterator = bibliographicEntityList.iterator(); iterator.hasNext(); ) {
                BibliographicEntity bibliographicEntity = iterator.next();
                BibEntityPreparerCallable bibEntityPreparerCallable = new BibEntityPreparerCallable(bibliographicEntity, bibItemMap.get(bibliographicEntity.getBibliographicId()));
                callables.add(bibEntityPreparerCallable);
            }

            List<Future<BibliographicEntity>> futureList = getExecutorService().invokeAll(callables);
            futureList.stream()
                    .map(future -> {
                        try {
                            return future.get();
                        } catch (InterruptedException | ExecutionException e) {
                            logger.error(RecapConstants.ERROR,e);<commit_msg>RECAP-884 - Fix for exporting orphan bibs in deleted feed<commit_after>package org.recap.camel.datadump.consumer;

import org.apache.camel.Exchange;
import org.apache.camel.FluentProducerTemplate;
import org.apache.camel.builder.DefaultFluentProducerTemplate;
import org.apache.commons.collections.CollectionUtils;
import org.recap.RecapConstants;
import org.recap.camel.datadump.callable.BibEntityPreparerCallable;
import org.recap.model.jpa.BibliographicEntity;
import org.recap.repository.BibliographicDetailsRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;
import java.util.concurrent.*;

/**
 * Created by peris on 11/1/16.
 */
public class BibEntityGeneratorActiveMQConsumer {

    private static final Logger logger = LoggerFactory.getLogger(BibEntityGeneratorActiveMQConsumer.class);

    private BibliographicDetailsRepository bibliographicDetailsRepository;
    private ExecutorService executorService;

    /**
     * Instantiates a new Bib entity generator active mq consumer.
     *
     * @param bibliographicDetailsRepository the bibliographic details repository
     */
    public BibEntityGeneratorActiveMQConsumer(BibliographicDetailsRepository bibliographicDetailsRepository) {
        this.bibliographicDetailsRepository = bibliographicDetailsRepository;
    }

    /**
     * This method is invoked by the route to build bibliographic entities using the data dump solr search results.
     * The bibliographic entities are then passed to another route for data dump export.
     *
     * @param exchange the exchange
     * @throws Exception the exception
     */
    public void processBibEntities(Exchange exchange) throws Exception {

        long startTime = System.currentTimeMillis();

        Map results = (Map) exchange.getIn().getBody();
        List<HashMap> dataDumpSearchResults = (List<HashMap>) results.get("dataDumpSearchResults");



        List<BibliographicEntity> bibliographicEntities = new ArrayList<>();

        List<Integer> bibIdList = new ArrayList<>();
        Map<Integer, List<Integer>> bibItemMap = new HashMap<>();
        for (Iterator<HashMap> iterator = dataDumpSearchResults.iterator(); iterator.hasNext(); ) {
            HashMap hashMap = iterator.next();
            Integer bibId = (Integer) hashMap.get("bibId");
            List<Integer> itemIds = (List<Integer>) hashMap.get("itemIds");
            bibIdList.add(bibId);
            bibItemMap.put(bibId, itemIds);
        }

        if(CollectionUtils.isNotEmpty(bibIdList)) {
            List<BibliographicEntity> bibliographicEntityList = bibliographicDetailsRepository.getBibliographicEntityList(bibIdList);

            List<Callable<BibliographicEntity>> callables = new ArrayList<>();

            for (Iterator<BibliographicEntity> iterator = bibliographicEntityList.iterator(); iterator.hasNext(); ) {
                BibliographicEntity bibliographicEntity = iterator.next();
                BibEntityPreparerCallable bibEntityPreparerCallable = new BibEntityPreparerCallable(bibliographicEntity, bibItemMap.get(bibliographicEntity.getBibliographicId()));
                callables.add(bibEntityPreparerCallable);
            }

            List<Future<BibliographicEntity>> futureList = getExecutorService().invokeAll(callables);
            futureList.stream()
                    .map(future -> {
                        try {
                            return future.get();
                        } catch (InterruptedException | ExecutionException e) {
                            logger.error(RecapConstants.ERROR,e);
{'fraction_non_alphanumeric': 0.08721575152523572, 'fraction_numerical': 0.0023571824736550195, 'mean_word_length': 3.350422195416164, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3114278', 'n_tokens_mistral': 1887, 'n_tokens_neox': 1788, 'n_words': 386}