<commit_before>package dk.statsbiblioteket.medieplatform.autonomous;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.slf4j.Logger;

import java.io.ByteArrayInputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;

public class SolrProxyIterator<T extends Item> implements Iterator<T> {
    private static Logger log = org.slf4j.LoggerFactory.getLogger(SolrProxyIterator.class);


    Iterator<T> items = null;


    private String queryString;
    private boolean details;
    private HttpSolrServer summaSearch;
    private PremisManipulatorFactory<T> premisManipulatorFactory;
    private DomsEventStorage<T> domsEventStorage;
    private final int rows = 100;
    private int start = 0;
    private int position = 0;


    public SolrProxyIterator(String queryString, boolean details, HttpSolrServer summaSearch,
                             PremisManipulatorFactory<T> premisManipulatorFactory, DomsEventStorage<T> domsEventStorage) {
        this.queryString = queryString;
        this.details = details;
        this.summaSearch = summaSearch;
        this.premisManipulatorFactory = premisManipulatorFactory;
        this.domsEventStorage = domsEventStorage;
        search();
    }

    @Override
    public synchronized boolean hasNext() {
        if (position >= rows) {
            start += rows;
            position = 0;
            search();
        }
        return items.hasNext();
    }

    private void search() {
        try {
            SolrQuery query = new SolrQuery();
            query.setQuery(queryString);
            query.setRows(rows); //Fetch size. Do not go over 1000 unless you specify fields to fetch which does not include content_text
            query.setStart(start);
            //IMPORTANT!Only use facets if needed.
            query.set("facet", "false"); //very important. Must overwrite to false. Facets are very slow and expensive.
            query.setFields(SBOIEventIndex.UUID, SBOIEventIndex.LAST_MODIFIED);
            if (!details) {
                query.addField(SBOIEventIndex.PREMIS_NO_DETAILS);
            }

            query.addSort(SBOIEventIndex.SORT_DATE, SolrQuery.ORDER.asc);

            QueryResponse response = summaSearch.query(query);
            SolrDocumentList results = response.getResults();
            List<T> hits = new ArrayList<>();
            for (SolrDocument result : results) {
                T hit;
                String uuid = result.getFirstValue(SBOIEventIndex.UUID).toString();
                String lastModified = result.getFirstValue(SBOIEventIndex.LAST_MODIFIED).toString();

                if (!details) { //no details, so we can retrieve everything from Summa
                    final String blob = result.getFirstValue(SBOIEventIndex.PREMIS_NO_DETAILS).toString();
                    hit= premisManipulatorFactory.createFromStringBlob(blob).toItem();
                    hit.setDomsID(uuid);
                } else {//Details requested so go to DOMS
                    hit = domsEventStorage.getItemFromDomsID(uuid);
                }
                hit.setLastModified(parseDate(lastModified));
                hits.add(hit);
            }
            items = hits.iterator();
        } catch (Exception e) {

            throw new RuntimeException(e);
        }
    }

    private Date parseDate(String lastModified) {
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
        try {
            return format.parse(lastModified);
        } catch (ParseException e) {
            log.warn("Failed to parse date {}",lastModified,e);
            return null;
        }
    }
<commit_msg>Ignore solr hits without premis<commit_after>package dk.statsbiblioteket.medieplatform.autonomous;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.slf4j.Logger;

import java.io.ByteArrayInputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;

public class SolrProxyIterator<T extends Item> implements Iterator<T> {
    private static Logger log = org.slf4j.LoggerFactory.getLogger(SolrProxyIterator.class);


    Iterator<T> items = null;


    private String queryString;
    private boolean details;
    private HttpSolrServer summaSearch;
    private PremisManipulatorFactory<T> premisManipulatorFactory;
    private DomsEventStorage<T> domsEventStorage;
    private final int rows = 100;
    private int start = 0;
    private int position = 0;


    public SolrProxyIterator(String queryString, boolean details, HttpSolrServer summaSearch,
                             PremisManipulatorFactory<T> premisManipulatorFactory,
                             DomsEventStorage<T> domsEventStorage) {
        this.queryString = queryString;
        this.details = details;
        this.summaSearch = summaSearch;
        this.premisManipulatorFactory = premisManipulatorFactory;
        this.domsEventStorage = domsEventStorage;
        search();
    }

    @Override
    public synchronized boolean hasNext() {
        if (position >= rows) {
            start += rows;
            position = 0;
            search();
        }
        return items.hasNext();
    }

    private void search() {
        try {
            SolrQuery query = new SolrQuery();
            query.setQuery(queryString);
            query.setRows(rows); //Fetch size. Do not go over 1000 unless you specify fields to fetch which does not include content_text
            query.setStart(start);
            //IMPORTANT!Only use facets if needed.
            query.set("facet", "false"); //very important. Must overwrite to false. Facets are very slow and expensive.
            query.setFields(SBOIEventIndex.UUID, SBOIEventIndex.LAST_MODIFIED);
            if (!details) {
                query.addField(SBOIEventIndex.PREMIS_NO_DETAILS);
            }

            query.addSort(SBOIEventIndex.SORT_DATE, SolrQuery.ORDER.asc);

            QueryResponse response = summaSearch.query(query);
            SolrDocumentList results = response.getResults();
            List<T> hits = new ArrayList<>();
            for (SolrDocument result : results) {
                T hit;
                String uuid = result.getFirstValue(SBOIEventIndex.UUID).toString();
                String lastModified = result.getFirstValue(SBOIEventIndex.LAST_MODIFIED).toString();

                if (!details) { //no details, so we can retrieve everything from Summa
                    if (result.getFirstValue(SBOIEventIndex.PREMIS_NO_DETAILS) == null) {
                        continue;
                    }
                    final String blob = result.getFirstValue(SBOIEventIndex.PREMIS_NO_DETAILS).toString();
                    hit = premisManipulatorFactory.createFromStringBlob(blob).toItem();
                    hit.setDomsID(uuid);
                } else {//Details requested so go to DOMS
                    hit = domsEventStorage.getItemFromDomsID(uuid);
                }
                hit.setLastModified(parseDate(lastModified));
                hits.add(hit);
            }
            items = hits.iterator();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    private Date parseDate(String lastModified) {
        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
        try {
            return format.parse(lastModified);
        } catch (ParseException e) {
            log.warn("Failed to parse date {}", lastModified, e);
            return null;
        }
    }

{'fraction_non_alphanumeric': 0.08861067217697824, 'fraction_numerical': 0.002917223775373769, 'mean_word_length': 2.4126918291165493, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3529844', 'n_tokens_mistral': 2330, 'n_tokens_neox': 2232, 'n_words': 499}