<commit_before>package org.gbif.nub.lookup;

import org.gbif.api.model.Constants;
import org.gbif.api.model.checklistbank.NameUsage;
import org.gbif.api.model.checklistbank.NameUsageContainer;
import org.gbif.checklistbank.index.ThreadPoolRunner;
import org.gbif.checklistbank.service.UsageService;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.lucene.index.IndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A multithreaded nub index builder reading the backbone from a checkist bank postgres database in batches.
 * It first reads out all usage id ints and partitions those in equal ranges of 20.000
 */
public class NubIndexBuilder extends ThreadPoolRunner<List<NameUsageContainer>> {

  private static final Logger LOG = LoggerFactory.getLogger(NubIndexBuilder.class);

  private final IndexWriter writer;
  private AtomicInteger counter = new AtomicInteger();
  private UsageService usageService;
  private int offset;
  private final int max;
  // the size of the key range used to get usages.
  // Not all keys exist and they are not evenly distributed, so this is the max of usages returned, but its usually less
  private final int window;

  NubIndexBuilder(IndexWriter writer, UsageService usageService, int threads) {
    super(threads);
    this.writer = writer;
    this.usageService = usageService;
    this.offset = 0;
    this.max = usageService.maxUsageKey(Constants.NUB_DATASET_KEY);
    this.window = 20000;
    // use sth like the following for local debugging: (puma concolor id)
    //this.window = 100;
    //this.offset = 2435099-window;
    //this.max = offset + 2*window;
  }

  @Override
  protected Callable<List<NameUsageContainer>> newJob() {<commit_msg>More nub index building logging<commit_after>package org.gbif.nub.lookup;

import org.gbif.api.model.Constants;
import org.gbif.api.model.checklistbank.NameUsage;
import org.gbif.api.model.checklistbank.NameUsageContainer;
import org.gbif.checklistbank.index.ThreadPoolRunner;
import org.gbif.checklistbank.service.UsageService;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.lucene.index.IndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A multithreaded nub index builder reading the backbone from a checkist bank postgres database in batches.
 * It first reads out all usage id ints and partitions those in equal ranges of 20.000
 */
public class NubIndexBuilder extends ThreadPoolRunner<List<NameUsageContainer>> {

  private static final Logger LOG = LoggerFactory.getLogger(NubIndexBuilder.class);

  private final IndexWriter writer;
  private AtomicInteger counter = new AtomicInteger();
  private UsageService usageService;
  private int offset;
  private final int max;
  // the size of the key range used to get usages.
  // Not all keys exist and they are not evenly distributed, so this is the max of usages returned, but its usually less
  private final int window;

  NubIndexBuilder(IndexWriter writer, UsageService usageService, int threads) {
    super(threads);
    this.writer = writer;
    this.usageService = usageService;
    this.offset = 0;
    this.max = usageService.maxUsageKey(Constants.NUB_DATASET_KEY);
    this.window = 20000;
    LOG.info("Retrieved biggest nub usage key={}. Building the index from {} batches with {} usages each", max, max/window, window);
    // use sth like the following for local debugging: (puma concolor id)
    //this.window = 100;
    //this.offset = 2435099-window;
    //this.max = offset + 2*window;
  }

  @Override
  protected Callable<List<NameUsageContainer>> newJob() {
{'fraction_non_alphanumeric': 0.08629308076821889, 'fraction_numerical': 0.012628255722178374, 'mean_word_length': 4.4314285714285715, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '140760', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1094, 'n_words': 346}