<commit_before>package org.gbif.checklistbank.index;

import org.gbif.api.model.checklistbank.Description;
import org.gbif.api.model.checklistbank.Distribution;
import org.gbif.api.model.checklistbank.SpeciesProfile;
import org.gbif.api.service.checklistbank.DescriptionService;
import org.gbif.api.service.checklistbank.DistributionService;
import org.gbif.api.service.checklistbank.SpeciesProfileService;
import org.gbif.api.service.checklistbank.VernacularNameService;
import org.gbif.checklistbank.service.UsageService;
import org.gbif.checklistbank.service.mybatis.DescriptionServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.DistributionServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.SpeciesProfileServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.VernacularNameServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.guice.ChecklistBankServiceMyBatisModule;
import org.gbif.utils.file.properties.PropertiesUtil;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import com.google.inject.Guice;
import com.google.inject.Injector;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**
 * Do a doc conversion using a real db. Manual test only to debug problems.
 */
@Ignore
public class NameUsageIndexingJobIT {
  private UsageService nameUsageService;
  private VernacularNameServiceMyBatis vernacularNameService;
  private DescriptionServiceMyBatis descriptionService;
  private DistributionServiceMyBatis distributionService;
  private SpeciesProfileServiceMyBatis speciesProfileService;

  @Before
  public void init() throws IOException {
    Properties props = PropertiesUtil.loadProperties("checklistbank.properties");
    props.list(System.out);
    Injector inj = Guice.createInjector(new ChecklistBankServiceMyBatisModule(props));
    nameUsageService = inj.getInstance(UsageService.class);
    vernacularNameService = (VernacularNameServiceMyBatis) inj.getInstance(VernacularNameService.class);
    descriptionService = (DescriptionServiceMyBatis) inj.getInstance(DescriptionService.class);
    distributionService = (DistributionServiceMyBatis) inj.getInstance(DistributionService.class);
    speciesProfileService = (SpeciesProfileServiceMyBatis) inj.getInstance(SpeciesProfileService.class);
  }

  @Test
  public void testSpeciesInfoRange() throws Exception {
    int start = 114989960;
    int end   = 114989970;
    Map<Integer, List<SpeciesProfile>> range = speciesProfileService.listRange(start, end);
    assertEquals(0, range.size());
  }

  @Test
  public void testDescriptionRange() throws Exception {
    int start = 110448010;
    int end   = 110448020;
    Map<Integer, List<Description>> range = descriptionService.listRange(start, end);
    assertEquals(10, range.size());
  }

  @Test
  public void testDistributionRange() throws Exception {
    int start = 110448010;
    int end   = 110448020;
    Map<Integer, List<Distribution>> range = distributionService.listRange(start, end);
    assertEquals(0, range.size());
  }

  @Test
  public void testCall() throws Exception {
    int start = 114989960;
    int end   = 114989970;
    start = 110448010;
    end   = 110448020;
    NameUsageIndexingJob job = new NameUsageIndexingJob(nameUsageService, start, end,
      vernacularNameService, descriptionService, distributionService, speciesProfileService);
    job.call();
  }
}
<commit_msg>reverting test to use Solr indexing<commit_after>package org.gbif.checklistbank.index;

import org.gbif.api.model.checklistbank.Description;
import org.gbif.api.model.checklistbank.Distribution;
import org.gbif.api.model.checklistbank.SpeciesProfile;
import org.gbif.api.service.checklistbank.DescriptionService;
import org.gbif.api.service.checklistbank.DistributionService;
import org.gbif.api.service.checklistbank.SpeciesProfileService;
import org.gbif.api.service.checklistbank.VernacularNameService;
import org.gbif.checklistbank.service.UsageService;
import org.gbif.checklistbank.service.mybatis.DescriptionServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.DistributionServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.SpeciesProfileServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.VernacularNameServiceMyBatis;
import org.gbif.checklistbank.service.mybatis.guice.ChecklistBankServiceMyBatisModule;
import org.gbif.common.search.solr.builders.EmbeddedServerBuilder;
import org.gbif.utils.file.properties.PropertiesUtil;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import com.google.inject.Guice;
import com.google.inject.Injector;
import org.apache.solr.client.solrj.SolrServer;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;

import static org.junit.Assert.assertEquals;

/**
 * Do a doc conversion using a real db. Manual test only to debug problems.
 */
@Ignore
public class NameUsageIndexingJobIT {
  private UsageService nameUsageService;
  private VernacularNameServiceMyBatis vernacularNameService;
  private DescriptionServiceMyBatis descriptionService;
  private DistributionServiceMyBatis distributionService;
  private SpeciesProfileServiceMyBatis speciesProfileService;
  private SolrServer solr;

  @Before
  public void init() throws IOException {
    Properties props = PropertiesUtil.loadProperties("checklistbank.properties");
    props.list(System.out);
    Injector inj = Guice.createInjector(new ChecklistBankServiceMyBatisModule(props));
    nameUsageService = inj.getInstance(UsageService.class);
    vernacularNameService = (VernacularNameServiceMyBatis) inj.getInstance(VernacularNameService.class);
    descriptionService = (DescriptionServiceMyBatis) inj.getInstance(DescriptionService.class);
    distributionService = (DistributionServiceMyBatis) inj.getInstance(DistributionService.class);
    speciesProfileService = (SpeciesProfileServiceMyBatis) inj.getInstance(SpeciesProfileService.class);
    solr = new EmbeddedServerBuilder().build();
  }

  @Test
  public void testSpeciesInfoRange() throws Exception {
    int start = 114989960;
    int end   = 114989970;
    Map<Integer, List<SpeciesProfile>> range = speciesProfileService.listRange(start, end);
    assertEquals(0, range.size());
  }

  @Test
  public void testDescriptionRange() throws Exception {
    int start = 110448010;
    int end   = 110448020;
    Map<Integer, List<Description>> range = descriptionService.listRange(start, end);
    assertEquals(10, range.size());
  }

  @Test
  public void testDistributionRange() throws Exception {
    int start = 110448010;
    int end   = 110448020;
    Map<Integer, List<Distribution>> range = distributionService.listRange(start, end);
    assertEquals(0, range.size());
  }

  @Test
  public void testCall() throws Exception {
    int start = 114989960;
    int end   = 114989970;
    start = 110448010;
    end   = 110448020;
    NameUsageIndexingJob job = new NameUsageIndexingJob(solr, nameUsageService, start, end, new NameUsageDocConverter(),
      vernacularNameService, descriptionService, distributionService, speciesProfileService);
    job.call();
  }
}
{'fraction_non_alphanumeric': 0.08948824343015215, 'fraction_numerical': 0.02600276625172891, 'mean_word_length': 5.6400367309458215, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6965514', 'n_tokens_mistral': 2246, 'n_tokens_neox': 2075, 'n_words': 410}