<commit_before>package org.gbif.checklistbank.nub.lookup;

import org.gbif.api.model.Constants;
import org.gbif.api.vocabulary.Kingdom;
import org.gbif.api.vocabulary.Rank;
import org.gbif.checklistbank.authorship.AuthorComparator;
import org.gbif.checklistbank.cli.common.ClbConfiguration;
import org.gbif.checklistbank.cli.common.MapDbObjectSerializer;
import org.gbif.checklistbank.model.Equality;
import org.gbif.checklistbank.postgres.TabMapperBase;

import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

import com.google.common.collect.ImmutableList;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IdLookup implements Iterable<LookupUsage> {
    private static final Logger LOG = LoggerFactory.getLogger(IdLookup.class);
    private final DB db;
    private final Map<String, List<LookupUsage>> usages;
    private final AuthorComparator authComp;
    private int counter = 0;
    private int deleted = 0;

    private IdLookup() {
        db = DBMaker.tempFileDB()
                .fileMmapEnableIfSupported()
                .transactionDisable()
                .make();
        usages = db.hashMapCreate("usages")
                .keySerializer(Serializer.STRING_ASCII)
                .valueSerializer(new MapDbObjectSerializer<ArrayList>(ArrayList.class))
                .make();
        //TODO: load known author abbreviations
        authComp = new AuthorComparator();
    }

    /**
     * Creates idlookup with explicit list of known ids.
     */
    public IdLookup(Collection<LookupUsage> usages) {
        this();
        for (LookupUsage u : usages) {
            add(u);
        }
    }

    /**
     * Loads known usages from checklistbank backbone.
     */
    public IdLookup(ClbConfiguration clb) throws SQLException, IOException {
        this();
        UsageWriter writer = new UsageWriter();
        LOG.info("Reading existing nub usages from postgres...");
        try (Connection c = clb.connect()){
            final CopyManager cm = new CopyManager((BaseConnection) c);
            cm.copyOut("COPY ("
                    + "SELECT u.id, n.canonical_name, n.authorship, n.year, u.rank, u.kingdom_fk, false"
                    + " FROM name_usage u join name n ON name_fk=n.id"
                    + " WHERE dataset_key = '" + Constants.NUB_DATASET_KEY + "')"
                    + " TO STDOUT WITH NULL ''", writer);
            LOG.info("Loaded existing nub with {} usages into id lookup", usages.size());
        } finally {
            writer.close();
        }
    }

    /**
     * int key
     * String canonical
     * String authorship
     * String year
     * Rank rank
     * Kingdom kingdom
     * boolean deleted
     */
    private class UsageWriter extends TabMapperBase {
        public UsageWriter() {
            // the number of columns in our query to consume
            super(7);
        }

        @Override
        protected void addRow(String[] row) {
            LookupUsage u = new LookupUsage(
                    toInt(row[0]),
                    row[1],
                    row[2],
                    row[3],
                    Rank.valueOf(row[4]),
                    Kingdom.byNubUsageId(Integer.valueOf(row[5])),
                    "t".equals(row[6])
            );
            add(u);
        }

        private Integer toInt(String x) {
            return x == null ? null : Integer.valueOf(x);
        }
    }

    private void add(LookupUsage u) {
        if (usages.containsKey(u.getCanonical())) {
            // we need to create a new list cause mapdb considers them immutable!
            usages.put(u.getCanonical(), ImmutableList.<LookupUsage>builder().addAll(usages.get(u.getCanonical())).add(u).build());
        } else {
            usages.put(u.getCanonical(), ImmutableList.of(u));
        }
        counter++;
        if (u.isDeleted()) {
            deleted++;
        }
    }

    public LookupUsage match(String canonicalName, Rank rank, Kingdom kingdom) {
        return match(canonicalName, null, null, rank, kingdom);
    }

    private boolean match(Rank r1, Rank r2) {
        if (r1 == Rank.INFRASPECIFIC_NAME) {
            return r2.isInfraspecific();
        } else if (r1 == Rank.INFRASUBSPECIFIC_NAME) {
            return r2.isInfraspecific() && r2 != Rank.SUBSPECIES;

        } else if (r2 == Rank.INFRASPECIFIC_NAME) {
            return r1.isInfraspecific();
        } else if (r2 == Rank.INFRASUBSPECIFIC_NAME) {
            return r1.isInfraspecific() && r1 != Rank.SUBSPECIES;
        }
<commit_msg>Ignore names without canonical<commit_after>package org.gbif.checklistbank.nub.lookup;

import org.gbif.api.model.Constants;
import org.gbif.api.vocabulary.Kingdom;
import org.gbif.api.vocabulary.Rank;
import org.gbif.checklistbank.authorship.AuthorComparator;
import org.gbif.checklistbank.cli.common.ClbConfiguration;
import org.gbif.checklistbank.cli.common.MapDbObjectSerializer;
import org.gbif.checklistbank.model.Equality;
import org.gbif.checklistbank.postgres.TabMapperBase;

import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;

import com.google.common.collect.ImmutableList;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.Serializer;
import org.postgresql.copy.CopyManager;
import org.postgresql.core.BaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class IdLookup implements Iterable<LookupUsage> {
    private static final Logger LOG = LoggerFactory.getLogger(IdLookup.class);
    private final DB db;
    private final Map<String, List<LookupUsage>> usages;
    private final AuthorComparator authComp;
    private int counter = 0;
    private int deleted = 0;

    private IdLookup() {
        db = DBMaker.tempFileDB()
                .fileMmapEnableIfSupported()
                .transactionDisable()
                .make();
        usages = db.hashMapCreate("usages")
                .keySerializer(Serializer.STRING_ASCII)
                .valueSerializer(new MapDbObjectSerializer<ArrayList>(ArrayList.class))
                .make();
        //TODO: load known author abbreviations
        authComp = new AuthorComparator();
    }

    /**
     * Creates idlookup with explicit list of known ids.
     */
    public IdLookup(Collection<LookupUsage> usages) {
        this();
        for (LookupUsage u : usages) {
            add(u);
        }
    }

    /**
     * Loads known usages from checklistbank backbone.
     */
    public IdLookup(ClbConfiguration clb) throws SQLException, IOException {
        this();
        UsageWriter writer = new UsageWriter();
        LOG.info("Reading existing nub usages from postgres...");
        try (Connection c = clb.connect()){
            final CopyManager cm = new CopyManager((BaseConnection) c);
            cm.copyOut("COPY ("
                    + "SELECT u.id, n.canonical_name, n.authorship, n.year, u.rank, u.kingdom_fk, false"
                    + " FROM name_usage u join name n ON name_fk=n.id"
                    + " WHERE dataset_key = '" + Constants.NUB_DATASET_KEY + "')"
                    + " TO STDOUT WITH NULL ''", writer);
            LOG.info("Loaded existing nub with {} usages into id lookup", usages.size());
        } finally {
            writer.close();
        }
    }

    /**
     * int key
     * String canonical
     * String authorship
     * String year
     * Rank rank
     * Kingdom kingdom
     * boolean deleted
     */
    private class UsageWriter extends TabMapperBase {
        public UsageWriter() {
            // the number of columns in our query to consume
            super(7);
        }

        @Override
        protected void addRow(String[] row) {
            LookupUsage u = new LookupUsage(
                    toInt(row[0]),
                    row[1],
                    row[2],
                    row[3],
                    Rank.valueOf(row[4]),
                    Kingdom.byNubUsageId(Integer.valueOf(row[5])),
                    "t".equals(row[6])
            );
            add(u);
        }

        private Integer toInt(String x) {
            return x == null ? null : Integer.valueOf(x);
        }
    }

    private void add(LookupUsage u) {
        if (u.getCanonical() == null) {
            LOG.warn("Ignore previous usage {} without canonical name", u.getKey());
        } else {
            if (usages.containsKey(u.getCanonical())) {
                // we need to create a new list cause mapdb considers them immutable!
                usages.put(u.getCanonical(), ImmutableList.<LookupUsage>builder().addAll(usages.get(u.getCanonical())).add(u).build());
            } else {
                usages.put(u.getCanonical(), ImmutableList.of(u));
            }
            counter++;
            if (u.isDeleted()) {
                deleted++;
            }
        }
    }

    public LookupUsage match(String canonicalName, Rank rank, Kingdom kingdom) {
        return match(canonicalName, null, null, rank, kingdom);
    }

    private boolean match(Rank r1, Rank r2) {
        if (r1 == Rank.INFRASPECIFIC_NAME) {
            return r2.isInfraspecific();
        } else if (r1 == Rank.INFRASUBSPECIFIC_NAME) {
            return r2.isInfraspecific() && r2 != Rank.SUBSPECIES;

        } else if (r2 == Rank.INFRASPECIFIC_NAME) {
            return r1.isInfraspecific();
        } else if (r2 == Rank.INFRASUBSPECIFIC_NAME) {
            return r1.isInfraspecific() && r1 != Rank.SUBSPECIES;
        }

{'fraction_non_alphanumeric': 0.10819737763987589, 'fraction_numerical': 0.004804323891502352, 'mean_word_length': 2.1700507614213196, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7394790', 'n_tokens_mistral': 2948, 'n_tokens_neox': 2854, 'n_words': 658}