<commit_before>package whelk.component

import groovy.transform.Canonical
import groovy.transform.CompileStatic
import whelk.util.LegacyIntegrationTools

import static groovy.transform.TypeCheckingMode.SKIP
import groovy.util.logging.Log4j2 as Log
import groovy.json.StringEscapeUtils
import org.apache.commons.dbcp2.BasicDataSource
import org.codehaus.jackson.map.ObjectMapper
import org.postgresql.PGStatement
import org.postgresql.util.PSQLException
import whelk.Document
import whelk.IdType
import whelk.JsonLd
import whelk.Location
import whelk.exception.StorageCreateFailedException
import whelk.exception.TooHighEncodingLevelException
import whelk.filter.LinkFinder

import java.sql.*
import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.Date
import java.util.regex.Matcher
import java.util.regex.Pattern

@Log
@CompileStatic
class PostgreSQLComponent {

    /**
     * Interface for performing atomic document updates
     */
    public interface UpdateAgent {
        public void update(Document doc)
    }

    private BasicDataSource connectionPool
    static String driverClass = "org.postgresql.Driver"

    public final static ObjectMapper mapper = new ObjectMapper()

    boolean versioning = true

    /**
     * This value is sensitiv. It must be strictly larger than the maxConnections parameter set in tomcat.
     * This is necessary in order to not have the very expensive deadlock/connection-starvation prevention code
     * in here.
     */
    final int MAX_CONNECTION_COUNT = 300

    // SQL statements
    protected String UPDATE_DOCUMENT, INSERT_DOCUMENT,
                     INSERT_DOCUMENT_VERSION, GET_DOCUMENT, GET_EMBELLISHED_DOCUMENT,
                     GET_DOCUMENT_VERSION, GET_ALL_DOCUMENT_VERSIONS,
                     GET_DOCUMENT_VERSION_BY_MAIN_ID,
                     GET_ALL_DOCUMENT_VERSIONS_BY_MAIN_ID,
                     GET_DOCUMENT_BY_SAMEAS_ID, LOAD_ALL_DOCUMENTS,
                     LOAD_ALL_DOCUMENTS_BY_COLLECTION,
                     DELETE_DOCUMENT_STATEMENT, STATUS_OF_DOCUMENT,
                     LOAD_ID_FROM_ALTERNATE, INSERT_IDENTIFIERS,
                     LOAD_RECORD_IDENTIFIERS, LOAD_THING_IDENTIFIERS, DELETE_IDENTIFIERS, LOAD_COLLECTIONS,
                     GET_DOCUMENT_FOR_UPDATE, GET_CONTEXT, GET_RECORD_ID_BY_THING_ID, GET_DEPENDENCIES, GET_DEPENDERS,
                     GET_DOCUMENT_BY_MAIN_ID, GET_RECORD_ID, GET_THING_ID, GET_MAIN_ID, GET_ID_TYPE
    protected String LOAD_SETTINGS, SAVE_SETTINGS
    protected String GET_DEPENDENCIES_OF_TYPE, GET_DEPENDERS_OF_TYPE
    protected String DELETE_DEPENDENCIES, INSERT_DEPENDENCIES
    protected String QUERY_LD_API
    protected String FIND_BY, COUNT_BY
    protected String GET_SYSTEMID_BY_IRI
    protected String GET_DOCUMENT_BY_IRI
    protected String GET_MINMAX_MODIFIED
    protected String UPDATE_MINMAX_MODIFIED
    protected String GET_LEGACY_PROFILE
    protected String INSERT_EMBELLISHED_DOCUMENT
    protected String DELETE_EMBELLISHED_DOCUMENT

    // Deprecated
    protected String LOAD_ALL_DOCUMENTS_WITH_LINKS, LOAD_ALL_DOCUMENTS_WITH_LINKS_BY_COLLECTION

    // Query defaults
    static final int DEFAULT_PAGE_SIZE = 50

    // Query idiomatic data
    static final Map<StorageType, String> SQL_PREFIXES = [
            (StorageType.JSONLD)                       : "data->'@graph'",
            (StorageType.JSONLD_FLAT_WITH_DESCRIPTIONS): "data->'descriptions'",
            (StorageType.MARC21_JSON)                  : "data->'fields'"
    ]

    String mainTableName

    LinkFinder linkFinder

    final boolean trackConnectionFetching

    class AcquireLockException extends RuntimeException { AcquireLockException(String s) { super(s) } }

    class ConflictingHoldException extends RuntimeException { ConflictingHoldException(String s) { super(s) } }

    // for testing
    PostgreSQLComponent() {}

    PostgreSQLComponent(Properties properties) {
        this.trackConnectionFetching = trackConnectionFetching
        setup(properties.getProperty("sqlUrl"), properties.getProperty("sqlMaintable"))
    }

    PostgreSQLComponent(String sqlUrl, String sqlMaintable, boolean trackConnectionFetching = true) {
        this.trackConnectionFetching = trackConnectionFetching
        setup(sqlUrl, sqlMaintable)
    }

    private void setup(String sqlUrl, String sqlMaintable) {
        mainTableName = sqlMaintable
        String idTableName = mainTableName + "__identifiers"
        String versionsTableName = mainTableName + "__versions"
        String settingsTableName = mainTableName + "__settings"
        String dependenciesTableName = mainTableName + "__dependencies"
        String profilesTableName = mainTableName + "__profiles"
        String embellishedTableName = mainTableName + "__embellished"

        connectionPool = new BasicDataSource()
<commit_msg>Remove unused trackConnectionFetching from PostgreSQLComponent<commit_after>package whelk.component

import groovy.transform.Canonical
import groovy.transform.CompileStatic
import whelk.util.LegacyIntegrationTools

import static groovy.transform.TypeCheckingMode.SKIP
import groovy.util.logging.Log4j2 as Log
import groovy.json.StringEscapeUtils
import org.apache.commons.dbcp2.BasicDataSource
import org.codehaus.jackson.map.ObjectMapper
import org.postgresql.PGStatement
import org.postgresql.util.PSQLException
import whelk.Document
import whelk.IdType
import whelk.JsonLd
import whelk.Location
import whelk.exception.StorageCreateFailedException
import whelk.exception.TooHighEncodingLevelException
import whelk.filter.LinkFinder

import java.sql.*
import java.time.ZoneId
import java.time.ZonedDateTime
import java.time.format.DateTimeFormatter
import java.util.Date
import java.util.regex.Matcher
import java.util.regex.Pattern

@Log
@CompileStatic
class PostgreSQLComponent {

    /**
     * Interface for performing atomic document updates
     */
    public interface UpdateAgent {
        public void update(Document doc)
    }

    private BasicDataSource connectionPool
    static String driverClass = "org.postgresql.Driver"

    public final static ObjectMapper mapper = new ObjectMapper()

    boolean versioning = true

    /**
     * This value is sensitiv. It must be strictly larger than the maxConnections parameter set in tomcat.
     * This is necessary in order to not have the very expensive deadlock/connection-starvation prevention code
     * in here.
     */
    final int MAX_CONNECTION_COUNT = 300

    // SQL statements
    protected String UPDATE_DOCUMENT, INSERT_DOCUMENT,
                     INSERT_DOCUMENT_VERSION, GET_DOCUMENT, GET_EMBELLISHED_DOCUMENT,
                     GET_DOCUMENT_VERSION, GET_ALL_DOCUMENT_VERSIONS,
                     GET_DOCUMENT_VERSION_BY_MAIN_ID,
                     GET_ALL_DOCUMENT_VERSIONS_BY_MAIN_ID,
                     GET_DOCUMENT_BY_SAMEAS_ID, LOAD_ALL_DOCUMENTS,
                     LOAD_ALL_DOCUMENTS_BY_COLLECTION,
                     DELETE_DOCUMENT_STATEMENT, STATUS_OF_DOCUMENT,
                     LOAD_ID_FROM_ALTERNATE, INSERT_IDENTIFIERS,
                     LOAD_RECORD_IDENTIFIERS, LOAD_THING_IDENTIFIERS, DELETE_IDENTIFIERS, LOAD_COLLECTIONS,
                     GET_DOCUMENT_FOR_UPDATE, GET_CONTEXT, GET_RECORD_ID_BY_THING_ID, GET_DEPENDENCIES, GET_DEPENDERS,
                     GET_DOCUMENT_BY_MAIN_ID, GET_RECORD_ID, GET_THING_ID, GET_MAIN_ID, GET_ID_TYPE
    protected String LOAD_SETTINGS, SAVE_SETTINGS
    protected String GET_DEPENDENCIES_OF_TYPE, GET_DEPENDERS_OF_TYPE
    protected String DELETE_DEPENDENCIES, INSERT_DEPENDENCIES
    protected String QUERY_LD_API
    protected String FIND_BY, COUNT_BY
    protected String GET_SYSTEMID_BY_IRI
    protected String GET_DOCUMENT_BY_IRI
    protected String GET_MINMAX_MODIFIED
    protected String UPDATE_MINMAX_MODIFIED
    protected String GET_LEGACY_PROFILE
    protected String INSERT_EMBELLISHED_DOCUMENT
    protected String DELETE_EMBELLISHED_DOCUMENT

    // Deprecated
    protected String LOAD_ALL_DOCUMENTS_WITH_LINKS, LOAD_ALL_DOCUMENTS_WITH_LINKS_BY_COLLECTION

    // Query defaults
    static final int DEFAULT_PAGE_SIZE = 50

    // Query idiomatic data
    static final Map<StorageType, String> SQL_PREFIXES = [
            (StorageType.JSONLD)                       : "data->'@graph'",
            (StorageType.JSONLD_FLAT_WITH_DESCRIPTIONS): "data->'descriptions'",
            (StorageType.MARC21_JSON)                  : "data->'fields'"
    ]

    String mainTableName

    LinkFinder linkFinder

    class AcquireLockException extends RuntimeException { AcquireLockException(String s) { super(s) } }

    class ConflictingHoldException extends RuntimeException { ConflictingHoldException(String s) { super(s) } }

    // for testing
    PostgreSQLComponent() {}

    PostgreSQLComponent(Properties properties) {
        setup(properties.getProperty("sqlUrl"), properties.getProperty("sqlMaintable"))
    }

    PostgreSQLComponent(String sqlUrl, String sqlMaintable) {
        setup(sqlUrl, sqlMaintable)
    }

    private void setup(String sqlUrl, String sqlMaintable) {
        mainTableName = sqlMaintable
        String idTableName = mainTableName + "__identifiers"
        String versionsTableName = mainTableName + "__versions"
        String settingsTableName = mainTableName + "__settings"
        String dependenciesTableName = mainTableName + "__dependencies"
        String profilesTableName = mainTableName + "__profiles"
        String embellishedTableName = mainTableName + "__embellished"

        connectionPool = new BasicDataSource()

{'fraction_non_alphanumeric': 0.05380141254673868, 'fraction_numerical': 0.002077274615704196, 'mean_word_length': 3.5527186761229315, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2277696', 'n_tokens_mistral': 2868, 'n_tokens_neox': 2697, 'n_words': 592}