<commit_before>
-- required extensions
CREATE EXTENSION IF NOT EXISTS hstore;
CREATE EXTENSION IF NOT EXISTS unaccent;
CREATE EXTENSION IF NOT EXISTS pg_trgm;

-- use unaccent by default for all simple search
CREATE TEXT SEARCH CONFIGURATION public.simple2 ( COPY = pg_catalog.simple );
ALTER TEXT SEARCH CONFIGURATION simple2 ALTER MAPPING FOR hword, hword_part, word WITH unaccent;

-- immutable unaccent function to be used in indexes
-- see https://stackoverflow.com/questions/11005036/does-postgresql-support-accent-insensitive-collations
CREATE OR REPLACE FUNCTION f_unaccent(text)
  RETURNS text AS
$func$
SELECT public.unaccent('public.unaccent', $1)  -- schema-qualify function and dictionary
$func$  LANGUAGE sql IMMUTABLE;


-- all enum types produces via PgSetupRuleTest.pgEnumSql()
CREATE TYPE AREASTANDARD AS ENUM (
  'TDWG',
  'ISO',
  'FAO',
  'FAO_FISHING',
  'LONGHURST',
  'TEOW',
  'IHO',
  'TEXT'
);

CREATE TYPE COLUSER_ROLE AS ENUM (
  'USER',
  'EDITOR',
  'ADMIN'
);

CREATE TYPE CONTINENT AS ENUM (
  'AFRICA',
  'ANTARCTICA',
  'ASIA',
  'OCEANIA',
  'EUROPE',
  'NORTH_AMERICA',
  'SOUTH_AMERICA'
);

CREATE TYPE DATAFORMAT AS ENUM (
  'DWCA',
  'ACEF',
  'TCS',
  'COLDP',
  'PROXY'
);

CREATE TYPE DATASETORIGIN AS ENUM (
  'EXTERNAL',
  'UPLOADED',
  'MANAGED'
);

CREATE TYPE DATASETTYPE AS ENUM (
  'NOMENCLATURAL',
  'TAXONOMIC',
  'ARTICLE',
  'PERSONAL',
  'OTU',
  'THEMATIC',
  'OTHER'
);

CREATE TYPE DESCRIPTIONCATEGORY AS ENUM (
  'BIOLOGY',
  'CONSERVATION',
  'HABITAT',
  'USE',
  'DISTRIBUTION',
  'DESCRIPTION',
  'ETYMOLOGY',
  'TREATMENT',
  'MISCELLANEOUS',
  'NOMENCLATURE',
  'STRATIGRAPHY',
  'TAXONOMY',
  'TYPIFICATION',
  'OTHER'
);

CREATE TYPE DISTRIBUTIONSTATUS AS ENUM (
  'NATIVE',
  'DOMESTICATED',
  'ALIEN',
  'UNCERTAIN'
);

CREATE TYPE EDITORIALDECISION_MODE AS ENUM (
  'BLOCK',
  'REVIEWED',
  'UPDATE',
  'UPDATE_RECURSIVE'
);

CREATE TYPE ENTITYTYPE AS ENUM (
  'ANY',
  'NAME',
  'NAME_RELATION',
  'NAME_USAGE',
  'TYPE_MATERIAL',
  'DESCRIPTION',
  'DISTRIBUTION',
  'MEDIA',
  'VERNACULAR',
  'REFERENCE'
);

CREATE TYPE ESTIMATETYPE AS ENUM (
  'DESCRIBED_SPECIES_LIVING',
  'DESCRIBED_SPECIES_EXTINCT',
  'ESTIMATED_SPECIES'
);

CREATE TYPE GAZETTEER AS ENUM (
  'TDWG',
  'ISO',
  'FAO',
  'LONGHURST',
  'TEOW',
  'IHO',
  'TEXT'
);

CREATE TYPE IMPORTSTATE AS ENUM (
  'WAITING',
  'DOWNLOADING',
  'PROCESSING',
  'INSERTING',
  'INDEXING',
  'BUILDING_METRICS',
  'UNCHANGED',
  'FINISHED',
  'CANCELED',
  'FAILED'
);

CREATE TYPE ISSUE AS ENUM (
  'NOT_INTERPRETED',
  'ESCAPED_CHARACTERS',
  'REFERENCE_ID_INVALID',
  'ID_NOT_UNIQUE',
  'URL_INVALID',
  'PARTIAL_DATE',
  'PREVIOUS_LINE_SKIPPED',
  'UNPARSABLE_NAME',
  'PARTIALLY_PARSABLE_NAME',
  'UNPARSABLE_AUTHORSHIP',
  'DOUBTFUL_NAME',
  'INCONSISTENT_AUTHORSHIP',
  'INCONSISTENT_NAME',
  'PARSED_NAME_DIFFERS',
  'UNUSUAL_NAME_CHARACTERS',
  'MULTI_WORD_EPITHET',<commit_msg>Add DECISION_MATCHING to pg enum<commit_after>
-- required extensions
CREATE EXTENSION IF NOT EXISTS hstore;
CREATE EXTENSION IF NOT EXISTS unaccent;
CREATE EXTENSION IF NOT EXISTS pg_trgm;

-- use unaccent by default for all simple search
CREATE TEXT SEARCH CONFIGURATION public.simple2 ( COPY = pg_catalog.simple );
ALTER TEXT SEARCH CONFIGURATION simple2 ALTER MAPPING FOR hword, hword_part, word WITH unaccent;

-- immutable unaccent function to be used in indexes
-- see https://stackoverflow.com/questions/11005036/does-postgresql-support-accent-insensitive-collations
CREATE OR REPLACE FUNCTION f_unaccent(text)
  RETURNS text AS
$func$
SELECT public.unaccent('public.unaccent', $1)  -- schema-qualify function and dictionary
$func$  LANGUAGE sql IMMUTABLE;


-- all enum types produces via PgSetupRuleTest.pgEnumSql()
CREATE TYPE AREASTANDARD AS ENUM (
  'TDWG',
  'ISO',
  'FAO',
  'FAO_FISHING',
  'LONGHURST',
  'TEOW',
  'IHO',
  'TEXT'
);

CREATE TYPE COLUSER_ROLE AS ENUM (
  'USER',
  'EDITOR',
  'ADMIN'
);

CREATE TYPE CONTINENT AS ENUM (
  'AFRICA',
  'ANTARCTICA',
  'ASIA',
  'OCEANIA',
  'EUROPE',
  'NORTH_AMERICA',
  'SOUTH_AMERICA'
);

CREATE TYPE DATAFORMAT AS ENUM (
  'DWCA',
  'ACEF',
  'TCS',
  'COLDP',
  'PROXY'
);

CREATE TYPE DATASETORIGIN AS ENUM (
  'EXTERNAL',
  'UPLOADED',
  'MANAGED'
);

CREATE TYPE DATASETTYPE AS ENUM (
  'NOMENCLATURAL',
  'TAXONOMIC',
  'ARTICLE',
  'PERSONAL',
  'OTU',
  'THEMATIC',
  'OTHER'
);

CREATE TYPE DESCRIPTIONCATEGORY AS ENUM (
  'BIOLOGY',
  'CONSERVATION',
  'HABITAT',
  'USE',
  'DISTRIBUTION',
  'DESCRIPTION',
  'ETYMOLOGY',
  'TREATMENT',
  'MISCELLANEOUS',
  'NOMENCLATURE',
  'STRATIGRAPHY',
  'TAXONOMY',
  'TYPIFICATION',
  'OTHER'
);

CREATE TYPE DISTRIBUTIONSTATUS AS ENUM (
  'NATIVE',
  'DOMESTICATED',
  'ALIEN',
  'UNCERTAIN'
);

CREATE TYPE EDITORIALDECISION_MODE AS ENUM (
  'BLOCK',
  'REVIEWED',
  'UPDATE',
  'UPDATE_RECURSIVE'
);

CREATE TYPE ENTITYTYPE AS ENUM (
  'ANY',
  'NAME',
  'NAME_RELATION',
  'NAME_USAGE',
  'TYPE_MATERIAL',
  'DESCRIPTION',
  'DISTRIBUTION',
  'MEDIA',
  'VERNACULAR',
  'REFERENCE'
);

CREATE TYPE ESTIMATETYPE AS ENUM (
  'DESCRIBED_SPECIES_LIVING',
  'DESCRIBED_SPECIES_EXTINCT',
  'ESTIMATED_SPECIES'
);

CREATE TYPE GAZETTEER AS ENUM (
  'TDWG',
  'ISO',
  'FAO',
  'LONGHURST',
  'TEOW',
  'IHO',
  'TEXT'
);

CREATE TYPE IMPORTSTATE AS ENUM (
  'WAITING',
  'DOWNLOADING',
  'PROCESSING',
  'INSERTING',
  'INDEXING',
  'DECISION_MATCHING',
  'BUILDING_METRICS',
  'UNCHANGED',
  'FINISHED',
  'CANCELED',
  'FAILED'
);

CREATE TYPE ISSUE AS ENUM (
  'NOT_INTERPRETED',
  'ESCAPED_CHARACTERS',
  'REFERENCE_ID_INVALID',
  'ID_NOT_UNIQUE',
  'URL_INVALID',
  'PARTIAL_DATE',
  'PREVIOUS_LINE_SKIPPED',
  'UNPARSABLE_NAME',
  'PARTIALLY_PARSABLE_NAME',
  'UNPARSABLE_AUTHORSHIP',
  'DOUBTFUL_NAME',
  'INCONSISTENT_AUTHORSHIP',
  'INCONSISTENT_NAME',
  'PARSED_NAME_DIFFERS',
  'UNUSUAL_NAME_CHARACTERS',
  'MULTI_WORD_EPITHET',
{'fraction_non_alphanumeric': 0.13257317903335603, 'fraction_numerical': 0.0037440435670524166, 'mean_word_length': 4.372029250457039, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4891289', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2529, 'n_words': 516}