<commit_before>/*Create database*/
CREATE DATABASE data_quality;
\connect data_quality;


/*Create schema*/
CREATE SCHEMA base;


/*Create functions*/
CREATE OR REPLACE FUNCTION base.update_updated_date_column()
RETURNS TRIGGER AS $$
BEGIN
   NEW.updated_date = now();
   RETURN NEW;
END;
$$ language 'plpgsql';


/*Create table data source type*/
CREATE TABLE base.data_source_type (
    id SERIAL PRIMARY KEY,
    data_source_type TEXT NOT NULL,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.data_source_type IS
'Data source types describe the types of a data sources the data quality framework can connect to.';

CREATE TRIGGER data_source_type_updated_date BEFORE UPDATE
ON base.data_source_type FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();

INSERT INTO base.data_source_type (data_source_type) VALUES
('Hive'),
('Impala'),
('Microsoft SQL Server'),
('MySQL'),
('PostgreSQL'),
('SQLite'),
('Teradata');


CREATE TABLE base.data_source (
    id SERIAL PRIMARY KEY,
    data_source TEXT NOT NULL,
    connection_string TEXT,
    login TEXT,
    password TEXT,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    data_source_type_id INTEGER NOT NULL REFERENCES base.data_source_type(id)
);

COMMENT ON TABLE base.data_source IS
'Data sources are systems containing or exposing data on which the data quality framework can compute indicators.';

CREATE TRIGGER data_source_updated_date BEFORE UPDATE
ON base.data_source FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.indicator_type (
    id SERIAL PRIMARY KEY,
    indicator_type TEXT NOT NULL,
    function TEXT NOT NULL,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.indicator_type IS
'Indicator types determine which function of the data quality framework is used to compute indicators.';

CREATE TRIGGER indicator_type_updated_date BEFORE UPDATE
ON base.indicator_type FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();

INSERT INTO base.indicator_type (indicator_type, function) VALUES
('Completeness', 'evaluate_completeness'),
('Freshness', 'evaluate_freshness'),
('Latency', 'evaluate_latency'),
('Validity', 'evaluate_validity');


CREATE TABLE base.indicator_group (
    id SERIAL PRIMARY KEY,
    indicator_group TEXT NOT NULL,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.indicator_group IS
'Indicator groups define collections of indicators to be computed in the same batch.';

CREATE TRIGGER indicator_group_updated_date BEFORE UPDATE
ON base.indicator_group FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.indicator (
    id SERIAL PRIMARY KEY,
    indicator TEXT NOT NULL,
    description TEXT,
    execution_order INTEGER DEFAULT 0,
    flag_active BOOLEAN DEFAULT FALSE,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    indicator_type_id INTEGER NOT NULL REFERENCES base.indicator_type(id),
    indicator_group_id INTEGER NOT NULL REFERENCES base.indicator_group(id)
);

COMMENT ON TABLE base.indicator IS
'Indicators compute data sets on one or several data sources in order to evaluate the quality of their data.';

CREATE TRIGGER indicator_updated_date BEFORE UPDATE
ON base.indicator FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.parameter_type (
    id SERIAL PRIMARY KEY,
    parameter_type TEXT NOT NULL,
    description TEXT,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
<commit_msg>Added unicity constraint on columns.<commit_after>/*Create database*/
CREATE DATABASE data_quality;
\connect data_quality;


/*Create schema*/
CREATE SCHEMA base;


/*Create functions*/
CREATE OR REPLACE FUNCTION base.update_updated_date_column()
RETURNS TRIGGER AS $$
BEGIN
   NEW.updated_date = now();
   RETURN NEW;
END;
$$ language 'plpgsql';


/*Create table data source type*/
CREATE TABLE base.data_source_type (
    id SERIAL PRIMARY KEY,
    data_source_type TEXT NOT NULL UNIQUE,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.data_source_type IS
'Data source types describe the types of a data sources the data quality framework can connect to.';

CREATE TRIGGER data_source_type_updated_date BEFORE UPDATE
ON base.data_source_type FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();

INSERT INTO base.data_source_type (data_source_type) VALUES
('Hive'),
('Impala'),
('Microsoft SQL Server'),
('MySQL'),
('PostgreSQL'),
('SQLite'),
('Teradata');


CREATE TABLE base.data_source (
    id SERIAL PRIMARY KEY,
    data_source TEXT NOT NULL UNIQUE,
    connection_string TEXT,
    login TEXT,
    password TEXT,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    data_source_type_id INTEGER NOT NULL REFERENCES base.data_source_type(id)
);

COMMENT ON TABLE base.data_source IS
'Data sources are systems containing or exposing data on which the data quality framework can compute indicators.';

CREATE TRIGGER data_source_updated_date BEFORE UPDATE
ON base.data_source FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.indicator_type (
    id SERIAL PRIMARY KEY,
    indicator_type TEXT NOT NULL UNIQUE,
    function TEXT NOT NULL,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.indicator_type IS
'Indicator types determine which function of the data quality framework is used to compute indicators.';

CREATE TRIGGER indicator_type_updated_date BEFORE UPDATE
ON base.indicator_type FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();

INSERT INTO base.indicator_type (indicator_type, function) VALUES
('Completeness', 'evaluate_completeness'),
('Freshness', 'evaluate_freshness'),
('Latency', 'evaluate_latency'),
('Validity', 'evaluate_validity');


CREATE TABLE base.indicator_group (
    id SERIAL PRIMARY KEY,
    indicator_group TEXT NOT NULL UNIQUE,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

COMMENT ON TABLE base.indicator_group IS
'Indicator groups define collections of indicators to be computed in the same batch.';

CREATE TRIGGER indicator_group_updated_date BEFORE UPDATE
ON base.indicator_group FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.indicator (
    id SERIAL PRIMARY KEY,
    indicator TEXT NOT NULL UNIQUE,
    description TEXT,
    execution_order INTEGER DEFAULT 0,
    flag_active BOOLEAN DEFAULT FALSE,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    indicator_type_id INTEGER NOT NULL REFERENCES base.indicator_type(id),
    indicator_group_id INTEGER NOT NULL REFERENCES base.indicator_group(id)
);

COMMENT ON TABLE base.indicator IS
'Indicators compute data sets on one or several data sources in order to evaluate the quality of their data.';

CREATE TRIGGER indicator_updated_date BEFORE UPDATE
ON base.indicator FOR EACH ROW EXECUTE PROCEDURE
base.update_updated_date_column();


CREATE TABLE base.parameter_type (
    id SERIAL PRIMARY KEY,
    parameter_type TEXT NOT NULL UNIQUE,
    description TEXT,
    created_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
    updated_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

{'fraction_non_alphanumeric': 0.05815894209783028, 'fraction_numerical': 0.0002567723712928489, 'mean_word_length': 5.232, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3068453', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2289, 'n_words': 786}