<commit_before>--
-- Accounting for Hadoop
--
-- How to add values:
--
-- INSERT INTO measure (name) VALUES ('quota');
-- INSERT INTO quota (id_measure, user, used) VALUES (last_insert_id(), 'valtri', 17);
-- INSERT INTO quota (id_measure, user, used) VALUES (last_insert_id(), 'nemo', 1);
--
-- INSERT INTO hdfs (full, disk, disk_used) VALUES (1024, 1023, 10);
-- or:
-- INSERT INTO measure (name) VALUES ('hdfs');
-- INSERT INTO hdfs (id_measure, full, disk, disk_used, block_under, block_corrupt, block_missing) VALUES (last_insert_id(), 10240, 10230, 100, 0, 0, 0);
-- INSERT INTO hdfs (id_measure, hostname, state, full, disk, disk_used) VALUES (last_insert_id(), 'hador1', 1, 1024, 1023, 10);
--
-- INSERT INTO measure (name, start, end) VALUES ('jobs', '2015-01-16', '2015-01-17');
-- INSERT INTO jobs (id_measure, user, jobs, fails, subjobs, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), 'valtri', 6, 10, 2, 1000, 500, 10, 100);
-- INSERT INTO jobs (id_measure, user, jobs, fails, subjobs, wait_min, wait_max) VALUES (last_insert_id(), 'nemo', 0, 0, 0, NULL, NULL);
--
-- How to read values:
--
-- a) all history
--
-- SELECT * FROM view_hdfs;
-- SELECT * FROM view_quotas;
--
-- b) current values
--
-- SELECT h.* FROM view_hdfs h, statistic s WHERE h.seq=s.last_seq;
-- SELECT h.* FROM view_hdfs h, statistic s WHERE h.seq=s.last_seq;
--

CREATE TABLE statistic (
	name CHAR(8) NOT NULL,
	last_id_measure INTEGER,
	last_seq INTEGER,

	INDEX (last_id_measure),
	INDEX (last_seq)
);

CREATE TABLE measure (
	id_measure INTEGER AUTO_INCREMENT PRIMARY KEY,
	name CHAR(8) NOT NULL,
	seq INTEGER NOT NULL,
	time TIMESTAMP DEFAULT NOW(),
	start TIMESTAMP NULL DEFAULT NULL,
	end TIMESTAMP NULL DEFAULT NULL,

	INDEX (id_measure),
	INDEX (name),
	INDEX (seq)
);

CREATE TABLE hdfs (
	id_measure INTEGER NOT NULL,
	hostname CHAR(50),
	state INTEGER,
	full BIGINT,
	disk BIGINT,
	disk_used BIGINT,
	disk_free BIGINT,
	block_under INTEGER,
	block_corrupt INTEGER,
	block_missing INTEGER,

	CONSTRAINT PRIMARY KEY (id_measure, hostname),
	INDEX(id_measure),
	INDEX(hostname)
);

CREATE TABLE quota (
	id_measure INTEGER NOT NULL,
	user CHAR(20) NOT NULL,
	used BIGINT,

	CONSTRAINT PRIMARY KEY (id_measure, user),
	INDEX(id_measure),
	INDEX(user)
);

CREATE TABLE jobs (
	id_measure INTEGER NOT NULL,
	user CHAR(20) NULL,
	jobs INTEGER,
	fails INTEGER,
	subjobs INTEGER,
	real_wait INTEGER,
	real_time INTEGER,
	wait_min INTEGER,
	wait_max INTEGER,

	CONSTRAINT PRIMARY KEY (id_measure, user),
	INDEX(id_measure),
	INDEX(user)
);

INSERT INTO statistic (name, last_seq) VALUES ('hdfs', 0);
INSERT INTO statistic (name, last_seq) VALUES ('quota', 0);
INSERT INTO statistic (name, last_seq) VALUES ('jobs', 0);

DELIMITER //<commit_msg>Fix Hadoop accounting database schema<commit_after>--
-- Accounting for Hadoop
--
-- How to add values:
--
-- INSERT INTO measure (name) VALUES ('quota');
-- INSERT INTO quota (id_measure, user, used) VALUES (last_insert_id(), 'valtri', 17);
-- INSERT INTO quota (id_measure, user, used) VALUES (last_insert_id(), 'nemo', 1);
--
-- INSERT INTO hdfs (full, disk, disk_used) VALUES (1024, 1023, 10);
-- or:
-- INSERT INTO measure (name) VALUES ('hdfs');
-- INSERT INTO hdfs (id_measure, full, disk, disk_used, block_under, block_corrupt, block_missing) VALUES (last_insert_id(), 10240, 10230, 100, 0, 0, 0);
-- INSERT INTO hdfs (id_measure, hostname, state, full, disk, disk_used) VALUES (last_insert_id(), 'hador1', 1, 1024, 1023, 10);
--
-- INSERT INTO measure (name, start, end) VALUES ('jobs', '2015-01-16', '2015-01-17');
-- INSERT INTO jobs (id_measure, user, jobs, fails, subjobs, real_wait, real_time, wait_min, wait_max) VALUES (last_insert_id(), 'valtri', 6, 10, 2, 1000, 500, 10, 100);
-- INSERT INTO jobs (id_measure, user, jobs, fails, subjobs, wait_min, wait_max) VALUES (last_insert_id(), 'nemo', 0, 0, 0, NULL, NULL);
--
-- How to read values:
--
-- a) all history
--
-- SELECT * FROM view_hdfs;
-- SELECT * FROM view_quotas;
--
-- b) current values
--
-- SELECT h.* FROM view_hdfs h, statistic s WHERE h.seq=s.last_seq;
-- SELECT h.* FROM view_hdfs h, statistic s WHERE h.seq=s.last_seq;
--

CREATE TABLE statistic (
	name CHAR(8) NOT NULL,
	last_id_measure INTEGER,
	last_seq INTEGER,

	INDEX (last_id_measure),
	INDEX (last_seq)
);

CREATE TABLE measure (
	id_measure INTEGER AUTO_INCREMENT PRIMARY KEY,
	name CHAR(8) NOT NULL,
	seq INTEGER NOT NULL,
	time TIMESTAMP DEFAULT NOW(),
	start TIMESTAMP NULL DEFAULT NULL,
	end TIMESTAMP NULL DEFAULT NULL,

	INDEX (id_measure),
	INDEX (name),
	INDEX (seq)
);

CREATE TABLE hdfs (
	id_measure INTEGER NOT NULL,
	hostname CHAR(50),
	state INTEGER,
	full BIGINT,
	disk BIGINT,
	disk_used BIGINT,
	disk_free BIGINT,
	block_under INTEGER,
	block_corrupt INTEGER,
	block_missing INTEGER,

	CONSTRAINT PRIMARY KEY (id_measure, hostname),
	INDEX(id_measure),
	INDEX(hostname)
);

CREATE TABLE quota (
	id_measure INTEGER NOT NULL,
	user CHAR(20) NOT NULL,
	used BIGINT,

	CONSTRAINT PRIMARY KEY (id_measure, user),
	INDEX(id_measure),
	INDEX(user)
);

CREATE TABLE jobs (
	id_measure INTEGER NOT NULL,
	user CHAR(20) NOT NULL,
	jobs INTEGER,
	fails INTEGER,
	subjobs INTEGER,
	real_wait INTEGER,
	real_time INTEGER,
	wait_min INTEGER,
	wait_max INTEGER,

	CONSTRAINT PRIMARY KEY (id_measure, user),
	INDEX(id_measure),
	INDEX(user)
);

INSERT INTO statistic (name, last_seq) VALUES ('hdfs', 0);
INSERT INTO statistic (name, last_seq) VALUES ('quota', 0);
INSERT INTO statistic (name, last_seq) VALUES ('jobs', 0);

DELIMITER //
{'fraction_non_alphanumeric': 0.13258670520231214, 'fraction_numerical': 0.03143063583815029, 'mean_word_length': 4.493055555555555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5802323', 'n_tokens_mistral': 2318, 'n_tokens_neox': 1930, 'n_words': 661}