<commit_before>CREATE TABLE IF NOT EXISTS `requests` (
  `request_id`     BINARY(12) NOT NULL,
  `type`           VARBINARY(75) NOT NULL,
  `state`          TINYINT UNSIGNED NOT NULL DEFAULT 0,
  `user`           VARCHAR(100) NULL DEFAULT NULL,
  `created_at`     TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `started_at`     TIMESTAMP NULL DEFAULT NULL,
  `finished_at`    TIMESTAMP NULL DEFAULT NULL,
  `total_jobs`     INT UNSIGNED NOT NULL DEFAULT 0,
  `finished_jobs`  INT UNSIGNED NOT NULL DEFAULT 0,

  PRIMARY KEY (`request_id`),
  INDEX (`created_at`),  -- recently created
  INDEX (`finished_at`), -- recently finished
  INDEX (`state`)        -- currently running
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS `raw_requests` (
  `request_id` BINARY(12) NOT NULL,
  `request`    BLOB NOT NULL,
  `job_chain`  BLOB NOT NULL,

  PRIMARY KEY (`request_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS `job_log` (
  `request_id`  BINARY(12) NOT NULL,
  `job_id`      BINARY(4) NOT NULL,
  `try`         SMALLINT NOT NULL DEFAULT 0,
  `type`        VARBINARY(75) NOT NULL,
  `state`       TINYINT UNSIGNED NOT NULL DEFAULT 0,
  `started_at`  BIGINT UNSIGNED NOT NULL DEFAULT 0, -- Unix time (nanoseconds)
  `finished_at` BIGINT UNSIGNED NOT NULL DEFAULT 0, -- Unix time (nanoseconds)
  `error`       TEXT NULL DEFAULT NULL,
  `exit`        TINYINT UNSIGNED NULL DEFAULT NULL,
  `stdout`      LONGBLOB NULL DEFAULT NULL,
  `stderr`      LONGBLOB NULL DEFAULT NULL,

  PRIMARY KEY (`request_id`, `job_id`, `try`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;<commit_msg>store request ids as 20 bytes for now<commit_after>CREATE TABLE IF NOT EXISTS `requests` (
  `request_id`     BINARY(20) NOT NULL,
  `type`           VARBINARY(75) NOT NULL,
  `state`          TINYINT UNSIGNED NOT NULL DEFAULT 0,
  `user`           VARCHAR(100) NULL DEFAULT NULL,
  `created_at`     TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
  `started_at`     TIMESTAMP NULL DEFAULT NULL,
  `finished_at`    TIMESTAMP NULL DEFAULT NULL,
  `total_jobs`     INT UNSIGNED NOT NULL DEFAULT 0,
  `finished_jobs`  INT UNSIGNED NOT NULL DEFAULT 0,

  PRIMARY KEY (`request_id`),
  INDEX (`created_at`),  -- recently created
  INDEX (`finished_at`), -- recently finished
  INDEX (`state`)        -- currently running
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS `raw_requests` (
  `request_id` BINARY(20) NOT NULL,
  `request`    BLOB NOT NULL,
  `job_chain`  BLOB NOT NULL,

  PRIMARY KEY (`request_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS `job_log` (
  `request_id`  BINARY(12) NOT NULL,
  `job_id`      BINARY(4) NOT NULL,
  `try`         SMALLINT NOT NULL DEFAULT 0,
  `type`        VARBINARY(75) NOT NULL,
  `state`       TINYINT UNSIGNED NOT NULL DEFAULT 0,
  `started_at`  BIGINT UNSIGNED NOT NULL DEFAULT 0, -- Unix time (nanoseconds)
  `finished_at` BIGINT UNSIGNED NOT NULL DEFAULT 0, -- Unix time (nanoseconds)
  `error`       TEXT NULL DEFAULT NULL,
  `exit`        TINYINT UNSIGNED NULL DEFAULT NULL,
  `stdout`      LONGBLOB NULL DEFAULT NULL,
  `stderr`      LONGBLOB NULL DEFAULT NULL,

  PRIMARY KEY (`request_id`, `job_id`, `try`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
{'fraction_non_alphanumeric': 0.0944206008583691, 'fraction_numerical': 0.017167381974248927, 'mean_word_length': 3.3275862068965516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1456594', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1096, 'n_words': 362}