<commit_before>CREATE TABLE public.access_tokens (
    "id" bigserial NOT NULL,
    "user_id" bigint,
    "token" text,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "enabled" boolean,
    PRIMARY KEY ("id")
);
CREATE INDEX index_access_tokens_on_enabled ON public.access_tokens USING btree (enabled);
CREATE INDEX index_access_tokens_on_user_id ON public.access_tokens USING btree (user_id);

CREATE TABLE public.last_updates (
    "id" integer NOT NULL,
    "cursor" bigint NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);

CREATE TABLE public.organization_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_organization_ranks_on_rank ON public.organization_ranks USING btree (rank);
CREATE UNIQUE INDEX index_organization_ranks_on_stargazers_count ON public.organization_ranks USING btree (stargazers_count);

CREATE TABLE public.repositories (
    "id" bigserial NOT NULL,
    "name" text,
    "full_name" citext,
    "owner_id" bigint,
    "author_id" bigint,
    "description" text,
    "fork" boolean,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "homepage" text,
    "stargazers_count" bigint NOT NULL DEFAULT 0,
    "language" text,
    "fetched_at" timestamp with time zone,
    PRIMARY KEY ("id")
);
CREATE INDEX index_repositories_on_author_id_and_stargazers_count ON public.repositories USING btree (author_id, stargazers_count);
CREATE INDEX index_repositories_on_full_name ON public.repositories USING btree (full_name);
CREATE INDEX index_repositories_on_owner_id_and_stargazers_count ON public.repositories USING btree (owner_id, stargazers_count);
CREATE INDEX index_repositories_on_stargazers_count ON public.repositories USING btree (stargazers_count);

CREATE TABLE public.repository_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_repository_ranks_on_rank ON public.repository_ranks USING btree (rank);
CREATE UNIQUE INDEX index_repository_ranks_on_stargazers_count ON public.repository_ranks USING btree (stargazers_count);

CREATE TABLE public.update_user_jobs (
    "id" bigserial NOT NULL,
    "payload" text,
    "timeout_at" timestamp with time zone,
    "owner" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE INDEX index_update_user_jobs_on_timeout_at ON public.update_user_jobs USING btree (timeout_at);

CREATE TABLE public.user_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_user_ranks_on_rank ON public.user_ranks USING btree (rank);
CREATE UNIQUE INDEX index_user_ranks_on_stargazers_count ON public.user_ranks USING btree (stargazers_count);

CREATE TABLE public.users (
    "id" bigserial NOT NULL,
    "login" citext,
    "avatar_url" text,
    "type" text,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "queued_at" timestamp with time zone,
    "stargazers_count" bigint NOT NULL DEFAULT 0,
    "location" text,
    PRIMARY KEY ("id")
);
CREATE INDEX index_users_on_login ON public.users USING btree (login);
CREATE INDEX index_users_on_stargazers_count ON public.users USING btree (stargazers_count);<commit_msg>Simplify the schema a bit<commit_after>CREATE TABLE access_tokens (
    "id" bigserial NOT NULL,
    "user_id" bigint,
    "token" text,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "enabled" boolean,
    PRIMARY KEY ("id")
);
CREATE INDEX index_access_tokens_on_enabled ON access_tokens (enabled);
CREATE INDEX index_access_tokens_on_user_id ON access_tokens (user_id);

CREATE TABLE last_updates (
    "id" integer NOT NULL,
    "cursor" bigint NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);

CREATE TABLE organization_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_organization_ranks_on_rank ON organization_ranks (rank);
CREATE UNIQUE INDEX index_organization_ranks_on_stargazers_count ON organization_ranks (stargazers_count);

CREATE TABLE repositories (
    "id" bigserial NOT NULL,
    "name" text,
    "full_name" citext,
    "owner_id" bigint,
    "author_id" bigint,
    "description" text,
    "fork" boolean,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "homepage" text,
    "stargazers_count" bigint NOT NULL DEFAULT 0,
    "language" text,
    "fetched_at" timestamp with time zone,
    PRIMARY KEY ("id")
);
CREATE INDEX index_repositories_on_author_id_and_stargazers_count ON repositories (author_id, stargazers_count);
CREATE INDEX index_repositories_on_full_name ON repositories (full_name);
CREATE INDEX index_repositories_on_owner_id_and_stargazers_count ON repositories (owner_id, stargazers_count);
CREATE INDEX index_repositories_on_stargazers_count ON repositories (stargazers_count);

CREATE TABLE repository_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_repository_ranks_on_rank ON repository_ranks (rank);
CREATE UNIQUE INDEX index_repository_ranks_on_stargazers_count ON repository_ranks (stargazers_count);

CREATE TABLE update_user_jobs (
    "id" bigserial NOT NULL,
    "payload" text,
    "timeout_at" timestamp with time zone,
    "owner" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE INDEX index_update_user_jobs_on_timeout_at ON update_user_jobs (timeout_at);

CREATE TABLE user_ranks (
    "id" bigserial NOT NULL,
    "stargazers_count" bigint,
    "rank" bigint,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    PRIMARY KEY ("id")
);
CREATE UNIQUE INDEX index_user_ranks_on_rank ON user_ranks (rank);
CREATE UNIQUE INDEX index_user_ranks_on_stargazers_count ON user_ranks (stargazers_count);

CREATE TABLE users (
    "id" bigserial NOT NULL,
    "login" citext,
    "avatar_url" text,
    "type" text,
    "created_at" timestamp with time zone NOT NULL,
    "updated_at" timestamp with time zone NOT NULL,
    "queued_at" timestamp with time zone,
    "stargazers_count" bigint NOT NULL DEFAULT 0,
    "location" text,
    PRIMARY KEY ("id")
);
CREATE INDEX index_users_on_login ON users (login);
CREATE INDEX index_users_on_stargazers_count ON users (stargazers_count);
{'fraction_non_alphanumeric': 0.07404900500879924, 'fraction_numerical': 0.0005414918099363748, 'mean_word_length': 4.674347158218126, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '709071', 'n_tokens_mistral': 2460, 'n_tokens_neox': 2393, 'n_words': 753}