<commit_before>CREATE TABLE snak (
    id        INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    property  VARCHAR(32),
    svalue    TEXT,
    dvalue    DECIMAL,
    lat       DOUBLE,
    lng       DOUBLE,
    tvalue    TIMESTAMP,
    precision INT,
    lang      VARCHAR(2),
    vtype     INT
);

CREATE TABLE statement (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    subject  VARCHAR(32),
    mainsnak INTEGER REFERENCES snak(id),
    state    INT DEFAULT 0
);

CREATE TABLE qualifier (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    stmt     INTEGER REFERENCES statement(id),
    snak     INTEGER REFERENCES snak(id)
);

CREATE TABLE source (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    stmt     INTEGER REFERENCES statement(id),
    snak     INTEGER REFERENCES snak(id)
);

CREATE TABLE userlog (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    user     VARCHAR(64),
    stmt     INTEGER REFERENCES statement(id),
    state    INT DEFAULT 0,
    changed  TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX idx_stmt_qid ON statement(subject);
CREATE INDEX idx_stmt_state ON statement(state);<commit_msg>Add indexes for efficient lookup of sources and qualifiers.<commit_after>CREATE TABLE snak (
    id        INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    property  VARCHAR(32),
    svalue    TEXT,
    dvalue    DECIMAL,
    lat       DOUBLE,
    lng       DOUBLE,
    tvalue    TIMESTAMP,
    precision INT,
    lang      VARCHAR(2),
    vtype     INT
);

CREATE TABLE statement (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    subject  VARCHAR(32),
    mainsnak INTEGER REFERENCES snak(id),
    state    INT DEFAULT 0
);

CREATE TABLE qualifier (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    stmt     INTEGER REFERENCES statement(id),
    snak     INTEGER REFERENCES snak(id)
);

CREATE TABLE source (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    stmt     INTEGER REFERENCES statement(id),
    snak     INTEGER REFERENCES snak(id)
);

CREATE TABLE userlog (
    id       INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
    user     VARCHAR(64),
    stmt     INTEGER REFERENCES statement(id),
    state    INT DEFAULT 0,
    changed  TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

CREATE INDEX idx_stmt_qid ON statement(subject);
CREATE INDEX idx_stmt_state ON statement(state);
CREATE INDEX idx_qualifier_stmt ON qualifier(stmt);
CREATE INDEX idx_source_stmt ON source(stmt);
{'fraction_non_alphanumeric': 0.05443548387096774, 'fraction_numerical': 0.007258064516129033, 'mean_word_length': 2.6919642857142856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7428133', 'n_tokens_mistral': 833, 'n_tokens_neox': 777, 'n_words': 253}