<commit_before># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""Sqlite coverage data."""

# TODO: get sys_info for data class, so we can see sqlite version etc
# TODO: get rid of skip_unless_data_storage_is
# TODO: get rid of "JSON message" and "SQL message" in the tests
# TODO: factor out dataop debugging to a wrapper class?
# TODO: make sure all dataop debugging is in place somehow
# TODO: should writes be batched?
# TODO: run_info

import collections
import glob
import itertools
import os
import sqlite3
import sys

from coverage.backward import get_thread_id, iitems
from coverage.data import filename_suffix
from coverage.debug import NoDebugging, SimpleReprMixin
from coverage.files import PathAliases
from coverage.misc import CoverageException, file_be_gone


# Schema versions:
# 1: Released in 5.0a2
# 2: Added contexts

SCHEMA_VERSION = 2

SCHEMA = """
create table coverage_schema (
    version integer
);

create table meta (
    has_lines boolean,
    has_arcs boolean,
    sys_argv text
);

create table file (
    id integer primary key,
    path text,
    unique(path)
);

create table context (
    id integer primary key,
    context text,
    unique(context)
);

create table line (
    file_id integer,
    context_id integer,
    lineno integer,
    unique(file_id, context_id, lineno)
);

create table arc (
    file_id integer,
    context_id integer,
    fromno integer,
    tono integer,
    unique(file_id, context_id, fromno, tono)
);

create table tracer (
    file_id integer primary key,
    tracer text
);
"""


class CoverageSqliteData(SimpleReprMixin):

    def __init__(self, basename=None, suffix=None, warn=None, debug=None):
        self._basename = os.path.abspath(basename or ".coverage")
        self._suffix = suffix
        self._warn = warn
        self._debug = debug or NoDebugging()

        self._choose_filename()
        self._file_map = {}
        self._dbs = {}
        self._pid = os.getpid()

        # Are we in sync with the data file?
        self._have_used = False

        self._has_lines = False
        self._has_arcs = False

        self._current_context = None
        self._current_context_id = None
        self._query_contexts = None
        self._query_context_ids = None

    def _choose_filename(self):
        self._filename = self._basename
        suffix = filename_suffix(self._suffix)
        if suffix:
            self._filename += "." + suffix

    def _reset(self):
        if self._dbs:
            for db in self._dbs.values():
                db.close()
        self._dbs = {}
        self._file_map = {}
        self._have_used = False
        self._current_context_id = None

    def _create_db(self):
        if self._debug.should('dataio'):
            self._debug.write("Creating data file {!r}".format(self._filename))
        self._dbs[get_thread_id()] = Sqlite(self._filename, self._debug)
        with self._dbs[get_thread_id()] as db:
            for stmt in SCHEMA.split(';'):
                stmt = " ".join(stmt.strip().split())
                if stmt:
                    db.execute(stmt)
            db.execute("insert into coverage_schema (version) values (?)", (SCHEMA_VERSION,))
            db.execute(
                "insert into meta (has_lines, has_arcs, sys_argv) values (?, ?, ?)",
                (self._has_lines, self._has_arcs, str(getattr(sys, 'argv', None)))<commit_msg>Add comments to the SQL schema, and keep them when creating the db<commit_after># Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt

"""Sqlite coverage data."""

# TODO: get sys_info for data class, so we can see sqlite version etc
# TODO: get rid of skip_unless_data_storage_is
# TODO: get rid of "JSON message" and "SQL message" in the tests
# TODO: factor out dataop debugging to a wrapper class?
# TODO: make sure all dataop debugging is in place somehow
# TODO: should writes be batched?
# TODO: run_info

import collections
import glob
import itertools
import os
import sqlite3
import sys

from coverage.backward import get_thread_id, iitems
from coverage.data import filename_suffix
from coverage.debug import NoDebugging, SimpleReprMixin
from coverage.files import PathAliases
from coverage.misc import CoverageException, file_be_gone


SCHEMA_VERSION = 2

SCHEMA = """
-- One row, to record the version of the schema store in this db.
CREATE TABLE coverage_schema (
    version integer
    -- Schema versions:
    -- 1: Released in 5.0a2
    -- 2: Added contexts in 5.0a3.  This is schema 2.
);

-- One row, to record some metadata about the data
CREATE TABLE meta (
    has_lines boolean,      -- Is this data recording lines?
    has_arcs boolean,       -- .. or branches?
    sys_argv text           -- The coverage command line that recorded the data.
);

-- A row per file measured.
CREATE TABLE file (
    id integer primary key,
    path text,
    unique(path)
);

-- A row per context measured.
CREATE TABLE context (
    id integer primary key,
    context text,
    unique(context)
);

-- If recording lines, a row per context per line executed.
CREATE TABLE line (
    file_id integer,            -- foreign key to `file`.
    context_id integer,         -- foreign key to `context`.
    lineno integer,             -- the line number.
    unique(file_id, context_id, lineno)
);

-- If recording branches, a row per context per from/to line transition executed.
CREATE TABLE arc (
    file_id integer,            -- foreign key to `file`.
    context_id integer,         -- foreign key to `context`.
    fromno integer,             -- line number jumped from.
    tono integer,               -- line number jumped to.
    unique(file_id, context_id, fromno, tono)
);

-- A row per file indicating the tracer used for that file.
CREATE TABLE tracer (
    file_id integer primary key,
    tracer text
);
"""


class CoverageSqliteData(SimpleReprMixin):

    def __init__(self, basename=None, suffix=None, warn=None, debug=None):
        self._basename = os.path.abspath(basename or ".coverage")
        self._suffix = suffix
        self._warn = warn
        self._debug = debug or NoDebugging()

        self._choose_filename()
        self._file_map = {}
        self._dbs = {}
        self._pid = os.getpid()

        # Are we in sync with the data file?
        self._have_used = False

        self._has_lines = False
        self._has_arcs = False

        self._current_context = None
        self._current_context_id = None
        self._query_contexts = None
        self._query_context_ids = None

    def _choose_filename(self):
        self._filename = self._basename
        suffix = filename_suffix(self._suffix)
        if suffix:
            self._filename += "." + suffix

    def _reset(self):
        if self._dbs:
            for db in self._dbs.values():
                db.close()
        self._dbs = {}
        self._file_map = {}
        self._have_used = False
        self._current_context_id = None

    def _create_db(self):
        if self._debug.should('dataio'):
            self._debug.write("Creating data file {!r}".format(self._filename))
        self._dbs[get_thread_id()] = Sqlite(self._filename, self._debug)
        with self._dbs[get_thread_id()] as db:
            for stmt in SCHEMA.split(';'):
                stmt = stmt.strip()
                if stmt:
                    db.execute(stmt)
            db.execute("insert into coverage_schema (version) values (?)", (SCHEMA_VERSION,))
            db.execute(
                "insert into meta (has_lines, has_arcs, sys_argv) values (?, ?, ?)",
                (self._has_lines, self._has_arcs, str(getattr(sys, 'argv', None)))
{'fraction_non_alphanumeric': 0.09438775510204081, 'fraction_numerical': 0.0028061224489795917, 'mean_word_length': 2.544755877034358, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2804951', 'n_tokens_mistral': 2355, 'n_tokens_neox': 2275, 'n_words': 726}