<commit_before>import os
import time
import sqlite3
import nanotime

from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.exceptions import DvcException
from dvc.logger import Logger


class StateDuplicateError(DvcException):
    pass


class State(object):
    STATE_FILE = 'state'
    STATE_TABLE = 'state'
    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
                         "mtime TEXT NOT NULL, " \
                         "md5 TEXT NOT NULL, " \
                         "timestamp TEXT NOT NULL"

    STATE_INFO_TABLE = 'state_info'
    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
    STATE_INFO_ROW = 1

    LINK_STATE_TABLE = 'link_state'
    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
                              "inode INTEGER NOT NULL, " \
                              "mtime TEXT NOT NULL"

    STATE_ROW_LIMIT = 10000000
    STATE_ROW_CLEANUP_QUOTA = 50

    def __init__(self, project, config):
        self.project = project
        self.dvc_dir = project.dvc_dir
        self.root_dir = project.root_dir

        self.row_limit = 100
        self.row_cleanup_quota = 50

        c = config.get(Config.SECTION_STATE, {})
        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
                               self.STATE_ROW_LIMIT)
        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
                                       self.STATE_ROW_CLEANUP_QUOTA)

        if not self.dvc_dir:
            self.state_file = None
            return

        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
        self.db = None
        self.c = None
        self.inserts = 0

    def __enter__(self):
        self.load()

    def __exit__(self, type, value, tb):
        self.dump()

    @staticmethod
    def init(project):
        return State(project)

    def _collect(self, path):
        if os.path.isdir(path):
            return self.project.cache.local.collect_dir_cache(path)
        else:
            return (file_md5(path)[0], None)

    def changed(self, path, md5):
        actual = self.update(path)

        if not md5 or not actual:
            return True

        return actual.split('.')[0] != md5.split('.')[0]

    def load(self):
        retries = 1
        while True:
            assert self.db is None
            assert self.c is None
            assert self.inserts == 0
            self.db = sqlite3.connect(self.state_file)
            self.c = self.db.cursor()

            # Try loading once to check that the file is indeed a database
            # and reformat it if it is not.
            try:
                # Check that the state file is indeed a database
                cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
                self.c.execute(cmd.format(self.STATE_TABLE,
                                          self.STATE_TABLE_LAYOUT))
                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
                                          self.STATE_INFO_TABLE_LAYOUT))
                self.c.execute(cmd.format(self.LINK_STATE_TABLE,
                                          self.LINK_STATE_TABLE_LAYOUT))

                cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
                      "WHERE NOT EXISTS (SELECT * FROM {})"
                self.c.execute(cmd.format(self.STATE_INFO_TABLE,
                                          self.STATE_INFO_TABLE))

                return
            except sqlite3.DatabaseError:
                self.c.close()
                self.db.close()
                self.db = None
                self.c = None
                self.inserts = 0<commit_msg>state: use state version<commit_after>import os
import time
import sqlite3
import nanotime

from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.exceptions import DvcException
from dvc.logger import Logger


class StateDuplicateError(DvcException):
    pass


class State(object):
    VERSION = 0
    STATE_FILE = 'state'
    STATE_TABLE = 'state'
    STATE_TABLE_LAYOUT = "inode INTEGER PRIMARY KEY, " \
                         "mtime TEXT NOT NULL, " \
                         "md5 TEXT NOT NULL, " \
                         "timestamp TEXT NOT NULL"

    STATE_INFO_TABLE = 'state_info'
    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
    STATE_INFO_ROW = 1

    LINK_STATE_TABLE = 'link_state'
    LINK_STATE_TABLE_LAYOUT = "path TEXT PRIMARY KEY, " \
                              "inode INTEGER NOT NULL, " \
                              "mtime TEXT NOT NULL"

    STATE_ROW_LIMIT = 10000000
    STATE_ROW_CLEANUP_QUOTA = 50

    def __init__(self, project, config):
        self.project = project
        self.dvc_dir = project.dvc_dir
        self.root_dir = project.root_dir

        self.row_limit = 100
        self.row_cleanup_quota = 50

        c = config.get(Config.SECTION_STATE, {})
        self.row_limit = c.get(Config.SECTION_STATE_ROW_LIMIT,
                               self.STATE_ROW_LIMIT)
        self.row_cleanup_quota = c.get(Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
                                       self.STATE_ROW_CLEANUP_QUOTA)

        if not self.dvc_dir:
            self.state_file = None
            return

        self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
        self.db = None
        self.c = None
        self.inserts = 0

    def __enter__(self):
        self.load()

    def __exit__(self, type, value, tb):
        self.dump()

    @staticmethod
    def init(project):
        return State(project)

    def _collect(self, path):
        if os.path.isdir(path):
            return self.project.cache.local.collect_dir_cache(path)
        else:
            return (file_md5(path)[0], None)

    def changed(self, path, md5):
        actual = self.update(path)

        if not md5 or not actual:
            return True

        return actual.split('.')[0] != md5.split('.')[0]

    def _load(self):
        from dvc import VERSION

        cmd = "PRAGMA user_version;"
        self.c.execute(cmd)
        ret = self.c.fetchall()
        assert len(ret) == 1
        assert len(ret[0]) == 1
        assert isinstance(ret[0][0], int)
        self.version = ret[0][0]

        if self.version > self.VERSION:
            msg = "You are using an old version '{}' of dvc that is using " \
                  "state file version '{}' which is not compatible with the " \
                  "state file version '{}' that is used in this projet. " \
                  "Please upgrade right now!"
            raise DvcException(msg.format(VERSION,
                                          self.VERSION,
                                          self.version))
        elif self.version < self.VERSION:
            msg = "State file version '{}' is too old. " \
                  "Reformatting to the current version '{}'."
            self.project.logger.warn(msg.format(self.version, self.VERSION))
            cmd = "DROP TABLE IF EXISTS {};"
            self.c.execute(cmd.format(self.STATE_TABLE))
            self.c.execute(cmd.format(self.STATE_INFO_TABLE))
            self.c.execute(cmd.format(self.LINK_STATE_TABLE))

        # Check that the state file is indeed a database
        cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
        self.c.execute(cmd.format(self.STATE_TABLE,
                                  self.STATE_TABLE_LAYOUT))
        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
                                  self.STATE_INFO_TABLE_LAYOUT))
        self.c.execute(cmd.format(self.LINK_STATE_TABLE,
                                  self.LINK_STATE_TABLE_LAYOUT))

        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
              "WHERE NOT EXISTS (SELECT * FROM {})"
        self.c.execute(cmd.format(self.STATE_INFO_TABLE,
                                  self.STATE_INFO_TABLE))

        cmd = "PRAGMA user_version = {};"
        self.c.execute(cmd.format(self.VERSION))

    def load(self):
        retries = 1
        while True:
            assert self.db is None
            assert self.c is None
            assert self.inserts == 0
            self.db = sqlite3.connect(self.state_file)
            self.c = self.db.cursor()

            # Try loading once to check that the file is indeed a database
            # and reformat it if it is not.
            try:
                self._load()
                return
            except sqlite3.DatabaseError:
                self.c.close()
                self.db.close()
                self.db = None
                self.c = None
                self.inserts = 0
{'fraction_non_alphanumeric': 0.0871925164568657, 'fraction_numerical': 0.008546021480540478, 'mean_word_length': 1.5912627169359665, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4171761', 'n_tokens_mistral': 2598, 'n_tokens_neox': 2418, 'n_words': 594}