<commit_before>module Dbox
  class DatabaseError < RuntimeError; end

  class Database
    include Loggable

    DB_FILENAME = ".dbox.sqlite3"

    def self.create(remote_path, local_path)
      db = new(local_path)
      if db.bootstrapped?
        raise DatabaseError, "Database already initialized -- please use 'dbox pull' or 'dbox push'."
      end
      db.bootstrap(remote_path, local_path)
      db
    end

    def self.load(local_path)
      db = new(local_path)
      unless db.bootstrapped?
        raise DatabaseError, "Database not initialized -- please run 'dbox create' or 'dbox clone'."
      end
      db
    end

    def self.exists?(local_path)
      File.exists?(File.join(local_path, DB_FILENAME))
    end

    def self.migrate_from_old_db_format(old_db)
      new_db = create(old_db.remote_path, old_db.local_path)
      new_db.delete_entry_by_path("") # clear out root record
      new_db.migrate_entry_from_old_db_format(old_db.root)
    end

    # IMPORTANT: Database.new is private. Please use Database.create
    # or Database.load as the entry point.
    private_class_method :new
    def initialize(local_path)
      FileUtils.mkdir_p(local_path)
      @db = SQLite3::Database.new(File.join(local_path, DB_FILENAME))
      @db.trace {|sql| log.debug sql.strip }
      @db.execute("PRAGMA foreign_keys = ON;")
      ensure_schema_exists
    end

    def ensure_schema_exists
      @db.execute_batch(%{
        CREATE TABLE IF NOT EXISTS metadata (
          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
          local_path   varchar(255) NOT NULL,
          remote_path  varchar(255) NOT NULL,
          version      integer NOT NULL
        );
        CREATE TABLE IF NOT EXISTS entries (
          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
          path         varchar(255) UNIQUE NOT NULL,
          is_dir       boolean NOT NULL,
          parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
          hash         varchar(255),
          modified     datetime,
          revision     integer
        );
        CREATE INDEX IF NOT EXISTS entry_parent_ids ON entries(parent_id);
      })
    end

    METADATA_COLS = [ :local_path, :remote_path, :version ] # don't need to return id
    ENTRY_COLS    = [ :id, :path, :is_dir, :parent_id, :hash, :modified, :revision ]

    def bootstrap(remote_path, local_path)
      @db.execute(%{
        INSERT INTO metadata (local_path, remote_path, version) VALUES (?, ?, ?);
      }, local_path, remote_path, 1)
      @db.execute(%{
        INSERT INTO entries (path, is_dir) VALUES (?, ?)
      }, "", 1)
    end

    def bootstrapped?
      n = @db.get_first_value(%{
        SELECT count(id) FROM metadata LIMIT 1;
      })
      n && n > 0
    end

    def metadata
      cols = METADATA_COLS
      res = @db.get_first_row(%{
        SELECT #{cols.join(',')} FROM metadata LIMIT 1;
      })
      make_fields(cols, res) if res
    end

    def update_metadata(fields)
      set_str = fields.keys.map {|k| "#{k}=?" }.join(",")
      @db.execute(%{
        UPDATE metadata SET #{set_str};
      }, *fields.values)
    end

    def root_dir
      find_entry("WHERE parent_id is NULL")
    end

    def find_by_path(path)
      raise(ArgumentError, "path cannot be null") unless path<commit_msg>Removed local_path from stored metadata so that local dir can be moved/renamed.<commit_after>module Dbox
  class DatabaseError < RuntimeError; end

  class Database
    include Loggable

    DB_FILENAME = ".dbox.sqlite3"

    def self.create(remote_path, local_path)
      db = new(local_path)
      if db.bootstrapped?
        raise DatabaseError, "Database already initialized -- please use 'dbox pull' or 'dbox push'."
      end
      db.bootstrap(remote_path)
      db.migrate()
      db
    end

    def self.load(local_path)
      db = new(local_path)
      unless db.bootstrapped?
        raise DatabaseError, "Database not initialized -- please run 'dbox create' or 'dbox clone'."
      end
      db.migrate()
      db
    end

    def self.exists?(local_path)
      File.exists?(File.join(local_path, DB_FILENAME))
    end

    def self.migrate_from_old_db_format(old_db)
      new_db = create(old_db.remote_path, old_db.local_path)
      new_db.delete_entry_by_path("") # clear out root record
      new_db.migrate_entry_from_old_db_format(old_db.root)
    end

    attr_reader :local_path

    # IMPORTANT: Database.new is private. Please use Database.create
    # or Database.load as the entry point.
    private_class_method :new
    def initialize(local_path)
      @local_path = local_path
      FileUtils.mkdir_p(local_path)
      @db = SQLite3::Database.new(File.join(local_path, DB_FILENAME))
      @db.trace {|sql| log.debug sql.strip }
      @db.execute("PRAGMA foreign_keys = ON;")
      ensure_schema_exists
    end

    def ensure_schema_exists
      @db.execute_batch(%{
        CREATE TABLE IF NOT EXISTS metadata (
          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
          remote_path  varchar(255) NOT NULL,
          version      integer NOT NULL
        );
        CREATE TABLE IF NOT EXISTS entries (
          id           integer PRIMARY KEY AUTOINCREMENT NOT NULL,
          path         varchar(255) UNIQUE NOT NULL,
          is_dir       boolean NOT NULL,
          parent_id    integer REFERENCES entries(id) ON DELETE CASCADE,
          hash         varchar(255),
          modified     datetime,
          revision     integer
        );
        CREATE INDEX IF NOT EXISTS entry_parent_ids ON entries(parent_id);
      })
    end

    def migrate
      if metadata[:version] < 2
        @db.execute_batch(%{
          ALTER TABLE metadata DROP COLUMN local_path;
          UPDATE_TABLE metadata SET version = 2;
        })
      end
    end

    METADATA_COLS = [ :remote_path, :version ] # don't need to return id
    ENTRY_COLS    = [ :id, :path, :is_dir, :parent_id, :hash, :modified, :revision ]

    def bootstrap(remote_path)
      @db.execute(%{
        INSERT INTO metadata (remote_path, version) VALUES (?, ?);
      }, remote_path, 2)
      @db.execute(%{
        INSERT INTO entries (path, is_dir) VALUES (?, ?)
      }, "", 1)
    end

    def bootstrapped?
      n = @db.get_first_value(%{
        SELECT count(id) FROM metadata LIMIT 1;
      })
      n && n > 0
    end

    def metadata
      cols = METADATA_COLS
      res = @db.get_first_row(%{
        SELECT #{cols.join(',')} FROM metadata LIMIT 1;
      })
      out = { :local_path => local_path }
      out.merge!(make_fields(cols, res)) if res
      out
    end

    def update_metadata(fields)
      set_str = fields.keys.map {|k| "#{k}=?" }.join(",")
      @db.execute(%{
        UPDATE metadata SET #{set_str};
      }, *fields.values)
    end

    def root_dir
      find_entry("WHERE parent_id is NULL")
    end

    def find_by_path(path)
      raise(ArgumentError, "path cannot be null") unless path
{'fraction_non_alphanumeric': 0.0932776738160357, 'fraction_numerical': 0.005326040017273643, 'mean_word_length': 2.3069966682532126, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4851980', 'n_tokens_mistral': 2279, 'n_tokens_neox': 2167, 'n_words': 586}