<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import Foundation
import XCGLogger
import Shared

private let log = Logger.syncLogger

typealias Args = [AnyObject?]

/* This is a base interface into our browser db. It holds arrays of tables and handles basic creation/updating of them. */
// Version 1 - Basic history table.
// Version 2 - Added a visits table, refactored the history table to be a GenericTable.
// Version 3 - Added a favicons table.
// Version 4 - Added a readinglist table.
// Version 5 - Added the clients and the tabs tables.
// Version 6 - Visit timestamps are now microseconds.
// Version 7 - Eliminate most tables.
public class BrowserDB {
    private let db: SwiftData
    // XXX: Increasing this should blow away old history, since we currently don't support any upgrades.
    private let Version: Int = 7
    private let files: FileAccessor
    private let filename: String
    private let secretKey: String?
    private let schemaTable: SchemaTable

    private var initialized = [String]()

    // SQLITE_MAX_VARIABLE_NUMBER = 999 by default. This controls how many ?s can
    // appear in a query string.
    static let MaxVariableNumber = 999

    public init(filename: String, secretKey: String? = nil, files: FileAccessor) {
        log.debug("Initializing BrowserDB: \(filename).")
        self.files = files
        self.filename = filename
        self.schemaTable = SchemaTable()
        self.secretKey = secretKey

        let file = ((try! files.getAndEnsureDirectory()) as NSString).stringByAppendingPathComponent(filename)
        self.db = SwiftData(filename: file, key: secretKey, prevKey: nil)

        if AppConstants.BuildChannel == .Developer && secretKey != nil {
            log.debug("Creating db: \(file) with secret = \(secretKey)")
        }

        // Create or update will also delete and create the database if our key was incorrect.
        self.createOrUpdate(self.schemaTable)
    }

    // Creates a table and writes its table info into the table-table database.
    private func createTable(conn: SQLiteDBConnection, table: SectionCreator) -> TableResult {
        log.debug("Try create \(table.name) version \(table.version)")
        if !table.create(conn) {
            // If creating failed, we'll bail without storing the table info
            log.debug("Creation failed.")
            return .Failed
        }

        var err: NSError? = nil
        return schemaTable.insert(conn, item: table, err: &err) > -1 ? .Created : .Failed
    }

    // Updates a table and writes its table into the table-table database.
    // Exposed internally for testing.
    func updateTable(conn: SQLiteDBConnection, table: SectionUpdater) -> TableResult {
        log.debug("Trying update \(table.name) version \(table.version)")
        var from = 0
        // Try to find the stored version of the table
        let cursor = schemaTable.query(conn, options: QueryOptions(filter: table.name))
        if cursor.count > 0 {
            if let info = cursor[0] as? TableInfoWrapper {
                from = info.version
            }
        }

        // If the versions match, no need to update
        if from == table.version {
            return .Exists
        }

        if !table.updateTable(conn, from: from) {
            // If the update failed, we'll bail without writing the change to the table-table.
            log.debug("Updating failed.")
            return .Failed
        }

        var err: NSError? = nil

        // Yes, we UPDATE OR INSERT… because we might be transferring ownership of a database table
        // to a different Table. It'll trigger exists, and thus take the update path, but we won't
        // necessarily have an existing schema entry -- i.e., we'll be updating from 0.
        if schemaTable.update(conn, item: table, err: &err) > 0 ||
           schemaTable.insert(conn, item: table, err: &err) > 0 {
            return .Updated
        }
        return .Failed
    }

    // Utility for table classes. They should call this when they're initialized to force
    // creation of the table in the database.
    func createOrUpdate(tables: Table...) -> Bool {
        var success = true
        let doCreate = { (table: Table, connection: SQLiteDBConnection) -> () in
            switch self.createTable(connection, table: table) {
            case .Created:
                success = true
                connection.checkpoint()
                return
            case .Exists:
                log.debug("Table already exists.")
                success = true
                return
            default:
                success = false
            }
        }

        if let _ = self.db.transaction({ connection -> Bool in
            let thread = NSThread.currentThread().description
            // If the table doesn't exist, we'll create it.
            for table in tables {
                log.debug("Create or update \(table.name) version \(table.version) on \(thread).")
                if !table.exists(connection) {
                    log.debug("Doesn't exist. Creating table \(table.name).")
                    doCreate(table, connection)
                } else {
                    // Otherwise, we'll update it
                    switch self.updateTable(connection, table: table) {
                    case .Updated:
                        log.debug("Updated table \(table.name).")
                        success = true
                        connection.checkpoint()
                        break
                    case .Exists:
                        log.debug("Table \(table.name) already exists.")
                        success = true
                        break
                    default:
                        log.error("Update failed for \(table.name). Dropping and recreating.")

                        table.drop(connection)
                        var err: NSError? = nil
                        self.schemaTable.delete(connection, item: table, err: &err)

                        doCreate(table, connection)
                    }
                }

                if !success {
                    log.warning("Failed to configure multiple tables. Aborting.")
                    return false
                }
            }
            return success
        }) {
            // Err getting a transaction
            success = false
        }

        // If we failed, move the file and try again. This will probably break things that are already
        // attached and expecting a working DB, but at least we should be able to restart.
        if !success {
            log.debug("Couldn't create or update \(tables.map { $0.name }).")
            log.debug("Attempting to move \(self.filename) to another location.")

            // Note that a backup file might already exist! We append a counter to avoid this.
            var bakCounter = 0
            var bak: String
            repeat {
                bak = "\(self.filename).bak.\(++bakCounter)"
            } while self.files.exists(bak)

            do {
                try self.files.move(self.filename, toRelativePath: bak)
                success = true
            } catch _ {
                success = false
            }
            assert(success)

            if let _ = db.transaction({ connection -> Bool in
                for table in tables {
                    doCreate(table, connection)
                    if !success {
                        return false<commit_msg>Bug 1226821 - Part 1: move .sqlite-shm and .sqlite-wal files when moving .sqlite.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import Foundation
import XCGLogger
import Shared

private let log = Logger.syncLogger

typealias Args = [AnyObject?]

/* This is a base interface into our browser db. It holds arrays of tables and handles basic creation/updating of them. */
// Version 1 - Basic history table.
// Version 2 - Added a visits table, refactored the history table to be a GenericTable.
// Version 3 - Added a favicons table.
// Version 4 - Added a readinglist table.
// Version 5 - Added the clients and the tabs tables.
// Version 6 - Visit timestamps are now microseconds.
// Version 7 - Eliminate most tables.
public class BrowserDB {
    private let db: SwiftData
    // XXX: Increasing this should blow away old history, since we currently don't support any upgrades.
    private let Version: Int = 7
    private let files: FileAccessor
    private let filename: String
    private let secretKey: String?
    private let schemaTable: SchemaTable

    private var initialized = [String]()

    // SQLITE_MAX_VARIABLE_NUMBER = 999 by default. This controls how many ?s can
    // appear in a query string.
    static let MaxVariableNumber = 999

    public init(filename: String, secretKey: String? = nil, files: FileAccessor) {
        log.debug("Initializing BrowserDB: \(filename).")
        self.files = files
        self.filename = filename
        self.schemaTable = SchemaTable()
        self.secretKey = secretKey

        let file = ((try! files.getAndEnsureDirectory()) as NSString).stringByAppendingPathComponent(filename)
        self.db = SwiftData(filename: file, key: secretKey, prevKey: nil)

        if AppConstants.BuildChannel == .Developer && secretKey != nil {
            log.debug("Creating db: \(file) with secret = \(secretKey)")
        }

        // Create or update will also delete and create the database if our key was incorrect.
        self.createOrUpdate(self.schemaTable)
    }

    // Creates a table and writes its table info into the table-table database.
    private func createTable(conn: SQLiteDBConnection, table: SectionCreator) -> TableResult {
        log.debug("Try create \(table.name) version \(table.version)")
        if !table.create(conn) {
            // If creating failed, we'll bail without storing the table info
            log.debug("Creation failed.")
            return .Failed
        }

        var err: NSError? = nil
        return schemaTable.insert(conn, item: table, err: &err) > -1 ? .Created : .Failed
    }

    // Updates a table and writes its table into the table-table database.
    // Exposed internally for testing.
    func updateTable(conn: SQLiteDBConnection, table: SectionUpdater) -> TableResult {
        log.debug("Trying update \(table.name) version \(table.version)")
        var from = 0
        // Try to find the stored version of the table
        let cursor = schemaTable.query(conn, options: QueryOptions(filter: table.name))
        if cursor.count > 0 {
            if let info = cursor[0] as? TableInfoWrapper {
                from = info.version
            }
        }

        // If the versions match, no need to update
        if from == table.version {
            return .Exists
        }

        if !table.updateTable(conn, from: from) {
            // If the update failed, we'll bail without writing the change to the table-table.
            log.debug("Updating failed.")
            return .Failed
        }

        var err: NSError? = nil

        // Yes, we UPDATE OR INSERT… because we might be transferring ownership of a database table
        // to a different Table. It'll trigger exists, and thus take the update path, but we won't
        // necessarily have an existing schema entry -- i.e., we'll be updating from 0.
        if schemaTable.update(conn, item: table, err: &err) > 0 ||
           schemaTable.insert(conn, item: table, err: &err) > 0 {
            return .Updated
        }
        return .Failed
    }

    // Utility for table classes. They should call this when they're initialized to force
    // creation of the table in the database.
    func createOrUpdate(tables: Table...) -> Bool {
        var success = true
        let doCreate = { (table: Table, connection: SQLiteDBConnection) -> () in
            switch self.createTable(connection, table: table) {
            case .Created:
                success = true
                connection.checkpoint()
                return
            case .Exists:
                log.debug("Table already exists.")
                success = true
                return
            default:
                success = false
            }
        }

        if let _ = self.db.transaction({ connection -> Bool in
            let thread = NSThread.currentThread().description
            // If the table doesn't exist, we'll create it.
            for table in tables {
                log.debug("Create or update \(table.name) version \(table.version) on \(thread).")
                if !table.exists(connection) {
                    log.debug("Doesn't exist. Creating table \(table.name).")
                    doCreate(table, connection)
                } else {
                    // Otherwise, we'll update it
                    switch self.updateTable(connection, table: table) {
                    case .Updated:
                        log.debug("Updated table \(table.name).")
                        success = true
                        connection.checkpoint()
                        break
                    case .Exists:
                        log.debug("Table \(table.name) already exists.")
                        success = true
                        break
                    default:
                        log.error("Update failed for \(table.name). Dropping and recreating.")

                        table.drop(connection)
                        var err: NSError? = nil
                        self.schemaTable.delete(connection, item: table, err: &err)

                        doCreate(table, connection)
                    }
                }

                if !success {
                    log.warning("Failed to configure multiple tables. Aborting.")
                    return false
                }
            }
            return success
        }) {
            // Err getting a transaction
            success = false
        }

        // If we failed, move the file and try again. This will probably break things that are already
        // attached and expecting a working DB, but at least we should be able to restart.
        if !success {
            log.debug("Couldn't create or update \(tables.map { $0.name }).")
            log.debug("Attempting to move \(self.filename) to another location.")

            // Note that a backup file might already exist! We append a counter to avoid this.
            var bakCounter = 0
            var bak: String
            repeat {
                bak = "\(self.filename).bak.\(++bakCounter)"
            } while self.files.exists(bak)

            do {
                try self.files.move(self.filename, toRelativePath: bak)
                try self.files.move(self.filename + "-shm", toRelativePath: bak + "-shm")
                try self.files.move(self.filename + "-wal", toRelativePath: bak + "-wal")
                success = true
            } catch _ {
                success = false
            }
            assert(success)

            if let _ = db.transaction({ connection -> Bool in
                for table in tables {
                    doCreate(table, connection)
                    if !success {
                        return false
{'fraction_non_alphanumeric': 0.0865211810012837, 'fraction_numerical': 0.003979460847240051, 'mean_word_length': 1.7003466204506066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2854932', 'n_tokens_mistral': 3966, 'n_tokens_neox': 3682, 'n_words': 1440}