<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import Foundation
import Shared
import XCGLogger

private let log = Logger.syncLogger

class NoSuchRecordError: MaybeErrorType {
    let guid: GUID
    init(guid: GUID) {
        self.guid = guid
    }
    var description: String {
        return "No such record: \(guid)."
    }
}

func failOrSucceed<T>(err: NSError?, op: String, val: T) -> Deferred<Maybe<T>> {
    if let err = err {
        log.debug("\(op) failed: \(err.localizedDescription)")
        return deferMaybe(DatabaseError(err: err))
    }

    return deferMaybe(val)
}

func failOrSucceed(err: NSError?, op: String) -> Success {
    return failOrSucceed(err, op: op, val: ())
}

private var ignoredSchemes = ["about"]

public func isIgnoredURL(url: NSURL) -> Bool {
    let scheme = url.scheme
    if let _ = ignoredSchemes.indexOf(scheme) {
        return true
    }

    if url.host == "localhost" {
        return true
    }

    return false
}

public func isIgnoredURL(url: String) -> Bool {
    if let url = NSURL(string: url) {
        return isIgnoredURL(url)
    }

    return false
}

/*
// Here's the Swift equivalent of the below.
func simulatedFrecency(now: MicrosecondTimestamp, then: MicrosecondTimestamp, visitCount: Int) -> Double {
    let ageMicroseconds = (now - then)
    let ageDays = Double(ageMicroseconds) / 86400000000.0         // In SQL the .0 does the coercion.
    let f = 100 * 225 / ((ageSeconds * ageSeconds) + 225)
    return Double(visitCount) * max(1.0, f)
}
*/

// The constants in these functions were arrived at by utterly unscientific experimentation.

func getRemoteFrecencySQL() -> String {
    let visitCountExpression = "remoteVisitCount"
    let now = NSDate.nowMicroseconds()
    let microsecondsPerDay = 86_400_000_000.0      // 1000 * 1000 * 60 * 60 * 24
    let ageDays = "((\(now) - remoteVisitDate) / \(microsecondsPerDay))"

    return "\(visitCountExpression) * max(1, 100 * 110 / (\(ageDays) * \(ageDays) + 110))"
}

func getLocalFrecencySQL() -> String {
    let visitCountExpression = "((2 + localVisitCount) * (2 + localVisitCount))"
    let now = NSDate.nowMicroseconds()
    let microsecondsPerDay = 86_400_000_000.0      // 1000 * 1000 * 60 * 60 * 24
    let ageDays = "((\(now) - localVisitDate) / \(microsecondsPerDay))"

    return "\(visitCountExpression) * max(2, 100 * 225 / (\(ageDays) * \(ageDays) + 225))"
}

extension SDRow {
    func getTimestamp(column: String) -> Timestamp? {
        return (self[column] as? NSNumber)?.unsignedLongLongValue
    }

    func getBoolean(column: String) -> Bool {
        if let val = self[column] as? Int {
            return val != 0
        }
        return false
    }
}

/**
 * The sqlite-backed implementation of the history protocol.
 */
public class SQLiteHistory {
    let db: BrowserDB
    let favicons: FaviconsTable<Favicon>

    required public init?(db: BrowserDB, version: Int? = nil) {
        self.db = db
        self.favicons = FaviconsTable<Favicon>()

        // BrowserTable exists only to perform create/update etc. operations -- it's not
        // a queryable thing that needs to stick around.
        if !db.createOrUpdate(BrowserTable(version: version ?? BrowserTable.DefaultVersion)) {
            return nil
        }
    }
}

extension SQLiteHistory: BrowserHistory {
    public func removeSiteFromTopSites(site: Site) -> Success {
        if let host = site.url.asURL?.normalizedHost() {
            return db.run([("UPDATE \(TableDomains) set showOnTopSites = 0 WHERE domain = ?", [host])])
        }
        return deferMaybe(DatabaseError(description: "Invalid url for site \(site.url)"))
    }

    public func removeHistoryForURL(url: String) -> Success {
        let visitArgs: Args = [url]
        let deleteVisits = "DELETE FROM \(TableVisits) WHERE siteID = (SELECT id FROM \(TableHistory) WHERE url = ?)"

        let markArgs: Args = [NSDate.nowNumber(), url]
        let markDeleted = "UPDATE \(TableHistory) SET url = NULL, is_deleted = 1, should_upload = 1, local_modified = ? WHERE url = ?"

        return db.run([(deleteVisits, visitArgs),
                       (markDeleted, markArgs),
                       favicons.getCleanupCommands()])
    }

    // Note: clearing history isn't really a sane concept in the presence of Sync.
    // This method should be split to do something else.
    public func clearHistory() -> Success {
        return db.run([("DELETE FROM \(TableVisits)", nil),
                       ("DELETE FROM \(TableHistory)", nil),
                       ("DELETE FROM \(TableDomains)", nil),
                       self.favicons.getCleanupCommands()])
    }

    func recordVisitedSite(site: Site) -> Success {
        var error: NSError? = nil

        // Don't store visits to sites with about: protocols
        if isIgnoredURL(site.url) {
            return deferMaybe(IgnoredSiteError())
        }

        db.withWritableConnection(&error) { (conn, inout err: NSError?) -> Int in
            let now = NSDate.nowNumber()

            let i = self.updateSite(site, atTime: now, withConnection: conn)
            if i > 0 {
                return i
            }

            // Insert instead.
            return self.insertSite(site, atTime: now, withConnection: conn)
        }

        return failOrSucceed(error, op: "Record site")
    }

    func updateSite(site: Site, atTime time: NSNumber, withConnection conn: SQLiteDBConnection) -> Int {
        // We know we're adding a new visit, so we'll need to upload this record.
        // If we ever switch to per-visit change flags, this should turn into a CASE statement like
        //   CASE WHEN title IS ? THEN max(should_upload, 1) ELSE should_upload END
        // so that we don't flag this as changed unless the title changed.
        //
        // Note that we will never match against a deleted item, because deleted items have no URL,
        // so we don't need to unset is_deleted here.
        if let host = site.url.asURL?.normalizedHost() {
            let update = "UPDATE \(TableHistory) SET title = ?, local_modified = ?, should_upload = 1, domain_id = (SELECT id FROM \(TableDomains) where domain = ?) WHERE url = ?"
            let updateArgs: Args? = [site.title, time, host, site.url]
            if Logger.logPII {
                log.debug("Setting title to \(site.title) for URL \(site.url)")
            }
            let error = conn.executeChange(update, withArgs: updateArgs)
            if error != nil {
                log.warning("Update failed with \(error?.localizedDescription)")
                return 0
            }
            return conn.numberOfRowsModified
        }
        return 0
    }

    private func insertSite(site: Site, atTime time: NSNumber, withConnection conn: SQLiteDBConnection) -> Int {

        if let host = site.url.asURL?.normalizedHost() {
            if let error = conn.executeChange("INSERT OR IGNORE INTO \(TableDomains) (domain) VALUES (?)", withArgs: [host]) {
                log.warning("Domain insertion failed with \(error.localizedDescription)")
                return 0
            }

            let insert = "INSERT INTO \(TableHistory) " +
                         "(guid, url, title, local_modified, is_deleted, should_upload, domain_id) " +
                         "SELECT ?, ?, ?, ?, 0, 1, id FROM \(TableDomains) WHERE domain = ?"
            let insertArgs: Args? = [site.guid ?? Bytes.generateGUID(), site.url, site.title, time, host]
            if let error = conn.executeChange(insert, withArgs: insertArgs) {
                log.warning("Site insertion failed with \(error.localizedDescription)")
                return 0
            }

            return 1
        }
        log.warning("Invalid URL \(site.url). Not stored in history.")
        return 0
    }

    // TODO: thread siteID into this to avoid the need to do the lookup.
    func addLocalVisitForExistingSite(visit: SiteVisit) -> Success {
        var error: NSError? = nil
        db.withWritableConnection(&error) { (conn, inout err: NSError?) -> Int in
            // INSERT OR IGNORE because we *might* have a clock error that causes a timestamp
            // collision with an existing visit, and it would really suck to error out for that reason.
            let insert = "INSERT OR IGNORE INTO \(TableVisits) (siteID, date, type, is_local) VALUES (" +
                         "(SELECT id FROM \(TableHistory) WHERE url = ?), ?, ?, 1)"
            let realDate = NSNumber(unsignedLongLong: visit.date)
            let insertArgs: Args? = [visit.site.url, realDate, visit.type.rawValue]
            error = conn.executeChange(insert, withArgs: insertArgs)
            if error != nil {
                log.warning("Visit insertion failed with \(err?.localizedDescription)")
                return 0
            }
            return 1
        }<commit_msg>Bug 1203692 - Don't log invalid history URLs without Logger.logPII.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

import Foundation
import Shared
import XCGLogger

private let log = Logger.syncLogger

class NoSuchRecordError: MaybeErrorType {
    let guid: GUID
    init(guid: GUID) {
        self.guid = guid
    }
    var description: String {
        return "No such record: \(guid)."
    }
}

func failOrSucceed<T>(err: NSError?, op: String, val: T) -> Deferred<Maybe<T>> {
    if let err = err {
        log.debug("\(op) failed: \(err.localizedDescription)")
        return deferMaybe(DatabaseError(err: err))
    }

    return deferMaybe(val)
}

func failOrSucceed(err: NSError?, op: String) -> Success {
    return failOrSucceed(err, op: op, val: ())
}

private var ignoredSchemes = ["about"]

public func isIgnoredURL(url: NSURL) -> Bool {
    let scheme = url.scheme
    if let _ = ignoredSchemes.indexOf(scheme) {
        return true
    }

    if url.host == "localhost" {
        return true
    }

    return false
}

public func isIgnoredURL(url: String) -> Bool {
    if let url = NSURL(string: url) {
        return isIgnoredURL(url)
    }

    return false
}

/*
// Here's the Swift equivalent of the below.
func simulatedFrecency(now: MicrosecondTimestamp, then: MicrosecondTimestamp, visitCount: Int) -> Double {
    let ageMicroseconds = (now - then)
    let ageDays = Double(ageMicroseconds) / 86400000000.0         // In SQL the .0 does the coercion.
    let f = 100 * 225 / ((ageSeconds * ageSeconds) + 225)
    return Double(visitCount) * max(1.0, f)
}
*/

// The constants in these functions were arrived at by utterly unscientific experimentation.

func getRemoteFrecencySQL() -> String {
    let visitCountExpression = "remoteVisitCount"
    let now = NSDate.nowMicroseconds()
    let microsecondsPerDay = 86_400_000_000.0      // 1000 * 1000 * 60 * 60 * 24
    let ageDays = "((\(now) - remoteVisitDate) / \(microsecondsPerDay))"

    return "\(visitCountExpression) * max(1, 100 * 110 / (\(ageDays) * \(ageDays) + 110))"
}

func getLocalFrecencySQL() -> String {
    let visitCountExpression = "((2 + localVisitCount) * (2 + localVisitCount))"
    let now = NSDate.nowMicroseconds()
    let microsecondsPerDay = 86_400_000_000.0      // 1000 * 1000 * 60 * 60 * 24
    let ageDays = "((\(now) - localVisitDate) / \(microsecondsPerDay))"

    return "\(visitCountExpression) * max(2, 100 * 225 / (\(ageDays) * \(ageDays) + 225))"
}

extension SDRow {
    func getTimestamp(column: String) -> Timestamp? {
        return (self[column] as? NSNumber)?.unsignedLongLongValue
    }

    func getBoolean(column: String) -> Bool {
        if let val = self[column] as? Int {
            return val != 0
        }
        return false
    }
}

/**
 * The sqlite-backed implementation of the history protocol.
 */
public class SQLiteHistory {
    let db: BrowserDB
    let favicons: FaviconsTable<Favicon>

    required public init?(db: BrowserDB, version: Int? = nil) {
        self.db = db
        self.favicons = FaviconsTable<Favicon>()

        // BrowserTable exists only to perform create/update etc. operations -- it's not
        // a queryable thing that needs to stick around.
        if !db.createOrUpdate(BrowserTable(version: version ?? BrowserTable.DefaultVersion)) {
            return nil
        }
    }
}

extension SQLiteHistory: BrowserHistory {
    public func removeSiteFromTopSites(site: Site) -> Success {
        if let host = site.url.asURL?.normalizedHost() {
            return db.run([("UPDATE \(TableDomains) set showOnTopSites = 0 WHERE domain = ?", [host])])
        }
        return deferMaybe(DatabaseError(description: "Invalid url for site \(site.url)"))
    }

    public func removeHistoryForURL(url: String) -> Success {
        let visitArgs: Args = [url]
        let deleteVisits = "DELETE FROM \(TableVisits) WHERE siteID = (SELECT id FROM \(TableHistory) WHERE url = ?)"

        let markArgs: Args = [NSDate.nowNumber(), url]
        let markDeleted = "UPDATE \(TableHistory) SET url = NULL, is_deleted = 1, should_upload = 1, local_modified = ? WHERE url = ?"

        return db.run([(deleteVisits, visitArgs),
                       (markDeleted, markArgs),
                       favicons.getCleanupCommands()])
    }

    // Note: clearing history isn't really a sane concept in the presence of Sync.
    // This method should be split to do something else.
    public func clearHistory() -> Success {
        return db.run([("DELETE FROM \(TableVisits)", nil),
                       ("DELETE FROM \(TableHistory)", nil),
                       ("DELETE FROM \(TableDomains)", nil),
                       self.favicons.getCleanupCommands()])
    }

    func recordVisitedSite(site: Site) -> Success {
        var error: NSError? = nil

        // Don't store visits to sites with about: protocols
        if isIgnoredURL(site.url) {
            return deferMaybe(IgnoredSiteError())
        }

        db.withWritableConnection(&error) { (conn, inout err: NSError?) -> Int in
            let now = NSDate.nowNumber()

            let i = self.updateSite(site, atTime: now, withConnection: conn)
            if i > 0 {
                return i
            }

            // Insert instead.
            return self.insertSite(site, atTime: now, withConnection: conn)
        }

        return failOrSucceed(error, op: "Record site")
    }

    func updateSite(site: Site, atTime time: NSNumber, withConnection conn: SQLiteDBConnection) -> Int {
        // We know we're adding a new visit, so we'll need to upload this record.
        // If we ever switch to per-visit change flags, this should turn into a CASE statement like
        //   CASE WHEN title IS ? THEN max(should_upload, 1) ELSE should_upload END
        // so that we don't flag this as changed unless the title changed.
        //
        // Note that we will never match against a deleted item, because deleted items have no URL,
        // so we don't need to unset is_deleted here.
        if let host = site.url.asURL?.normalizedHost() {
            let update = "UPDATE \(TableHistory) SET title = ?, local_modified = ?, should_upload = 1, domain_id = (SELECT id FROM \(TableDomains) where domain = ?) WHERE url = ?"
            let updateArgs: Args? = [site.title, time, host, site.url]
            if Logger.logPII {
                log.debug("Setting title to \(site.title) for URL \(site.url)")
            }
            let error = conn.executeChange(update, withArgs: updateArgs)
            if error != nil {
                log.warning("Update failed with \(error?.localizedDescription)")
                return 0
            }
            return conn.numberOfRowsModified
        }
        return 0
    }

    private func insertSite(site: Site, atTime time: NSNumber, withConnection conn: SQLiteDBConnection) -> Int {

        if let host = site.url.asURL?.normalizedHost() {
            if let error = conn.executeChange("INSERT OR IGNORE INTO \(TableDomains) (domain) VALUES (?)", withArgs: [host]) {
                log.warning("Domain insertion failed with \(error.localizedDescription)")
                return 0
            }

            let insert = "INSERT INTO \(TableHistory) " +
                         "(guid, url, title, local_modified, is_deleted, should_upload, domain_id) " +
                         "SELECT ?, ?, ?, ?, 0, 1, id FROM \(TableDomains) WHERE domain = ?"
            let insertArgs: Args? = [site.guid ?? Bytes.generateGUID(), site.url, site.title, time, host]
            if let error = conn.executeChange(insert, withArgs: insertArgs) {
                log.warning("Site insertion failed with \(error.localizedDescription)")
                return 0
            }

            return 1
        }

        if Logger.logPII {
            log.warning("Invalid URL \(site.url). Not stored in history.")
        }
        return 0
    }

    // TODO: thread siteID into this to avoid the need to do the lookup.
    func addLocalVisitForExistingSite(visit: SiteVisit) -> Success {
        var error: NSError? = nil
        db.withWritableConnection(&error) { (conn, inout err: NSError?) -> Int in
            // INSERT OR IGNORE because we *might* have a clock error that causes a timestamp
            // collision with an existing visit, and it would really suck to error out for that reason.
            let insert = "INSERT OR IGNORE INTO \(TableVisits) (siteID, date, type, is_local) VALUES (" +
                         "(SELECT id FROM \(TableHistory) WHERE url = ?), ?, ?, 1)"
            let realDate = NSNumber(unsignedLongLong: visit.date)
            let insertArgs: Args? = [visit.site.url, realDate, visit.type.rawValue]
            error = conn.executeChange(insert, withArgs: insertArgs)
            if error != nil {
                log.warning("Visit insertion failed with \(err?.localizedDescription)")
                return 0
            }
            return 1
        }
{'fraction_non_alphanumeric': 0.10861196656401231, 'fraction_numerical': 0.013583369995600529, 'mean_word_length': 2.294980974814278, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3721482', 'n_tokens_mistral': 5484, 'n_tokens_neox': 5057, 'n_words': 1662}