<commit_before>//
//  DataCache.swift
//  Quadrat
//
//  Created by <NAME> on 01/12/14.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//

import Foundation

#if os(iOS)
import UIKit
#endif

/** Configs for data cache. */
struct DataCacheConfiguration {
    
    /** Time to keep a data in cache in seconds. Default is 1 week.  */
    private let maxCacheAge         = (60 * 60 * 24 * 7) as NSTimeInterval
    
    /** The maximum size for disk cache in bytes. Default is 10MB. */
    private let maxDiskCacheSize    = (1024 * 1024 * 10) as UInt
    
    /** The maximum size for memory cache in bytes. Default is 10MB. */
    private let maxMemoryCacheSize  = (1024 * 1024 * 10) as UInt
}

/** The queue for I/O operations. Shared between several instances. */
private let privateQueue    = NSOperationQueue()

/** Responsible for caching data on disk and memory. Thread safe. */
class DataCache {
    
    /** Logger to log all errors. */
    var logger: Logger?
    
    /** The URL to directory where we put all the files. */
    private let directoryURL: NSURL
    
    /** In memory cache for NSData. */
    private let cache           = NSCache()
    
    /** Obsever objects from NSNotificationCenter. */
    private var observers       = [AnyObject]()
    
    /** The file manager to use for all file operations. */
    private let fileManager     = NSFileManager.defaultManager()
    
    /** The configuration for cache. */
    private let cacheConfiguration  = DataCacheConfiguration()
    
    init(name: String?) {
        cache.totalCostLimit = Int(cacheConfiguration.maxMemoryCacheSize)
        let directoryName = "net.foursquare.quadrat"
        let subdirectiryName = (name != nil) ? ( "Cache" + name! ) : "DefaultCache"
        let cacheURL: NSURL?
        do {
            cacheURL = try fileManager.URLForDirectory(.CachesDirectory, inDomain: .UserDomainMask,
                        appropriateForURL: nil, create: true)
        } catch _ {
            fatalError("Can't get access to cache directory")
        }
        self.directoryURL = cacheURL!.URLByAppendingPathComponent("\(directoryName)/DataCache/\(subdirectiryName)")
        privateQueue.maxConcurrentOperationCount = 1
        createBaseDirectory()
        subscribeForNotifications()
    }
    
    deinit {
        for observer in observers {
            NSNotificationCenter.defaultCenter().removeObserver(observer)
        }
    }
    
    /** Returns data for key. */
    func dataForKey(key: String) -> NSData? {
        var result: NSData?
        privateQueue.addOperationWithBlock {
            result = self.cache.objectForKey(key) as? NSData
            if result == nil {
                let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
                result = NSData(contentsOfURL: targetURL)
                if let result = result {
                    self.cache.setObject(result, forKey: key, cost: result.length)
                }
            }
        }
        privateQueue.waitUntilAllOperationsAreFinished()
        return result
    }
    
    /** Copies file into cache. */
    func addFileAtURL(URL: NSURL, withKey key: String) {
        privateQueue.addOperationWithBlock { () -> Void in
            let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
            do {
                try self.fileManager.copyItemAtURL(URL, toURL: targetURL)
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't copy file into cache directory.")
            } catch {
            }
        }
        privateQueue.waitUntilAllOperationsAreFinished()
    }
    
    /** Saves data into cache. */
    func addData(data: NSData, withKey key: String) {
        privateQueue.addOperationWithBlock { () -> Void in
            let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
            do {
                try data.writeToURL(targetURL, options: .DataWritingAtomic)
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't save file into cache directory.")
            } catch {
            }
        }
    }
    
    /** Subcribes for iOS specific notifications to perform cache cleaning. */
    private func subscribeForNotifications() {
        #if os(iOS)
            
            let center = NSNotificationCenter.defaultCenter()
            let didEnterBackground = UIApplicationDidEnterBackgroundNotification
            let firstObserver = center.addObserverForName(didEnterBackground, object: nil, queue: nil) {
                [weak self] (notification) -> Void in
                self?.cleanupCache()
                self?.cache.removeAllObjects()
            }
            observers.append(firstObserver)
            
            let didReceiveMemoryWaring = UIApplicationDidReceiveMemoryWarningNotification
            let secondObserver = center.addObserverForName(didReceiveMemoryWaring, object: nil, queue: nil) {
                [weak self] (notification) -> Void in
                self?.cache.removeAllObjects()
            }
            observers.append(secondObserver)
        #endif
    }
    
    /** Creates base directory. */
    private func createBaseDirectory() {
        do {
            try fileManager.createDirectoryAtURL(directoryURL,
                        withIntermediateDirectories: true, attributes: nil)
        } catch let error as NSError {
            self.logger?.logError(error, withMessage: "Cache can't create base directory.")
        }
    }
    
    /** Removes all cached files. */
    func clearCache() {
        privateQueue.addOperationWithBlock {
            self.cache.removeAllObjects()
            do {
                try self.fileManager.removeItemAtURL(self.directoryURL)
                self.createBaseDirectory()
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't remove base directory.")
            } catch {
            }
        }
    }
    
    /** Removes expired files. In addition removes 1/4 of files if total size exceeds `maxDiskCacheSize`. */
    private func cleanupCache() {
        privateQueue.addOperationWithBlock {
            let fileURLs = self.getFilesToRemove()
            fileURLs.forEach {
                _ = try? self.fileManager.removeItemAtURL($0)
            }
        }
    }
    
    private func getFilesToRemove() -> [NSURL] {
        let expirationDate = NSDate(timeIntervalSinceNow: -self.cacheConfiguration.maxCacheAge)
        let properties = [NSURLContentModificationDateKey, NSURLTotalFileAllocatedSizeKey]
        var cacheSize: UInt = 0
        var expiredFiles = [NSURL]()
        var validFiles = [NSURL]()
        let fileURLs = self.getCachedFileURLs()
        /** Searches for expired files and calculates total size. */
        fileURLs.forEach {
            (aFileURL) -> () in
            let values: [String : AnyObject]? = try? aFileURL.resourceValuesForKeys(properties)
            if let values = values, let modificationDate = values[NSURLContentModificationDateKey] as? NSDate {
                if modificationDate.laterDate(expirationDate).isEqualToDate(modificationDate) {
                    validFiles.append(aFileURL)
                    if let fileSize = values[NSURLTotalFileAllocatedSizeKey] as? UInt {
                        cacheSize += fileSize
                    }
                } else {
                    expiredFiles.append(aFileURL)
                }
            }
        }
        
        if cacheSize > self.cacheConfiguration.maxDiskCacheSize {
            validFiles = self.sortFileURLByModificationDate(validFiles)
            /** Let's just remove 1/4 of all files. */
            validFiles.removeRange(Range(start: 0, end: validFiles.count / 4))
            expiredFiles += validFiles
        }
        return expiredFiles
    }
    
    private func getCachedFileURLs() -> [NSURL] {
        let properties = [NSURLContentModificationDateKey, NSURLTotalFileAllocatedSizeKey]
        do {
            return try self.fileManager.contentsOfDirectoryAtURL(self.directoryURL,
                includingPropertiesForKeys: properties, options: .SkipsHiddenFiles)
        } catch let error as NSError {
            self.logger?.logError(error, withMessage: "Cache can't get properties of files in base directory.")
            return [NSURL]()
        }
    }
    
    private func sortFileURLByModificationDate(urls: [NSURL]) -> [NSURL] {
        return urls.sort {
            (url1, url2) -> Bool in
            let dateKey = [NSURLContentModificationDateKey]
            do {
                let values1 = try url1.resourceValuesForKeys(dateKey) as? [String: NSDate]
                let values2 = try url2.resourceValuesForKeys(dateKey) as? [String: NSDate]
                if let date1 = values1?[NSURLContentModificationDateKey] {
                    if let date2 = values2?[NSURLContentModificationDateKey] {
                        return date1.compare(date2) == .OrderedAscending
                    }
                }
            } catch {
                return false
            }
            return false
        }
    }
    
}
<commit_msg>Fix various code style issues.<commit_after>//
//  DataCache.swift
//  Quadrat
//
//  Created by <NAME> on 01/12/14.
//  Copyright (c) 2014 <NAME>. All rights reserved.
//

import Foundation

#if os(iOS)
import UIKit
#endif

/** Configs for data cache. */
struct DataCacheConfiguration {
    
    /** Time to keep a data in cache in seconds. Default is 1 week.  */
    private let maxCacheAge = (60 * 60 * 24 * 7) as NSTimeInterval
    
    /** The maximum size for disk cache in bytes. Default is 10MB. */
    private let maxDiskCacheSize = (1024 * 1024 * 10) as UInt
    
    /** The maximum size for memory cache in bytes. Default is 10MB. */
    private let maxMemoryCacheSize = (1024 * 1024 * 10) as UInt
}

/** The queue for I/O operations. Shared between several instances. */
private let privateQueue    = NSOperationQueue()

/** Responsible for caching data on disk and memory. Thread safe. */
class DataCache {
    
    /** Logger to log all errors. */
    var logger: Logger?
    
    /** The URL to directory where we put all the files. */
    private let directoryURL: NSURL
    
    /** In memory cache for NSData. */
    private let cache = NSCache()
    
    /** Obsever objects from NSNotificationCenter. */
    private var observers = [AnyObject]()
    
    /** The file manager to use for all file operations. */
    private let fileManager = NSFileManager.defaultManager()
    
    /** The configuration for cache. */
    private let cacheConfiguration  = DataCacheConfiguration()
    
    init(name: String?) {
        cache.totalCostLimit = Int(cacheConfiguration.maxMemoryCacheSize)
        let directoryName = "net.foursquare.quadrat"
        let subdirectiryName = (name != nil) ? ( "Cache" + name! ) : "DefaultCache"
        let cacheURL: NSURL?
        do {
            cacheURL = try fileManager.URLForDirectory(.CachesDirectory, inDomain: .UserDomainMask,
                        appropriateForURL: nil, create: true)
        } catch _ {
            fatalError("Can't get access to cache directory")
        }
        self.directoryURL = cacheURL!.URLByAppendingPathComponent("\(directoryName)/DataCache/\(subdirectiryName)")
        privateQueue.maxConcurrentOperationCount = 1
        createBaseDirectory()
        subscribeForNotifications()
    }
    
    deinit {
        observers.forEach {
            NSNotificationCenter.defaultCenter().removeObserver($0)
        }
    }
    
    /** Returns data for key. */
    func dataForKey(key: String) -> NSData? {
        var result: NSData?
        privateQueue.addOperationWithBlock {
            result = self.cache.objectForKey(key) as? NSData
            if result == nil {
                let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
                result = NSData(contentsOfURL: targetURL)
                if let result = result {
                    self.cache.setObject(result, forKey: key, cost: result.length)
                }
            }
        }
        privateQueue.waitUntilAllOperationsAreFinished()
        return result
    }
    
    /** Copies file into cache. */
    func addFileAtURL(URL: NSURL, withKey key: String) {
        privateQueue.addOperationWithBlock { () -> Void in
            let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
            do {
                try self.fileManager.copyItemAtURL(URL, toURL: targetURL)
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't copy file into cache directory.")
            } catch {
            }
        }
        privateQueue.waitUntilAllOperationsAreFinished()
    }
    
    /** Saves data into cache. */
    func addData(data: NSData, withKey key: String) {
        privateQueue.addOperationWithBlock { () -> Void in
            let targetURL = self.directoryURL.URLByAppendingPathComponent(key)
            do {
                try data.writeToURL(targetURL, options: .DataWritingAtomic)
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't save file into cache directory.")
            } catch {
            }
        }
    }
    
    /** Subcribes for iOS specific notifications to perform cache cleaning. */
    private func subscribeForNotifications() {
        #if os(iOS)
            
            let center = NSNotificationCenter.defaultCenter()
            let didEnterBackground = UIApplicationDidEnterBackgroundNotification
            let firstObserver = center.addObserverForName(didEnterBackground, object: nil, queue: nil) {
                [weak self] (notification) -> Void in
                self?.cleanupCache()
                self?.cache.removeAllObjects()
            }
            observers.append(firstObserver)
            
            let didReceiveMemoryWaring = UIApplicationDidReceiveMemoryWarningNotification
            let secondObserver = center.addObserverForName(didReceiveMemoryWaring, object: nil, queue: nil) {
                [weak self] (notification) -> Void in
                self?.cache.removeAllObjects()
            }
            observers.append(secondObserver)
        #endif
    }
    
    /** Creates base directory. */
    private func createBaseDirectory() {
        do {
            try fileManager.createDirectoryAtURL(directoryURL,
                        withIntermediateDirectories: true, attributes: nil)
        } catch let error as NSError {
            self.logger?.logError(error, withMessage: "Cache can't create base directory.")
        }
    }
    
    /** Removes all cached files. */
    func clearCache() {
        privateQueue.addOperationWithBlock {
            self.cache.removeAllObjects()
            do {
                try self.fileManager.removeItemAtURL(self.directoryURL)
                self.createBaseDirectory()
            } catch let error as NSError {
                self.logger?.logError(error, withMessage: "Cache can't remove base directory.")
            } catch {
            }
        }
    }
    
    /** Removes expired files. In addition removes 1/4 of files if total size exceeds `maxDiskCacheSize`. */
    private func cleanupCache() {
        privateQueue.addOperationWithBlock {
            let fileURLs = self.getFilesToRemove()
            fileURLs.forEach {
                _ = try? self.fileManager.removeItemAtURL($0)
            }
        }
    }
    
    private func getFilesToRemove() -> [NSURL] {
        let expirationDate = NSDate(timeIntervalSinceNow: -self.cacheConfiguration.maxCacheAge)
        let properties = [NSURLContentModificationDateKey, NSURLTotalFileAllocatedSizeKey]
        var cacheSize: UInt = 0
        var expiredFiles = [NSURL]()
        var validFiles = [NSURL]()
        let fileURLs = self.getCachedFileURLs()
        /** Searches for expired files and calculates total size. */
        fileURLs.forEach {
            (aFileURL) -> () in
            let values: [String : AnyObject]? = try? aFileURL.resourceValuesForKeys(properties)
            if let values = values, let modificationDate = values[NSURLContentModificationDateKey] as? NSDate {
                if modificationDate.laterDate(expirationDate).isEqualToDate(modificationDate) {
                    validFiles.append(aFileURL)
                    if let fileSize = values[NSURLTotalFileAllocatedSizeKey] as? UInt {
                        cacheSize += fileSize
                    }
                } else {
                    expiredFiles.append(aFileURL)
                }
            }
        }
        
        if cacheSize > self.cacheConfiguration.maxDiskCacheSize {
            validFiles = self.sortFileURLByModificationDate(validFiles)
            /** Let's just remove 1/4 of all files. */
            validFiles.removeRange(Range(start: 0, end: validFiles.count / 4))
            expiredFiles += validFiles
        }
        return expiredFiles
    }
    
    private func getCachedFileURLs() -> [NSURL] {
        let properties = [NSURLContentModificationDateKey, NSURLTotalFileAllocatedSizeKey]
        do {
            return try self.fileManager.contentsOfDirectoryAtURL(self.directoryURL,
                includingPropertiesForKeys: properties, options: .SkipsHiddenFiles)
        } catch let error as NSError {
            self.logger?.logError(error, withMessage: "Cache can't get properties of files in base directory.")
            return [NSURL]()
        }
    }
    
    private func sortFileURLByModificationDate(urls: [NSURL]) -> [NSURL] {
        return urls.sort {
            (url1, url2) -> Bool in
            let dateKey = [NSURLContentModificationDateKey]
            do {
                let values1 = try url1.resourceValuesForKeys(dateKey) as? [String: NSDate]
                let values2 = try url2.resourceValuesForKeys(dateKey) as? [String: NSDate]
                if let date1 = values1?[NSURLContentModificationDateKey] {
                    if let date2 = values2?[NSURLContentModificationDateKey] {
                        return date1.compare(date2) == .OrderedAscending
                    }
                }
            } catch {
                return false
            }
            return false
        }
    }
    
}

{'fraction_non_alphanumeric': 0.08580732700135685, 'fraction_numerical': 0.00689280868385346, 'mean_word_length': 1.9719354838709677, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '790029', 'n_tokens_mistral': 4959, 'n_tokens_neox': 4721, 'n_words': 1241}