<commit_before>//
// https://github.com/SteveKChiu/WebCacheMonk
//
// Copyright 2015, <NAME> <<EMAIL>>
//
// The MIT License (http://www.opensource.org/licenses/mit-license.php)
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//

import Foundation

//---------------------------------------------------------------------------

public class WebObjectCache<OBJECT> {
    private var queue: dispatch_queue_t
    private let cache = NSCache()
    private let dataSource: WebCacheSource
    
    public var decoder: ObjectDecoder?
    public var evaluator: ObjectEvaluator?
    
    public typealias ObjectDecoder = (NSData, options: [String: Any]?, completion: (OBJECT?) -> Void) -> Void
    public typealias ObjectEvaluator = (OBJECT) -> Int
    
    public init(name: String? = nil, configuration: NSURLSessionConfiguration? = nil, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = WebCacheFileStore(name: name) | WebCacheFetcher(configuration: configuration)
        self.decoder = decoder
        self.cache.name = name ?? "WebObjectCache"
    }

    public init(path: String, configuration: NSURLSessionConfiguration? = nil, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = WebCacheFileStore(path: path) | WebCacheFetcher(configuration: configuration)
        self.decoder = decoder
        self.cache.name = "WebObjectCache"
    }

    public init(source: WebCacheSource, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = source
        self.decoder = decoder
        self.cache.name = "WebObjectCache"
    }

    public var totalCostLimit: Int {
        get {
            return self.cache.totalCostLimit
        }
        set {
            self.cache.totalCostLimit = newValue
        }
    }
    
    public var countLimit: Int {
        get {
            return self.cache.countLimit
        }
        set {
            self.cache.countLimit = newValue
        }
    }
    
    public func decode(data: NSData, options: [String: Any]?, completion: (OBJECT?) -> Void) {
        if let decoder = self.decoder {
            decoder(data, options: options, completion: completion)
        } else {
            completion(nil)
        }
    }
    
    public func evaluate(object: OBJECT) -> Int {
        return self.evaluator?(object) ?? 0
    }
    
    public func fetch(url: String, tag: String? = nil, options: [String: Any]? = nil, expired: WebCacheExpiration = .Default, progress: NSProgress? = nil, completion: (OBJECT?) -> Void) {
        dispatch_async(self.queue) {
            if let entry = self.cache.objectForKey(url) as? WebObjectEntry {
                if let object = entry.get(tag) as? OBJECT {
                    completion(object)
                    return
                }
            }
        
            let receiver = WebCacheDataReceiver(url: url) {
                receiver in
                
                guard let data = receiver.buffer else {
                    completion(nil)
                    return
                }
                
                self.decode(data, options: options) {
                    object in
                    
                    if let object = object {
                        self.store(url, tag: tag, object: object)
                    }
                    
                    completion(object)
                }
            }
            
            self.dataSource.fetch(url, offset: nil, length: nil, expired: expired, progress: progress, receiver: receiver)
        }
    }
    
    public func store(url: String, tag: String? = nil, object: OBJECT) {
        dispatch_async(self.queue) {
            let cost = self.evaluate(object)
            if let entry = self.cache.objectForKey(url) as? WebObjectEntry {
                entry.set(tag, object: object, cost: cost)
                self.cache.setObject(entry, forKey: url, cost: entry.cost)
            } else {
                let entry = WebObjectEntry(tag: tag, object: object, cost: cost)
                self.cache.setObject(entry, forKey: url, cost: entry.cost)
            }
        }
    }
    
    public func change(url: String, expired: WebCacheExpiration) {
        dispatch_async(self.queue) {
            if expired.isExpired {
                self.remove(url)
            } else if let sourceStore = self.dataSource as? WebCacheMutableStore {
                sourceStore.change(url, expired: expired)
            }
        }
    }<commit_msg>Add get method<commit_after>//
// https://github.com/SteveKChiu/WebCacheMonk
//
// Copyright 2015, <NAME> <<EMAIL>>
//
// The MIT License (http://www.opensource.org/licenses/mit-license.php)
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
//

import Foundation

//---------------------------------------------------------------------------

public class WebObjectCache<OBJECT> {
    private var queue: dispatch_queue_t
    private let cache = NSCache()
    private let dataSource: WebCacheSource
    
    public var decoder: ObjectDecoder?
    public var evaluator: ObjectEvaluator?
    
    public typealias ObjectDecoder = (NSData, options: [String: Any]?, completion: (OBJECT?) -> Void) -> Void
    public typealias ObjectEvaluator = (OBJECT) -> Int
    
    public init(name: String? = nil, configuration: NSURLSessionConfiguration? = nil, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = WebCacheFileStore(name: name) | WebCacheFetcher(configuration: configuration)
        self.decoder = decoder
        self.cache.name = name ?? "WebObjectCache"
    }

    public init(path: String, configuration: NSURLSessionConfiguration? = nil, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = WebCacheFileStore(path: path) | WebCacheFetcher(configuration: configuration)
        self.decoder = decoder
        self.cache.name = "WebObjectCache"
    }

    public init(source: WebCacheSource, decoder: ObjectDecoder?) {
        self.queue = dispatch_queue_create("WebObjectCache", DISPATCH_QUEUE_SERIAL)
        self.dataSource = source
        self.decoder = decoder
        self.cache.name = "WebObjectCache"
    }

    public var totalCostLimit: Int {
        get {
            return self.cache.totalCostLimit
        }
        set {
            self.cache.totalCostLimit = newValue
        }
    }
    
    public var countLimit: Int {
        get {
            return self.cache.countLimit
        }
        set {
            self.cache.countLimit = newValue
        }
    }
    
    public func decode(data: NSData, options: [String: Any]?, completion: (OBJECT?) -> Void) {
        if let decoder = self.decoder {
            decoder(data, options: options, completion: completion)
        } else {
            completion(nil)
        }
    }
    
    public func evaluate(object: OBJECT) -> Int {
        return self.evaluator?(object) ?? 0
    }
    
    public func fetch(url: String, tag: String? = nil, options: [String: Any]? = nil, expired: WebCacheExpiration = .Default, progress: NSProgress? = nil, completion: (OBJECT?) -> Void) {
        dispatch_async(self.queue) {
            if let entry = self.cache.objectForKey(url) as? WebObjectEntry {
                if let object = entry.get(tag) as? OBJECT {
                    completion(object)
                    return
                }
            }
        
            let receiver = WebCacheDataReceiver(url: url) {
                receiver in
                
                guard let data = receiver.buffer else {
                    completion(nil)
                    return
                }
                
                self.decode(data, options: options) {
                    object in
                    
                    if let object = object {
                        self.set(url, tag: tag, object: object)
                    }
                    
                    completion(object)
                }
            }
            
            self.dataSource.fetch(url, offset: nil, length: nil, expired: expired, progress: progress, receiver: receiver)
        }
    }
    
    public func get(url: String, tag: String? = nil) -> OBJECT? {
        var object: OBJECT?
        dispatch_sync(self.queue) {
            if let entry = self.cache.objectForKey(url) as? WebObjectEntry {
                object = entry.get(tag) as? OBJECT
            }
        }
        return object
    }
    
    public func set(url: String, tag: String? = nil, object: OBJECT) {
        dispatch_async(self.queue) {
            let cost = self.evaluate(object)
            if let entry = self.cache.objectForKey(url) as? WebObjectEntry {
                entry.set(tag, object: object, cost: cost)
                self.cache.setObject(entry, forKey: url, cost: entry.cost)
            } else {
                let entry = WebObjectEntry(tag: tag, object: object, cost: cost)
                self.cache.setObject(entry, forKey: url, cost: entry.cost)
            }
        }
    }
    
    public func change(url: String, expired: WebCacheExpiration) {
        dispatch_async(self.queue) {
            if expired.isExpired {
                self.remove(url)
            } else if let sourceStore = self.dataSource as? WebCacheMutableStore {
                sourceStore.change(url, expired: expired)
            }
        }
    }
{'fraction_non_alphanumeric': 0.10159055926115956, 'fraction_numerical': 0.0008551393877201984, 'mean_word_length': 1.9948783610755443, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 24, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7567747', 'n_tokens_mistral': 3100, 'n_tokens_neox': 2961, 'n_words': 936}