<commit_before>//
//  CMLogProcessor.swift
//  BuildTimeAnalyzer
//
//  Created by <NAME> on 01/05/2016.
//  Copyright © 2016 <NAME>. All rights reserved.
//

typealias CMUpdateClosure = (result: [CMCompileMeasure], didComplete: Bool) -> ()

protocol CMLogProcessorProtocol: class {
    var unprocessedResult: [CMRawMeasure] { get set }
    var updateHandler: CMUpdateClosure? { get set }
    var workspace: CMXcodeWorkSpace? { get set }
    var shouldCancel: Bool { get set }
    
    func processingDidStart()
    func processingDidFinish()
}

private let processRx = try! NSRegularExpression(pattern:  "^\\d*\\.?\\dms\\t/", options: [])

extension CMLogProcessorProtocol {
    func process(productName: String, buildCompletionDate: NSDate?, updateHandler: CMUpdateClosure?) {
        workspace = CMXcodeWorkSpace(productName: productName, buildCompletionDate: buildCompletionDate)
        workspace?.logTextForProduct() { [weak self] (text) in
            guard let text = text else {
                updateHandler?(result: [], didComplete: true)
                return
            }
            
            self?.updateHandler = updateHandler
            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)) {
                self?.process(text: text)
            }
        }
    }
    
    // MARK: Private methods
    
    private func process(text text: String) {
        let characterSet = NSCharacterSet(charactersInString:"\r\"")
        var remainingRange = text.startIndex..<text.endIndex
        var rawMeasures: [String:Double] = [:]

        unprocessedResult.removeAll()
        processingDidStart()
        
        while let nextRange = text.rangeOfCharacterFromSet(characterSet, options: [], range: remainingRange) {
            let currentRange = remainingRange.startIndex..<nextRange.endIndex
            let text = text.substringWithRange(currentRange)
            
            defer {
                remainingRange = nextRange.endIndex..<remainingRange.endIndex
            }
            
            let range = NSMakeRange(0, text.characters.count)
            guard let match = processRx.firstMatchInString(text, options: [], range: range) else { continue }
            
            let timeString = text.substringToIndex(text.startIndex.advancedBy(match.range.length - 4))
            if let time = Double(timeString) {
                let value = text.substringFromIndex(text.startIndex.advancedBy(match.range.length - 1))

                let cumulativeTime = rawMeasures[value] ?? 0
                rawMeasures[value] = cumulativeTime + time
            }
            if shouldCancel {
                break
            }
        }

        for (k, v) in rawMeasures {
            unprocessedResult.append(CMRawMeasure(time: v, text: k))
        }

        processingDidFinish()
    }
    
    private func updateResults(didComplete: Bool) {
        let cappedResult = capEntries(unprocessedResult)
        updateHandler?(result: processResult(cappedResult), didComplete: didComplete)
        
        if didComplete {
            unprocessedResult.removeAll()
        }
    }
    
    private func capEntries(entries: [CMRawMeasure]) -> [CMRawMeasure] {
        let limit = 20
        
        let distinct = Array(Set(entries))
        var sorted = distinct.sort{ $0.time > $1.time }
        if sorted.count > limit {
            sorted = Array(sorted[0..<limit])
        }
        return sorted
    }
    
    private func processResult(unprocessedResult: [CMRawMeasure]) -> [CMCompileMeasure] {
        let unprocessedResult = capEntries(unprocessedResult)
        
        var result: [CMCompileMeasure] = []
        for entry in unprocessedResult {
            let code = entry.text.characters.split("\t").map(String.init)
            if code.count >= 2, let measure = CMCompileMeasure(time: entry.time, rawPath: code[0], code: trimPrefixes(code[1])) {
                result.append(measure)
            }
        }
        return result
    }
    
    private func trimPrefixes(code: String) -> String {
        var code = code
        ["@objc ", "final ", "@IBAction "].forEach { (prefix) in
            if code.hasPrefix(prefix) {
                code = code.substringFromIndex(code.startIndex.advancedBy(prefix.characters.count))
            }
        }
        return code
    }
}

class CMLogProcessor: NSObject, CMLogProcessorProtocol {
    
    var unprocessedResult: [CMRawMeasure] = []
    var updateHandler: CMUpdateClosure?
    var workspace: CMXcodeWorkSpace?
    var shouldCancel = false
    var timer: NSTimer?
    
    func processingDidStart() {
        dispatch_async(dispatch_get_main_queue()) {
            self.timer = NSTimer.scheduledTimerWithTimeInterval(1, target: self, selector: #selector(self.timerCallback(_:)), userInfo: nil, repeats: true)
        }
    }
    
    func processingDidFinish() {
        dispatch_async(dispatch_get_main_queue()) {
            self.timer?.invalidate()
            self.timer = nil
            self.shouldCancel = false
            self.updateResults(true)
        }
    }
    
    func timerCallback(timer: NSTimer) {
        updateResults(false)
    }
}<commit_msg>Remove results cap<commit_after>//
//  CMLogProcessor.swift
//  BuildTimeAnalyzer
//
//  Created by <NAME> on 01/05/2016.
//  Copyright © 2016 <NAME>. All rights reserved.
//

typealias CMUpdateClosure = (result: [CMCompileMeasure], didComplete: Bool) -> ()

protocol CMLogProcessorProtocol: class {
    var unprocessedResult: [CMRawMeasure] { get set }
    var updateHandler: CMUpdateClosure? { get set }
    var workspace: CMXcodeWorkSpace? { get set }
    var shouldCancel: Bool { get set }
    
    func processingDidStart()
    func processingDidFinish()
}

private let processRx = try! NSRegularExpression(pattern:  "^\\d*\\.?\\dms\\t/", options: [])

extension CMLogProcessorProtocol {
    func process(productName: String, buildCompletionDate: NSDate?, updateHandler: CMUpdateClosure?) {
        workspace = CMXcodeWorkSpace(productName: productName, buildCompletionDate: buildCompletionDate)
        workspace?.logTextForProduct() { [weak self] (text) in
            guard let text = text else {
                updateHandler?(result: [], didComplete: true)
                return
            }
            
            self?.updateHandler = updateHandler
            dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0)) {
                self?.process(text: text)
            }
        }
    }
    
    // MARK: Private methods
    
    private func process(text text: String) {
        let characterSet = NSCharacterSet(charactersInString:"\r\"")
        var remainingRange = text.startIndex..<text.endIndex
        var rawMeasures: [String:Double] = [:]

        unprocessedResult.removeAll()
        processingDidStart()
        
        while let nextRange = text.rangeOfCharacterFromSet(characterSet, options: [], range: remainingRange) {
            let currentRange = remainingRange.startIndex..<nextRange.endIndex
            let text = text.substringWithRange(currentRange)
            
            defer {
                remainingRange = nextRange.endIndex..<remainingRange.endIndex
            }
            
            let range = NSMakeRange(0, text.characters.count)
            guard let match = processRx.firstMatchInString(text, options: [], range: range) else { continue }
            
            let timeString = text.substringToIndex(text.startIndex.advancedBy(match.range.length - 4))
            if let time = Double(timeString) {
                let value = text.substringFromIndex(text.startIndex.advancedBy(match.range.length - 1))

                let cumulativeTime = rawMeasures[value] ?? 0
                rawMeasures[value] = cumulativeTime + time
            }
            if shouldCancel {
                break
            }
        }

        for (k, v) in rawMeasures {
            // Only show files whose compile time is above 10ms
            if v > 10 {
                unprocessedResult.append(CMRawMeasure(time: v, text: k))
            }
        }

        unprocessedResult.sortInPlace{ $0.time > $1.time }

        processingDidFinish()
    }
    
    private func updateResults(didComplete: Bool) {
        updateHandler?(result: processResult(unprocessedResult), didComplete: didComplete)
        if didComplete {
            unprocessedResult.removeAll()
        }
    }

    private func processResult(unprocessedResult: [CMRawMeasure]) -> [CMCompileMeasure] {
        var result: [CMCompileMeasure] = []
        for entry in unprocessedResult {
            let code = entry.text.characters.split("\t").map(String.init)
            if code.count >= 2, let measure = CMCompileMeasure(time: entry.time, rawPath: code[0], code: trimPrefixes(code[1])) {
                result.append(measure)
            }
        }
        return result
    }
    
    private func trimPrefixes(code: String) -> String {
        var code = code
        ["@objc ", "final ", "@IBAction "].forEach { (prefix) in
            if code.hasPrefix(prefix) {
                code = code.substringFromIndex(code.startIndex.advancedBy(prefix.characters.count))
            }
        }
        return code
    }
}

class CMLogProcessor: NSObject, CMLogProcessorProtocol {
    
    var unprocessedResult: [CMRawMeasure] = []
    var updateHandler: CMUpdateClosure?
    var workspace: CMXcodeWorkSpace?
    var shouldCancel = false
    var timer: NSTimer?
    
    func processingDidStart() {
        dispatch_async(dispatch_get_main_queue()) {
            self.timer = NSTimer.scheduledTimerWithTimeInterval(1, target: self, selector: #selector(self.timerCallback(_:)), userInfo: nil, repeats: true)
        }
    }
    
    func processingDidFinish() {
        dispatch_async(dispatch_get_main_queue()) {
            self.timer?.invalidate()
            self.timer = nil
            self.shouldCancel = false
            self.updateResults(true)
        }
    }
    
    func timerCallback(timer: NSTimer) {
        updateResults(false)
    }
}
{'fraction_non_alphanumeric': 0.09861248761149653, 'fraction_numerical': 0.005252725470763132, 'mean_word_length': 2.2136942675159235, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3383691', 'n_tokens_mistral': 2830, 'n_tokens_neox': 2710, 'n_words': 594}