<commit_before>//
//  FuzzyMatchTokenTests.swift
//  Mustard
//
//  Created by <NAME> on 12/31/16.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import XCTest
import Mustard

class FuzzyLiteralMatch: TokenType {
    
    let target: String
    let exclusions: CharacterSet
    var position: String.UnicodeScalarIndex
    
    required convenience init() {
        self.init(target: "", ignoring: CharacterSet.whitespaces)
    }
    
    init(target: String, ignoring exclusions: CharacterSet) {
        self.target = target
        self.position = target.unicodeScalars.startIndex
        self.exclusions = exclusions
    }
    
    func canAppend(next scalar: UnicodeScalar) -> Bool {
        
        guard position < target.unicodeScalars.endIndex else {
            // we've matched all of the target
            return false
        }
        
        if scalar == target.unicodeScalars[position] {
            // scalar matches the next target scalar!
            // advance the position and return true
            position = target.unicodeScalars.index(after: position)
            return true
        }
        
        else if position > target.unicodeScalars.startIndex && exclusions.contains(scalar) {
            // if: 
            // - we've matched at least one of the target scalars; and
            // - this scalar matches a scalar that's can be ignored
            // then:
            // - return true without advancing the position
            return true
        }
        else {
            // scalar isn't the next target scalar, or a scalar that can be ignored
            return false
        }
    }
    
    func canCompleteWhenNextScalar(is scalar: UnicodeScalar) -> Bool {
        
        if position == target.unicodeScalars.endIndex {
            resetToken()
            return true
        }
        else {
            resetToken()
            return false
        }
    }
    
    private func resetToken() {
        position = target.unicodeScalars.startIndex
    }
}

class FuzzyMatchTokenTests: XCTestCase {
    
    func testSpecialFormat() {
        
        let messyInput = "Serial: #YF 1942-B 12/01/27 (Scanned)"
        let fuzzyTokenzier = FuzzyLiteralMatch(target: "#YF1942B",
                                               ignoring: CharacterSet.whitespaces.union(.punctuationCharacters))
        
        let tokens = messyInput.tokens(from: fuzzyTokenzier)
        
        XCTAssert(tokens.count == 1, "Unexpected number of tokens [\(tokens.count)]")
        
        XCTAssert(tokens[0].tokenType is FuzzyLiteralMatch)
        XCTAssert(tokens[0].text == "#YF 1942-B")
        
    }
}<commit_msg>Add a token that matches a template.<commit_after>//
//  FuzzyMatchTokenTests.swift
//  Mustard
//
//  Created by <NAME> on 12/31/16.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import XCTest
import Mustard

infix operator ~=
func ~= (option: CharacterSet, input: UnicodeScalar) -> Bool {
    return option.contains(input)
}

class FuzzyLiteralMatch: TokenType {
    
    let target: String
    let exclusions: CharacterSet
    var position: String.UnicodeScalarIndex
    
    required convenience init() {
        self.init(target: "", ignoring: CharacterSet.whitespaces)
    }
    
    init(target: String, ignoring exclusions: CharacterSet) {
        self.target = target
        self.position = target.unicodeScalars.startIndex
        self.exclusions = exclusions
    }
    
    func canAppend(next scalar: UnicodeScalar) -> Bool {
        
        guard position < target.unicodeScalars.endIndex else {
            // we've matched all of the target
            return false
        }
        
        let targetScalar = target.unicodeScalars[position]
        
        switch (scalar, targetScalar) {
        case (_, _)
                where scalar == targetScalar,
             (CharacterSet.lowercaseLetters, CharacterSet.uppercaseLetters)
                where scalar.value - targetScalar.value == 32,
             (CharacterSet.uppercaseLetters, CharacterSet.lowercaseLetters)
                where targetScalar.value - scalar.value == 32:
            // scalar and target scalar are either an exact match,
            // or equivilant upper/lowercase pair
            // advance the position and return true
            position = target.unicodeScalars.index(after: position)
            return true
        
        case (exclusions, _)
            where position > target.unicodeScalars.startIndex:
            // if:
            // - we've matched at least one of the target scalars; and
            // - this scalar matches a scalar that's can be ignored
            // then:
            // - return true without advancing the position
            return true
            
        default:
            // scalar isn't the next target scalar, or a scalar that can be ignored
            return false
        }
    }
    
    func canCompleteWhenNextScalar(is scalar: UnicodeScalar) -> Bool {
        
        if position == target.unicodeScalars.endIndex {
            resetToken()
            return true
        }
        else {
            resetToken()
            return false
        }
    }
    
    private func resetToken() {
        position = target.unicodeScalars.startIndex
    }
}

class DateMatch: TokenType {
    
    let template = "00/00/00"
    var position: String.UnicodeScalarIndex
    
    required init() {
        position = template.unicodeScalars.startIndex
    }
    
    func canAppend(next scalar: UnicodeScalar) -> Bool {
        
        guard position < template.unicodeScalars.endIndex else {
            // we've matched all of the template
            return false
        }
        
        switch (template.unicodeScalars[position], scalar) {
        case ("\u{0030}", CharacterSet.decimalDigits), // match with a decimal digit
             ("\u{002F}", "\u{002F}"):                 // match with the '/' character
            
            position = template.unicodeScalars.index(after: position)
            return true
            
        default:
            return false
        }
    }
    
    func canCompleteWhenNextScalar(is scalar: UnicodeScalar) -> Bool {
        
        if position == template.unicodeScalars.endIndex {
            resetToken()
            return true
        }
        else {
            resetToken()
            return false
        }
    }
    
    private func resetToken() {
        position = template.unicodeScalars.startIndex
    }
}

class FuzzyMatchTokenTests: XCTestCase {
    
    func testSpecialFormat() {
        
        let messyInput = "Serial: #YF 1942-b 12/01/27 (Scanned)"
        
        let fuzzyTokenzier = FuzzyLiteralMatch(target: "#YF1942B",
                                               ignoring: CharacterSet.whitespaces.union(.punctuationCharacters))
        
        let tokens = messyInput.tokens(from: fuzzyTokenzier, DateMatch.tokenizer)
        
        XCTAssert(tokens.count == 2, "Unexpected number of tokens [\(tokens.count)]")
        
        XCTAssert(tokens[0].tokenType is FuzzyLiteralMatch)
        XCTAssert(tokens[0].text == "#YF 1942-b")
        
        XCTAssert(tokens[1].tokenType is DateMatch)
        XCTAssert(tokens[1].text == "12/01/27")
        
    }
}
{'fraction_non_alphanumeric': 0.08421632035227741, 'fraction_numerical': 0.012384752992981973, 'mean_word_length': 1.7519878833775084, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5387632', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1855, 'n_words': 503}