<commit_before>//
//  TextualDataImport.swift
//  SimpleSimilarityFramework
//
//  Created by <NAME> on 20.08.17.
//  Copyright © 2017 <NAME>. All rights reserved.
//

import Foundation


/// Error object thrown when reading a file fails
public struct FileReadError: Error {}

/// Data structure for a single textual input
public struct TextualData: Hashable {
    /// The input string
    public let inputString: String

    /// The origin where this string is found. I.e. a book or a webpage or software
    public let origin: String?
    
    /// Reference to an object containig textual data
    public let originObject: AnyObject?
    
    public init(inputString: String, origin: String?, originObject: AnyObject?) {
        self.inputString = inputString
        self.origin = origin
        self.originObject = originObject
    }
    
    public var hashValue: Int {
        if let origin = origin {
            return origin.appending(inputString).hashValue
        } else {
            return inputString.hashValue
        }
    }
    
    public static func ==(lhs: TextualData, rhs: TextualData) -> Bool {
        var lhsText = lhs.inputString
        var rhsText = rhs.inputString
        
        if let lhsOrigin = lhs.origin {
            lhsText = lhsText.appending(lhsOrigin)
        }
        
        if let rhsOrigin = rhs.origin {
            rhsText = rhsOrigin.appending(rhsOrigin)
        }
        
        return lhsText == rhsText
    }
}


/// Base protocol used by objects that load data for the textual similarity search
public protocol TextualDataImport {

    /// Load's the content of the given file
    ///
    /// - Parameter fileName: this file is being read
    /// - Throws: a FileReadError error when loading the file fails<commit_msg>Performance<commit_after>//
//  TextualDataImport.swift
//  SimpleSimilarityFramework
//
//  Created by <NAME> on 20.08.17.
//  Copyright © 2017 <NAME>. All rights reserved.
//

import Foundation


/// Error object thrown when reading a file fails
public struct FileReadError: Error {}

/// Data structure for a single textual input
public struct TextualData: Hashable {
    /// The input string
    public let inputString: String

    /// The origin where this string is found. I.e. a book or a webpage or software
    public let origin: String?
    
    /// Reference to an object containig textual data
    public let originObject: AnyObject?
    
    public init(inputString: String, origin: String?, originObject: AnyObject?) {
        self.inputString = inputString
        self.origin = origin
        self.originObject = originObject

        if let origin = origin {
            self.computedHash = origin.appending(inputString).hashValue
        } else {
            self.computedHash = inputString.hashValue
        }
    }

    private let computedHash: Int

    // hashInto is too slow
    public var hashValue: Int {
        return computedHash
    }
    
    public static func ==(lhs: TextualData, rhs: TextualData) -> Bool {
        var lhsText = lhs.inputString
        var rhsText = rhs.inputString
        
        if let lhsOrigin = lhs.origin {
            lhsText = lhsText.appending(lhsOrigin)
        }
        
        if let rhsOrigin = rhs.origin {
            rhsText = rhsOrigin.appending(rhsOrigin)
        }
        
        return lhsText == rhsText
    }
}


/// Base protocol used by objects that load data for the textual similarity search
public protocol TextualDataImport {

    /// Load's the content of the given file
    ///
    /// - Parameter fileName: this file is being read
    /// - Throws: a FileReadError error when loading the file fails
{'fraction_non_alphanumeric': 0.0789186237028946, 'fraction_numerical': 0.005461496450027308, 'mean_word_length': 2.1741767764298094, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2361097', 'n_tokens_mistral': 1006, 'n_tokens_neox': 965, 'n_words': 351}