<commit_before>//
// SwiftGenKit
// Copyright (c) 2017 SwiftGen
// MIT Licence
//

import Foundation
import PathKit

public enum StringsFileParserError: Error, CustomStringConvertible {
  case FailureOnLoading(path: String)
  case InvalidFormat

  public var description: String {
    switch self {
    case .FailureOnLoading(let path):
      return "Failed to load a file at \"\(path)\""
    case .InvalidFormat:
      return "Invalid strings file"
    }
  }
}

public final class StringsFileParser {
  var entries = [Entry]()

  public init() {}

  public func addEntry(_ entry: Entry) {
    entries.append(entry)
  }

  // Localizable.strings files are generally UTF16, not UTF8!
  public func parseFile(at path: Path) throws {
    guard let data = try? path.read() else {
      throw StringsFileParserError.FailureOnLoading(path: path.description)
    }

    let plist = try PropertyListSerialization
        .propertyList(from: data, format: nil)

    guard let dict = plist as? [String: String] else {
      throw StringsFileParserError.InvalidFormat
    }

    for (key, translation) in dict {
      addEntry(Entry(key: key, translation: translation))
    }
  }

  // MARK: - Public Enum types

  public enum PlaceholderType: String {
    case Object = "String"
    case Float = "Float"
    case Int = "Int"
    case Char = "Character"
    case CString = "UnsafePointer<unichar>"
    case Pointer = "UnsafePointer<Void>"
    case Unknown = "UnsafePointer<()>"

    init?(formatChar char: Character) {
      guard let lcChar = String(char).lowercased().characters.first else {
        return nil
      }
      switch lcChar {
      case "@":
        self = .Object
      case "a", "e", "f", "g":
        self = .Float
      case "d", "i", "o", "u", "x":
        self = .Int
      case "c":
        self = .Char
      case "s":
        self = .CString
      case "p":
        self = .Pointer
      default:
        return nil
      }
    }

    public static func placeholders(fromFormat str: String) -> [PlaceholderType] {
      return StringsFileParser.placeholders(fromFormat:  str)
    }
  }

  public struct Entry {
    let key: String
    var keyStructure: [String] {
        return key.components(separatedBy: CharacterSet(charactersIn: "."))
    }
    let translation: String
    let types: [PlaceholderType]

    public init(key: String, translation: String, types: [PlaceholderType]) {
      self.key = key
      self.translation = translation
      self.types = types
    }

    public init(key: String, translation: String, types: PlaceholderType...) {
      self.init(key: key, translation: translation, types: types)
    }

    public init(key: String, translation: String) {
      let types = PlaceholderType.placeholders(fromFormat: translation)
      self.init(key: key, translation: translation, types: types)
    }
  }

  // MARK: - Private Helpers

  private static let formatTypesRegEx: NSRegularExpression = {
    // %d/%i/%o/%u/%x with their optional length modifiers like in "%lld"
    let pattern_int = "(?:h|hh|l|ll|q|z|t|j)?([dioux])"
    // valid flags for float
    let pattern_float = "[aefg]"
    // like in "%3$" to make positional specifiers
    let position = "([1-9]\\d*\\$)?"
    // precision like in "%1.2f"
    let precision = "[-+]?\\d?(?:\\.\\d)?"

    do {
      return try NSRegularExpression(
        pattern: "(?<!%)%\(position)\(precision)(@|\(pattern_int)|\(pattern_float)|[csp])",
        options: [.caseInsensitive]
      )
    } catch {
      fatalError("Error building the regular expression used to match string formats")
    }
  }()

  // "I give %d apples to %@" --> [.Int, .String]
  private static func placeholders(fromFormat formatString: String) -> [PlaceholderType] {
    let range = NSRange(location: 0, length: (formatString as NSString).length)

    // Extract the list of chars (conversion specifiers) and their optional positional specifier
    let chars = formatTypesRegEx.matches(in: formatString, options: [], range: range)
      .map({ match -> (String, Int?) in
        let range: NSRange
        if match.rangeAt(3).location != NSNotFound {
          // [dioux] are in range #3 because in #2 there may be length modifiers (like in "lld")
          range = match.rangeAt(3)<commit_msg>Corrected format string regex to support placeholders that are directly prefixed with an escaped %<commit_after>//
// SwiftGenKit
// Copyright (c) 2017 SwiftGen
// MIT Licence
//

import Foundation
import PathKit

public enum StringsFileParserError: Error, CustomStringConvertible {
  case FailureOnLoading(path: String)
  case InvalidFormat

  public var description: String {
    switch self {
    case .FailureOnLoading(let path):
      return "Failed to load a file at \"\(path)\""
    case .InvalidFormat:
      return "Invalid strings file"
    }
  }
}

public final class StringsFileParser {
  var entries = [Entry]()

  public init() {}

  public func addEntry(_ entry: Entry) {
    entries.append(entry)
  }

  // Localizable.strings files are generally UTF16, not UTF8!
  public func parseFile(at path: Path) throws {
    guard let data = try? path.read() else {
      throw StringsFileParserError.FailureOnLoading(path: path.description)
    }

    let plist = try PropertyListSerialization
        .propertyList(from: data, format: nil)

    guard let dict = plist as? [String: String] else {
      throw StringsFileParserError.InvalidFormat
    }

    for (key, translation) in dict {
      addEntry(Entry(key: key, translation: translation))
    }
  }

  // MARK: - Public Enum types

  public enum PlaceholderType: String {
    case Object = "String"
    case Float = "Float"
    case Int = "Int"
    case Char = "Character"
    case CString = "UnsafePointer<unichar>"
    case Pointer = "UnsafePointer<Void>"
    case Unknown = "UnsafePointer<()>"

    init?(formatChar char: Character) {
      guard let lcChar = String(char).lowercased().characters.first else {
        return nil
      }
      switch lcChar {
      case "@":
        self = .Object
      case "a", "e", "f", "g":
        self = .Float
      case "d", "i", "o", "u", "x":
        self = .Int
      case "c":
        self = .Char
      case "s":
        self = .CString
      case "p":
        self = .Pointer
      default:
        return nil
      }
    }

    public static func placeholders(fromFormat str: String) -> [PlaceholderType] {
      return StringsFileParser.placeholders(fromFormat:  str)
    }
  }

  public struct Entry {
    let key: String
    var keyStructure: [String] {
        return key.components(separatedBy: CharacterSet(charactersIn: "."))
    }
    let translation: String
    let types: [PlaceholderType]

    public init(key: String, translation: String, types: [PlaceholderType]) {
      self.key = key
      self.translation = translation
      self.types = types
    }

    public init(key: String, translation: String, types: PlaceholderType...) {
      self.init(key: key, translation: translation, types: types)
    }

    public init(key: String, translation: String) {
      let types = PlaceholderType.placeholders(fromFormat: translation)
      self.init(key: key, translation: translation, types: types)
    }
  }

  // MARK: - Private Helpers

  private static let formatTypesRegEx: NSRegularExpression = {
    // %d/%i/%o/%u/%x with their optional length modifiers like in "%lld"
    let pattern_int = "(?:h|hh|l|ll|q|z|t|j)?([dioux])"
    // valid flags for float
    let pattern_float = "[aefg]"
    // like in "%3$" to make positional specifiers
    let position = "([1-9]\\d*\\$)?"
    // precision like in "%1.2f"
    let precision = "[-+]?\\d?(?:\\.\\d)?"

    do {
      return try NSRegularExpression(
        pattern: "(?:^|[^%]|(?<!%)(?:%%)+)%\(position)\(precision)(@|\(pattern_int)|\(pattern_float)|[csp])",
        options: [.caseInsensitive]
      )
    } catch {
      fatalError("Error building the regular expression used to match string formats")
    }
  }()

  // "I give %d apples to %@" --> [.Int, .String]
  private static func placeholders(fromFormat formatString: String) -> [PlaceholderType] {
    let range = NSRange(location: 0, length: (formatString as NSString).length)

    // Extract the list of chars (conversion specifiers) and their optional positional specifier
    let chars = formatTypesRegEx.matches(in: formatString, options: [], range: range)
      .map({ match -> (String, Int?) in
        let range: NSRange
        if match.rangeAt(3).location != NSNotFound {
          // [dioux] are in range #3 because in #2 there may be length modifiers (like in "lld")
          range = match.rangeAt(3)
{'fraction_non_alphanumeric': 0.12539148590650737, 'fraction_numerical': 0.00394385802111124, 'mean_word_length': 2.7601395551679024, 'pattern_counts': {'":': 12, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3210124', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2535, 'n_words': 783}