<commit_before>import Unbox

open class NameClassifier {
  public struct Item: Unboxable {
    public let id: String
    public let name: String

    public init(unboxer: Unboxer) throws {
      self.id = try unboxer.unbox(key: "id")
      self.name = try unboxer.unbox(key: "name")
    }
  }

  public struct ItemsGroup: Unboxable {
    public let key: String
    public let value: [Item]

    public init(unboxer: Unboxer) throws {
      self.key = try unboxer.unbox(key: "key")
      self.value = try unboxer.unbox(key: "value")
    }
  }

  public func classify(items: [Item]) throws -> [(key: String, value: [Any])] {
    var groups = [String: [Item]]()

    for item in items {
      let name = item.name
      let id = item.id

      let index = name.characters.count < 3 ? name.index(name.startIndex, offsetBy: name.characters.count) : name.index(name.startIndex, offsetBy: 3)
      let groupName = name[name.startIndex..<index].uppercased()

      if !groups.keys.contains(groupName) {
        let group: [Item] = []

        groups[groupName] = group
      }

      groups[groupName]!.append(try unbox(dictionary: ["id": id, "name": name]))
    }

    let sortedGroups = groups.sorted { $0.key < $1.key }

    return mergeSmallGroups(sortedGroups)
  }

  public func classify2(items: [Item]) throws -> [ItemsGroup] {
    let result = try classify(items: items)

    var items: [ItemsGroup] = []

    for item in result {
      items.append(try unbox(dictionary: ["key": item.key, "value": item.value as! [Item]]))
    }

    return items
  }

  func mergeSmallGroups(_ groups: [(key: String, value: [Item])]) -> [(key: String, value: [Any])] {
    // merge groups into bigger groups with size ~ 20 records

    var classifier: [[String]] = []

    var groupSize = 0

    classifier.append([])

    var index = 0

    for (groupName, group) in groups {
      let groupWeight = group.count
      groupSize += groupWeight

      if groupSize > 20 || startsWithDifferentLetter(classifier[index], name: groupName) {
        groupSize = 0
        classifier.append([])
        index = index + 1
      }

      classifier[index].append(groupName)
    }

    // flatten records from different group within same classification
    // assign new name in format firstName-lastName, e.g. ABC-AZZ

    var newGroups: [(key: String, value: [Any])] = []

    for groupNames in classifier {
      let key = groupNames[0] + "-" + groupNames[groupNames.count - 1]

      var value: [Any] = []

      for groupName in groupNames {
        let group = groups.filter { $0.key == groupName }.first

        for item in group!.value {
          value.append(item)
        }
      }

      newGroups.append((key: key, value: value))
    }
<commit_msg>add site<commit_after>import Unbox

open class NameClassifier {
  public struct Item: Unboxable {
    public let id: String
    public let name: String

    public init(unboxer: Unboxer) throws {
      self.id = try unboxer.unbox(key: "id")
      self.name = try unboxer.unbox(key: "name")
    }
  }

  public struct ItemsGroup: Unboxable {
    public let key: String
    public let value: [Item]

    public init(unboxer: Unboxer) throws {
      self.key = try unboxer.unbox(key: "key")
      self.value = try unboxer.unbox(key: "value")
    }
  }

  public func classify(items: [Item]) throws -> [(key: String, value: [Any])] {
    var groups = [String: [Item]]()

    for item in items {
      let name = item.name
      let id = item.id

      let index = name.characters.count < 3 ? name.index(name.startIndex, offsetBy: name.characters.count) : name.index(name.startIndex, offsetBy: 3)
      let groupName = name[name.startIndex..<index].uppercased()

      if !groups.keys.contains(groupName) {
        let group: [Item] = []

        groups[groupName] = group
      }

      groups[groupName]!.append(try unbox(dictionary: ["id": id, "name": name]))
    }

    let sortedGroups = groups.sorted { $0.key < $1.key }

    return mergeSmallGroups(sortedGroups)
  }

  public func classify2(items: [Item]) throws -> [ItemsGroup] {
    let result = try classify(items: items)

    var items: [ItemsGroup] = []

    for item in result {
      items.append(try unbox(dictionary: ["key": item.key, "value": item.value as! [Item]]))
    }

    return items
  }

  func mergeSmallGroups(_ groups: [(key: String, value: [Item])]) -> [(key: String, value: [Any])] {
    // merge groups into bigger groups with size ~ 20 records

    var classifier: [[String]] = []

    var groupSize = 0

    classifier.append([])

    var index = 0

    for (groupName, group) in groups {
      let groupWeight = group.count
      groupSize += groupWeight

      if groupSize > 20 || startsWithDifferentLetter(classifier[index], name: groupName) {
        groupSize = 0
        classifier.append([])
        index = index + 1
      }

      classifier[index].append(groupName)
    }

    // flatten records from different group within same classification
    // assign new name in format firstName-lastName, e.g. ABC-AZZ

    var newGroups: [(key: String, value: [Any])] = []

    for groupNames in classifier {
      if groupNames.count > 0 {
        let key = groupNames[0] + "-" + groupNames[groupNames.count - 1]

        var value: [Any] = []

        for groupName in groupNames {
          let group = groups.filter { $0.key == groupName }.first

          for item in group!.value {
            value.append(item)
          }
        }

        newGroups.append((key: key, value: value))
      }
    }

{'fraction_non_alphanumeric': 0.12773656594897775, 'fraction_numerical': 0.0059706893432241726, 'mean_word_length': 2.5595621377978106, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363012', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1694, 'n_words': 490}