<commit_before>import Foundation

public enum HtmlTag: String {
  case p, a, b, i, u, br, pre, code
}

public typealias HtmlAttributes = [String]

public enum HtmlToken {
  case open(HtmlTag, HtmlAttributes)
  case close(HtmlTag)
  case entity(UnicodeScalar)
  case text(UnicodeScalar)
}

public enum HtmlTokenError: Error {
  case unknownTag
  case invalidTag
  case invalidEntity
}

public struct HtmlTokenizer: Tokenizer {
  typealias Token = HtmlToken
  typealias Error = HtmlTokenError

  private var iterator: String.UnicodeScalarView.Iterator
  private var pushedBackScalar: UnicodeScalar?

  public init(text: String) {
    iterator = text.unicodeScalars.makeIterator()
  }

  public mutating func nextToken() -> Result<Token, Error>? {
    while let char = nextCharacter() {
      switch char {
      case HtmlTagId.start:
        return parseTag()
      case HtmlEntity.start:
        return parseEntity()
      default:
        return .success(.text(char))
      }
    }
    return .none
  }

  private mutating func nextCharacter() -> UnicodeScalar? {
    return iterator.next()
  }

  private mutating func parseTag() -> Result<Token, Error> {
    var tag: String? = .none
    var text = String()
    var attributes = HtmlAttributes()
    var isCloseTag = false

    let pushTag = {
      if tag == .none {
        tag = text
      } else {
        attributes.append(text)
      }
    }

    while let char = nextCharacter() {
      switch char {
      case HtmlTagId.separator:
        if !text.isEmpty {
          pushTag()
          text = ""
        }
      case HtmlTagId.close where tag == .none:
        isCloseTag = true
      case HtmlTagId.end:
        pushTag()
        return tag.flatMap(HtmlTag.init(rawValue:))
          .map { isCloseTag ? .close($0) : .open($0, attributes) }
          .map(Result.success)
          .or(.failure(.unknownTag))
      default:
        text.unicodeScalars.append(char)
      }
    }
    return .failure(.invalidTag)
  }

  private mutating func parseEntity() -> Result<Token, Error> {
    var char = nextCharacter()
    guard char == HtmlEntity.number else {
      var entityName = String()
      while let ch = char, ch != HtmlEntity.end {
        entityName.unicodeScalars.append(ch)
        char = nextCharacter()
      }
      return HtmlEntity.decode(entityName)
        .map(HtmlToken.entity)
        .map(Result.success)
        .or(.failure(.invalidEntity))
    }

    let radix: Int
    char = nextCharacter()
    if char == HtmlEntity.hexLower || char == HtmlEntity.hexUpper {
      radix = 16
      char = nextCharacter()
    } else {
      radix = 10
    }

    var number = String()
    while let digit = char, digit != HtmlEntity.end {
      number.unicodeScalars.append(digit)
      char = nextCharacter()
    }

    return UInt32(number, radix: radix)
      .flatMap(UnicodeScalar.init)
      .flatMap(HtmlToken.entity)
      .map(Result.success)
      .or(.failure(.invalidEntity))
  }
}

private enum HtmlTagId {
  static let start: UnicodeScalar = "<"
  static let end: UnicodeScalar = ">"
  static let close: UnicodeScalar = "/"
  static let separator: UnicodeScalar = " "
}

private enum HtmlEntity {
  static let start: UnicodeScalar = "&"
  static let end: UnicodeScalar = ";"
  static let number: UnicodeScalar = "#"<commit_msg>Define constants on private extension<commit_after>import Foundation

public enum HtmlTag: String {
  case p, a, b, i, u, br, pre, code
}

public typealias HtmlAttributes = [String]

public enum HtmlToken {
  case open(HtmlTag, HtmlAttributes)
  case close(HtmlTag)
  case entity(UnicodeScalar)
  case text(UnicodeScalar)
}

public enum HtmlTokenError: Error {
  case unknownTag
  case invalidTag
  case invalidEntity
}

public struct HtmlTokenizer: Tokenizer {
  typealias Token = HtmlToken
  typealias Error = HtmlTokenError

  private var iterator: String.UnicodeScalarView.Iterator
  private var pushedBackScalar: UnicodeScalar?

  public init(text: String) {
    iterator = text.unicodeScalars.makeIterator()
  }

  public mutating func nextToken() -> Result<Token, Error>? {
    while let char = nextCharacter() {
      switch char {
      case HtmlTag.start:
        return parseTag()
      case HtmlEntity.start:
        return parseEntity()
      default:
        return .success(.text(char))
      }
    }
    return .none
  }

  private mutating func nextCharacter() -> UnicodeScalar? {
    return iterator.next()
  }

  private mutating func parseTag() -> Result<Token, Error> {
    var tag: String? = .none
    var text = String()
    var attributes = HtmlAttributes()
    var isCloseTag = false

    let pushTag = {
      if tag == .none {
        tag = text
      } else {
        attributes.append(text)
      }
    }

    while let char = nextCharacter() {
      switch char {
      case HtmlTag.separator:
        if !text.isEmpty {
          pushTag()
          text = ""
        }
      case HtmlTag.close where tag == .none:
        isCloseTag = true
      case HtmlTag.end:
        pushTag()
        return tag.flatMap(HtmlTag.init(rawValue:))
          .map { isCloseTag ? .close($0) : .open($0, attributes) }
          .map(Result.success)
          .or(.failure(.unknownTag))
      default:
        text.unicodeScalars.append(char)
      }
    }
    return .failure(.invalidTag)
  }

  private mutating func parseEntity() -> Result<Token, Error> {
    var char = nextCharacter()
    guard char == HtmlEntity.number else {
      var entityName = String()
      while let ch = char, ch != HtmlEntity.end {
        entityName.unicodeScalars.append(ch)
        char = nextCharacter()
      }
      return HtmlEntity.decode(entityName)
        .map(HtmlToken.entity)
        .map(Result.success)
        .or(.failure(.invalidEntity))
    }

    let radix: Int
    char = nextCharacter()
    if char == HtmlEntity.hexLower || char == HtmlEntity.hexUpper {
      radix = 16
      char = nextCharacter()
    } else {
      radix = 10
    }

    var number = String()
    while let digit = char, digit != HtmlEntity.end {
      number.unicodeScalars.append(digit)
      char = nextCharacter()
    }

    return UInt32(number, radix: radix)
      .flatMap(UnicodeScalar.init)
      .flatMap(HtmlToken.entity)
      .map(Result.success)
      .or(.failure(.invalidEntity))
  }
}

private extension HtmlTag {
  static let start: UnicodeScalar = "<"
  static let end: UnicodeScalar = ">"
  static let close: UnicodeScalar = "/"
  static let separator: UnicodeScalar = " "
}

private enum HtmlEntity {
  static let start: UnicodeScalar = "&"
  static let end: UnicodeScalar = ";"
  static let number: UnicodeScalar = "#"
{'fraction_non_alphanumeric': 0.10416980279993979, 'fraction_numerical': 0.0024085503537558334, 'mean_word_length': 2.502372166578809, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7112373', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1971, 'n_words': 524}