<commit_before>import XCTest
@testable import Util

let allScalars = Unicode.Scalar.allScalars

class AllScalarsTests: XCTestCase {
  func testCollectionConformance() {
    let calculatedCount = (0...0xD7FF).count + (0xE000...0x10FFFF).count
    XCTAssertEqual(calculatedCount, allScalars.count)
    XCTAssertEqual(calculatedCount, allScalars.reduce(0, { sum, _ in sum + 1 }))
  }
  
  func testIndexOf() throws {
    for scalar in "béड🥰 \u{0} \u{D7FF} \u{E000} \u{10FFFF}".unicodeScalars {
      let i = try XCTUnwrap(allScalars.firstIndex(of: scalar))
      XCTAssertEqual(scalar, allScalars[i])
    }
  }
  
  func testProperties() throws {
    let whitespaces = allScalars.filter { $0.properties.isWhitespace }
    XCTAssertEqual(25, whitespaces.count)
    
    let numericIndices = allScalars
      .indices
      .filter { allScalars[$0].properties.numericType == .decimal }
    XCTAssertEqual(650, numericIndices.count)
    
    let digitSum = try numericIndices
      .map { try XCTUnwrap(allScalars[$0].properties.numericValue) }
      .reduce(0, +)
    XCTAssertEqual(2925, digitSum)
    XCTAssertEqual(4.5, digitSum / Double(numericIndices.count))
  }
}
<commit_msg>Add version checks to Unicode.AllScalars tests<commit_after>import XCTest
@testable import Util

let allScalars = Unicode.Scalar.allScalars

extension Unicode.Scalar {
  var isUnicode12OrEarlier: Bool {
    properties.age.map { age in age <= (12, 0) } ?? false
  }
}

class AllScalarsTests: XCTestCase {
  func testCollectionConformance() {
    let calculatedCount = (0...0xD7FF).count + (0xE000...0x10FFFF).count
    XCTAssertEqual(calculatedCount, allScalars.count)
    XCTAssertEqual(calculatedCount, allScalars.reduce(0, { sum, _ in sum + 1 }))
  }
  
  func testIndexOf() throws {
    for scalar in "béड🥰 \u{0} \u{D7FF} \u{E000} \u{10FFFF}".unicodeScalars {
      let i = try XCTUnwrap(allScalars.firstIndex(of: scalar))
      XCTAssertEqual(scalar, allScalars[i])
    }
  }
  
  func testProperties() throws {
    let whitespaces = allScalars.filter { $0.properties.isWhitespace }
    XCTAssertEqual(25, whitespaces.count)
    
    let numericIndices = allScalars
      .indices
      .filter {
        allScalars[$0].isUnicode12OrEarlier
          && allScalars[$0].properties.numericType == .decimal
      }
    XCTAssertEqual(630, numericIndices.count)
    
    let digitSum = try numericIndices
      .map { try XCTUnwrap(allScalars[$0].properties.numericValue) }
      .reduce(0, +)
    XCTAssertEqual(2835, digitSum)
    XCTAssertEqual(4.5, digitSum / Double(numericIndices.count))
  }
}

{'fraction_non_alphanumeric': 0.11486224291812185, 'fraction_numerical': 0.029491656965463717, 'mean_word_length': 3.506993006993007, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3613341', 'n_tokens_mistral': 920, 'n_tokens_neox': 888, 'n_words': 166}