<commit_before>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2021 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
*/

import Foundation
import TSCBasic

extension Bitcode {
  /// Traverse a bitstream using the specified `visitor`, which will receive
  /// callbacks when blocks and records are encountered.
  public static func read<Visitor: BitstreamVisitor>(bytes: ByteString, using visitor: inout Visitor) throws {
    precondition(bytes.count > 4)
    var reader = BitstreamReader(buffer: bytes)
    try visitor.validate(signature: reader.readSignature())
    try reader.readBlock(id: BitstreamReader.fakeTopLevelBlockID,
                         abbrevWidth: 2,
                         abbrevInfo: [],
                         visitor: &visitor)
  }
}

/// A basic visitor that collects all the blocks and records in a stream.
private struct CollectingVisitor: BitstreamVisitor {
  var stack: [(UInt64, [BitcodeElement])] = [(BitstreamReader.fakeTopLevelBlockID, [])]

  func validate(signature: Bitcode.Signature) throws {}

  mutating func shouldEnterBlock(id: UInt64) throws -> Bool {
    stack.append((id, []))
    return true
  }

  mutating func didExitBlock() throws {
    guard let (id, elements) = stack.popLast() else {
      fatalError("Unbalanced calls to shouldEnterBlock/didExitBlock")
    }

    let block = BitcodeElement.Block(id: id, elements: elements)
    stack[stack.endIndex-1].1.append(.block(block))
  }

  mutating func visit(record: BitcodeElement.Record) throws {
    stack[stack.endIndex-1].1.append(.record(record))
  }

  func finalizeTopLevelElements() -> [BitcodeElement] {
    assert(stack.count == 1)
    return stack[0].1
  }
}

private extension Bits.Cursor {
  enum BitcodeError: Swift.Error {
    case vbrOverflow
  }

  mutating func readVBR(_ width: Int) throws -> UInt64 {
    precondition(width > 1)
    let testBit = UInt64(1 << (width &- 1))
    let mask = testBit &- 1

    var result: UInt64 = 0
    var offset: UInt64 = 0
    var next: UInt64
    repeat {
      next = try self.read(width)
      result |= (next & mask) << offset
      offset += UInt64(width &- 1)
      if offset > 64 { throw BitcodeError.vbrOverflow }
    } while next & testBit != 0

    return result
  }
}
<commit_msg>Strike the Unused CollectingVisitor<commit_after>/*
 This source file is part of the Swift.org open source project

 Copyright (c) 2021 Apple Inc. and the Swift project authors
 Licensed under Apache License v2.0 with Runtime Library Exception

 See http://swift.org/LICENSE.txt for license information
 See http://swift.org/CONTRIBUTORS.txt for Swift project authors
*/

import Foundation
import TSCBasic

extension Bitcode {
  /// Traverse a bitstream using the specified `visitor`, which will receive
  /// callbacks when blocks and records are encountered.
  public static func read<Visitor: BitstreamVisitor>(bytes: ByteString, using visitor: inout Visitor) throws {
    precondition(bytes.count > 4)
    var reader = BitstreamReader(buffer: bytes)
    try visitor.validate(signature: reader.readSignature())
    try reader.readBlock(id: BitstreamReader.fakeTopLevelBlockID,
                         abbrevWidth: 2,
                         abbrevInfo: [],
                         visitor: &visitor)
  }
}

private extension Bits.Cursor {
  enum BitcodeError: Swift.Error {
    case vbrOverflow
  }

  mutating func readVBR(_ width: Int) throws -> UInt64 {
    precondition(width > 1)
    let testBit = UInt64(1 << (width &- 1))
    let mask = testBit &- 1

    var result: UInt64 = 0
    var offset: UInt64 = 0
    var next: UInt64
    repeat {
      next = try self.read(width)
      result |= (next & mask) << offset
      offset += UInt64(width &- 1)
      if offset > 64 { throw BitcodeError.vbrOverflow }
    } while next & testBit != 0

    return result
  }
}

{'fraction_non_alphanumeric': 0.09512013871686896, 'fraction_numerical': 0.01758731731483775, 'mean_word_length': 3.009930486593843, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1069712', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1138, 'n_words': 376}