<commit_before>#if SWIFT_PACKAGE
  import cclang
#endif

import Foundation

/// Represents a C, C++, or Objective-C token.
public protocol Token {
  var clang: CXToken { get }
}

extension Token {
  
  /// Determine the spelling of the given token.
  /// The spelling of a token is the textual representation of that token,
  /// e.g., the text of an identifier or keyword.
  public func spelling(in translationUnit: TranslationUnit) -> String {
    return clang_getTokenSpelling(translationUnit.clang, clang).asSwift()
  }
  
  /// Retrieve the source location of the given token.
  /// - param translationUnit: The translation unit in which you're looking
  ///                          for this token.
  public func location(in translationUnit: TranslationUnit) -> SourceLocation {
    return SourceLocation(clang: clang_getTokenLocation(translationUnit.clang,
                                                        clang))
  }
  
  /// Retrieve a source range that covers the given token.
  /// - param translationUnit: The translation unit in which you're looking
  ///                          for this token.
  public func range(in translationUnit: TranslationUnit) -> SourceRange {
    return SourceRange(clang: clang_getTokenExtent(translationUnit.clang,
                                                   clang))
  }
  
  /// Returns the underlying CXToken value.
  public func asClang() -> CXToken {
    return self.clang
  }
}

/// A token that contains some kind of punctuation.
public struct PunctuationToken: Token {
  public let clang: CXToken
}

/// A language keyword.
public struct KeywordToken: Token {
  public let clang: CXToken
}

/// An identifier (that is not a keyword).
public struct IdentifierToken: Token {
  public let clang: CXToken
}

/// A numeric, string, or character literal.
public struct LiteralToken: Token {
  public let clang: CXToken
}

/// A comment.
public struct CommentToken: Token {
  public let clang: CXToken
}

/// Converts a CXToken to a Token, returning `nil` if it was unsuccessful
func convertToken(_ clang: CXToken) -> Token {
  switch clang_getTokenKind(clang) {
  case CXToken_Punctuation: return PunctuationToken(clang: clang)
  case CXToken_Keyword: return KeywordToken(clang: clang)
  case CXToken_Identifier: return IdentifierToken(clang: clang)
  case CXToken_Literal: return LiteralToken(clang: clang)
  case CXToken_Comment: return CommentToken(clang: clang)
  default: fatalError("invalid CXTokenKind \(clang)")
  }
}

public struct SourceLocation {
  let clang: CXSourceLocation

  /// Creates a SourceLocation.
  /// - parameter clang: A CXSourceLocation.
  init(clang: CXSourceLocation) {
    self.clang = clang
  }

  /// Creates a source location associated with a given file/line/column
  /// in a particular translation unit
  /// - parameters:
  ///   - translationUnit: The translation unit associated with the location
  ///       to extract.
  ///   - file: Source file.
  ///   - line: The line number in the source file.
  ///   - column: The column number in the source file.
  init(translationUnit: TranslationUnit, file: File, line: Int, column: Int) {
    self.clang = clang_getLocation(
      translationUnit.clang, file.clang, UInt32(line), UInt32(column))
  }

  /// Creates a source location associated with a given character offset
  /// in a particular translation unit
  /// - parameters:
  ///   - translationUnit: The translation unit associated with the location
  ///       to extract.
  ///   - file: Source file.
  ///   - offset: character offset in the source file.
  init(translationUnit: TranslationUnit, file: File, offset: Int) {
    self.clang = clang_getLocationForOffset(
      translationUnit.clang, file.clang, UInt32(offset))
  }

  /// Retrieves all file, line, column, and offset attributes of the provided
  /// source location.
  internal var locations: (file: File, line: Int, column: Int, offset: Int) {
    var l = 0 as UInt32
    var c = 0 as UInt32
    var o = 0 as UInt32
    var f: CXFile?
    clang_getFileLocation(clang, &f, &l, &c, &o)
    return (file: File(clang: f!), line: Int(l), column: Int(c),
            offset: Int(o))
  }
  
  public func cursor(in translationUnit: TranslationUnit) -> Cursor? {
    return clang_getCursor(translationUnit.clang, clang)
  }
  
  /// The line to which the given source location points.
  public var line: Int {<commit_msg>make init(translationUnit:file:line:column) public make init(translationUnit:file:offset) public<commit_after>#if SWIFT_PACKAGE
  import cclang
#endif

import Foundation

/// Represents a C, C++, or Objective-C token.
public protocol Token {
  var clang: CXToken { get }
}

extension Token {
  
  /// Determine the spelling of the given token.
  /// The spelling of a token is the textual representation of that token,
  /// e.g., the text of an identifier or keyword.
  public func spelling(in translationUnit: TranslationUnit) -> String {
    return clang_getTokenSpelling(translationUnit.clang, clang).asSwift()
  }
  
  /// Retrieve the source location of the given token.
  /// - param translationUnit: The translation unit in which you're looking
  ///                          for this token.
  public func location(in translationUnit: TranslationUnit) -> SourceLocation {
    return SourceLocation(clang: clang_getTokenLocation(translationUnit.clang,
                                                        clang))
  }
  
  /// Retrieve a source range that covers the given token.
  /// - param translationUnit: The translation unit in which you're looking
  ///                          for this token.
  public func range(in translationUnit: TranslationUnit) -> SourceRange {
    return SourceRange(clang: clang_getTokenExtent(translationUnit.clang,
                                                   clang))
  }
  
  /// Returns the underlying CXToken value.
  public func asClang() -> CXToken {
    return self.clang
  }
}

/// A token that contains some kind of punctuation.
public struct PunctuationToken: Token {
  public let clang: CXToken
}

/// A language keyword.
public struct KeywordToken: Token {
  public let clang: CXToken
}

/// An identifier (that is not a keyword).
public struct IdentifierToken: Token {
  public let clang: CXToken
}

/// A numeric, string, or character literal.
public struct LiteralToken: Token {
  public let clang: CXToken
}

/// A comment.
public struct CommentToken: Token {
  public let clang: CXToken
}

/// Converts a CXToken to a Token, returning `nil` if it was unsuccessful
func convertToken(_ clang: CXToken) -> Token {
  switch clang_getTokenKind(clang) {
  case CXToken_Punctuation: return PunctuationToken(clang: clang)
  case CXToken_Keyword: return KeywordToken(clang: clang)
  case CXToken_Identifier: return IdentifierToken(clang: clang)
  case CXToken_Literal: return LiteralToken(clang: clang)
  case CXToken_Comment: return CommentToken(clang: clang)
  default: fatalError("invalid CXTokenKind \(clang)")
  }
}

public struct SourceLocation {
  let clang: CXSourceLocation

  /// Creates a SourceLocation.
  /// - parameter clang: A CXSourceLocation.
  init(clang: CXSourceLocation) {
    self.clang = clang
  }

  /// Creates a source location associated with a given file/line/column
  /// in a particular translation unit
  /// - parameters:
  ///   - translationUnit: The translation unit associated with the location
  ///       to extract.
  ///   - file: Source file.
  ///   - line: The line number in the source file.
  ///   - column: The column number in the source file.
  public init(translationUnit: TranslationUnit,
              file: File, line: Int, column: Int) {
    self.clang = clang_getLocation(
      translationUnit.clang, file.clang, UInt32(line), UInt32(column))
  }

  /// Creates a source location associated with a given character offset
  /// in a particular translation unit
  /// - parameters:
  ///   - translationUnit: The translation unit associated with the location
  ///       to extract.
  ///   - file: Source file.
  ///   - offset: character offset in the source file.
  public init(translationUnit: TranslationUnit, file: File, offset: Int) {
    self.clang = clang_getLocationForOffset(
      translationUnit.clang, file.clang, UInt32(offset))
  }

  /// Retrieves all file, line, column, and offset attributes of the provided
  /// source location.
  internal var locations: (file: File, line: Int, column: Int, offset: Int) {
    var l = 0 as UInt32
    var c = 0 as UInt32
    var o = 0 as UInt32
    var f: CXFile?
    clang_getFileLocation(clang, &f, &l, &c, &o)
    return (file: File(clang: f!), line: Int(l), column: Int(c),
            offset: Int(o))
  }
  
  public func cursor(in translationUnit: TranslationUnit) -> Cursor? {
    return clang_getCursor(translationUnit.clang, clang)
  }
  
  /// The line to which the given source location points.
  public var line: Int {
{'fraction_non_alphanumeric': 0.09420371412492966, 'fraction_numerical': 0.0033764772087788407, 'mean_word_length': 2.850086655112652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2610951', 'n_tokens_mistral': 2604, 'n_tokens_neox': 2501, 'n_words': 899}