<commit_before>//
//  File+SwiftLint.swift
//  SwiftLint
//
//  Created by <NAME> on 5/16/15.
//  Copyright © 2015 Realm. All rights reserved.
//

import Foundation
import SourceKittenFramework

internal func regex(_ pattern: String,
                    options: NSRegularExpression.Options? = nil) -> NSRegularExpression {
    // all patterns used for regular expressions in SwiftLint are string literals which have been
    // confirmed to work, so it's ok to force-try here.

    let options = options ?? [.anchorsMatchLines, .dotMatchesLineSeparators]
    // swiftlint:disable:next force_try
    return try! .cached(pattern: pattern, options: options)
}

extension File {
    internal func regions() -> [Region] {
        var regions = [Region]()
        var disabledRules = Set<String>()
        let commands = self.commands()
        let commandPairs = zip(commands, Array(commands.dropFirst().map(Optional.init)) + [nil])
        for (command, nextCommand) in commandPairs {
            switch command.action {
            case .disable: disabledRules.formUnion(command.ruleIdentifiers)
            case .enable: disabledRules.subtract(command.ruleIdentifiers)
            }
            let start = Location(file: path, line: command.line, character: command.character)
            let end = endOfNextCommand(nextCommand)
            regions.append(Region(start: start, end: end, disabledRuleIdentifiers: disabledRules))
        }
        return regions
    }

    fileprivate func commands() -> [Command] {
        if sourcekitdFailed {
            return []
        }
        let contents = self.contents.bridge()
        let pattern = "swiftlint:(enable|disable)(:previous|:this|:next)?\\ [^\\n]+"
        return matchPattern(pattern, withSyntaxKinds: [.comment]).flatMap { range in
            return Command(string: contents, range: range)
        }.flatMap { command in
            return command.expand()
        }
    }

    fileprivate func endOfNextCommand(_ nextCommand: Command?) -> Location {
        guard let nextCommand = nextCommand else {
            return Location(file: path, line: .max, character: .max)
        }
        let nextLine: Int
        let nextCharacter: Int?
        if let nextCommandCharacter = nextCommand.character {
            nextLine = nextCommand.line
            if nextCommandCharacter > 0 {
                nextCharacter = nextCommandCharacter - 1
            } else {
                nextCharacter = nil
            }
        } else {
            nextLine = max(nextCommand.line - 1, 0)
            nextCharacter = .max
        }
        return Location(file: path, line: nextLine, character: nextCharacter)
    }

    internal func matchPattern(_ pattern: String,
                               withSyntaxKinds syntaxKinds: [SyntaxKind],
                               range: NSRange? = nil) -> [NSRange] {
        return matchPattern(pattern, range: range).filter({ $0.1 == syntaxKinds }).map { $0.0 }
    }

    internal func rangesAndTokensMatching(_ pattern: String,
                                          range: NSRange? = nil) -> [(NSRange, [SyntaxToken])] {
        return rangesAndTokensMatching(regex(pattern), range: range)
    }

    internal func rangesAndTokensMatching(_ regex: NSRegularExpression,
                                          range: NSRange? = nil) -> [(NSRange, [SyntaxToken])] {
        let contents = self.contents.bridge()
        let range = range ?? NSRange(location: 0, length: contents.length)
        let syntax = syntaxMap
        return regex.matches(in: self.contents, options: [], range: range).map { match in
            let matchByteRange = contents.NSRangeToByteRange(start: match.range.location,
                length: match.range.length) ?? match.range
            let tokensInRange = syntax.tokensIn(matchByteRange)
            return (match.range, tokensInRange)
        }
    }

    internal func matchPattern(_ pattern: String, range: NSRange? = nil) -> [(NSRange, [SyntaxKind])] {
        return matchPattern(regex(pattern), range: range)
    }

    internal func matchPattern(_ regex: NSRegularExpression, range: NSRange? = nil) -> [(NSRange, [SyntaxKind])] {
        return rangesAndTokensMatching(regex, range: range).map { range, tokens in
            (range, tokens.flatMap { SyntaxKind(rawValue: $0.type) })
        }
    }

    internal func syntaxTokensByLine() -> [[SyntaxToken]]? {
        if sourcekitdFailed {
            return nil
        }
        var results = [[SyntaxToken]](repeating: [], count: lines.count + 1)
        var tokenGenerator = syntaxMap.tokens.makeIterator()
        var lineGenerator = lines.makeIterator()
        var maybeLine = lineGenerator.next()
        var maybeToken = tokenGenerator.next()
        while let line = maybeLine, let token = maybeToken {
            let tokenRange = NSRange(location: token.offset, length: token.length)
            if NSLocationInRange(token.offset, line.byteRange) ||
                NSLocationInRange(line.byteRange.location, tokenRange) {
                    results[line.index].append(token)
            }
            let tokenEnd = NSMaxRange(tokenRange)
            let lineEnd = NSMaxRange(line.byteRange)
            if tokenEnd < lineEnd {
                maybeToken = tokenGenerator.next()
            } else if tokenEnd > lineEnd {
                maybeLine = lineGenerator.next()
            } else {
                maybeLine = lineGenerator.next()
                maybeToken = tokenGenerator.next()
            }
        }
        return results
    }

    internal func syntaxKindsByLine() -> [[SyntaxKind]]? {
        guard !sourcekitdFailed, let tokens = syntaxTokensByLine() else {
            return nil
        }

        return tokens.map { $0.flatMap { SyntaxKind(rawValue: $0.type) } }
    }

    //Added by S2dent
    /**
     This function returns only matches that are not contained in a syntax kind
     specified.

     - parameter pattern: regex pattern to be matched inside file.
     - parameter excludingSyntaxKinds: syntax kinds the matches to be filtered
     when inside them.

     - returns: An array of [NSRange] objects consisting of regex matches inside
     file contents.
     */
    internal func matchPattern(_ pattern: String,
                               excludingSyntaxKinds syntaxKinds: [SyntaxKind],
                               range: NSRange? = nil) -> [NSRange] {
        return matchPattern(pattern, range: range).filter {
            $0.1.filter(syntaxKinds.contains).isEmpty
        }.map { $0.0 }
    }

    internal typealias MatchMapping = (NSTextCheckingResult) -> NSRange

    internal func matchPattern(_ pattern: String,
                               excludingSyntaxKinds: [SyntaxKind],
                               excludingPattern: String,
                               exclusionMapping: MatchMapping = { $0.range }) -> [NSRange] {
        let contents = self.contents.bridge()
        let range = NSRange(location: 0, length: contents.length)
        let matches = matchPattern(pattern, excludingSyntaxKinds: excludingSyntaxKinds)
        if matches.isEmpty {
            return []
        }
        let exclusionRanges = regex(excludingPattern).matches(in: self.contents, options: [],
                                                              range: range).map(exclusionMapping)
        return matches.filter { !$0.intersectsRanges(exclusionRanges) }
    }

    internal func validateVariableName(_ dictionary: [String: SourceKitRepresentable],
                                       kind: SwiftDeclarationKind) -> (name: String, offset: Int)? {
        guard let name = dictionary["key.name"] as? String,
            let offset = (dictionary["key.offset"] as? Int64).flatMap({ Int($0) }),
            SwiftDeclarationKind.variableKinds().contains(kind) && !name.hasPrefix("$") else {
                return nil
        }
        return (name.nameStrippingLeadingUnderscoreIfPrivate(dictionary), offset)
    }

    internal func append(_ string: String) {
        guard let stringData = string.data(using: .utf8) else {
            fatalError("can't encode '\(string)' with UTF8")
        }
        guard let path = path, let fileHandle = FileHandle(forWritingAtPath: path) else {
            fatalError("can't write to path '\(self.path)'")
        }
        _ = fileHandle.seekToEndOfFile()<commit_msg>Add range to avoid whole file lookup<commit_after>//
//  File+SwiftLint.swift
//  SwiftLint
//
//  Created by <NAME> on 5/16/15.
//  Copyright © 2015 Realm. All rights reserved.
//

import Foundation
import SourceKittenFramework

internal func regex(_ pattern: String,
                    options: NSRegularExpression.Options? = nil) -> NSRegularExpression {
    // all patterns used for regular expressions in SwiftLint are string literals which have been
    // confirmed to work, so it's ok to force-try here.

    let options = options ?? [.anchorsMatchLines, .dotMatchesLineSeparators]
    // swiftlint:disable:next force_try
    return try! .cached(pattern: pattern, options: options)
}

extension File {
    internal func regions() -> [Region] {
        var regions = [Region]()
        var disabledRules = Set<String>()
        let commands = self.commands()
        let commandPairs = zip(commands, Array(commands.dropFirst().map(Optional.init)) + [nil])
        for (command, nextCommand) in commandPairs {
            switch command.action {
            case .disable: disabledRules.formUnion(command.ruleIdentifiers)
            case .enable: disabledRules.subtract(command.ruleIdentifiers)
            }
            let start = Location(file: path, line: command.line, character: command.character)
            let end = endOfNextCommand(nextCommand)
            regions.append(Region(start: start, end: end, disabledRuleIdentifiers: disabledRules))
        }
        return regions
    }

    fileprivate func commands() -> [Command] {
        if sourcekitdFailed {
            return []
        }
        let contents = self.contents.bridge()
        let pattern = "swiftlint:(enable|disable)(:previous|:this|:next)?\\ [^\\n]+"
        return matchPattern(pattern, withSyntaxKinds: [.comment]).flatMap { range in
            return Command(string: contents, range: range)
        }.flatMap { command in
            return command.expand()
        }
    }

    fileprivate func endOfNextCommand(_ nextCommand: Command?) -> Location {
        guard let nextCommand = nextCommand else {
            return Location(file: path, line: .max, character: .max)
        }
        let nextLine: Int
        let nextCharacter: Int?
        if let nextCommandCharacter = nextCommand.character {
            nextLine = nextCommand.line
            if nextCommandCharacter > 0 {
                nextCharacter = nextCommandCharacter - 1
            } else {
                nextCharacter = nil
            }
        } else {
            nextLine = max(nextCommand.line - 1, 0)
            nextCharacter = .max
        }
        return Location(file: path, line: nextLine, character: nextCharacter)
    }

    internal func matchPattern(_ pattern: String,
                               withSyntaxKinds syntaxKinds: [SyntaxKind],
                               range: NSRange? = nil) -> [NSRange] {
        return matchPattern(pattern, range: range).filter({ $0.1 == syntaxKinds }).map { $0.0 }
    }

    internal func rangesAndTokensMatching(_ pattern: String,
                                          range: NSRange? = nil) -> [(NSRange, [SyntaxToken])] {
        return rangesAndTokensMatching(regex(pattern), range: range)
    }

    internal func rangesAndTokensMatching(_ regex: NSRegularExpression,
                                          range: NSRange? = nil) -> [(NSRange, [SyntaxToken])] {
        let contents = self.contents.bridge()
        let range = range ?? NSRange(location: 0, length: contents.length)
        let syntax = syntaxMap
        return regex.matches(in: self.contents, options: [], range: range).map { match in
            let matchByteRange = contents.NSRangeToByteRange(start: match.range.location,
                length: match.range.length) ?? match.range
            let tokensInRange = syntax.tokensIn(matchByteRange)
            return (match.range, tokensInRange)
        }
    }

    internal func matchPattern(_ pattern: String, range: NSRange? = nil) -> [(NSRange, [SyntaxKind])] {
        return matchPattern(regex(pattern), range: range)
    }

    internal func matchPattern(_ regex: NSRegularExpression, range: NSRange? = nil) -> [(NSRange, [SyntaxKind])] {
        return rangesAndTokensMatching(regex, range: range).map { range, tokens in
            (range, tokens.flatMap { SyntaxKind(rawValue: $0.type) })
        }
    }

    internal func syntaxTokensByLine() -> [[SyntaxToken]]? {
        if sourcekitdFailed {
            return nil
        }
        var results = [[SyntaxToken]](repeating: [], count: lines.count + 1)
        var tokenGenerator = syntaxMap.tokens.makeIterator()
        var lineGenerator = lines.makeIterator()
        var maybeLine = lineGenerator.next()
        var maybeToken = tokenGenerator.next()
        while let line = maybeLine, let token = maybeToken {
            let tokenRange = NSRange(location: token.offset, length: token.length)
            if NSLocationInRange(token.offset, line.byteRange) ||
                NSLocationInRange(line.byteRange.location, tokenRange) {
                    results[line.index].append(token)
            }
            let tokenEnd = NSMaxRange(tokenRange)
            let lineEnd = NSMaxRange(line.byteRange)
            if tokenEnd < lineEnd {
                maybeToken = tokenGenerator.next()
            } else if tokenEnd > lineEnd {
                maybeLine = lineGenerator.next()
            } else {
                maybeLine = lineGenerator.next()
                maybeToken = tokenGenerator.next()
            }
        }
        return results
    }

    internal func syntaxKindsByLine() -> [[SyntaxKind]]? {
        guard !sourcekitdFailed, let tokens = syntaxTokensByLine() else {
            return nil
        }

        return tokens.map { $0.flatMap { SyntaxKind(rawValue: $0.type) } }
    }

    //Added by S2dent
    /**
     This function returns only matches that are not contained in a syntax kind
     specified.

     - parameter pattern: regex pattern to be matched inside file.
     - parameter excludingSyntaxKinds: syntax kinds the matches to be filtered
     when inside them.

     - returns: An array of [NSRange] objects consisting of regex matches inside
     file contents.
     */
    internal func matchPattern(_ pattern: String,
                               excludingSyntaxKinds syntaxKinds: [SyntaxKind],
                               range: NSRange? = nil) -> [NSRange] {
        return matchPattern(pattern, range: range).filter {
            $0.1.filter(syntaxKinds.contains).isEmpty
        }.map { $0.0 }
    }

    internal typealias MatchMapping = (NSTextCheckingResult) -> NSRange

    internal func matchPattern(_ pattern: String,
                               excludingSyntaxKinds: [SyntaxKind],
                               excludingPattern: String,
                               range: NSRange? = nil,
                               exclusionMapping: MatchMapping = { $0.range }) -> [NSRange] {
        let contents = self.contents.bridge()
        let range = NSRange(location: 0, length: contents.length)
        let matches = matchPattern(pattern, excludingSyntaxKinds: excludingSyntaxKinds, range: range)
        if matches.isEmpty {
            return []
        }
        let exclusionRanges = regex(excludingPattern).matches(in: self.contents, options: [],
                                                              range: range).map(exclusionMapping)
        return matches.filter { !$0.intersectsRanges(exclusionRanges) }
    }

    internal func validateVariableName(_ dictionary: [String: SourceKitRepresentable],
                                       kind: SwiftDeclarationKind) -> (name: String, offset: Int)? {
        guard let name = dictionary["key.name"] as? String,
            let offset = (dictionary["key.offset"] as? Int64).flatMap({ Int($0) }),
            SwiftDeclarationKind.variableKinds().contains(kind) && !name.hasPrefix("$") else {
                return nil
        }
        return (name.nameStrippingLeadingUnderscoreIfPrivate(dictionary), offset)
    }

    internal func append(_ string: String) {
        guard let stringData = string.data(using: .utf8) else {
            fatalError("can't encode '\(string)' with UTF8")
        }
        guard let path = path, let fileHandle = FileHandle(forWritingAtPath: path) else {
            fatalError("can't write to path '\(self.path)'")
        }
        _ = fileHandle.seekToEndOfFile()
{'fraction_non_alphanumeric': 0.09911462356646265, 'fraction_numerical': 0.004159486600511022, 'mean_word_length': 2.1201334816462736, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4737258', 'n_tokens_mistral': 4374, 'n_tokens_neox': 4227, 'n_words': 1162}