<commit_before>require 'strscan'

module Sass
  module Script
    class Lexer # :nodoc:
      OPERATORS = {
        '+' => :plus,
        '-' => :minus,
        '*' => :times,
        '/' => :div,
        '%' => :mod,
        '(' => :lparen,
        ')' => :rparen,
        ',' => :comma,
        'and' => :and,
        'or' => :or,
        'not' => :not,
        '==' => :eq,
        '!=' => :neq,
        '>=' => :gte,
        '<=' => :lte,
        '>' => :gt,
        '<' => :lt,
      }

      # We'll want to match longer names first
      # so that > and < don't clobber >= and <=
      OP_NAMES = OPERATORS.keys.sort_by {|o| -o.size}

      REGULAR_EXPRESSIONS = {
        :whitespace => /\s*/,
        :variable => /!(\w+)/,
        :ident => /(\\.|[^\s\\+\-*\/%(),=!])+/,
        :string => /"((?:\\.|[^"\\])*)"/,
        :number => /(-)?(?:(\d*\.\d+)|(\d+))([a-zA-Z%]+)?/,
        :color => /\##{"([0-9a-fA-F]{1,2})" * 3}|(#{Color::HTML4_COLORS.keys.join("|")})/,
        :bool => /(true|false)\b/,
        :op => %r{(#{Regexp.union(*OP_NAMES.map{|s| Regexp.new(Regexp.escape(s) + (s =~ /\w$/ ? '(?:\b|$)' : ''))})})}
      }

      def initialize(str)
        @scanner = StringScanner.new(str)
      end

      def token
        return if done?

        if @tok
          @tok, tok = nil, @tok
          return tok
        end

        whitespace
        variable || string || number || color || bool || op || ident ||
          (raise SyntaxError.new("Syntax error in '#{@scanner.string}' at '#{@scanner.rest}'."))
      end

      def peek
        @tok ||= token
      end

      def done?
        whitespace
        @scanner.eos? && @tok.nil?
      end

      def rest
        @scanner.rest
      end

      private

      def whitespace
        @scanner.scan(REGULAR_EXPRESSIONS[:whitespace])
      end

      def variable
        return unless @scanner.scan(REGULAR_EXPRESSIONS[:variable])
        [:const, @scanner[1]]
      end

      def ident
        return unless s = @scanner.scan(REGULAR_EXPRESSIONS[:ident])
        [:ident, s.gsub(/\\(.)/, '\1')]<commit_msg>Small optimization for the token method.<commit_after>require 'strscan'

module Sass
  module Script
    class Lexer # :nodoc:
      OPERATORS = {
        '+' => :plus,
        '-' => :minus,
        '*' => :times,
        '/' => :div,
        '%' => :mod,
        '(' => :lparen,
        ')' => :rparen,
        ',' => :comma,
        'and' => :and,
        'or' => :or,
        'not' => :not,
        '==' => :eq,
        '!=' => :neq,
        '>=' => :gte,
        '<=' => :lte,
        '>' => :gt,
        '<' => :lt,
      }

      # We'll want to match longer names first
      # so that > and < don't clobber >= and <=
      OP_NAMES = OPERATORS.keys.sort_by {|o| -o.size}

      REGULAR_EXPRESSIONS = {
        :whitespace => /\s*/,
        :variable => /!(\w+)/,
        :ident => /(\\.|[^\s\\+\-*\/%(),=!])+/,
        :string => /"((?:\\.|[^"\\])*)"/,
        :number => /(-)?(?:(\d*\.\d+)|(\d+))([a-zA-Z%]+)?/,
        :color => /\##{"([0-9a-fA-F]{1,2})" * 3}|(#{Color::HTML4_COLORS.keys.join("|")})/,
        :bool => /(true|false)\b/,
        :op => %r{(#{Regexp.union(*OP_NAMES.map{|s| Regexp.new(Regexp.escape(s) + (s =~ /\w$/ ? '(?:\b|$)' : ''))})})}
      }

      def initialize(str)
        @scanner = StringScanner.new(str)
      end

      def token
        if @tok
          @tok, tok = nil, @tok
          return tok
        end

        return if done?

        variable || string || number || color || bool || op || ident ||
          (raise SyntaxError.new("Syntax error in '#{@scanner.string}' at '#{@scanner.rest}'."))
      end

      def peek
        @tok ||= token
      end

      def done?
        whitespace
        @scanner.eos? && @tok.nil?
      end

      def rest
        @scanner.rest
      end

      private

      def whitespace
        @scanner.scan(REGULAR_EXPRESSIONS[:whitespace])
      end

      def variable
        return unless @scanner.scan(REGULAR_EXPRESSIONS[:variable])
        [:const, @scanner[1]]
      end

      def ident
        return unless s = @scanner.scan(REGULAR_EXPRESSIONS[:ident])
        [:ident, s.gsub(/\\(.)/, '\1')]
{'fraction_non_alphanumeric': 0.2268806899856253, 'fraction_numerical': 0.003833253473885961, 'mean_word_length': 1.6224874371859297, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3794347', 'n_tokens_mistral': 1549, 'n_tokens_neox': 1493, 'n_words': 283}