<commit_before>require 'ripper'
require 'hamlit/ruby_expression'

module Hamlit::StringInterpolation
  class << self
    # `code` param must be valid string literal
    def compile(code)
      [].tap do |exps|
        code   = Hamlit::RubyExpression.strip_comment(code)
        tokens = Ripper.lex(code)

        if tokens.size < 2
          raise Hamlit::InternalError.new("Expected token size >= 2 but got: #{tokens.size}")
        end
        compile_tokens!(exps, tokens)
      end
    end

    # Compile Hamlit::HamlParser::ParseNode into Temple AST.
    def compile_node(node, key)
      [:multi].tap do |temple|
        compile(node.value[key]).each do |type, value|
          case type
          when :static
            value = Temple::Utils.escape_html(value) if node.value[:escape_html]
            value = ::Hamlit::HamlHelpers.find_and_preserve(value, %w(textarea pre code)) if node.value[:preserve]
            temple << [:static, value]
          when :dynamic
            temple << [:escape, node.value[:escape_html] || node.value[:escape_interpolation], [:dynamic, value]]<commit_msg>Optimize string compilation<commit_after>require 'ripper'
require 'hamlit/ruby_expression'

module Hamlit::StringInterpolation
  class << self
    # `code` param must be valid string literal
    def compile(code)
      [].tap do |exps|
        code   = Hamlit::RubyExpression.strip_comment(code)
        tokens = Ripper.lex(code)

        if tokens.size < 2
          raise Hamlit::InternalError.new("Expected token size >= 2 but got: #{tokens.size}")
        end
        compile_tokens!(exps, tokens)
      end
    end

    # Compile Hamlit::HamlParser::ParseNode into Temple AST.
    def compile_node(node, key)
      [:multi].tap do |temple|
        compile(node.value[key]).each do |type, value|
          case type
          when :static
            value = Temple::Utils.escape_html(value) if node.value[:escape_html]
            temple << [:static, value]
          when :dynamic
            if Hamlit::StaticAnalyzer.static?(value)
              value = eval(value).to_s
              value = Temple::Utils.escape_html(value) if node.value[:escape_html] || node.value[:escape_interpolation]
              temple << [:static, value]
            else
              temple << [:escape, node.value[:escape_html] || node.value[:escape_interpolation], [:dynamic, value]]
            end
{'fraction_non_alphanumeric': 0.11636668061950607, 'fraction_numerical': 0.0016743407283382169, 'mean_word_length': 2.5993975903614457, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5197023', 'n_tokens_mistral': 703, 'n_tokens_neox': 689, 'n_words': 170}