<commit_before># encoding: utf-8

require 'set'
require 'abstract_type'
require 'procto'
require 'concord'
require 'parser/all'
require 'parser/current'

# Library namespace
module Unparser

  EMPTY_STRING = ''.freeze

  EMPTY_ARRAY = [].freeze

  # Unparse an AST (and, optionally, comments) into a string
  #
  # @param [Parser::Node, nil] node
  # @param [Array] comment_array
  #
  # @return [String]
  #
  # @api private
  #
  def self.unparse(node, comment_array = [])
    node = Preprocessor.run(node)
    buffer = Buffer.new
    comments = Comments.new(comment_array)
    root = Emitter::Root.new(Parser::AST::Node.new(:root, [node]), buffer, comments)
    Emitter.emitter(node, root).write_to_buffer
    buffer.content
  end

  # Transquote string
  #
  # @param [String] raw_quoted
  # @param [String] current_delimiter
  # @param [String] target_delimiter
  #
  # @return [String]
  #
  # @api private
  #
  def self.transquote(raw_quoted, current_delimiter, target_delimiter)
    raw = raw_quoted.gsub("\\#{current_delimiter}", current_delimiter)
    raw.gsub(target_delimiter, "\\#{target_delimiter}").freeze
  end

end # Unparser

require 'unparser/strip_helper'
require 'unparser/buffer'
require 'unparser/node_helpers'
require 'unparser/preprocessor'
require 'unparser/comments'
require 'unparser/constants'
require 'unparser/ast'
require 'unparser/ast/local_variable_scope'
require 'unparser/emitter'
require 'unparser/emitter/literal'<commit_msg>Remove dead code<commit_after># encoding: utf-8

require 'set'
require 'abstract_type'
require 'procto'
require 'concord'
require 'parser/all'
require 'parser/current'

# Library namespace
module Unparser

  EMPTY_STRING = ''.freeze

  EMPTY_ARRAY = [].freeze

  # Unparse an AST (and, optionally, comments) into a string
  #
  # @param [Parser::Node, nil] node
  # @param [Array] comment_array
  #
  # @return [String]
  #
  # @api private
  #
  def self.unparse(node, comment_array = [])
    node = Preprocessor.run(node)
    buffer = Buffer.new
    comments = Comments.new(comment_array)
    root = Emitter::Root.new(Parser::AST::Node.new(:root, [node]), buffer, comments)
    Emitter.emitter(node, root).write_to_buffer
    buffer.content
  end

end # Unparser

require 'unparser/strip_helper'
require 'unparser/buffer'
require 'unparser/node_helpers'
require 'unparser/preprocessor'
require 'unparser/comments'
require 'unparser/constants'
require 'unparser/ast'
require 'unparser/ast/local_variable_scope'
require 'unparser/emitter'
require 'unparser/emitter/literal'
{'fraction_non_alphanumeric': 0.12297989751675206, 'fraction_numerical': 0.0007883326763894363, 'mean_word_length': 4.665178571428571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4892288', 'n_tokens_mistral': 869, 'n_tokens_neox': 848, 'n_words': 192}