<commit_before># frozen_string_literal: true

# A simple library to prepend magic comments to multiple ".rb" files

module AddMagicComment
  MAGIC_COMMENT_PREFIX  = "frozen_string_literal"
  MAGIC_COMMENT_PATTERN = /^-?# *#{MAGIC_COMMENT_PREFIX}/
  MAGIC_COMMENT         = "#{MAGIC_COMMENT_PREFIX}: true"

  EXTENSIONS = {
    'rb'   => '# {text}',
    'rake' => '# {text}',
    'haml' => '-# {text}',
  }

  # Options :
  # 1 : Encoding
  # 2 : Path
  # TODO : check that the encoding specified is a valid encoding
  # TODO : allow use of only one option, so the encoding would be guessed (maybe using `file --mime`?)
  def self.process(options)

    directory = options[0] || Dir.pwd


    # TODO : add options for recursivity (and application of the script to a single file)

    count = 0
    EXTENSIONS.each do |ext, comment_style|
      rbfiles = File.join(directory, "**", "*.#{ext}")
      Dir.glob(rbfiles).each do |filename|
        next unless File.size(filename) > 0
        File.open(filename, "r+") do |file|
          lines = file.readlines

          # remove current encoding comment(s)
          while lines[0].match(MAGIC_COMMENT_PATTERN)
            lines.shift
          end

          # set current encoding
          lines.insert(0, comment_style.sub('{text}', MAGIC_COMMENT + "\n"))
          count += 1

          file.pos = 0
          file.puts(lines.join)
        end
      end
    end

    puts "Magic comments added to #{count} source file(s)"
  end
end<commit_msg>Truncate output file.<commit_after># frozen_string_literal: true

# A simple library to prepend magic comments to multiple ".rb" files

module AddMagicComment
  MAGIC_COMMENT_PREFIX  = "frozen_string_literal"
  MAGIC_COMMENT_PATTERN = /^-?# *#{MAGIC_COMMENT_PREFIX}/
  MAGIC_COMMENT         = "#{MAGIC_COMMENT_PREFIX}: true"

  EXTENSIONS = {
    'rb'   => '# {text}',
    'rake' => '# {text}',
    'haml' => '-# {text}',
  }

  # Options :
  # 1 : Encoding
  # 2 : Path
  # TODO : check that the encoding specified is a valid encoding
  # TODO : allow use of only one option, so the encoding would be guessed (maybe using `file --mime`?)
  def self.process(options)

    directory = options[0] || Dir.pwd


    # TODO : add options for recursivity (and application of the script to a single file)

    count = 0
    EXTENSIONS.each do |ext, comment_style|
      rbfiles = File.join(directory, "**", "*.#{ext}")
      Dir.glob(rbfiles).each do |filename|
        next unless File.size(filename) > 0
        File.open(filename, "r+") do |file|
          lines = file.readlines

          # remove current encoding comment(s)
          while lines[0].match(MAGIC_COMMENT_PATTERN)
            lines.shift
          end

          # set current encoding
          lines.insert(0, comment_style.sub('{text}', MAGIC_COMMENT + "\n"))
          count += 1

          file.pos = 0
          file.puts(lines.join)
          file.truncate(file.pos)
        end
      end
    end

    puts "Magic comments added to #{count} source file(s)"
  end
end
{'fraction_non_alphanumeric': 0.11972295514511873, 'fraction_numerical': 0.005936675461741424, 'mean_word_length': 2.4982698961937717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '205104', 'n_tokens_mistral': 999, 'n_tokens_neox': 953, 'n_words': 274}