<commit_before>require 'source_board/parser/base'

module SourceBoard

  module Parser

    class Simple < Base

      def parse!(source_tree)
        loc_data = Dir.chdir(source_tree.directory) do
          process_with_wc(source_tree)
        end
        def_data = Dir.chdir(source_tree.directory) do
          def_data = process_with_ctags(source_tree)
        end
        source_tree.files.map do |file|
          {
            'id'        => file.path,
            'path'      => file.path,
            'name'      => File.basename(file.path),
            'metrics'   => {
              'loc'     => loc_data[file.path],
              'nom'     => def_data[file.path] || 0,
            },
          }
        end
      end

      private

      def process_with_wc(source_tree)
        process_pipe(source_tree, 'xargs wc -l') do |data,line|
          loc, path = line.split
          data[path] = loc.to_i
        end
      end

      def process_with_ctags(source_tree)
        process_pipe(source_tree, 'xargs ctags -x') do |data,line|
          tag, kind, line_number, path, signature = line.split
          data[path] ||= 0
          if ["member", "function", "method"].include?(kind)
            data[path] += 1
          end
        end
      end

      def process_pipe(source_tree, command)
        pipe = IO.popen(command, 'w+')
        source_tree.files.each do |source_file|
          pipe.puts source_file.path
        end
        pipe.close_write
        data = {}
        pipe.each_line do |line|
          yield(data, line)
        end
        pipe.close
        data
      end

    end

  end

end
<commit_msg>Rename def_data → nom_data<commit_after>require 'source_board/parser/base'

module SourceBoard

  module Parser

    class Simple < Base

      def parse!(source_tree)
        loc_data = Dir.chdir(source_tree.directory) do
          process_with_wc(source_tree)
        end
        nom_data = Dir.chdir(source_tree.directory) do
          process_with_ctags(source_tree)
        end
        source_tree.files.map do |file|
          {
            'id'        => file.path,
            'path'      => file.path,
            'name'      => File.basename(file.path),
            'metrics'   => {
              'loc'     => loc_data[file.path],
              'nom'     => nom_data[file.path] || 0,
            },
          }
        end
      end

      private

      def process_with_wc(source_tree)
        process_pipe(source_tree, 'xargs wc -l') do |data,line|
          loc, path = line.split
          data[path] = loc.to_i
        end
      end

      def process_with_ctags(source_tree)
        process_pipe(source_tree, 'xargs ctags -x') do |data,line|
          tag, kind, line_number, path, signature = line.split
          data[path] ||= 0
          if ["member", "function", "method"].include?(kind)
            data[path] += 1
          end
        end
      end

      def process_pipe(source_tree, command)
        pipe = IO.popen(command, 'w+')
        source_tree.files.each do |source_file|
          pipe.puts source_file.path
        end
        pipe.close_write
        data = {}
        pipe.each_line do |line|
          yield(data, line)
        end
        pipe.close
        data
      end

    end

  end

end

{'fraction_non_alphanumeric': 0.1003671970624235, 'fraction_numerical': 0.0018359853121175031, 'mean_word_length': 1.5780757097791798, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4019986', 'n_tokens_mistral': 1023, 'n_tokens_neox': 1002, 'n_words': 214}