<commit_before>require 'yaml'

require 'spout/helpers/subject_loader'
require 'spout/models/coverage_result'

module Spout
  module Commands
    class Coverage
      def initialize(standard_version)
        @standard_version = standard_version

        @variable_files = []
        @valid_ids = []
        @number_of_rows = 100

        spout_config = YAML.load_file('.spout.yml')
        @visit = (spout_config.kind_of?(Hash) ? spout_config['visit'].to_s.strip : '')

        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
        @subject_loader.load_subjects_from_csvs_part_one! # Not Part Two which is essentially cleaning the data
        @subjects = @subject_loader.subjects

        run_coverage_report!
      end

      def run_coverage_report!
        choice_variables = []

        Dir.glob("variables/**/*.json").each do |file|
          if json = JSON.parse(File.read(file)) rescue false
            choice_variables << json['id'] if json['type'] == 'choices'
          end
        end

        @matching_results = []
        csv_names = ['tmp_csv_file.csv']

        all_column_headers = @subjects.size > 0 ? @subjects.first.class.instance_methods(false).select{|k| k != :_visit}.reject{|k| k.to_s[-1] == '='} : []
        all_column_headers.each do |column|<commit_msg>Show all lines<commit_after>require 'yaml'

require 'spout/helpers/subject_loader'
require 'spout/models/coverage_result'

module Spout
  module Commands
    class Coverage
      def initialize(standard_version)
        @standard_version = standard_version

        @variable_files = []
        @valid_ids = []
        @number_of_rows = nil

        spout_config = YAML.load_file('.spout.yml')
        @visit = (spout_config.kind_of?(Hash) ? spout_config['visit'].to_s.strip : '')

        @subject_loader = Spout::Helpers::SubjectLoader.new(@variable_files, @valid_ids, @standard_version, @number_of_rows, @visit)
        @subject_loader.load_subjects_from_csvs_part_one! # Not Part Two which is essentially cleaning the data
        @subjects = @subject_loader.subjects

        run_coverage_report!
      end

      def run_coverage_report!
        choice_variables = []

        Dir.glob("variables/**/*.json").each do |file|
          if json = JSON.parse(File.read(file)) rescue false
            choice_variables << json['id'] if json['type'] == 'choices'
          end
        end

        @matching_results = []
        csv_names = ['tmp_csv_file.csv']

        all_column_headers = @subjects.size > 0 ? @subjects.first.class.instance_methods(false).select{|k| k != :_visit}.reject{|k| k.to_s[-1] == '='} : []
        all_column_headers.each do |column|
{'fraction_non_alphanumeric': 0.12339331619537275, 'fraction_numerical': 0.002570694087403599, 'mean_word_length': 3.0236336779911372, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642611', 'n_tokens_mistral': 870, 'n_tokens_neox': 841, 'n_words': 153}