<commit_before>require 'csv'


class CSVDiff

    # Defines functionality for exporting a Diff report in TEXT format. This is
    # a CSV format where only fields with differences have values.
    module Text

        private

        # Generare a diff report in TEXT format.
        def text_output(output)
            path = "#{File.dirname(output)}/#{File.basename(output, File.extname(output))}.csv"
            CSV.open(path, 'w') do |csv|
                @diffs.each do |file_diff|
                    text_diff(csv, file_diff) if file_diff.diffs.size > 0
                end
            end
            path
        end


        def text_diff(csv, file_diff)
            count = 0

            all_fields = [:row, :action]
            all_fields << :sibling_position unless file_diff.options[:ignore_moves]
            all_fields.concat(file_diff.diff_fields)

            csv << all_fields.map{ |fld| fld.is_a?(Symbol) ? titleize(fld) : fld }
            file_diff.diffs.each do |key, diff|
                row = all_fields.map do |field|
                    d = diff[field]
                    d = d.last if d.is_a?(Array)
                    d
                end
                csv << row
            end<commit_msg>Remove unused variable to silence warning<commit_after>require 'csv'


class CSVDiff

    # Defines functionality for exporting a Diff report in TEXT format. This is
    # a CSV format where only fields with differences have values.
    module Text

        private

        # Generare a diff report in TEXT format.
        def text_output(output)
            path = "#{File.dirname(output)}/#{File.basename(output, File.extname(output))}.csv"
            CSV.open(path, 'w') do |csv|
                @diffs.each do |file_diff|
                    text_diff(csv, file_diff) if file_diff.diffs.size > 0
                end
            end
            path
        end


        def text_diff(csv, file_diff)
            all_fields = [:row, :action]
            all_fields << :sibling_position unless file_diff.options[:ignore_moves]
            all_fields.concat(file_diff.diff_fields)

            csv << all_fields.map{ |fld| fld.is_a?(Symbol) ? titleize(fld) : fld }
            file_diff.diffs.each do |key, diff|
                row = all_fields.map do |field|
                    d = diff[field]
                    d = d.last if d.is_a?(Array)
                    d
                end
                csv << row
            end
{'fraction_non_alphanumeric': 0.08729192042224929, 'fraction_numerical': 0.001218026796589525, 'mean_word_length': 1.5666666666666667, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4483005', 'n_tokens_mistral': 731, 'n_tokens_neox': 694, 'n_words': 192}