<commit_before>require 'optparse'
require 'csv'


module CSVProc

    PROG_NAME = 'csvproc'

    @opt_parsers = {
        'global' => OptionParser.new do |opts|
        opts.banner = "Usage: #{PROG_NAME} [options] command [cmd_options]

commands:
    help            shows help for a command
    keyval  	    converts csv to key=value. First line contains keys.
        "
            opts.on('-h', '--help', 'Show help') do
                puts opts
                exit
            end
        end,

        'keyval' => OptionParser.new do |opts|
            opts.banner = "Usage: #{PROG_NAME} keyval CSVFILE..."
        end,
    }

    def self.cmd_help()
        if ARGV.size == 0
            puts @opt_parsers['help']
        else
            puts @opt_parsers[ARGV[0]]
        end
    end

    def self.parse_cmdline()
        @opt_parsers['global'].parse!
        command = ARGV.shift

        if !command
            puts @opt_parsers['global']
            exit
        end

        raise "Unknown command: #{command}"  if !@opt_parsers.key?(command)

        @opt_parsers[command].parse!
        send('cmd_' + command)
    end

    def self.cmd_keyval()
        ARGV.each do |f|
            first = true
            csv = CSV.foreach(f, :headers => :first_row) do |row|
                if !first
                    puts '-'*8
                end
                first = false
                row.each do | (k, v) |
                    puts "#{k}=#{v}"
                end
            end
        end
    end

    begin
        parse_cmdline
    rescue Errno::EPIPE
    end


end # module<commit_msg>Escapes line breaks in key and value.<commit_after>require 'optparse'
require 'csv'


module CSVProc

    PROG_NAME = 'csvproc'

    @opt_parsers = {
        'global' => OptionParser.new do |opts|
        opts.banner = "Usage: #{PROG_NAME} [options] command [cmd_options]

commands:
    help            shows help for a command
    keyval  	    converts csv to key=value. First line of file contains key names.
                    Escapes \\n, \\r, \\f, and \\ with backslash.
        "
            opts.on('-h', '--help', 'Show help') do
                puts opts
                exit
            end
        end,

        'keyval' => OptionParser.new do |opts|
            opts.banner = "Usage: #{PROG_NAME} keyval CSVFILE..."
        end,
    }

    def self.cmd_help()
        if ARGV.size == 0
            puts @opt_parsers['help']
        else
            puts @opt_parsers[ARGV[0]]
        end
    end

    def self.parse_cmdline()
        @opt_parsers['global'].parse!
        command = ARGV.shift

        if !command
            puts @opt_parsers['global']
            exit
        end

        raise "Unknown command: #{command}"  if !@opt_parsers.key?(command)

        @opt_parsers[command].parse!
        send('cmd_' + command)
    end

    def self.cmd_keyval()
        ARGV.each do |f|
            first = true
            csv = CSV.foreach(f, :headers => :first_row) do |row|
                if !first
                    puts '-'*8
                end
                first = false
                row.each do | (k, v) |
                    puts "#{escape(k)}=#{escape(v)}"
                end
            end
        end
    end

    @h = {
            "\\" => Regexp.escape("\\\\"),
            "\n" => Regexp.escape("\n"),
            "\r" => Regexp.escape("\r"),
            "\f" => Regexp.escape("\f"),
    }

    def self.escape(s)
        return nil unless s
        s.gsub(/\\|\n/) { |m|
            @h[m]
        }
    end

    begin
        parse_cmdline
    rescue Errno::EPIPE
    end


end # module
{'fraction_non_alphanumeric': 0.11822930987077261, 'fraction_numerical': 0.0016497113005224085, 'mean_word_length': 1.327575175943698, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3547076', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1106, 'n_words': 283}