<commit_before># -*- coding: utf-8 -*-
if RUBY_VERSION < '1.9'
  require 'fastercsv'
else
  require 'csv'
  FasterCSV = CSV
end

module Gyunyu
  module Command
    module Export
      module Format
        module Csv
          @num_tags  = nil
          @num_notes = nil

          class << self
            def export( tasks, fields )
              FasterCSV.generate { |csv|
                tasks, fields = parse( tasks, fields )
                csv << fields
                fields.delete('')
                tasks.each { |t|
                  csv << fields.map { |f|
                    case f
                    when 'tags', 'notes'
                      num = instance_variable_get( "@num_#{f}" )
                      while t[f].size < num
                        t[f] += ['']
                      end
                    end
                    t[f]
                  }.flatten
                }
              }
            end

            def parse( tasks, fields )
              if !fields.include?( 'tags' ) and !fields.include?( 'notes' )
                [tasks, fields]
              else
                if fields.include?( 'tags' )
                  tasks = tasks.map { |t| parse_tags( t ) }
                end
                if fields.include?( 'notes' )
                  tasks = tasks.map { |t| parse_notes( t ) }
                end

                [tasks, expand_fields( fields )]
              end
            end

            def parse_tags( t )
              tags = if t['tags'].first.has_key? 'tag'
                       t['tags'].first['tag']
                     else
                       []
                     end
              if !@num_tags or tags.size > @num_tags
                @num_tags = tags.size
              end
              t['tags'] = tags.sort

              t
            end

            def parse_notes( t )
              notes = if t['notes'].first.has_key? 'note'
                        t['notes'].first['note'].map { |n|
                          n['content']
                        }
                      else
                        []
                      end
              if !@num_notes or notes.size > @num_notes
                @num_notes = notes.size
              end
              t['notes'] = notes

              t
            end

            def expand_fields( fields )
              fields.map { |e|
                case e
                when 'tags', 'notes'
                  num = instance_variable_get("@num_#{e}")
                  if num and num > 1
                    [e] + Array.new(num - 1, '')
                  else
                    e
                  end
                else
                  e
                end
              }.flatten
            end
          end<commit_msg>enhance : stable tags fields<commit_after># -*- coding: utf-8 -*-
if RUBY_VERSION < '1.9'
  require 'fastercsv'
else
  require 'csv'
  FasterCSV = CSV
end

module Gyunyu
  module Command
    module Export
      module Format
        module Csv
          @tags      = []
          @num_tags  = nil
          @num_notes = nil

          class << self
            def export( tasks, fields )
              FasterCSV.generate { |csv|
                tasks, fields = parse( tasks, fields )
                csv << fields
                fields.delete('')
                tasks.each { |t|
                  csv << fields.map { |f|
                    case f
                    when 'tags'
                      t[f] = expand_tags( t[f] )
                    when 'notes'
                      num = instance_variable_get( "@num_#{f}" )
                      while t[f].size < num
                        t[f] += ['']
                      end
                    end
                    t[f]
                  }.flatten
                }
              }
            end

            def parse( tasks, fields )
              if !fields.include?( 'tags' ) and !fields.include?( 'notes' )
                [tasks, fields]
              else
                if fields.include?( 'tags' )
                  tasks = tasks.map { |t| parse_tags( t ) }
                end
                if fields.include?( 'notes' )
                  tasks = tasks.map { |t| parse_notes( t ) }
                end

                [tasks, expand_fields( fields )]
              end
            end

            def parse_tags( t )
              tags = if t['tags'].first.has_key? 'tag'
                       t['tags'].first['tag']
                     else
                       []
                     end
              update_tags_field( tags )

              t['tags'] = tags.sort

              t
            end

            def parse_notes( t )
              notes = if t['notes'].first.has_key? 'note'
                        t['notes'].first['note'].map { |n|
                          n['content']
                        }
                      else
                        []
                      end
              if !@num_notes or notes.size > @num_notes
                @num_notes = notes.size
              end
              t['notes'] = notes

              t
            end

            def update_tags_field( tags )
              @tags = (@tags | tags).sort

              @num_tags = @tags.size
            end

            def expand_fields( fields )
              fields.map { |e|
                case e
                when 'tags', 'notes'
                  num = instance_variable_get("@num_#{e}")
                  if num and num > 1
                    [e] + Array.new(num - 1, '')
                  else
                    e
                  end
                else
                  e
                end
              }.flatten
            end

            def expand_tags( tags )
              @tags.map { |e|
                tags.include?(e) ? e : ''
              }
            end
          end
{'fraction_non_alphanumeric': 0.09288873766587274, 'fraction_numerical': 0.001701258931609391, 'mean_word_length': 0.7617620617320947, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1675079', 'n_tokens_mistral': 1533, 'n_tokens_neox': 1413, 'n_words': 372}