<commit_before>class Completion
  include Mongoid::Document
  field :run,     type: Integer
  field :step,    type: String
  field :phone,   type: String
  field :text,    type: String
  field :flow,    type: Integer
  field :values,  type: Array
  field :primary, type: String
  field :ids,     type: Array
  field :arrived_on, type: DateTime
  field :left_on,    type: DateTime
  # field :steps,   type: Array
  
  embeds_many :steps

  def self.primary_node_text(id)
    Record.where(primary: id).first.steps.first.text
  end


  def self.total_instances(id)
    Completion.where(primary: id).all.count
  end


  def self.flare(diagram)

  end

  def self.diagram(node)
    node2question = {}
    node2response = {}
    node2label   = {}
    hits         = {}

    Completion.all.group_by(&:ids).each do |arry|
      arry[1].each do |completion| 
        completion.steps.each do |s|

          if !!s.text
            s.text.rstrip!
            if s.type == "A"
              node2question[s.node] = s.text
            else # Type is R. Maintain record of responses with frequency 
              if !node2response[s.node]
                node2response[s.node] = {}
              end
              resp = s.text.downcase
              if resp.split.size == 1
                resp.sub!(/[!.?]$/,'')
              end

              if !node2response[s.node][resp]
                node2response[s.node][resp] = 1
              else
                node2response[s.node][resp] = node2response[s.node][resp] + 1
              end
            end
          end
        end

        completion.values.each do |v|
          node2label[v['node']] = v['label']
        end
      end
    end

    Completion.all.each do |c|
      c.steps.each do |s|
        hits[s.node] = !!hits[s.node] ? hits[s.node] + 1 : 1
      end
    end


    arrays = Completion.where(primary: node).map(&:ids).uniq
    puts "#{arrays} !!!!!!"
    seen   = {}    
    nodes  = []<commit_msg>strip responses<commit_after>class Completion
  include Mongoid::Document
  field :run,     type: Integer
  field :step,    type: String
  field :phone,   type: String
  field :text,    type: String
  field :flow,    type: Integer
  field :values,  type: Array
  field :primary, type: String
  field :ids,     type: Array
  field :arrived_on, type: DateTime
  field :left_on,    type: DateTime
  # field :steps,   type: Array
  
  embeds_many :steps

  def self.primary_node_text(id)
    Record.where(primary: id).first.steps.first.text
  end


  def self.total_instances(id)
    Completion.where(primary: id).all.count
  end


  def self.flare(diagram)

  end

  def self.diagram(node)
    node2question = {}
    node2response = {}
    node2label   = {}
    hits         = {}

    Completion.all.group_by(&:ids).each do |arry|
      arry[1].each do |completion| 
        completion.steps.each do |s|

          if !!s.text
            s.text.rstrip!
            if s.type == "A"
              node2question[s.node] = s.text
            else # Type is R. Maintain record of responses with frequency 
              if !node2response[s.node]
                node2response[s.node] = {}
              end
              resp = s.text.downcase.strip
              if resp.split.size == 1
                resp.sub!(/[!.?]$/,'')
              end

              if !node2response[s.node][resp]
                node2response[s.node][resp] = 1
              else
                node2response[s.node][resp] = node2response[s.node][resp] + 1
              end
            end
          end
        end

        completion.values.each do |v|
          node2label[v['node']] = v['label']
        end
      end
    end

    Completion.all.each do |c|
      c.steps.each do |s|
        hits[s.node] = !!hits[s.node] ? hits[s.node] + 1 : 1
      end
    end


    arrays = Completion.where(primary: node).map(&:ids).uniq
    puts "#{arrays} !!!!!!"
    seen   = {}    
    nodes  = []
{'fraction_non_alphanumeric': 0.12041751527494908, 'fraction_numerical': 0.008655804480651732, 'mean_word_length': 1.737979094076655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708748', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1247, 'n_words': 295}