<commit_before># data types that are used in compilation
require 'mfactor/graph.rb'

module MFactor
  def convert_tailcall(b)
    if b[-1].is_a? MFWord
      b[-1].is_tail = true
      # puts "#{b[-1].err_loc}:Info: tailcall"
    end
    # b.each do |elt|
    #   if elt.is_a?(Array)
    #     convert_tailcalls(elt)
    #   end
    # end
  end
  module_function :convert_tailcall
  class Literal < Struct.new(:value)
    include GraphNode
    def dot_label
      value.to_s
    end
    def dot_node_shape
      "none"
    end
  end
  # integer literal
  class MFIntLit < Literal
    def see
      "#I#{value}"
    end
  end
  # special case: byte-sized literal
  class MFByteLit < MFIntLit
    def see
      "#B#{value}"
    end
  end
  class MFStringLit < Literal
    def see
      value.inspect
    end
    def dot_label
      '\"'+value+'\"'
    end
  end
  class MFWord < Struct.new(:name,:definition,:is_tail)
    def initialize(*a)
      super *a
      @file=$current_mfactor_file
    end
    def see
      name.to_s.downcase
    end
    def err_loc
      line,col=name.line_and_column
      "#{@file}:#{line}:#{col}"
    end
    def change_name(newname)
      old=name
      name=Parslet::Slice.new(old.position,newname,old.line_cache)
    end
  end

  # Definition object, which can be moved into dictionary
  # file: source file of the definition
  # graph: if set, points to the graph resulted by partial evaluation.  basis for code generation and petri-net simulation
  class MFDefinition < Struct.new(:name,:definer,:effect,:code,:mods,:vocabulary,:file,:graph,:compile_log)
    def syntax_word?
      definer == "SYNTAX:"
    end
    def normal_word?
      definer == ":"
    end
    def primitive?
      definer == "PRIM:"
    end
    def compilable?
      (definer == ":") &&
        # !(mods.member?("inline")) &&
        !(mods.member?("nocompile"))
    end
    def inline?
      mods.member? "inline"
    end
    def recursive?
      mods.member? "recursive"
    end
    # return printed location of definition
    def err_loc
      line,col=definer.line_and_column
      "#{file}:#{line}:#{col}"
    end
    def see
      ": #{name} #{effect} "+
        code.body.map{ |elt| see_word(elt) }.join(" ")
    end
    # TODO: move out of here, into emitter
    # attach something to the compilation log
    def log (s)
      self.compile_log ||= ""
      self.compile_log += s
      self.compile_log += "\n"
      s
    end
    # return a flat list of all words that might be somehow inside
    def flatten_words
      res=[]
      flatten_quotation_words(code.body,res)
      res
    end
    private
    def flatten_quotation_words (arr,acc=[])
      arr.each do |elt|
        acc.push elt if elt.is_a? MFWord
        case elt
          when MFComplexSequence then
          flatten_quotation_words(elt.content,acc)
          when Quotation then
          flatten_quotation_words(elt.body,acc)
        end
      end
    end
  end

  class MFInput < Struct.new(:name,:type)
    include GraphNode
    def see
      self.name
    end
    def dot_label
      name
    end
  end

  # represents a result of a compiled definition
  class Output < Struct.new(:name,:type)
    include GraphNode
    def dot_label
      name
    end
  end

  #parser item: represents literal quotation
  #hack: expand case statements here
  class Quotation
    attr_accessor :body
    def initialize body
      @body = body
      MFactor::convert_tailcall(@body)
      convert_case_statements
    end
    def case_if_chain (assoc)
      if assoc.content.length == 1
        t_case = assoc.content.shift
         return t_case
      else
        a = assoc.content.shift
        a[1].body.unshift(MFWord.new("drop"))
        q = Quotation.new([MFWord.new("dup"), a[0], MFWord.new("="),
                           a[1]])
        q.body << case_if_chain(assoc)
        q.body << MFWord.new("if")
        return q
      end
    end
    def convert_case_statements
      code=@body
      case_indices = code.each_with_index.select{|x,ind|
        x.is_a?(MFWord) and (x.name.to_s == "case") }.map{|l| l[1]}
      case_indices.each do |i|
        assoc = code[i - 1]
        # remove the assoc and the case statement, insert quotation
        new_code = case_if_chain(assoc).body
        code.delete_at(i-1)
        code.delete_at(i-1)
        code.insert(i-1,*new_code)
        puts code
      end
    end
  end

  class WrappedWord < Struct.new(:name)
    def see
      "\\ #{self.name}"
    end
  end

  class MFEffectItem < Struct.new(:name,:type)<commit_msg>fix tail calls in case statements, remove debug output<commit_after># data types that are used in compilation
require 'mfactor/graph.rb'

module MFactor
  def convert_tailcall(b)
    if b[-1].is_a? MFWord
      b[-1].is_tail = true
      # puts "#{b[-1].err_loc}:Info: tailcall"
    end
    # b.each do |elt|
    #   if elt.is_a?(Array)
    #     convert_tailcalls(elt)
    #   end
    # end
  end
  module_function :convert_tailcall
  class Literal < Struct.new(:value)
    include GraphNode
    def dot_label
      value.to_s
    end
    def dot_node_shape
      "none"
    end
  end
  # integer literal
  class MFIntLit < Literal
    def see
      "#I#{value}"
    end
  end
  # special case: byte-sized literal
  class MFByteLit < MFIntLit
    def see
      "#B#{value}"
    end
  end
  class MFStringLit < Literal
    def see
      value.inspect
    end
    def dot_label
      '\"'+value+'\"'
    end
  end
  class MFWord < Struct.new(:name,:definition,:is_tail)
    def initialize(*a)
      super *a
      @file=$current_mfactor_file
    end
    def see
      name.to_s.downcase
    end
    def err_loc
      line,col=name.line_and_column
      "#{@file}:#{line}:#{col}"
    end
    def change_name(newname)
      old=name
      name=Parslet::Slice.new(old.position,newname,old.line_cache)
    end
  end

  # Definition object, which can be moved into dictionary
  # file: source file of the definition
  # graph: if set, points to the graph resulted by partial evaluation.  basis for code generation and petri-net simulation
  class MFDefinition < Struct.new(:name,:definer,:effect,:code,:mods,:vocabulary,:file,:graph,:compile_log)
    def syntax_word?
      definer == "SYNTAX:"
    end
    def normal_word?
      definer == ":"
    end
    def primitive?
      definer == "PRIM:"
    end
    def compilable?
      (definer == ":") &&
        # !(mods.member?("inline")) &&
        !(mods.member?("nocompile"))
    end
    def inline?
      mods.member? "inline"
    end
    def recursive?
      mods.member? "recursive"
    end
    # return printed location of definition
    def err_loc
      line,col=definer.line_and_column
      "#{file}:#{line}:#{col}"
    end
    def see
      ": #{name} #{effect} "+
        code.body.map{ |elt| see_word(elt) }.join(" ")
    end
    # TODO: move out of here, into emitter
    # attach something to the compilation log
    def log (s)
      self.compile_log ||= ""
      self.compile_log += s
      self.compile_log += "\n"
      s
    end
    # return a flat list of all words that might be somehow inside
    def flatten_words
      res=[]
      flatten_quotation_words(code.body,res)
      res
    end
    private
    def flatten_quotation_words (arr,acc=[])
      arr.each do |elt|
        acc.push elt if elt.is_a? MFWord
        case elt
          when MFComplexSequence then
          flatten_quotation_words(elt.content,acc)
          when Quotation then
          flatten_quotation_words(elt.body,acc)
        end
      end
    end
  end

  class MFInput < Struct.new(:name,:type)
    include GraphNode
    def see
      self.name
    end
    def dot_label
      name
    end
  end

  # represents a result of a compiled definition
  class Output < Struct.new(:name,:type)
    include GraphNode
    def dot_label
      name
    end
  end

  #parser item: represents literal quotation
  #hack: expand case statements here
  class Quotation
    attr_accessor :body
    def initialize body
      @body = body
      MFactor::convert_tailcall(@body)
      convert_case_statements
    end
    def case_if_chain (assoc)
      if assoc.content.length == 1
        t_case = assoc.content.shift
         return t_case
      else
        a = assoc.content.shift
        a[1].body.unshift(MFWord.new("drop"))
        q = Quotation.new([MFWord.new("dup"), a[0], MFWord.new("="),
                           a[1]])
        q.body << case_if_chain(assoc)
        q.body << MFWord.new("if",nil,true)
        return q
      end
    end
    def convert_case_statements
      code=@body
      case_indices = code.each_with_index.select{|x,ind|
        x.is_a?(MFWord) and (x.name.to_s == "case") }.map{|l| l[1]}
      case_indices.each do |i|
        assoc = code[i - 1]
        # remove the assoc and the case statement, insert quotation
        new_code = case_if_chain(assoc).body
        code.delete_at(i-1)
        code.delete_at(i-1)
        code.insert(i-1,*new_code)
      end
    end
  end

  class WrappedWord < Struct.new(:name)
    def see
      "\\ #{self.name}"
    end
  end

  class MFEffectItem < Struct.new(:name,:type)
{'fraction_non_alphanumeric': 0.1044498027181061, 'fraction_numerical': 0.00263042525208242, 'mean_word_length': 2.2232426704344754, 'pattern_counts': {'":': 6, '<': 31, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '84844', 'n_tokens_mistral': 3084, 'n_tokens_neox': 3067, 'n_words': 773}