<commit_before>class OtrunkExampleImport
  require 'hpricot'
  require 'open-uri'
  require 'fileutils'

  require 'set'
  require 'arrayfields'

  ViewEntry = Array.struct :fq_view_classname, :fq_object_classname

  class OtFile
    attr_accessor :path, :name, :category, :last_modified, :body, :doc, :imports, :view_entries
    def initialize(path)
      @path = path
      @name = File.basename(path)
      @category =  File.basename(File.dirname(@path))
      @last_modified =  File.ctime(@path)
      @body = File.read(path)
      @doc = Hpricot::XML(@body)
      @imports = @doc.search("import").collect { |e| e['class'] }
      @view_entries = @doc.search("OTViewEntry").collect {|ve| ViewEntry.new([ve['viewClass'], ve['objectClass']])}
    end
  end

  class OtImport
    attr_accessor :fq_classname, :classname, :otml_files
    def initialize(klass, otml_files=nil)
      @fq_classname = klass
      @classname = klass.split('.')[-1]
      @otml_files = Set.new
      if otml_files then @otml_files.merge(otml_files)
      end
    end
  end

  class OtViewEntry
    attr_accessor :fq_view_classname, :view_classname, :fq_object_classname, :object_classname, :otml_files
    def initialize(view_entry, otml_files=nil)
      @fq_view_classname = view_entry[:fq_view_classname]
      @view_classname = @fq_view_classname.split('.')[-1]
      @fq_object_classname = view_entry[:fq_object_classname]
      @object_classname = @fq_object_classname.split('.')[-1]
      @otml_files = Set.new
      if otml_files then @otml_files.merge(otml_files)
      end
    end
  end

  class OtLaunchFile
    attr_accessor :path, :name, :body, :doc, :launch_type, :project_name, :projects_dir, :class_dir, :category,
                  :projects, :internal_archives, :main_type, :program_arguments, :vm_arguments, :valid
    def initialize(path)
      @path = path
      @name = File.basename(@path).chomp('.launch')
      @body = File.read(@path)
      @doc = Hpricot::XML(@body)
      @launch_type = @doc.search("launchConfiguration").attr(:type).split('.')[-1]
      list_entry_values = @doc.search("listAttribute[@key='org.<EMAIL>.jdt.launching.CLASSPATH']/listEntry").collect {|le| le['value']}
      @project_name = @doc.search("stringAttribute[@key='<EMAIL>.launching.PROJECT_ATTR']").attr(:value)
      @projects_dir = @path.split(project_name)[0]
      @category = @path[/#{@project_name}\/(.*)\//, 1]
      @internal_archives = []
      @projects = []
      @valid = true
      list_entry_values.each do |val|
        value = Hpricot::XML(val)
        if project = value.search("runtimeClasspathEntry")[0]['projectName']
          path = @projects_dir + project
          if File.exists?("#{path}/.classpath")
            classpath = Hpricot.XML(open("#{path}/.classpath"))
            output = classpath.at("classpathentry[@kind=output]")['path']
            @projects <<  "#{path}/#{output}"
          else
            @valid = false
            @projects <<  "#{path}/no-classpath-found"
          end
        elsif archive = value.search("runtimeClasspathEntry")[0]['internalArchive']
          @internal_archives << @projects_dir.chomp('/') + archive
        end
      end
      @main_type = @doc.search("stringAttribute[@key='<EMAIL>.launching.MAIN_TYPE']").collect {|i| i['value']}.to_s
      @program_arguments = @doc.search("stringAttribute[@key='org.<EMAIL>.jdt.launching.PROGRAM_ARGUMENTS']").collect {|i| i['value']}.to_s
      @vm_arguments = @doc.search("stringAttribute[@key='org.eclipse.jdt.launching.VM_ARGUMENTS']").collect {|i| i['value']}.to_s
    end
  end<commit_msg>work-around possibility of unnamed eclipse projects<commit_after>class OtrunkExampleImport
  require 'hpricot'
  require 'open-uri'
  require 'fileutils'

  require 'set'
  require 'arrayfields'

  ViewEntry = Array.struct :fq_view_classname, :fq_object_classname

  class OtFile
    attr_accessor :path, :name, :category, :last_modified, :body, :doc, :imports, :view_entries
    def initialize(path)
      @path = path
      @name = File.basename(path)
      @category =  File.basename(File.dirname(@path))
      @last_modified =  File.ctime(@path)
      @body = File.read(path)
      @doc = Hpricot::XML(@body)
      @imports = @doc.search("import").collect { |e| e['class'] }
      @view_entries = @doc.search("OTViewEntry").collect {|ve| ViewEntry.new([ve['viewClass'], ve['objectClass']])}
    end
  end

  class OtImport
    attr_accessor :fq_classname, :classname, :otml_files
    def initialize(klass, otml_files=nil)
      @fq_classname = klass
      @classname = klass.split('.')[-1]
      @otml_files = Set.new
      if otml_files then @otml_files.merge(otml_files)
      end
    end
  end

  class OtViewEntry
    attr_accessor :fq_view_classname, :view_classname, :fq_object_classname, :object_classname, :otml_files
    def initialize(view_entry, otml_files=nil)
      @fq_view_classname = view_entry[:fq_view_classname]
      @view_classname = @fq_view_classname.split('.')[-1]
      @fq_object_classname = view_entry[:fq_object_classname]
      @object_classname = @fq_object_classname.split('.')[-1]
      @otml_files = Set.new
      if otml_files then @otml_files.merge(otml_files)
      end
    end
  end

  class OtLaunchFile
    attr_accessor :path, :name, :body, :doc, :launch_type, :project_name, :projects_dir, :class_dir, :category,
                  :projects, :internal_archives, :main_type, :program_arguments, :vm_arguments, :valid
    def initialize(path)
      @path = path
      @name = File.basename(@path).chomp('.launch')
      @body = File.read(@path)
      @doc = Hpricot::XML(@body)
      @launch_type = @doc.search("launchConfiguration").attr(:type).split('.')[-1]
      list_entry_values = @doc.search("listAttribute[@key='org.<EMAIL>.jdt.launching.CLASSPATH']/listEntry").collect {|le| le['value']}
      project_attr = @doc.search("stringAttribute[@key='org.eclipse.jdt.launching.PROJECT_ATTR']")
      if project_attr
        @project_name = project_attr.attr(:value)
      else
        @project_name = 'unnamed'
      end
      @projects_dir = @path.split(project_name)[0]
      @category = @path[/#{@project_name}\/(.*)\//, 1]
      @internal_archives = []
      @projects = []
      @valid = true
      list_entry_values.each do |val|
        value = Hpricot::XML(val)
        if project = value.search("runtimeClasspathEntry")[0]['projectName']
          path = @projects_dir + project
          if File.exists?("#{path}/.classpath")
            classpath = Hpricot.XML(open("#{path}/.classpath"))
            output = classpath.at("classpathentry[@kind=output]")['path']
            @projects <<  "#{path}/#{output}"
          else
            @valid = false
            @projects <<  "#{path}/no-classpath-found"
          end
        elsif archive = value.search("runtimeClasspathEntry")[0]['internalArchive']
          @internal_archives << @projects_dir.chomp('/') + archive
        end
      end
      @main_type = @doc.search("stringAttribute[@key='<EMAIL>.launching.MAIN_TYPE']").collect {|i| i['value']}.to_s
      @program_arguments = @doc.search("stringAttribute[@key='org.eclipse.jdt.launching.PROGRAM_ARGUMENTS']").collect {|i| i['value']}.to_s
      @vm_arguments = @doc.search("stringAttribute[@key='org.eclipse.jdt.launching.VM_ARGUMENTS']").collect {|i| i['value']}.to_s
    end
  end
{'fraction_non_alphanumeric': 0.1531961292081232, 'fraction_numerical': 0.0021807278179092273, 'mean_word_length': 3.4553734061930785, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5874212', 'n_tokens_mistral': 2501, 'n_tokens_neox': 2472, 'n_words': 400}