<commit_before>module Etsource
  ETSOURCE_DIR = 'etsource'

  class Commit
    attr_reader :commit

    def initialize(commit)
      @git = Git.open('etsource')
      @commit = @git.gcommit(commit)
      @git.checkout(commit)
    end

    def import!
      Gquery.transaction do 
        GqlTestCases.new.import!
        Gqueries.new.import!
        Inputs.new.import!
      end
      # DEBT fix this properly
      `curl http://beta.et-model.com/pages/refresh_gqueries > /dev/null`
    end
  end
  
  class Inputs
    def import!
      # Do not delete inputs because input ids are important and referenced by et-model
      import
    end
    
    def export
      base_dir = "#{ETSOURCE_DIR}/inputs"

      FileUtils.mkdir_p(base_dir)
      Input.find_each do |input|
        attrs = input.attributes
        attrs.delete('created_at')
        attrs.delete('updated_at')
        File.open("#{base_dir}/#{input.key}.yml", 'w') do |f|
          f << YAML::dump(attrs)
        end
      end
    end
    
    def import
      base_dir = "#{ETSOURCE_DIR}/inputs"
      
      Dir.glob("#{base_dir}/**/*.yml").each do |f|
        attributes = YAML::load_file(f)
        begin
          input = Input.find(attributes.delete('id'))
          input.update_attributes(attributes)
        rescue ActiveRecord::RecordNotFound
          Rails.logger.debug "*** ETSource::Inputs#import: Input not found"
        end
      end
    end
  end

  class GqlTestCases
    def import!
      GqlTestCase.delete_all
      import
    end
    
    def import
      base_dir = "#{ETSOURCE_DIR}/test_suites"
      Dir.glob("#{base_dir}/**/*.js").each do |f|
        key = f.split('/').last.split('.').first
        GqlTestCase.create(:name => key, :instruction => File.read(f))
      end
    end
  end

  class Gqueries
    VARIABLE_PREFIX = '-'
    FILE_SUFFIX = 'gql'
    
    def import!
      Gquery.transaction do
        Gquery.delete_all
        GqueryGroup.delete_all
        import
        Rails.cache.clear
        system("touch tmp/restart.txt")
      end
    end

    def export
      base_dir = "#{ETSOURCE_DIR}/gqueries"
      Gquery.includes(:gquery_groups).all.each do |gquery|
        group = group_key(gquery.gquery_groups.first)
        path = [base_dir, group].compact.join('/')

        FileUtils.mkdir_p(path)
        File.open("#{path}/#{gquery.key}.#{FILE_SUFFIX}", 'w') do |f|
          f << to_file(gquery)
        end
      end
    end

    def import
      base_dir = "#{ETSOURCE_DIR}/gqueries"
      groups = GqueryGroup.all.inject({}) {|hsh,g| hsh.merge group_key(g) => g}
      
      gqueries = []
      Dir.glob("#{base_dir}/**/*.#{FILE_SUFFIX}").each do |f|
        group_key, rest = f.gsub(base_dir+"/", '').split('/')
        gquery = from_file(f)
        groups[group_key] ||= GqueryGroup.create(:group_key => group_key)
        gquery.gquery_group_id = groups[group_key].id
        gqueries << gquery
      end
      Gquery.import gqueries
    end

  #protected
    def to_file(gquery)
      commented_description = "# #{gquery.description.to_s.strip.gsub("\n", "\n# ")}\n\n"
      unit = gquery.unit.to_s.downcase.strip.gsub(' ', '_')
      unit = 'boolean' if unit.include?('true')
      <commit_msg>better gquery group names importing from etsource<commit_after>module Etsource
  ETSOURCE_DIR = 'etsource'

  class Commit
    attr_reader :commit

    def initialize(commit)
      @git = Git.open('etsource')
      @commit = @git.gcommit(commit)
      @git.checkout(commit)
    end

    def import!
      Gquery.transaction do 
        GqlTestCases.new.import!
        Gqueries.new.import!
        Inputs.new.import!
      end
      # DEBT fix this properly
      `curl http://beta.et-model.com/pages/refresh_gqueries > /dev/null`
    end
  end
  
  class Inputs
    def import!
      # Do not delete inputs because input ids are important and referenced by et-model
      import
    end
    
    def export
      base_dir = "#{ETSOURCE_DIR}/inputs"

      FileUtils.mkdir_p(base_dir)
      Input.find_each do |input|
        attrs = input.attributes
        attrs.delete('created_at')
        attrs.delete('updated_at')
        File.open("#{base_dir}/#{input.key}.yml", 'w') do |f|
          f << YAML::dump(attrs)
        end
      end
    end
    
    def import
      base_dir = "#{ETSOURCE_DIR}/inputs"
      
      Dir.glob("#{base_dir}/**/*.yml").each do |f|
        attributes = YAML::load_file(f)
        begin
          input = Input.find(attributes.delete('id'))
          input.update_attributes(attributes)
        rescue ActiveRecord::RecordNotFound
          Rails.logger.debug "*** ETSource::Inputs#import: Input not found"
        end
      end
    end
  end

  class GqlTestCases
    def import!
      GqlTestCase.delete_all
      import
    end
    
    def import
      base_dir = "#{ETSOURCE_DIR}/test_suites"
      Dir.glob("#{base_dir}/**/*.js").each do |f|
        key = f.split('/').last.split('.').first
        GqlTestCase.create(:name => key, :instruction => File.read(f))
      end
    end
  end

  class Gqueries
    VARIABLE_PREFIX = '-'
    FILE_SUFFIX = 'gql'
    
    def import!
      Gquery.transaction do
        Gquery.delete_all
        GqueryGroup.delete_all
        import
        Rails.cache.clear
        system("touch tmp/restart.txt")
      end
    end

    def export
      base_dir = "#{ETSOURCE_DIR}/gqueries"
      Gquery.includes(:gquery_groups).all.each do |gquery|
        group = group_key(gquery.gquery_groups.first)
        path = [base_dir, group].compact.join('/')

        FileUtils.mkdir_p(path)
        File.open("#{path}/#{gquery.key}.#{FILE_SUFFIX}", 'w') do |f|
          f << to_file(gquery)
        end
      end
    end

    def import
      base_dir = "#{ETSOURCE_DIR}/gqueries"
      groups = GqueryGroup.all.inject({}) {|hsh,g| hsh.merge group_key(g) => g}
      
      gqueries = []
      Dir.glob("#{base_dir}/**/*.#{FILE_SUFFIX}").each do |f|
        tokens = f.gsub(base_dir+"/", '').split('/')
        # the group name concatenates the directory names
        group_key = tokens[0..-2].join('_').gsub(' ', '_') rescue nil
        gquery = from_file(f)
        groups[group_key] ||= GqueryGroup.create(:group_key => group_key)
        gquery.gquery_group_id = groups[group_key].id
        gqueries << gquery
      end
      Gquery.import gqueries
    end

  #protected
    def to_file(gquery)
      commented_description = "# #{gquery.description.to_s.strip.gsub("\n", "\n# ")}\n\n"
      unit = gquery.unit.to_s.downcase.strip.gsub(' ', '_')
      unit = 'boolean' if unit.include?('true')
      
{'fraction_non_alphanumeric': 0.1322703113135915, 'fraction_numerical': 0.00030372057706909645, 'mean_word_length': 2.1847195357833655, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2720834', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2186, 'n_words': 422}