<commit_before>require 'resque'

class Parsing
  @queue = :parse

  def self.perform(genotype_id, temp_file)
    Rails.logger.level = 0
    Rails.logger = Logger.new("#{Rails.root}/log/parsing_#{Rails.env}.log")
    genotype_id = genotype_id["genotype"]["id"].to_i if genotype_id.is_a?(Hash)
    @genotype = Genotype.find(genotype_id)
    
    if @genotype.filetype != "other"



      # IYG filetype needs proper dbSNP-names from marshalled file
      if @genotype.filetype == "IYG"
        db_snp_snps = {"MT-T3027C"=>"rs199838004", "MT-T4336C"=>"rs41456348", "MT-G4580A"=>"rs28357975", "MT-T5004C"=>"rs41419549", "MT-C5178a"=>"rs28357984", "MT-A5390G"=>"rs41333444", "MT-C6371T"=>"rs41366755", "MT-G8697A"=>"rs28358886", "MT-G9477A"=>"rs2853825", "MT-G10310A"=>"rs41467651", "MT-A10550G"=>"rs28358280", "MT-C10873T"=>"rs2857284", "MT-C11332T"=>"rs55714831", "MT-A11947G"=>"rs28359168", "MT-A12308G"=>"rs2853498", "MT-A12612G"=>"rs28359172", "MT-T14318C"=>"rs28357675", "MT-T14766C"=>"rs3135031", "MT-T14783C"=>"rs28357680"}
      end

      genotype_file = File.open(temp_file, "r")
      log "Loading known Snps."
      known_snps = {}
      Snp.find_each do |s| known_snps[s.name] = true end
      
      known_user_snps = {}  
      UserSnp.where("user_id" => @genotype.user_id).find_each do |us| known_user_snps[us.snp_name] = true end
        
      new_snps = []
      new_user_snps = []

      log "Parsing file #{temp_file}"
      # open that file, go through each line
      genotype_file.each do |single_snp|
        next if single_snp[0] == "#" 

        # make a nice array if line is no comment
        if @genotype.filetype == "IYG"
          prior_snp_array = single_snp.split("\t")
          name = prior_snp_array[0]
          if name.starts_with? "MT"
            # check whether it's in db_snp_snps, use that name
            position = name.tr('0-9','') # MT-G1234G -> 1234

            if db_snp_snps[name] # do we have a dbSNP-name?
                name = db_snp_snps[name]
            end
            
            snp_array = [name, "MT", position, prior_snp_array[1]]
          else
            snp_array = [prior_snp_array[0], "1", "1", prior_snp_array[1]]
          end
        elsif @genotype.filetype == "23andme"
          snp_array = single_snp.split("\t")

        elsif @genotype.filetype == "decodeme"
          temp_array = single_snp.split(",")
          if temp_array[0] != "Name"
            snp_array = [temp_array[0],temp_array[2],temp_array[3],temp_array[5]]
          else
            next
          end
          
        elsif @genotype.filetype == "ftdna-illumina"
          temp_array = single_snp.split("\",\"")<commit_msg>Proper position-parsing in YIG-parser<commit_after>require 'resque'

class Parsing
  @queue = :parse

  def self.perform(genotype_id, temp_file)
    Rails.logger.level = 0
    Rails.logger = Logger.new("#{Rails.root}/log/parsing_#{Rails.env}.log")
    genotype_id = genotype_id["genotype"]["id"].to_i if genotype_id.is_a?(Hash)
    @genotype = Genotype.find(genotype_id)
    
    if @genotype.filetype != "other"



      # IYG filetype needs proper dbSNP-names from marshalled file
      if @genotype.filetype == "IYG"
        db_snp_snps = {"MT-T3027C"=>"rs199838004", "MT-T4336C"=>"rs41456348", "MT-G4580A"=>"rs28357975", "MT-T5004C"=>"rs41419549", "MT-C5178a"=>"rs28357984", "MT-A5390G"=>"rs41333444", "MT-C6371T"=>"rs41366755", "MT-G8697A"=>"rs28358886", "MT-G9477A"=>"rs2853825", "MT-G10310A"=>"rs41467651", "MT-A10550G"=>"rs28358280", "MT-C10873T"=>"rs2857284", "MT-C11332T"=>"rs55714831", "MT-A11947G"=>"rs28359168", "MT-A12308G"=>"rs2853498", "MT-A12612G"=>"rs28359172", "MT-T14318C"=>"rs28357675", "MT-T14766C"=>"rs3135031", "MT-T14783C"=>"rs28357680"}
      end

      genotype_file = File.open(temp_file, "r")
      log "Loading known Snps."
      known_snps = {}
      Snp.find_each do |s| known_snps[s.name] = true end
      
      known_user_snps = {}  
      UserSnp.where("user_id" => @genotype.user_id).find_each do |us| known_user_snps[us.snp_name] = true end
        
      new_snps = []
      new_user_snps = []

      log "Parsing file #{temp_file}"
      # open that file, go through each line
      genotype_file.each do |single_snp|
        next if single_snp[0] == "#" 

        # make a nice array if line is no comment
        if @genotype.filetype == "IYG"
          prior_snp_array = single_snp.split("\t")
          name = prior_snp_array[0]
          if name.starts_with? "MT"
            # check whether it's in db_snp_snps, use that name
            position = name.tr('A-Za-z-','') # MT-G1234G -> 1234

            if db_snp_snps[name] # do we have a dbSNP-name?
                name = db_snp_snps[name]
            end
            
            snp_array = [name, "MT", position, prior_snp_array[1]]
          else
            snp_array = [prior_snp_array[0], "1", "1", prior_snp_array[1]]
          end
        elsif @genotype.filetype == "23andme"
          snp_array = single_snp.split("\t")

        elsif @genotype.filetype == "decodeme"
          temp_array = single_snp.split(",")
          if temp_array[0] != "Name"
            snp_array = [temp_array[0],temp_array[2],temp_array[3],temp_array[5]]
          else
            next
          end
          
        elsif @genotype.filetype == "ftdna-illumina"
          temp_array = single_snp.split("\",\"")
{'fraction_non_alphanumeric': 0.15562174236783322, 'fraction_numerical': 0.0964259121370067, 'mean_word_length': 2.7055172413793103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2313888', 'n_tokens_mistral': 2411, 'n_tokens_neox': 2097, 'n_words': 344}