<commit_before>#!/usr/bin/env ruby
# encoding: utf-8

require 'rubygems'
require 'Bio'
require 'hpricot'

# always tell NCBI who they're messing with
Bio::NCBI.default_email = "<EMAIL>"

ncbi = Bio::NCBI::REST.new
file = File.new(ARGV.pop, 'r')
records = []

file.each do |line|
  if line.include? '>'
    id = line.split('|')[3]
    records << id
  end
end

records.uniq!

while records.length > 0
  query = records.pop(100)
  retries = 5
  begin
    rec = ncbi.efetch(query.join(','), {"db"=>"nucleotide", "rettype"=>"gb", "retmode"=>"xml"})
  rescue SocketError
    if (retries -= 1) > 0
      retry
    else
      next
    end
  end
  doc = Hpricot::XML(rec)
  numb = query.pop
  
  doc.search('GBSeq').each do |result|
    gi = result.search('GBSeq_accession-version>')
    result.search('GBQualifier_value').each do |h|
      if h.to_s.include? 'taxon:'
        a = h.to_s.split(':')
        a = a[1].split('</')
        id = a[0]
        puts '%s => %s' % [gi, id]
      end
    end
  end
end<commit_msg>nothing. just refactorage<commit_after>#!/usr/bin/env ruby
# encoding: utf-8

require 'rubygems'
require 'Bio'
require 'hpricot'

# always tell NCBI who they're messing with
Bio::NCBI.default_email = "<EMAIL>"

if ARGV.length == 0
  $stdout.puts "usage #{__FILE__} fasta_file_with_accessions > output"
  exit
end

# Get accession numbers
records = []
File.open(ARGV.pop) do |handle|
  handle.each do |line|
    if line[0] == ">"
      id = line.split('|')[3]
      records << id
    end
  end
end
records.uniq!
puts records
# Query NCBI
ncbi = Bio::NCBI::REST.new

while records.length > 0
  query = records.pop 100
  retries = 5
  begin # get record
    rec = ncbi.efetch(query.join(','),
      {"db" => "nucleotide", "rettype" => "gb", "retmode" => "xml"})
  rescue SocketError # NCBI sometimes messes up
    if (retries -= 1) > 0
      retry
    else
      next
    end
  end
  
  # Parse XML reply
  doc = Hpricot::XML(rec)
  numb = query.pop
  
  # Do some weird obfuscated hpricot shit to get the taxid
  doc.search('GBSeq').each do |result|
    gi = result.search('GBSeq_accession-version>')
    result.search('GBQualifier_value').each do |h|
      if h.to_s.include? 'taxon:'
        a = h.to_s.split(':')
        a = a[1].split('</')
        id = a[0]
        
        # print to STDOUT
        puts '%s => %s' % [gi, id]
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.1574504010130857, 'fraction_numerical': 0.010130856901646263, 'mean_word_length': 2.703125, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7344863', 'n_tokens_mistral': 943, 'n_tokens_neox': 902, 'n_words': 227}