<commit_before>require 'json'
require 'tempfile'
require 'zlib'

class Genome < ActiveRecord::Base
  has_many :scaffolds
  has_many :features

  validates :assembly_id, numericality: { only_integer: true }, presence: true

  after_create do
    self.pull_from_ncbi
  end

  def pull_from_ncbi

    @genome = self

    # get metadata from ncbi
    ncbi_res = `bionode-ncbi search assembly #{self.assembly_id}`
    p ncbi_res
    dat = JSON.parse(ncbi_res)

    # remove features and scaffolds if there are any
    @genome.features.delete_all
    @genome.scaffolds.delete_all

    @genome.update organism: dat['organism']

    dir = Dir.mktmpdir self.assembly_id.to_s
    Dir.chdir dir

    fna = JSON.parse(`bionode-ncbi download assembly #{self.assembly_id}`.split("\n").last)
    gff = JSON.parse(`bionode-ncbi download gff #{self.assembly_id}`.split("\n").last)

    # has to be loaded before features so that features can reference a
    # scaffold
    read_scaffolds(fna["path"]).each_slice(10) do |scaffolds|
      scaffolds.map! { |x| x.genome = @genome; x }
      Scaffold.import scaffolds
    end

    # has to be loaded after scaffolds
    read_features(gff["path"]).each_slice(1_000) do |features|
      features.map! { |x| x.genome = @genome; x }
      Feature.import features
    end

  end

#  handle_asynchronously :pull_from_ncbi
end
<commit_msg>fixed job runner :)<commit_after>require 'json'
require 'tempfile'
require 'zlib'

class Genome < ActiveRecord::Base
  has_many :scaffolds
  has_many :features

  validates :assembly_id, numericality: { only_integer: true }, presence: true

  after_create do
    self.pull_from_ncbi
  end

  def pull_from_ncbi

    @genome = self

    # get metadata from ncbi
    ncbi_res = `bionode-ncbi search assembly #{self.assembly_id}`
    p ncbi_res
    dat = JSON.parse(ncbi_res)

    # remove features and scaffolds if there are any
    @genome.features.delete_all
    @genome.scaffolds.delete_all

    @genome.update organism: dat['organism']

    dir = Dir.mktmpdir self.assembly_id.to_s
    Dir.chdir dir

    fna = JSON.parse(`bionode-ncbi download assembly #{self.assembly_id}`.split("\n").last)
    gff = JSON.parse(`bionode-ncbi download gff #{self.assembly_id}`.split("\n").last)

    # has to be loaded before features so that features can reference a
    # scaffold
    read_scaffolds(fna["path"]).each_slice(10) do |scaffolds|
      scaffolds.map! { |x| x.genome = @genome; x }
      Scaffold.import scaffolds
    end

    # has to be loaded after scaffolds
    read_features(gff["path"]).each_slice(1_000) do |features|
      features.map! { |x| x.genome = @genome; x }
      Feature.import features
    end

  end

  handle_asynchronously :pull_from_ncbi
end

{'fraction_non_alphanumeric': 0.10451045104510451, 'fraction_numerical': 0.0044004400440044, 'mean_word_length': 3.44299674267101, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1161092', 'n_tokens_mistral': 991, 'n_tokens_neox': 893, 'n_words': 231}