<commit_before>class Source < ActiveRecord::Base
  
  has_many :data_files, :dependent => :destroy
  has_many :data_file_filters, :dependent => :destroy
  
  def parsed_options
    output = {}
    options.scan(/(.*?):\"(.*?)\"/i).each do |label,value|      
      output[label.to_sym] = value
    end
    return output
  end
  
  def uri
    URI.parse(location)
  end
  
  def source_type_object
    SourceTypes::SourceType.build(source_type)
  end
  
  def data options={}
    SCRAPER_LOG.info( "Gathering data from: '#{name}'" )
    self.update_attribute(:last_scraped_at, Time.now) unless options[:manual_scrape]
    source_type_object.data(location,self)    
  end
  
  def download location
    to = download_location.blank? ? Setting.get("DEFAULT_DOWNLOAD_LCOATION") : download_location
    SCRAPER_LOG.info( "Downloading #{location} to '#{to}'" )
    source_type_object.download(location,to)
  end
  
end<commit_msg>Fixed type<commit_after>class Source < ActiveRecord::Base
  
  has_many :data_files, :dependent => :destroy
  has_many :data_file_filters, :dependent => :destroy
  
  def parsed_options
    output = {}
    options.scan(/(.*?):\"(.*?)\"/i).each do |label,value|      
      output[label.to_sym] = value
    end
    return output
  end
  
  def uri
    URI.parse(location)
  end
  
  def source_type_object
    SourceTypes::SourceType.build(source_type)
  end
  
  def data options={}
    SCRAPER_LOG.info( "Gathering data from: '#{name}'" )
    self.update_attribute(:last_scraped_at, Time.now) unless options[:manual_scrape]
    source_type_object.data(location,self)    
  end
  
  def download location
    to = download_location.blank? ? Setting.get("DEFAULT_DOWNLOAD_PATH") : download_location
    SCRAPER_LOG.info( "Downloading #{location} to '#{to}'" )
    source_type_object.download(location,to)
  end
  
end
{'fraction_non_alphanumeric': 0.12071778140293637, 'fraction_numerical': 0.0, 'mean_word_length': 3.6231155778894473, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512245', 'n_tokens_mistral': 632, 'n_tokens_neox': 623, 'n_words': 120}