<commit_before>require 'sequel'
require 'yaml'
require 'uri'
require 'trollop'
require './snapshot'

class Database
  attr_accessor :opts, :db, :urls, :links, :queue, :SHELF, :domain, :start_time, :output, :crawled
  def initialize(opts)
    # Load configuration file
    # Used to load the base domain to be crawled, and the path to the database
    if opts[:config]
      # Use the given config file if it exists, otherwise create one
      if File.exists?(opts[:config])
        config = YAML::load(File.open(opts[:config]))
      else
        config = YAML::load(File.open(create_config))
      end
    # Else load the default config file
    else
      config = YAML::load(File.open('config.yaml'))
    end

    if opts[:init] then config = YAML::load(File.open(create_config)) end

    if config['db_path']
      @db = Sequel.connect(config['db_path'])
    else
      @db = Sequel.connect(:adapter => 'mysql', :user => config['user'], :socket => config['sock'], :database => config['name'], :password => config['<PASSWORD>'])
    end
    @crawled = 0
    @avg_response = 0
    @urls = db[:urls]
    @links = db[:links]
    @queue = db[:queue]
    @domain = config['domain']
    @start_time = Time.now<commit_msg>Added support for exceptions.<commit_after>require 'sequel'
require 'yaml'
require 'uri'
require 'trollop'
require './snapshot'

class Database
  attr_accessor :opts, :db, :urls, :links, :queue, :SHELF, :domain, :start_time, :output, :crawled
  def initialize(opts)
    # Load configuration file
    # Used to load the base domain to be crawled, and the path to the database
    if opts[:config]
      # Use the given config file if it exists, otherwise create one
      if File.exists?(opts[:config])
        config = YAML::load(File.open(opts[:config]))
      else
        config = YAML::load(File.open(create_config))
      end
    # Else load the default config file
    else
      config = YAML::load(File.open('config.yaml'))
    end

    if opts[:init] then config = YAML::load(File.open(create_config)) end

    if config['db_path']
      @db = Sequel.connect(config['db_path'])
    else
      @db = Sequel.connect(:adapter => 'mysql', :user => config['user'], :socket => config['sock'], :database => config['name'], :password => config['<PASSWORD>'])
    end
    @crawled = 0
    @avg_response = 0
    @urls = db[:urls]
    @links = db[:links]
    @queue = db[:queue]
    @domain = config['domain']
    @exceptions = config['exceptions']
    @start_time = Time.now
{'fraction_non_alphanumeric': 0.14572460859092734, 'fraction_numerical': 0.0016057808109193096, 'mean_word_length': 3.4184397163120566, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1811933', 'n_tokens_mistral': 797, 'n_tokens_neox': 784, 'n_words': 226}