<commit_before>require 'sequel'
require 'yaml'
require 'logging'

unless defined? DB
    basedir = File.dirname(__FILE__)
    config = YAML.load_file("#{basedir}/config.yml")

    if `hostname` =~ /^ip-/ and ENV['AHS_DATABASE_TYPE'].nil?
        ENV['AHS_DATABASE_TYPE']='mysql'
    end

    mode = nil
    if ENV['AHS_DATABASE_TYPE'].nil? or 
      !(["mysql", "sqlite"].include? ENV['AHS_DATABASE_TYPE'])
        puts "environment variable AHS_DATABASE_TYPE must be set to"
        puts "mysql or sqlite."
        exit
    else
        mode = ENV['AHS_DATABASE_TYPE'].to_sym
    end

    url = nil
    if mode == :mysql
        url = config['mysql_url']
    else
        url = "sqlite://#{basedir}/#{config['sqlite_filename']}"
    end

    DB = Sequel.connect(url)
end

DB.loggers << Logger.new($stdout)
<commit_msg>correct lib name<commit_after>require 'sequel'
require 'yaml'
require 'logger'

unless defined? DB
    basedir = File.dirname(__FILE__)
    config = YAML.load_file("#{basedir}/config.yml")

    if `hostname` =~ /^ip-/ and ENV['AHS_DATABASE_TYPE'].nil?
        ENV['AHS_DATABASE_TYPE']='mysql'
    end

    mode = nil
    if ENV['AHS_DATABASE_TYPE'].nil? or 
      !(["mysql", "sqlite"].include? ENV['AHS_DATABASE_TYPE'])
        puts "environment variable AHS_DATABASE_TYPE must be set to"
        puts "mysql or sqlite."
        exit
    else
        mode = ENV['AHS_DATABASE_TYPE'].to_sym
    end

    url = nil
    if mode == :mysql
        url = config['mysql_url']
    else
        url = "sqlite://#{basedir}/#{config['sqlite_filename']}"
    end

    DB = Sequel.connect(url)
end

DB.loggers << Logger.new($stdout)

{'fraction_non_alphanumeric': 0.14268292682926828, 'fraction_numerical': 0.0, 'mean_word_length': 2.8611764705882354, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6756932', 'n_tokens_mistral': 591, 'n_tokens_neox': 582, 'n_words': 124}