<commit_before># 下载器的主要职责是抓取网页并将网页内容返还给蜘蛛(Spiders)

require 'active_record'
require 'mysql2'

module FreeSpider
  module Downloader
  	ActiveRecord::Base.logger = Logger.new(STDERR)

    # 数据库配置文件
    # spec = Gem::Specification.find_by_name("database")
    # gem_root = spec.gem_dir
    environment = ENV['RACK_ENV'] || 'development'
    dbconfig = YAML.load(File.read("#{Gem.paths.home}/gems/free_spider-0.0.2/lib/free_spider/config/database.yml"))

		# 链接数据库
		puts "----database_connection-----"
  	ActiveRecord::Base.establish_connection(dbconfig[environment])

  	# 创建表结构
  	puts "----table_create-----"
    ActiveRecord::Schema.define do
		  unless ActiveRecord::Base.connection.tables.include? 'news_teachings'
		    create_table :news_teachings do |table|
		      table.column :title,     :string
		      table.column :content,   :text
		    end
		  end
		end

  	class NewsTeaching < ActiveRecord::Base
  		validates_presence_of :title, :content
  	end

  end
end<commit_msg>数据库配置<commit_after># 下载器的主要职责是抓取网页并将网页内容返还给蜘蛛(Spiders)

require 'active_record'
require 'mysql2'

module FreeSpider
  module Downloader
  	ActiveRecord::Base.logger = Logger.new(STDERR)

    # 数据库配置文件
    # spec = Gem::Specification.find_by_name("database")
    # gem_root = spec.gem_dir
    # environment = ENV['RACK_ENV'] || 'development'
    # dbconfig = YAML.load(File.read("#{Gem.paths.home}/gems/free_spider-0.0.2/lib/free_spider/config/database.yml"))

		# 链接数据库
		puts "----database_connection-----"
  	ActiveRecord::Base.establish_connection(dbconfig[environment])

  	# 创建表结构
  	puts "----table_create-----"
    ActiveRecord::Schema.define do
		  unless ActiveRecord::Base.connection.tables.include? 'news_teachings'
		    create_table :news_teachings do |table|
		      table.column :title,     :string
		      table.column :content,   :text
		    end
		  end
		end

  	class NewsTeaching < ActiveRecord::Base
  		validates_presence_of :title, :content
  	end

  end
end
{'fraction_non_alphanumeric': 0.12512716174974567, 'fraction_numerical': 0.004069175991861648, 'mean_word_length': 3.5427251732101617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7460787', 'n_tokens_mistral': 782, 'n_tokens_neox': 756, 'n_words': 111}