<commit_before>class Bot::Plugin::Image < Bot::Plugin
  require 'digest/md5'
  require 'digest/sha2'
  require 'uri'
  require 'open-uri'
  require 'fileutils'
  require 'nokogiri'

  def initialize(bot)
    @s = {
      trigger: {
        image: [:random_image_link, 0, 'Image scraper plugin. Call this for a random image.'],
        images: [:images_link, 0, 'Gets the images listing webpage.']
      },
      subscribe: true,
      filters: ['http.*png', 'http.*gif', 'http.*jpg', 'http.*jpeg', 'http.*bmp', 'http.*webm'],
      relative_database_path: ['lib', 'databases', 'images.sqlite3'],
      image_directory: ['lib', 'public', 'i'],
      get_google_guess: false
    }
    super(bot)

    @db = Bot::Database.new(File.join(@s[:relative_database_path]))

    if !@db.table_exists?(:images)
      @db.create_table :images do
        primary_key :id
        String      :sha256, size: 64
        String      :md5
        String      :path
        String      :google_description
        Integer     :bytesize
      end
    end

    if !@db.table_exists?(:sources)
      @db.create_table :sources do
        primary_key :id
        foreign_key :image_id, :images, key: :id
        DateTime    :timestamp
        String      :url
        String      :source_name
        String      :context
        String      :filename
      end<commit_msg>Image: Add mp4 to default filter<commit_after>class Bot::Plugin::Image < Bot::Plugin
  require 'digest/md5'
  require 'digest/sha2'
  require 'uri'
  require 'open-uri'
  require 'fileutils'
  require 'nokogiri'

  def initialize(bot)
    @s = {
      trigger: {
        image: [:random_image_link, 0, 'Image scraper plugin. Call this for a random image.'],
        images: [:images_link, 0, 'Gets the images listing webpage.']
      },
      subscribe: true,
      filters: ['http.*png', 'http.*gif', 'http.*jpg', 'http.*jpeg', 'http.*bmp', 'http.*webm', 'http.*mp4'],
      relative_database_path: ['lib', 'databases', 'images.sqlite3'],
      image_directory: ['lib', 'public', 'i'],
      get_google_guess: false
    }
    super(bot)

    @db = Bot::Database.new(File.join(@s[:relative_database_path]))

    if !@db.table_exists?(:images)
      @db.create_table :images do
        primary_key :id
        String      :sha256, size: 64
        String      :md5
        String      :path
        String      :google_description
        Integer     :bytesize
      end
    end

    if !@db.table_exists?(:sources)
      @db.create_table :sources do
        primary_key :id
        foreign_key :image_id, :images, key: :id
        DateTime    :timestamp
        String      :url
        String      :source_name
        String      :context
        String      :filename
      end
{'fraction_non_alphanumeric': 0.1282991202346041, 'fraction_numerical': 0.008797653958944282, 'mean_word_length': 2.303874092009685, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4861849', 'n_tokens_mistral': 880, 'n_tokens_neox': 849, 'n_words': 215}