<commit_before>require 'db'

class Linkbot
  class Dupe
    def self.regexp
      Regexp.new('(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))')
    end
  
    def self.check_dupe(user, url)
      return unless url =~ self.regexp
    
      messages = []
      rows = Linkbot.db.execute("select user_id, url from links where url = '#{url}'")
      if rows.empty?
        Linkbot::Plugin.plugins.each {|k,v|
          messages << v[:ptr].on_newlink(user,url).join("\n") if(v[:ptr].respond_to?(:on_newlink)) 
        }
        Linkbot.db.execute("insert into links (user_id, dt, url) VALUES ('#{user['id']}', '#{Time.now.getgm.to_i}', '#{url}')")
      else
        Linkbot::Plugin.plugins.each {|k,v|
          messages << v[:ptr].on_dupe(user,url).join("\n") if(v[:ptr].respond_to?(:on_dupe)) 
        }
      end  
        s = messages.join("\n")
        print "\n----\n" + s + "\n---\n" if s.length > 1
        Linkbot.msg LINKCHAT, s if defined?(LINKCHAT) && s.length > 1
    end
  
    if Linkbot.db.table_info('links').empty?
      db.execute('CREATE TABLE links (user_id INTEGER, dt DATETIME, url TEXT)');
    end
    if Linkbot.db.table_info('stats').empty?
      Linkbot.db.execute('CREATE TABLE stats (user_id INTEGER, dupes INTEGER, total INTEGER)');
    end
  end
end<commit_msg>missed during the last db change<commit_after>require 'db'

class Linkbot
  class Dupe
    def self.regexp
      Regexp.new('(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))')
    end
  
    def self.check_dupe(user, url)
      return unless url =~ self.regexp
    
      messages = []
      rows = Linkbot.db.execute("select user_id, url from links where url = '#{url}'")
      if rows.empty?
        Linkbot::Plugin.plugins.each {|k,v|
          messages << v[:ptr].on_newlink(user,url).join("\n") if(v[:ptr].respond_to?(:on_newlink)) 
        }
        Linkbot.db.execute("insert into links (user_id, dt, url) VALUES ('#{user['id']}', '#{Time.now.getgm.to_i}', '#{url}')")
      else
        Linkbot::Plugin.plugins.each {|k,v|
          messages << v[:ptr].on_dupe(user,url).join("\n") if(v[:ptr].respond_to?(:on_dupe)) 
        }
      end  
        s = messages.join("\n")
        print "\n----\n" + s + "\n---\n" if s.length > 1
        Linkbot.msg LINKCHAT, s if defined?(LINKCHAT) && s.length > 1
    end
  
    if Linkbot.db.table_info('links').empty?
      Linkbot.db.execute('CREATE TABLE links (user_id INTEGER, dt DATETIME, url TEXT)');
    end
    if Linkbot.db.table_info('stats').empty?
      Linkbot.db.execute('CREATE TABLE stats (user_id INTEGER, dupes INTEGER, total INTEGER)');
    end
  end
end
{'fraction_non_alphanumeric': 0.26532009585758304, 'fraction_numerical': 0.008216364258815475, 'mean_word_length': 2.78988326848249, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1008149', 'n_tokens_mistral': 1182, 'n_tokens_neox': 1155, 'n_words': 196}