<commit_before>#!/usr/bin/env ruby19

# goodlog traffic monitor

require 'eventmachine'
require 'eventmachine-tail'
require 'haml'
require 'fileutils'
require 'json'
require 'pg'

class Traffic
  attr_reader :config

  def initialize
    @host_traffic = {}
    @config = eval(File.read('traffic.conf.rb'))
    @frontend_dir = @config[:frontend_dir]
    @frontend_dir += '/' unless @frontend_dir.match(/\/$/)
    puts "config: #{@config}"
    @db = PG::Connection.open(@config[:postgresql])
    begin
      @db.exec('CREATE TABLE traffic (
                  hostname VARCHAR(200),
                  month INTEGER,
                  year INTEGER,
                  requests INTEGER,
                  bytes INTEGER,
                  PRIMARY KEY (hostname, month, year)
                )')
    rescue PG::Error => e
      # table does exist
    end
    update_frontend_static
    update_frontend_data
  end

  def increment_host(hostname, bytes)
    @host_traffic[hostname] ||= {:requests => 0, :bytes => 0}
    @host_traffic[hostname][:requests] += 1
    @host_traffic[hostname][:bytes] += bytes
  end

  def update_stats
    puts "update stats"
    if not @host_traffic.empty?
      time = Time.now
      month, year = time.month, time.year
      @host_traffic.keys.each do |hostname|
        puts "#{hostname}: #{@host_traffic[hostname][:requests]} reqs / #{@host_traffic[hostname][:bytes]} bytes"
        result = @db.exec("UPDATE traffic SET requests = requests + #{@host_traffic[hostname][:requests]}, bytes = bytes + #{@host_traffic[hostname][:bytes]} " +
                          "WHERE hostname = '#{hostname}' AND month = #{month} AND year = #{year}")
        if result.cmd_tuples == 0
          @db.exec("INSERT INTO traffic (hostname, month, year, requests, bytes) VALUES ('#{hostname}', #{month}, #{year}, " +
                   "#{@host_traffic[hostname][:requests]}, #{@host_traffic[hostname][:bytes]})")
        end
      end
      update_frontend_data
      @host_traffic = {}
    end
  end

  def update_frontend_static
    puts "update frontend static"
    index = Haml::Engine.new(File.read('index.haml'))
    File.open(@frontend_dir + 'index.html', 'w') {|f| f.write index.render(self) }
    FileUtils.cp_r('js', @frontend_dir)
    FileUtils.cp_r('css', @frontend_dir)
    FileUtils.cp_r('img', @frontend_dir)
  end<commit_msg>parameterise DB queries<commit_after>#!/usr/bin/env ruby19

# goodlog traffic monitor

require 'eventmachine'
require 'eventmachine-tail'
require 'haml'
require 'fileutils'
require 'json'
require 'pg'

class Traffic
  attr_reader :config

  def initialize
    @host_traffic = {}
    @config = eval(File.read('traffic.conf.rb'))
    @frontend_dir = @config[:frontend_dir]
    @frontend_dir += '/' unless @frontend_dir.match(/\/$/)
    puts "config: #{@config}"
    @db = PG::Connection.open(@config[:postgresql])
    begin
      @db.exec('CREATE TABLE traffic (
                  hostname VARCHAR(200),
                  month INTEGER,
                  year INTEGER,
                  requests INTEGER,
                  bytes INTEGER,
                  PRIMARY KEY (hostname, month, year)
                )')
    rescue PG::Error => e
      # table does exist
    end
    update_frontend_static
    update_frontend_data
  end

  def increment_host(hostname, bytes)
    @host_traffic[hostname] ||= {:requests => 0, :bytes => 0}
    @host_traffic[hostname][:requests] += 1
    @host_traffic[hostname][:bytes] += bytes
  end

  def update_stats
    puts "update stats"
    if not @host_traffic.empty?
      time = Time.now
      month, year = time.month, time.year
      @host_traffic.keys.each do |hostname|
        puts "#{hostname}: #{@host_traffic[hostname][:requests]} reqs / #{@host_traffic[hostname][:bytes]} bytes"
        result = @db.exec_params('UPDATE traffic SET requests = requests + $1::int, bytes = bytes + $2::int ' +
                                 'WHERE hostname = $3 AND month = $4::int AND year = $5::int',
                                 [@host_traffic[hostname][:requests], @host_traffic[hostname][:bytes], hostname, month, year])
        if result.cmd_tuples == 0
          @db.exec_params('INSERT INTO traffic (hostname, month, year, requests, bytes) VALUES ($1, $2::int, $3::int, $4::int, $5::int)',
                          [hostname, month, year, @host_traffic[hostname][:requests], @host_traffic[hostname][:bytes]])
        end
      end
      update_frontend_data
      @host_traffic = {}
    end
  end

  def update_frontend_static
    puts "update frontend static"
    index = Haml::Engine.new(File.read('index.haml'))
    File.open(@frontend_dir + 'index.html', 'w') {|f| f.write index.render(self) }
    FileUtils.cp_r('js', @frontend_dir)
    FileUtils.cp_r('css', @frontend_dir)
    FileUtils.cp_r('img', @frontend_dir)
  end
{'fraction_non_alphanumeric': 0.13991683991683992, 'fraction_numerical': 0.0058212058212058215, 'mean_word_length': 2.7527301092043683, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2579714', 'n_tokens_mistral': 1558, 'n_tokens_neox': 1531, 'n_words': 339}