<commit_before>require 'csv'
require 'httparty'
require 'rss'
require 'open-uri'
require 'logger'
require_relative 'vending_machine'
require_relative 'trigger'

class FeedMonitor

  FileUtils.mkdir_p 'log'
  FileUtils.touch 'log/monitor.log'
  LOG = Logger.new('log/monitor.log')
  LOG.level = Logger::INFO

  def self.keywords
    c = CSV.read 'config/keywords.csv'
    c.map!{ |i| i.first }
  end

  def self.flavours
    VendingMachine.flavours
  end

  def self.perform
    LOG.info "I am running now - looking for headlines..."
    last_run_headlines = (CSV.read 'config/headlines.csv').map { |line| line[0] }
    headlines = []
    url = 'http://feeds.bbci.co.uk/news/rss.xml'
    open(url) do |rss|
      feed = RSS::Parser.parse(rss)
      feed.items.each do |item|
        triggered = false
        unless last_run_headlines.include? item.title
          keywords.each do |kw|
            if item.title =~ /\b#{kw}\b/
              trigger(item, kw)
              triggered = true
            end
          end
        end

        headlines << "\"#{item.pubDate}\",\"#{item.title}\",#{item.link},#{triggered}"
      end
    end

    out = File.open 'config/headlines.csv', 'w'
    headlines.each do |headline|
      out.write headline
      out.write "\n"
    end
    out.close
  end

  def self.trigger(item, kw)
    if date_stamp.nil? || item.pubDate > date_stamp
      LOG.info "'#{item.title}' matched '#{kw}'"
      flav = self.flavours.keys.sample
      LOG.info "Dispensing #{flav}"
      File.write(marker, Marshal.dump(item.pubDate))
      HTTParty.post('http://localhost:9292/dispense', body: { flavour: flav })
      Trigger.perform(item.link, flav)
    end<commit_msg>Remove headlines csv<commit_after>require 'csv'
require 'httparty'
require 'rss'
require 'open-uri'
require 'logger'
require_relative 'vending_machine'
require_relative 'trigger'

class FeedMonitor

  FileUtils.mkdir_p 'log'
  FileUtils.touch 'log/monitor.log'
  LOG = Logger.new('log/monitor.log')
  LOG.level = Logger::INFO

  def self.keywords
    c = CSV.read 'config/keywords.csv'
    c.map!{ |i| i.first }
  end

  def self.flavours
    VendingMachine.flavours
  end

  def self.perform
    LOG.info "I am running now - looking for headlines..."
    last_run_headlines = (CSV.read 'config/headlines.csv').map { |line| line[0] }
    headlines = []
    url = 'http://feeds.bbci.co.uk/news/rss.xml'
    open(url) do |rss|
      feed = RSS::Parser.parse(rss)
      feed.items.each do |item|
        triggered = false
        unless last_run_headlines.include? item.title
          keywords.each do |kw|
            if item.title =~ /\b#{kw}\b/
              trigger(item, kw)
              triggered = true
            end
          end
        end

        headlines << "\"#{item.pubDate}\",\"#{item.title}\",#{item.link},#{triggered}"
      end
    end

    FileUtils.rm 'config/headlines.csv'
    out = File.open 'config/headlines.csv', 'w'
    headlines.each do |headline|
      out.write headline
      out.write "\n"
    end
    out.close
  end

  def self.trigger(item, kw)
    if date_stamp.nil? || item.pubDate > date_stamp
      LOG.info "'#{item.title}' matched '#{kw}'"
      flav = self.flavours.keys.sample
      LOG.info "Dispensing #{flav}"
      File.write(marker, Marshal.dump(item.pubDate))
      HTTParty.post('http://localhost:9292/dispense', body: { flavour: flav })
      Trigger.perform(item.link, flav)
    end
{'fraction_non_alphanumeric': 0.13977239568135397, 'fraction_numerical': 0.0029180040852057193, 'mean_word_length': 2.4979591836734696, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5789080', 'n_tokens_mistral': 1185, 'n_tokens_neox': 1160, 'n_words': 251}