<commit_before>require 'eventmachine'
require 'yaml'
require 'logger'
require 'redis'
require 'restclient/components'
require 'rack/cache'
require 'redis-store'
require 'mongo_mapper'

require 'pushme/push'
require 'pushme/parser'
#require 'pushme/parsers'
require 'pushme/feed'

require 'choice'

Choice.options do
  option :config, :required => true do
    short '-c'
    long '--config=FILE'
    desc 'the file with configuration of pushme'
  end

  option :log do
    short '-l'
    long '--log=FILE'
    desc 'the file where all log are write'
    default STDOUT
  end

  option :jid, :required => true do
    long '--jid=jid'
    desc 'your jid listen'
  end

  option :jid_password, :required => true do
    long '--jid-password=jid-password'
    desc 'your password of your jid'
  end

  option :help do
    long '--help'
    desc 'Show this message'
  end
end

class ProxyLogger
  def initialize(logger)
    @logger = logger
  end

  def write(msg)
    @logger.info(msg)
  end

  def flush
    #nothing
  end

  def puts(msg)
    if msg.is_a?(Array)
      @logger.debug(msg.map(&:chomp).join("\n"))
    elsif msg.is_a?(String)
      @logger.debug(msg.chomp)
    else
      @logger.debug(msg)
    end
  end

  def method_missing(method, *args)
    @logger.send(method, *args)
  end
end

class LogError
  def initialize(app, logger)
    @app = app
    @logger = logger
  end

  def call(env)
    env['rack.errors'] = @logger
    @app.call(env)
  end
end



Pushme::OPTIONS = YAML.load_file(Choice.choices[:config])

Pushme::Logger = Logger.new(Choice.choices[:log])
Pushme::Logger.level = eval(Pushme::OPTIONS[:log_level]) || Logger::ERROR

MongoMapper.connection = Mongo::Connection.new(Pushme::OPTIONS[:mongodb][:host],
                                               Pushme::OPTIONS[:mongodb][:port],
                                               :logger => Pushme::Logger)
MongoMapper.database = Pushme::OPTIONS[:mongodb][:database]
MongoMapper.database.authenticate(Pushme::OPTIONS[:mongodb][:username],
                                  Pushme::OPTIONS[:mongodb][:password]) if Pushme::OPTIONS[:mongodb][:username] && Pushme::OPTIONS[:mongodb][:password]

#parsers = Pushme::Parsers.new(Pushme::OPTIONS[:datas])
Pushme::Parser.all.each {|f| f.delete }
Pushme::OPTIONS[:datas].each do |data|
  unless Pushme::Parser.exists?(:feed_url => data[:feed_url],
                      :feed_type => data[:feed_type])
    Pushme::Parser.create(:feed_url => data[:feed_url],
                          :feed_type => data[:feed_type],
                          :redis_key => data[:redis_key],
                          :pusher => Pushme::Push.new(data[:push]))
  end
end
feed = Pushme::Feed.new

RestClient.enable LogError, ProxyLogger.new(Pushme::Logger)
RestClient.enable Rack::Cache,
  :verbose => true,
  :metastore   => 'redis://localhost:6379/',
  :entitystore => 'redis://localhost:6379/'

RestClient.enable Rack::CommonLogger, ProxyLogger.new(Pushme::Logger)
RestClient.log = Pushme::Logger

require 'blather/client/dsl'
$stdout.sync = true

module Ping
  extend Blather::DSL

  def self.run; client.run; end

  setup Choice.choices[:jid], Choice.choices[:jid_password]

  when_ready do
    p 'ok'
    Pushme::Logger.debug "Connected ! send messages to #{jid.stripped}."
  end

  message :chat?, :body do |m|
    puts 'ok'
    p Blather::Stanza::PubSub::Subscribe.new(:set, Choice.choices[:jid], 'http://google.com', '<EMAIL>')
    client.write Blather::Stanza::PubSub::Subscribe.new(:set, Choice.choices[:jid], 'http://google.com', '<EMAIL>')
    puts 'write'
  end

  iq '/iq/pubsub' do |s|
    puts 'sub'
    p s
  end

  disconnected {
    p 'dis'
    client.run }

end

EventMachine.run {
  Ping.run

  EventMachine::PeriodicTimer.new(Pushme::OPTIONS[:cycle]) do
    Pushme::Logger.info('check feeds')
    Pushme::Parser.all.each do |parser|
      Pushme::Logger.debug("parse feeds : #{parser.inspect}")
      parser.items do |item|
        if feed.exists?(parser, item)
          parser.push(item)
          Pushme::Logger.info("push item #{item.inspect}")
        end
      end
    end
  end
  Pushme::Logger.debug('start eventmachine')

}

Pushme::Logger.debug("stop eventmachine")<commit_msg>add pubsub_subscribe event in blather to cache subscribe<commit_after>require 'eventmachine'
require 'yaml'
require 'logger'
require 'redis'
require 'restclient/components'
require 'rack/cache'
require 'redis-store'
require 'mongo_mapper'

require 'pushme/push'
require 'pushme/parser'
#require 'pushme/parsers'
require 'pushme/feed'

require 'choice'

Choice.options do
  option :config, :required => true do
    short '-c'
    long '--config=FILE'
    desc 'the file with configuration of pushme'
  end

  option :log do
    short '-l'
    long '--log=FILE'
    desc 'the file where all log are write'
    default STDOUT
  end

  option :jid, :required => true do
    long '--jid=jid'
    desc 'your jid listen'
  end

  option :jid_password, :required => true do
    long '--jid-password=jid-password'
    desc 'your password of your jid'
  end

  option :help do
    long '--help'
    desc 'Show this message'
  end
end

class ProxyLogger
  def initialize(logger)
    @logger = logger
  end

  def write(msg)
    @logger.info(msg)
  end

  def flush
    #nothing
  end

  def puts(msg)
    if msg.is_a?(Array)
      @logger.debug(msg.map(&:chomp).join("\n"))
    elsif msg.is_a?(String)
      @logger.debug(msg.chomp)
    else
      @logger.debug(msg)
    end
  end

  def method_missing(method, *args)
    @logger.send(method, *args)
  end
end

class LogError
  def initialize(app, logger)
    @app = app
    @logger = logger
  end

  def call(env)
    env['rack.errors'] = @logger
    @app.call(env)
  end
end



Pushme::OPTIONS = YAML.load_file(Choice.choices[:config])

Pushme::Logger = Logger.new(Choice.choices[:log])
Pushme::Logger.level = eval(Pushme::OPTIONS[:log_level]) || Logger::ERROR

MongoMapper.connection = Mongo::Connection.new(Pushme::OPTIONS[:mongodb][:host],
                                               Pushme::OPTIONS[:mongodb][:port],
                                               :logger => Pushme::Logger)
MongoMapper.database = Pushme::OPTIONS[:mongodb][:database]
MongoMapper.database.authenticate(Pushme::OPTIONS[:mongodb][:username],
                                  Pushme::OPTIONS[:mongodb][:password]) if Pushme::OPTIONS[:mongodb][:username] && Pushme::OPTIONS[:mongodb][:password]

#parsers = Pushme::Parsers.new(Pushme::OPTIONS[:datas])
Pushme::Parser.all.each {|f| f.delete }
Pushme::OPTIONS[:datas].each do |data|
  unless Pushme::Parser.exists?(:feed_url => data[:feed_url],
                      :feed_type => data[:feed_type])
    Pushme::Parser.create(:feed_url => data[:feed_url],
                          :feed_type => data[:feed_type],
                          :redis_key => data[:redis_key],
                          :pusher => Pushme::Push.new(data[:push]))
  end
end
feed = Pushme::Feed.new

RestClient.enable LogError, ProxyLogger.new(Pushme::Logger)
RestClient.enable Rack::Cache,
  :verbose => true,
  :metastore   => 'redis://localhost:6379/',
  :entitystore => 'redis://localhost:6379/'

RestClient.enable Rack::CommonLogger, ProxyLogger.new(Pushme::Logger)
RestClient.log = Pushme::Logger

require 'blather/client/dsl'
$stdout.sync = true

module Ping
  extend Blather::DSL

  def self.run; client.run; end

  setup Choice.choices[:jid], Choice.choices[:jid_password]

  when_ready do
    p 'ok'
    Pushme::Logger.debug "Connected ! send messages to #{jid.stripped}."
  end

  message :chat?, :body do |m|
    puts 'ok'
    p Blather::Stanza::PubSub::Subscribe.new(:set, Choice.choices[:jid], 'http://google.com', '<EMAIL>')
    node = Blather::Stanza::PubSub::Subscribe.new(:set, Choice.choices[:jid], 'http://google.com', '<EMAIL>')
    # node.subscribe['pushme:type'] = 'open-notification'
    # node.subscribe['pushme:api_key'] = '4b1ede9b7e21025d9d000001'
    # node.subscribe['pushme:channel'] = 'jabber'
    # node.subscribe['pushme:to'] = '<EMAIL>'
    # p node
    client.write(node)
    puts 'write'
  end

  pubsub_subscribe   do |n|
    p n
    {:url => n.node,
      :username => n.jid,
      :redis_key => "#{n.jid}::#{n.node}"}
  end

  disconnected do
    p 'dis'
    client.run
  end

end

EventMachine.run {
  Ping.run

  EventMachine::PeriodicTimer.new(Pushme::OPTIONS[:cycle]) do
    Pushme::Logger.info('check feeds')
    Pushme::Parser.all.each do |parser|
      Pushme::Logger.debug("parse feeds : #{parser.inspect}")
      parser.items do |item|
        if feed.exists?(parser, item)
          parser.push(item)
          Pushme::Logger.info("push item #{item.inspect}")
        end
      end
    end
  end
  Pushme::Logger.debug('start eventmachine')

}

Pushme::Logger.debug("stop eventmachine")
{'fraction_non_alphanumeric': 0.14669092563111213, 'fraction_numerical': 0.003638844666818285, 'mean_word_length': 3.1485849056603774, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2521920', 'n_tokens_mistral': 3072, 'n_tokens_neox': 2956, 'n_words': 624}