<commit_before>#!/usr/bin/env perl

=head1 NAME

apim-harvester.pl


=head1 DESCRIPTION

This is glue code that takes data from activemq [Apache messaging bus]
and writes it to MongoDB Because of the current limitation in fedora-commons, 
it only has api-m records at the moment.

=cut

use Net::Stomp;
use XML::XML2JSON;
use JSON::XS;
use YAML::Syck;
use MongoDB;
use Data::Dumper;
use Log::Log4perl;

my $config = YAML::Syck::LoadFile('/etc/phaidra.yml');
Log::Log4perl->init_once("/etc/phaidra_logging.conf");

=head2 configuration, put onto the end of phaidra.yml
apimharvester:
   stomp_host: 'localhost.localdomain'
   stomp_port: 61613
   stomp_user: 'not used at present'
   stomp_password: '<PASSWORD>'
   mongo_host: 'host'
   mongo_user: 'user'
   mongo_password: '<PASSWORD>'
   mongo_port: 27017 # not needed, if standard 27017
   mongo_db:  'db'
   mongo_collection: 'apim'
   update_topic: 'fedora.apim.update'
   access_topic: 'fedora.apim.access'
=cut

# format is: my $client = MongoDB::MongoClient->new( "host" =>
#      "mongodb://username:pass\@host\/db" );
# one in the doc, doesn't seem to work?

my $log = Log::Log4perl->get_logger("Phaidra::Apimharvester");

=cut
my $uri = "mongodb://".$config->{apimharvester}->{mongo_user}.":".$config->{apimharvester}->{mongo_password}."@". $config->{apimharvester}->{mongo_host}."/".$config->{apimharvester}->{mongo_db};
my $client = MongoDB->connect($uri);
=cut

=cut
  MongoDB::MongoClient->new( "host" =>
"mongodb://$config->{'apimharvester'}->{'mongo_user'}:$config->{'apimharvester'}->{'mongo_password'}\@$config->{'apimharvester'}->{'mongo_host'}\/$config->{'apimharvester'}->{'mongo_db'}"
  );
=cut

my $client = MongoDB::Connection->new(
  host => $config->{apimharvester}->{mongo_host}, 
  port => "27017",
  username => $config->{apimharvester}->{mongo_user},
  password => $config->{apimharvester}->{mongo_password},
  db_name => $config->{apimharvester}->{mongo_db}
);

my $db = $client->get_database( $config->{'apimharvester'}->{'mongo_db'} );
my $messages =
  $db->get_collection( $config->{'apimharvester'}->{'mongo_collection'} );

# store the frame bodies here for debugging before trying to put into Mongo
my $debug_file = 'debug.txt';
open my $fh, '>>', $debug_file;

# set up connection to activemq on default port for non-ssl stomp<commit_msg>mongo<commit_after>#!/usr/bin/env perl

=head1 NAME

apim-harvester.pl


=head1 DESCRIPTION

This is glue code that takes data from activemq [Apache messaging bus]
and writes it to MongoDB Because of the current limitation in fedora-commons, 
it only has api-m records at the moment.

=cut

use Net::Stomp;
use XML::XML2JSON;
use JSON::XS;
use YAML::Syck;
use MongoDB;
use Data::Dumper;
use Log::Log4perl qw(:easy);

my $config = YAML::Syck::LoadFile('/etc/phaidra.yml');

Log::Log4perl->easy_init( { level => $DEBUG, file => ">>/usr/local/phaidra/logs/apim-harvester.log" } );

=head2 configuration, put onto the end of phaidra.yml
apimharvester:
   stomp_host: 'localhost.localdomain'
   stomp_port: 61613
   stomp_user: 'not used at present'
   stomp_password: '<PASSWORD>'
   mongo_host: 'host'
   mongo_user: 'user'
   mongo_password: '<PASSWORD>'
   mongo_port: 27017 # not needed, if standard 27017
   mongo_db:  'db'
   mongo_collection: 'apim'
   update_topic: 'fedora.apim.update'
   access_topic: 'fedora.apim.access'
=cut

# format is: my $client = MongoDB::MongoClient->new( "host" =>
#      "mongodb://username:pass\@host\/db" );
# one in the doc, doesn't seem to work?

my $log = Log::Log4perl->get_logger("Phaidra::Apimharvester");


my $uri = "mongodb://".$config->{apimharvester}->{mongo_user}.":".$config->{apimharvester}->{mongo_password}."@". $config->{apimharvester}->{mongo_host}."/".$config->{apimharvester}->{mongo_db};
my $client = MongoDB->connect($uri);


=cut
my $client = MongoDB::MongoClient->new( "host" =>
"mongodb://$config->{'apimharvester'}->{'mongo_user'}:$config->{'apimharvester'}->{'mongo_password'}\@$config->{'apimharvester'}->{'mongo_host'}\/$config->{'apimharvester'}->{'mongo_db'}"
  );
=cut

=cut
my $client = MongoDB::Connection->new(
  host => $config->{apimharvester}->{mongo_host}, 
  port => "27017",
  username => $config->{apimharvester}->{mongo_user},
  password => $config->{apimharvester}->{mongo_password},
  db_name => $config->{apimharvester}->{mongo_db}
);
=cut

my $db = $client->get_database( $config->{'apimharvester'}->{'mongo_db'} );
my $messages =
  $db->get_collection( $config->{'apimharvester'}->{'mongo_collection'} );

# store the frame bodies here for debugging before trying to put into Mongo
my $debug_file = 'debug.txt';
open my $fh, '>>', $debug_file;

# set up connection to activemq on default port for non-ssl stomp
{'fraction_non_alphanumeric': 0.1823379923761118, 'fraction_numerical': 0.011435832274459974, 'mean_word_length': 5.288948069241012, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6416015', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1749, 'n_words': 374}