<commit_before><commit_msg>script to import emm_sources.xml into the database<commit_after>#!/usr/bin/env perl
#
# Import emm_sources.xml as media sources
#
# Usage: ./mediawords_import_emm_sources.pl --input_file=~/Desktop/emm_sources.xml
#        - or -
#        ./mediawords_import_emm_sources.pl \
#            --input_file=~/Desktop/emm_sources.xml \
#            --collection_tag=europe_media_monitor_20121015
#

use strict;
use warnings;

BEGIN
{
    use FindBin;
    use lib "$FindBin::Bin/../lib";
}

use Modern::Perl "2012";
use MediaWords::CommonLibs;

use utf8;

use XML::TreePP;
use Data::Dumper;
use Getopt::Long;
use Encode;
use MediaWords::DB;

# Strip string from whitespace
sub _strip_string
{
    my $string = shift;
    $string =~ s/^\s+//;
    $string =~ s/\s+$//;
    return $string;
}

# Don't leave spaces in tags, make lowercase too
sub _tagify_string
{
    my $string = shift;
    $string = _strip_string( $string );
    $string =~ s/\s/_/gs;
    $string = lc( $string );
    $string = encode( 'UTF-8', $string );
    return $string;
}

# Stoplist generator
sub main
{
    binmode( STDOUT, ':utf8' );
    binmode( STDERR, ':utf8' );

    my $input_file     = undef;                              # Input file to read emm_sources.xml from
    my $collection_tag = 'europe_media_monitor_20121015';    # Collection tag to add to every medium

    my Readonly $usage = "Usage: $0" . ' --input_file=emm_sources.xml' . ' [--collection_tag=europe_media_monitor_20121015]';

    GetOptions(
        'input_file=s'     => \$input_file,
        'collection_tag:s' => \$collection_tag,
    ) or die "$usage\n";
    die "$usage\n" unless ( $input_file and $collection_tag );

    say STDERR "starting --  " . localtime();

    my $dbis = MediaWords::DB::connect_to_db;

    my $tpp = XML::TreePP->new();
    $tpp->set( utf8_flag => 1 );
    my $tree = $tpp->parsefile( $input_file );

    # Start transaction
    $dbis->begin_work;

    for my $source ( @{ $tree->{ 'sources' }->{ 'source' } } )
    {

        # print 'Source: ' . Dumper($source);

        my @required_arguments = ( '-name', '-url', '-type', '-subject', '-country', '-region', '-category', '-lang' );
        my $skip_source = 0;
        for my $argument ( @required_arguments )
        {
            unless ( $source->{ $argument } )
            {
                print STDERR "Skipping source that does not have a required '$argument' argument: " . Dumper( $source );
                $skip_source = 1;
            }
        }
        if ( $skip_source )
        {
            next;
        }

        my $media_to_add = {
            name        => encode( 'UTF-8', _strip_string( $source->{ '-name' } ) ),
            url         => encode( 'UTF-8', _strip_string( $source->{ '-url' } ) ),
            moderated   => 'f',
            feeds_added => 'f',
        };
        my $media_tags_to_add = [

            # Blanket tag for emm_sources.xml media
            'collection:' . _tagify_string( $collection_tag ),

            # Additional arguments
            'emm_type:' . _tagify_string( $source->{ '-type' } ),
            'emm_subject:' . _tagify_string( $source->{ '-subject' } ),
            'emm_country:' . _tagify_string( $source->{ '-country' } ),
            'emm_region:' . _tagify_string( $source->{ '-region' } ),
            'emm_category:' . _tagify_string( $source->{ '-category' } ),
            'emm_lang:' . _tagify_string( $source->{ '-lang' } ),
        ];

        # Create / fetch media
        my $medium = undef;
        if (
            $medium = $dbis->query(
                <<"EOF",
            SELECT *
            FROM media
            WHERE name = ? OR url = ?
EOF
                $media_to_add->{ name }, $media_to_add->{ url }
            )->hash
          )
        {
            print STDERR "Using existing medium with duplicate title '" . $media_to_add->{ name } . "' or URL '" .
              $media_to_add->{ url } . "'.\n";
        }
        else
        {
            $medium = $dbis->create( 'media', $media_to_add );
        }

        for my $tag_set_and_tag ( @{ $media_tags_to_add } )
        {
            my ( $tag_set_name, $tag_name ) = split( ':', $tag_set_and_tag );

            unless ( $tag_set_name and $tag_name )
            {
                $dbis->rollback;
                die "Tag set or tag is empty.\n";
            }

            # Create / fetch tag set
            my $tag_set = $dbis->find_or_create( 'tag_sets', { name => $tag_set_name } );
            unless ( $tag_set )
            {
                $dbis->rollback;
                die "Unable to create / fetch tag set '$tag_set_name'.\n";
            }

            # Create / fetch tag
            my $tag = $dbis->find_or_create( 'tags', { tag => $tag_name, tag_sets_id => $tag_set->{ tag_sets_id } } );
            unless ( $tag )
            {
                $dbis->rollback;
                die "Unable to create / fetch tag '$tag_name' in tag set '$tag_set_name' (" . $tag_set->{ tag_sets_id } .
                  ").\n";
            }

            # Map media to a tag
            unless (
                $dbis->find_or_create(
                    'media_tags_map', { tags_id => $tag->{ tags_id }, media_id => $medium->{ media_id } }
                )
              )
            {
                $dbis->rollback;
                die "Unable to assign '$tag_name' (" . $tag->{ tags_id } . ") to media '" . $medium->{ name } . "' (" .
                  $medium->{ media_id } . ").\n";
            }

        }

        print STDERR "Will add / update media '" . $medium->{ name } . "'\n";
    }

    print STDERR "Committing...\n";
    $dbis->commit;

    say STDERR "finished --  " . localtime();
}

main();
{'fraction_non_alphanumeric': 0.15210186638758066, 'fraction_numerical': 0.006802721088435374, 'mean_word_length': 1.7047169811320755, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1675308', 'n_tokens_mistral': 1922, 'n_tokens_neox': 1833, 'n_words': 422}