<commit_before>package NonameTV::Importer::Mtve;

#
# This importer imports data from MTV Europe's press service. 
#
# One xml-file per day. Each entry contains a start-time with a timezone,
# (CET), title, description, url and image for the show.
#

use strict;
use warnings;

use DateTime;
use XML::LibXML;

use NonameTV qw/MyGet Utf8Conv/;

use NonameTV::Importer::BaseDaily;

use base 'NonameTV::Importer::BaseDaily';

sub new {
    my $proto = shift;
    my $class = ref($proto) || $proto;
    my $self  = $class->SUPER::new( @_ );
    bless ($self, $class);

    $self->{grabber_name} = "Mtve";

    defined( $self->{UrlRoot} ) or die "You must specify UrlRoot";

    return $self;
}

sub ImportContent
{
  my $self = shift;
  my( $batch_id, $cref, $chd ) = @_;

  my $l = $self->{logger};
  my $ds = $self->{datastore};
  my $dsh = $self->{datastorehelper};

  my( $date ) = ($batch_id =~ /_(.*)$/);

  # Add a proper entity set reference to allow &Aring; etc.
  $$cref =~ s(\?>)(?><!DOCTYPE MTV [
<!ENTITY \% ISOlat1 PUBLIC "ISO 8879:1986//ENTITIES Added Latin 1//EN"
    "http://www.w3.org/2003/entities/iso8879/isolat1.ent">
\%ISOlat1; ]>);

  my $xml = XML::LibXML->new;
  my $doc;
  eval { $doc = $xml->parse_string($$cref); };
  if( $@ ne "" )
  {
    $l->error( "$batch_id: Failed to parse: $@" );
    return;
  }
  
  # Verify the assumption that each file only contains data for
  # one channel.
  
  my $channelname = $doc->findvalue( '//Channel/@name' );<commit_msg>Add one more entity set reference to the xml-files.<commit_after>package NonameTV::Importer::Mtve;

#
# This importer imports data from MTV Europe's press service. 
#
# One xml-file per day. Each entry contains a start-time with a timezone,
# (CET), title, description, url and image for the show.
#

use strict;
use warnings;

use DateTime;
use XML::LibXML;

use NonameTV qw/MyGet Utf8Conv/;

use NonameTV::Importer::BaseDaily;

use base 'NonameTV::Importer::BaseDaily';

sub new {
    my $proto = shift;
    my $class = ref($proto) || $proto;
    my $self  = $class->SUPER::new( @_ );
    bless ($self, $class);

    $self->{grabber_name} = "Mtve";

    defined( $self->{UrlRoot} ) or die "You must specify UrlRoot";

    return $self;
}

sub ImportContent
{
  my $self = shift;
  my( $batch_id, $cref, $chd ) = @_;

  my $l = $self->{logger};
  my $ds = $self->{datastore};
  my $dsh = $self->{datastorehelper};

  my( $date ) = ($batch_id =~ /_(.*)$/);

  # Add proper entity set references
  # ISOlat1 contains &Aring; etc.
  # ISOdia contains &acute;
  $$cref =~ s(\?>)(?><!DOCTYPE MTV [
<!ENTITY \% ISOlat1 PUBLIC "ISO 8879:1986//ENTITIES Added Latin 1//EN"
    "http://www.w3.org/2003/entities/iso8879/isolat1.ent">
\%ISOlat1; 
<!ENTITY \% ISOdia PUBLIC "ISO 8879:1986//ENTITIES Diacritical Marks//EN"
    "http://www.w3.org/2003/entities/iso8879/isodia.ent">
\%ISOdia; 
]>);

  my $xml = XML::LibXML->new;
  my $doc;
  eval { $doc = $xml->parse_string($$cref); };
  if( $@ ne "" )
  {
    $l->error( "$batch_id: Failed to parse: $@" );
    return;
  }
  
  # Verify the assumption that each file only contains data for
  # one channel.
  
  my $channelname = $doc->findvalue( '//Channel/@name' );
{'fraction_non_alphanumeric': 0.17654437127626216, 'fraction_numerical': 0.019441831295076827, 'mean_word_length': 3.4060773480662982, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3076464', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1201, 'n_words': 323}