<commit_before>package NonameTV::Importer::SverigesRadio;

use strict;
use warnings;

=pod

Import data from Sveriges Radio. They publish data in xmltv-format
with one file per day containing all channels.

TODO:

Make sure that all available info is included. URL to programme!

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Encode qw/encode/;

use NonameTV qw/ParseXml ParseXmltv/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseDaily;

use base 'NonameTV::Importer::BaseDaily';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);


  push @{$self->{OptionSpec}}, "list-channels";
  $self->{OptionDefaults}->{'list-channels'} = 0;

  return $self;
}

sub InitiateDownload {
  my $self = shift;
  my( $p ) = @_;

  if( $p->{'list-channels'} ) {
    $self->PrintChannelList();
    exit;
  }

  return undef;
}

sub FilterContent {
  my $self = shift;
  my( $cref, $chd ) = @_;

  my $doc = ParseXml( $cref );
  
  if( not defined $doc ) {
    return (undef, "ParseXml failed" );
  } 

  my $fdoc = XML::LibXML::Document->new( "1.0", "UTF-8" );

  my $root = $fdoc->createElement( 'tv' );
  $fdoc->setDocumentElement( $root );

  my $ns = $doc->find( '//programme[@channel="' . $chd->{xmltvid} . '"]' );

  foreach my $node ($ns->get_nodelist()) {
    $fdoc->adoptNode( $node );
    $root->appendChild( $node );
  }
 
  # Stringify

  my $str = $fdoc->toString( 1 );
  return( \$str, undef );
}

sub ImportContent {
  my $self = shift;
  my( $batch_id, $cref, $chd ) = @_;

  my $ds = $self->{datastore};

  my $data = ParseXmltv( $cref );

  foreach my $e (@{$data})
  {
    $e->{start_dt}->set_time_zone( "UTC" );

    $e->{stop_dt}->set_time_zone( "UTC" );
    
    $e->{start_time} = $e->{start_dt}->ymd('-') . " " . 
        $e->{start_dt}->hms(':');
    delete $e->{start_dt};
    $e->{end_time} = $e->{stop_dt}->ymd('-') . " " . 
        $e->{stop_dt}->hms(':');
    delete $e->{stop_dt};
    $e->{channel_id} = $chd->{id};
    
    $ds->AddProgrammeRaw( $e );
  }

  return 1;
}

sub Object2Url {
  my $self = shift;
  my( $objectname, $chd ) = @_;

  my( $date ) = ( $objectname =~ /(\d+-\d+-\d+)$/ );
 
  my $url = sprintf( "%s%s.xml", $self->{UrlRoot}, $date );
  
  return( $url, undef );
}
<commit_msg>Ignore program blocks.<commit_after>package NonameTV::Importer::SverigesRadio;

use strict;
use warnings;

=pod

Import data from Sveriges Radio. They publish data in xmltv-format
with one file per day containing all channels.

TODO:

Make sure that all available info is included. URL to programme!

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Encode qw/encode/;

use NonameTV qw/ParseXml ParseXmltv/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseDaily;

use base 'NonameTV::Importer::BaseDaily';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);


  push @{$self->{OptionSpec}}, "list-channels";
  $self->{OptionDefaults}->{'list-channels'} = 0;

  return $self;
}

sub InitiateDownload {
  my $self = shift;
  my( $p ) = @_;

  if( $p->{'list-channels'} ) {
    $self->PrintChannelList();
    exit;
  }

  return undef;
}

sub FilterContent {
  my $self = shift;
  my( $cref, $chd ) = @_;

  my $doc = ParseXml( $cref );
  
  if( not defined $doc ) {
    return (undef, "ParseXml failed" );
  } 

  my $fdoc = XML::LibXML::Document->new( "1.0", "UTF-8" );

  my $root = $fdoc->createElement( 'tv' );
  $fdoc->setDocumentElement( $root );

  my $ns = $doc->find( '//programme[@channel="' . $chd->{xmltvid} . '"]' );

  foreach my $node ($ns->get_nodelist()) {
    $fdoc->adoptNode( $node );
    $root->appendChild( $node );
  }
 
  # Stringify

  my $str = $fdoc->toString( 1 );
  return( \$str, undef );
}

sub ImportContent {
  my $self = shift;
  my( $batch_id, $cref, $chd ) = @_;

  my $ds = $self->{datastore};

  my $data = ParseXmltv( $cref );

  foreach my $e (@{$data})
  {
    $e->{start_dt}->set_time_zone( "UTC" );

    $e->{stop_dt}->set_time_zone( "UTC" );
    
    $e->{start_time} = $e->{start_dt}->ymd('-') . " " . 
        $e->{start_dt}->hms(':');
    delete $e->{start_dt};
    $e->{end_time} = $e->{stop_dt}->ymd('-') . " " . 
        $e->{stop_dt}->hms(':');
    delete $e->{stop_dt};

    # SR uses zero-length programmes to signal the start of a
    # block of programmes.
    next if $e->{start_time} eq $e->{end_time};

    $e->{channel_id} = $chd->{id};
    
    $ds->AddProgrammeRaw( $e );
  }

  return 1;
}

sub Object2Url {
  my $self = shift;
  my( $objectname, $chd ) = @_;

  my( $date ) = ( $objectname =~ /(\d+-\d+-\d+)$/ );
 
  my $url = sprintf( "%s%s.xml", $self->{UrlRoot}, $date );
  
  return( $url, undef );
}

{'fraction_non_alphanumeric': 0.207492795389049, 'fraction_numerical': 0.0032935364347468094, 'mean_word_length': 3.11082910321489, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5145404', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1933, 'n_words': 440}