<commit_before>package NonameTV::Importer::KanalLokal;

use strict;
use warnings;

=pod

Import data from Xml-files delivered via e-mail.  Each day
is handled as a separate batch.

Features:

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Text::Capitalize qw/capitalize_title/;

use NonameTV qw/MyGet Wordfile2Xml Htmlfile2Xml norm/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/info progress error logdie 
                     log_to_string log_to_string_result/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);

  $self->{grabber_name} = "KanalLokal";

  return $self;
}

sub ImportContentFile {
  my $self = shift;
  my( $file, $chd ) = @_;

  progress( "KanalLokal: Processing $file" );
  
  $self->{fileerror} = 0;

  my $xmltvid=$chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $ds = $self->{datastore};
  $ds->{SILENCE_END_START_OVERLAP}=1;

  my $doc;
  my $xml = XML::LibXML->new;
  eval { $doc = $xml->parse_file($file); };

  if( not defined( $doc ) ) {
    error( "KanalLokal $file: Failed to parse" );
    return;
  }
  
  my $ns = $doc->find( "//Event" );
  
  if( $ns->size() == 0 ) {
    error( "KanalLokal $file: No Events found." ) ;
    return;
  }

  my $batch_id;

  foreach my $div ($ns->get_nodelist) {
    my $date = norm( $div->findvalue( 'StartDate' ) );
    my $starttime = norm( $div->findvalue( 'StartTime' ) );
    my $endtime = norm( $div->findvalue( 'EndTime' ) );
    my $title = norm( $div->findvalue( 'Title1' ) );
    my $synopsis = norm( $div->findvalue( 'Synopsis1' ) );
<commit_msg>Prefer Title2 if it exists.<commit_after>package NonameTV::Importer::KanalLokal;

use strict;
use warnings;

=pod

Import data from Xml-files delivered via e-mail.  Each day
is handled as a separate batch.

Features:

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Text::Capitalize qw/capitalize_title/;

use NonameTV qw/MyGet Wordfile2Xml Htmlfile2Xml norm/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/info progress error logdie 
                     log_to_string log_to_string_result/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);

  $self->{grabber_name} = "KanalLokal";

  return $self;
}

sub ImportContentFile {
  my $self = shift;
  my( $file, $chd ) = @_;

  progress( "KanalLokal: Processing $file" );
  
  $self->{fileerror} = 0;

  my $xmltvid=$chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $ds = $self->{datastore};
  $ds->{SILENCE_END_START_OVERLAP}=1;

  my $doc;
  my $xml = XML::LibXML->new;
  eval { $doc = $xml->parse_file($file); };

  if( not defined( $doc ) ) {
    error( "KanalLokal $file: Failed to parse" );
    return;
  }
  
  my $ns = $doc->find( "//Event" );
  
  if( $ns->size() == 0 ) {
    error( "KanalLokal $file: No Events found." ) ;
    return;
  }

  my $batch_id;

  foreach my $div ($ns->get_nodelist) {
    my $date = norm( $div->findvalue( 'StartDate' ) );
    my $starttime = norm( $div->findvalue( 'StartTime' ) );
    my $endtime = norm( $div->findvalue( 'EndTime' ) );
    my $title1 = norm( $div->findvalue( 'Title1' ) );
    my $title2 = norm( $div->findvalue( 'Title2' ) );
    my $synopsis = norm( $div->findvalue( 'Synopsis1' ) );

    # Prefer Title2 before Title1
    my $title = $title2 eq "" ? $title1 : $title2;

{'fraction_non_alphanumeric': 0.1685866816521495, 'fraction_numerical': 0.0070244450688395615, 'mean_word_length': 3.455569461827284, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2537752', 'n_tokens_mistral': 1410, 'n_tokens_neox': 1382, 'n_words': 339}