<commit_before>package NonameTV::Importer::Kanal5_doc;

use strict;
use warnings;

=pod

Importer for Kanal5's Word-format.
Data is downloaded in one file per week. The file is in Microsoft Word-
format. There is no consistent markup used for describing the data. The
parsing is done by iterating over each <div> in the resulting html and
looking at the text inside the <div> to decide what type of data is
in this <div>. This is then fed to a state-machine.

The Importer also accepts data in html-format as produced by wvHtml.
This makes it possible to provide overrides in html-format.

Categorization-data is fetched from the xml-files that Kanal5 publish.
These files contain more data than the Word-files, but they are not
updated with last minute changes. Programs are matched between the
two formats by looking for identical titles.

Features:

Episode-information parsed from description.

=cut

use DateTime;
use XML::LibXML;
use POSIX qw/floor/;

use NonameTV qw/MyGet Word2Xml Html2Xml norm AddCategory ParseDescCatSwe/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/info progress error logdie/;

use NonameTV::Importer::BaseWeekly;

use base 'NonameTV::Importer::BaseWeekly';

sub new {
    my $proto = shift;
    my $class = ref($proto) || $proto;
    my $self  = $class->SUPER::new( @_ );
    bless ($self, $class);

    $self->{grabber_name} = "Kanal5";

    defined( $self->{UrlRoot} ) or die "You must specify UrlRoot";

    my $dsh = NonameTV::DataStore::Helper->new( $self->{datastore} );
    $self->{datastorehelper} = $dsh;

    return $self;
}

sub ImportContent
{
  my $self = shift;

  my( $batch_id, $cref, $chd ) = @_;

  my $cat = $self->FetchCategories( $batch_id, $chd );

  my $dsh = $self->{datastorehelper};

  my $doc;
    
  if( $$cref =~ /^\<\!DOCTYPE HTML/ )
  {
    # This is an override that has already been run through wvHtml
    $doc = Html2Xml( $$cref );
  }
  else
  {
    $doc = Word2Xml( $$cref );
  }

  if( not defined( $doc ) )
  {
    error( "$batch_id: Failed to parse" );
    return 0;
  }
  
  # Find all div-entries.
  my $ns = $doc->find( "//div" );
  
  if( $ns->size() == 0 )
  {
    error( "$batch_id: No programme entries found" );
    return 0;
  }
  
  # States
  use constant {
    ST_START  => 0,
    ST_FDATE  => 1,   # Found date
    ST_FTIME  => 2,   # Found starttime
    ST_FTITLE => 3,   # Found title
    ST_FDESC  => 4,   # Found description
  };
  
  use constant {
    T_TIME => 10,
    T_TIME_TITLE => 11, # Both time and title on the same line.
    T_DATE => 12,
    T_TEXT => 13,
        };
  
  my $state=ST_START;
  my $currdate;
  
  my $ce = {};
  
  foreach my $div ($ns->get_nodelist)
  {
    my( $text ) = norm( $div->findvalue( '.' ) );

    next if $text eq "";
    
    my $type = T_TEXT;
    
    my( $date ) = ($text =~ 
		   /^\s*\S+\s+
		   (\d+-\d+-\d+),\s+
		   vecka\s+\d+,\s+
		   Kanal\s+5\s*$/x )
	and $type = T_DATE;
    
    my( $start, $stop, $text2 );
    
    if( ($start, $stop) = ( $text =~ /^(\d+:\d+)\s*\-\s*(\d+:\d+)$/ ) )
    {
      $type = T_TIME;
    }
    elsif( ( $start ) = ( $text =~ /^(\d+:\d+)$/ ) )
    {
      $type = T_TIME;
    }
    elsif( ($start, $stop, $text2) = 
           ( $text =~ /^(\d+:\d+)\s*\-\s*(\d+:\d+)\s+(.*)$/ ) )
    {
      $type = T_TIME_TITLE;
    }
    elsif( ( $start, $text2 ) = ( $text =~ /^(\d+:\d+)\s+(.*)$/ ) )
    {<commit_msg>Be more permissive when matching dates.<commit_after>package NonameTV::Importer::Kanal5_doc;

use strict;
use warnings;

=pod

Importer for Kanal5's Word-format.
Data is downloaded in one file per week. The file is in Microsoft Word-
format. There is no consistent markup used for describing the data. The
parsing is done by iterating over each <div> in the resulting html and
looking at the text inside the <div> to decide what type of data is
in this <div>. This is then fed to a state-machine.

The Importer also accepts data in html-format as produced by wvHtml.
This makes it possible to provide overrides in html-format.

Categorization-data is fetched from the xml-files that Kanal5 publish.
These files contain more data than the Word-files, but they are not
updated with last minute changes. Programs are matched between the
two formats by looking for identical titles.

Features:

Episode-information parsed from description.

=cut

use DateTime;
use XML::LibXML;
use POSIX qw/floor/;

use NonameTV qw/MyGet Word2Xml Html2Xml norm AddCategory ParseDescCatSwe/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/info progress error logdie/;

use NonameTV::Importer::BaseWeekly;

use base 'NonameTV::Importer::BaseWeekly';

sub new {
    my $proto = shift;
    my $class = ref($proto) || $proto;
    my $self  = $class->SUPER::new( @_ );
    bless ($self, $class);

    $self->{grabber_name} = "Kanal5";

    defined( $self->{UrlRoot} ) or die "You must specify UrlRoot";

    my $dsh = NonameTV::DataStore::Helper->new( $self->{datastore} );
    $self->{datastorehelper} = $dsh;

    return $self;
}

sub ImportContent
{
  my $self = shift;

  my( $batch_id, $cref, $chd ) = @_;

  my $cat = $self->FetchCategories( $batch_id, $chd );

  my $dsh = $self->{datastorehelper};

  my $doc;
    
  if( $$cref =~ /^\<\!DOCTYPE HTML/ )
  {
    # This is an override that has already been run through wvHtml
    $doc = Html2Xml( $$cref );
  }
  else
  {
    $doc = Word2Xml( $$cref );
  }

  if( not defined( $doc ) )
  {
    error( "$batch_id: Failed to parse" );
    return 0;
  }
  
  # Find all div-entries.
  my $ns = $doc->find( "//div" );
  
  if( $ns->size() == 0 )
  {
    error( "$batch_id: No programme entries found" );
    return 0;
  }
  
  # States
  use constant {
    ST_START  => 0,
    ST_FDATE  => 1,   # Found date
    ST_FTIME  => 2,   # Found starttime
    ST_FTITLE => 3,   # Found title
    ST_FDESC  => 4,   # Found description
  };
  
  use constant {
    T_TIME => 10,
    T_TIME_TITLE => 11, # Both time and title on the same line.
    T_DATE => 12,
    T_TEXT => 13,
        };
  
  my $state=ST_START;
  my $currdate;
  
  my $ce = {};
  
  foreach my $div ($ns->get_nodelist)
  {
    my( $text ) = norm( $div->findvalue( '.' ) );

    next if $text eq "";
    
    my $type = T_TEXT;
    
    my( $date ) = ($text =~ 
		   /^.*?
		   (\d+-\d+-\d+),\s+
		   vecka\s+\d+,\s+
		   Kanal\s+5\s*$/x )
	and $type = T_DATE;
    
    my( $start, $stop, $text2 );
    
    if( ($start, $stop) = ( $text =~ /^(\d+:\d+)\s*\-\s*(\d+:\d+)$/ ) )
    {
      $type = T_TIME;
    }
    elsif( ( $start ) = ( $text =~ /^(\d+:\d+)$/ ) )
    {
      $type = T_TIME;
    }
    elsif( ($start, $stop, $text2) = 
           ( $text =~ /^(\d+:\d+)\s*\-\s*(\d+:\d+)\s+(.*)$/ ) )
    {
      $type = T_TIME_TITLE;
    }
    elsif( ( $start, $text2 ) = ( $text =~ /^(\d+:\d+)\s+(.*)$/ ) )
    {
{'fraction_non_alphanumeric': 0.1614138438880707, 'fraction_numerical': 0.008247422680412371, 'mean_word_length': 2.59502382212811, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7040457', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2548, 'n_words': 742}