<commit_before>package NonameTV::Importer::TCM_doc;

use strict;
use warnings;

=pod

Channels: TCM Nordic (SE, NO, DK, EN)

Import data from Word-files delivered via e-mail.  Each day
is handled as a separate batch.

Features:

=cut

use utf8;

use POSIX;
use DateTime;
use Data::Dumper;
use XML::LibXML;
use File::Basename;
use Encode qw/decode/;

use NonameTV qw/MyGet Wordfile2Xml AddCategory Htmlfile2Xml norm MonthNumber/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);


  my $dsh = NonameTV::DataStore::Helper->new( $self->{datastore} );
  $self->{datastorehelper} = $dsh;

  # use augment
  $self->{datastore}->{augment} = 1;

  return $self;
}

sub ImportContentFile
{
  my $self = shift;
  my( $file, $chd ) = @_;

  $self->{fileerror} = 0;

  my $xmltvid = $chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $dsh = $self->{datastorehelper};
  my $ds = $self->{datastore};

  return if( $file !~ /\.doc$/i );

  progress( "Turner_doc: $xmltvid: Processing $file" );

  my $doc;
  $doc = Wordfile2Xml( $file );

  if( not defined( $doc ) ) {
    error( "Turner_doc - $xmltvid: $file: Failed to parse" );
    return;
  }

  my @nodes = $doc->findnodes( '//span[@style="text-transform:uppercase"]/text()' );
  foreach my $node (@nodes) {
    my $str = $node->getData();
    $node->setData( uc( $str ) );
  }

  # Find all paragraphs.
  my $ns = $doc->find( "//p" );

  if( $ns->size() == 0 ) {
    error( "Turner_doc - $xmltvid: $file: No ps found." ) ;
    return;
  }

  my $currdate = "x";
  my $date = undef;
  my @ces;
  my $description;



  my $actors = 0;
  my $batched = 0;

  foreach my $div ($ns->get_nodelist) {

    my( $text ) = tcm_norm( $div->findvalue( '.' ) );

    if( isDate( $text ) ) {

    } elsif( isShow( $text ) ) {

      $actors = 0;

      my( $time, $date, $title, $director, $prodyear ) = ParseShow( $text );
      next if( ! $time );
      next if( ! $title );

      if(!$batched) {
        $batched = 1;
        my( $year, $month, $day ) = split( '-', $date );
        $ds->StartBatch( $xmltvid."_".$year."-".$month, $channel_id );
      }

      my $ce = {
        channel_id => $chd->{id},
        start_time => $date." ".$time,
        title => $title,
      };

      $ce->{directors} = join( ", ", split( /\s*,\s*/, norm($director) ) ) if defined $director and norm($director) ne "";
      $ce->{production_date} = norm($prodyear)."-01-01" if defined $prodyear and norm($prodyear) ne "";
      $ce->{program_type} = "movie" if defined $director and norm($director) ne "";

      # add the programme to the array
      # as we have to add description later
      push( @ces , $ce );

    } else {
        $actors += 1 if norm($text) ne "";
        # the last element is the one to which
        # this description belongs to
        my $element = $ces[$#ces];

        # Genre<commit_msg>Fix where prod year can include B/W etc<commit_after>package NonameTV::Importer::TCM_doc;

use strict;
use warnings;

=pod

Channels: TCM Nordic (SE, NO, DK, EN)

Import data from Word-files delivered via e-mail.  Each day
is handled as a separate batch.

Features:

=cut

use utf8;

use POSIX;
use DateTime;
use Data::Dumper;
use XML::LibXML;
use File::Basename;
use Encode qw/decode/;

use NonameTV qw/MyGet Wordfile2Xml AddCategory Htmlfile2Xml norm MonthNumber/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);


  my $dsh = NonameTV::DataStore::Helper->new( $self->{datastore} );
  $self->{datastorehelper} = $dsh;

  # use augment
  $self->{datastore}->{augment} = 1;

  return $self;
}

sub ImportContentFile
{
  my $self = shift;
  my( $file, $chd ) = @_;

  $self->{fileerror} = 0;

  my $xmltvid = $chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $dsh = $self->{datastorehelper};
  my $ds = $self->{datastore};

  return if( $file !~ /\.doc$/i );

  progress( "Turner_doc: $xmltvid: Processing $file" );

  my $doc;
  $doc = Wordfile2Xml( $file );

  if( not defined( $doc ) ) {
    error( "Turner_doc - $xmltvid: $file: Failed to parse" );
    return;
  }

  my @nodes = $doc->findnodes( '//span[@style="text-transform:uppercase"]/text()' );
  foreach my $node (@nodes) {
    my $str = $node->getData();
    $node->setData( uc( $str ) );
  }

  # Find all paragraphs.
  my $ns = $doc->find( "//p" );

  if( $ns->size() == 0 ) {
    error( "Turner_doc - $xmltvid: $file: No ps found." ) ;
    return;
  }

  my $currdate = "x";
  my $date = undef;
  my @ces;
  my $description;



  my $actors = 0;
  my $batched = 0;

  foreach my $div ($ns->get_nodelist) {

    my( $text ) = tcm_norm( $div->findvalue( '.' ) );

    if( isDate( $text ) ) {

    } elsif( isShow( $text ) ) {

      $actors = 0;

      my( $time, $date, $title, $director, $prodyear ) = ParseShow( $text );
      next if( ! $time );
      next if( ! $title );

      if(!$batched) {
        $batched = 1;
        my( $year, $month, $day ) = split( '-', $date );
        $ds->StartBatch( $xmltvid."_".$year."-".$month, $channel_id );
      }

      my $ce = {
        channel_id => $chd->{id},
        start_time => $date." ".$time,
        title => $title,
      };

      $ce->{directors} = join( ", ", split( /\s*,\s*/, norm($director) ) ) if defined $director and norm($director) ne "";
      if( defined( $prodyear ) and ($prodyear =~ /(\d\d\d\d)/) )
      {
        $ce->{production_date} = "$1-01-01";
      }

      $ce->{program_type} = "movie" if defined $director and norm($director) ne "";

      # add the programme to the array
      # as we have to add description later
      push( @ces , $ce );

    } else {
        $actors += 1 if norm($text) ne "";
        # the last element is the one to which
        # this description belongs to
        my $element = $ces[$#ces];

        # Genre
{'fraction_non_alphanumeric': 0.18381521563560008, 'fraction_numerical': 0.00533031820384429, 'mean_word_length': 2.7504542701393095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4129834', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2295, 'n_words': 614}