<commit_before>package NonameTV::Importer::DisneyChannel;

use strict;
use warnings;

=pod

Import data from Xml-files delivered via e-mail in zip-files.  Each
day is handled as a separate batch.

Features:

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Archive::Zip;
use Data::Dumper;
use File::Temp qw/tempfile/;

use NonameTV qw/norm/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);

  $self->{grabber_name} = "DisneyChannel";

  return $self;
}

sub ImportContentFile {
  my $self = shift;
  my( $file, $chd ) = @_;

  progress( "DisneyChannel: Processing $file" );
  
  $self->{fileerror} = 0;

  my $xmltvid=$chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $ds = $self->{datastore};

  my $datafile;

  if( $file =~ /\.xml$/i ) {
    $datafile = $file;
  }
  elsif( $file =~ /\.zip$/i ) {
    my $zip = Archive::Zip->new( $file );
    if( not defined $zip ) {
      error( "DisneyChannel $file: Failed to read zip." );
      return;
    }

    my @swedish_files;
    
    my @members = $zip->members();
    foreach my $member (@members) {
      push( @swedish_files, $member->{fileName} ) 
	  if $member->{fileName} =~ /swed.*xml$/i;
    }
    
    my $numfiles = scalar( @swedish_files );
    if( $numfiles != 1 ) {
      error( "DisneyChannel $file: Found $numfiles matching files, expected 1." );
      return;
    }

    progress( "Using file $swedish_files[0]" );

    (undef, $datafile) = tempfile();
    
    $zip->extractMember( $swedish_files[0], $datafile );
  }<commit_msg>Adjust filename parsing slightly for Disney.<commit_after>package NonameTV::Importer::DisneyChannel;

use strict;
use warnings;

=pod

Import data from Xml-files delivered via e-mail in zip-files.  Each
day is handled as a separate batch.

Features:

=cut

use utf8;

use DateTime;
use XML::LibXML;
use Archive::Zip;
use Data::Dumper;
use File::Temp qw/tempfile/;

use NonameTV qw/norm/;
use NonameTV::DataStore::Helper;
use NonameTV::Log qw/progress error/;

use NonameTV::Importer::BaseFile;

use base 'NonameTV::Importer::BaseFile';

sub new {
  my $proto = shift;
  my $class = ref($proto) || $proto;
  my $self  = $class->SUPER::new( @_ );
  bless ($self, $class);

  $self->{grabber_name} = "DisneyChannel";

  return $self;
}

sub ImportContentFile {
  my $self = shift;
  my( $file, $chd ) = @_;

  progress( "DisneyChannel: Processing $file" );
  
  $self->{fileerror} = 0;

  my $xmltvid=$chd->{xmltvid};
  my $channel_id = $chd->{id};
  my $ds = $self->{datastore};

  my $datafile;

  if( $file =~ /\.xml$/i ) {
    $datafile = $file;
  }
  elsif( $file =~ /\.zip$/i ) {
    my $zip = Archive::Zip->new( $file );
    if( not defined $zip ) {
      error( "DisneyChannel $file: Failed to read zip." );
      return;
    }

    my @swedish_files;
    
    my @members = $zip->members();
    foreach my $member (@members) {
      push( @swedish_files, $member->{fileName} ) 
	  if $member->{fileName} =~ /swe.*xml$/i;
    }
    
    my $numfiles = scalar( @swedish_files );
    if( $numfiles != 1 ) {
      error( "DisneyChannel $file: Found $numfiles matching files, expected 1." );
      return;
    }

    progress( "Using file $swedish_files[0]" );

    (undef, $datafile) = tempfile();
    
    $zip->extractMember( $swedish_files[0], $datafile );
  }
{'fraction_non_alphanumeric': 0.1682857142857143, 'fraction_numerical': 0.0034285714285714284, 'mean_word_length': 3.2231604342581424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5986339', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1301, 'n_words': 329}