<commit_before>package CXGN::Phenotypes::ParseUpload::Plugin::DataCollectorSpreadsheet;

use Moose;
#use File::Slurp;
use Spreadsheet::ParseExcel;

sub name {
    return "datacollector spreadsheet";
}

sub validate {
    my $self = shift;
    my $filename = shift;
    my @file_lines;
    my $delimiter = ',';
    my $header;
    my @header_row;
    my $parser   = Spreadsheet::ParseExcel->new();
    my $excel_obj;
    my $worksheet;
    my @errors;
   
    #try to open the excel file and report any errors
    $excel_obj = $parser->parse($filename);
    if ( !$excel_obj ) {
      push @errors,  $parser->error();
      #$self->_set_parse_errors(\@errors);
      print STDERR "validate error: ".$parser->error()."\n";
      return;
    }

    $worksheet = ($excel_obj->worksheets())[7]; #support only one worksheet
    if (!$worksheet) {
	print STDERR "No 7th tab found in your Excel file.\n";
	return;
    }

   my  ( $row_min, $row_max ) = $worksheet->row_range();
   my  ( $col_min, $col_max ) = $worksheet->col_range();

    if (($col_max - $col_min)  < 1 || ($row_max - $row_min) < 1 ) { #must have header and at least one row of phenotypes
      push @errors, "Spreadsheet is missing header";
      #$self->_set_parse_errors(\@errors);
      print STDERR "Spreadsheet is missing header\n";
      return;
    }

    my $plot_name_head;
    if ($worksheet->get_cell(0,0)) {
      $plot_name_head  = $worksheet->get_cell(0,0)->value();
    }

    if (!$plot_name_head || $plot_name_head ne 'plot_name') {
      print STDERR "No plot name in header\n";
      return;
    }


    #if the rest of the header rows are not two caps followed by colon followed by text then return

    return 1;
}

sub parse {
    my $self = shift;
    my $filename = shift;
    my %parse_result;
    my @file_lines;
    my $delimiter = ',';
    my $header;
    my @header_row;
    my $header_column_number = 0;
    my %header_column_info; #column numbers of key info indexed from 0;
    my %plots_seen;
    my %traits_seen;
    my @plots;
    my @traits;
    my %data;
    my $parser   = Spreadsheet::ParseExcel->new();
    my $excel_obj;
    my $worksheet;
    my @errors;

    #try to open the excel file and report any errors
    $excel_obj = $parser->parse($filename);
    if ( !$excel_obj ) {
      push @errors,  $parser->error();
      #$self->_set_parse_errors(\@errors);
      print STDERR "Could not open excel file";
      return;
    }

    $worksheet = ( $excel_obj->worksheets() )[7];
    my ( $row_min, $row_max ) = $worksheet->row_range();
    my ( $col_min, $col_max ) = $worksheet->col_range();

    #get trait names and column numbers;
    for my $col (6 .. $col_max) {
      my $cell_val;
      if ($worksheet->get_cell(0,$col)) {
	$cell_val = $worksheet->get_cell(0,$col)->value();
      }
      if ($cell_val) {
	$header_column_info{$cell_val} = $col;
	$traits_seen{$cell_val} = 1;<commit_msg>datacollector phenotype upload catch if excel file does not have 7th tab<commit_after>package CXGN::Phenotypes::ParseUpload::Plugin::DataCollectorSpreadsheet;

use Moose;
#use File::Slurp;
use Spreadsheet::ParseExcel;

sub name {
    return "datacollector spreadsheet";
}

sub validate {
    my $self = shift;
    my $filename = shift;
    my @file_lines;
    my $delimiter = ',';
    my $header;
    my @header_row;
    my $parser   = Spreadsheet::ParseExcel->new();
    my $excel_obj;
    my $worksheet;
    my @errors;
   
    #try to open the excel file and report any errors
    $excel_obj = $parser->parse($filename);
    if ( !$excel_obj ) {
      push @errors,  $parser->error();
      #$self->_set_parse_errors(\@errors);
      print STDERR "validate error: ".$parser->error()."\n";
      return;
    }

    $worksheet = ($excel_obj->worksheets())[7]; #support only one worksheet
    if (!$worksheet) {
	print STDERR "No 7th tab found in your Excel file.\n";
	return;
    }

   my  ( $row_min, $row_max ) = $worksheet->row_range();
   my  ( $col_min, $col_max ) = $worksheet->col_range();

    if (($col_max - $col_min)  < 1 || ($row_max - $row_min) < 1 ) { #must have header and at least one row of phenotypes
      push @errors, "Spreadsheet is missing header";
      #$self->_set_parse_errors(\@errors);
      print STDERR "Spreadsheet is missing header\n";
      return;
    }

    my $plot_name_head;
    if ($worksheet->get_cell(0,0)) {
      $plot_name_head  = $worksheet->get_cell(0,0)->value();
    }

    if (!$plot_name_head || $plot_name_head ne 'plot_name') {
      print STDERR "No plot name in header\n";
      return;
    }


    #if the rest of the header rows are not two caps followed by colon followed by text then return

    return 1;
}

sub parse {
    my $self = shift;
    my $filename = shift;
    my %parse_result;
    my @file_lines;
    my $delimiter = ',';
    my $header;
    my @header_row;
    my $header_column_number = 0;
    my %header_column_info; #column numbers of key info indexed from 0;
    my %plots_seen;
    my %traits_seen;
    my @plots;
    my @traits;
    my %data;
    my $parser   = Spreadsheet::ParseExcel->new();
    my $excel_obj;
    my $worksheet;
    my @errors;

    #try to open the excel file and report any errors
    $excel_obj = $parser->parse($filename);
    if ( !$excel_obj ) {
      push @errors,  $parser->error();
      #$self->_set_parse_errors(\@errors);
      print STDERR "Could not open excel file";
      return;
    }

    $worksheet = ( $excel_obj->worksheets() )[7];
    if (!$worksheet) {
	print STDERR "No 7th tab found in your Excel file.\n";
	return;
    }

    my ( $row_min, $row_max ) = $worksheet->row_range();
    my ( $col_min, $col_max ) = $worksheet->col_range();

    #get trait names and column numbers;
    for my $col (6 .. $col_max) {
      my $cell_val;
      if ($worksheet->get_cell(0,$col)) {
	$cell_val = $worksheet->get_cell(0,$col)->value();
      }
      if ($cell_val) {
	$header_column_info{$cell_val} = $col;
	$traits_seen{$cell_val} = 1;
{'fraction_non_alphanumeric': 0.14213709677419356, 'fraction_numerical': 0.00571236559139785, 'mean_word_length': 2.9164473684210526, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3315877', 'n_tokens_mistral': 2158, 'n_tokens_neox': 2080, 'n_words': 563}