<commit_before>#
# EnsEMBL module for Bio::EnsEMBL::Funcgen::Parsers::Bed
#

=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=head1 CONTACT

  Please email comments or questions to the public Ensembl
  developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

  Questions may also be sent to the Ensembl help desk at
  <http://www.ensembl.org/Help/Contact>.

=head1 NAME

Bio::EnsEMBL::Funcgen::Parsers::Bed

=head1 SYNOPSIS

  my $parser_type = "Bio::EnsEMBL::Funcgen::Parsers::Bed";
  push @INC, $parser_type;
  my $imp = $class->SUPER::new(@_);


=head1 DESCRIPTION

This is a definitions class which should not be instatiated directly, it 
normally set by the Importer as the parent class.  Bed contains meta 
data and methods specific to data in bed format, to aid 
parsing and importing of experimental data.

=cut

#Import/Parser rework
#We now have potential to use indexed DBFile and Parsers
#Importer should become BaseImporter, inherited from InputSet/Nimblegen
#Have Bed(format) importer which sets the generic Bed(format) Parser

package Bio::EnsEMBL::Funcgen::Parsers::Bed;

use strict;
use warnings;
use File::Basename;
use Bio::EnsEMBL::Utils::Exception         qw( throw );
use Bio::EnsEMBL::Utils::Argument          qw( rearrange );
use Bio::EnsEMBL::Funcgen::Utils::EFGUtils qw( convert_strand
                                               open_file 
                                               is_bed 
                                               is_gzipped);
use Bio::EnsEMBL::Funcgen::FeatureSet;
use Bio::EnsEMBL::Funcgen::AnnotatedFeature;
use Bio::EnsEMBL::Funcgen::Utils::Helper;

use base qw( Bio::EnsEMBL::Funcgen::Parsers::InputSet );

#To do
# Extend this so we can import features to annotated_feature
# and profiles/reads to result_feature
# This will replace individual tables for current bed das sources
# Altho' idea of separate tables for each set is not necessarily a bad one
# We would have to have extra table or fields details registration
# Easier admin i.e. drop tables
# What about partitioning? Irrelevant if we are moving to matrix
# Not easy to patch dynamically named tables? Can't assign values to user varaible from query!
# Would need to be a stored procedure
# Could have separate matrix files for result_features as we probably wouldn't want 
# to do any cross set querying at this level.
# Could we also use this in the RunnableDBs?
# Simply separate those methods required by both normal bed annotated_feature import
# and RunnableDB based annotated_feature import
# Importing into result_feature requires a result_set which assumes a chip experiment
# We need to alter result_set such that it can be optionally associated with ExperimentalSets
# Or should we just go for a different table? read_feature?
# NOTE: PARITION by key may run into problems if different sets have different windows
# Would also need to modify ResultFeatureAdaptor?

=head2 new

  Arg[0]     : hash containing optional attributes:
                          -bed_reads => 0|1, #Set input as read alignments rather than peak calls (default is 0)
  Example    : my $self = $class->SUPER::new(@_);
  Description: Constructor method for Bed class
  Returntype : Bio::EnsEMBL::Funcgen::Parsers::Bed
  Exceptions : throws if Experiment name not defined or if caller is not Importer
  Caller     : Bio::EnsEMBL::Funcgen::Importer
  Status     : at risk

=cut

sub new{
  my $caller = shift;
  my $class = ref($caller) || $caller;
  my $self = $class->SUPER::new(@_, no_disconnect => 1);
  
  throw("This is a skeleton class for Bio::EnsEMBL::Funcgen::Importer, should not be used directly") 
	if(! $self->isa("Bio::EnsEMBL::Funcgen::Importer"));

 
  #This was over-riding InputSet new config
  #$self->{'config'} =  
  #	{(
  #      #order of these method arrays is important!
  #	  #remove method arrays and execute serially?
  #	  #Just move these to individual Parser register_experiment methods
  #      array_data   => [],#['experiment'],
  #      probe_data   => [],#["probe"],
  #      results_data => ["and_import"],
  #      norm_method => undef,
  #
  #	  #Need to make these definable?
  #	  #have protocolfile arg and just parse tab2mage protocol section format
  #	  #SEE NIMBLEGEN FOR EXAMPLE
  #	  protocols => {()},
  #     )};
	
  $self->{'overhang_features'} = []; #Move to InputSet?
  #Maybe used by other formats

  return $self;
}


=head2 set_config

  Example    : my $self->set_config;
  Description: Sets attribute dependent config
  Returntype : None
  Exceptions : None
  Caller     : Bio::EnsEMBL::Funcgen::Importer
  Status     : at risk -remove

=cut


sub set_config{
  my $self = shift;

  $self->SUPER::set_config;
  #dir are not set in config to enable generic get_dir method access
  #Need to define output dir if we are processing reads

  return;
}

#We aren't really preprocessing the file here
#this is done in read_and_import_data
#initialise_files?

#do we even want to call this?
#Can't we completely skip this in the caller
#based on another flag?
#prepare (does the sorting and pre_processing) and pre_process just does the slice
#caching?
#out_file will be infile if we are just pre-processing
#if is already prepared, then we don't need to make a copy
#Do we even need to add prepared to the file name?
#This is filtered for the chr names


#This also set's the outputfile for use when preparing
#simply don't write the no sort!


sub initialise_input_file{
  my ($self, $filepath, $prepare) = @_;

  #Test file format
  throw("Input file is not bed format:\t$filepath") if ! &is_bed($filepath);
  

  #separate sort keys stop lexical sorting of start/end
  #when faced with a non numerical seq_region_name
  my $sort = ($prepare || ! $self->prepared) ? 'sort -k1,1 -k2,2n -k3,3n ' : '';
  my $is_gzipped = &is_gzipped($filepath);


  if($is_gzipped){
	$sort .= '|' if $sort;
	$self->input_file_operator("gzip -dc %s | $sort ");
  }<commit_msg>Minor<commit_after>#
# EnsEMBL module for Bio::EnsEMBL::Funcgen::Parsers::Bed
#

=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=head1 CONTACT

  Please email comments or questions to the public Ensembl
  developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

  Questions may also be sent to the Ensembl help desk at
  <http://www.ensembl.org/Help/Contact>.

=head1 NAME

Bio::EnsEMBL::Funcgen::Parsers::Bed

=head1 SYNOPSIS

  my $parser_type = "Bio::EnsEMBL::Funcgen::Parsers::Bed";
  push @INC, $parser_type;
  my $imp = $class->SUPER::new(@_);


=head1 DESCRIPTION

This is a definitions class which should not be instatiated directly, it 
normally set by the Importer as the parent class.  Bed contains meta 
data and methods specific to data in bed format, to aid 
parsing and importing of experimental data.

=cut

#Import/Parser rework
#We now have potential to use indexed DBFile and Parsers
#Importer should become BaseImporter, inherited from InputSet/Nimblegen
#Have Bed(format) importer which sets the generic Bed(format) Parser

package Bio::EnsEMBL::Funcgen::Parsers::Bed;

use strict;
use warnings;
use File::Basename;
use Bio::EnsEMBL::Utils::Exception         qw( throw );
use Bio::EnsEMBL::Utils::Argument          qw( rearrange );
use Bio::EnsEMBL::Funcgen::Utils::EFGUtils qw( convert_strand
                                               open_file 
                                               is_bed 
                                               is_gzipped);
use Bio::EnsEMBL::Funcgen::FeatureSet;
use Bio::EnsEMBL::Funcgen::AnnotatedFeature;
use Bio::EnsEMBL::Funcgen::Utils::Helper;

use base qw( Bio::EnsEMBL::Funcgen::Parsers::InputSet );

#To do
# Extend this so we can import features to annotated_feature
# and profiles/reads to result_feature
# This will replace individual tables for current bed das sources
# Altho' idea of separate tables for each set is not necessarily a bad one
# We would have to have extra table or fields details registration
# Easier admin i.e. drop tables
# What about partitioning? Irrelevant if we are moving to matrix
# Not easy to patch dynamically named tables? Can't assign values to user varaible from query!
# Would need to be a stored procedure
# Could have separate matrix files for result_features as we probably wouldn't want 
# to do any cross set querying at this level.
# Could we also use this in the RunnableDBs?
# Simply separate those methods required by both normal bed annotated_feature import
# and RunnableDB based annotated_feature import
# Importing into result_feature requires a result_set which assumes a chip experiment
# We need to alter result_set such that it can be optionally associated with ExperimentalSets
# Or should we just go for a different table? read_feature?
# NOTE: PARITION by key may run into problems if different sets have different windows
# Would also need to modify ResultFeatureAdaptor?

=head2 new

  Arg[0]     : hash containing optional attributes:
                          -bed_reads => 0|1, #Set input as read alignments rather than peak calls (default is 0)
  Example    : my $self = $class->SUPER::new(@_);
  Description: Constructor method for Bed class
  Returntype : Bio::EnsEMBL::Funcgen::Parsers::Bed
  Exceptions : throws if Experiment name not defined or if caller is not Importer
  Caller     : Bio::EnsEMBL::Funcgen::Importer
  Status     : at risk

=cut

sub new{
  my $caller = shift;
  my $class = ref($caller) || $caller;
  my $self = $class->SUPER::new(@_, no_disconnect => 1);
  
  throw("This is a skeleton class for Bio::EnsEMBL::Funcgen::Importer, should not be used directly") 
	if(! $self->isa("Bio::EnsEMBL::Funcgen::Importer"));

 
  #This was over-riding InputSet new config
  #$self->{'config'} =  
  #	{(
  #      #order of these method arrays is important!
  #	  #remove method arrays and execute serially?
  #	  #Just move these to individual Parser register_experiment methods
  #      array_data   => [],#['experiment'],
  #      probe_data   => [],#["probe"],
  #      results_data => ["and_import"],
  #      norm_method => undef,
  #
  #	  #Need to make these definable?
  #	  #have protocolfile arg and just parse tab2mage protocol section format
  #	  #SEE NIMBLEGEN FOR EXAMPLE
  #	  protocols => {()},
  #     )};
	
  $self->{'overhang_features'} = []; #Move to InputSet?
  #Maybe used by other formats

  return $self;
}


=head2 set_config

  Example    : my $self->set_config;
  Description: Sets attribute dependent config
  Returntype : None
  Exceptions : None
  Caller     : Bio::EnsEMBL::Funcgen::Importer
  Status     : at risk -remove

=cut


sub set_config{
  my $self = shift;

  $self->SUPER::set_config;
  #dir are not set in config to enable generic get_dir method access
  #Need to define output dir if we are processing reads

  return;
}

#We aren't really preprocessing the file here
#this is done in read_and_import_data
#initialise_files?

#do we even want to call this?
#Can't we completely skip this in the caller
#based on another flag?
#prepare (does the sorting and pre_processing) and pre_process just does the slice
#caching?
#out_file will be infile if we are just pre-processing
#if is already prepared, then we don't need to make a copy
#Do we even need to add prepared to the file name?
#This is filtered for the chr names


#This also set's the outputfile for use when preparing
#simply don't write the no sort!


sub initialise_input_file {
  my ($self, $filepath, $prepare) = @_;

  #Test file format
  throw("Input file is not bed format:\t$filepath") if ! &is_bed($filepath);
  

  #separate sort keys stop lexical sorting of start/end
  #when faced with a non numerical seq_region_name
  my $sort = ($prepare || ! $self->prepared) ? 'sort -k1,1 -k2,2n -k3,3n ' : '';
  my $is_gzipped = &is_gzipped($filepath);


  if($is_gzipped){
	$sort .= '|' if $sort;
	$self->input_file_operator("gzip -dc %s | $sort ");
  }
{'fraction_non_alphanumeric': 0.08784975086239939, 'fraction_numerical': 0.004752778842468379, 'mean_word_length': 3.782258064516129, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3213918', 'n_tokens_mistral': 4022, 'n_tokens_neox': 3790, 'n_words': 1552}