<commit_before>'''
This module is part the ngs_backbone. It performas analyses related to sequence
annotation.

Created on 12/03/2010

@author: peio
'''
# Copyright 2009 <NAME>, <NAME>, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import shutil, os, copy
from os.path import join, exists
from franklin.backbone.analysis import Analyzer
from tempfile import NamedTemporaryFile
from franklin.pipelines.pipelines import seq_pipeline_runner
from franklin.backbone.blast_runner import (backbone_blast_runner,
                                            make_backbone_blast_db,
                                            guess_blastdb_kind)
from franklin.pipelines.annotation_steps import (annotate_cdna_introns,
                                                 annotate_orthologs,
                                                 annotate_with_descriptions,
                                                 annotate_microsatellites,
                                                 annotate_orfs,
                                                 annotate_gos)
from franklin.sam import create_bam_index
from franklin.pipelines.snv_pipeline_steps import (
                                            unique_contiguous_region_filter,
                                            close_to_intron_filter,
                                            high_variable_region_filter,
                                            close_to_snv_filter,
                                            close_to_limit_filter,
                                            major_allele_freq_filter,
                                            kind_filter,
                                            cap_enzyme_filter,
                                            is_variable_filter,
                                            is_not_variable_filter,
                                            ref_not_in_list,
                                            min_groups)
from franklin.seq.writers import (SequenceWriter, GffWriter, SsrWriter,
                                  OrfWriter, OrthologWriter)
from franklin.snv.writers import VariantCallFormatWriter, SnvIlluminaWriter

from franklin.utils.misc_utils import VersionedPath, xml_itemize
from franklin.utils.cmd_utils import b2gpipe_runner

class AnnotationAnalyzer(Analyzer):
    'It annotates the introns in cdna sequences'

    def _get_inputs_and_prepare_outputs(self):
        'It creates the output dir and it returns the inputs'
        output_dirs = self._create_output_dirs()
        inputs = self._get_input_fpaths()
        return inputs, output_dirs

    def _run_annotation(self, pipeline, configuration, inputs, output_dir):
        'It runs the analysis.'

        self._log({'analysis_started':True})
        pickle_fpaths = inputs['pickle']
        try:
            seqs_fpaths = inputs['input']
        except KeyError:
            seqs_fpaths = []
        seqs_paths = self._get_seq_or_pickle_path(seqs_fpaths, pickle_fpaths)
        for seq_path in seqs_paths:
            seq_fpath = seq_path.last_version
            temp_pickle = NamedTemporaryFile(suffix='.pickle', mode='a',
                                           delete=False)
            in_fhands = {'in_seq': open(seq_fpath)}

            writer = SequenceWriter(fhand=temp_pickle,
                                    file_format='pickle')

            if seq_path.basename in configuration:
                #there is a different configuration for every file to annotate
                config = configuration[seq_path.basename]
            else:
                config = configuration

            seq_pipeline_runner(pipeline, configuration=config,
                                in_fhands=in_fhands,
                                processes=self.threads,
                                writers={'repr': writer})
            temp_pickle.close()
            repr_path = VersionedPath(os.path.join(output_dir,
                                                 seq_path.basename + '.pickle'))
            repr_fpath = repr_path.next_version
            shutil.move(temp_pickle.name, repr_fpath)
        self._log({'analysis_finished':True})

    def _get_seq_or_pickle_path(self, seqs_paths, pickle_paths):
        'It returns for every file the pickle or the seq file'
        pickle_paths_ = dict([(path.basename, path) for path in pickle_paths])
        if not seqs_paths:
            return pickle_paths
        new_seq_paths = []
        for path in seqs_paths:
            basename = path.basename
            if basename in pickle_paths_:
                new_seq_paths.append(pickle_paths_[basename])
            else:
                new_seq_paths.append(path)<commit_msg>bugfix: empty db file should not be used<commit_after>'''
This module is part the ngs_backbone. It performas analyses related to sequence
annotation.

Created on 12/03/2010

@author: peio
'''
# Copyright 2009 <NAME>, <NAME>, COMAV-Univ. Politecnica Valencia
# This file is part of franklin.
# franklin is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# franklin is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR  PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with franklin. If not, see <http://www.gnu.org/licenses/>.

import shutil, os, copy
from os.path import join, exists, getsize
from franklin.backbone.analysis import Analyzer
from tempfile import NamedTemporaryFile
from franklin.pipelines.pipelines import seq_pipeline_runner
from franklin.backbone.blast_runner import (backbone_blast_runner,
                                            make_backbone_blast_db,
                                            guess_blastdb_kind)
from franklin.pipelines.annotation_steps import (annotate_cdna_introns,
                                                 annotate_orthologs,
                                                 annotate_with_descriptions,
                                                 annotate_microsatellites,
                                                 annotate_orfs,
                                                 annotate_gos)
from franklin.sam import create_bam_index
from franklin.pipelines.snv_pipeline_steps import (
                                            unique_contiguous_region_filter,
                                            close_to_intron_filter,
                                            high_variable_region_filter,
                                            close_to_snv_filter,
                                            close_to_limit_filter,
                                            major_allele_freq_filter,
                                            kind_filter,
                                            cap_enzyme_filter,
                                            is_variable_filter,
                                            is_not_variable_filter,
                                            ref_not_in_list,
                                            min_groups)
from franklin.seq.writers import (SequenceWriter, GffWriter, SsrWriter,
                                  OrfWriter, OrthologWriter)
from franklin.snv.writers import VariantCallFormatWriter, SnvIlluminaWriter

from franklin.utils.misc_utils import VersionedPath, xml_itemize
from franklin.utils.cmd_utils import b2gpipe_runner

class AnnotationAnalyzer(Analyzer):
    'It annotates the introns in cdna sequences'

    def _get_inputs_and_prepare_outputs(self):
        'It creates the output dir and it returns the inputs'
        output_dirs = self._create_output_dirs()
        inputs = self._get_input_fpaths()
        return inputs, output_dirs

    def _run_annotation(self, pipeline, configuration, inputs, output_dir):
        'It runs the analysis.'

        self._log({'analysis_started':True})
        pickle_fpaths = inputs['pickle']
        try:
            seqs_fpaths = inputs['input']
        except KeyError:
            seqs_fpaths = []
        seqs_paths = self._get_seq_or_pickle_path(seqs_fpaths, pickle_fpaths)
        for seq_path in seqs_paths:
            seq_fpath = seq_path.last_version
            temp_pickle = NamedTemporaryFile(suffix='.pickle', mode='a',
                                           delete=False)
            in_fhands = {'in_seq': open(seq_fpath)}

            writer = SequenceWriter(fhand=temp_pickle,
                                    file_format='pickle')

            if seq_path.basename in configuration:
                #there is a different configuration for every file to annotate
                config = configuration[seq_path.basename]
            else:
                config = configuration

            seq_pipeline_runner(pipeline, configuration=config,
                                in_fhands=in_fhands,
                                processes=self.threads,
                                writers={'repr': writer})
            temp_pickle.close()
            repr_path = VersionedPath(os.path.join(output_dir,
                                                 seq_path.basename + '.pickle'))
            repr_fpath = repr_path.next_version
            shutil.move(temp_pickle.name, repr_fpath)
        self._log({'analysis_finished':True})

    def _get_seq_or_pickle_path(self, seqs_paths, pickle_paths):
        'It returns for every file the pickle or the seq file'
        pickle_paths_ = dict([(path.basename, path) for path in pickle_paths])
        if not seqs_paths:
            return pickle_paths
        new_seq_paths = []
        for path in seqs_paths:
            basename = path.basename
            if (basename in pickle_paths_ and
                getsize(pickle_paths_[basename].last_version)):
                new_seq_paths.append(pickle_paths_[basename])
            else:
                new_seq_paths.append(path)
{'fraction_non_alphanumeric': 0.054856614246068455, 'fraction_numerical': 0.0025901942645698427, 'mean_word_length': 1.662807881773399, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 88, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1983824', 'n_tokens_mistral': 2670, 'n_tokens_neox': 2560, 'n_words': 709}