<commit_before>import abc
import collections
import copy
import hashlib
import logging
import multiprocessing
import multiprocessing.pool
import os
import time

import annoy
import numexpr as ne
import numpy as np
import tqdm

import reader
import spectrum
import spectrum_match
from config import config


class SpectralLibrary(metaclass=abc.ABCMeta):
    """
    Spectral library search engine.

    Spectral library search engines identify unknown query spectra by comparing each query spectrum against relevant
    known spectra in the spectral library, after which the query spectrum is assigned the identity of the library
    spectrum to which is matches best.
    """

    _config_match_keys = []

    def __init__(self, lib_filename, lib_spectra=None):
        """
        Initialize the spectral library search engine from the given spectral library file.

        Args:
            lib_filename: The spectral library file name.
            lib_spectra: All valid spectra from the spectral library. Avoids re-reading a large spectral library if not
                         `None`.

        Raises:
            FileNotFoundError: The given spectral library file wasn't found or isn't supported.
        """
        try:
            self._library_reader = reader.get_spectral_library_reader(lib_filename, self._get_config_hash())
        except FileNotFoundError as e:
            logging.error(e)
            raise
            
    def _get_config_hash(self):
        config_match = {config_key: config[config_key] for config_key in self._config_match_keys}
        return hashlib.sha1(str(config_match).encode('utf-8')).hexdigest()

    def shutdown(self):
        """
        Release any resources to gracefully shut down.
        """
        pass

    def search(self, query_filename):
        """
        Identify all unknown spectra in the given query file.

        Args:
            query_filename: The query file in .mgf format containing the unknown spectra.

        Returns:
            A list of SpectrumMatch identifications.
        """
        logging.info('Identifying file %s', query_filename)

        # read all spectra in the query file and split based on their precursor charge
        logging.info('Reading all query spectra')
        query_spectra = []
        for query_spectrum in tqdm.tqdm(reader.read_mgf(query_filename), desc='Query spectra read', unit='spectra', smoothing=0):
            # for queries with an unknown charge, try all possible charge states
            for charge in [2, 3] if query_spectrum.precursor_charge is None else [query_spectrum.precursor_charge]:
                query_spectrum_charge = copy.copy(query_spectrum)   # TODO: don't needlessly copy
                query_spectrum_charge.precursor_charge = charge
                query_spectrum_charge.process_peaks()
                if query_spectrum_charge.is_valid():      # discard low-quality spectra
                    query_spectra.append(query_spectrum_charge)

        # sort the spectra based on their precursor charge and precursor mass
        query_spectra.sort(key=lambda spec: (spec.precursor_charge, spec.precursor_mz))

        # identify all spectra
        logging.info('Identifying all query spectra')
        query_matches = {}
        for query_spectrum in tqdm.tqdm(query_spectra, desc='Query spectra identified', unit='spectra'):
            query_match = self._find_match(query_spectrum)

            if query_match.sequence is not None:
                # make sure we only retain the best identification
                # (i.e. for duplicated spectra if the precursor charge was unknown)
                if query_match.query_id not in query_matches or\
                   query_match.search_engine_score > query_matches[query_match.query_id].search_engine_score:
                    query_matches[query_match.query_id] = query_match

        logging.info('Finished identifying file %s', query_filename)

        return query_matches.values()

    def _find_match(self, query):
        """
        Identifies the given query Spectrum.

        Args:
            query: The query Spectrum to be identified.

        Returns:
            A SpectrumMatch identification. If the query couldn't be identified SpectrumMatch.sequence will be None.<commit_msg>Don't needlessly create new query Spectrum objects<commit_after>import abc
import collections
import copy
import hashlib
import logging
import multiprocessing
import multiprocessing.pool
import os
import time

import annoy
import numexpr as ne
import numpy as np
import tqdm

import reader
import spectrum
import spectrum_match
from config import config


class SpectralLibrary(metaclass=abc.ABCMeta):
    """
    Spectral library search engine.

    Spectral library search engines identify unknown query spectra by comparing each query spectrum against relevant
    known spectra in the spectral library, after which the query spectrum is assigned the identity of the library
    spectrum to which is matches best.
    """

    _config_match_keys = []

    def __init__(self, lib_filename, lib_spectra=None):
        """
        Initialize the spectral library search engine from the given spectral library file.

        Args:
            lib_filename: The spectral library file name.
            lib_spectra: All valid spectra from the spectral library. Avoids re-reading a large spectral library if not
                         `None`.

        Raises:
            FileNotFoundError: The given spectral library file wasn't found or isn't supported.
        """
        try:
            self._library_reader = reader.get_spectral_library_reader(lib_filename, self._get_config_hash())
        except FileNotFoundError as e:
            logging.error(e)
            raise
            
    def _get_config_hash(self):
        config_match = {config_key: config[config_key] for config_key in self._config_match_keys}
        return hashlib.sha1(str(config_match).encode('utf-8')).hexdigest()

    def shutdown(self):
        """
        Release any resources to gracefully shut down.
        """
        pass

    def search(self, query_filename):
        """
        Identify all unknown spectra in the given query file.

        Args:
            query_filename: The query file in .mgf format containing the unknown spectra.

        Returns:
            A list of SpectrumMatch identifications.
        """
        logging.info('Identifying file %s', query_filename)

        # read all spectra in the query file and split based on their precursor charge
        logging.info('Reading all query spectra')
        query_spectra = []
        for query_spectrum in tqdm.tqdm(reader.read_mgf(query_filename), desc='Query spectra read', unit='spectra', smoothing=0):
            # for queries with an unknown charge, try all possible charge states
            for charge in [2, 3] if query_spectrum.precursor_charge is None else [query_spectrum.precursor_charge]:
                query_spectrum_charge = copy.copy(query_spectrum) if query_spectrum.precursor_charge is None else query_spectrum
                query_spectrum_charge.precursor_charge = charge
                if query_spectrum_charge.process_peaks().is_valid():      # discard low-quality spectra
                    query_spectra.append(query_spectrum_charge)

        # sort the spectra based on their precursor charge and precursor mass
        query_spectra.sort(key=lambda spec: (spec.precursor_charge, spec.precursor_mz))

        # identify all spectra
        logging.info('Identifying all query spectra')
        query_matches = {}
        for query_spectrum in tqdm.tqdm(query_spectra, desc='Query spectra identified', unit='spectra', smoothing=0):
            query_match = self._find_match(query_spectrum)

            if query_match.sequence is not None:
                # make sure we only retain the best identification
                # (i.e. for duplicated spectra if the precursor charge was unknown)
                if query_match.query_id not in query_matches or\
                   query_match.search_engine_score > query_matches[query_match.query_id].search_engine_score:
                    query_matches[query_match.query_id] = query_match

        logging.info('Finished identifying file %s', query_filename)

        return query_matches.values()

    def _find_match(self, query):
        """
        Identifies the given query Spectrum.

        Args:
            query: The query Spectrum to be identified.

        Returns:
            A SpectrumMatch identification. If the query couldn't be identified SpectrumMatch.sequence will be None.
{'fraction_non_alphanumeric': 0.060433824382322233, 'fraction_numerical': 0.0012759540656536365, 'mean_word_length': 2.630315789473684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5070546', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2089, 'n_words': 749}