<commit_before>from app.lookups.sqlite.base import DatabaseConnection


class SearchSpaceDB(DatabaseConnection):
    def add_tables(self):
        """Creates a searchspace lookup sqlite."""
        self.create_tables(['known_searchspace'])

    def write_peps(self, peps, reverse_seqs):
        """Writes peps to db. We can reverse to be able to look up
        peptides that have some amino acids missing at the N-terminal.
        This way we can still use the index.
        """
        if reverse_seqs:
            peps = [(x[0][::-1],) for x in peps]
        cursor = self.get_cursor()
        cursor.executemany(
            'INSERT INTO known_searchspace(seqs) VALUES (?)', peps)
        self.conn.commit()

    def index_peps(self):
        self.index_column('seqs_index', 'known_searchspace', 'seqs')

    def check_seq_exists(self, seq, amount_ntermwildcards):
        """Look up sequence in sqlite DB. Returns True or False if it
        exists (or not). When looking up a reversed DB with
        ntermwildcards: we reverse the sequence of the pep and add
        a LIKE and %-suffix to the query.
        """
        cursor = self.get_cursor()
        if amount_ntermwildcards > 0:
            seq = seq[::-1]
            sqlseq = '{}{}'.format(seq, '%')
            sql = 'select seqs from known_searchspace where seqs LIKE ?'
            for match in cursor.execute(sql, (sqlseq, )):
                if match[0][:-amount_ntermwildcards] in seq:
                    return True
            return False
        else:
            sql = ('select exists(select seqs from known_searchspace '
                   'where seqs=? limit 1)')
            return cursor.execute(sql, (seq, )).fetchone()[0] == 1<commit_msg>Index searchspace table with COLLATE NOCASE because otherwise the index is not used when having a LIKE statement<commit_after>from app.lookups.sqlite.base import DatabaseConnection


class SearchSpaceDB(DatabaseConnection):
    def add_tables(self):
        """Creates a searchspace lookup sqlite."""
        self.create_tables(['known_searchspace'])

    def write_peps(self, peps, reverse_seqs):
        """Writes peps to db. We can reverse to be able to look up
        peptides that have some amino acids missing at the N-terminal.
        This way we can still use the index.
        """
        if reverse_seqs:
            peps = [(x[0][::-1],) for x in peps]
        cursor = self.get_cursor()
        cursor.executemany(
            'INSERT INTO known_searchspace(seqs) VALUES (?)', peps)
        self.conn.commit()

    def index_peps(self):
        self.index_column('seqs_index', 'known_searchspace',
                          'seqs COLLATE NOCASE')

    def check_seq_exists(self, seq, amount_ntermwildcards):
        """Look up sequence in sqlite DB. Returns True or False if it
        exists (or not). When looking up a reversed DB with
        ntermwildcards: we reverse the sequence of the pep and add
        a LIKE and %-suffix to the query.
        """
        cursor = self.get_cursor()
        if amount_ntermwildcards > 0:
            seq = seq[::-1]
            sqlseq = '{}{}'.format(seq, '%')
            sql = 'select seqs from known_searchspace where seqs LIKE ?'
            for match in cursor.execute(sql, (sqlseq, )):
                if match[0][:-amount_ntermwildcards] in seq:
                    return True
            return False
        else:
            sql = ('select exists(select seqs from known_searchspace '
                   'where seqs=? limit 1)')
            return cursor.execute(sql, (seq, )).fetchone()[0] == 1
{'fraction_non_alphanumeric': 0.09921962095875139, 'fraction_numerical': 0.004459308807134894, 'mean_word_length': 2.2044642857142858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6524439', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1001, 'n_words': 338}