<commit_before>import csv
from collections import OrderedDict
from tqdm import tqdm
from database import db
from database import get_or_create
from helpers.parsers import parse_fasta_file
from helpers.parsers import parse_tsv_file
from helpers.parsers import parse_text_file
from helpers.parsers import count_lines
from models import Domain
from models import Gene
from models import InterproDomain
from models import Cancer
from models import Kinase
from models import KinaseGroup
from models import Protein
from models import Site
from models import BadWord
from models import GeneList
from models import CancerGeneListEntry
from import_mutations import load_mutations
from import_mutations import import_mappings
from flask import current_app


IMPORTERS = OrderedDict()


def importer(func):
    IMPORTERS[func.__name__] = func


def import_data(restrict_mutations_to):
    global genes
    genes, proteins = create_proteins_and_genes()
    load_sequences(proteins)
    select_preferred_isoforms(genes)
    load_disorder(proteins)
    load_domains(proteins)
    load_domains_hierarchy()
    load_domains_types()
    load_cancers()
    global kinase_protein_mappings
    kinase_protein_mappings = load_kinase_mapings()
    kinases, groups = load_sites(proteins)
    kinases, groups = load_kinase_classification(proteins, kinases, groups)
    load_pathways(genes)
    print('Adding kinases to the session...')
    db.session.add_all(kinases.values())
    print('Adding groups to the session...')
    db.session.add_all(groups.values())
    del kinases
    del groups
    remove_wrong_proteins(proteins)
    calculate_interactors(proteins)
    db.session.commit()
    with current_app.app_context():
        load_mutations(proteins, restrict_mutations_to)


def load_bad_words(filename='data/bad-words.txt'):

    list_of_profanities = []
    parse_text_file(
        filename,
        list_of_profanities.append,
        file_opener=lambda name: open(name, encoding='utf-8')
    )
    bad_words = [
        BadWord(word=word)
        for word in list_of_profanities
    ]
    return bad_words


@importer
def load_active_driver_gene_lists():
    tcga_gene_list = GeneList(name='TCGA')
    with open(<commit_msg>Make load_bad_words an independent importer<commit_after>import csv
from collections import OrderedDict
from tqdm import tqdm
from database import db
from database import get_or_create
from helpers.parsers import parse_fasta_file
from helpers.parsers import parse_tsv_file
from helpers.parsers import parse_text_file
from helpers.parsers import count_lines
from models import Domain
from models import Gene
from models import InterproDomain
from models import Cancer
from models import Kinase
from models import KinaseGroup
from models import Protein
from models import Site
from models import BadWord
from models import GeneList
from models import CancerGeneListEntry
from import_mutations import load_mutations
from import_mutations import import_mappings
from flask import current_app


IMPORTERS = OrderedDict()


def importer(func):
    IMPORTERS[func.__name__] = func
    return func


def import_data(restrict_mutations_to):
    global genes
    genes, proteins = create_proteins_and_genes()
    load_sequences(proteins)
    select_preferred_isoforms(genes)
    load_disorder(proteins)
    load_domains(proteins)
    load_domains_hierarchy()
    load_domains_types()
    load_cancers()
    global kinase_protein_mappings
    kinase_protein_mappings = load_kinase_mapings()
    kinases, groups = load_sites(proteins)
    kinases, groups = load_kinase_classification(proteins, kinases, groups)
    load_pathways(genes)
    print('Adding kinases to the session...')
    db.session.add_all(kinases.values())
    print('Adding groups to the session...')
    db.session.add_all(groups.values())
    del kinases
    del groups
    remove_wrong_proteins(proteins)
    calculate_interactors(proteins)
    db.session.commit()
    with current_app.app_context():
        load_mutations(proteins, restrict_mutations_to)


@importer
def load_bad_words(filename='data/bad-words.txt'):

    list_of_profanities = []
    parse_text_file(
        filename,
        list_of_profanities.append,
        file_opener=lambda name: open(name, encoding='utf-8')
    )
    bad_words = [
        BadWord(word=word)
        for word in list_of_profanities
    ]
    return bad_words


@importer
def load_active_driver_gene_lists():
    tcga_gene_list = GeneList(name='TCGA')
    with open(
{'fraction_non_alphanumeric': 0.06224899598393574, 'fraction_numerical': 0.0004462293618920125, 'mean_word_length': 4.534567901234568, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6502918', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1328, 'n_words': 348}