<commit_before>import time
import psutil
from tqdm import tqdm
from app import app
from app import db
from website.models import Protein
from website.models import Cancer
from website.models import Mutation
from website.models import Site
from website.models import Kinase
from website.models import KinaseGroup
# from website.models import CodingSequenceVariant
# from website.models import SingleNucleotideVariation
from website.models import Gene
from website.models import Domain
from website.models import InterproDomain


# remember to `set global max_allowed_packet=1073741824;` (it's max - 1GB)
# (otherwise MySQL server will be gone)
MEMORY_LIMIT = 2e9  # it can be greater than sql ma packet, since we will be
# counting a lot of overhead into the current memory usage. Adjust manually.

MEMORY_PERCENT_LIMIT = 80


def system_memory_percent():
    return psutil.virtual_memory().percent


def import_data():
    proteins = create_proteins_and_genes()
    load_sequences(proteins)
    select_preferred_isoforms()
    load_domains(proteins)
    load_disorder(proteins)
    # cancers = load_cancers()
    # load_mutations(proteins, cancers)
    #kinases, groups = load_sites(proteins)
    #kinases, groups = load_kinase_classification(proteins, kinases, groups)
    #db.session.add_all(kinases.values())
    #db.session.add_all(groups.values())
    print('Added kinases')
    print('Added proteins')
    #del kinases
    #del groups
    #del proteins
    # db.session.add_all(cancers.values())
    # print('Added cancers')
    print('Memory usage before commit: ', memory_usage())
    db.session.commit()
    print('Memory usage before cleaning: ', memory_usage())
    # del cancers
    print('Memory usage after cleaning: ', memory_usage())
    with app.app_context():
        load_mimp_mutations()   # this requires having sites already loaded
    start = time.clock()
    with app.app_context():
        proteins = {protein.refseq: protein for protein in Protein.query.all()}
        import_mappings(proteins)
    end = time.clock()
    print('Imported mappings in:', end - start)
    print('Memory usage after mappings: ', memory_usage())


def buffered_readlines(file_handle, line_count=5000):
    is_eof = False
    while not is_eof:
        buffer = []<commit_msg>Adjust log message<commit_after>import time
import psutil
from tqdm import tqdm
from app import app
from app import db
from website.models import Protein
from website.models import Cancer
from website.models import Mutation
from website.models import Site
from website.models import Kinase
from website.models import KinaseGroup
# from website.models import CodingSequenceVariant
# from website.models import SingleNucleotideVariation
from website.models import Gene
from website.models import Domain
from website.models import InterproDomain


# remember to `set global max_allowed_packet=1073741824;` (it's max - 1GB)
# (otherwise MySQL server will be gone)
MEMORY_LIMIT = 2e9  # it can be greater than sql ma packet, since we will be
# counting a lot of overhead into the current memory usage. Adjust manually.

MEMORY_PERCENT_LIMIT = 80


def system_memory_percent():
    return psutil.virtual_memory().percent


def import_data():
    proteins = create_proteins_and_genes()
    load_sequences(proteins)
    select_preferred_isoforms()
    load_domains(proteins)
    load_disorder(proteins)
    # cancers = load_cancers()
    # load_mutations(proteins, cancers)
    #kinases, groups = load_sites(proteins)
    #kinases, groups = load_kinase_classification(proteins, kinases, groups)
    #print('Adding kinases to the session...')
    #db.session.add_all(kinases.values())
    #print('Adding groups to the session...')
    #db.session.add_all(groups.values())
    #del kinases
    #del groups
    #del proteins
    # db.session.add_all(cancers.values())
    # print('Added cancers')
    print('Memory usage before commit: ', memory_usage())
    db.session.commit()
    print('Memory usage before cleaning: ', memory_usage())
    # del cancers
    print('Memory usage after cleaning: ', memory_usage())
    with app.app_context():
        load_mimp_mutations()   # this requires having sites already loaded
    start = time.clock()
    with app.app_context():
        proteins = {protein.refseq: protein for protein in Protein.query.all()}
        import_mappings(proteins)
    end = time.clock()
    print('Imported mappings in:', end - start)
    print('Memory usage after mappings: ', memory_usage())


def buffered_readlines(file_handle, line_count=5000):
    is_eof = False
    while not is_eof:
        buffer = []
{'fraction_non_alphanumeric': 0.08867637470115192, 'fraction_numerical': 0.008259074114322973, 'mean_word_length': 4.1766029246344205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4589484', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1300, 'n_words': 410}