<commit_before># Genenetwork2 specific methods and callback handler
#
# Copyright (C) 2015  <NAME> (<EMAIL>)
#

from __future__ import absolute_import, print_function, division

import numpy as np
import sys
import logging

# logging.basicConfig(level=logging.DEBUG)
# np.set_printoptions()

last_location = None
last_progress = 0

def set_progress_storage(location):
    global storage
    storage = location
    
def progress(location, count, total):
    global last_location
    global last_progress
    
    perc = round(count*100.0/total)
    # print(last_progress,";",perc)
    if perc != last_progress and (location != last_location or perc > 98 or perc > last_progress + 5):
        storage.store("percent_complete",perc)
        logger.info("Progress: %s %d%%" % (location,perc))
        last_location = location
        last_progress = perc

    
def mprint(msg,data):
    """
    Array/matrix print function
    """
    m = np.array(data)
    print(msg,m.shape,"=\n",m)

def callbacks():
    return dict(
        write = sys.stdout.write,
        writeln = print,
        debug = logging.debug,
        info = logging.info,
        warning = logging.warning,
        error = logging.error,
        critical = logging.critical,
        progress = progress,
        mprint = mprint
    )

def uses(*funcs):
    """<commit_msg>Aligned gn2 handlers with standalone<commit_after># Genenetwork2 specific methods and callback handler
#
# Copyright (C) 2015  <NAME> (<EMAIL>)
#

from __future__ import absolute_import, print_function, division

import numpy as np
import sys
import logging

# logging.basicConfig(level=logging.DEBUG)
# np.set_printoptions()

progress_location = None 
progress_current  = None
progress_prev_perc     = None

def progress_default_func(location,count,total):
    global progress_current
    value = round(count*100.0/total)
    progress_current = value
    
progress_func = progress_default_func

def progress_set_func(func):
    global progress_func
    progress_func = func
    
def progress(location, count, total):
    global progress_location
    global progress_prev_perc
    
    perc = round(count*100.0/total)
    if perc != progress_prev_perc and (location != progress_location or perc > 98 or perc > progress_prev_perc + 5):
        progress_func(location, count, total)
        logger.info("Progress: %s %d%%" % (location,perc))
        progress_location = location
        progress_prev_perc = perc
    
def mprint(msg,data):
    """
    Array/matrix print function
    """
    m = np.array(data)
    print(msg,m.shape,"=\n",m)

def callbacks():
    return dict(
        write = sys.stdout.write,
        writeln = print,
        debug = logging.debug,
        info = logging.info,
        warning = logging.warning,
        error = logging.error,
        critical = logging.critical,
        progress = progress,
        mprint = mprint
    )

def uses(*funcs):
    """
{'fraction_non_alphanumeric': 0.10078848131642099, 'fraction_numerical': 0.010284538909838875, 'mean_word_length': 3.0471567267683772, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4136871', 'n_tokens_mistral': 905, 'n_tokens_neox': 867, 'n_words': 223}