<commit_before>import prolog
import py
import time
from prolog.interpreter import helper, term, error
from prolog.builtin.register import expose_builtin


# TODO: make this continuation based and return all statistics
# RPYTHON??
@expose_builtin("statistics", unwrap_spec=["atom", "obj"])
def impl_statistics(engine, heap, stat_name, value):
    t = []
    if stat_name == 'runtime':        
        t = [clock_time(engine), clocktime_since_last_call(engine)]
    if stat_name == 'walltime':
        t = [walltime(engine), walltime_since_last_call(engine)]
    l = [term.Number(x) for x in t]
    helper.wrap_list(l).unify(value, heap)

def clock_time(engine):
    engine.clocks['cpu_now'] = int(time.clock()*1000)
    return engine.clocks['cpu_now']
    
def clocktime_since_last_call(engine):
    t = engine.clocks['cpu_now'] - engine.clocks['cpu_last']
    engine.clocks['cpu_last'] = engine.clocks['cpu_now']
    return t
    
def walltime(engine):
    engine.clocks['wall_now'] = int((time.time()-engine.start)*1000)
    return engine.clocks['wall_now']

def walltime_since_last_call(engine):
    t = engine.clocks['wall_now'] - engine.clocks['wall_last']
    engine.clocks['wall_last'] = engine.clocks['wall_now']
    return t
    
def reset_clocks(engine):
    engine.clocks = {'cpu_last': 0, 'cpu_now': 0, 'wall_now':0, 'wall_last':0}<commit_msg>improve walltime results and do not overflow ;)<commit_after>import prolog
import py
import time
from prolog.interpreter import helper, term, error
from prolog.builtin.register import expose_builtin


# TODO: make this continuation based and return all statistics
# RPYTHON??
@expose_builtin("statistics", unwrap_spec=["atom", "obj"])
def impl_statistics(engine, heap, stat_name, value):
    t = []
    if stat_name == 'runtime':
        t = [clock_time(engine), clocktime_since_last_call(engine)]
    if stat_name == 'walltime':
        t = [walltime(engine), walltime_since_last_call(engine)]
    l = [term.Number(x) for x in t]
    helper.wrap_list(l).unify(value, heap)

def clock_time(engine):
    engine.clocks['cpu_now'] = int(time.clock()*1000)
    return engine.clocks['cpu_now']

def clocktime_since_last_call(engine):
    t = engine.clocks['cpu_now'] - engine.clocks['cpu_last']
    engine.clocks['cpu_last'] = engine.clocks['cpu_now']
    return t

def walltime(engine):
    # XXX Unhack
    engine.clocks['wall_now'] = millis()
    return engine.clocks['wall_now']

def walltime_since_last_call(engine):
    t = engine.clocks['wall_now'] - engine.clocks['wall_last']
    engine.clocks['wall_last'] = engine.clocks['wall_now']
    return t

def reset_clocks(engine):
    engine.clocks = {'cpu_last': 0, 'cpu_now': 0, 'wall_now':0, 'wall_last':0}

def millis():
    x = time.time()
    y = int(x)
    dec = int((x - y)*100)
    h = y % 100
    return (h*100+dec)
{'fraction_non_alphanumeric': 0.14290767598160595, 'fraction_numerical': 0.010258224266006368, 'mean_word_length': 4.428023032629558, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4598495', 'n_tokens_mistral': 1004, 'n_tokens_neox': 983, 'n_words': 210}