<commit_before># author: <NAME>
from cassandra.cluster import Cluster
from hecuba.datastore import *
from hecuba.iter import Block
from hecuba.dict import *
from hecuba.storageobj import *
from hecuba.storageobjix import *
import collections

from conf.hecuba_params import *
from conf.apppath import apppath


def start_task(params):
    if not 'prefetch_activated' in globals():
        global prefetch_activated
        prefetch_activated = True
    if not 'batch_activated' in globals():
        global batch_activated
        batch_activated = True
    if prefetch_activated or batch_activated:
        path = apppath + '/conf/imports.py'
        file = open(path, 'r')
        for line in file:
            exec line

    if batch_activated:
        for param in params:
            if isinstance(param, StorageObj) or isinstance(param, StorageObjIx) or isinstance(param, Block):
                param.cntxt = context(param)
                param.cntxt.__enter__()

def end_task(params):
    if not 'prefetch_activated' in globals():
        global prefetch_activated
        prefetch_activated = True
    if not 'batch_activated' in globals():
        global batch_activated
        batch_activated = True
    if prefetch_activated or batch_activated:
        path = apppath + '/conf/imports.py'
        file = open(path, 'r')
        for line in file:
            exec line

    if batch_activated:
        for param in params:
            if isinstance(param, Block) or isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
                try:
                    param.cntxt.__exit__()
                except Exception as e:
                    print "error trying to exit context:", e

    if prefetch_activated:
        for param in params:
            if isinstance(param, Block):
                keys = param.storageobj.keyList[param.storageobj.__class__.__name__]
                exec("persistentdict = param.storageobj." + str(keys[0]))
                if persistentdict.prefetch == True:
                    try:
                        persistentdict.end_prefetch()
                    except Exception as e:
                        print "error trying to prefetch:", e

    if not 'statistics_activated' in globals():
        global statistics_activated
        statistics_activated = True
    if statistics_activated:
        for param in params:
            if isinstance(param, Block):
                param.storageobj.statistics()
            if isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
                param.statistics()

def getByID(objid):
    path = apppath + '/conf/imports.py'
    file = open(path, 'r')
    for line in file:
        exec(line)
    objidsplit = objid.split("_")
    print "objidsplit:", objidsplit

    if len(objidsplit) == 2:
        result = ''
        exec('result = ' + objidsplit[0] + '()')
        print "str(objidsplit[0]):", str(objidsplit[0])
        result.getByName(str(objidsplit[0]))
        return result

    else:
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        (ksp,tab,dict_name,obj_type,entryPoint,port,tokens) = session.execute("SELECT ksp,tab,dict_name,obj_type,entryPoint,port,tkns FROM hecuba.workers WHERE workerid = %s",(objectID))[0]
        if obj_type == 'qbeast':
            print "indexed storageobj"
            blockid = objid
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            if not 'prefetch_activated' in globals():
                global prefetch_activated
                prefetch_activated = True
            for position in tokens: 
                for key, val in odtokenmap.iteritems():
                    # (self, peer, keynames, tablename, blockkeyspace, myuuid)
                    b = IxBlock(str(val), str(objidsplit[1]), objidsplit[2], objidsplit[0], myuuid)
                    exec("b.storageobj = " + str(objidsplit[2]) + "('" + str(objidsplit[2]) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()<commit_msg>fixing the api<commit_after># author: <NAME>
from cassandra.cluster import Cluster
from hecuba.datastore import *
from hecuba.iter import Block
from hecuba.dict import *
from hecuba.storageobj import *
from hecuba.storageobjix import *
import collections

from conf.hecuba_params import *
from conf.apppath import apppath


def start_task(params):
    if not 'prefetch_activated' in globals():
        global prefetch_activated
        prefetch_activated = True
    if not 'batch_activated' in globals():
        global batch_activated
        batch_activated = True
    if prefetch_activated or batch_activated:
        path = apppath + '/conf/imports.py'
        file = open(path, 'r')
        for line in file:
            exec line

    if batch_activated:
        for param in params:
            if isinstance(param, StorageObj) or isinstance(param, StorageObjIx) or isinstance(param, Block):
                param.cntxt = context(param)
                param.cntxt.__enter__()

def end_task(params):
    if not 'prefetch_activated' in globals():
        global prefetch_activated
        prefetch_activated = True
    if not 'batch_activated' in globals():
        global batch_activated
        batch_activated = True
    if prefetch_activated or batch_activated:
        path = apppath + '/conf/imports.py'
        file = open(path, 'r')
        for line in file:
            exec line

    if batch_activated:
        for param in params:
            if isinstance(param, Block) or isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
                try:
                    param.cntxt.__exit__()
                except Exception as e:
                    print "error trying to exit context:", e

    if prefetch_activated:
        for param in params:
            if isinstance(param, Block):
                keys = param.storageobj.keyList[param.storageobj.__class__.__name__]
                exec("persistentdict = param.storageobj." + str(keys[0]))
                if persistentdict.prefetch == True:
                    try:
                        persistentdict.end_prefetch()
                    except Exception as e:
                        print "error trying to prefetch:", e

    if not 'statistics_activated' in globals():
        global statistics_activated
        statistics_activated = True
    if statistics_activated:
        for param in params:
            if isinstance(param, Block):
                param.storageobj.statistics()
            if isinstance(param, StorageObj) or isinstance(param, StorageObjIx):
                param.statistics()

def getByID(objid):
    path = apppath + '/conf/imports.py'
    file = open(path, 'r')
    for line in file:
        exec(line)
    objidsplit = objid.split("_")
    print "objidsplit:", objidsplit

    if len(objidsplit) == 2:
        result = ''
        exec('result = ' + objidsplit[0] + '()')
        print "str(objidsplit[0]):", str(objidsplit[0])
        result.getByName(str(objidsplit[0]))
        return result

    else:
        cluster = Cluster(contact_points=contact_names, port=nodePort, protocol_version=2)
        session = cluster.connect()
        (ksp,tab,dict_name,obj_type,entryPoint,port,tokens) = session.execute("SELECT ksp,tab,dict_name,obj_type,entryPoint,port,tkns FROM hecuba.workers WHERE workerid = %s",(objid))[0]
        if obj_type == 'qbeast':
            print "indexed storageobj"
            blockid = objid
            metadata = cluster.metadata
            tokenmap = metadata.token_map.token_to_host_owner
            odtokenmap = collections.OrderedDict(sorted(tokenmap.items()))
            if not 'prefetch_activated' in globals():
                global prefetch_activated
                prefetch_activated = True
            for position in tokens: 
                for key, val in odtokenmap.iteritems():
                    # (self, peer, keynames, tablename, blockkeyspace, myuuid)
                    b = IxBlock(str(val), str(objidsplit[1]), objidsplit[2], objidsplit[0], myuuid)
                    exec("b.storageobj = " + str(objidsplit[2]) + "('" + str(objidsplit[2]) + "')")
                    if prefetch_activated:
                        b.storageobj.init_prefetch(b)
                    session.shutdown()
                    cluster.shutdown()
{'fraction_non_alphanumeric': 0.09131040892193308, 'fraction_numerical': 0.0030204460966542753, 'mean_word_length': 2.1615864854939404, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454577', 'n_tokens_mistral': 2458, 'n_tokens_neox': 2333, 'n_words': 605}