<commit_before>import json

import libdlt.protocol.ibp.factory as ibp
import libdlt.protocol.ceph.factory as ceph
#import libdlt.protocol.rdma.factory as rdma
#import libdlt.protocol.gdrive.factory as gdrive

from lace import logging
from lace.logging import trace
from libdlt.protocol.ibp.allocation import IBP_EXTENT_URI
from libdlt.protocol.ceph.allocation import CEPH_EXTENT_URI
#from libdlt.protocol.rdma.allocation import RDMA_EXTENT_URI
#from libdlt.protocol.gdrive.allocation import GOOGLE_EXTENT_URI

from unis.models import Extent

PROTOCOL_MAP = {
    "ceph": ceph,
    "ibp": ibp,
#    "rdma": rdma,
#    "google": gdrive
}

SCHEMA_MAP = {
    CEPH_EXTENT_URI: ceph,
    IBP_EXTENT_URI: ibp,
#    RDMA_EXTENT_URI: rdma,
#    GOOGLE_EXTENT_URI: gdrive
}

@trace.info("libdlt.factory")
def buildAllocation(json):
    if type(json) is str:
        try:
            json = json.loads(json)
        except Exception as exp:
            logging.getLogger("libdlt").warn("{func:>20}| Could not decode allocation - {exp}".format(func = "buildAllocation", exp = exp))
            return False

    if isinstance(json, Extent):
        schema = getattr(json, "$schema")
    else:
        schema = json["schema"]
    return SCHEMA_MAP[schema].buildAllocation(json)

@trace.info("libdlt.factory")
def makeProxy(alloc):
    return SCHEMA_MAP[getattr(alloc, "$schema", IBP_EXTENT_URI)].services.ProtocolService()

@trace.info("libdlt.factory")
def makeAllocation(data, offset, depot, **kwds):
    return PROTOCOL_MAP[depot.scheme].makeAllocation(data, offset, depot, **kwds)<commit_msg>Proxies automatically insert depot information for allocations<commit_after>import json

import libdlt.protocol.ibp.factory as ibp
import libdlt.protocol.ceph.factory as ceph
#import libdlt.protocol.rdma.factory as rdma
#import libdlt.protocol.gdrive.factory as gdrive

from lace import logging
from lace.logging import trace
from libdlt.protocol.ibp.allocation import IBP_EXTENT_URI
from libdlt.protocol.ceph.allocation import CEPH_EXTENT_URI
from libdlt.depot import Depot
#from libdlt.protocol.rdma.allocation import RDMA_EXTENT_URI
#from libdlt.protocol.gdrive.allocation import GOOGLE_EXTENT_URI

from unis.models import Extent

PROTOCOL_MAP = {
    "ceph": ceph,
    "ibp": ibp,
#    "rdma": rdma,
#    "google": gdrive
}

SCHEMA_MAP = {
    CEPH_EXTENT_URI: ceph,
    IBP_EXTENT_URI: ibp,
#    RDMA_EXTENT_URI: rdma,
#    GOOGLE_EXTENT_URI: gdrive
}

@trace.info("libdlt.factory")
def buildAllocation(json):
    if type(json) is str:
        try:
            json = json.loads(json)
        except Exception as exp:
            logging.getLogger("libdlt").warn("{func:>20}| Could not decode allocation - {exp}".format(func = "buildAllocation", exp = exp))
            return False

    if isinstance(json, Extent):
        schema = getattr(json, "$schema")
    else:
        schema = json["schema"]
    return SCHEMA_MAP[schema].buildAllocation(json)

@trace.info("libdlt.factory")
def makeProxy(alloc):
    if not hasattr(alloc, 'depot'):
        alloc.depot = Depot(getattr(alloc, 'location', ''))
    return SCHEMA_MAP[getattr(alloc, "$schema", IBP_EXTENT_URI)].services.ProtocolService()

@trace.info("libdlt.factory")
def makeAllocation(data, offset, depot, **kwds):
    return PROTOCOL_MAP[depot.scheme].makeAllocation(data, offset, depot, **kwds)
{'fraction_non_alphanumeric': 0.1139354066985646, 'fraction_numerical': 0.0011961722488038277, 'mean_word_length': 4.210280373831775, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6481045', 'n_tokens_mistral': 1167, 'n_tokens_neox': 1155, 'n_words': 252}