<commit_before>import sys
import json
import os
import logging
from gatelfdata import Dataset
#from gatelfkerasjson import ???
from gatelfkerasjson import KerasWrapperImpl1

modelprefix=sys.argv[1]
metafile=sys.argv[2]
datadir=sys.argv[3]

# Set up logging
logger = logging.getLogger("gatelfdata")
logger.setLevel(logging.ERROR)
logger = logging.getLogger("gatelfkerasjson")
logger.setLevel(logging.DEBUG)
streamhandler = logging.StreamHandler(stream=sys.stderr)
formatter = logging.Formatter(
                '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
streamhandler.setFormatter(formatter)
logger.addHandler(streamhandler)
filehandler = logging.FileHandler(os.path.join(datadir,"FileJsonKerasWrapper.train.log"))
logger.addHandler(filehandler)

# restore the wrapper
ds = Dataset(metafile, targets_need_padding=False)
wrapper = KerasWrapperImpl1(ds)
wrapper.loadModel(modelprefix)

with sys.stdin as infile:
    for line in infile:
        print("PYTHON FileJsonKeras APPLICATION, input=",line,file=sys.stderr)
        if line == "STOP":
            break
        # TODO: currently the LF sends individual instances here, we may want to change
        # However we need to always apply to a set of instances, so wrap into another array
        instancedata = json.loads(line)
        # TODO: better error handling: put the apply call into a try block and catch any error, also
        # check returned data. If there is a problem send back in the map we return!!
        # NOTE: the  LF expects to get a map with the following elements:
        # status: must be "ok", anything else is interpreted as an error
        # output: the actual prediction: gets extracted from the returned data here
        # confidence: some confidence/probability score for the output, may be null: this gets extracted
        # from our returned data here
        # confidences: a map with confidences for all labels, may be null: this is NOT SUPPORTED in the LF yet!
        preds=wrapper.applyModel(instancedata)
        print("PYTHON  APPLICATION, preds=", preds, file=sys.stderr)
        # preds are a list of one or two lists, where the first list contains all the labels and the second
        # list contains all the confidences in the order used by the model. 
        # For now we just extract the label or for a sequence, the list of labels, knowing that for now we always process only one instance/sequence!
        ret = {"status":"ok", "output":preds[0][0]}
        print("PYTHON FileJsonKeras APPLICATION, return=", ret, file=sys.stderr)
        print(json.dumps(ret))
        # TODO: IMPORTANT!!! What the model returns is currently different from what the LF code expects!!!
        sys.stdout.flush()<commit_msg>fix keras apply.py<commit_after>import sys
import json
import os
import logging
from gatelfdata import Dataset
#from gatelfkerasjson import ???
from gatelfkerasjson import KerasWrapperImpl1

modelprefix=sys.argv[1]
metafile=sys.argv[2]
datadir=sys.argv[3]

# Set up logging
logger = logging.getLogger("gatelfdata")
logger.setLevel(logging.ERROR)
logger = logging.getLogger("gatelfkerasjson")
logger.setLevel(logging.DEBUG)
streamhandler = logging.StreamHandler(stream=sys.stderr)
formatter = logging.Formatter(
                '%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
streamhandler.setFormatter(formatter)
logger.addHandler(streamhandler)
filehandler = logging.FileHandler(os.path.join(datadir,"FileJsonKerasWrapper.train.log"))
logger.addHandler(filehandler)

# restore the wrapper
ds = Dataset(metafile, targets_need_padding=False)
wrapper = KerasWrapperImpl1(ds)
wrapper.loadModel(modelprefix)

with sys.stdin as infile:
    for line in infile:
        print("PYTHON FileJsonKeras APPLICATION, input=",line,file=sys.stderr)
        if line == "STOP":
            break
        # TODO: currently the LF sends individual instances here, we may want to change
        # However we need to always apply to a set of instances, so wrap into another array
        instancedata = json.loads(line)
        # TODO: better error handling: put the apply call into a try block and catch any error, also
        # check returned data. If there is a problem send back in the map we return!!
        # NOTE: the  LF expects to get a map with the following elements:
        # status: must be "ok", anything else is interpreted as an error
        # output: the actual prediction: gets extracted from the returned data here
        # confidence: some confidence/probability score for the output, may be null: this gets extracted
        # from our returned data here
        # confidences: a map with confidences for all labels, may be null: this is NOT SUPPORTED in the LF yet!
        preds=wrapper.applyModel(instancedata)
        print("PYTHON  APPLICATION, preds=", preds, file=sys.stderr)
        # preds are a list of one or two lists, where the first list contains all the labels and the second
        # list contains all the confidences in the order used by the model. 
        # For now we just extract the label or for a sequence, the list of labels, knowing that for now we always process only one instance/sequence!
        ret = {"status":"ok", "output":preds[0]}
        print("PYTHON FileJsonKeras APPLICATION, return=", ret, file=sys.stderr)
        print(json.dumps(ret))
        # TODO: IMPORTANT!!! What the model returns is currently different from what the LF code expects!!!
        sys.stdout.flush()
{'fraction_non_alphanumeric': 0.0816813509544787, 'fraction_numerical': 0.0034875183553597653, 'mean_word_length': 3.7055267702936097, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3833056', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1499, 'n_words': 585}