<commit_before>
import sys
import json
import base64

import service
from service import engine_api_name

from tabulate import tabulate
from stats import human_fmt

MAX_INLINE_ATTACHMENT = 1024*1024

stream_shortcuts = {
  "discard": '{"Transport":"discard"}'
}

def run(args):
  model_name = args["model-name"]

  in_name = args["in-stream-name"]
  in_desc = get_stream_desc(in_name)

  if "out-stream-name" in args:
    out_name = args["out-stream-name"]
    out_desc = get_stream_desc(out_name)
  else:
    out_name = "discard"
    out_desc = stream_shortcuts["discard"]

  code,body,ctype = service.get_with_ct("model-manage", "/1/model/%s" % model_name)
  if code == 200:
    att = attachments(model_name)
    run1(in_name, in_desc, out_name, out_desc, ctype, model_name, body, attachments=att)
  elif code == 404:
    print "Model '%s' not found" % model_name
  else:
    raise Exception(body)

def attachments(model_name):
  code,body = service.get("model-manage", "/1/model/%s/attachment" % model_name)
  if code == 200:
    return [ get_att(model_name, att_name) for att_name in json.loads(body) ]
  else:
    raise Exception(body)

def get_att(model_name, att_name):
  _,headers = service.head("model-manage",
                  "/1/model/%s/attachment/%s" % (model_name,att_name))
  ctype = headers["content-type"]
  size = int(headers["content-length"])
  if size > MAX_INLINE_ATTACHMENT:

    ## See https://opendatagoup.atlassian.net/wiki/display/FAS/Working+with+large+attachments
    ##
    ## An example of an externalized attachment:
    ##
    ## Content-Type: message/external-body; access-type=x-model-manage; name="att1.zip"
    ##
    ## Content-Type: application/zip
    ## Content-Disposition: attachment; filename="att1.zip"
    ## Content-Length: 1234
    ##

    ext_type = "message/external-body; " + \
               "access-type=x-model-manage; " + \
               "ref=\"urn:fastscore:attachment:%s:%s\"" % (model_name,att_name)

    body = "Content-Type: %s\r\n" % ctype + \
           "Content-Disposition: attachment; filename=\"%s\"\r\n" % att_name + \
           "Content-Length: %d\r\n" % size + \
           "\r\n"

    return (att_name,body,ext_type)

  else:
    code,body = service.get("model-manage",
                    "/1/model/%s/attachment/%s" % (model_name,att_name))
    if code == 200:
      return (att_name,body,ctype)
    else:
      raise Exception(body)

def run1(in_name, in_desc, out_name, out_desc, ctype, model_name, body, attachments=[]):
  headers1 = {"content-type": "application/json",
              "content-disposition": "x-stream; name=\"" + out_name + "\""}
  code3,body3 = service.put_with_headers(engine_api_name(), "/1/job/stream/out", headers1, out_desc)
  if code3 != 204:
    raise Exception(body3)
  print "Output stream set"
  headers2 = {"content-type": "application/json",
              "content-disposition": "x-stream; name=\"" + in_name + "\""}
  code2,body2 = service.put_with_headers(engine_api_name(), "/1/job/stream/in", headers2, in_desc)
  if code2 != 204:
    raise Exception(body2)
  print "Input stream set"
  if attachments == []:
    headers3 = {"content-type": ctype,
                "content-disposition": "x-model; name=\"" + model_name + "\""}
    code1,body1 = service.put_with_headers(engine_api_name(), "/1/job/model", headers3, body)
  else:
    parts = [ ("attachment",x) for x in attachments ]
    parts.append( ("x-model",(model_name,body,ctype)) )
    code1,body1 = service.put_multi(engine_api_name(), "/1/job/model", parts)
  if code1 != 204:
    raise Exception(body1)
  print "Model sent to engine"
  print "The engine is running"

def debug(args):
  model_name = args["model-name"]
  in_desc = get_stream_desc(args["in-stream-name"])
  out_desc = get_stream_desc(args["out-stream-name"]) \
                  if "out-stream-name" in args else stream_shortcuts["discard"]
  code,body,ctype = service.get_with_ct("model-manage", "/1/model/%s" % model_name)
  att = attachments(model_name)
  if code == 200:
    spec = {
      "input": in_desc,
      "output": out_desc,
      "type": ctype_to_type(ctype),
      "model": body,
      "attachments": [{
        "name": att_name,
        "data": base64.b64encode(att_body),
        "type": att_ctype_to_type(att_ctype)
      } for (att_name,att_body,att_ctype) in att ]
    }
    debug1(json.dumps(spec))
  elif code == 404:
    print "Model '%s' not found" % model_name
  else:
    raise Exception(body)

def debug1(data):
  code,report = service.post(engine_api_name(), "/1/job/debug", ctype="application/json", data=data)
  if code == 200:
    print report
  else:
    raise Exception(report)

def get_stream_desc(name):<commit_msg>Forces the debugger to inline attachments<commit_after>
import sys
import json
import base64

import service
from service import engine_api_name

from tabulate import tabulate
from stats import human_fmt

MAX_INLINE_ATTACHMENT = 1024*1024

stream_shortcuts = {
  "discard": '{"Transport":"discard"}'
}

def run(args):
  model_name = args["model-name"]

  in_name = args["in-stream-name"]
  in_desc = get_stream_desc(in_name)

  if "out-stream-name" in args:
    out_name = args["out-stream-name"]
    out_desc = get_stream_desc(out_name)
  else:
    out_name = "discard"
    out_desc = stream_shortcuts["discard"]

  code,body,ctype = service.get_with_ct("model-manage", "/1/model/%s" % model_name)
  if code == 200:
    att = attachments(model_name)
    run1(in_name, in_desc, out_name, out_desc, ctype, model_name, body, attachments=att)
  elif code == 404:
    print "Model '%s' not found" % model_name
  else:
    raise Exception(body)

def attachments(model_name, force_inline=False):
  code,body = service.get("model-manage", "/1/model/%s/attachment" % model_name)
  if code == 200:
    return [ get_att(model_name, att_name, force_inline) for att_name in json.loads(body) ]
  else:
    raise Exception(body)

def get_att(model_name, att_name, force_inline):
  _,headers = service.head("model-manage",
                  "/1/model/%s/attachment/%s" % (model_name,att_name))
  ctype = headers["content-type"]
  size = int(headers["content-length"])
  if size > MAX_INLINE_ATTACHMENT and not force_inline:

    ## See https://opendatagoup.atlassian.net/wiki/display/FAS/Working+with+large+attachments
    ##
    ## An example of an externalized attachment:
    ##
    ## Content-Type: message/external-body; access-type=x-model-manage; name="att1.zip"
    ##
    ## Content-Type: application/zip
    ## Content-Disposition: attachment; filename="att1.zip"
    ## Content-Length: 1234
    ##

    ext_type = "message/external-body; " + \
               "access-type=x-model-manage; " + \
               "ref=\"urn:fastscore:attachment:%s:%s\"" % (model_name,att_name)

    body = "Content-Type: %s\r\n" % ctype + \
           "Content-Disposition: attachment; filename=\"%s\"\r\n" % att_name + \
           "Content-Length: %d\r\n" % size + \
           "\r\n"

    return (att_name,body,ext_type)

  else:
    code,body = service.get("model-manage",
                    "/1/model/%s/attachment/%s" % (model_name,att_name))
    if code == 200:
      return (att_name,body,ctype)
    else:
      raise Exception(body)

def run1(in_name, in_desc, out_name, out_desc, ctype, model_name, body, attachments=[]):
  headers1 = {"content-type": "application/json",
              "content-disposition": "x-stream; name=\"" + out_name + "\""}
  code3,body3 = service.put_with_headers(engine_api_name(), "/1/job/stream/out", headers1, out_desc)
  if code3 != 204:
    raise Exception(body3)
  print "Output stream set"
  headers2 = {"content-type": "application/json",
              "content-disposition": "x-stream; name=\"" + in_name + "\""}
  code2,body2 = service.put_with_headers(engine_api_name(), "/1/job/stream/in", headers2, in_desc)
  if code2 != 204:
    raise Exception(body2)
  print "Input stream set"
  if attachments == []:
    headers3 = {"content-type": ctype,
                "content-disposition": "x-model; name=\"" + model_name + "\""}
    code1,body1 = service.put_with_headers(engine_api_name(), "/1/job/model", headers3, body)
  else:
    parts = [ ("attachment",x) for x in attachments ]
    parts.append( ("x-model",(model_name,body,ctype)) )
    code1,body1 = service.put_multi(engine_api_name(), "/1/job/model", parts)
  if code1 != 204:
    raise Exception(body1)
  print "Model sent to engine"
  print "The engine is running"

def debug(args):
  model_name = args["model-name"]
  in_desc = get_stream_desc(args["in-stream-name"])
  out_desc = get_stream_desc(args["out-stream-name"]) \
                  if "out-stream-name" in args else stream_shortcuts["discard"]
  code,body,ctype = service.get_with_ct("model-manage", "/1/model/%s" % model_name)
  att = attachments(model_name, force_inline=True)
  if code == 200:
    spec = {
      "input": in_desc,
      "output": out_desc,
      "type": ctype_to_type(ctype),
      "model": body,
      "attachments": [{
        "name": att_name,
        "data": base64.b64encode(att_body),
        "type": att_ctype_to_type(att_ctype)
      } for (att_name,att_body,att_ctype) in att ]
    }
    debug1(json.dumps(spec))
  elif code == 404:
    print "Model '%s' not found" % model_name
  else:
    raise Exception(body)

def debug1(data):
  code,report = service.post(engine_api_name(), "/1/job/debug", ctype="application/json", data=data)
  if code == 200:
    print report
  else:
    raise Exception(report)

def get_stream_desc(name):
{'fraction_non_alphanumeric': 0.15149599662874, 'fraction_numerical': 0.017699115044247787, 'mean_word_length': 3.4297713485767614, 'pattern_counts': {'":': 32, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2591393', 'n_tokens_mistral': 3422, 'n_tokens_neox': 3254, 'n_words': 663}