<commit_before># Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
from __future__ import absolute_import

import json
import os
import requests
import tempfile
import uuid

import flask

from digits.log import logger
from digits.pretrained_model import PretrainedModelJob
from digits.utils import auth
from digits.utils.store import StoreParser
from digits.webapp import app, scheduler

blueprint = flask.Blueprint(__name__, __name__)

def save_binary(url, file_name, tmp_dir):
    r = requests.get(os.path.join(url, file_name), stream=True)
    full_path = os.path.join(tmp_dir, file_name)
    with open(full_path,'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
    return full_path

def retrieve_files(url, directory):
    model_url = os.path.join(url, directory)
    tmp_dir = tempfile.mkdtemp()
    info = json.loads(requests.get(os.path.join(model_url, 'info.json')).content)
    weights = save_binary(model_url, info["snapshot file"], tmp_dir)
    if "model file" in info:
        remote_model_file = info["model file"]
    elif "network file" in info:
        remote_model_file = info["network file"]
    else:
        return flask.jsonify({"status": "Missing model definition in info.json"}), 500
    model = save_binary(model_url, remote_model_file, tmp_dir)
    label = save_binary(model_url, info["labels file"], tmp_dir)
    meta_data = info
    return weights, model, label, meta_data

@blueprint.route('/push', methods=['GET'])
def push():
    """
    Create a pre-trained model from model store
    """
    model_id = flask.request.args.get('id')
    model_grand_list = app.config['store_cache'].read()
    found = False
    if model_grand_list is not None:
        for store in model_grand_list:
            for model in model_grand_list[store]['model_list']:
                if model['id'] == model_id:
                    url = model_grand_list[store]['base_url']
                    directory = model['dir_name']
                    found = True
                    break
            if found:
                break
    if not found:
        return 'Error', 404
    else:
        weights, model, label, meta_data = retrieve_files(url, directory)
        job = PretrainedModelJob(
            weights,
            model,
            label,
            meta_data['framework'],<commit_msg>Handle when labels file not in info.json.<commit_after># Copyright (c) 2014-2016, NVIDIA CORPORATION.  All rights reserved.
from __future__ import absolute_import

import json
import os
import requests
import tempfile
import uuid

import flask

from digits.log import logger
from digits.pretrained_model import PretrainedModelJob
from digits.utils import auth
from digits.utils.store import StoreParser
from digits.webapp import app, scheduler

blueprint = flask.Blueprint(__name__, __name__)

def save_binary(url, file_name, tmp_dir):
    r = requests.get(os.path.join(url, file_name), stream=True)
    full_path = os.path.join(tmp_dir, file_name)
    with open(full_path,'wb') as f:
        for chunk in r.iter_content(chunk_size=1024):
            if chunk:  # filter out keep-alive new chunks
                f.write(chunk)
    return full_path

def retrieve_files(url, directory):
    model_url = os.path.join(url, directory)
    tmp_dir = tempfile.mkdtemp()
    info = json.loads(requests.get(os.path.join(model_url, 'info.json')).content)
    weights = save_binary(model_url, info["snapshot file"], tmp_dir)
    if "model file" in info:
        remote_model_file = info["model file"]
    elif "network file" in info:
        remote_model_file = info["network file"]
    else:
        return flask.jsonify({"status": "Missing model definition in info.json"}), 500
    model = save_binary(model_url, remote_model_file, tmp_dir)
    if "labels file" in info:
        label = save_binary(model_url, info["labels file"], tmp_dir)
    else:
        label = None
    meta_data = info
    return weights, model, label, meta_data

@blueprint.route('/push', methods=['GET'])
def push():
    """
    Create a pre-trained model from model store
    """
    model_id = flask.request.args.get('id')
    model_grand_list = app.config['store_cache'].read()
    found = False
    if model_grand_list is not None:
        for store in model_grand_list:
            for model in model_grand_list[store]['model_list']:
                if model['id'] == model_id:
                    url = model_grand_list[store]['base_url']
                    directory = model['dir_name']
                    found = True
                    break
            if found:
                break
    if not found:
        return 'Error', 404
    else:
        weights, model, label, meta_data = retrieve_files(url, directory)
        job = PretrainedModelJob(
            weights,
            model,
            label,
            meta_data['framework'],
{'fraction_non_alphanumeric': 0.09832424793054714, 'fraction_numerical': 0.0072683222289521504, 'mean_word_length': 2.7701674277016743, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5134122', 'n_tokens_mistral': 1504, 'n_tokens_neox': 1441, 'n_words': 407}