<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2017-2022 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A local Predictive Deepnet.

This module defines a Deepnet to make predictions locally or
embedded into your application without needing to send requests to
BigML.io.

This module can help you enormously to
reduce the latency for each prediction and let you use your models
offline.

You can also visualize your predictive model in IF-THEN rule format
and even generate a python function that implements the model.

Example usage (assuming that you have previously set up the BIGML_USERNAME
and BIGML_API_KEY environment variables and that you own the model/id below):

from bigml.api import BigML
from bigml.deepnet import Deepnet

api = BigML()

deepnet = Deepnet('deepnet/5026965515526876630001b2')
deepnet.predict({"petal length": 3, "petal width": 1})

"""
import logging
import os
import sys
import tempfile

from functools import cmp_to_key

# avoiding tensorflow info logging
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
logging.getLogger('tensorflow').setLevel(logging.ERROR)

laminar_version = False

try:
    import tensorflow as tf
    tf.autograph.set_verbosity(0)
    from PIL import Image
except ModuleNotFoundError:
    laminar_version = True

from bigml.api import FINISHED
from bigml.api import get_status, get_api_connection, get_deepnet_id
from bigml.util import cast, use_cache, load, PRECISION
from bigml.basemodel import get_resource_dict, extract_objective
from bigml.modelfields import ModelFields
from bigml.laminar.constants import NUMERIC
from bigml.model import parse_operating_point, sort_categories
from bigml.constants import REGIONS, REGIONS_OPERATION_SETTINGS, \
    DEFAULT_OPERATION_SETTINGS, REGION_SCORE_ALIAS, REGION_SCORE_THRESHOLD, \
    DECIMALS

import bigml.laminar.numpy_ops as net
import bigml.laminar.preprocess_np as pp
try:
    from sensenet.models.wrappers import create_model
except ModuleNotFoundError:
    laminar_version = True

LOGGER = logging.getLogger('BigML')

MEAN = "mean"
STANDARD_DEVIATION = "stdev"

IMAGE = "image"
REMOTE_SETTINGS = {"iou_threshold": 0.2}
TEMP_DIR = "/tmp"
TOP_SIZE = 512


def moments(amap):
    """Extracts mean and stdev

    """
    return amap[MEAN], amap[STANDARD_DEVIATION]


def expand_terms(terms_list, input_terms):
    """Builds a list of occurrences for all the available terms

    """
    terms_occurrences = [0.0] * len(terms_list)
    for term, occurrences in input_terms:
        index = terms_list.index(term)
        terms_occurrences[index] = occurrences
    return terms_occurrences


def to_relative_coordinates(image_file, regions_list):
    """Transforms predictions with regions having absolute pixels regions
    to the relative format used remotely and rounds to the same precision.
    """

    if regions_list:
        image_obj = Image.open(image_file)
        width, height = image_obj.size
        for index, region in enumerate(regions_list):
            [xmin, ymin, xmax, ymax] = region["box"]
            region["box"] = [round(xmin / width, DECIMALS),
                             round(ymin / height, DECIMALS),
                             round(xmax / width, DECIMALS),
                             round(ymax / height, DECIMALS)]
            region["score"] = round(region["score"], DECIMALS)
            regions_list[index] = region
    return regions_list


def remote_preprocess(image_file):
    """Emulating the preprocessing of images done in the backend to
    get closer results in local predictions
    """
    # converting to jpg
    image = Image.open(image_file)
    if not (image_file.lower().endswith(".jpg") or
            image_file.lower().endswith(".jpeg")):
        image = image.convert('RGB')
    # resizing to TOP_SIZE
    width, height = image.size
    if width > TOP_SIZE or height > TOP_SIZE:
        if width > height:
            ratio = height / width
            image = image.resize((TOP_SIZE , int(ratio * TOP_SIZE)),
                                 Image.BICUBIC)
        else:
            ratio = width / height
            image = image.resize((int(ratio * TOP_SIZE), TOP_SIZE),
                                  Image.BICUBIC)
    with tempfile.NamedTemporaryFile() as temp_fp:
        tmp_file_name = os.path.join(TEMP_DIR, "%s.jpg" % temp_fp.name)
        # compressing to 90%
        image.save(tmp_file_name, quality=90)
    return tmp_file_name


class Deepnet(ModelFields):
    """ A lightweight wrapper around Deepnet model.
<commit_msg>Make temporary file stick around after close<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2017-2022 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""A local Predictive Deepnet.

This module defines a Deepnet to make predictions locally or
embedded into your application without needing to send requests to
BigML.io.

This module can help you enormously to
reduce the latency for each prediction and let you use your models
offline.

You can also visualize your predictive model in IF-THEN rule format
and even generate a python function that implements the model.

Example usage (assuming that you have previously set up the BIGML_USERNAME
and BIGML_API_KEY environment variables and that you own the model/id below):

from bigml.api import BigML
from bigml.deepnet import Deepnet

api = BigML()

deepnet = Deepnet('deepnet/5026965515526876630001b2')
deepnet.predict({"petal length": 3, "petal width": 1})

"""
import logging
import os
import sys
import tempfile

from functools import cmp_to_key

# avoiding tensorflow info logging
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
logging.getLogger('tensorflow').setLevel(logging.ERROR)

laminar_version = False

try:
    import tensorflow as tf
    tf.autograph.set_verbosity(0)
    from PIL import Image
except ModuleNotFoundError:
    laminar_version = True

from bigml.api import FINISHED
from bigml.api import get_status, get_api_connection, get_deepnet_id
from bigml.util import cast, use_cache, load, PRECISION
from bigml.basemodel import get_resource_dict, extract_objective
from bigml.modelfields import ModelFields
from bigml.laminar.constants import NUMERIC
from bigml.model import parse_operating_point, sort_categories
from bigml.constants import REGIONS, REGIONS_OPERATION_SETTINGS, \
    DEFAULT_OPERATION_SETTINGS, REGION_SCORE_ALIAS, REGION_SCORE_THRESHOLD, \
    DECIMALS

import bigml.laminar.numpy_ops as net
import bigml.laminar.preprocess_np as pp
try:
    from sensenet.models.wrappers import create_model
except ModuleNotFoundError:
    laminar_version = True

LOGGER = logging.getLogger('BigML')

MEAN = "mean"
STANDARD_DEVIATION = "stdev"

IMAGE = "image"
REMOTE_SETTINGS = {"iou_threshold": 0.2}
TEMP_DIR = "/tmp"
TOP_SIZE = 512


def moments(amap):
    """Extracts mean and stdev

    """
    return amap[MEAN], amap[STANDARD_DEVIATION]


def expand_terms(terms_list, input_terms):
    """Builds a list of occurrences for all the available terms

    """
    terms_occurrences = [0.0] * len(terms_list)
    for term, occurrences in input_terms:
        index = terms_list.index(term)
        terms_occurrences[index] = occurrences
    return terms_occurrences


def to_relative_coordinates(image_file, regions_list):
    """Transforms predictions with regions having absolute pixels regions
    to the relative format used remotely and rounds to the same precision.
    """

    if regions_list:
        image_obj = Image.open(image_file)
        width, height = image_obj.size
        for index, region in enumerate(regions_list):
            [xmin, ymin, xmax, ymax] = region["box"]
            region["box"] = [round(xmin / width, DECIMALS),
                             round(ymin / height, DECIMALS),
                             round(xmax / width, DECIMALS),
                             round(ymax / height, DECIMALS)]
            region["score"] = round(region["score"], DECIMALS)
            regions_list[index] = region
    return regions_list


def remote_preprocess(image_file):
    """Emulating the preprocessing of images done in the backend to
    get closer results in local predictions
    """
    # converting to jpg
    image = Image.open(image_file)
    if not (image_file.lower().endswith(".jpg") or
            image_file.lower().endswith(".jpeg")):
        image = image.convert('RGB')
    # resizing to TOP_SIZE
    width, height = image.size
    if width > TOP_SIZE or height > TOP_SIZE:
        if width > height:
            ratio = height / width
            image = image.resize((TOP_SIZE , int(ratio * TOP_SIZE)),
                                 Image.BICUBIC)
        else:
            ratio = width / height
            image = image.resize((int(ratio * TOP_SIZE), TOP_SIZE),
                                  Image.BICUBIC)
    with tempfile.NamedTemporaryFile(delete=False) as temp_fp:
        tmp_file_name = os.path.join(TEMP_DIR, "%s.jpg" % temp_fp.name)
        # compressing to 90%
        image.save(tmp_file_name, quality=90)
    return tmp_file_name


class Deepnet(ModelFields):
    """ A lightweight wrapper around Deepnet model.

{'fraction_non_alphanumeric': 0.07909099930534881, 'fraction_numerical': 0.01012206013694552, 'mean_word_length': 3.3420939250323136, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1950022', 'n_tokens_mistral': 3097, 'n_tokens_neox': 2924, 'n_words': 970}