text:
"""
Mask R-CNN
- The main Mask R-CNN model implementation.
- Copyright (c) 2017 Matterport, Inc.
- Licensed under the MIT License (see LICENSE for details)
- Written by Waleed Abdulla
- https://github.com/matterport/Mask_RCNN

iMaterialist Fashion 2019 at FGVC6
- https://www.kaggle.com/c/imaterialist-fashion-2019-FGVC6
- Fine-grained segmentation task for fashion and apparel

Training Mask R-CNN to be a Fashionista (LB=0.07)
- https://www.kaggle.com/pednoi/training-mask-r-cnn-to-be-a-fashionista-lb-0-07
- This kernel was used

Gachon University Graduation Project
- Choi Hyung-Kyu (Image Processing)
- Jung Yoo-Ji (Image Processing)
- Lee Se-Jin (Text Processing)
"""

# Import Modules
from module.Mask_RCNN.mrcnn import config as maskconfig
from module.Mask_RCNN.mrcnn import model as maskmodel
from module.Mask_RCNN.mrcnn import utils
from sklearn.model_selection import KFold
from imgaug import augmenters as iaa
from pathlib import Path
import numpy as np
import pandas as pd
import warnings
import json
import cv2

# Ignore Warnings
warnings.filterwarnings(action='ignore')

# Directories
data_dir = Path('../../../data/image/mask_rcnn')
save_dir = Path('../../../data/weight')

# Initialize NUM_CATS, IMAGE_SIZE
NUM_CATS = 46
IMAGE_SIZE = 512

# Import label_description
with open(data_dir/"label_descriptions.json") as f:
    label_descriptions = json.load(f)
label_names = [x['name'] for x in label_descriptions['categories']]

# Import mask_rcnn_coco
mrCoco = '../../../data/weight/mask_rcnn_coco.h5'

# Import train.csv for segmentation
segment = pd.read_csv(data_dir/"train.csv")
segment['CategoryId'] = segment['ClassId'].str.split('_').str[0]

# Grouping data in dataFrame by Pixels, Height-Width
image = segment.groupby('ImageId')['EncodedPixels', 'CategoryId']\
    .agg(lambda x: list(x)).join(segment.groupby('ImageId')['Height', 'Width'].mean())

# Configuration Class
class FashionConfig(maskconfig.Config):
    NAME = "fashion"
    NUM_CLASSES = NUM_CATS + 1
    GPU_COUNT = 1
    IMAGES_PER_GPU = 4
    BACKBONE = 'resnet101'
    IMAGE_MIN_DIM = IMAGE_SIZE
    IMAGE_MAX_DIM = IMAGE_SIZE
    IMAGE_RESIZE_MODE = 'none'
    RPN_ANCHOR_SCALES = (16, 32, 64, 128, 256)
    RPN_NMS_THRESHOLD = 0.8
    STEPS_PER_EPOCH = 1500
    VALIDATION_STEPS = 300

# Fashion Dataset Class
class FashionDataset(utils.Dataset):
    def __init__(self, df):
        super().__init__(self)

        # Add classes
        for i, name in enumerate(label_names):
            self.add_class("fashion", i + 1, name)

        # Add images
        for i, row in df.iterrows():
            self.add_image("fashion",
                           image_id=row.name,
                           path=str(data_dir / 'train' / row.name),
                           labels=row['CategoryId'],
                           annotations=row['EncodedPixels'],
                           height=row['Height'], width=row['Width'])

    def image_reference(self, image_id):
        info = self.image_info[image_id]
        return info['path'], [label_names[int(x)] for x in info['labels']]

    def load_image(self, image_id):
        return resize_image(self.image_info[image_id]['path'])

    def load_mask(self, image_id):
        info = self.image_info[image_id]

        mask = np.zeros((IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])), dtype=np.uint8)
        labels = []

        for m, (annotation, label) in enumerate(zip(info['annotations'], info['labels'])):
            sub_mask = np.full(info['height'] * info['width'], 0, dtype=np.uint8)
            annotation = [int(x) for x in annotation.split(' ')]

            for i, start_pixel in enumerate(annotation[::2]):
                sub_mask[start_pixel: start_pixel + annotation[2 * i + 1]] = 1

            sub_mask = sub_mask.reshape((info['height'], info['width']), order='F')
            sub_mask = cv2.resize(sub_mask, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)

            mask[:, :, m] = sub_mask
            labels.append(int(label) + 1)

        return mask, np.array(labels)

# Prepare dataset, config
dataset = FashionDataset(image)
dataset.prepare()
config = FashionConfig()

# Functions
def resize_image(image_path):
    img = cv2.imread(image_path)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    img = cv2.resize(img, (IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_AREA)
    return img
def get_fold(fold, splits):
    for i, (train_idx, valid_idx) in enumerate(splits):
        if i == fold:
            return image.iloc[train_idx], image.iloc[valid_idx]

# k-Fold Cross Validation
k = 10
f = 0
kf = KFold(n_splits=k, random_state=42, shuffle=True)
s = kf.split(image)
train, valid = get_fold(f, s)

# Dataset
train_dataset = FashionDataset(train)
train_dataset.prepare()
valid_dataset = FashionDataset(valid)
valid_dataset.prepare()

# Load Model
model = maskmodel.MaskRCNN(mode='training', config=config, model_dir=save_dir)
model.load_weights(mrCoco, by_name=True, exclude=['mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'])
augmentation = iaa.Sequential([iaa.Fliplr(0.5)])

# Training Parameters
LR = 1e-4
EPOCHS = [2, 6, 8]

# Training
model.train(train_dataset, valid_dataset,
            learning_rate=LR*2, epochs=EPOCHS[0],
            layers='heads', augmentation=None)
history = model.keras_model.history.history

model.train(train_dataset, valid_dataset,
            learning_rate=LR, epochs=EPOCHS[1],
            layers='all', augmentation=augmentation)

new_history = model.keras_model.history.history
for k in new_history:
    history[k] = history[k] + new_history[k]

model.train(train_dataset, valid_dataset,
            learning_rate=LR/5, epochs=EPOCHS[2],
            layers='all', augmentation=augmentation)

new_history = model.keras_model.history.history
for k in new_history:
    history[k] = history[k] + new_history[k]

best_epoch = np.argmin(history["val_loss"]) + 1
print("Best epoch: ", best_epoch)
print("Valid loss: ", history["val_loss"][best_epoch-1])

max_stars_repo_name: ieesejin/graduation-project

max_issues_repo_name: ieesejin/graduation-project

max_forks_repo_name: ieesejin/graduation-project


text:
#  Copyright 2019-2020 The Lux Authors.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

from .context import lux
import pytest
import pandas as pd

from lux.vis.Vis import Vis

def test_vary_filter_val(global_var):
    lux.config.set_executor_type("Pandas")
    df = pytest.olympic
    vis = Vis(["Height", "SportType=Ball"], df)
    df.set_intent_as_vis(vis)
    df._ipython_display_()
    assert len(df.recommendation["Filter"]) == len(df["SportType"].unique()) - 1
    linechart = list(filter(lambda x: x.mark == "line", df.recommendation["Enhance"]))[0]
    assert (
        linechart.get_attr_by_channel("x")[0].attribute == "Year"
    ), "Ensure recommended vis does not inherit input vis channel"

def test_filter_inequality(global_var):
    df = pytest.car_df
    df["Year"] = pd.to_datetime(df["Year"], format="%Y")

    df.set_intent(
        [
            lux.Clause(attribute="Horsepower"),
            lux.Clause(attribute="MilesPerGal"),
            lux.Clause(attribute="Acceleration", filter_op=">", value=10),
        ]
    )
    df._ipython_display_()

    from lux.utils.utils import get_filter_specs

    complement_vis = df.recommendation["Filter"][0]
    fltr_clause = get_filter_specs(complement_vis._intent)[0]
    assert fltr_clause.filter_op == "<="
    assert fltr_clause.value == 10

def test_generalize_action(global_var):
    # test that generalize action creates all unique visualizations
    df = pytest.car_df
    df["Year"] = pd.to_datetime(
        df["Year"], format="%Y"
    )  # change pandas dtype for the column "Year" to datetype
    df.set_intent(["Acceleration", "MilesPerGal", "Cylinders", "Origin=USA"])
    df._ipython_display_()
    assert len(df.recommendation["Generalize"]) == 4
    v1 = df.recommendation["Generalize"][0]
    v2 = df.recommendation["Generalize"][1]
    v3 = df.recommendation["Generalize"][2]
    v4 = df.recommendation["Generalize"][3]

    for clause in v4._inferred_intent:
        assert clause.value == ""  # No filter value
    assert v4.title == "Overall"

    check1 = v1 != v2 and v1 != v3 and v1 != v4
    check2 = v2 != v3 and v2 != v4
    check3 = v3 != v4
    assert check1 and check2 and check3

def test_row_column_group(global_var):
    df = pd.read_csv(
        "https://github.com/lux-org/lux-datasets/blob/master/data/state_timeseries.csv?raw=true"
    )
    df["Date"] = pd.to_datetime(df["Date"])
    tseries = df.pivot(index="State", columns="Date", values="Value")
    # Interpolating missing values
    tseries[tseries.columns.min()] = tseries[tseries.columns.min()].fillna(0)
    tseries[tseries.columns.max()] = tseries[tseries.columns.max()].fillna(tseries.max(axis=1))
    tseries = tseries.interpolate("zero", axis=1)
    tseries._ipython_display_()
    assert list(tseries.recommendation.keys()) == ["Temporal"]

def test_groupby(global_var):
    df = pytest.college_df
    groupbyResult = df.groupby("Region").sum()
    groupbyResult._ipython_display_()
    assert list(groupbyResult.recommendation.keys()) == ["Column Groups"]

def test_crosstab():
    # Example from http://www.datasciencemadesimple.com/cross-tab-cross-table-python-pandas/
    d = {
        "Name": [
            "Alisa",
            "Bobby",
            "Cathrine",
            "Alisa",
            "Bobby",
            "Cathrine",
            "Alisa",
            "Bobby",
            "Cathrine",
            "Alisa",
            "Bobby",
            "Cathrine",
        ],
        "Exam": [
            "Semester 1",
            "Semester 1",
            "Semester 1",
            "Semester 1",
            "Semester 1",
            "Semester 1",
            "Semester 2",
            "Semester 2",
            "Semester 2",
            "Semester 2",
            "Semester 2",
            "Semester 2",
        ],
        "Subject": [
            "Mathematics",
            "Mathematics",
            "Mathematics",
            "Science",
            "Science",
            "Science",
            "Mathematics",
            "Mathematics",
            "Mathematics",
            "Science",
            "Science",
            "Science",
        ],
        "Result": [
            "Pass",
            "Pass",
            "Fail",
            "Pass",
            "Fail",
            "Pass",
            "Pass",
            "Fail",
            "Fail",
            "Pass",
            "Pass",
            "Fail",
        ],
    }

    df = pd.DataFrame(d, columns=["Name", "Exam", "Subject", "Result"])
    result = pd.crosstab([df.Exam], df.Result)
    result._ipython_display_()
    assert list(result.recommendation.keys()) == ["Row Groups", "Column Groups"]

def test_custom_aggregation(global_var):
    import numpy as np

    df = pytest.college_df
    df.set_intent(["HighestDegree", lux.Clause("AverageCost", aggregation=np.ptp)])
    df._ipython_display_()
    assert list(df.recommendation.keys()) == ["Enhance", "Filter", "Generalize"]
    df.clear_intent()

def test_year_filter_value(global_var):
    df = pytest.car_df
    df["Year"] = pd.to_datetime(df["Year"], format="%Y")
    df.set_intent(["Acceleration", "Horsepower"])
    df._ipython_display_()
    list_of_vis_with_year_filter = list(
        filter(
            lambda vis: len(
                list(
                    filter(
                        lambda clause: clause.value != "" and clause.attribute == "Year",
                        vis._intent,
                    )
                )
            )
            != 0,
            df.recommendation["Filter"],
        )
    )
    vis = list_of_vis_with_year_filter[0]
    assert (
        "T00:00:00.000000000" not in vis.to_altair()
    ), "Year filter title contains extraneous string, not displayed as summarized string"
    df.clear_intent()

def test_similarity(global_var):
    lux.config.early_pruning = False
    df = pytest.car_df
    df["Year"] = pd.to_datetime(df["Year"], format="%Y")
    df.set_intent(
        [
            lux.Clause("Year", channel="x"),
            lux.Clause("Displacement", channel="y"),
            lux.Clause("Origin=USA"),
        ]
    )
    df._ipython_display_()
    assert len(df.recommendation["Similarity"]) == 2
    ranked_list = df.recommendation["Similarity"]

    japan_vis = list(
        filter(
            lambda vis: vis.get_attr_by_attr_name("Origin")[0].value == "Japan",
            ranked_list,
        )
    )[0]
    europe_vis = list(
        filter(
            lambda vis: vis.get_attr_by_attr_name("Origin")[0].value == "Europe",
            ranked_list,
        )
    )[0]
    assert japan_vis.score > europe_vis.score
    df.clear_intent()
    lux.config.early_pruning = True

def test_similarity2():
    df = pd.read_csv(
        "https://raw.githubusercontent.com/lux-org/lux-datasets/master/data/real_estate_tutorial.csv"
    )

    df["Month"] = pd.to_datetime(df["Month"], format="%m")
    df["Year"] = pd.to_datetime(df["Year"], format="%Y")

    df.intent = [
        lux.Clause("Year"),
        lux.Clause("PctForeclosured"),
        lux.Clause("City=Crofton"),
    ]

    ranked_list = df.recommendation["Similarity"]

    morrisville_vis = list(
        filter(
            lambda vis: vis.get_attr_by_attr_name("City")[0].value == "Morrisville",
            ranked_list,
        )
    )[0]
    watertown_vis = list(
        filter(
            lambda vis: vis.get_attr_by_attr_name("City")[0].value == "Watertown",
            ranked_list,
        )
    )[0]
    assert morrisville_vis.score > watertown_vis.score

def test_intent_retained():
    df = pd.read_csv("https://raw.githubusercontent.com/lux-org/lux-datasets/master/data/employee.csv")
    df.intent = ["Attrition"]
    df._ipython_display_()

    df["%WorkingYearsAtCompany"] = df["YearsAtCompany"] / df["TotalWorkingYears"]
    assert df.current_vis != None
    assert df.intent != None
    assert df._recs_fresh == False
    assert df._metadata_fresh == False

    df._ipython_display_()
    assert list(df.recommendation.keys()) == ["Enhance", "Filter"]

def test_metadata_propogate_invalid_intent():
    df = pd.read_csv("https://raw.githubusercontent.com/lux-org/lux-datasets/master/data/employee.csv")
    df.intent = ["Attrition"]
    new_df = df.groupby("BusinessTravel").mean()
    assert new_df.intent[0].attribute == "Attrition", "User-specified intent is retained"
    assert new_df._inferred_intent == [], "Invalid inferred intent is cleared"
    new_df._ipython_display_()
    assert new_df.current_vis == []

ext: py

max_stars_repo_name: cjachekang/lux

max_issues_repo_name: cjachekang/lux

max_forks_repo_name: cjachekang/lux

reason: import numpy


text:
#!/usr/bin/env python

# Plot the bernoulli entropy.

import matplotlib.pyplot as pl
import numpy as np
from utils.util import save_fig

def entropy(p):
    """calculate the entropy"""
    h = -p * np.log2(p) - (1 - p) * np.log2(1 - p)
    return h

x = np.linspace(0.01, 0.99, 100)
y = entropy(x)

pl.plot(x, y)
pl.xlabel('p(X=1)')
pl.ylabel('H(X)')

save_fig('bernoulliEntropyFig.pdf')
pl.show()


text:
# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
import inspect
from mock import Mock
import os
from sagemaker.fw_utils import create_image_uri, framework_name_from_image, framework_version_from_tag
from sagemaker.fw_utils import tar_and_upload_dir, parse_s3_url, UploadedCode
import pytest

DATA_DIR = 'data_dir'
BUCKET_NAME = 'mybucket'
ROLE = 'Sagemaker'
REGION = 'us-west-2'
SCRIPT_PATH = 'script.py'

@pytest.fixture()
def sagemaker_session():
    boto_mock = Mock(name='boto_session', region_name=REGION)
    ims = Mock(name='sagemaker_session', boto_session=boto_mock)
    ims.default_bucket = Mock(name='default_bucket', return_value=BUCKET_NAME)
    ims.expand_role = Mock(name="expand_role", return_value=ROLE)
    ims.sagemaker_client.describe_training_job = Mock(return_value={'ModelArtifacts':
                                                                    {'S3ModelArtifacts': 's3://m/m.tar.gz'}})
    return ims

def test_create_image_uri_cpu():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.c4.large', '1.0rc', 'py2', '23')
    assert image_uri == '23.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0rc-cpu-py2'

    image_uri = create_image_uri('mars-south-3', 'mlfw', 'local', '1.0rc', 'py2', '23')
    assert image_uri == '23.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0rc-cpu-py2'

def test_create_image_uri_gpu():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.p3.2xlarge', '1.0rc', 'py3', '23')
    assert image_uri == '23.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0rc-gpu-py3'

    image_uri = create_image_uri('mars-south-3', 'mlfw', 'local_gpu', '1.0rc', 'py3', '23')
    assert image_uri == '23.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0rc-gpu-py3'

def test_create_image_uri_default_account():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.p3.2xlarge', '1.0rc', 'py3')
    assert image_uri == '520713654638.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0rc-gpu-py3'

def test_invalid_instance_type():
    # instance type is missing 'ml.' prefix
    with pytest.raises(ValueError):
        create_image_uri('mars-south-3', 'mlfw', 'p3.2xlarge', '1.0.0', 'py3')

def test_optimized_family():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.p3.2xlarge', '1.0.0', 'py3',
                                 optimized_families=['c5', 'p3'])
    assert image_uri == '520713654638.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0.0-p3-py3'

def test_unoptimized_cpu_family():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.m4.xlarge', '1.0.0', 'py3',
                                 optimized_families=['c5', 'p3'])
    assert image_uri == '520713654638.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0.0-cpu-py3'

def test_unoptimized_gpu_family():
    image_uri = create_image_uri('mars-south-3', 'mlfw', 'ml.p2.xlarge', '1.0.0', 'py3',
                                 optimized_families=['c5', 'p3'])
    assert image_uri == '520713654638.dkr.ecr.mars-south-3.amazonaws.com/sagemaker-mlfw:1.0.0-gpu-py3'

def test_tar_and_upload_dir_s3(sagemaker_session):
    bucket = 'mybucker'
    s3_key_prefix = 'something/source'
    script = 'mnist.py'
    directory = 's3://m'
    result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
    assert result == UploadedCode('s3://m', 'mnist.py')

def test_tar_and_upload_dir_does_not_exits(sagemaker_session):
    bucket = 'mybucker'
    s3_key_prefix = 'something/source'
    script = 'mnist.py'
    directory = ' !@#$%^&*()path probably in not there.!@#$%^&*()'
    with pytest.raises(ValueError) as error:
        tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
    assert 'does not exist' in str(error)

def test_tar_and_upload_dir_is_not_directory(sagemaker_session):
    bucket = 'mybucker'
    s3_key_prefix = 'something/source'
    script = 'mnist.py'
    directory = inspect.getfile(inspect.currentframe())
    with pytest.raises(ValueError) as error:
        tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
    assert 'is not a directory' in str(error)

def test_tar_and_upload_dir_file_not_in_dir(sagemaker_session):
    bucket = 'mybucker'
    s3_key_prefix = 'something/source'
    script = ' !@#$%^&*() .myscript. !@#$%^&*() '
    directory = '.'
    with pytest.raises(ValueError) as error:
        tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
    assert 'No file named' in str(error)

def test_tar_and_upload_dir_not_s3(sagemaker_session):
    bucket = 'mybucker'
    s3_key_prefix = 'something/source'
    script = os.path.basename(__file__)
    directory = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    result = tar_and_upload_dir(sagemaker_session, bucket, s3_key_prefix, script, directory)
    assert result == UploadedCode('s3://{}/{}/sourcedir.tar.gz'.format(bucket, s3_key_prefix), script)

def test_framework_name_from_image_mxnet():
    image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet:1.1-gpu-py3'
    assert ('mxnet', 'py3', '1.1-gpu-py3') == framework_name_from_image(image_name)

def test_framework_name_from_image_tf():
    image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow:1.6-cpu-py2'
    assert ('tensorflow', 'py2', '1.6-cpu-py2') == framework_name_from_image(image_name)

def test_legacy_name_from_framework_image():
    image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-mxnet-py3-gpu:2.5.6-gpu-py2'
    framework, py_ver, tag = framework_name_from_image(image_name)
    assert framework == 'mxnet'
    assert py_ver == 'py3'
    assert tag == '2.5.6-gpu-py2'

def test_legacy_name_from_wrong_framework():
    framework, py_ver, tag = framework_name_from_image('123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py2-gpu:1')
    assert framework is None
    assert py_ver is None
    assert tag is None

def test_legacy_name_from_wrong_python():
    framework, py_ver, tag = framework_name_from_image('123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1')
    assert framework is None
    assert py_ver is None
    assert tag is None

def test_legacy_name_from_wrong_device():
    framework, py_ver, tag = framework_name_from_image('123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-myown-py4-gpu:1')
    assert framework is None
    assert py_ver is None
    assert tag is None

def test_legacy_name_from_image_any_tag():
    image_name = '123.dkr.ecr.us-west-2.amazonaws.com/sagemaker-tensorflow-py2-cpu:any-tag'
    framework, py_ver, tag = framework_name_from_image(image_name)
    assert framework == 'tensorflow'
    assert py_ver == 'py2'
    assert tag == 'any-tag'

def test_framework_version_from_tag():
    version = framework_version_from_tag('1.5rc-keras-gpu-py2')
    assert version == '1.5rc-keras'

def test_framework_version_from_tag_other():
    version = framework_version_from_tag('weird-tag-py2')
    assert version is None

def test_parse_s3_url():
    bucket, key_prefix = parse_s3_url('s3://bucket/code_location')
    assert 'bucket' == bucket
    assert 'code_location' == key_prefix

def test_parse_s3_url_fail():
    with pytest.raises(ValueError) as error:
        parse_s3_url('t3://code_location')
    assert 'Expecting \'s3\' scheme' in str(error)

max_stars_repo_name: besirkurtulmus/sagemaker-python-sdk

max_issues_repo_name: besirkurtulmus/sagemaker-python-sdk

max_forks_repo_name: besirkurtulmus/sagemaker-python-sdk

reason: from sage


text:
import argparse
import json
import os
import pickle
import zlib
from base64 import b64decode
from typing import Iterable, Optional, Tuple

import numpy as np
import tqdm

def create_file_descriptions(
    filename: str, input_column: str, random_seed: Optional[int] = None
) -> Tuple[Iterable[int], Iterable[int]]:
    """Create seeking points and sequence lengths of dataset.

    Args:
        filename: The name of dataset file.
        input_column: The name of input column.
        random_seed: A random seed of index shuffling. Note that this function will
            shuffle the collected seeking points and text lengths. Default is `None`.

    Returns:
        The collected seeking points and sequence lengths of the dataset.
    """
    seeking_points, sequence_lengths = [], []

    # Get total number of lines in the file to show the progress bar for calculating and
    # creating file index.
    total_lines = 0
    with open(filename, "r") as fp:
        for _ in fp:
            total_lines += 1

    with open(filename, "r") as fp:
        columns = fp.readline().strip().split(",")
        input_column_idx = columns.index(input_column)

        last_seeking_point = fp.tell()
        for line in tqdm.tqdm(fp, total=total_lines - 1):
            structure = line.strip().split(",")[input_column_idx]
            structure = json.loads(zlib.decompress(b64decode(structure)).decode())
            sequence_lengths.append(len(structure["atoms"]))

            seeking_points.append(last_seeking_point)
            last_seeking_point += len(line)

    if random_seed is not None:
        np.random.seed(random_seed)
    random_indices = np.random.permutation(len(seeking_points))

    seeking_points = np.array(seeking_points)
    sequence_lengths = np.array(sequence_lengths)
    return seeking_points[random_indices], sequence_lengths[random_indices]

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("input_file")
    parser.add_argument("--input_column", default="structure")
    parser.add_argument("--random_seed", type=int, default=42)
    args = parser.parse_args()

    seeking_points, sequence_lengths = create_file_descriptions(
        args.input_file, args.input_column, args.random_seed
    )
    seeking_points = seeking_points.tolist()
    sequence_lengths = sequence_lengths.tolist()

    with open(os.path.splitext(args.input_file)[0] + ".index", "wb") as fp:
        pickle.dump((seeking_points, sequence_lengths), fp)


text:
import math
import timeit
import random
import sympy
import warnings
from random import randint, seed
import sys
from ecpy.curves import Curve, Point
from Crypto.Hash import SHA3_256
import requests
from Crypto.Cipher import AES
from Crypto import Random
from Crypto.Util.Padding import pad
from Crypto.Util.Padding import unpad
import random
import re
import json

API_URL = 'http://cryptlygos.pythonanywhere.com'

stuID = 25308

def random_prime(bitsize):
    # random.seed(42)
    warnings.simplefilter('ignore')
    chck = False
    while chck == False:
        p = random.randrange(2 ** (bitsize - 1), 2 ** bitsize - 1)
        chck = sympy.isprime(p)
    warnings.simplefilter('default')
    return p

curve = Curve.get_curve('secp256k1')

# TODO: HERE CREATE A LONG TERM KEY

random.seed(42)
sL = randint(1, random_prime(256) - 1)
print("sL: ", sL)

# base point P is the generator
P = curve.generator

lkey = sL * P
print("lkey: ", lkey)

n = curve.order
print("n: ", n)

k = randint(1, n - 2)
print("k: ", k)

R = k * P
print("R: ", R)

r = (R.x) % n
print("r: ", r)

m = "25097"

h_ = SHA3_256.new(b'25097' + r.to_bytes((r.bit_length() + 7) // 8, byteorder='big'))

h = (int.from_bytes(h_.digest(), byteorder='big')) % n
print("h: ", h)

s = (sL * h + k) % n
print("s: ", s)

# print("sL: ", sL)
print("LKey.x: ", lkey.x)
print("LKey.y: ", lkey.y)
# print("LKey: ", lkey)

V = (s * P) - (h * lkey)
print("V: ", V)

v = V.x % n
print("v: ", v)

h_2 = SHA3_256.new(b'25097' + v.to_bytes((v.bit_length() + 7) // 8, byteorder='big'))

h_new = (int.from_bytes(h_2.digest(), byteorder='big')) % n

if h == h_new:
    print("true")

else:
    print("false")

n = curve.order

# HERE GENERATE A EPHEMERAL KEY

e_sL = randint(1, random_prime(256) - 1)
print("e_sL: ", e_sL)

# base point P is the generator
ekey = e_sL * P
print("e_Lkey: ", ekey)
print("e_Lkey.x: ", ekey.x)
print("e_Lkey.y: ", ekey.y)

# server's long term key
QSer_long = Point(0xc1bc6c9063b6985fe4b93be9b8f9d9149c353ae83c34a434ac91c85f61ddd1e9,
                  0x931bd623cf52ee6009ed3f50f6b4f92c564431306d284be7e97af8e443e69a8c, curve)

try:

    # REGISTRATION
    mes = {'ID': stuID, 'h': h, 's': s, 'LKEY.X': lkey.x, 'LKEY.Y': lkey.y}
    response = requests.put('{}/{}'.format(API_URL, "RegStep1"), json=mes)
    if ((response.ok) == False): raise Exception(response.json())
    print(response.json())

    print("Enter verification code which is sent to you: ")
    code = int(input())

    mes = {'ID': stuID, 'CODE': code}
    response = requests.put('{}/{}'.format(API_URL, "RegStep3"), json=mes)
    if ((response.ok) == False): raise Exception(response.json())
    print(response.json())

    # STS PROTOCOL

    mes = {'ID': stuID, 'EKEY.X': ekey.x, 'EKEY.Y': ekey.y}
    response = requests.put('{}/{}'.format(API_URL, "STSStep1&2"), json=mes)
    if ((response.ok) == False): raise Exception(response.json())
    res = response.json()

    #calculate T,K,U

    qB = Point(res['SKEY.X'], res['SKEY.Y'], curve)
    T = e_sL * qB
    print("x coordinate of T: ",T.x)
    print("y coordinate of T: ",T.y)

    a = "BeYourselfNoMatterWhatTheySay"
    U = str(T.x) + str(T.y) + a
    U = bytes(U, 'utf-8')
    print("U:",U)

    K = SHA3_256.new(U)
    print("K: ", K)

    W1 = str(ekey.x) + str(ekey.y) + str(qB.x) + str(qB.y)
    print("W1: ", W1)

    #Sign Message

    sig_k = randint(1, n - 2)

    new_R = sig_k * P

    new_r = new_R.x % n

    m = bytes(W1, 'utf-8')

    byte_r = new_r.to_bytes((new_r.bit_length() + 7) // 8, byteorder='big')

    h_3 = SHA3_256.new(m + byte_r)

    sig_h = (int.from_bytes(h_3.digest(), byteorder='big'))
    sig_h = sig_h % n

    sig_s = (sL * sig_h + sig_k) % n

    print("sig_s: ", sig_s)
    print("sig_h: ", sig_h)

    Y1 = 's' + str(sig_s) + 'h' + str(sig_h)
    Y1 = bytes(Y1, 'utf-8')
    print("plaintext: ", Y1)

    #Encryption

    crypto = AES.new(K.digest(), AES.MODE_CTR)
    Y1 = crypto.encrypt(Y1)
    nonce = crypto.nonce
    print("Y1: ", Y1)
    print("nonce: ", nonce)

    final_message = nonce + Y1
    print("nonce + y1", final_message)
    ctext = int.from_bytes(final_message, byteorder='big')
    print("ctext", ctext)

    ###Send encrypted-signed keys and retrive server's signed keys
    mes = {'ID': stuID, 'FINAL MESSAGE': ctext}
    response = requests.put('{}/{}'.format(API_URL, "STSStep4&5"), json=mes)
    if ((response.ok) == False):
        raise Exception(response.json())
    ctext = response.json()

    #Decrypt

    W2 = ctext.to_bytes((ctext.bit_length() + 7) // 8, byteorder='big')
    print("Received encrypted ciphertext: ", W2)

    crypto = AES.new(K.digest(), AES.MODE_CTR, nonce=W2[0:8])
    decrypted = crypto.decrypt(W2[8:])

    decoded = decrypted.decode('UTF-8')
    print("Decrypted text: ", decoded)

    message = str(qB.x) + str(qB.y) + str(ekey.x) + str(ekey.y)
    message = bytes(message, 'utf-8')
    print("The message is:", message)

    s_nw = decoded[1:decoded.index('h')]
    h_nw = decoded[decoded.index('h') + 1:]
    s_nw = int(s_nw)
    h_nw = int(h_nw)

    #verify

    V = (s * P) - (h * lkey)
    print("V: ", V)

    v = V.x % n
    print("v: ", v)

    h_2 = SHA3_256.new(b'25097' + v.to_bytes((v.bit_length() + 7) // 8, byteorder='big'))

    h_new = (int.from_bytes(h_2.digest(), byteorder='big')) % n

    if h == h_new:
        print("true")

    else:
        print("false")

    # get a message from server for
    mes = {'ID': stuID}
    response = requests.get('{}/{}'.format(API_URL, "STSStep6"), json=mes)
    ctext = response.json()

    print(ctext)

    #Decrypt
    num = ctext.to_bytes((ctext.bit_length() + 7) // 8, byteorder='big')
    crypto = AES.new(K.digest(), AES.MODE_CTR, nonce=num[0:8])
    dtext = crypto.decrypt(num[8:])

    decoded_dtext = dtext.decode('UTF-8')
    print("Decrypted text: ", decoded_dtext)

    #Add 1 to random to create the new message and encrypt it

    random = decoded_dtext[decoded_dtext.index('.') + 2:]
    text = decoded_dtext[:decoded_dtext.index('.') + 1]
    #print("Text: ", text)
    #print("Random: ", rand)
    random = int(random) + 1

    text = text + " " + str(random)
    print(text)
    text = bytes(text, 'utf-8')

    crypto = AES.new(K.digest(), AES.MODE_CTR)
    ctext = crypto.nonce + crypto.encrypt(text)
    ct = int.from_bytes(ctext, byteorder='big')
    print("Plaintext: ", text)

    # send the message and get response of the server
    mes = {'ID': stuID, 'ctext': ct}
    response = requests.put('{}/{}'.format(API_URL, "STSStep7&8"), json=mes)
    ctext = response.json()
    print("Response: ", ctext)

    num = ctext.to_bytes((ctext.bit_length() + 7) // 8, byteorder='big')
    crypto = AES.new(K.digest(), AES.MODE_CTR, nonce=num[0:8])
    dtext = crypto.decrypt(num[8:])
    print("Decrypted text: ", dtext.decode('UTF-8'))
    decoded_dtext = dtext.decode('UTF-8')
    #print(decoded_dtext)

except Exception as e:
    print(e)

reason: import sympy


text:
"""
4x4 transfer matrix method functions for 2d data
"""

from __future__ import absolute_import, print_function, division
import numpy as np

from dtmm.conf import  BETAMAX, CDTYPE, DTMMConfig
from dtmm.linalg import dotmdm, inv, dotmv, bdotmm, bdotmd, bdotdm
from dtmm.print_tools import print_progress

import dtmm.tmm as tmm
from dtmm.tmm import alphaffi, phase_mat

from dtmm.wave import eigenbetax1, eigenindices1, eigenmask1, eigenwave1, betaxy2beta, mask2betax1, mask2indices1,betaxy2phi
from dtmm.wave import k0 as wavenumber
from dtmm.field import field2modes1, modes2field1
from dtmm.fft import mfft

def layer_mat2d(k0, d, epsv,epsa, betay = 0., method = "4x4", mask = None):
    """Computes characteristic matrix of a single layer M=F.P.Fi,

    Numpy broadcasting rules apply.

    Parameters
    ----------
    k0 : float or sequence of floats
        A scalar or a vector of wavenumbers
    d : array_like
        Layer thickness
    epsv : array_like
        Epsilon eigenvalues.
    epsa : array_like
        Optical axes orientation angles (psi, theta, phi).
    method : str, optional
        Either a 4x4 or 4x4_1

    Returns
    -------
    cmat : ndarray
        Characteristic matrix of the layer.
    """
    if method not in ("4x4","4x4_1"):
        raise ValueError("Unsupported method: '{}'".format(method))
    k0 = np.asarray(k0)
    shape = epsv.shape[-2]
    if mask is None:
        mask = eigenmask1(shape, k0)
        betas = eigenbetax1(shape, k0)
        indices = eigenindices1(shape, k0)
    else:
        betas = mask2betax1(mask,k0)
        indices = mask2indices1(mask)
    if k0.ndim == 0:
        return _layer_mat2d(k0,d,epsv,epsa, mask, betas,betay, indices, method)
    else:
        out = (_layer_mat2d(k0[i],d,epsv,epsa, mask[i],betas[i], betay, indices[i],method) for i in range(len(k0)))
        return tuple(out)

def _layer_mat2d(k0,d,epsv,epsa, mask, betaxs, betay,indices, method):
    n = len(betaxs)
    kd = k0*d
    shape = epsv.shape[-2]
    out = np.empty(shape = (n, n, 4, 4), dtype = CDTYPE)
    beta = betaxy2beta(betaxs,betay)
    phi = betaxy2phi(betaxs,betay)
    for j,(beta,phi) in enumerate(zip(beta, phi)):
        alpha,f,fi = alphaffi(beta,phi,epsv,epsa)
        pmat = phase_mat(alpha,-kd)
        if method == "4x4_1":
            pmat[...,1::2] = 0.
        elif method != "4x4":
            raise ValueError("Unsupported method!")

        wave = eigenwave1(shape, indices[j], amplitude = 1.)

        #m is shape (...,4,4)
        m = dotmdm(f,pmat,fi)

        #wave is shape (...) make it broadcastable to (...,4,4)

        mw = m*wave[...,None,None]
        mf = mfft(mw, overwrite_x = True)
        mf = mf[mask,...]

        out[:,j,:,:] = mf

        #for i,mfj in enumerate(mf):
        #    out[i,j,:,:] = mfj

    return out

def stack_mat2d(k,d,epsv,epsa, betay = 0., method = "4x4" ,mask = None):
    n = len(d)
    indices = range(n)
    if method.startswith("2x2"):
        indices = reversed(indices)
    verbose_level = DTMMConfig.verbose
    if verbose_level > 1:
        print ("Building stack matrix.")
    for i in range(n):
        print_progress(i,n)
        mat = layer_mat2d(k,d[i],epsv[i],epsa[i], betay = betay, mask = mask, method = method)

        if i == 0:
            if isinstance(mat, tuple):
                out = tuple((m.copy() for m in mat))
            else:
                out = mat.copy()
        else:
            if isinstance(mat, tuple):
                out = tuple((bdotmm(o,m) for o,m in zip(out,mat)))
            else:
                out = bdotmm(out,mat)

    print_progress(n,n)

    return out

def f_iso2d(shape, k0, n = 1., betay = 0, betamax = BETAMAX):
    k0 = np.asarray(k0)
    betax = eigenbetax1(shape,k0, betamax)

    if k0.ndim == 0:
        beta = betaxy2beta(betax,betay)
        phi = betaxy2phi(betax,betay)
        fmat = tmm.f_iso(n = n, beta = beta, phi = phi)
        return fmat
    else:
        beta = tuple((betaxy2beta(beta,betay) for beta in betax))
        phi = tuple((betaxy2phi(beta,betay) for beta in betax))
        fmat = (tmm.f_iso(n = n, beta = beta[i], phi = phi[i]) for i in range(len(k0)))
        return tuple(fmat)

def fi_iso2d(shape, k0, n = 1., betay = 0, betamax = BETAMAX):
    fmat = f_iso2d(shape, k0, n, betay, betamax)
    if isinstance(fmat, tuple):
        out = (inv(f) for f in fmat)
        return tuple(out)
    else:
        return inv(fmat)

def _system_mat2d(fmatin, cmat, fmatout):
    """Computes a system matrix from a characteristic matrix Fin-1.C.Fout"""
    fmatini = inv(fmatin)
    out = bdotdm(fmatini,cmat)
    return bdotmd(out,fmatout)

def system_mat2d(fmatin, cmat, fmatout):
    """Computes a system matrix from a characteristic matrix Fin-1.C.Fout"""
    if isinstance(fmatin, tuple):
        if cmat is not None:
            out = (_system_mat2d(fi, c, fo) for fi,c,fo in zip(fmatin,cmat,fmatout))
        else:
            out = (_system_mat2d(fi, None, fo) for fi,fo in zip(fmatin,fmatout))
        return tuple(out)
    else:
        return _system_mat2d(fmatin, cmat, fmatout)

def _reflection_mat2d(smat):
    """Computes a 4x4 reflection matrix.
    """
    shape = smat.shape[0:-4] + (smat.shape[-4] * 4,smat.shape[-4] * 4)
    smat = np.moveaxis(smat, -2,-3)
    smat = smat.reshape(shape)
    return tmm.reflection_mat(smat)

def reflection_mat2d(smat):
    verbose_level = DTMMConfig.verbose
    if verbose_level > 1:
        print ("Building reflectance and transmittance matrix.")
    if isinstance(smat, tuple):
        out = []
        n = len(smat)
        for i,s in enumerate(smat):
            print_progress(i,n)
            out.append(_reflection_mat2d(s))
        print_progress(n,n)
        return tuple(out)
    else:
        return _reflection_mat2d(smat)

def _reflect2d(fvec_in, fmat_in, rmat, fmat_out, fvec_out = None):
    """Transmits/reflects field vector using 4x4 method.

    This functions takes a field vector that describes the input field and
    computes the output transmited field and also updates the input field
    with the reflected waves.
    """

    fmat_ini = inv(fmat_in)

    avec = dotmv(fmat_ini,fvec_in)

    a = np.zeros(avec.shape, avec.dtype)
    a[...,0::2] = avec[...,0::2]

    if fvec_out is not None:
        fmat_outi = inv(fmat_out)
        bvec = dotmv(fmat_outi,fvec_out)
        a[...,1::2] = bvec[...,1::2]
    else:
        bvec = np.zeros_like(avec)

    av = a.reshape(a.shape[:-2] + (a.shape[-2]*a.shape[-1],))
    out = dotmv(rmat,av).reshape(a.shape)

    avec[...,1::2] = out[...,1::2]
    bvec[...,::2] = out[...,::2]

    dotmv(fmat_in,avec,out = fvec_in)
    return dotmv(fmat_out,bvec,out = out)

def reflect2d(fvecin, fmatin, rmat, fmatout, fvecout = None):
    """Transmits/reflects field vector using 4x4 method.

    This functions takes a field vector that describes the input field and
    computes the output transmited field vector and also updates the input field
    with the reflected waves.
    """
    verbose_level = DTMMConfig.verbose
    if verbose_level > 2:
        print ("Transmitting field.")
    if isinstance(fvecin, tuple):
        n = len(fvecin)
        if fvecout is None:
            return tuple((_reflect2d(fvecin[i], fmatin[i], rmat[i], fmatout[i]) for i in range(n)))
        else:
            return tuple((_reflect2d(fvecin[i], fmatin[i], rmat[i], fmatout[i], fvecout[i]) for i in range(n)))
    else:
        return _reflect2d(fvecin, fmatin, rmat, fmatout, fvecout)

def transfer2d(field_data_in, optical_data, betay = 0., nin = 1., nout = 1., method = "4x4", betamax = BETAMAX, field_out = None):

    f,w,p = field_data_in
    shape = f.shape[-1]
    d,epsv,epsa = optical_data
    k0 = wavenumber(w, p)

    if field_out is not None:
        mask, fmode_out = field2modes1(field_out,k0, betamax = betamax)
    else:
        fmode_out = None

    mask, fmode_in = field2modes1(f,k0, betamax = betamax)

    fmatin = f_iso2d(shape = shape, betay = betay, k0 = k0, n=nin, betamax = betamax)
    fmatout = f_iso2d(shape = shape, betay = betay, k0 = k0, n=nout, betamax = betamax)

    cmat = stack_mat2d(k0,d, epsv, epsa, betay = betay, mask = mask, method = method)
    smat = system_mat2d(fmatin = fmatin, cmat = cmat, fmatout = fmatout)
    rmat = reflection_mat2d(smat)

    fmode_out = reflect2d(fmode_in, rmat = rmat, fmatin = fmatin, fmatout = fmatout, fvecout = fmode_out)

    field_out = modes2field1(mask, fmode_out)
    f[...] = modes2field1(mask, fmode_in)

    return field_out,w,p

max_stars_repo_name: IJSComplexMatter/dtmm

max_issues_repo_name: IJSComplexMatter/dtmm

max_forks_repo_name: IJSComplexMatter/dtmm


text:
import numpy as np
import tensorflow as tf

from pararealml.core.operators.ml.pidon.auto_differentiator import \
    AutoDifferentiator

def test_gradient_int_x_axis():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.ones((2, 2), dtype=tf.float32)
        diff.watch(x)

        c = tf.constant([[1., 2.], [-4., -5.]], dtype=tf.float32)
        y = c * x

        x_axis = 1
        expected_gradient = [[0., 2.], [0., -5.]]
        actual_gradient = diff.batch_gradient(x, y, x_axis).numpy()
        assert np.allclose(actual_gradient, expected_gradient)

def test_gradient_tensor_x_axis():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.ones((3, 2), dtype=tf.float32)
        diff.watch(x)

        c = tf.constant([[3., -7.], [2., -1.], [4., 5.]], dtype=tf.float32)
        y = c * x

        x_axis = tf.constant([1, 0, 0], dtype=tf.int32)
        expected_gradient = [[0., -7.], [2., 0.], [4., 0.]]
        actual_gradient = diff.batch_gradient(x, y, x_axis).numpy()
        assert np.allclose(actual_gradient, expected_gradient)

def test_hessian():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.constant([[1., 2.], [3., 4.]], dtype=tf.float32)
        diff.watch(x)

        c = tf.constant([[1., 2.], [-4., -5.]], dtype=tf.float32)
        y = c * x ** 2

        x_axis1 = x_axis2 = 0
        expected_hessian = [[2., 0.], [-8., 0.]]
        actual_hessian = diff.batch_hessian(x, y, x_axis1, x_axis2).numpy()
        assert np.allclose(actual_hessian, expected_hessian)

def test_mixed_hessian():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.zeros((3, 2), dtype=tf.float32)
        diff.watch(x)

        y = tf.reduce_sum(x, axis=1, keepdims=True) ** 2

        x_axis1 = 1
        x_axis2 = 0
        expected_hessian = [[2.], [2.], [2.]]
        actual_hessian = diff.batch_hessian(x, y, x_axis1, x_axis2).numpy()
        assert np.allclose(actual_hessian, expected_hessian)

def test_divergence():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.ones((2, 2), dtype=tf.float32)
        diff.watch(x)

        c = tf.constant([[1., 2.], [-4., -5.]], dtype=tf.float32)
        y = c * x

        expected_divergence = [[3.], [-9.]]
        actual_divergence = diff.batch_divergence(x, y).numpy()
        assert np.allclose(expected_divergence, actual_divergence)

def test_curl():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.ones((3, 3), dtype=tf.float32)
        diff.watch(x)

        c = tf.constant(
            [[1., 2., 3.], [-4., -5., -6.], [0., -2., 2.]],
            dtype=tf.float32)
        y = tf.concat(
            [
                tf.reduce_sum(c * x, axis=1, keepdims=True),
                tf.reduce_sum(c * x ** 2, axis=1, keepdims=True),
                tf.reduce_sum(x, axis=1, keepdims=True),
            ], axis=1)

        curl_ind = 2
        expected_curl = [[0.], [-3.], [2.]]
        actual_curl = diff.batch_curl(x, y, curl_ind).numpy()
        assert np.allclose(expected_curl, actual_curl)

def test_laplacian():
    with AutoDifferentiator(persistent=True) as diff:
        x = tf.constant([[1., 2.], [3., 4.]], dtype=tf.float32)
        diff.watch(x)

        c = tf.constant([[-1.], [2.]], dtype=tf.float32)
        y = c * tf.reduce_sum(x * x, axis=1, keepdims=True)

        expected_laplacian = [[-4.], [8.]]
        actual_laplacian = diff.batch_laplacian(x, y).numpy()
        assert np.allclose(expected_laplacian, actual_laplacian)

max_stars_repo_name: ViktorC/PararealML

max_issues_repo_name: ViktorC/PararealML

max_forks_repo_name: ViktorC/PararealML


text:
# coding: utf-8

# In[1]:

import os

import numpy as np
import pyspark.sql.functions as F
from pyspark import SparkContext, SQLContext, SparkConf
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.feature import PCA
from pyspark.ml.feature import VectorAssembler
from pyspark.sql import Window
from pyspark.sql.functions import col, row_number, lit
from pyspark.sql.types import FloatType
from pyspark.sql.utils import AnalysisException

# In[2]:

# In[3]:

SPARK_IP = "192.168.240.10"
SPARK_URL = "spark://" + SPARK_IP + ":7077"
MIN_RATINGS = 10
TRAIN_TEST_SPLIT = 0.8
DIR_ROOT = "hdfs://" + SPARK_IP
FOLDS = 5
CREATE_FOLDS = False
CREATE_DISTANCES = False

# In[4]:

conf = SparkConf()
conf.set("spark.executor.memory", "4g")
# conf.set("spark.sql.shuffle.partitions", "400")
# conf.set("spark.yarn.executor.memoryOverhead", "256m")
conf.set("spark.network.timeout", "2000")
conf.set("spark.sql.broadcastTimeout", "300000")
# conf.set("spark.dynamicAllocation.enabled","true")
# conf.set("spark.shuffle.service.enabled", "true")
# conf.set("spark.local.dir", "/yelp-dataset/spark-tmp")
conf.set("spark.driver.memory", "1g")
# conf.set("spark.driver.maxResultSize","10g")
# sc = SparkContext("local[*]", "Simple App", conf=conf)
# sc.setCheckpointDir('/tmp')
os.environ['PYSPARK_PYTHON'] = '/usr/bin/python3.6'
conf.setMaster(SPARK_URL)
sc = SparkContext(conf=conf)
sql_sc = SQLContext(sc)

# In[5]:

# users.persist()

# In[6]:

# + cat_features + geo_features

#     train_df = train_df.select(["user_id", "user_id_2"] + feature_columns + other_columns)
#     assembler = VectorAssembler(inputCols=feature_columns, outputCol="input_features")
#     features_df = assembler.transform(train_df)

#     scaler = StandardScaler(inputCol="input_features", outputCol="scaled_features",
#                             withStd=True, withMean=True)
#     # Compute summary statistics by fitting the StandardScaler
#     scalerModel = scaler.fit(features_df)
#     # Normalize each feature to have unit standard deviation.
#     scaledData = scalerModel.transform(features_df)
#     normalizer = Normalizer(inputCol="scaled_features", outputCol="normalized_features", p=2.0)
#     l1NormData = normalizer.transform(scaledData)
#     return l1NormData

# check_users = sql_sc.createDataFrame([("u1", 3.0),
#                              ("u2", 2.0),
#                             ("u3", 1.0)
#                             ], ["user_id", "average_stars"])

# check_df = sql_sc.createDataFrame([("u1", "a", 3.5, None),
#                              ("u1", "c", 3.5, 3.5),
#                              ("u1", "d", 3.5, 3.5),
#                              ("u1", "e", 3.5, 3.5),
#                              ("u1", "g", 3.5, 3.5),
#                              ("u2", "f", 2.0, None),
#                              ("u2", "a", 5.0, 5.0),
#                              ("u2", "f", 2.0, 2.0),
#                              ("u2", "a", 5.0, 5.0),
#                              ("u2", "f", 2.0, 2.0),
#                             ("u2", "a", 5.0, 5.0),
#                             ("u3", "b", 3.0, 3.0),
#                              ("u3", "b", 3.0, None),
#                              ("u3", "b", 3.0, 3.0),
#                              ("u3", "b", 3.0, 3.0),
#                              ("u3", "b", 3.0, 3.0)
#                             ], ["user_id", "business_id", "stars", "pred_stars"])
# mae, cov, prec, prec_liked = evaluate(check_df, check_users, 100)
# print("{},{},{},{}".format(mae, cov, prec, prec_liked))

# In[7]:

# In[9]:

max_stars_repo_path: Collaborative Filtering.py

max_issues_repo_path: Collaborative Filtering.py

max_forks_repo_path: Collaborative Filtering.py


text:
import h5py
import os
import numpy as np

base_path = os.path.dirname(os.path.realpath(__file__))
feature_file = '/media/uttaran/FCE1-7BF3/Gamma/Gait/classifier_stgcn/model_classifier_stgcn/featuresCombineddeep_features.txt'
f = np.loadtxt(feature_file)
fCombined = h5py.File('/media/uttaran/FCE1-7BF3/Gamma/Gait/data/featuresCombined.h5', 'r')
fkeys = fCombined.keys()
dfCombined = h5py.File('/media/uttaran/FCE1-7BF3/Gamma/Gait/data/deepFeaturesCombined.h5', 'w')
for i, fkey in enumerate(fkeys):
    fname = [fkey][0]
    feature = f[i, :]
    dfCombined.create_dataset(fname, data=feature)
dfCombined.close()


text:
import logging
from ast import literal_eval

import numpy as np
import pandas as pd
from PyQt5 import uic
from PyQt5.QtWidgets import (QPushButton, QTableWidget, QTableWidgetItem,
                             QWidget)

logger = logging.getLogger(__name__)

class ItemType(int):
    TYPE_DEFINITION = {
        object: 1000,
        int: 1001,
        np.integer: 1001,
        float: 1002,
        np.floating: 1002,
        str: 1003,
        np.character: 1003
    }

    def __new__(cls, item_type: type):
        key = item_type
        while(key != None):
            if key in ItemType.TYPE_DEFINITION:
                obj = int.__new__(cls, ItemType.TYPE_DEFINITION[key])
                obj.type = key
                obj.original_type = item_type

                if key == object:
                    logger.warn(f'Unknown Type: {item_type}')
                break
            else:
                key = key.__base__

        return obj

    def __init__(self, type: type) -> None:
        super().__init__()
        self.type = self.type
        self.original_type = self.original_type

class DataSheetItem(QTableWidgetItem):
    def __init__(self, text: str, item_type: ItemType = ItemType(object)):
        super().__init__(text, item_type)
        self.item_type = item_type

    def value(self):
        return self.item_type.type(self.text())

class DataSheetWidget(QWidget, uic.loadUiType('src/datasheet.ui')[0]):
    def __init__(self, excel_path=None) -> None:
        super().__init__()
        self.setupUi(self)
        self.data_widget: QTableWidget = self.data_widget
        self.add_col_button: QPushButton = self.add_col_button

        self.data_widget.cellChanged.connect(self.add_row)
        self.add_col_button.clicked.connect(self.add_col)

        if excel_path is not None:
            self.name, df = self.load_excel(excel_path)
            self.initialize_all(df)
        else:
            self.initialize_all()

    def load_excel(self, path: str):
        temp = pd.ExcelFile(path)
        df = temp.parse(header=0, index_col=0)
        name = temp.sheet_names[0]
        temp.close()

        return name, df

    def add_row(self, row: int):
        if row == self.data_widget.rowCount() - 1:
            self.data_widget.setRowCount(self.data_widget.rowCount() + 1)
            self.data_widget.setVerticalHeaderItem(self.data_widget.rowCount() - 2, None)

        self.data_widget.setVerticalHeaderItem(
            self.data_widget.rowCount() - 1, QTableWidgetItem('...'))

    def add_col(self):
        self.data_widget.setColumnCount(self.data_widget.columnCount() + 1)

    def initialize_all(self, data=pd.DataFrame(columns=['Value'])) -> None:
        self.data_widget.clear()
        data = data.fillna('')

        self.data_widget.setRowCount(data.shape[0] + 1)
        self.data_widget.setColumnCount(data.shape[1])

        for cidx, col_name in enumerate(data):
            col = data[col_name]
            for ridx, value in enumerate(col):
                val_type = ItemType(type(value))
                item = DataSheetItem(str(value), val_type)
                self.data_widget.setItem(ridx, cidx, item)

        self.data_widget.setHorizontalHeaderLabels([str(e) for e in data.columns])
        self.data_widget.setVerticalHeaderLabels([str(e) for e in data.index])

        self.data_widget.setVerticalHeaderItem(
            self.data_widget.rowCount() - 1, QTableWidgetItem('...'))

    def delete_row(self) -> None:
        for selection_range in self.data_widget.selectedRanges():
            print(
                f't: {selection_range.topRow()}, b: {selection_range.bottomRow()}')

        self.initialize_all()

    def export_to_dataframe(self):
        number_of_rows = self.data_widget.rowCount()
        number_of_columns = self.data_widget.columnCount()

        data = []

        for row_num in range(number_of_rows - 1):
            row = []
            for col_num in range(number_of_columns):
                item: DataSheetItem = self.data_widget.item(row_num, col_num)
                if item is not None:
                    row.append(item.value())
                else:
                    row.append('')
            data.append(row)

        column_labels = [
            self.data_widget.horizontalHeaderItem(n).text()
            if self.data_widget.horizontalHeaderItem(n) is not None
            else n
            for n in range(number_of_columns)
        ]
        index_labels = [
            self.data_widget.verticalHeaderItem(n).text()
            if self.data_widget.verticalHeaderItem(n) is not None
            else n
            for n in range(number_of_rows - 1)
        ]

        return pd.DataFrame(data, index=index_labels, columns=column_labels)


text:
import numpy as np

class Perceptron(object):
    """
    Implements a Perceptron
    """
    def __init__(self, learning_rate = 0.0001, max_iters = 100):
        """

        :param learning_rate: float  The learning rate for the perceptron algorithm.
        :param max_iters: int  The maximum allowed iterations of the learning algorithm.
        """
        self.learning_rate = learning_rate
        self.max_iters = max_iters
        self.weights = []
        self.errors_ = []

    def fit(self, data_frame, y):
        """

        :param data_frame: A matrix of m x n where m is the number of samples and n is the number of features.
        :param y: An array like object with the true labels for the m samples
        :return: self
        """
        self.weights = np.zeros(1 + data_frame.shape[1])
        self.errors_ = []

        for _ in range(self.max_iters):
            preds = self.predict(data_frame)
            indicator = (y - preds)
            row_update = indicator * self.learning_rate

            self.weights[0] = self.weights[0] + np.sum(row_update)
            self.weights[1:] = self.weights[1:] + np.sum( ((data_frame[:,1:].transpose() * row_update).transpose()), axis=0 )

            self.errors_.append((preds != y).sum())

        return self

    def score(self, sample, weights):
        """

        :param sample: A data point with n features
        :param weights: An array of weights of size n + 1
        :return: The "score" of the current data point for the provided weights
        """
        return np.dot(sample, weights[1:]) + weights[0]

    def predict(self, sample):
        """

        :param sample: A data frame with m data points and n features
        :return: The predicted class of the data point
        """
        pred = []

        for row in sample:
            if self.score(row, self.weights) >= 0:
                pred.append(1)
            else:
                pred.append(-1)

        return np.array(pred)

max_stars_repo_name: prasannals/perceptron

max_issues_repo_name: prasannals/perceptron

max_forks_repo_name: prasannals/perceptron


text:
#!/usr/bin/env python3
"""
video.py
---

Video reading utilities.

"""

import os

import imageio
import collections

import numpy as np

class VideoReader:
  def __init__(self, vid_path, start=0, seq_len=1, fps=29.97):
    """ The VideoReader serves a generator of frame-sequences as needed. To increase performance, sequential
    frame access is enforced, and frames are cached into a buffer to allow for quicker repeated and
    sequential accesses within the allocated buffer.
    TODO: Buffer implementation. Could be implemented by taking a cache_size param for max(caption_size) and
    keeping track lo/hi indices for when to update. See `_updateCache` for draft.

    Note memory usage may grow significantly with high-resolution video or large cache sizes, calculated by the
    following formula:
    ```
    bytes = seq_len * height * width * channels
    ```

    A 1080p (1920x1080) video with sequence length of 30, or approximately 1 second of 30fps footage equates to:
    ```
    30 * 1080 * 1920 * 3 bytes ~ 186MB
    ```

    :param vid_path: Path of the video to read from.
    :param start: The starting frame index to begin reading from.
    :param seq_len: The length of the sequence of frames to serve.
    """
    assert os.path.isfile(vid_path)
    reader = imageio.get_reader(vid_path)
    vid_len = reader.get_length()

    # State.
    self.lo = start
    self._reader = reader
    self.cache = collections.deque(maxlen=seq_len)
    self.buf = np.empty(shape=(seq_len,), dtype=np.ndarray)

    # For convenience.
    self._seq_len = seq_len
    self._vid_len = vid_len
    self._fps = fps

  def get_frame_idx(self, seconds):
    return int(seconds * self._fps)

  def getNumFrames(self):
    return self._vid_len

  def genFrames(self, lo, hi):
    # assert isinstance(self.buf, np.ndarray) and self.buf.ndim == 1 and len(self.buf) == self._seq_len
    # assert isinstance(self.cache, collections.deque) and self.cache.maxlen == len(self.cache) == self._seq_len
    # self._updateCache(lo, hi)

    # Populate ndarray vector with cache.
    # REVIEW josephz: Can this be improved with a buffer?
    assert self.lo <= lo <= hi
    self.lo = lo
    return [self._reader.get_data(x) for x in range(lo, min(hi, self.getNumFrames()))]
    # for x in range(lo, hi):
    #   yield self._reader.get_data(x)

  def _updateCache(self, lo, hi):
    raise NotImplementedError
    # assert isinstance(self.cache, collections.deque) and self.cache.maxlen == self._seq_len
    # assert self.lo <= lo < hi < self._vid_len
    #
    # # The subsequent sequence may jump ahead. If so, we only wish to load the minimum number of
    # # frames to catch-up since our previous sequence.
    # assert lo + len(self.cache) <= hi + self._seq_len
    # cacheRange = range(max(lo + len(self.cache), hi), hi + self._seq_len)
    # self.cache.extend(self._reader.get_data(x) for x in cacheRange)

max_stars_repo_name: joseph-zhong/LipReading

max_issues_repo_name: BirdofRain/LipReading

max_forks_repo_name: BirdofRain/LipReading


text:
from pyjamas_core import Supermodel
from pyjamas_core.util import Input, Output, Property
import numpy as np
from datetime import datetime as dt
from Models._utils.time import utc_time2datetime, datetime2utc_time

# define the model class and inherit from class "Supermodel"
class Model(Supermodel):
    # model constructor
    def __init__(self, id, name: str):
        # instantiate supermodel
        super(Model, self).__init__(id, name)

        # define inputs
        self.inputs['futures'] = Input('Futures', unit='s', info='utc time array in seconds since epoch')
        self.inputs['kwDaten'] = Input('PowerPlantsData', info='dict, power plant id required')

        # define outputs
        self.outputs['load'] = Output('Load', info='load of all running-water power plants, value[0-1]')

        self.ref_year = 2016
        ref_dates = [[self.ref_year, 1, 1],   [self.ref_year, 1, 8],   [self.ref_year, 1, 12],  [self.ref_year, 1, 17],  [self.ref_year, 1, 26],
                     [self.ref_year, 2, 1],   [self.ref_year, 2, 8],   [self.ref_year, 2, 10],  [self.ref_year, 2, 12],  [self.ref_year, 2, 14], [self.ref_year, 2, 20], [self.ref_year, 2, 26],
                     [self.ref_year, 3, 1],   [self.ref_year, 3, 6],   [self.ref_year, 3, 14],  [self.ref_year, 3, 28],
                     [self.ref_year, 4, 7],   [self.ref_year, 4, 11],  [self.ref_year, 4, 19],  [self.ref_year, 4, 22],  [self.ref_year, 4, 24], [self.ref_year, 4, 30],
                     [self.ref_year, 5, 2],   [self.ref_year, 5, 8],   [self.ref_year, 5, 11],  [self.ref_year, 5, 13],  [self.ref_year, 5, 22],
                     [self.ref_year, 6, 1],
                     [self.ref_year, 7, 7],   [self.ref_year, 7, 10],  [self.ref_year, 7, 13],  [self.ref_year, 7, 20],  [self.ref_year, 7, 30],
                     [self.ref_year, 8, 4],   [self.ref_year, 8, 5],   [self.ref_year, 8, 7],   [self.ref_year, 8, 16],  [self.ref_year, 8, 19],
                     [self.ref_year, 9, 4],   [self.ref_year, 9, 6],   [self.ref_year, 9, 18],  [self.ref_year, 9, 19],
                     [self.ref_year, 10, 14], [self.ref_year, 10, 24], [self.ref_year, 10, 26],
                     [self.ref_year, 11, 4],  [self.ref_year, 11, 6],  [self.ref_year, 11, 9],  [self.ref_year, 11, 11], [self.ref_year, 11, 16], [self.ref_year, 11, 19],
                     [self.ref_year, 12, 1],  [self.ref_year, 12, 11], [self.ref_year, 12, 18], [self.ref_year, 12, 1],
                     [self.ref_year + 1, 1, 1]]
        self.ref_dates = np.array([datetime2utc_time(dt(d[0], d[1], d[2])) for d in ref_dates])
        self.ref_loads = np.array([0.300, 0.512,  1.0,   0.5447, 0.432,
                                   1,     0.748,  0.834, 0.742,  0.8473, 0.6227, 0.7887,
                                   0.674, 0.8247, 0.541, 0.431,
                                   0.646, 0.5747, 1.0,   0.8,    0.9687, 0.778,
                                   0.925, 0.6827, 0.682, 1.0,    0.994,
                                   1.0,
                                   1.0,   0.882,  1.0,   1.0,    0.827,
                                   0.774, 1.0,    1.0,   0.742,  0.805,
                                   0.550, 0.77,   0.524, 0.552,
                                   0.328, 0.336,  0.542,
                                   0.372, 0.534,  0.49, 0.762, 0.534, 0.704,
                                   0.425, 0.340,  0.316, 0.284,
                                   0.300])

    async def func_peri(self, prep_to_peri=None):
        # get inputs
        futures = await self.get_input('futures')
        kwDaten = await self.get_input('kwDaten')

        load = self.laufwasserKWauslastung(kwDaten, futures)

        # set output
        self.set_output("load", load)

    # define additional methods (normal)
    def laufwasserKWauslastung(self, KWDaten, Futures):
        # Determine the load(Auslastung) running water power plant
        ###################################################################################################################
        # Input Arguments:
        # KWDaten: Dictionary holding the different parameters of power plants
        # ----------------------------------------------------------------------------------------------
        #   id  fk_kwt   kw_bezeichnung    power[W]          spez_info             Capex   Spez_Opex,  usw...
        # ----------------------------------------------------------------------------------------------
        #   1     2       Windturbine      1000000       NH: 150,  Z0: 0.03          1     0.01
        #   2     1      Photovoltaik      2000000       NH: 0,    Z0: {}            2     0.02
        #   3     2       Windturbine      3000000       NH: 200,  Z0: 0.2           3     0.03
        #   4     1      Photovoltaik      4000000       NH: 0,    Z0: {}            4     0.04
        #   5     2       Windturbine      5000000       NH: 250,  Z0: 0.03          5     0.05
        #   6     1      Photovoltaik      6000000       NH: 0,    Z0: {}            6     0.06
        #   8     3        Others          1000000       NH: 0,    Z0: {}            7     0.07
        #   10    3        Others          1000000       NH: 0,    Z0: {}            8     0.08
        #   11    4        Others          1000000       NH: 0,    Z0: {}            9     0.09
        # [KWID, FKKWT, KWBezeichnung, Power, Weitere spezifische parameter(Nabenhoehe, Z0, usw.), Capex,
        #  Spez_Opex, KEV, Brennstoffkosten, Entsorgungskostne, CO2-Kosten, usw.]
        #
        # Futures: Incoming datetime values (produced by Scheduler/Cronjob/V2), required for interpolation
        #
        # Output Arguments:
        # Auslastung: Dictionary containing Power plant IDs(id) in the first list and corresponding calculated
        # load(Auslastung) of running-water power plants in second list, output values are between [0-1] except ids
        # ---------------------------
        #   id   Auslastung   Note: Output matrix contains the load of running water power plants only
        # ---------------------------
        #   10    array(96)
        #   11    array(96)
        ###################################################################################################################
        KWBezeichnung = 'Laufwasserkraftwerk'  # ForeignKeyKWTyp = 1  # ForeignKey Kraftwerkstyp z.B. 1= PV-Anlage, 2= WindKraftwerk
        KWDaten = np.array([KWDaten['id'], KWDaten['bez_kraftwerkstyp'], KWDaten['spez_info']]).transpose()

        # Extracting data corresponding solely to running water power plant
        KraftwerksDaten = KWDaten[KWDaten[:, 1] == KWBezeichnung]

        def make_load_for_one_plant():

            futures = [utc_time2datetime(f).replace(year=self.ref_year) for f in Futures]
            futures = [datetime2utc_time(f) for f in futures]
            # One-dimensional linear interpolation
            load = np.interp(futures, self.ref_dates, self.ref_loads).tolist()
            return load

        KWid = [kw[0] for kw in KraftwerksDaten]
        loads = [make_load_for_one_plant() for kw in KraftwerksDaten]

        Auslastung = {'power_plant_id': KWid, 'load': loads}
        return Auslastung

# For testing purposes
if __name__ == "__main__":
    kw_daten = {'id': [1, 2, 3, 4, 5, 6, 8, 10, 11], 'fk_kwt': [2, 1, 2, 1, 2, 1, 5, 3, 4],
             'kw_bezeichnung': ['WT','PV','WT','PV','WT','PV','Laufwasserkraftwerk','Laufwasserkraftwerk','OTHER'],
             'power': [1000000, 2000000, 3000000, 4000000, 5000000, 6000000, 8000000, 9000000, 7000000],
             'spez_info': [{'NH': 150, 'Z0': 0.03}, {}, {'NH': 100, 'Z0': 0.2}, {}, {'NH': 250, 'Z0': 0.03}, {}, {}, {}, {}],
             'capex': [1, 2, 3, 4, 5, 6, 7, 8, 9],
             'spez_opex': [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09]}

    Props={}
    Inputs = {
        'futures': [1531227300, 1531227400, 1531227500, 1531227600],
        'kwDaten': kw_daten
    }
    Output = Model.test(Inputs, Props)

    print(Output)

max_stars_repo_name: schmocker/Pyjamas

max_issues_repo_name: schmocker/Pyjamas

max_forks_repo_name: schmocker/Pyjamas


text:
from math import factorial

import numpy
from sympy import Rational as frac
from sympy import sqrt

from ..helpers import techreport, untangle
from ._helpers import NSimplexScheme

citation = techreport(
    authors=["Noel J. Walkington"],
    title="Quadrature on simplices of arbitrary dimension",
    institution="CMU",
    year="2000",
    url="https://www.math.cmu.edu/~nw0z/publications/00-CNA-023/023abs/",
)

def walkington_1(d):
    degree = 1
    data = [(frac(1, factorial(d)), _c(d, frac))]
    points, weights = untangle(data)
    # normalize weights
    weights /= numpy.sum(weights)
    return NSimplexScheme("Walkington 1", d, weights, points, degree, citation)

def walkington_2(d):
    # The article claims order 2, but tests really only show order 1. Also, the article
    # says:
    #
    # > The points are inside the simplex when the positive square root is selected.
    #
    # Not sure what this means, but for d>=2, the points are outside the simplex.
    degree = 1
    data = [(frac(1, factorial(d + 1)), _xi1(d, 1 / sqrt(d + 1)))]
    points, weights = untangle(data)
    # normalize weights
    weights /= numpy.sum(weights)
    return NSimplexScheme("Walkington 2", d, weights, points, degree, citation)

def walkington_3(d):
    degree = 3
    data = [
        (frac(-((d + 1) ** 3), 4 * factorial(d + 2)), _c(d, frac)),
        (frac(+((d + 3) ** 3), 4 * factorial(d + 3)), _xi1(d, frac(1, (d + 3)))),
    ]
    points, weights = untangle(data)
    # normalize weights
    weights /= numpy.sum(weights)
    return NSimplexScheme("Walkington 3", d, weights, points, degree, citation)

def walkington_5(d):
    degree = 5
    w0 = frac(+((d + 1) ** 5), 32 * factorial(d + 3))
    w1 = frac(-((d + 3) ** 5), 16 * factorial(d + 4))
    w2 = frac(+((d + 5) ** 5), 16 * factorial(d + 5))
    data = [
        (w0, _c(d, frac)),
        (w1, _xi1(d, frac(1, d + 3))),
        (w2, _xi1(d, frac(1, d + 5))),
        (w2, _xi11(d, frac(1, d + 5), frac)),
    ]

    points, weights = untangle(data)
    # normalize weights
    weights /= numpy.sum(weights)
    return NSimplexScheme("Walkington 5", d, weights, points, degree, citation)

def walkington_7(d):
    degree = 7
    w0 = -frac(1, 384) * frac((d + 1) ** 7, factorial(d + 4))
    w1 = +frac(1, 128) * frac((d + 3) ** 7, factorial(d + 5))
    w2 = -frac(1, 64) * frac((d + 5) ** 7, factorial(d + 6))
    w3 = +frac(1, 64) * frac((d + 7) ** 7, factorial(d + 7))
    data = [
        (w0, _c(d, frac)),
        (w1, _xi1(d, frac(1, d + 3))),
        (w2, _xi1(d, frac(1, d + 5))),
        (w2, _xi11(d, frac(1, d + 5), frac)),
        (w3, _xi1(d, frac(1, d + 7))),
        (w3, _xi21(d, frac(1, d + 7), frac)),
        (w3, _xi111(d, frac(1, d + 7), frac)),
    ]

    points, weights = untangle(data)
    # normalize weights
    weights /= numpy.sum(weights)
    return NSimplexScheme("Walkington 7", d, weights, points, degree, citation)

def _c(d, frac):
    return numpy.array([numpy.full(d + 1, frac(1, d + 1))])

def _xi1(d, a):
    out = numpy.full((d + 1, d + 1), a)
    b = 1 - d * a
    numpy.fill_diagonal(out, b)
    return out

def _xi11(d, a, frac):
    assert d > 1
    b = frac(1 - (d - 1) * a, 2)
    if d == 2:
        out = numpy.array([[b, b, a], [b, a, b], [a, b, b]])
    else:
        assert d == 3
        out = numpy.array(
            [
                [b, b, a, a],
                [b, a, b, a],
                [b, a, a, b],
                [a, b, a, b],
                [a, a, b, b],
                [a, b, b, a],
            ]
        )
    return out

def _xi21(d, a, frac):
    assert d > 1
    b = frac(1 - (d - 2) * a, 3)
    # ERR Note that the article incorrectly states (d-2) the the expression for c.
    c = 1 - (d - 1) * a - b
    if d == 2:
        out = numpy.array(
            [[b, c, a], [c, b, a], [c, a, b], [b, a, c], [a, b, c], [a, c, b]]
        )
    else:
        assert d == 3
        out = numpy.array(
            [
                [b, c, a, a],
                [b, a, c, a],
                [b, a, a, c],
                [a, b, a, c],
                [a, a, b, c],
                [a, b, c, a],
                [c, b, a, a],
                [c, a, b, a],
                [c, a, a, b],
                [a, c, a, b],
                [a, a, c, b],
                [a, c, b, a],
            ]
        )

    return out

def _xi111(d, a, frac):
    assert d == 3
    b = frac(1 - (d - 2) * a, 3)
    out = numpy.array([[b, b, b, a], [b, b, a, b], [b, a, b, b], [a, b, b, b]])
    return out

max_stars_repo_name: whzup/quadpy

max_issues_repo_name: whzup/quadpy

max_forks_repo_name: whzup/quadpy

reason: import numpy,from sympy


text:
"""This module contains NetatmoRepository which contains all necessary methods for a DtssHost to handle communications
with a Netatmo weather station service."""

import typing as ty
from abc import abstractmethod, ABC
import logging

import numpy as np
import lnetatmo
from shyft.time_series import (StringVector, UtcPeriod, TimeAxisByPoints, TimeSeries, POINT_INSTANT_VALUE, TsVector,
                               TsInfoVector,
                               TsInfo, time, utctime_now, Calendar)

from weather.data_sources.netatmo.domain import NetatmoDomain
from weather.interfaces.config import RepositoryConfigBase, EnvVarConfig, EncryptedEnvVarConfig
from weather.interfaces.data_collection_repository import DataCollectionRepository
from weather.data_sources.netatmo.identifiers import parse_ts_id, parse_ts_query
from weather.utilities import rate_limiter

DeviceMetadataType = ty.Dict[str, object]
TimeType = ty.Union[float, int, time]
Number = ty.Union[float, int]
NoneType = type(None)
NanType = type(np.nan)

class NetatmoRepositoryError(Exception):
    """Exceptions raised by the NetatmoRepository."""
    pass

class NetatmoRepository(DataCollectionRepository):
    """The NetatmoRepository contains methods complying to the the DataCollectionRepository standard."""
    name = 'netatmo'

    def __init__(self, username: str,
                 password: str,
                 client_id: str,
                 client_secret: str,
                 rate_limiters: ty.Dict[str, ty.Dict[str, int]] = None) -> None:
        """

        :param username: Netatmo login username.
        :param password: Netatmo login password.
        :param client_id: Client id for communicating with the netatmo api.
        :param client_secret: Client id for communicating with the netatmo api.
        :param rate_limiters: Parameters for the RateLimiters that make sure we do not trip api communication limits.
        :param direct_login: Bool dictating if the api connection is established on init or if we want to initiate
        the api connection later.
        """

        self.username = username
        self.password = password
        self.client_id = client_id
        self.client_secret = client_secret

        # We don't want to breach the Netatmo API rate limiting policy, so we apply rate limiters to the functions
        # performing API calls to the Netatmo servers:
        self.rate_limiters = [rate_limiter.RateLimiter(**api_limit) for api_limit in rate_limiters.values()]
        for limiter in self.rate_limiters:
            self._get_measurements_block = limiter.rate_limit_decorator(self._get_measurements_block)

        # noinspection PyArgumentList
        self.utc = Calendar()

    def create_netatmo_connection(self) -> ty.Tuple[lnetatmo.WeatherStationData, NetatmoDomain]:
        """Refresh the netatmo connection."""
        auth = lnetatmo.ClientAuth(clientId=self.client_id,
                                   clientSecret=self.client_secret,
                                   username=self.username,
                                   password=self.password,
                                   scope='read_station')
        device_data = lnetatmo.WeatherStationData(auth)
        domain = NetatmoDomain(device_data.stations)

        return device_data, domain

    def wait_for_rate_limiters(self) -> None:
        """Check with the rate limiters, and wait if needed."""
        for limiter in self.rate_limiters:
            limiter.check_next_and_wait()

    def add_action_timestamp_to_rate_limiters(self, timestamp: TimeType) -> None:
        """Add a current action to rate limiters, so they keep track of actions."""
        for limiter in self.rate_limiters:
            limiter.add_action_timestamp(timestamp)

    @staticmethod
    def set_none_to_nan(values: ty.Sequence[ty.Union[Number, NoneType]]) -> ty.Sequence[ty.Union[Number, NanType]]:
        """Method that replaces None with np.nan in a sequence."""
        return [np.nan if value is None else value for value in values]

    def _get_measurements_block(self, *,
                                device_data: lnetatmo.WeatherStationData,
                                device_id: str,
                                module_id: str,
                                measurements: ty.Sequence[str],
                                utc_period: UtcPeriod = None
                                ) -> TsVector:
        """Get data for a specific device and set of measurements. utc_period is the timespan for which we ask for
        data, but it is optional, as utc_period=None asks for the longest possible timespan of data.

        NB: Calls are limited to 1024 values. Must split to get all data in period (50 req pr sec, 500 req pr hour).

        Args:
            device_id: Unique identifier for the netatmo device.
            module_id: Unique identifier for the netatmo module (can be None, '').
            measurements: A ty.Sequence of strings representing the measurements we want to fetch.
            utc_period: Inclusive start/end. The period we want data for (if none, the longest possible period
                        (up to 1024 values).

        Returns:
            A TsVector with timeseries containing data for each measurement type, in the order of the input.
        """

        date_start = float(utc_period.start) if utc_period else None
        date_end = float(utc_period.end) if utc_period else None

        self.wait_for_rate_limiters()
        self.add_action_timestamp_to_rate_limiters(utctime_now())

        measurement_types_str = ','.join([m for m in measurements])

        data = device_data.getMeasure(
            device_id=device_id,
            module_id=module_id,
            scale='max',
            mtype=measurement_types_str,
            date_begin=date_start,
            date_end=date_end)

        if not data['body']:
            # noinspection PyArgumentList
            output = [TimeSeries() for _ in measurements]
        else:
            t = [float(timestamp) for timestamp in data['body'].keys()]
            # Add an additional timestep fmod(dt) forward in time to indicate the validness of the last value.
            dt_list = [t2 - t1 for t1, t2 in zip(t[0:-2], t[1:-1])]
            dt_mode = max(set(dt_list), key=dt_list.count)
            ta = TimeAxisByPoints(t + [t[-1] + dt_mode])

            values_pr_time = [value for value in data['body'].values()]
            values = list(map(list, zip(*values_pr_time)))

            # Remove nan:
            output = [TimeSeries(ta, self.set_none_to_nan(vector), POINT_INSTANT_VALUE) for vector in values]

        return TsVector(output)

    def get_measurements(self, *,
                         device_data: lnetatmo.WeatherStationData,
                         station_id: str,
                         module_id: str,
                         measurements: ty.Sequence[str],
                         utc_period: UtcPeriod
                         ) -> TsVector:
        """Get data for a specific device and set of measurements. utc_period defines the data period.
        Netatmo only returns data in 1024 point chunks, so this method performs multiple calls to retrieve all queried
        data.

        Args:
            station_id: Unique identifier for the netatmo device.
            module_id: Unique identifier for the netatmo module (can be None, '').
            measurements: A ty.Sequence of strings representing the measurements we want to fetch.
            utc_period: Inclusive start/end. The period we want data for (if none, the longest possible period
                        (up to 1024 values).

        Returns:
            A TsVector with timeseries containing data for each measurement type, in the order of the input.
        """

        result_end = utc_period.start
        # noinspection PyArgumentList
        output = [TimeSeries() for _ in measurements]
        while result_end < utc_period.end:
            utc_period = UtcPeriod(result_end, utc_period.end)  # Define a UtcPeriod for the remaining data.

            result = self._get_measurements_block(
                device_data=device_data,
                device_id=station_id,
                module_id=module_id,
                measurements=measurements,
                utc_period=utc_period)

            if not any([bool(ts) for ts in result]):  # None data in period. Return blank.
                break
            for ind, res in enumerate(result):
                if res:
                    if not output[ind]:
                        output[ind] = res
                    else:
                        output[ind] = output[ind].extend(res)

            result_end = result[0].time_axis.time_points_double[-1]  # Set the start of the new calls UtcPeriod.
            # noinspection PyArgumentList
            logging.info(f'Got {len(result[0])} data points from '
                         f'{self.utc.to_string(result[0].time_axis.time_points_double[0])} to '
                         f'{self.utc.to_string(result_end)}')

        return TsVector(output)

    def read(self, list_of_ts_id: ty.Sequence[str], period: UtcPeriod) -> ty.Dict[str, TimeSeries]:
        """Take a sequence of strings identifying specific timeseries and get data from these series according to
        the timespan defined in period.

        Note: The read_callback is a less specialized function than the TsRepository.read, so this method just calls
        the read_callback.

        Args:
            list_of_ts_id: A sequence of strings identifying specific timeseries available from the netatmo login.
            period: A period defined by a utc timestamp for the start and end of the analysis period.

        Returns:
            A TsVector containing the resulting timeseries containing data enough to cover the query period.
        """

        return {ts_id: ts for ts_id, ts in zip(list_of_ts_id, self.read_callback(ts_ids=StringVector([list_of_ts_id]),
                                                                                read_period=period))}

    def read_callback(self, ts_ids: StringVector, read_period: UtcPeriod) -> TsVector:
        """This callback is passed as the default read_callback for a shyft.time_series.DtsServer.

        Args:
            ts_ids: A sequence of strings identifying specific timeseries available from the netatmo login.
            read_period: A period defined by a utc timestamp for the start and end of the analysis period.

        Returns:
            A TsVector containing the resulting timeseries containing data enough to cover the query period.
        """

        device_data, domain = self.create_netatmo_connection()
        data = dict()  # Group ts_ids by repo.name (scheme).
        for enum, ts_id in enumerate(ts_ids):
            ts_id_props = parse_ts_id(ts_id=ts_id)
            measurement = domain.get_measurement(**ts_id_props)

            if measurement.module.id not in data:
                data[measurement.module.id] = []
            data[measurement.module.id].append(
                dict(enum=enum, ts=None, measurement=measurement))

        for module_id in data:
            measurements = [meas['measurement'] for meas in data[module_id]]

            if module_id == measurements[0].station.id:  # The current module is the actual station itself.
                module_id_arg = None
            else:
                module_id_arg = module_id
            measurement_types = [m.data_type.name for m in measurements]
            tsvec = self.get_measurements(device_data=device_data,
                                          station_id=measurements[0].station.id,
                                          module_id=module_id_arg,
                                          measurements=measurement_types,
                                          utc_period=read_period
                                          )
            for index, ts in enumerate(tsvec):
                data[module_id][index]['ts'] = ts

        # Collapse nested lists and sort by initial enumerate:
        transpose_data = []
        for items in data.values():
            transpose_data.extend(items)
        sort = sorted(transpose_data, key=lambda item: item['enum'])

        return TsVector([item['ts'] for item in sort])

    def find_callback(self, query: str) -> TsInfoVector:
        """This callback is passed as the default find_callback for a shyft.time_series.DtsServer.

        Args:
            query: The url representing a relevant query for this DataCollectionRepository.

        Returns:
            A sequence of results matching the query.
        """

        return self.find(query)

    def find(self, query: str) -> TsInfoVector:
        """Check if data matching the query exists in the data source.

        Args:
            query: The url representing a relevant query for this DataCollectionRepository.

        Returns:
            A sequence of results matching the query.
        """
        info = parse_ts_query(ts_query=query)
        domain = self.create_netatmo_connection()[1]
        meas = domain.get_measurement(**info)

        # noinspection PyArgumentList
        tsi = TsInfo(
            name=meas.ts_id,
            point_fx=meas.data_type.point_interpretation,
            delta_t=np.nan,
            olson_tz_id=meas.station.place['timezone'],
            data_period=UtcPeriod(meas.module.last_setup, meas.module.last_seen),
            created=meas.module.last_setup,
            modified=meas.module.last_seen
        )

        # noinspection PyArgumentList
        tsiv = TsInfoVector()
        tsiv.append(tsi)
        return tsiv

    def read_forecast(self, list_of_fc_id: ty.List[str], period: UtcPeriod):
        """
        read and return the newest forecast that have the biggest overlap with specified period
        note that we should check that the semantic of this is reasonable
        """
        raise NotImplementedError("read_forecast")

    def store(self, timeseries_dict: ty.Dict[str, TimeSeries]):
        """ Store the supplied time-series to the underlying db-system.
            Parameters
            ----------
            timeseries_dict: dict string:timeseries
                the keys are the wanted ts(-path) names
                and the values are shyft api.time-series.
                If the named time-series does not exist, create it.
        """
        raise NotImplementedError("read_forecast")

class NetatmoConfigBase(ABC, RepositoryConfigBase):
    """Metaclass defining the properties that are expected from a NetatmoConfig."""

    _unpack_props = ['username', 'password', 'client_id', 'client_secret', 'rate_limiters']

    @property
    @abstractmethod
    def username(self) -> str:
        """This property must contain the username for Netatmo."""

    @property
    @abstractmethod
    def password(self) -> str:
        """This property must contain the password for Netatmo."""

    @property
    @abstractmethod
    def client_id(self) -> str:
        """This property must contain the client_id for Netatmo."""

    @property
    @abstractmethod
    def client_secret(self) -> str:
        """This property must contain the client_secret for Netatmo."""

    @property
    @abstractmethod
    def rate_limiters(self) -> ty.Dict[str, ty.Dict[str, int]]:
        """This property must contain the optional api limits for Netatmo."""

class NetatmoEnvVarConfig(NetatmoConfigBase, EnvVarConfig):
    """Netatmo config information fetched partially from environment_variables."""

    def __init__(self,
                 username_var: str,
                 password_var: str,
                 client_id_var: str,
                 client_secret_var: str,
                 rate_limiters: ty.Dict[str, ty.Dict[str, int]] = None,
                 ) -> None:
        self.username_var = self.verify_env_var(username_var)
        self.password_var = self.verify_env_var(password_var)
        self.client_id_var = self.verify_env_var(client_id_var)
        self.client_secret_var = self.verify_env_var(client_secret_var)

        self._rate_limiters = rate_limiters or {}

    @property
    def username(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.username_var)

    @property
    def password(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.password_var)

    @property
    def client_id(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.client_id_var)

    @property
    def client_secret(self) -> str:
        """Return the client_secret for the Netatmo instance."""
        return self.get_env_var(self.client_secret_var)

    @property
    def rate_limiters(self) -> ty.Dict[str, ty.Dict[str, int]]:
        """Return the api limits for the RateLimiter for the Netatmo instance."""
        return self._rate_limiters

class NetatmoEncryptedEnvVarConfig(NetatmoConfigBase, EncryptedEnvVarConfig):
    """Netatmo config information fetched partially from environment_variables."""

    def __init__(self,
                 username_var: str,
                 password_var: str,
                 client_id_var: str,
                 client_secret_var: str,
                 password: str,
                 salt: str,
                 rate_limiters: ty.Dict[str, ty.Dict[str, int]] = None
                 ) -> None:
        self.username_var = self.verify_env_var(username_var)
        self.password_var = self.verify_env_var(password_var)
        self.client_id_var = self.verify_env_var(client_id_var)
        self.client_secret_var = self.verify_env_var(client_secret_var)

        self._rate_limiters = rate_limiters or {}

        super(NetatmoEncryptedEnvVarConfig, self).__init__(password=password, salt=salt)

    @property
    def username(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.username_var)

    @property
    def password(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.password_var)

    @property
    def client_id(self) -> str:
        """Return the username for the Netatmo instance."""
        return self.get_env_var(self.client_id_var)

    @property
    def client_secret(self) -> str:
        """Return the client_secret for the Netatmo instance."""
        return self.get_env_var(self.client_secret_var)

    @property
    def rate_limiters(self) -> ty.Dict[str, ty.Dict[str, int]]:
        """Return the api limits for the RateLimiter for the Netatmo instance."""
        return self._rate_limiters


text:
# import what you need
import os

import numpy as np
import cv2

import config_main
from Application.Frame.global_variables import JobInitStateReturn
from Application.Frame.transferJobPorts import get_port_from_wave
from Utils.log_handler import log_to_console, log_to_file, log_error_to_console

from Application.Config.create_config import jobs_dict, create_dictionary_element
from config_main import PYRAMID_LEVEL, APPL_INPUT_IMG_DIR
from Application.Config.util import transform_port_name_lvl, transform_port_size_lvl, job_name_create, get_module_name_from_file

from Application.Frame.global_variables import global_var_handler

"""
Module handles DESCRIPTION OF THE MODULE jobs for the APPL block.
"""

############################################################################################################################################
# Internal functions
############################################################################################################################################

############################################################################################################################################
# Init functions
############################################################################################################################################

cube_matrix = list()

# define a init function, function that will be executed at the begging of the wave
def init_func_global(param_list) -> JobInitStateReturn:
    """
    Init function for the job.
    Remember this function is called before the framework gets pictures.
    :return: INIT or NOT_INIT state for the job
    """
    return JobInitStateReturn(True)

############################################################################################################################################
# Main functions
############################################################################################################################################

# define a main function, function that will be executed at the begging of the wave
def main_func(param_list: list = None) -> bool:
    """
    Main function for {job} calculation job.
    :param param_list: Param needed to respect the following list:
                       [enumerate list]
    :return: True if the job executed OK.
    """
    # variables for position of param needed
    # ex:

    # noinspection PyPep8Naming
    PORT_IN_POS = 0
    # noinspection PyPep8Naming
    PORT_IN_WAVE = 1
    # noinspection PyPep8Naming
    PORT_OUT_NAME = 2
    # noinspection PyPep8Naming
    PORT_OUT_LOCATION = 3
    # verify that the number of parameters are OK.
    if len(param_list) != 4:
        log_error_to_console("IMAGE CUBE JOB MAIN FUNCTION PARAM NOK", str(len(param_list)))
        return False
    else:
        # get needed ports
        port_in = get_port_from_wave(name=param_list[PORT_IN_POS], wave_offset=param_list[PORT_IN_WAVE])

        # check if port's you want to use are valid
        if port_in.is_valid() is True:
            if True:
            # try:
                global cube_matrix
                cube_matrix.append(port_in.arr)

                if len(cube_matrix) == len(APPL_INPUT_IMG_DIR):
                    # Cast the list into a numpy array
                    img_array = np.array(cube_matrix)
                    for line in range(img_array.shape[1]):
                        if len(img_array.shape) == 4:
                            line_image = img_array[:, line, :, :]
                        else:
                            line_image = img_array[:, line, :]
                        # img_location = 'Logs/process_cube/IMG_CUBE_GRAY_RAW_LINE_L0'
                        img_location = param_list[PORT_OUT_LOCATION] + '/' + 'LINE_SLICING_' + param_list[PORT_OUT_NAME]
                        if not os.path.exists(img_location):
                            os.makedirs(img_location)
                        cv2.imwrite(img_location + '/' + '{:08d}'.format(line) + config_main.APPl_SAVE_PICT_EXTENSION, line_image)

                    for col in range(img_array.shape[2]):
                        if len(img_array.shape) == 4:
                            col_image = img_array[:, :, col, :]
                            col_image = np.transpose(col_image, (1, 0, 2))
                        else:
                            col_image = img_array[:, :, col]
                            col_image = np.transpose(col_image)
                        # img_location = 'Logs/process_cube/IMG_CUBE_GRAY_RAW_COL_L0'
                        img_location = param_list[PORT_OUT_LOCATION] + '/' + 'COLUMN_SLICING_' + param_list[PORT_OUT_NAME]
                        if not os.path.exists(img_location):
                            os.makedirs(img_location)
                        cv2.imwrite(img_location + '/' + '{:08d}'.format(col) + config_main.APPl_SAVE_PICT_EXTENSION, col_image)

            # except BaseException as error:
            #     log_error_to_console("JOB_NAME JOB NOK: ", str(error))
            #     pass
        else:
            return False

        return True

############################################################################################################################################
# Job create functions
############################################################################################################################################

def create_image_cube(port_input_name: str, location_to_save: str,
                      port_img_output: str = None, is_rgb: bool = False,
                      level: PYRAMID_LEVEL = PYRAMID_LEVEL.LEVEL_0, wave_offset: int = 0) -> str:
    """
    User-interface of {job}
    Add details of job
    Please add paper from feature
    Add as many other parameters needed
    :param port_input_name:  One or several input ports
    :param port_img_output:
    :param level: Level of input port, please correlate with each input port name parameter
    :param wave_offset: wave of input port, please correlate with each input port name parameter
    :return: Name of output port or ports
    """
    # Do this for each input port this function has
    input_port_name = transform_port_name_lvl(name=port_input_name, lvl=level)

    if port_img_output is None:
        port_img_output = 'IMG_CUBE_{Input}'.format(Input=port_input_name)

    # size can be custom as needed
    port_img_output_name = transform_port_name_lvl(name=port_img_output, lvl=level)
    port_img_output_name_size = transform_port_size_lvl(lvl=level, rgb=True)

    input_port_list = [input_port_name]
    init_func_param = [is_rgb]
    main_func_list = [input_port_name, wave_offset, port_img_output_name, location_to_save]
    # output_port_list = [(port_img_output_name, port_img_output_name_size, 'B', True)]
    output_port_list = []

    job_name = job_name_create(action='IMG_CUBE_{Input}'.format(Input=port_input_name), input_list=input_port_list, wave_offset=[wave_offset], level=level)

    d = create_dictionary_element(job_module=get_module_name_from_file(__file__),
                                  job_name=job_name,
                                  input_ports=input_port_list,
                                  init_func_name='init_func_global', init_func_param=init_func_param,
                                  main_func_name='main_func',
                                  main_func_param=main_func_list,
                                  output_ports=output_port_list)

    jobs_dict.append(d)

    return port_img_output

if __name__ == "__main__":
    # If you want to run something stand-alone
    pass