text:
# ===========================================================================
# ===========================================================================
# Copyright (c) 2021 Nghia T. Vo. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===========================================================================
# Author: Nghia T. Vo
# E-mail: algotomography@gmail.com
# Description: Examples of how to use the Algotom package.
# ===========================================================================

"""
The following examples show how to use Algotom to perform full reconstruction
of a standard tomographic data.

Raw data is at: https://zenodo.org/record/1443568
There're two files: "pco1-68067.hdf" contains flat-field, dark-field, and
projection images; "68067.nxs" contains metadata with a link to the
"pco1-68067.hdf" file.

Referring to "example_01_*.py" to know how to find key-paths and datasets
in a hdf/nxs file.

Referring to "example_06_*.py" to know how to include distortion correction.
"""

import numpy as np
import timeit
import algotom.io.loadersaver as losa
import algotom.prep.correction as corr
import algotom.prep.calculation as calc
import algotom.rec.reconstruction as reco
import algotom.prep.removal as remo
import algotom.prep.filtering as filt
import algotom.util.utility as util

file_path = "E:/Tomo_data/68067.nxs"
output_base = "E:/tmp/output4/"

# Optional parameters
start_slice = 10
stop_slice = -1
slice_chunk = 10 # Number of slices to be reconstructed in one go to reduce
                 # IO overhead (in loading a hdf file) and process in
                 # parallel (for CPU-based methods).

# Options to include artifact removal methods in the flat-field
# correction method.
opt1 = {"method": "remove_zinger", "para1": 0.08, "para2": 1}
opt2 = {"method": "remove_all_stripe", "para1": 3.0, "para2": 51, "para3": 17}
opt3 = None
# opt3 = {"method": "fresnel_filter", "para1": 100, "para2": 1} # Denoising

# Provide metadata for loading hdf file, get data shape and rotation angles.
data_key = "/entry1/flyScanDetector/data"
image_key = "/entry1/flyScanDetector/image_key"
angle_key = "/entry1/tomo_entry/data/rotation_angle"
ikey = np.squeeze(np.asarray(losa.load_hdf(file_path, image_key)))
angles = np.squeeze(np.asarray(losa.load_hdf(file_path, angle_key)))
data = losa.load_hdf(file_path, data_key)  # This is an object not ndarray.
(depth, height, width) = data.shape
# Get indices of projection images
proj_idx = np.squeeze(np.where(ikey == 0))
thetas = angles[proj_idx[0]:proj_idx[-1]] * np.pi / 180

time_start = timeit.default_timer()
print("---------------------------------------------------------------")
print("-----------------------------Start-----------------------------\n")
# Load dark-field images and flat-field images, averaging each result.
print("1 -> Load dark-field and flat-field images, average each result")
dark_field = np.mean(np.asarray(data[np.squeeze(np.where(ikey == 2.0)), :, :]),
                     axis=0)
flat_field = np.mean(np.asarray(data[np.squeeze(np.where(ikey == 1.0)), :, :]),
                     axis=0)
print("2 -> Calculate the center-of-rotation")
index = height // 2
sinogram = corr.flat_field_correction(data[proj_idx[0]:proj_idx[-1], index, :],
                                      flat_field[index, :],
                                      dark_field[index, :])
center = calc.find_center_vo(sinogram)
print("Center-of-rotation is {}".format(center))

if (stop_slice == -1) or (stop_slice > height):
    stop_slice = height
total_slice = stop_slice - start_slice
offset = start_slice
if slice_chunk > total_slice:
    slice_chunk = total_slice
num_iter = total_slice // slice_chunk
num_rest = total_slice - num_iter * slice_chunk

# Perform full reconstruction and save results as 32-bit tif images
for i in range(num_iter):
    start_sino = i * slice_chunk + offset
    stop_sino = start_sino + slice_chunk
    sinograms = corr.flat_field_correction(
        data[proj_idx[0]:proj_idx[-1], start_sino:stop_sino, :],
        flat_field[start_sino:stop_sino, :],
        dark_field[start_sino:stop_sino, :],
        option1=opt1, option2=opt2, option3=opt3)
    # Reconstruct a chunk of slices in parallel if using CPU-based method.
    recon_img = util.apply_method_to_multiple_sinograms(sinograms,
                                                        "dfi_reconstruction",
                                                        [center])
    # Save the results to tif images
    for j in range(start_sino, stop_sino):
        name = "0000" + str(j)
        losa.save_image(output_base + "/rec_" + name[-5:] + ".tif",
                        recon_img[:, j - start_sino, :])

    # # Reconstruct the slices using a GPU-based method
    # for j in range(start_sino, stop_sino):
    #     recon_img = reco.fbp_reconstruction(sinograms[:, j - start_sino, :],
    #                                         center, angles=thetas)
    #     name = "0000" + str(j)
    #     losa.save_image(output_base + "/rec_" + name[-5:] + ".tif", recon_img)

    t_stop = timeit.default_timer()
    print("Done slice: {0} - {1} . Time {2}".format(start_sino, stop_sino,
                                                    t_stop - time_start))
if num_rest != 0:
    for i in range(num_rest):
        start_sino = num_iter * slice_chunk + offset
        stop_sino = start_sino + num_rest
        sinograms = corr.flat_field_correction(
            data[proj_idx[0]:proj_idx[-1], start_sino:stop_sino, :],
            flat_field[start_sino:stop_sino, :],
            dark_field[start_sino:stop_sino, :],
            option1=opt1, option2=opt2, option3=opt3)
        # Reconstruct a chunk of slices in parallel if using CPU-based method.
        recon_img = util.apply_method_to_multiple_sinograms(sinograms,
                                                            "dfi_reconstruction",
                                                            [center])
        # Save the results to tif images
        for j in range(start_sino, stop_sino):
            name = "0000" + str(j)
            losa.save_image(output_base + "/rec_" + name[-5:] + ".tif",
                            recon_img[:, j - start_sino, :])

        # # Reconstruct the slices using a GPU-based method
        # for j in range(start_sino, stop_sino):
        #     recon_img = reco.fbp_reconstruction(sinograms[:, j - start_sino, :],
        #                                         center, angles=thetas)
        #     name = "0000" + str(j)
        #     losa.save_image(output_base + "/rec_" + name[-5:] + ".tif",
        #                     recon_img)

        t_stop = timeit.default_timer()
        print("Done slice: {0} - {1} . Time {2}".format(start_sino, stop_sino,
                                                        t_stop - time_start))
time_stop = timeit.default_timer()
print("!!! All Done. Time cost {} !!!".format(time_stop - time_start))

max_stars_repo_name: decarlof/algotom

max_issues_repo_name: decarlof/algotom

max_forks_repo_name: algotom/algotom


text:
# This file is part of QuTiP: Quantum Toolbox in Python.
#
#    Copyright (c) 2011 and later, Paul D. Nation and Robert J. Johansson.
#    All rights reserved.
#
#    Redistribution and use in source and binary forms, with or without
#    modification, are permitted provided that the following conditions are
#    met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of the QuTiP: Quantum Toolbox in Python nor the names
#       of its contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
#    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
#    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#    HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
###############################################################################

__all__ = ['essolve', 'ode2es']

import numpy as np
import scipy.linalg as la
import scipy.sparse as sp

from qutip.qobj import Qobj, issuper, isket, isoper
from qutip.eseries import eseries, estidy, esval
from qutip.expect import expect
from qutip.superoperator import liouvillian, mat2vec, vec2mat
from qutip.solver import Result
from qutip.operators import qzero

# Only used for deprecation warnings.
import functools
import warnings

def _deprecate(alternative):
    def decorated(f):
        message = (
            f"{f.__name__} is to be removed in QuTiP 5.0"
            f", consider swapping to {alternative}."
        )

        @functools.wraps(f)
        def out(*args, **kwargs):
            warnings.warn(message, DeprecationWarning, stacklevel=2)
            return f(*args, **kwargs)
        return out
    return decorated

# -----------------------------------------------------------------------------
# pass on to wavefunction solver or master equation solver depending on whether
# any collapse operators were given.
#
@_deprecate("mesolve")
def essolve(H, rho0, tlist, c_op_list, e_ops):
    """
    Evolution of a state vector or density matrix (`rho0`) for a given
    Hamiltonian (`H`) and set of collapse operators (`c_op_list`), by
    expressing the ODE as an exponential series. The output is either
    the state vector at arbitrary points in time (`tlist`), or the
    expectation values of the supplied operators (`e_ops`).

    .. deprecated:: 4.6.0
        :obj:`~essolev` will be removed in QuTiP 5.  Please use :obj:`~sesolve`
        or :obj:`~mesolve` for general-purpose integration of the
        Schroedinger/Lindblad master equation.  This will likely be faster than
        :obj:`~essolve` for you.

    Parameters
    ----------
    H : qobj/function_type
        System Hamiltonian.

    rho0 : :class:`qutip.qobj`
        Initial state density matrix.

    tlist : list/array
        ``list`` of times for :math:`t`.

    c_op_list : list of :class:`qutip.qobj`
        ``list`` of :class:`qutip.qobj` collapse operators.

    e_ops : list of :class:`qutip.qobj`
        ``list`` of :class:`qutip.qobj` operators for which to evaluate
        expectation values.

    Returns
    -------
     expt_array : array
        Expectation values of wavefunctions/density matrices for the
        times specified in ``tlist``.

    .. note:: This solver does not support time-dependent Hamiltonians.

    """
    n_expt_op = len(e_ops)
    n_tsteps = len(tlist)

    # Calculate the Liouvillian
    if (c_op_list is None or len(c_op_list) == 0) and isket(rho0):
        L = H
    else:
        L = liouvillian(H, c_op_list)

    es = ode2es(L, rho0)

    # evaluate the expectation values
    if n_expt_op == 0:
        results = [Qobj()] * n_tsteps
    else:
        results = np.zeros([n_expt_op, n_tsteps], dtype=complex)

    for n, e in enumerate(e_ops):
        results[n, :] = expect(e, esval(es, tlist))

    data = Result()
    data.solver = "essolve"
    data.times = tlist
    data.expect = [np.real(results[n, :]) if e.isherm else results[n, :]
                   for n, e in enumerate(e_ops)]

    return data

# -----------------------------------------------------------------------------
#
#
@_deprecate("direct eigenstate and -value calculation")
def ode2es(L, rho0):
    """Creates an exponential series that describes the time evolution for the
    initial density matrix (or state vector) `rho0`, given the Liouvillian
    (or Hamiltonian) `L`.

    .. deprecated:: 4.6.0
        :obj:`~ode2es` will be removed in QuTiP 5.  Please use
        :obj:`Qobj.eigenstates` to get the eigenstates and -values, and use
        :obj:`~QobjEvo` for general time-dependence.

    Parameters
    ----------
    L : qobj
        Liouvillian of the system.

    rho0 : qobj
        Initial state vector or density matrix.

    Returns
    -------
    eseries : :class:`qutip.eseries`
        ``eseries`` represention of the system dynamics.

    """
    if issuper(L):
        # check initial state
        if isket(rho0):
            # Got a wave function as initial state: convert to density matrix.
            rho0 = rho0 * rho0.dag()
        # check if state is below error threshold
        if abs(rho0.full()).sum() < 1e-10 + 1e-24:
            # enforce zero operator
            return eseries(qzero(rho0.dims[0]))
        w, v = L.eigenstates()
        v = np.hstack([ket.full() for ket in v])
        # w[i]   = eigenvalue i
        # v[:,i] = eigenvector i

        rlen = np.prod(rho0.shape)
        r0 = mat2vec(rho0.full())
        v0 = la.solve(v, r0)
        vv = v * sp.spdiags(v0.T, 0, rlen, rlen)

        out = None
        for i in range(rlen):
            qo = Qobj(vec2mat(vv[:, i]), dims=rho0.dims, shape=rho0.shape)
            if out:
                out += eseries(qo, w[i])
            else:
                out = eseries(qo, w[i])

    elif isoper(L):

        if not isket(rho0):
            raise TypeError('Second argument must be a ket if first' +
                            'is a Hamiltonian.')

        # check if state is below error threshold
        if abs(rho0.full()).sum() < 1e-5 + 1e-20:
            # enforce zero operator
            dims = rho0.dims
            return eseries(Qobj(sp.csr_matrix((dims[0][0], dims[1][0]),
                                              dtype=complex)))

        w, v = L.eigenstates()
        v = np.hstack([ket.full() for ket in v])
        # w[i]   = eigenvalue i
        # v[:,i] = eigenvector i

        rlen = np.prod(rho0.shape)
        r0 = rho0.full()
        v0 = la.solve(v, r0)
        vv = v * sp.spdiags(v0.T, 0, rlen, rlen)

        out = None
        for i in range(rlen):
            qo = Qobj(np.matrix(vv[:, i]).T, dims=rho0.dims, shape=rho0.shape)
            if out:
                out += eseries(qo, -1.0j * w[i])
            else:
                out = eseries(qo, -1.0j * w[i])

    else:
        raise TypeError('First argument must be a Hamiltonian or Liouvillian.')

    return estidy(out)

ext: py

max_stars_repo_name: leofang/qutip

max_issues_repo_name: leofang/qutip

max_forks_repo_name: leofang/qutip

reason: import numpy,import scipy


text:
import os
import argparse
import numpy as np
from algos import construct_classifier, classifier_types
from utils.data_utils import get_dataset, dataset_names
from utils.misc import increment_path
from utils.tf_utils import launch_tensorboard
from utils.vis_utils import plot_embeddings

masterdir = "/tmp/fairml-farm/"
base_datadir = masterdir + "data/"
base_logdir = masterdir + "logs/"
parser = argparse.ArgumentParser(description="Evaluate an individual fairness "
                                 "algorithm.\nNOTE: classifier-specific "
                                 "arguments should be specified in area "
                                 "in the script itself.")
parser.add_argument("--experiment-name", default="default",
                    help="Name for the experiment base directory, "
                    "used as the extension relative to {}".format(base_logdir))
parser.add_argument("--load-dir",
                    help="Path to a previous experiment subdirectory, used to "
                    "load model weights, relative to {}.".format(base_logdir))
parser.add_argument("--train", action="store_true",
                    help="train the classifier")
parser.add_argument("-epochs", type=int, default=20)
parser.add_argument("--visualize", action="store_true", help="visualize "
                    "learned latent space")
parser.add_argument("-classifier", choices=[c.name for c in classifier_types],
                    default="simplenn",
                    help="Name of the type of fairness algorithm to use.")
parser.add_argument("-dataset", choices=dataset_names,
                    default="adult",
                    help="Name of dataset to train on.")
args = parser.parse_args()

loaddir = None
if args.load_dir is not None:
    loaddir = os.path.join(base_logdir, args.load_dir)
logdir = increment_path(os.path.join(base_logdir, args.experiment_name, "run"))
os.makedirs(logdir, exist_ok=True)
print("Logging data to {}".format(logdir))
print("Loading {} dataset...".format(args.dataset))
train_dataset, validation_dataset = get_dataset(args.dataset,
                                                base_datadir=base_datadir)
print("Launching Tensorboard.\nTo visualize, navigate to "
      "http://0.0.0.0:6006/\nTo close Tensorboard,"
      " press ctrl+C")
tensorboard_process = launch_tensorboard(logdir)
# ===== SPECIFY HYPERPARAMETERS (INCLUDING CLASSIFIER-TYPE) =====
inputsize = train_dataset["data"].shape[1]
layersizes = [100]
classifier_type = "paritynn"
hparams = {
    "classifier_type": classifier_type,
    "layersizes": layersizes,
    "inputsize": inputsize,
}
# ===============================================================
print("Initializing classifier...")
classifier = construct_classifier(hparams, loaddir=loaddir)
if args.train:
    print("Training network...")
    classifier.train(train_dataset, logdir, epochs=args.epochs,
                     validation_dataset=validation_dataset)
savepath = classifier.save_model(logdir)
if args.visualize: # Plot out the learned embedding space
    n = validation_dataset["label"].shape[0]
    # get an equal number of male and female points
    n_males = sum(validation_dataset["label"])
    limiting_gender = n_males > n - n_males # 1 if men, 0 if women
    n_limiting_gender = sum(validation_dataset["label"] == limiting_gender)
    max_points_per_gender = 500
    n_per_gender = min(max_points_per_gender, n_limiting_gender)
    inds = np.concatenate([
        np.where(validation_dataset["label"] == limiting_gender)[0][:n_per_gender],
        np.where(validation_dataset["label"] != limiting_gender)[0][:n_per_gender]],
        axis=0)
    vis_dataset = {k:v[inds, ...] for k, v in validation_dataset.items()}
    val_embeddings = classifier.compute_embedding(vis_dataset["data"])
    plot_embeddings(val_embeddings,
                    vis_dataset["label"],
                    vis_dataset["protected"],
                    plot3d=True,
                    subsample=False,
                    label_names=["income<=50k", "income>50k"],
                    protected_names=["female", "male"])
tensorboard_process.join()

max_stars_repo_name: yoshavit/fairml-farm

max_issues_repo_name: yo-shavit/fairml-farm

max_forks_repo_name: yo-shavit/fairml-farm

reason: import numpy


text:
""" Training script for steps_with_decay policy"""

import argparse
import os
import sys
import pickle
import resource
import traceback
import logging
from collections import defaultdict

import numpy as np
import yaml
import torch
from torch.autograd import Variable
import torch.nn as nn
import cv2
cv2.setNumThreads(0)  # pytorch issue 1355: possible deadlock in dataloader

import _init_paths  # pylint: disable=unused-import
import nn as mynn
import utils.net as net_utils
import utils.misc as misc_utils
from core.config import cfg, cfg_from_file, assert_and_infer_cfg
from datasets.roidb import combined_roidb_for_training
from roi_data.loader import RoiDataLoader, MinibatchSampler, BatchSampler, collate_minibatch
from modeling.model_builder import Generalized_RCNN
from utils.detectron_weight_helper import load_detectron_weight
from utils.logging import setup_logging
from utils.timer import Timer
from utils.training_stats import TrainingStats

# Set up logging and load config options
logger = setup_logging(__name__)
logging.getLogger('roi_data.loader').setLevel(logging.INFO)

# RuntimeError: received 0 items of ancdata. Issue: pytorch/pytorch#973
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (4096, rlimit[1]))

def parseArgs():
    """Parse input arguments"""
    parser = argparse.ArgumentParser(description='Train a X-RCNN network')
    parser.add_argument('--dataset', dest='dataset', required=True, default='fsod', help='Dataset')
    parser.add_argument('--save_dir', dest='save_dir', required=True, help='where to save training results')
    parser.add_argument('--cfg', dest='cfg_file', required=True, help='Config file for training')
    parser.add_argument('--bs', dest='batch_size', help='batch size, must equal to num of GPU', type=int)
    parser.add_argument('--nw', dest='num_workers', help='number of workers', type=int)
    parser.add_argument('--iter_size', help='update interval', default=1, type=int)
    parser.add_argument('--load_ckpt', help='load checkpoint')
    parser.add_argument('--load_detectron', help='load detectron weight')
    parser.add_argument('--use_tfboard', help='Use tensorboard', action='store_true')
    return parser.parse_args()

def save_ckpt(args, step, train_size, model, optimizer):
    """Save checkpoint"""
    output_dir = './Outputs/' + args.save_dir
    ckpt_dir = os.path.join(output_dir, 'ckpt')
    if not os.path.exists(ckpt_dir):
        os.makedirs(ckpt_dir)
    save_name = os.path.join(ckpt_dir, 'model_step{}.pth'.format(step))
    if isinstance(model, mynn.DataParallel):
        model = model.module
    model_state_dict = model.state_dict()
    torch.save({
        'step': step,
        'train_size': train_size,
        'batch_size': args.batch_size,
        'model': model.state_dict(),
        'optimizer': optimizer.state_dict()}, save_name)
    logger.info('save model: %s', save_name)

def main():
    """Main function"""
    args = parseArgs()
    print('Run with argument:')
    print(args)

    # must use cuda to run code
    if not torch.cuda.is_available():
        sys.exit("CUDA not available!")
    cfg.CUDA = True

    if args.dataset == "fsod":
        cfg.TRAIN.DATASETS = ('fsod_train',)
        cfg.MODEL.NUM_CLASSES = 801

    cfg_from_file(args.cfg_file)

    ### Adaptively adjust some configs ###
    original_ims_per_batch = cfg.TRAIN.IMS_PER_BATCH
    original_num_gpus = cfg.NUM_GPUS
    original_batch_size = original_num_gpus * original_ims_per_batch
    if args.batch_size is None:
        args.batch_size = original_batch_size
    cfg.NUM_GPUS = torch.cuda.device_count()
    assert (args.batch_size % cfg.NUM_GPUS) == 0, 'batch_size: %d, NUM_GPUS: %d' % (args.batch_size, cfg.NUM_GPUS)
    cfg.TRAIN.IMS_PER_BATCH = args.batch_size // cfg.NUM_GPUS
    effective_batch_size = args.iter_size * args.batch_size
    print('effective_batch_size = batch_size * iter_size = %d * %d' % (args.batch_size, args.iter_size))

    print('Adaptive config changes:')
    print('    effective_batch_size: %d --> %d' % (original_batch_size, effective_batch_size))
    print('    NUM_GPUS:             %d --> %d' % (original_num_gpus, cfg.NUM_GPUS))
    print('    IMS_PER_BATCH:        %d --> %d' % (original_ims_per_batch, cfg.TRAIN.IMS_PER_BATCH))

    ### Adjust learning based on batch size change linearly
    # For iter_size > 1, gradients are `accumulated`, so lr is scaled based
    # on batch_size instead of effective_batch_size
    old_base_lr = cfg.SOLVER.BASE_LR
    cfg.SOLVER.BASE_LR *= args.batch_size / original_batch_size
    print('Adjust BASE_LR linearly according to batch_size change:\n'
          '    BASE_LR: {} --> {}'.format(old_base_lr, cfg.SOLVER.BASE_LR))

    ### Adjust solver steps
    step_scale = original_batch_size / effective_batch_size
    old_solver_steps = cfg.SOLVER.STEPS
    old_max_iter = cfg.SOLVER.MAX_ITER
    cfg.SOLVER.STEPS = list(map(lambda x: int(x * step_scale + 0.5), cfg.SOLVER.STEPS))
    cfg.SOLVER.MAX_ITER = int(cfg.SOLVER.MAX_ITER * step_scale + 0.5)
    print('Adjust SOLVER.STEPS and SOLVER.MAX_ITER linearly based on effective_batch_size change:\n'
          '    SOLVER.STEPS: {} --> {}\n'
          '    SOLVER.MAX_ITER: {} --> {}'.format(old_solver_steps, cfg.SOLVER.STEPS,
                                                  old_max_iter, cfg.SOLVER.MAX_ITER))

    if args.num_workers is not None:
        cfg.DATA_LOADER.NUM_THREADS = args.num_workers
    print('Number of data loading threads: %d' % cfg.DATA_LOADER.NUM_THREADS)

    assert_and_infer_cfg()

    timers = defaultdict(Timer)

    ### Dataset ###
    timers['roidb'].tic()
    roidb, ratio_list, ratio_index, cls_list, id_list = combined_roidb_for_training(
        cfg.TRAIN.DATASETS, cfg.TRAIN.PROPOSAL_FILES)
    info_list = np.concatenate([ratio_index[:, np.newaxis], cls_list[:, np.newaxis]], axis=1)

    info_list = np.concatenate([info_list, id_list[:, np.newaxis]], axis=1)
    timers['roidb'].toc()
    roidb_size = len(roidb)
    logger.info('{:d} roidb entries'.format(roidb_size))
    logger.info('Takes %.2f sec(s) to construct roidb', timers['roidb'].average_time)

    # Effective training sample size for one epoch
    train_size = roidb_size // args.batch_size * args.batch_size

    batchSampler = BatchSampler(
        sampler=MinibatchSampler(ratio_list, ratio_index),
        batch_size=args.batch_size,
        drop_last=True
    )
    dataset = RoiDataLoader(
        roidb,
        cfg.MODEL.NUM_CLASSES,
        info_list,
        ratio_list,
        training=True)
    dataloader = torch.utils.data.DataLoader(
        dataset,
        batch_sampler=batchSampler,
        num_workers=cfg.DATA_LOADER.NUM_THREADS,
        collate_fn=collate_minibatch)
    dataiterator = iter(dataloader)

    ### Model ###
    maskRCNN = Generalized_RCNN()
    maskRCNN.cuda()

    ### Optimizer ###
    gn_param_nameset = set()
    for name, module in maskRCNN.named_modules():  # Returns an iterator over all modules in the network
        if isinstance(module, nn.GroupNorm):
            gn_param_nameset.add(name+'.weight')
            gn_param_nameset.add(name+'.bias')
    gn_params = []
    gn_param_names = []
    bias_params = []
    bias_param_names = []
    nonbias_params = []
    nonbias_param_names = []
    nograd_param_names = []
    box_outs_params = []
    box_outs_param_names = []
    for key, value in maskRCNN.named_parameters():
        #print(key, value.requires_grad)
        if value.requires_grad:
            if 'bias' in key and 'Box_Outs' not in key:
                bias_params.append(value)
                bias_param_names.append(key)
            elif key in gn_param_nameset:
                gn_params.append(value)
                gn_param_names.append(key)
            elif 'Box_Outs' in key:
                box_outs_params.append(value)
                box_outs_param_names.append(key)
            else:
                nonbias_params.append(value)
                nonbias_param_names.append(key)
        else:
            nograd_param_names.append(key)
    assert (gn_param_nameset - set(nograd_param_names) - set(bias_param_names)) == set(gn_param_names)

    # Learning rate of 0 is a dummy value to be set properly at the start of training
    params = [
        {'params': nonbias_params,
         'lr': 0,
         'weight_decay': cfg.SOLVER.WEIGHT_DECAY},
        {'params': bias_params,
         'lr': 0 * (cfg.SOLVER.BIAS_DOUBLE_LR + 1),
         'weight_decay': cfg.SOLVER.WEIGHT_DECAY if cfg.SOLVER.BIAS_WEIGHT_DECAY else 0},
        {'params': gn_params,
         'lr': 0,
         'weight_decay': cfg.SOLVER.WEIGHT_DECAY_GN},
        {'params': box_outs_params,
         'lr': 0,
         'weight_decay': cfg.SOLVER.WEIGHT_DECAY},
    ]

    if cfg.SOLVER.TYPE == "SGD":
        optimizer = torch.optim.SGD(params, momentum=cfg.SOLVER.MOMENTUM)

    ### Load checkpoint
    if args.load_ckpt:
        logging.info("loading checkpoint %s", args.load_ckpt)
        checkpoint = torch.load(args.load_ckpt, map_location=lambda storage, loc: storage)
        net_utils.load_ckpt(maskRCNN, checkpoint['model'])
        del checkpoint
        torch.cuda.empty_cache()

    if args.load_detectron:  # TODO resume for detectron weights (load sgd momentum values)
        logging.info("loading Detectron weights %s", args.load_detectron)
        load_detectron_weight(maskRCNN, args.load_detectron)

    lr = optimizer.param_groups[0]['lr']  # lr of non-bias parameters, for commmand line outputs.

    maskRCNN = mynn.DataParallel(maskRCNN, cpu_keywords=['im_info', 'roidb'], minibatch=True)

    ### Training Setups ###
    args.run_name = misc_utils.get_run_name() + '_step'
    output_dir = './Outputs/' + args.save_dir
    if os.path.exists(output_dir) == False:
        os.makedirs(output_dir)

    args.cfg_filename = os.path.basename(args.cfg_file)
    blob = {'cfg': yaml.dump(cfg), 'args': args}
    with open(os.path.join(output_dir, 'config_and_args.pkl'), 'wb') as f:
        pickle.dump(blob, f, pickle.HIGHEST_PROTOCOL)

    if args.use_tfboard:
        from tensorboardX import SummaryWriter
        # Set the Tensorboard logger
        tblogger = SummaryWriter(output_dir)

    ### Training Loop ###
    maskRCNN.train()
    CHECKPOINT_PERIOD = 2000  # int(cfg.TRAIN.SNAPSHOT_ITERS / cfg.NUM_GPUS)  # Stephane on 4/28 11:00PM
    # Set index for decay steps
    decay_steps_ind = None
    for i in range(1, len(cfg.SOLVER.STEPS)):
        if cfg.SOLVER.STEPS[i] >= 0:
            decay_steps_ind = i
            break
    if decay_steps_ind is None:
        decay_steps_ind = len(cfg.SOLVER.STEPS)

    training_stats = TrainingStats(args, tensorboard_logger=tblogger if args.use_tfboard else None)
    try:
        logger.info('Training starts !')
        step = 0
        for step in range(0, cfg.SOLVER.MAX_ITER):

            # Warm up
            if step < cfg.SOLVER.WARM_UP_ITERS:
                method = cfg.SOLVER.WARM_UP_METHOD
                if method == 'constant':
                    warmup_factor = cfg.SOLVER.WARM_UP_FACTOR
                elif method == 'linear':
                    alpha = step / cfg.SOLVER.WARM_UP_ITERS
                    warmup_factor = cfg.SOLVER.WARM_UP_FACTOR * (1 - alpha) + alpha
                else:
                    raise KeyError('Unknown SOLVER.WARM_UP_METHOD: {}'.format(method))
                lr_new = cfg.SOLVER.BASE_LR * warmup_factor
                net_utils.update_learning_rate(optimizer, lr, lr_new)
                lr = optimizer.param_groups[0]['lr']
                assert lr == lr_new
            elif step == cfg.SOLVER.WARM_UP_ITERS:
                net_utils.update_learning_rate(optimizer, lr, cfg.SOLVER.BASE_LR)
                lr = optimizer.param_groups[0]['lr']
                assert lr == cfg.SOLVER.BASE_LR

            # Learning rate decay
            if decay_steps_ind < len(cfg.SOLVER.STEPS) and \
                    step == cfg.SOLVER.STEPS[decay_steps_ind]:
                logger.info('Decay the learning on step %d', step)
                lr_new = lr * cfg.SOLVER.GAMMA
                net_utils.update_learning_rate(optimizer, lr, lr_new)
                lr = optimizer.param_groups[0]['lr']
                assert lr == lr_new
                decay_steps_ind += 1

            training_stats.IterTic()
            optimizer.zero_grad()
            for inner_iter in range(args.iter_size):
                try:
                    input_data = next(dataiterator)
                except StopIteration:
                    dataiterator = iter(dataloader)
                    input_data = next(dataiterator)

                for key in input_data:
                    if key != 'roidb': # roidb is a list of ndarrays with inconsistent length
                        input_data[key] = list(map(Variable, input_data[key]))

                net_outputs = maskRCNN(**input_data)
                training_stats.UpdateIterStats(net_outputs, inner_iter)
                loss = net_outputs['total_loss']
                loss.backward()
            optimizer.step()
            training_stats.IterToc()

            training_stats.LogIterStats(step, lr)

            if (step+1) % CHECKPOINT_PERIOD == 0:
                save_ckpt(args, step, train_size, maskRCNN, optimizer)

        # ---- Training ends ----
        # Save last checkpoint
        save_ckpt(args, step, train_size, maskRCNN, optimizer)

    except (RuntimeError, KeyboardInterrupt):
        del dataiterator
        logger.info('Save ckpt on exception ...')
        save_ckpt(args, step, train_size, maskRCNN, optimizer)
        logger.info('Save ckpt done.')
        stack_trace = traceback.format_exc()
        print(stack_trace)

    finally:
        if args.use_tfboard:
            tblogger.close()

if __name__ == '__main__':
    main()

max_stars_repo_name: DanyuHuang/DL-Project

max_issues_repo_name: DanyuHuang/DL-Project

max_forks_repo_name: DanyuHuang/DL-Project


text:
import numpy as np
import pytest

from pymoo.algorithms.moo.age import AGEMOEA
from pymoo.algorithms.moo.moead import MOEAD, ParallelMOEAD
from pymoo.algorithms.moo.nsga2 import NSGA2
from pymoo.algorithms.moo.rvea import RVEA
from pymoo.algorithms.soo.nonconvex.cmaes import CMAES
from pymoo.algorithms.soo.nonconvex.de import DE
from pymoo.algorithms.soo.nonconvex.direct import DIRECT
from pymoo.algorithms.soo.nonconvex.es import ES
from pymoo.algorithms.soo.nonconvex.nelder_mead import NelderMead
from pymoo.algorithms.soo.nonconvex.pattern_search import PatternSearch
from pymoo.algorithms.soo.nonconvex.pso import PSO
from pymoo.factory import Sphere, Rosenbrock, Himmelblau, ZDT1, get_reference_directions
from pymoo.indicators.igd import IGD
from pymoo.optimize import minimize

SINGLE_OBJECTIVE_PROBLEMS = [Sphere(n_var=10), Himmelblau(), Rosenbrock()]

SINGLE_OBJECTIVE_ALGORITHMS = [PatternSearch(), CMAES(), NelderMead(), DIRECT(), PSO(), DE(), ES()]

@pytest.mark.parametrize('problem', SINGLE_OBJECTIVE_PROBLEMS)
@pytest.mark.parametrize('algorithm', SINGLE_OBJECTIVE_ALGORITHMS)
def test_singe_obj(problem, algorithm):
    res = minimize(problem, algorithm, seed=1, verbose=True)
    fmin = problem.pareto_front().flatten()[0]
    np.testing.assert_almost_equal(fmin, res.F[0], decimal=3)

MULTI_OBJECTIVE_PROBLEMS = [ZDT1()]

ref_dirs = get_reference_directions("das-dennis", 2, n_partitions=99)
MULTI_OBJECTIVE_ALGORITHMS = [NSGA2(),
                              RVEA(ref_dirs),
                              MOEAD(ref_dirs),
                              ParallelMOEAD(ref_dirs),
                              AGEMOEA()]

@pytest.mark.parametrize('problem', MULTI_OBJECTIVE_PROBLEMS)
@pytest.mark.parametrize('algorithm', MULTI_OBJECTIVE_ALGORITHMS)
def test_multi_obj(problem, algorithm):
    res = minimize(problem, algorithm, ('n_gen', 300), seed=1, verbose=True)
    pf = problem.pareto_front()
    assert IGD(pf).do(res.F) < 0.05

max_stars_repo_name: jarreguit/pymoo

max_issues_repo_name: jarreguit/pymoo

max_forks_repo_name: jarreguit/pymoo


text:
# 1. Import all libraries
import numpy as np
import matplotlib
import matplotlib.pyplot as ppt
from matplotlib.colors import ListedColormap
from sklearn.datasets import load_breast_cancer
from sklearn import metrics
from sklearn import model_selection
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import PCA
from matplotlib.colors import ListedColormap

# 2. Load data
breastcancer = load_breast_cancer()
X, headers, y = breastcancer.data, breastcancer.feature_names, breastcancer.target

# 3. Standardize data
scaler = StandardScaler()
X = scaler.fit_transform(X)

# 3. Split data into training data and test data
X_train, X_test, Y_train, Y_test = \
    train_test_split(X, y, test_size=.4, random_state=42)

# 3a. Make data two dimensional
pca_model = PCA(n_components=2)
pca_model.fit(X_train)
X_train = pca_model.transform(X_train)
X_test = pca_model.transform(X_test)
X_train[:5]

# 5. Perform all classifications using various models
# 5a. Define all models
cla_names = ["Logistic Regression", "SVM", "Stochastic Gradient Descent", "Nearest Neighbors", "Naive Bayes", "Decision Trees", "Bagging Meta-Estimator", "Random Forest"]
classifiers = [
    LogisticRegression(random_state=0),
    SVC(kernel='rbf', random_state=0),
    SGDClassifier(loss="hinge", penalty='l2', max_iter=5),
    KNeighborsClassifier(n_neighbors=7),
    GaussianNB(),
    DecisionTreeClassifier(random_state=0),
    BaggingClassifier(random_state=0),
    RandomForestClassifier(random_state=0)]

# 5b. Apply all models in for loop
name_list = []
score = []
for name, clf in zip(cla_names, classifiers):
    clf.fit(X_train, Y_train)
    name_list.append(name)
    score.append(clf.score(X_test, Y_test))
# 5c Plot performance
fig, ax = ppt.subplots(figsize=(20, 10))
ind = np.arange(8)
width = 0.9
ppt.bar(ind, score, width, 0)
ppt.xticks(ind, name_list)
ppt.title("Classifier Performance")
i=0
for i in range(len(score)):
    ppt.annotate(score[i], xy=(i-0.4,0.99))
    i = i+1
ppt.savefig ("Classifiers_Performance.png", format="PNG")

# 6. Plot all classification plots
fig = ppt.figure(figsize=(27,27))

# 6a. Plot three attributes that were my GOAT
matplotlib.style.use('ggplot')
h = 0.02

for i in [1,2,3,4,5,6,7,8,9]:
	x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
	y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
	xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
	                     np.arange(y_min, y_max, h))
	cm = ppt.cm.RdBu
	cm_bright = ListedColormap(['#FF0000', '#0000FF'])
	ax = ppt.subplot(3,3,i)
	ax.set_title("Input Data")
	plot1=ax.scatter(X_train[:,0], X_train[:,1], c=Y_train, cmap=cm_bright, edgecolors='k')
	plot2=ax.scatter(X_test[:,0], X_test[:,1], c=Y_test, cmap=cm_bright, alpha=0.6, edgecolors='k')
	ax.legend((plot1, plot2), ('Training Data', 'Test Data'))
	ax.set_xlim(xx.min(), xx.max())
	ax.set_ylim(yy.min(), yy.max())
	ax.set_xticks(())
	ax.set_yticks(())

# 6b. Plot decision boundary function
def plot_decision_boundary(model,data,position):
    ax = ppt.subplot(3,3,position)
    model.fit(X_train, Y_train)
    score = model.score(X_test, Y_test)

    if hasattr(model, "decision_function"):
    	Z = model.decision_function(np.c_[xx.ravel(), yy.ravel()])
    else:
        Z = model.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:,1]

    Z = Z.reshape(xx.shape)
    ax.contourf(xx, yy, Z, cmap=cm, alpha=.8, norm=None)

    ax.scatter(X_train[:,0], X_train[:,1], c=Y_train, cmap=cm_bright, edgecolors='k')
    ax.scatter(X_test[:,0], X_test[:,1], c=Y_test, cmap=cm_bright, alpha=0.3, edgecolors='k')

    ax.text(xx.max() - .3, yy.min() + .3, ('%.2f' % score).lstrip('0'),
            size=15, horizontalalignment='right')
    ax.set_xlim(xx.min(), xx.max())
    ax.set_ylim(yy.min(), yy.max())
    ax.set_xticks(())
    ax.set_yticks(())
    ax.set_title(cla_names[i])

for i in (range(len(classifiers))):
	plot_decision_boundary(classifiers[i], X, i+2)
ppt.tight_layout()
ppt.savefig ("Classifiers_Plots.png", format="PNG")


text:
import torch
import torch.nn as nn
import torch.utils.data
import torch.utils.data.distributed
import torch.nn.functional as F
import numpy as np
from SPPE.src.utils.img import flip_v, shuffleLR
from SPPE.src.utils.eval import getPrediction
from SPPE.src.models.hgPRM import createModel_Inference

import visdom
import time

import torch._utils
try:
    torch._utils._rebuild_tensor_v2
except AttributeError:
    def _rebuild_tensor_v2(storage, storage_offset, size, stride, requires_grad, backward_hooks):
        tensor = torch._utils._rebuild_tensor(storage, storage_offset, size, stride)
        tensor.requires_grad = requires_grad
        tensor._backward_hooks = backward_hooks
        return tensor
    torch._utils._rebuild_tensor_v2 = _rebuild_tensor_v2

# batch_size = 128 * 8

def gaussian(size):
    '''
    Generate a 2D gaussian array
    '''
    sigma = 1
    x = np.arange(0, size, 1, float)
    y = x[:, np.newaxis]
    x0 = y0 = size // 2
    sigma = size / 4.0
    # The gaussian is not normalized, we want the center value to equal 1
    g = np.exp(- ((x - x0) ** 2 + (y - y0) ** 2) / (2 * sigma ** 2))
    g = g[np.newaxis, :]
    return g

class InferenNet(nn.Module):
    def __init__(self, kernel_size, dataset):
        super(InferenNet, self).__init__()

        model = createModel_Inference().cuda()
        print('Loading pose model from {}'.format('./models/sppe/pyra_4.pth'))
        model.load_state_dict(torch.load('./models/sppe/pyra_4.pth'))
        model.eval()
        self.pyranet = model
        self.gaussian = nn.Conv2d(17, 17, kernel_size=kernel_size,
                                  stride=1, padding=2, groups=17, bias=False)

        g = torch.from_numpy(gaussian(kernel_size)).clone()
        g = torch.unsqueeze(g, 1)
        g = g.repeat(17, 1, 1, 1)
        assert g.shape == self.gaussian.weight.data.shape
        self.gaussian.weight.data = g.float()

        self.dataset = dataset

    def forward(self, x):
        out = self.pyranet(x)
        out = out.narrow(1, 0, 17)

        flip_out = self.pyranet(flip_v(x))
        flip_out = flip_out.narrow(1, 0, 17)

        flip_out = flip_v(shuffleLR(
            flip_out, self.dataset))

        out = (flip_out + out) / 2
        out = self.gaussian(F.relu(out, inplace=True))

        return out

class InferenNet_fast(nn.Module):
    def __init__(self, kernel_size, dataset):
        super(InferenNet_fast, self).__init__()

        model = createModel_Inference().cuda()
        print('Loading pose model from {}'.format('./models/sppe/pyra_4.pth'))
        model.load_state_dict(torch.load('./models/sppe/pyra_4.pth'))
        model.eval()
        self.pyranet = model
        self.gaussian = nn.Conv2d(17, 17, kernel_size=kernel_size,
                                  stride=1, padding=2, groups=17, bias=False)

        g = torch.from_numpy(gaussian(kernel_size)).clone()
        g = torch.unsqueeze(g, 1)
        g = g.repeat(17, 1, 1, 1)
        assert g.shape == self.gaussian.weight.data.shape
        self.gaussian.weight.data = g.float()

        self.dataset = dataset

    def forward(self, x):
        out = self.pyranet(x)
        out = out.narrow(1, 0, 17)

        out = self.gaussian(F.relu(out, inplace=True))

        return out


text:
"""
This is where the user interface is made using the streamlit package.
"""

import joblib
import numpy as np
import pandas as pd
import streamlit as st
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder

st.title('CdSe Quantum Dots Synthesis Project  - Cossairt Laboratory')

st.header('Predicting Properties of CdSe Quantum Dots')

st.markdown('****')
st.write('Answer the questions below about your CdSe quandum dots synthesis and we will predict the diameter, absorbance max, and emission wavelength of your dots.')
st.markdown('****')

#Creating questions with multiple choice answer
RADIO_QUESTIONS_LIST = ['1. What is your cadmium source?',
                        '2. What is your carboxylic acid source?',
                        '3. What is your amine source?',
                        '4. What is your phosphine source?',
                        '5. What is your first solvent?',
                        '6. What is your second solvent?'
                        ]
#Creating multiple choice answers for each question above
RADIO_SELECTIONS = [['cadmium stearate', 'cadmium oxide', 'dimethylcadmium',
                            'cadmium acetate', 'cadmium acetate dihydrate'],
                    ['None', 'myrstic acid', 'oleic acid', 'stearic acid',
                                'benzoic acid', 'dodecylphosphonic acid',
                                'ethylphosphonic acid', 'lauric acid'],
                    ['None', '2-6-dimethylpyridine', 'aniline', 'benzylamine',
                                'dioctylamine/hexadecylamine', 'dodecylamine',
                            'heptylamine', 'hexadecylamine', 'octadecylamine',
                     'octylamine', 'oleylamine', 'pyridine', 'trioctylamine'],
                    ['None', 'diphenylphosphine', 'tributylphosphine',
                        'trioctylphosphine', 'triphenylphosphine'],
                    ['None', 'liquid parafin', 'octadecene',
                        'phenyl ether', 'trioctylphosphine oxide'],
                    ['None', 'phosphinic acid', 'trioctylphosphine oxide']
                    ]
#Creating questions with slider
SLIDER_QUESTIONS_LIST = ['7. How much Cadmium do you plan to use? (mmol)',
                         '8. Selenium power is used; how much Selenium do you plan to use? (mmol)',
                         '9. How much carboxylic acid  do you plan to use? (mmol)',
                         '10. How much amine do you plan to use? (mmol)',
                         '11. How much phosphine do you plan to use? (mmol)',
                         '12. How much first solvent do you plan to use? (g)',
                         '13. How much second solvent do you plan to use? (g)',
                         '14. What is the growth temperature? (Degree Celsius)',
                         '15. How long do you plan to grow the quantum dots (minute)?'
                         ]
#Creating sliders for each question above
SLIDER_SELECTIONS = [[0.1, 14.0, 0.15, 0.001],
                     [0.001, 1.0, 0.01, 0.0001],
                     [0.0, 60.0, 10.0, 0.001],
                     [0.0, 40.0, 1.0, 0.001],
                     [0.0, 60.0, 1.0, 0.001],
                     [0.0, 60.0, 10.0, 0.1],
                     [0.0, 60.0, 10.0, 0.1],
                     [45.0, 350.0, 200.0, 1.0],
                     [0.5, 360.0, 50.0, 0.5],]

#Initiate lists for answers
radio_answers = []
slider_answers = []

def get_radio_input(question, selection):
    """
    This function creates questions and
    multiple choice answers in the user interface.
    """
    answer = st.radio(question, selection)
    return answer

def get_slider_input(question, mmin_val, max_val, default_val, interval):
    """
    This function creates questions and
    slider answers in the user interface.
    """
    answer = st.slider(question, mmin_val, max_val, default_val, interval)
    return answer

#List of answers for multiple choice questions
for i in range(len(RADIO_QUESTIONS_LIST)):
    radio_answers.append(get_radio_input(
        RADIO_QUESTIONS_LIST[i], RADIO_SELECTIONS[i]))

#List of answers for slider questions
for i in range(len(SLIDER_QUESTIONS_LIST)):
    slider_answers.append(get_slider_input(
        SLIDER_QUESTIONS_LIST[i], SLIDER_SELECTIONS[i][0],
        SLIDER_SELECTIONS[i][1], SLIDER_SELECTIONS[i][2],
        SLIDER_SELECTIONS[i][3]))

#Rearange users' choice into a list to input to the ML model
user_input = [slider_answers[7], radio_answers[0], slider_answers[0], slider_answers[1],
              radio_answers[1], slider_answers[2], radio_answers[2], slider_answers[3],
              radio_answers[3], slider_answers[4], radio_answers[4], slider_answers[5],
              radio_answers[5], slider_answers[6], slider_answers[8]
              ]

#Naming each choice in the user input
user_df = pd.DataFrame(np.array(user_input).reshape(1, -1), columns=['Growth Temp (Celsius)',
                                            'Metal_source', 'Metal_mmol (mmol)',
                                            'Chalcogen_mmol (mmol)', 'Carboxylic_Acid',
                                            'CA_mmol (mmol)', 'Amines', 'Amines_mmol (mmol)',
                                            'Phosphines', 'Phosphines_mmol (mmol)',
                                            'Solvent I', 'S_I_amount (g)',
                                            'Solvent II', 'S_II_amount (g)', 'Time_min (min)'
                                            ])
#Print user inputs
st.write(user_df)

#Scaling and encoding user input using the raw dataset
df = pd.read_csv('CdSe/CdSe_BetterthanRaw.csv')

#Separate out initial DataFrame into the input features and output features
df_input = df.drop(columns =['Injection Temp (Celsius)', 'Metal_amount (g)',
'Metal_concentration (mmol/g)', 'Chalcogen_amount (g)', 'Chalcogen_concentration (mmol/g)',
'Metal/Se_ratio', 'CA_amount (g)', 'Cd/CA_ratio', 'Amines_amount (g)', 'Phosphines_amount (g)',
'Chalcogen/Ph_ratio','Total_amount (g)','Chalcogen_source','Diameter_nm', 'Absorbance max (nm)',
'PL max (nm)', 'Diameter from', 'Citation'],
inplace = False, axis = 1)

#Checks the column names, and ensures that they do not have any leading or trailing spaces
df_input.columns = df_input.columns.str.strip()

#Converts the values in Growth Temperature Column into float types
df_input['Growth Temp (Celsius)'] = df_input['Growth Temp (Celsius)'].astype(float)

#Initializes 2 lists to contain all of the numerical and categorical input columns
input_num_cols = [col for col in df_input.columns if df[col].dtypes !='O']
input_cat_cols = [col for col in df_input.columns if df[col].dtypes =='O']

ct = ColumnTransformer([
    ('step1', StandardScaler(), input_num_cols),
    ('step2', OneHotEncoder(sparse=False, handle_unknown='ignore'), input_cat_cols)
], remainder = 'passthrough')

ct.fit_transform(df_input)

#Use same column transformer on user input
X = ct.transform(user_df)

#Load and use ExtraTrees ML model to predict outcomes
load_Extra_Trees = joblib.load('CdSe/Extra_Trees.joblib')
predicted = load_Extra_Trees.predict(X)
st.markdown('****')
st.markdown('****')

st.write('Predicted diameter is', round(predicted[0, 0], 3))
st.write('Predicted absorbance max is', round(predicted[0, 1], 3))
st.write('Predicted emission is', round(predicted[0, 2], 3))

st.subheader('Updated 04/04/2021')
st.write('Contact: haon02@uw.edu')


text:
from data_reader import Vocabulary, transform_data, load_vocabularies, FORM, Corpus
from model import build_model, load_model
import numpy as np
import sys

def predict(args):

    # ARGUMENTS
    test_file=args.data
    minibatch=64
    max_sent_len_words=30

    print("fsdfdfsdfs",file=sys.stderr)

    ## VOCABULARY
    vs=load_vocabularies(args.model_name+"-vocab.pickle")

    corpus=Corpus(test_file,test_time=True)

    sentences,x,y=transform_data(corpus,vs,max_sent_len_words,test_time=True)

    print(len(sentences),flush=True,file=sys.stderr)

    model=load_model(args.model_name)
    predictions=model.predict(x,batch_size=64,verbose=1)

    from collections import defaultdict
    # reconstruct file
    last_sentence=None
    for (sent_id,predicate_id),pred in zip(sentences,predictions):
        if last_sentence!=None and sent_id!=last_sentence.id: # new sentence
            corpus.sentences[last_sentence.id]=last_sentence
            last_sentence=corpus.sentences[sent_id]
            last_sentence.arguments=defaultdict(list) # clear arguments to be sure there is nothing
        elif last_sentence==None:
            last_sentence=corpus.sentences[sent_id]
        # add predicted arguments for this predicate
        # predicate_id is the token[ID] of the current predicate
        labels=[vs.idx2label[np.argmax(t)] for t in pred]
        for i,r in enumerate(labels): # i is now token counter, r is predicted role
            if r!="<MASK>" and r!="None":
                last_sentence.arguments[str(i+1)].append((predicate_id,r))

    corpus.save(args.output)

if __name__=="__main__":

    import argparse

    parser = argparse.ArgumentParser(description='')
    g=parser.add_argument_group("Reguired arguments")
    g.add_argument('-d', '--data', type=str, required=True, help='Training file')
    g.add_argument('-m', '--model_name', type=str, required=True, help='Name of the saved model')
    g.add_argument('-o', '--output', type=str, required=True, help='Output file name')

    args = parser.parse_args()

    predict(args)

max_stars_repo_name: jmnybl/finnish-srl

max_issues_repo_name: jmnybl/finnish-srl

max_forks_repo_name: jmnybl/finnish-srl


text:
from dolfin import *
from dolfin_adjoint import *
import windse
import numpy as np

parameters['form_compiler']['quadrature_degree'] = 6
set_log_level(20)

### Create an Instance of the Options ###
options = windse.initialize("params3Dterrain.yaml")

### Generate Domain ###
dom = windse.ImportedDomain()
# dom = windse.BoxDomain()

### Generate Wind Farm ###
farm = windse.GridWindFarm(dom)
# farm = windse.RandomWindFarm(dom)

# farm.Plot(False)

### Warp the mesh and refine ###
# dom.Warp(200,0.75)
region = [[-1000,1500],[-1000,1000],[0,300]]
dom.Refine(1,region=region)
# dom.Save()

print(len(dom.mesh.coordinates()[:]))
print(len(farm.dom.mesh.coordinates()[:]))
print(print(farm.dom.mesh.hmin()))
# exit()

### Function Space ###
fs = windse.LinearFunctionSpace(dom)

print(fs.Q.dim())

### Setup Boundary Conditions ###
bc = windse.PowerInflow(dom,fs)

### Generate the problem ###
problem = windse.StabilizedProblem(dom,farm,fs,bc)

### Solve ###
solver = windse.SteadySolver(problem)
solver.Solve()

### Output Results ###
# solver.Save()

######
# control = windse.CreateAxialControl(farm.ma,farm)
# bounds = windse.CreateAxialBounds(farm.ma,farm)
control = windse.CreateLayoutControl(farm.mx,farm.my,farm)
bounds = windse.CreateLayoutBounds(farm.mx,farm.my,farm)

J=windse.PowerFunctional(problem.tf,solver.u_next)
# rf=ReducedFunctional(J,control)

# def iter_cb(m):
# 	# if MPI.rank(mpi_comm_world()) == 0:
# 	print("m = ")
# 	for mm in m:
# 		print("Constant("+ str(mm)+ "),")

# m_opt=minimize(rf, method="L-BFGS-B", options = {"disp": True}, bounds = bounds, callback = iter_cb)
# # m_opt=minimize(rf, method="SLSQP", options = {"disp": True}, bounds = bounds, callback = iter_cb)

# print([float(mm) for mm in m_opt])
# # farm.mx,farm.my=splitSolution(m_opt,farm.numturbs)
# farm.ma = m_opt
# solver.Solve()

mtest=[]
for i in range(farm.numturbs):
    mtest.append((farm.mx[i]))
    mtest.append((farm.my[i]))

h = [Constant(0.001)]*(2*farm.numturbs)  # the direction of the perturbation
Jhat = ReducedFunctional(J, control)
conv_rate = taylor_test(Jhat, mtest, h)
print(conv_rate)

# ### Output Results ###
solver.Save()

max_stars_repo_name: jefalon/WindSE

max_issues_repo_name: jefalon/WindSE

max_forks_repo_name: jefalon/WindSE


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 25 16:02:58 2022

@author: erri
"""
import os
import numpy as np
import math
from morph_quantities_func_v2 import morph_quantities
import matplotlib.pyplot as plt

# SINGLE RUN NAME
run = 'q07_1'
DoD_name = 'DoD_s1-s0_filt_nozero_rst.txt'
# Step between surveys
DoD_delta = 1
# Base length in terms of columns. If the windows dimensions are channel width
# multiples, the windows_length_base is 12 columns
windows_length_base = 12
window_mode = 1

'''
windows_mode:
    0 = fixed windows (all the channel)
    1 = expanding window
    2 = floating fixed windows (WxW, Wx2W, Wx3W, ...) without overlapping
    3 = floating fixed windows (WxW, Wx2W, Wx3W, ...) with overlapping
'''

plot_mode = 2
'''
plot_mode:
    1 = only summary plot
    2 = all single DoD plot
'''
# Parameters
# Survey pixel dimension
px_x = 50 # [mm]
px_y = 5 # [mm]
W = 0.6 # Width [m]
d50 = 0.001
NaN = -999

# setup working directory and DEM's name
home_dir = os.getcwd()
# Source DoDs folder
DoDs_folder = os.path.join(home_dir, 'DoDs', 'DoD_'+run)

DoDs_name_array = [] # List the file's name of the DoDs with step of delta_step
for f in sorted(os.listdir(DoDs_folder)):
    if f.endswith('_filt_nozero_rst.txt') and f.startswith('DoD_'):
        delta = eval(f[5]) - eval(f[8])
        if delta == DoD_delta:
            DoDs_name_array = np.append(DoDs_name_array, f)
        else:
            pass

# Initialize overall arrays
dep_vol_w_array_all = []
sco_vol_w_array_all = []

# Loop over the DoDs with step of delta_step
for f in DoDs_name_array:
    DoD_name = f
    print(f)

    DoD_path = os.path.join(DoDs_folder,DoD_name)
    DoD_filt_nozero = np.loadtxt(DoD_path, delimiter='\t')

    # DoD length
    DoD_length = DoD_filt_nozero.shape[1]*px_x/1000 # DoD length [m]
    dim_x = DoD_filt_nozero.shape[1]

    # Initialize array

    # Define total volume matrix, Deposition matrix and Scour matrix
    DoD_vol = np.where(np.isnan(DoD_filt_nozero), 0, DoD_filt_nozero) # Total volume matrix
    DoD_vol = np.where(DoD_vol==NaN, 0, DoD_vol)
    dep_DoD = (DoD_vol>0)*DoD_vol # DoD of only deposition data
    sco_DoD = (DoD_vol<0)*DoD_vol # DoD of only scour data

    # Active pixel matrix:
    act_px_matrix = np.where(DoD_vol!=0, 1, 0) # Active pixel matrix, both scour and deposition
    act_px_matrix_dep = np.where(dep_DoD != 0, 1, 0) # Active deposition matrix
    act_px_matrix_sco = np.where(sco_DoD != 0, 1, 0) # Active scour matrix

    # Initialize array for each window dimension

    ###################################################################
    # MOVING WINDOWS ANALYSIS
    ###################################################################
    array = DoD_filt_nozero

    W=windows_length_base
    mean_array_tot = []
    std_array_tot= []

    window_boundary = np.array([0,0])

    x_data_tot=[]
    tot_vol_array=[] # Tot volume
    tot_vol_mean_array=[]
    tot_vol_std_array=[]
    sum_vol_array=[] # Sum of scour and deposition volume
    dep_vol_array=[] # Deposition volume
    sco_vol_array=[] # Scour volume
    morph_act_area_array=[] # Total active area array
    morph_act_area_dep_array=[] # Deposition active area array
    morph_act_area_sco_array=[] # Active active area array
    act_width_mean_array=[] # Total active width mean array
    act_width_mean_dep_array=[] # Deposition active width mean array
    act_width_mean_sco_array=[] # Scour active width mean array

    if window_mode == 1:
        # With overlapping
        for w in range(1, int(math.floor(array.shape[1]/W))+1): # W*w is the dimension of every possible window
        # Initialize arrays that stock data for each window position
            x_data=[]
            tot_vol_w_array = []
            sum_vol_w_array = []
            dep_vol_w_array = []
            sco_vol_w_array =[]
            morph_act_area_w_array = []
            morph_act_area_dep_w_array = []
            morph_act_area_sco_w_array = []
            act_width_mean_w_array = []
            act_width_mean_dep_w_array = []
            act_width_mean_sco_w_array = []
            act_thickness_w_array = []
            act_thickness_dep_w_array = []
            act_thickness_sco_w_array = []
            for i in range(0, array.shape[1]+1):
                if i+w*W <= array.shape[1]:
                    window = array[:, i:W*w+i]
                    boundary = np.array([i,W*w+i])
                    window_boundary = np.vstack((window_boundary, boundary))
                    x_data=np.append(x_data, w)

                    # Calculate morphological quantities
                    tot_vol, sum_vol, dep_vol, sco_vol, morph_act_area, morph_act_area_dep, morph_act_area_sco, act_width_mean, act_width_mean_dep, act_width_mean_sco, act_thickness, act_thickness_dep, act_thickness_sco = morph_quantities(window)

                    # Append single data to array
                    # For each window position the calculated parameters will be appended to _array
                    tot_vol_w_array=np.append(tot_vol_w_array, tot_vol)
                    sum_vol_w_array=np.append(sum_vol_w_array, sum_vol)
                    dep_vol_w_array=np.append(dep_vol_w_array, dep_vol)
                    sco_vol_w_array=np.append(sco_vol_w_array, sco_vol)
                    morph_act_area_w_array=np.append(morph_act_area_w_array, morph_act_area)
                    morph_act_area_dep_w_array=np.append(morph_act_area_dep_w_array, morph_act_area_dep)
                    morph_act_area_sco_w_array=np.append(morph_act_area_sco_w_array, morph_act_area_sco)
                    act_width_mean_w_array=np.append(act_width_mean_w_array, act_width_mean)
                    act_width_mean_dep_w_array=np.append(act_width_mean_dep_w_array, act_width_mean_dep)
                    act_width_mean_sco_w_array=np.append(act_width_mean_sco_w_array, act_width_mean_sco)
                    act_thickness_w_array=np.append(act_thickness_w_array, act_thickness)
                    act_thickness_dep_w_array=np.append(act_thickness_dep_w_array, act_thickness_dep)
                    act_thickness_sco_w_array=np.append(act_thickness_sco_w_array, act_thickness_sco)
            # For each window dimension w*W,
            x_data_tot=np.append(x_data_tot, np.nanmean(x_data)) # Append one value of x_data
            tot_vol_mean_array=np.append(tot_vol_mean_array, np.nanmean(tot_vol_w_array)) # Append the tot_vol_array mean
            tot_vol_std_array=np.append(tot_vol_std_array, np.nanstd(tot_vol_w_array)) # Append the tot_vol_array mean
            # sum_vol_array=
            # dep_vol_array=
            # sco_vol_array=
            # morph_act_area_array=
            # morph_act_area_dep_array=
            # morph_act_area_sco_array=
            # act_width_mean_array=
            # act_width_mean_dep_array=
            # act_width_mean_sco_array=

            # Slice window boundaries array to delete [0,0] when initialized
            window_boundary = window_boundary[1,:]

    if window_mode == 2:
        # Without overlapping
        for w in range(1, int(math.floor(array.shape[1]/W))+1): # W*w is the dimension of every possible window
            mean_array = []
            std_array= []
            x_data=[]
            for i in range(0, array.shape[1]+1):
                if W*w*(i+1) <= array.shape[1]:
                    window = array[:, W*w*i:W*w*(i+1)]
                    boundary = np.array([W*w*i,W*w*(i+1)])
                    window_boundary = np.vstack((window_boundary, boundary))
                    mean = np.nanmean(window)
                    std = np.nanstd(window)
                    mean_array = np.append(mean_array, mean)
                    std_array = np.append(std_array, std)
                    x_data=np.append(x_data, w)
            mean_array_tot = np.append(mean_array_tot, np.nanmean(mean_array))
            std_array_tot= np.append(std_array_tot, np.nanstd(std_array)) #TODO check this
            x_data_tot=np.append(x_data_tot, np.nanmean(x_data))

            # Slice window boundaries array to delete [0,0] when initialized
            window_boundary = window_boundary[1,:]

    if window_mode == 3:
        # Increasing window dimension keeping still the upstream cross section
        mean_array = []
        std_array= []
        x_data=[]
        for i in range(0, array.shape[1]+1):
            if W*(i+1) <= array.shape[1]:
                window = array[:, 0:W*(i+1)]
                boundary = np.array([0,W*(i+1)])
                window_boundary = np.vstack((window_boundary, boundary))
                mean = np.nanmean(window)
                std = np.nanstd(window)
                mean_array = np.append(mean_array, mean)
                std_array = np.append(std_array, std)
                x_data=np.append(x_data, i)
        mean_array_tot = np.append(mean_array_tot, np.nanmean(mean_array))
        std_array_tot= np.append(std_array_tot, np.nanstd(std_array)) #TODO check this
        x_data_tot=np.append(x_data_tot, np.nanmean(x_data))

        # Slice window boundaries array to delete [0,0] when initialized
        window_boundary = window_boundary[1,:]

#     # TODO Go on with this section
#     if windows_mode == 1:

#         # Define x_data for plots
#         x_data = np.linspace(W,dim_x,math.floor(DoD_length/W))*px_x/1e03
#         for n in range(1,math.floor(DoD_length/W)+1):
#             w_cols = n*round(W/(px_x/1000)) # Window analysis length in number of columns
#             w_len = round(n*W,1) # Window analysis lenght im meter [m]

#             # Define total volume matrix, Deposition matrix and Scour matrix
#             DoD_vol_w = DoD_vol[:,0:w_cols] # Total volume matrix
#             dep_DoD_w = dep_DoD[:,0:w_cols] # DoD of only deposition data
#             sco_DoD_w = sco_DoD[:,0:w_cols] # DoD of only scour data

#             # Define active pixel matrix
#             act_px_matrix_w = act_px_matrix[:,0:w_cols] # Active pixel matrix, both scour and deposition
#             act_px_matrix_dep_w = act_px_matrix_dep[:,0:w_cols] # Active deposition matrix
#             act_px_matrix_sco_w = act_px_matrix_sco[:,0:w_cols] # Active scour matrix

#             # Calculate principal quantities:
#             # Volumes
#             tot_vol_w = np.sum(DoD_vol_w)*px_x*px_y/(W*w_len*d50*1e09)# Total volume as V/(L*W*d50) [-] considering negative sign for scour
#             sum_vol_w = np.sum(np.abs(DoD_vol_w))*px_x*px_y/(W*w_len*d50*1e09) # Sum of scour and deposition volume as V/(L*W*d50) [-]
#             dep_vol_w = np.sum(dep_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Deposition volume as V/(L*W*d50) [-]
#             sco_vol_w = np.sum(sco_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Scour volume as V/(L*W*d50) [-]

#             # Areas:
#             morph_act_area_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Active area both in terms of scour and deposition as A/(W*L) [-]
#             morph_act_area_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Active deposition area as A/(W*L) [-]
#             morph_act_area_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Active scour area as A/(W*L) [-]

#             # Widths:
#             act_width_mean_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Total mean active width [%] - Wact/W
#             act_width_mean_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Deposition mean active width [%] - Wact/W
#             act_width_mean_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Scour mean active width [%] - Wact/W

#             # Thicknesses:
#             act_thickness_w = sum_vol_w/morph_act_area_w*(d50*1e03) # Total active thickness (abs(V_sco) + V_dep)/act_area [mm]
#             act_thickness_dep_w = dep_vol_w/morph_act_area_dep_w*(d50*1e03) # Deposition active thickness V_dep/act_area [mm]
#             act_thickness_sco_w = sco_vol_w/act_width_mean_sco_w*(d50*1e03) # Scour active thickness V_sco/act_area [mm]

#             # Append all values in arrays
#             tot_vol_w_array = np.append(tot_vol_w_array, tot_vol_w)
#             sum_vol_w_array = np.append(sum_vol_w_array, sum_vol_w)
#             dep_vol_w_array = np.append(dep_vol_w_array, dep_vol_w)
#             sco_vol_w_array = np.append(sco_vol_w_array, sco_vol_w)

#             morph_act_area_w_array = np.append(morph_act_area_w_array, morph_act_area_w)
#             morph_act_area_dep_w_array = np.append(morph_act_area_dep_w_array, morph_act_area_dep_w)
#             morph_act_area_sco_w_array = np.append(morph_act_area_sco_w_array, morph_act_area_sco_w)

#             act_width_mean_w_array = np.append(act_width_mean_w_array, act_width_mean_w)
#             act_width_mean_dep_w_array = np.append(act_width_mean_dep_w_array, act_width_mean_dep_w)
#             act_width_mean_sco_w_array = np.append(act_width_mean_sco_w_array, act_width_mean_sco_w)

#             act_thickness_w_array = np.append(act_thickness_w_array, act_thickness_w)
#             act_thickness_dep_w_array = np.append(act_thickness_dep_w_array, act_thickness_dep_w)
#             act_thickness_sco_w_array = np.append(act_thickness_sco_w_array, act_thickness_sco_w)

#         if plot_mode ==2:

#             # Plots
#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, dep_vol_w_array, '-', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Deposition volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, sco_vol_w_array, '-', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Scour volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_width_mean_w_array, '-', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Active width actW/W [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_thickness_w_array, '-', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Longitudinal coordinate [m]')
#             axs.set_ylabel('Active thickness [mm]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#     # Fixed window without overlapping
#     if windows_mode == 2:

#         # Calculate the number of suitable windows in the channel length
#         c_array = []
#         W_cols = int(W/px_x*1e03)
#         for i in range(1, round(dim_x/W_cols)):
#             c = math.floor(dim_x/(W_cols*i))
#             if c*W_cols*i<=dim_x:
#                 c_array = np.append(c_array, c)
#             else:
#                 pass

#         # Define the components of the slicing operation (exclude the first one)
#         f_cols_array = [0,0]
#         x_data = [] # X data for the plot
#         n = 0 # Initialize variable count
#         for m in range(0,len(c_array)):
#             # m is the window dimension in columns
#             n+=1
#             for i in range(1,(math.floor(dim_x/(W_cols*(m+1)))+1)):
#                 f_cols = [round(W_cols*(m+1)*(i-1), 1), round(W_cols*(m+1)*(i),1)]
#                 f_cols_array = np.vstack((f_cols_array, f_cols))
#                 x_data = np.append(x_data, n)

#         x_data = (x_data)*W

#         # Resize f_cols_array
#         f_cols_array = f_cols_array[1:]

#         for p in range(0, f_cols_array.shape[0]): # Loop over all the available window

#             w_len = (f_cols_array[p,1] - f_cols_array[p,0])*px_x/1e03 # Define the window lwgth

#             # Define total volume matrix, Deposition matrix and Scour matrix
#             DoD_vol_w = DoD_vol[:, f_cols_array[p,0]:f_cols_array[p,1]] # Total volume matrix
#             dep_DoD_w = dep_DoD[:, f_cols_array[p,0]:f_cols_array[p,1]] # DoD of only deposition data
#             sco_DoD_w = sco_DoD[:, f_cols_array[p,0]:f_cols_array[p,1]] # DoD of only scour data

#             # Define active pixel matrix
#             act_px_matrix_w = act_px_matrix[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active pixel matrix, both scour and deposition
#             act_px_matrix_dep_w = act_px_matrix_dep[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active deposition matrix
#             act_px_matrix_sco_w = act_px_matrix_sco[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active scour matrix

#             # Calculate principal quantities:
#             # Volumes
#             tot_vol_w = np.sum(DoD_vol_w)*px_x*px_y/(W*w_len*d50*1e09)# Total volume as V/(L*W*d50) [-] considering negative sign for scour
#             sum_vol_w = np.sum(np.abs(DoD_vol_w))*px_x*px_y/(W*w_len*d50*1e09) # Sum of scour and deposition volume as V/(L*W*d50) [-]
#             dep_vol_w = np.sum(dep_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Deposition volume as V/(L*W*d50) [-]
#             sco_vol_w = np.sum(sco_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Scour volume as V/(L*W*d50) [-]

#             # Areas:
#             morph_act_area_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Active area both in terms of scour and deposition as A/(W*L) [-]
#             morph_act_area_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Active deposition area as A/(W*L) [-]
#             morph_act_area_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Active scour area as A/(W*L) [-]

#             # Widths:
#             act_width_mean_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Total mean active width [%] - Wact/W
#             act_width_mean_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Deposition mean active width [%] - Wact/W
#             act_width_mean_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Scour mean active width [%] - Wact/W

#             # Thicknesses:
#             act_thickness_w = sum_vol_w/morph_act_area_w*(d50*1e03) # Total active thickness (abs(V_sco) + V_dep)/act_area [mm]
#             act_thickness_dep_w = dep_vol_w/morph_act_area_dep_w*(d50*1e03) # Deposition active thickness V_dep/act_area [mm]
#             act_thickness_sco_w = sco_vol_w/act_width_mean_sco_w*(d50*1e03) # Scour active thickness V_sco/act_area [mm]

#             # Append all values in arrays
#             tot_vol_w_array = np.append(tot_vol_w_array, tot_vol_w)
#             sum_vol_w_array = np.append(sum_vol_w_array, sum_vol_w)
#             dep_vol_w_array = np.append(dep_vol_w_array, dep_vol_w)
#             sco_vol_w_array = np.append(sco_vol_w_array, sco_vol_w)

#             morph_act_area_w_array = np.append(morph_act_area_w_array, morph_act_area_w)
#             morph_act_area_dep_w_array = np.append(morph_act_area_dep_w_array, morph_act_area_dep_w)
#             morph_act_area_sco_w_array = np.append(morph_act_area_sco_w_array, morph_act_area_sco_w)

#             act_width_mean_w_array = np.append(act_width_mean_w_array, act_width_mean_w)
#             act_width_mean_dep_w_array = np.append(act_width_mean_dep_w_array, act_width_mean_dep_w)
#             act_width_mean_sco_w_array = np.append(act_width_mean_sco_w_array, act_width_mean_sco_w)

#             act_thickness_w_array = np.append(act_thickness_w_array, act_thickness_w)
#             act_thickness_dep_w_array = np.append(act_thickness_dep_w_array, act_thickness_dep_w)
#             act_thickness_sco_w_array = np.append(act_thickness_sco_w_array, act_thickness_sco_w)

#         if plot_mode ==2:
#             # Plots
#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, dep_vol_w_array, 'o', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Deposition volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, sco_vol_w_array, 'o', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Scour volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_width_mean_w_array, 'o', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Active width actW/W [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_thickness_w_array, 'o', c='brown')
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Active thickness [mm]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#     # Fixed window with overlapping
#     if windows_mode == 3:

#         # Calculate the number of suitable windows in the channel length
#         c_array = []
#         W_cols = int(W/px_x*1e03) # Minimum windows length WxW dimension in columns
#         for i in range(1, math.floor(dim_x/W_cols)+1): # per each windows analysis WxWi
#             c = dim_x - W_cols*i
#             c_array = np.append(c_array, c) # Contains the number of windows for each dimension WxW*i
#         else:
#             pass

#         f_cols_array = [0,0]
#         x_data = []
#         n = 0
#         for m in range(1,int(dim_x/W_cols)+1):
#             w_length = m*W_cols # Analysis windows length
#             # print(w_length)
#             n+=1
#             for i in range(0,dim_x): # i is the lower limit of the analysis window
#                 low_lim = i # Analisys window lower limit
#                 upp_lim = i + w_length # Analisys window upper limit

#                 if upp_lim<=dim_x:
#                     # print(low_lim, upp_lim)
#                     # print(i+w_length)
#                     f_cols = [low_lim, upp_lim] # Lower and upper boundary of the analysis window
#                     f_cols_array = np.vstack((f_cols_array, f_cols))
#                     x_data = np.append(x_data, n)
#                 else:
#                     pass

#         x_data = x_data*W
#         # Resize f_cols_array
#         f_cols_array = f_cols_array[1:]

#         for p in range(0, f_cols_array.shape[0]):

#             w_len = (f_cols_array[p,1] - f_cols_array[p,0])*px_x/1e03 # Define the window length

#             # print()
#             # print(f_cols_array[p,:])
#             # print(w_len)

#             # Define total volume matrix, Deposition matrix and Scour matrix
#             DoD_vol_w = DoD_vol[:, f_cols_array[p,0]:f_cols_array[p,1]] # Total volume matrix
#             dep_DoD_w = dep_DoD[:, f_cols_array[p,0]:f_cols_array[p,1]] # DoD of only deposition data
#             sco_DoD_w = sco_DoD[:, f_cols_array[p,0]:f_cols_array[p,1]] # DoD of only scour data

#             # Define active pixel matrix
#             act_px_matrix_w = act_px_matrix[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active pixel matrix, both scour and deposition
#             act_px_matrix_dep_w = act_px_matrix_dep[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active deposition matrix
#             act_px_matrix_sco_w = act_px_matrix_sco[:, f_cols_array[p,0]:f_cols_array[p,1]] # Active scour matrix

#             # Calculate principal quantities:
#             # Volumes
#             tot_vol_w = np.sum(DoD_vol_w)*px_x*px_y/(W*w_len*d50*1e09)# Total volume as V/(L*W*d50) [-] considering negative sign for scour
#             sum_vol_w = np.sum(np.abs(DoD_vol_w))*px_x*px_y/(W*w_len*d50*1e09) # Sum of scour and deposition volume as V/(L*W*d50) [-]
#             dep_vol_w = np.sum(dep_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Deposition volume as V/(L*W*d50) [-]
#             sco_vol_w = np.sum(sco_DoD_w)*px_x*px_y/(W*w_len*d50*1e09) # Scour volume as V/(L*W*d50) [-]

#             # Areas:
#             morph_act_area_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Active area both in terms of scour and deposition as A/(W*L) [-]
#             morph_act_area_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Active deposition area as A/(W*L) [-]
#             morph_act_area_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Active scour area as A/(W*L) [-]

#             # Widths:
#             act_width_mean_w = np.count_nonzero(act_px_matrix_w)*px_x*px_y/(W*w_len*1e06) # Total mean active width [%] - Wact/W
#             act_width_mean_dep_w = np.count_nonzero(act_px_matrix_dep_w)*px_x*px_y/(W*w_len*1e06) # Deposition mean active width [%] - Wact/W
#             act_width_mean_sco_w = np.count_nonzero(act_px_matrix_sco_w)*px_x*px_y/(W*w_len*1e06) # Scour mean active width [%] - Wact/W

#             # Thicknesses:
#             act_thickness_w = sum_vol_w/morph_act_area_w*(d50*1e03) # Total active thickness (abs(V_sco) + V_dep)/act_area [mm]
#             act_thickness_dep_w = dep_vol_w/morph_act_area_dep_w*(d50*1e03) # Deposition active thickness V_dep/act_area [mm]
#             act_thickness_sco_w = sco_vol_w/act_width_mean_sco_w*(d50*1e03) # Scour active thickness V_sco/act_area [mm]

#             # Append all values in arrays
#             tot_vol_w_array = np.append(tot_vol_w_array, tot_vol_w)
#             sum_vol_w_array = np.append(sum_vol_w_array, sum_vol_w)
#             dep_vol_w_array = np.append(dep_vol_w_array, dep_vol_w)
#             sco_vol_w_array = np.append(sco_vol_w_array, sco_vol_w)

#             morph_act_area_w_array = np.append(morph_act_area_w_array, morph_act_area_w)
#             morph_act_area_dep_w_array = np.append(morph_act_area_dep_w_array, morph_act_area_dep_w)
#             morph_act_area_sco_w_array = np.append(morph_act_area_sco_w_array, morph_act_area_sco_w)

#             act_width_mean_w_array = np.append(act_width_mean_w_array, act_width_mean_w)
#             act_width_mean_dep_w_array = np.append(act_width_mean_dep_w_array, act_width_mean_dep_w)
#             act_width_mean_sco_w_array = np.append(act_width_mean_sco_w_array, act_width_mean_sco_w)

#             act_thickness_w_array = np.append(act_thickness_w_array, act_thickness_w)
#             act_thickness_dep_w_array = np.append(act_thickness_dep_w_array, act_thickness_dep_w)
#             act_thickness_sco_w_array = np.append(act_thickness_sco_w_array, act_thickness_sco_w)

#         if plot_mode ==2:
#             # Plots
#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, dep_vol_w_array, 'o', c='brown', markersize=0.1)
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Deposition volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, sco_vol_w_array, 'o', c='brown', markersize=0.1)
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Scour volumes V/(W*L*d50) [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_width_mean_w_array, 'o', c='brown', markersize=0.1)
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Active width actW/W [-]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#             fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
#             axs.plot(x_data, act_thickness_w_array, 'o', c='brown', markersize=0.1)
#             axs.set_title(run)
#             axs.set_xlabel('Window analysis length [m]')
#             axs.set_ylabel('Active thickness [mm]')
#             # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
#             plt.show()

#     if f == DoDs_name_array[0]:
#         dep_vol_w_array_all = np.transpose(np.array(dep_vol_w_array))
#         sco_vol_w_array_all = np.transpose(np.array(sco_vol_w_array))
#     else:
#         pass

#     dep_vol_w_array_all = np.vstack((dep_vol_w_array_all,dep_vol_w_array))
#     dep_vol_mean = np.mean(dep_vol_w_array_all, axis=0)
#     dep_vol_std = np.std(dep_vol_w_array_all, axis=0)

#     sco_vol_w_array_all = np.vstack((sco_vol_w_array_all,sco_vol_w_array))
#     sco_vol_mean = np.mean(sco_vol_w_array_all, axis=0)
#     sco_vol_std = np.std(sco_vol_w_array_all, axis=0)

#     if windows_mode==2:

#         # Loop to define the windows to clusterize data
#         array = [0]
#         num=0
#         for n in range(0,len(c_array)):
#             num += c_array[n]
#             array = np.append(array, num) # Clusterize window dimension

#         dep_vol_mean = []
#         sco_vol_mean = []
#         dep_vol_std = []
#         sco_vol_std = []

#         x_data_full = x_data
#         x_data = []
#         for n in range(0, len(array)-1):
#             x_data = np.append(x_data, x_data_full[int(array[n])])

#         for n in f_cols_array:
#             dep_vol_mean = np.append(dep_vol_mean, np.mean(dep_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             sco_vol_mean = np.append(sco_vol_mean, np.mean(sco_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             dep_vol_std = np.append(dep_vol_std, np.std(dep_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             sco_vol_std = np.append(sco_vol_std, np.std(sco_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             # To finish

#     if windows_mode == 3:
#         # Loop to define the windows to clusterize data
#         array = [0]
#         num=0
#         for n in range(0,len(c_array)):
#             num += c_array[n]
#             array = np.append(array, num) # Clusterize window dimension

#         dep_vol_mean = []
#         sco_vol_mean = []
#         dep_vol_std = []
#         sco_vol_std = []

#         x_data_full = x_data
#         x_data = []
#         for n in range(0, len(array)-1):
#             # low_lim = int(f_cols_array[n,0])
#             # upp_lim = int(f_cols_array[n,1])
#             x_data = np.append(x_data, round(x_data_full[int(array[n])+n],1))
#             # dep_vol_mean = np.append(dep_vol_mean, np.mean(dep_vol_w_array_all[:,low_lim:upp_lim]))
#             # sco_vol_mean = np.append(sco_vol_mean, np.mean(sco_vol_w_array_all[:,low_lim:upp_lim]))
#             # dep_vol_std = np.append(dep_vol_std, np.std(dep_vol_w_array_all[:,low_lim:upp_lim]))
#             # sco_vol_std = np.append(sco_vol_std, np.std(sco_vol_w_array_all[:,low_lim:upp_lim]))
#             dep_vol_mean = np.append(dep_vol_mean, np.mean(dep_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             sco_vol_mean = np.append(sco_vol_mean, np.mean(sco_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             dep_vol_std = np.append(dep_vol_std, np.std(dep_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             sco_vol_std = np.append(sco_vol_std, np.std(sco_vol_w_array_all[:,int(array[n]):int(array[n+1])]))
#             # print(int(array[n]),int(array[n+1]))
#             # TODO To finish

# fig3, axs = plt.subplots(2,1,dpi=80, figsize=(10,6), sharex=True, tight_layout=True)
# fig3.suptitle(run + ' - Volume')
# axs[0].errorbar(x_data, sco_vol_mean, sco_vol_std, linestyle='--', marker='^', color='red')
# # axs[0].set_ylim(bottom=0)
# axs[0].set_title('Scour')
# # axs[0].set_xlabel()
# axs[0].set_ylabel('Scour volume V/(L*W*d50) [-]')
# axs[1].errorbar(x_data, dep_vol_mean, dep_vol_std, linestyle='--', marker='^', color='blue')
# axs[1].set_ylim(bottom=0)
# axs[1].set_title('Deposition')
# axs[1].set_xlabel('Analysis window length [m]')
# axs[1].set_ylabel('Deposition volume V/(L*W*d50) [-]')
# # plt.savefig(os.path.join(plot_dir, run +'dep_scour.png'), dpi=200)
# plt.show()

# # # Plots
# # fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
# # axs.plot(x_data, dep_vol_w_array, 'o', c='brown')
# # axs.set_title(run)
# # axs.set_xlabel('Longitudinal coordinate [m]')
# # axs.set_ylabel('Deposition volumes V/(W*L*d50) [-]')
# # # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
# # plt.show()

# # fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
# # axs.plot(x_data, sco_vol_w_array, 'o', c='brown')
# # axs.set_title(run)
# # axs.set_xlabel('Longitudinal coordinate [m]')
# # axs.set_ylabel('Scour volumes V/(W*L*d50) [-]')
# # # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
# # plt.show()

# # fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
# # axs.plot(x_data, act_width_mean_w_array, 'o', c='brown')
# # axs.set_title(run)
# # axs.set_xlabel('Longitudinal coordinate [m]')
# # axs.set_ylabel('Active width actW/W [-]')
# # # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
# # plt.show()

# # fig1, axs = plt.subplots(1,1,dpi=200, sharex=True, tight_layout=True)
# # axs.plot(x_data, act_thickness_w_array, 'o', c='brown')
# # axs.set_title(run)
# # axs.set_xlabel('Longitudinal coordinate [m]')
# # axs.set_ylabel('Active thickness [mm]')
# # # plt.savefig(os.path.join(plot_dir, run +'_morphW_interp.png'), dpi=200)
# # plt.show()


text:
from random import randint
from subprocess import Popen, PIPE

from hcbr import HCBRClassifier

import numpy as np
from sklearn.metrics import accuracy_score, matthews_corrcoef
from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold, RepeatedKFold

HCBR_BIN = '../../../build/hcbr'
DATASET = 'breast'

def example_1(clf, X, y):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)
    clf.fit(X_train, y_train)

    pred = clf.predict(X_test, y_test)
    score = accuracy_score(y_test, pred)
    mcc = matthews_corrcoef(y_test, pred)
    print(score, mcc)

def example_2(clf, X, y):
    random_state = randint(0,2**32)
    k=10
    n=5
    rkf = RepeatedStratifiedKFold(n_splits=k, n_repeats=n, random_state=random_state)
    averages = [0] * n
    for i, (train, test) in enumerate(rkf.split(X, y)):
        it = i // k
        X_train, X_test, y_train, y_test = X[train], X[test], y[train], y[test]
        clf.fit(X_train, y_train)
        pred = clf.predict(X_test, y_test)

        acc = accuracy_score(y_test, pred)
        mcc = matthews_corrcoef(y_test, pred)
        print(it, acc, mcc)
        averages[it] += acc
    averages = map(lambda x: x / k, averages)
    print('BEST ACCURACY: {}'.format(max(averages)))

def main():

    # Load the dataset
    with open('../../../data/{}_casebase.txt'.format(DATASET)) as file:
        X = file.readlines()
        X = np.array([map(int, d.split()) for d in X])
        file.close()

    with open('../../../data/{}_outcomes.txt'.format(DATASET)) as file:
        y = file.readlines()
        y = np.array([int(o.strip()) for o in y])
        file.close()

    # Create the classifier by specifying the configuration file
    clf = HCBRClassifier(params_file='../../../data/parameters/{}.params.json'.format(DATASET))

    # example 1: simple split
    print('----- EXAMPLE 1 -----')
    example_1(clf, X, y)

    # example 2: stratified K-fold cross-validation
    print('----- EXAMPLE 2 -----')
    example_2(clf, X, y)

if __name__ == '__main__':
    main()

max_stars_repo_name: aquemy/HCBR

max_issues_repo_name: aquemy/HCBR

max_forks_repo_name: aquemy/HCBR


text:
# coding = utf-8
import encodings
from bs4 import BeautifulSoup
import re
import urllib.error
from numpy.distutils.misc_util import is_string
from numpy.ma.core import append
import requests
import os
import time
import json
import codecs

def fix_file():
    actor_files = os.listdir(os.path.abspath('./stardata'))
    for actor_file in actor_files:
        c = ''
        with codecs.open(os.path.abspath('./stardata') + '/' + actor_file, 'r+', encoding='utf-8') as fp:
            f = fp.read()
            c = f.split('}')[1] + '}'
        with codecs.open(os.path.abspath('./stardata') + '/' + actor_file, 'w', encoding='utf-8') as fp:
            fp.write(c)

def build_map():
    film_files = os.listdir(os.path.abspath('./filmdata'))
    for film_file in film_files:
        film_id = re.compile(r'(\d+).json').findall(film_file)[0]
        with codecs.open(os.path.abspath('./filmdata') + '/' + film_file, 'r', encoding='utf-8') as fp:
            film_json = json.load(fp)
        for actor in film_json['actor']:
            actor_id = re.compile(r'/celebrity/(\d+)/').findall(actor)[0]
            if os.path.exists(('./stardata') + '/' + actor_id + '.json'):
                afp = open(os.path.abspath('./stardata') + '/' + actor_id + '.json', 'r+', encoding='utf-8')
                try:
                    actor_json = json.load(afp)
                    actor_json['movies'].append(film_id)
                    afp.truncate()
                    afp.close()
                    wfp = open(os.path.abspath('./stardata') + '/' + actor_id + '.json', 'w', encoding='utf-8')
                    json.dump(actor_json, wfp, indent=2, sort_keys=False, ensure_ascii=False)
                    wfp.close()
                except ValueError:
                    continue

def find_co():
    actor_files = os.listdir(os.path.abspath('./stardata'))
    for actor_file in actor_files:
        print(actor_file)
        actor_id = re.compile(r'(\d+).json').findall(actor_file)[0]
        fp = open(os.path.abspath('./stardata') + '/' + actor_file, 'r+', encoding='utf-8')
        try:
            actor_json = json.load(fp)
        except  ValueError:
            continue
        fp.truncate()
        fp.close()

        cooperators = {}
        actor_json['cooperators'] = {}
        for movie in actor_json['movies']:
            ffp = open(os.path.abspath('./filmdata') + '/' + movie + '.json', 'r', encoding='utf-8')
            try:
                film_json = json.load(ffp)
                for co in film_json['actor']:
                    co_id = re.compile(r'/celebrity/(\d+)/').findall(co)[0]
                    if co_id != actor_id:
                        if co_id in cooperators:
                            cooperators[co_id] += 1
                        else:
                            cooperators[co_id] = 1
            except ValueError:
                continue
        l = sorted(cooperators.items(), key=lambda x: x[1], reverse=True)
        for key, value in l:
            actor_json['cooperators'][key] = value
        wfp = open(os.path.abspath('./stardata') + '/' + actor_file, 'w', encoding='utf-8')
        json.dump(actor_json, wfp, indent=2, sort_keys=False, ensure_ascii=False)
        wfp.close()

if __name__ == "__main__":
    find_co()

reason: from numpy