text:
import numpy as np
import time

class mynet(object):

	def sigmoid(x, dif=False):
		if dif is False:
			return 1/(1 + np.exp(-x))
		else:
			return np.exp(-x)/((1 + np.exp(-x))**2)

	def softmax(x, dif=False):
		if dif is False:
			return np.exp(x)/np.sum(np.exp(x))
		else:
			y = np.exp(x)/np.sum(np.exp(x))
			y = y[:, np.newaxis]
			matrixd = np.zeros((len(y), len(y)))
			for i in range(len(y)):
				matrixd[i, i] = y[i]
			matrix = np.matmul(y, y.T)
			return matrixd - matrix

	def __init__(self, ninput, noutput, activation=sigmoid, finalf=softmax):
		if type(ninput) != int:
			err_msg = 'ninput must be an int'
			raise TypeError(err_msg)
		if type(noutput) != int:
			err_msg = 'noutput must be an int'
			raise TypeError(err_msg)
		if callable(activation) is False:
			err_msg = 'activation must be a callable function'
			raise TypeError(err_msg)
		if callable(finalf) is False:
			err_msg = 'finalf must be a callable function'
			raise TypeError(err_msg)
		self.nlayers = 1
		self.ninput = ninput
		self.layers = [noutput]
		self.weights = [np.random.randn(noutput, ninput)*0.1]
		self.biases = [np.random.randn(noutput)*0.1]
		self.activation = activation
		self.finalf = finalf
		self.fulldata = None
		self.traindata = None
		self.testdata = None

	def add_single_layer(self, nnodes, position=None):
		if type(nnodes) != int:
			err_msg = 'nnodes must be an int'
			raise TypeError(err_msg)
		position = self.nlayers-1 if position is None else position
		position = self.nlayers + position if position < 0 else position
		if position >= self.nlayers:
			err_msg = 'position must be less than the number of layers'
			raise ValueError(err_msg)
		self.nlayers += 1
		self.layers.insert(position, nnodes)
		self.weights.insert(position, np.random.randn(nnodes, self.layers[position-1] if position > 0 else self.ninput)*0.1)
		self.weights[position+1] = np.random.randn(self.layers[position+1], nnodes)*0.1
		self.biases.insert(position, np.random.randn(nnodes)*0.1)

	def add_layers(self, nodelist, position=None):
		if type(nodelist) != list:
			err_msg = 'nodelist must be a list'
			raise TypeError(err_msg)
		position = self.nlayers-1 if position is None else position
		position = self.nlayers + position if position < 0 else position
		for nnodes in nodelist:
			self.add_single_layer(nnodes, position=position)
			position += 1

	def print_layers(self):
		print(self.ninput, end=' ')
		for l, w, b in zip(self.layers, self.weights, self.biases):
			print(np.shape(w), np.shape(b), l, end=' ')
		print('')

	def evaluate(self, ineval):
		if np.shape(ineval) != np.array(self.ninput):
			err_msg = 'Input must be same size as the input layer (=' + str(self.ninput) + ')'
			raise ValueError(err_msg)
		else:
			current = np.matmul(self.weights[0], ineval) + self.biases[0]
			for i in range(1, self.nlayers):
				current = np.matmul(self.weights[i], self.activation(current)) + self.biases[i]
			return self.finalf(current)

	def give_data(self, dataset, labels, datatype='all', ftrain=0.5):
		if len(dataset[0]) != self.ninput:
			err_msg = 'Dataset must be of the form [#instances, shape of input]'
			raise ValueError(err_msg)
		if len(labels[0]) != self.layers[-1]:
			err_msg = 'Labels must be of the form [#instances, shape of output]'
			raise ValueError(err_msg)
		if datatype not in ['all', 'train', 'test']:
			err_msg = 'Datatype must be "all", "train" or "test"'
			raise ValueError(err_msg)
		if (ftrain < 0) or (ftrain > 1):
			err_msg = 'ftrain must be between 0 and 1'
			raise ValueError(err_msg)
		if self.fulldata is None:
			self.fulldata = [dataset, labels]
		else:
			self.fulldata[0] = np.concatenate((self.fulldata[0], dataset), axis=0)
			self.fulldata[1] = np.concatenate((self.fulldata[1], labels), axis=0)
		if datatype == 'train' and self.traindata == None:
			self.traindata = [dataset, labels]
		elif datatype == 'train' and self.traindata != None:
			self.traindata[0] = np.concatenate((self.traindata[0], dataset), axis=0)
			self.traindata[1] = np.concatenate((self.traindata[1], labels), axis=0)
		if datatype == 'test' and self.testdata == None:
			self.testdata = [dataset, labels]
		elif datatype == 'test' and self.testdata != None:
			self.testdata[0] = np.concatenate((self.testdata[0], dataset), axis=0)
			self.testdata[1] = np.concatenate((self.testdata[1], labels), axis=0)
		if datatype == 'all':
			cut = int(ftrain*np.size(self.fulldata[0], axis=0))
			self.traindata = [self.fulldata[0][:cut], self.fulldata[1][:cut]]
			self.testdata = [self.fulldata[0][cut:], self.fulldata[1][cut:]]

	def backprop(self, ineval, expected):
		if np.shape(ineval) != np.array(self.ninput):
			err_msg = 'Input must be same size as the input layer (=' + str(self.ninput) + ')'
			raise ValueError(err_msg)
		else:
			Zs = [np.matmul(self.weights[0], ineval) + self.biases[0]]
			As = [self.activation(Zs[0])]
			for i in range(1, self.nlayers-1):
				Zs.append(np.matmul(self.weights[i], As[-1]) + self.biases[i])
				As.append(self.activation(Zs[-1]))
			Zs.append(np.matmul(self.weights[-1], As[-1]) + self.biases[-1])
			As.append(self.finalf(Zs[-1]))
			deltaC = As[-1] - expected
			deltaBs = [np.matmul(self.finalf(Zs[-1], dif=True), deltaC)]
			deltaWs = [np.matmul(deltaBs[0][:, np.newaxis], np.array(As[-2])[:, np.newaxis].T)]
			for i in range(1, self.nlayers-1):
				deltaBs.insert(0, np.matmul(self.weights[-i].T, deltaBs[0])*self.activation(Zs[-i-1], dif=True))
				deltaWs.insert(0, np.matmul(deltaBs[0][:, np.newaxis], np.array(As[-i-2])[:, np.newaxis].T))
			deltaBs.insert(0, np.matmul(self.weights[1].T, deltaBs[0])*self.activation(Zs[0], dif=True))
			deltaWs.insert(0, np.matmul(deltaBs[0][:, np.newaxis], np.array(ineval)[:, np.newaxis].T))
			return deltaBs, deltaWs

	def mini_batch(self, start, end, eta):
		n = end-start
		comBs = []
		comWs = []
		for i in range(start, end):
			deltaBs, deltaWs = self.backprop(self.traindata[0][i], self.traindata[1][i])
			if i == start:
				for deltaB in deltaBs:
					comBs.append(deltaB/n)
				for deltaW in deltaWs:
					comWs.append(deltaW/n)
			else:
				for j in range(self.nlayers):
					comBs[j] += deltaBs[j]/n
				for j in range(self.nlayers):
					comWs[j] += deltaWs[j]/n
		for i in range(self.nlayers):
			self.biases[i] += -eta*comBs[i]
		for i in range(self.nlayers):
			self.weights[i] += -eta*comWs[i]

	def training_montage(self, batches, rounds, eta, exams=True):
		trainsize = np.size(self.traindata[0], axis=0)
		mostsize = trainsize//batches
		lastsize = mostsize + trainsize%batches
		if exams is True:
			examsize = np.size(self.testdata[0], axis=0)
			passes = 0
			averageC = 0
			for j in range(examsize):
				output = self.evaluate(self.testdata[0][j])
				averageC += 0.5/examsize*np.sum((self.testdata[1][j] - output)**2)
				if np.argmax(output) == np.argmax(self.testdata[1][j]) and np.max(output) > 0.5:
					passes += 1
			grade = np.round(100*passes/examsize, decimals=2)
			print('Before training achieved a grade of ' + str(grade) + ' with an average cost of ' + str(np.round(averageC, decimals=5)))
		for i in range(rounds):
			print('Starting round ' + str(i+1) + '/' + str(rounds))
			time0 = time.time()
			for j in range(batches-1):
				averageC = self.mini_batch(j*mostsize, (j+1)*mostsize, eta)
			averageC = self.mini_batch((batches-1)*mostsize, (batches-1)*mostsize + lastsize, eta)
			if exams is True:
				passes = 0
				averageC = 0
				for j in range(examsize):
					output = self.evaluate(self.testdata[0][j])
					averageC += 0.5/examsize*np.sum((self.testdata[1][j] - output)**2)
					if np.argmax(output) == np.argmax(self.testdata[1][j]) and np.max(output) > 0.5:
						passes += 1
				grade = np.round(100*passes/examsize, decimals=2)
				announce = ' Achieved a grade of ' + str(grade) + ' with an average cost of ' + str(np.round(averageC, decimals=5))
			else:
				announce = ''
			print('Round ' + str(i+1) + '/' + str(rounds) + ' complete in ' + str(np.round(time.time()-time0, decimals=3)) + ' seconds!' + announce)

max_stars_repo_name: connor-mcisaac/myml

max_issues_repo_name: connor-mcisaac/myml

max_forks_repo_name: connor-mcisaac/myml


text:
import sys,time
import numpy as np
import torch
from copy import deepcopy
import torch.nn.functional as F
from collections import defaultdict

import utils
from .approach import BaseApproach

class Appr(BaseApproach):
    """ Class implementing the Incremental Moment Matching (mean) approach described in https://arxiv.org/abs/1703.08475 """

    def __init__(self,model,nepochs=100,sbatch=64,lr=0.05,lr_min=1e-4,lr_factor=3,lr_patience=5,warmup=None,clipgrad=10000,curriculum=None,log_path=None,regularizer=0.0001,alpha=0.7):
        super().__init__(model, nepochs, sbatch, lr, lr_min, lr_factor, lr_patience, warmup, clipgrad,curriculum, log_path)
        self.model_old=None

        print("Setting regularizer to {}".format(regularizer))
        self.reg=regularizer    # Grid search = [0.01,0.005,0.001,0.0005,0.0001,0.00005,0.000001]; best was 0.0001
        #self.alpha=alpha       # We assume the same alpha for all tasks. Unrealistic to tune one alpha per task (num_task-1 alphas) when we have a lot of tasks.

        return

    def post_train(self, t,xtrain,ytrain,xvalid,yvalid):
        # Restore best, save model as old
        if t>0:
            model_state = utils.get_model(self.model)
            model_old_state = utils.get_model(self.model_old)
            for name, param in self.model.named_parameters():
                #model_state[name]=(1-self.alpha)*model_old_state[name]+self.alpha*model_state[name]
                model_state[name]=(model_state[name]+model_old_state[name]*t)/(t+1)
            utils.set_model_(self.model,model_state)

        self.model_old=deepcopy(self.model)
        utils.freeze_model(self.model_old)
        self.model_old.eval()

        return

    def train_batch(self,t,tt,i,x,y,c,b,r,e):
        with torch.no_grad():
            images=torch.autograd.Variable(x[b])
            targets=torch.autograd.Variable(y[b])

        # Forward current model
        outputs=self.model.forward(images)
        output=outputs[t]
        loss=self.imm_criterion(output, targets, t)

        # Backward
        self.optimizer.zero_grad()
        loss.backward()
        torch.nn.utils.clip_grad_norm_(self.model.parameters(),self.clipgrad)
        self.optimizer.step()

        return

    def eval_batch(self,b,t,tt,x,y,c,items):
        with torch.no_grad():
            images=torch.autograd.Variable(x[b])
            targets=torch.autograd.Variable(y[b])

        # Forward
        outputs=self.model.forward(images)
        output=outputs[t]
        loss=self.imm_criterion(output,targets,t)
        _,pred=output.max(1)
        hits=(pred==targets).float()

        # Log
        items["loss"]+=loss.data.cpu().numpy()*len(b)
        items["acc"]+=hits.sum().data.cpu().numpy()

        return items

    def imm_criterion(self, output, targets, t):
        # L2 multiplier loss
        loss_reg=0
        if t>0:
            for (name,param),(_,param_old) in zip(self.model.named_parameters(), self.model_old.named_parameters()):
                loss_reg += torch.sum((param_old-param).pow(2))/2

        # Cross entropy loss
        loss_ce=self.criterion(output,targets)

        return loss_ce + self.reg*loss_reg

ext: py

max_stars_repo_name: felixnext/dwa

max_issues_repo_name: felixnext/dwa

max_forks_repo_name: felixnext/dwa

reason: import numpy


text:
"""Prepare files that will go into LN2_GEODISTANCE."""

import os
import subprocess
import numpy as np
import nibabel as nb

RIMS = [
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/00_segmentation/HG_RH/sub-05_ses-T2s_segm_rim_HG_RH_v02.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/00_segmentation/HG_LH/sub-05_ses-T2s_segm_rim_HG_LH_v02.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/00_segmentation/CS_RH/sub-05_ses-T2s_segm_rim_CS_RH_v02.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/00_segmentation/CS_LH/sub-05_ses-T2s_segm_rim_CS_LH_v02.nii.gz",
]

BORDERS = [
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/01_rim_prep/sub-05_ses-T2s_segm_rim_HG_RH_v02_borders.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/01_rim_prep/sub-05_ses-T2s_segm_rim_HG_LH_v02_borders.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/01_rim_prep/sub-05_ses-T2s_segm_rim_CS_RH_v02_borders.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/01_rim_prep/sub-05_ses-T2s_segm_rim_CS_LH_v02_borders.nii.gz",
]

CHUNKS = [
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/03_multilaterate/sub-05_ses-T2s_segm_rim_HG_RH_v02_borderized_multilaterate_perimeter_chunk.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/03_multilaterate/sub-05_ses-T2s_segm_rim_HG_LH_v02_borderized_multilaterate_perimeter_chunk.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/03_multilaterate/sub-05_ses-T2s_segm_rim_CS_RH_v02_borderized_multilaterate_perimeter_chunk.nii.gz",
    "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/03_multilaterate/sub-05_ses-T2s_segm_rim_CS_LH_v02_borderized_multilaterate_perimeter_chunk.nii.gz",
    ]

MAX_DIST = 0.35 * 2

OUTDIR = "/home/faruk/data2/DATA_MRI_NIFTI/derived/sub-05/segmentation/05_beyond_gm_prep"

# -----------------------------------------------------------------------------
# Output directory
if not os.path.exists(OUTDIR):
    os.makedirs(OUTDIR)
    print("  Output directory: {}\n".format(OUTDIR))

for i in range(len(RIMS)):
    rim = RIMS[i]
    border = BORDERS[i]

    nii1 = nb.load(rim)
    nii2 = nb.load(border)

    # Mask WM
    data1 = np.asarray(nii1.dataobj)
    data2 = np.asarray(nii2.dataobj)
    data1[data1 != 2] = 0
    data1[data1 != 0] = 1
    data2[data2 != 2] = 0
    data2[data2 != 0] = 1

    # Save nifti
    filename = os.path.basename(rim)
    basename, ext = filename.split(os.extsep, 1)
    outname = os.path.join(OUTDIR, "{}_domain-wm.{}".format(basename, ext))
    out_img = nb.Nifti1Image(data1, header=nii1.header, affine=nii1.affine)
    nb.save(out_img, outname)
    outname = os.path.join(OUTDIR, "{}_border-wm.{}".format(basename, ext))
    out_img = nb.Nifti1Image(data2, header=nii2.header, affine=nii2.affine)
    nb.save(out_img, outname)

    # -------------------------------------------------------------------------
    # Mask CSF
    data1 = np.asarray(nii1.dataobj)
    data2 = np.asarray(nii2.dataobj)
    data1[data1 != 1] = 0
    data1[data1 != 0] = 1
    data2[data2 != 1] = 0
    data2[data2 != 0] = 1

    # Save nifti
    filename = os.path.basename(RIMS[i])
    basename, ext = filename.split(os.extsep, 1)
    outname = os.path.join(OUTDIR, "{}_domain-csf.{}".format(basename, ext))
    out_img = nb.Nifti1Image(data1, header=nii1.header, affine=nii1.affine)
    nb.save(out_img, outname)
    outname = os.path.join(OUTDIR, "{}_border-csf.{}".format(basename, ext))
    out_img = nb.Nifti1Image(data2, header=nii2.header, affine=nii2.affine)
    nb.save(out_img, outname)

    # =========================================================================
    # Dilate chunks
    chunk = CHUNKS[i]

    # Determine output basename
    filename = os.path.basename(chunk)
    basename, ext = filename.split(os.extsep, 1)
    outname = os.path.join(OUTDIR, "{}_voronoi_dilated.{}".format(basename, ext))

    # Layers and middle gray matter
    command = "/home/faruk/Git/LAYNII/LN2_VORONOI "
    command += "-domain {} ".format(rim)
    command += "-init {} ".format(chunk)
    command += "-max_dist {} ".format(MAX_DIST)
    command += "-output {} ".format(outname)
    print(command)
    subprocess.run(command, shell=True)

print('Finished.\n')

max_stars_repo_name: ofgulban/meso-MRI

max_issues_repo_name: ofgulban/meso-MRI

max_forks_repo_name: ofgulban/meso-MRI


text:
import numpy as np
from sklearn.utils import class_weight

def load_data(features=[0,1,2,3,4,5], shuffle=True, data_path="../dana/datasets/MotionSense/"):

    X_train = np.load(data_path+"X_train.npy")
    Y_train = np.load(data_path+"y_train.npy")
    X_test = np.load(data_path+"X_test.npy")
    Y_test = np.load(data_path+"y_test.npy")

    X_train = X_train[:,:,features]
    X_test = X_test[:,:,features]

    if shuffle:
        indices = np.random.permutation(len(X_train))
        X_train = X_train[indices]
        Y_train = Y_train[indices]

    act_lbls = ["STN","STU","WAL","JOG","STD","SIT"]
    act_weights = class_weight.compute_class_weight('balanced',range(len(act_lbls)),Y_train)
    act_weights = act_weights.round(4)
    act_weights = dict(zip(range(len(act_weights)),act_weights))

    return (X_train, Y_train, X_test, Y_test, act_lbls, act_weights)

max_stars_repo_name: mmalekzadeh/dana

max_issues_repo_name: mmalekzadeh/dana

max_forks_repo_name: mmalekzadeh/dana


text:
"""S3 indexing module."""
from __future__ import absolute_import

import logging
from uuid import uuid4

import numpy as np
from sqlalchemy import select, and_

from datacube.drivers.postgres._core import pg_exists
from datacube.drivers.s3aio_index.schema import S3_DATASET, S3_DATASET_CHUNK, S3_DATASET_MAPPING, S3_METADATA
from datacube.index._datasets import DatasetResource as BaseDatasetResource
from datacube.index.index import Index

_LOG = logging.getLogger(__name__)
FORMAT = 'aio'

class S3DatabaseException(Exception):
    """Raised on errors to do with the S3 block specific database"""

class S3AIOIndex(Index):
    """The s3 indexer extends the existing postgres indexer functionality
    by writing additional s3 information to specific tables.
    """

    def __init__(self, db):
        """Initialise the index and its dataset resource."""
        super(S3AIOIndex, self).__init__(db)
        # if not self.connected_to_s3_database():
        #     raise S3DatabaseException('Not connected to an S3 Database')

        self.datasets = DatasetResource(db, self.products)

    def connected_to_s3_database(self):
        """Check we are connected to an appropriately initialised database.

        :return: True if requirements is satisfied, otherwise returns False
        """
        try:
            with self._db.give_me_a_connection() as connection:
                return (pg_exists(connection, "agdc.s3_dataset") and
                        pg_exists(connection, "agdc.s3_dataset_chunk") and
                        pg_exists(connection, "agdc.s3_dataset_mapping"))
        except AttributeError:
            _LOG.warning('Should only be here for tests.')
            return True

    def init_db(self, with_default_types=True, with_permissions=True):
        is_new = super(S3AIOIndex, self).init_db(with_default_types, with_permissions)

        if is_new:
            with self._db.give_me_a_connection() as connection:
                try:
                    connection.execute('begin')
                    _LOG.info('Creating s3 block tables.')

                    S3_METADATA.create_all(connection)
                    connection.execute('commit')
                except Exception:
                    connection.execute('rollback')
                    raise

        return is_new

    def __repr__(self):
        return "S3Index<db={!r}>".format(self._db)

class DatasetResource(BaseDatasetResource):
    """The s3 dataset resource extends the postgres one by writing
    additional s3 information to specific tables.
    """

    def add(self, dataset, with_lineage=None, **kwargs):
        saved_dataset = super(DatasetResource, self).add(dataset, with_lineage=with_lineage, **kwargs)

        if dataset.format == FORMAT:
            storage_metadata = kwargs['storage_metadata']  # It's an error to not include this
            self.add_datasets_to_s3_tables([dataset.id], storage_metadata)
        return saved_dataset

    def add_multiple(self, datasets, with_lineage=None):
        """Index several datasets.

        Perform the normal indexing, followed by the s3 specific
        indexing. If the normal indexing fails for any dataset, then
        no s3 indexing takes place and a `ValueError` is raised.

        :param datasets: The datasets to be indexed. It must contain
          an attribute named `storage_metadata` otherwise a ValueError
          is raised.
        :param bool with_lineage: Whether to recursively add lineage, default: yes
        :return: The number of datasets indexed.
        :rtype: int

        """
        raise SystemError("I don't think this is called or used, it will need to be fixed when "
                          " storage chunks across multiple datasets are used")

        # if 'storage_metadata' not in datasets.attrs:
        #     raise ValueError('s3 storage output not received, indexing aborted.')
        # dataset_refs = []
        # n = 0
        # for dataset in datasets.values:
        #     self.add(dataset, with_lineage=with_lineage)
        #     dataset_refs.append(dataset.id)
        #     n += 1
        # if n == len(datasets):
        #     self.add_datasets_to_s3_tables(dataset_refs, datasets.attrs['storage_metadata'])
        # else:
        #     raise ValueError('Some datasets could not be indexed, hence no s3 indexing will happen.')
        # return n

    def _add_s3_dataset(self, transaction, s3_dataset_id, band, output):
        """Add the new s3 dataset to DB.

        :param transaction: Postgres transaction.
        :param uuid s3_dataset_id: The uuid of the s3 dataset.
        :param str band: The band to index this set for.
        :param dict output: Dictionary of metadata consigning
          the s3 storage information for that band.
        """
        # Build regular indices as list of triple scalars (as
        # numpy types are not accepted by sqlalchemy)
        regular_indices = [list(map(np.asscalar, index))
                           for index in output['regular_index'] if index is not None]

        # Build irregular indices list, padding them all to
        # the same size as required by Postgres
        # multidimensional arrays
        irregular_indices = [list(map(np.asscalar, index))
                             for index in output['irregular_index'] if index is not None]
        if irregular_indices:
            irregular_max_size = max(map(len, irregular_indices))
            irregular_indices = [index + [None] * (irregular_max_size - len(index))
                                 for index in irregular_indices]

        _LOG.debug('put_s3_dataset(%s, %s, %s, %s, %s, %s, %s, %s, %s ,%s, %s)', s3_dataset_id,
                   output['base_name'], band, output['bucket'],
                   output['macro_shape'], output['chunk_size'],
                   output['numpy_type'], output['dimensions'],
                   output['regular_dims'], regular_indices,
                   irregular_indices)
        self._put_s3_dataset(transaction,
                             s3_dataset_id,
                             output['base_name'],
                             band,
                             output['bucket'],
                             output['macro_shape'],
                             output['chunk_size'],
                             output['numpy_type'],
                             output['dimensions'],
                             output['regular_dims'],
                             regular_indices,
                             irregular_indices)

    def _add_s3_dataset_chunks(self, transaction, s3_dataset_id, band, output):
        """Add details of chunks composing this s3 dataset to DB.

        :param transaction: Postgres transaction.
        :param uuid s3_dataset_id: The uuid of the s3 dataset.
        :param str band: The band to index this set for.
        :param dict output: Dictionary of metadata consigning
          the s3 storage information for that band.
        """
        for key_map in output['key_maps']:
            micro_shape = [chunk_dim.stop - chunk_dim.start for chunk_dim in key_map['chunk']]
            # Convert index_min and index_max to scalars
            index_min = list(map(np.asscalar, key_map['index_min']))
            index_max = list(map(np.asscalar, key_map['index_max']))

            _LOG.debug('put_s3_dataset_chunk(%s, %s, %s, %s, %s, %s, %s)',
                       s3_dataset_id, key_map['s3_key'],
                       key_map['chunk_id'], key_map['compression'], micro_shape,
                       index_min, index_max)
            self._put_s3_dataset_chunk(transaction,
                                       s3_dataset_id,
                                       key_map['s3_key'],
                                       key_map['chunk_id'],
                                       key_map['compression'],
                                       micro_shape,
                                       index_min,
                                       index_max)

    def _add_s3_dataset_mappings(self, transaction, s3_dataset_id, band, dataset_refs):
        """Add mappings between postgres datsets and s3 datasets to DB.

        :param transaction: Postgres transaction.
        :param UUID s3_dataset_id: The uuid of the s3 dataset.
        :param str band: The band to index this set for.
        :param list dataset_refs: The list of dataset references
          (uuids) that all point to the s3 dataset entry being
          created.
        """
        for dataset_ref in dataset_refs:
            _LOG.debug('put_s3_mapping(%s, %s, %s)', dataset_ref, band, s3_dataset_id)
            self._put_s3_mapping(transaction,
                                 dataset_ref,
                                 band,
                                 s3_dataset_id)

    def add_datasets_to_s3_tables(self, dataset_refs, storage_metadata):
        """Add extra dataset metadata to s3 tables.

        :param list dataset_refs: The list of dataset references
          (uuids) that all point to the s3 dataset entry being
          created.
        :param dict storage_metadata: Dictionary of metadata consigning
          the s3 storage information.
        """
        # Roll back if any exception arise
        with self._db.begin() as transaction:
            for band, output in storage_metadata.items():
                # Create a random UUID for this s3 dataset/band pair
                s3_dataset_id = uuid4()

                # Add s3 dataset
                self._add_s3_dataset(transaction, s3_dataset_id, band, output)

                # Add chunks
                self._add_s3_dataset_chunks(transaction, s3_dataset_id, band, output)

                # Add mappings
                self._add_s3_dataset_mappings(transaction, s3_dataset_id, band, dataset_refs)

    def _make(self, dataset_res, full_info=False):
        """
        :rtype Dataset

        :param bool full_info: Include all available fields
        """
        dataset = super(DatasetResource, self)._make(dataset_res, full_info)
        self._extend_dataset_with_s3_metadata(dataset)
        return dataset

    def _extend_dataset_with_s3_metadata(self, dataset):
        """Extend the dataset doc with driver specific index data.

        This methods extends the dataset document with a `s3_metadata`
        variable containing the s3 indexing metadata.

        The dataset is modified in place.

        :param :cls:`datacube.model.Dataset` dataset: The dataset to
          add NetCDF-specific indexing data to.
        """
        dataset.s3_metadata = {}
        if dataset.measurements:
            with self._db.begin() as transaction:
                for band in dataset.measurements.keys():
                    s3_datasets = self._get_s3_dataset(transaction, dataset.id, band)
                    for s3_dataset in s3_datasets:
                        dataset.s3_metadata[band] = {
                            's3_dataset': s3_dataset,
                            # TODO(csiro): commenting this out for now, not using it yet.
                            # 's3_chunks': transaction.get_s3_dataset_chunk(s3_dataset.id)
                        }

    ### S3 specific functions
    # See .tables for description of each column
    def _put_s3_mapping(self, _connection, dataset_ref, band, s3_dataset_id):
        """:type dataset_ref: uuid.UUID
        :type band: str
        :type s3_dataset_id: uuid.UUID"""
        res = _connection.execute(
            S3_DATASET_MAPPING.insert().values(
                id=uuid4(),
                dataset_ref=dataset_ref,
                band=band,
                s3_dataset_id=s3_dataset_id,
            )
        )
        return res.inserted_primary_key[0]

    # pylint: disable=too-many-arguments
    def _put_s3_dataset(self, _connection, s3_dataset_id, base_name, band, bucket,
                        macro_shape, chunk_size, numpy_type,
                        dimensions, regular_dims, regular_index,
                        irregular_index):
        """:type s3_dataset_id: uuid.UUID
        :type base_name: str
        :type band: str
        :type bucket: str
        :type macro_shape: array[int]
        :type chunk_size: array[int]
        :type numpy_type: str
        :type dimensions: array[str]
        :type regular_dims: array[bool]
        :type regular_index: array[array[float]]
        :type irregular_index: array[array[float]]"""
        res = _connection.execute(
            S3_DATASET.insert().values(
                id=s3_dataset_id,
                base_name=base_name,
                band=band,
                bucket=bucket,
                macro_shape=macro_shape,
                chunk_size=chunk_size,
                numpy_type=numpy_type,
                dimensions=dimensions,
                regular_dims=regular_dims,
                regular_index=regular_index,
                irregular_index=irregular_index
            )
        )

        return res.inserted_primary_key[0]

    def _get_s3_dataset(self, _connection, dataset_ref, band):
        """:type dataset_ref: uuid.UUID
        :type band: str"""
        return _connection.execute(
            select(
                [S3_DATASET.c.id,
                 S3_DATASET.c.base_name,
                 S3_DATASET.c.band,
                 S3_DATASET.c.bucket,
                 S3_DATASET.c.macro_shape,
                 S3_DATASET.c.chunk_size,
                 S3_DATASET.c.numpy_type,
                 S3_DATASET.c.dimensions,
                 S3_DATASET.c.regular_dims,
                 S3_DATASET.c.regular_index,
                 S3_DATASET.c.irregular_index]
            ).select_from(
                S3_DATASET.join(S3_DATASET_MAPPING,
                                S3_DATASET_MAPPING.c.s3_dataset_id == S3_DATASET.c.id,
                                isouter=True)
            ).where(
                and_(
                    S3_DATASET_MAPPING.c.dataset_ref == dataset_ref,
                    S3_DATASET_MAPPING.c.band == band
                )
            )
        ).fetchall()

    def _put_s3_dataset_chunk(self, _connection, s3_dataset_id, s3_key,
                              chunk_id, compression_scheme,
                              micro_shape, index_min, index_max):
        """:type s3_dataset_id: uuid.UUID
        :type s3_key: str
        :type chunk_id: str
        :type compression_scheme: str
        :type micro_shape: array[int]
        :type index_min: array[float]
        :type index_max: array[float]"""
        res = _connection.execute(
            S3_DATASET_CHUNK.insert().values(
                id=uuid4(),
                s3_dataset_id=s3_dataset_id,
                s3_key=s3_key,
                chunk_id=chunk_id,
                compression_scheme=compression_scheme,
                micro_shape=micro_shape,
                index_min=index_min,
                index_max=index_max
            )
        )
        return res.inserted_primary_key[0]

    def _get_s3_dataset_chunk(self, _connection, s3_dataset_id):
        """:type s3_dataset_id: uuid.UUID"""
        return _connection.execute(
            select(
                [S3_DATASET_CHUNK.c.s3_key,
                 S3_DATASET_CHUNK.c.chunk_id,
                 S3_DATASET_CHUNK.c.compression_scheme,
                 S3_DATASET_CHUNK.c.micro_shape,
                 S3_DATASET_CHUNK.c.index_min,
                 S3_DATASET_CHUNK.c.index_max]
            ).where(
                S3_DATASET_CHUNK.c.s3_dataset_id == s3_dataset_id,
            )
        ).fetchall()

max_stars_repo_name: eric-erki/datacube-core

max_issues_repo_name: eric-erki/datacube-core

max_forks_repo_name: eric-erki/datacube-core


text:
# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors and The HugginFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""BERT finetuning runner."""
import sys
import os
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)

import time

import logging
import os
import random

import numpy as np
import torch

from pytorch_pretrained_bert.tokenization import BertTokenizer
from pytorch_pretrained_bert.optimization import BertAdam
from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE
from dcmn_pytorch.model import BertForMultipleChoiceWithMatch

from dcmn_pytorch.utils import  get_arg_parser
from dcmn_pytorch.preprocess import get_dataloader
from dcmn_pytorch.train import train_valid

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)
TASK_NAME = 'mctest' #'roc' 'coin' 'race' 'mctest'
NUM_CHOICE = 4

def main():
    parser = get_arg_parser()
    args = parser.parse_args()
    logger.info(args)
    output_eval_file = os.path.join(args.output_dir, args.output_file)

    if os.path.exists(output_eval_file) and args.output_file != 'output_test.txt':
        raise ValueError("Output file ({}) already exists and is not empty.".format(output_eval_file))
    with open(output_eval_file, "w") as writer:
        writer.write("***** Eval results Epoch  %s *****\t\n" % (
            time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))))
        writer.write("%s\t\n" % args)

    args.model_name = args.output_file[:-4] + ".bin"
    # print(args.model_name)

    if args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        n_gpu = args.n_gpu
    else:
        torch.cuda.set_device(args.gpu_id)
        device = torch.device("cuda", args.gpu_id)
        n_gpu = args.n_gpu
        # Initializes the distributed backend which will take care of sychronizing nodes/GPUs

    if args.gradient_accumulation_steps < 1:
        raise ValueError("Invalid gradient_accumulation_steps parameter: {}, should be >= 1".format(
            args.gradient_accumulation_steps))

    args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)

    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

    tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)

    train_dataloader, train_examples_size  = get_dataloader(data_dir=args.data_dir, data_file=args.train_file, num_choices=args.num_choices,
                                      tokenizer=tokenizer, max_seq_length=args.max_seq_length, batch_size=args.train_batch_size)
    num_train_steps = int(
        train_examples_size / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
    t_total = num_train_steps

    eval_dataloader, eval_examples_size = get_dataloader(data_dir=args.data_dir, data_file=args.test_file,
                                                           num_choices=args.num_choices,
                                                           tokenizer=tokenizer, max_seq_length=args.max_seq_length,
                                                           batch_size=args.eval_batch_size)

    model = BertForMultipleChoiceWithMatch.from_pretrained(args.bert_model,
                                                           cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format(-1),
                                                           num_choices=args.num_choices)

    model.to(device)

    param_optimizer = list(model.named_parameters())

    # hack to remove pooler, which is not used
    # thus it produce None grad that break apex
    param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]

    no_decay = ['bias', 'gamma', 'beta']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},
        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}
    ]

    optimizer = BertAdam(optimizer_grouped_parameters,
                         lr=args.learning_rate,
                         warmup=args.warmup_proportion,
                         t_total=t_total)

    train_valid(model, args.num_train_epochs, train_dataloader, eval_dataloader, device,
                optimizer, t_total, args.gradient_accumulation_steps, args.warmup_proportion, args.learning_rate,
                args.output_dir, args.output_file, args.model_name)

if __name__ == '__main__':
    main()

max_stars_repo_name: cscyuge/medlane

max_issues_repo_name: cscyuge/medlane

max_forks_repo_name: cscyuge/medlane


text:
"""
Exemple de lancement de ce programme:
    python3 server.py --archi data/MLP_3x3x4_2018-12-09-01-29-58.json --weights data/MLP_3x3x4_2018-12-09-01-29-58.hdf5
"""

try:
    import sys
    import socket
    import getopt
    import struct
    from agentModelAdapter import AgentModelAdapter
    from neuralController import NeuralVelocityController
    from threading import Thread, Lock
    import numpy as np
except ImportError as err:
    print(err)
    sys.exit(1)

class Server():
    # Constructor
    def __init__(self, maxNbrOfClients, address, port):
        self.__maxNbrOfClients = maxNbrOfClients
        self.__address        = address
        self.__port           = port
        self.__socket         = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.__threads        = []
        self.__threadId       = []
        self.__state          = ""
        self.__mutex          = Lock()

        # Uncomment for complete map
        # Inputs
        self.__maxX, self.__minX     = 3038.19, 928.88
        self.__maxY, self.__minY     = 543.43, -2562.1
        # Outputs
        self.__maxVx, self.__minVx   = 790.65, -440.0
        self.__maxVy, self.__minVy   = 759.32, -779.81
        self.__maxVz, self.__minVz   = 328.52, -500.23

    # This method is the main method of the class Server
    def run(self):
        print("running")

        opts, args = self.__parseCommandLine()

        self.__behaviorController = NeuralVelocityController()
        self.__behaviorController.configure(opts, args)
        self.__behaviorController.build()

        # Bind socket
        self.__socket.bind((self.__address, self.__port))
        # Listen to maxNbrOfClients clients
        self.__socket.listen(self.__maxNbrOfClients)

        #thread_stats = Thread(target=self.__stats).start()
        #self.__threads.append(thread_stats)
        #self.__threadId.append("stats")

        while (self.__state != "stop"):
            client, address = self.__socket.accept()

            if ((address[0] == '' or address[0] == 'localhost' or address[0] == '127.0.0.1') and (len(self.__threads) <= self.__maxNbrOfClients)):
                print("new client : " + address[0])
                #thread_client = Thread(target=self.__runClient, args=(client,)).start()
                self.__runClient(client)
                #self.__threads.append(thread_client)
                #self.__threadId.append("client")
            else:
                client.close()

    def __stats(self):
        while (self.__state != "stop"):
            self.__state = sys.stdin.readline().split("\n")[0]
            if (self.__state == "nbrOfClients"):
                print(len(self.__threads) - 1)
        self.__close()

    def __parseCommandLine(self):
        try:
            opts, args = getopt.getopt(sys.argv[1:], "halw:v", ["help", "archi=", "length=", "weights="])
        except getopt.GetoptError as err:
            print(str(err))
            sys.exit(2)
        verbose = False
        for opt, arg in opts:
            if opt == "-v":
                verbose = True
            elif opt in ("-h", "--help"):
                sys.exit()

        return opts, args

    def __runClient(self, client):
        print("runClient")

        while (self.__state != "stop"):
            try:
                req = client.recv(8192)
                if req != "":
                    # Print the request
                    self.__mutex.acquire()
                    values = self.__splitData(req)
                    result = self.__processData(values)
                    self.__mutex.release()
                    client.send(bytes("[", "utf-8")
                                + bytes(str(result[0]),"utf-8")
                                + bytes(" ", "utf-8")
                                + bytes(str(result[1]),"utf-8")
                                + bytes(" ", "utf-8")
                                + bytes(str(result[2]),"utf-8")
                                + bytes("]", "utf-8")
                                + bytes("\n", "utf-8")) # MUST NOT forget end of line
            except socket.error as error:
                client.close()
                break

        # Close the connection
        client.close()

    def __processData(self, data):
        result = self.__behaviorController.process(data)
        vx, vy, vz = self.__transformOutputDatas(result[0], result[1], result[2])
        print(str(data) + ' --> ' + str(vx) + ', ' + str(vy)+ ', ' + str(vz))
        return [vx, vy, vz]

    def __splitData(self, data):
        print(data)

        data = data.decode("utf-8")
        d = data.split("[")[1].split(']')[0]
        splitValues = d.split(" ")

        # Normalize input datas
        x, y = self.__normalizeInputDatas(float(splitValues[0]), float(splitValues[1]))

        values = [x, y]
        datas = np.array([values])

        return datas

    def __normalizeInputDatas(self, x, y):
        if x > self.__maxX:
            x = self.__maxX
        elif x < self.__minX:
            x = self.__minX
        nx = (x - self.__minX) / (self.__maxX - self.__minX)

        if y > self.__maxY:
            y = self.__maxY
        elif y < self.__minY:
            y = self.__minY
        ny = (y - self.__minY) / (self.__maxY - self.__minY)

        return nx, ny

    def __transformOutputDatas(self, vx, vy, vz):
        tvx = ((vx + 1)/2)*(self.__maxVx - self.__minVx) + self.__minVx
        tvy = ((vy + 1)/2)*(self.__maxVy - self.__minVy) + self.__minVy
        tvz = ((vz + 1)/2)*(self.__maxVz - self.__minVz) + self.__minVz
        return tvx, tvy, tvz

    def __close(self):
        # Close the main socket
        self.__socket.close()

        for i in range (0, len(self.__threads)):
            if (self.__threadId == "client"):
                self.__threads[i].join()

        for i in range (0, len(self.__threads)):
            if (self.__threadId == "client"):
                self.__threads[i].close()

if __name__ == "__main__":
    # Instatiate the server
    server = Server(8, 'localhost', 12400)
    # Run it
    server.run()

max_stars_repo_name: etiennelndr/neo

max_issues_repo_name: etiennelndr/neo

max_forks_repo_name: etiennelndr/neo


text:
from model import *
import cv2
import numpy as np

h = 256
w = 256

x_ = tf.placeholder(tf.float32, [None, w*h*1])
x_image = tf.reshape(x_, [-1, h, w, 1])
enhanced = Generator(x_image)

with tf.Session() as sess:

    saver = tf.train.Saver()
    saver.restore(sess, "./para/" + "model.ckpt")

    for i in range(1000):
        I = cv2.imread("./test/distortion/"+str(i)+".jpg", cv2.IMREAD_GRAYSCALE)
        II = np.float32(np.reshape(I, [1, h, w, 1])) / 255
        enhanced_2d = sess.run(enhanced, feed_dict={x_image: II})
        crop1 = np.reshape(enhanced_2d, [h, w, 1])
        cv2.imwrite("res/" + str(i) + ".jpg", crop1 * 255)


text:
import pandas as pd
import numpy as np
import xml.etree.ElementTree as ElementTree

from traffic_analysis.d05_evaluation.parse_annotation import parse_annotation

class VideoLevelEvaluator:
    """
    Purpose of this class is to conduct video level evaluation for multiple videos.
    """
    def __init__(self,
                 videos_to_eval: pd.DataFrame,
                 video_level_df: pd.DataFrame,
                 video_level_column_order: list,
                 selected_labels: list):

        # data frames to work with
        self.videos_to_eval = videos_to_eval
        self.video_level_df = video_level_df
        self.video_level_ground_truth = pd.DataFrame({})

        # parameters
        self.video_level_column_order = video_level_column_order
        self.selected_labels = selected_labels
        self.stats_to_evaluate = ['counts', 'starts', 'stops']

    def evaluate(self) -> (pd.DataFrame, pd.DataFrame):
        """Get video level evaluation results
        Returns:
          performance_df -- data frame summarising the performance
          diff_df -- data frame with raw differences of
                     predictions vs ground truth
        """
        self.video_level_ground_truth = self.get_ground_truth()
        diff_df = self.compute_diffs()
        performance_df = self.summarise_performance(diff_df)
        return performance_df, diff_df

    def compute_diffs(self) -> pd.DataFrame:
        """Get raw diffs of pred minus true for each statistic we are
        evaluating.

        Returns:
            diff_df -- data frame with raw differences of
                       predictions vs ground truth
        """
        # get data sets
        video_level_ground_truth = self.video_level_ground_truth
        video_level_estimates = pd.merge(left=self.videos_to_eval[['camera_id', 'video_upload_datetime']],
                                         right=self.video_level_df,
                                         on=['camera_id', 'video_upload_datetime'],
                                         how='inner')

        # calculate diffs
        truth_rename_dict = {stat: stat + '_true' for stat in self.stats_to_evaluate}
        pred_rename_dict = {stat: stat + '_pred' for stat in self.stats_to_evaluate}
        video_level_ground_truth.rename(columns=truth_rename_dict, inplace=True)
        video_level_estimates.rename(columns=pred_rename_dict, inplace=True)

        id_cols = ['camera_id', 'video_upload_datetime', 'vehicle_type']
        diff_df = pd.merge(left=video_level_ground_truth[id_cols + list(truth_rename_dict.values())],
                           right=video_level_estimates[id_cols + list(pred_rename_dict.values())],
                           on=id_cols,
                           how='left').fillna(0)
        for stat in self.stats_to_evaluate:
            diff_df[stat + '_diff'] = diff_df[stat + '_pred'] - diff_df[stat + '_true']
        return diff_df

    def summarise_performance(self,
                              diff_df: pd.DataFrame) -> pd.DataFrame:
        # reshape for convenient summarising
        id_cols = ['camera_id', 'video_upload_datetime', 'vehicle_type']
        diff_cols = [stat + '_diff' for stat in self.stats_to_evaluate]
        diff_melted = diff_df.melt(id_vars=id_cols,
                                   value_vars=diff_cols,
                                   value_name='stat_diff',
                                   var_name='stat')
        diff_melted['stat'] = diff_melted['stat'].str.replace("_diff", "")

        # summarise
        performance_df = (diff_melted
                          .groupby(['vehicle_type', 'stat'])
                          .stat_diff
                          .agg({'bias': np.mean,
                                'MAE': lambda x: np.mean(abs(x)),
                                'RMSE': lambda x: np.sqrt(np.mean(x ** 2)),
                                'sd': np.std,
                                'n_videos': 'count'})
                          .reset_index())

        return performance_df

    def get_ground_truth(self):
        """Parse ground truth XMLs into corresponding ground truth
        video level dataframe
        """

        video_level_ground_truth_list = []
        for idx, video in self.videos_to_eval.iterrows():
            # get frame level ground truth
            xml_root = ElementTree.parse(video['xml_path']).getroot()
            frame_level_ground_truth = parse_annotation(xml_root)

            # get video level ground truth
            video_level_ground_truth = self.compute_video_level_ground_truth(frame_level_ground_truth)

            video_level_ground_truth['camera_id'] = video['camera_id']
            video_level_ground_truth['video_upload_datetime'] = video['video_upload_datetime']
            video_level_ground_truth_list.append(video_level_ground_truth)

        video_level_ground_truth = pd.concat(video_level_ground_truth_list, axis=0)

        missing_cols = list(set(self.video_level_column_order)
                            - set(video_level_ground_truth.columns))
        for column_name in missing_cols:
            video_level_ground_truth[column_name] = 0

        return video_level_ground_truth

    def compute_video_level_ground_truth(self,
                                         frame_level_ground_truth: pd.DataFrame) -> pd.DataFrame:
        """Parse frame level ground truth dataframe into a video level ground
        truth dataframe. Gets parked, starts, stops, counts
        """

        # prepare for evaluation
        ground_truth = frame_level_ground_truth.sort_values(['vehicle_id', 'frame_id'])
        ground_truth['parked'] = (ground_truth['parked']
                                  .replace({'false': False,
                                            'true': True}))
        ground_truth['stopped'] = (ground_truth['stopped']
                                   .replace({'false': False,
                                             'true': True}))

        ground_truth['motion_change'] = ((ground_truth['stopped'] -
                                          (ground_truth
                                           .groupby('vehicle_id')
                                           .stopped
                                           .shift(1)))
                                         .fillna(0)  # the first frame does not count as motion change
                                         )
        ground_truth['starts'] = (ground_truth['motion_change'] == -1).astype(int)
        ground_truth['stops'] = (ground_truth['motion_change'] == 1).astype(int)

        # aggregate to vehicle level
        vehicle_df = (ground_truth
                      .groupby('vehicle_id')
                      ['vehicle_type', 'parked']
                      .first()  # take initial status
                      .reset_index())

        start_stop = (ground_truth
                      .groupby(['vehicle_id'])
                      ['starts', 'stops']
                      .sum()
                      .reset_index())

        vehicle_df = pd.merge(left=vehicle_df,
                              right=start_stop,
                              on='vehicle_id',
                              how='inner')

        # aggregate to video level
        motion_df = (vehicle_df
                     .groupby('vehicle_type')
                     ['parked', 'starts', 'stops']
                     .sum()
                     .reset_index()
                     )
        motion_df['parked'] = motion_df['parked'].astype(int)

        count_df = (vehicle_df
                    .groupby('vehicle_type')
                    .vehicle_id
                    .count()
                    .reset_index()
                    .rename(columns={'vehicle_id': 'counts'})
                    )
        video_level_ground_truth = pd.merge(left=count_df,
                                            right=motion_df,
                                            on='vehicle_type',
                                            how='inner')

        # guarantee that all required types are reported
        all_types = pd.DataFrame({'vehicle_type': self.selected_labels})
        video_level_ground_truth = pd.merge(left=all_types,
                                            right=video_level_ground_truth,
                                            on='vehicle_type',
                                            how='left')
        for col in ['counts', 'parked', 'starts', 'stops']:
            video_level_ground_truth[col] = video_level_ground_truth[col].fillna(0).astype(int)

        return video_level_ground_truth


text:
# +
import sys
sys.path.append("..")

import minitorch
from hypothesis import settings
from hypothesis.strategies import composite, floats, integers, lists
import numpy as np
# -

settings.register_profile("ci", deadline=None)
settings.load_profile("ci")

small_ints = integers(min_value=1, max_value=3)
small_floats = floats(min_value=-100, max_value=100)

@composite
def vals(draw, size, number):
    pts = draw(lists(number, min_size=size, max_size=size,))
    return minitorch.tensor(pts)

@composite
def scalars(draw, min_value=-100000, max_value=100000):
    val = draw(floats(min_value=min_value, max_value=max_value))
    return minitorch.Scalar(val)

@composite
def shapes(draw):
    lsize = draw(lists(small_ints, min_size=1, max_size=4))
    return tuple(lsize)

@composite
def tensor_data(draw, numbers=floats()):
    shape = draw(shapes())
    size = int(minitorch.prod(shape))
    data = draw(lists(numbers, min_size=size, max_size=size))
    return minitorch.TensorData(data, shape)

@composite
def indices(draw, layout):
    return tuple((draw(integers(min_value=0, max_value=s - 1)) for s in layout.shape))

@composite
def tensors(draw, numbers=floats(allow_nan=False, min_value=-100, max_value=100)):
    td = draw(tensor_data(numbers))
    return minitorch.Tensor(td)

@composite
def shaped_tensors(
    draw, n, numbers=floats(allow_nan=False, min_value=-100, max_value=100)
):
    td = draw(tensor_data(numbers))
    values = []
    for i in range(n):
        data = draw(lists(numbers, min_size=td.size, max_size=td.size))
        values.append(minitorch.Tensor(minitorch.TensorData(data, td.shape)))
    return values

@composite
def matmul_tensors(
    draw, numbers=floats(allow_nan=False, min_value=-100, max_value=100)
):

    i, j, k = [draw(integers(min_value=1, max_value=10)) for _ in range(3)]

    l1 = (i, j)
    l2 = (j, k)
    values = []
    for shape in [l1, l2]:
        size = int(minitorch.prod(shape))
        data = draw(lists(numbers, min_size=size, max_size=size))
        values.append(minitorch.Tensor(minitorch.TensorData(data, shape)))
    return values

def assert_isclose(a, b):
    return np.testing.assert_allclose(a, b, 1e-2, 1e-2)

max_stars_repo_name: chris-tng/minitorch

max_issues_repo_name: chris-tng/minitorch

max_forks_repo_name: chris-tng/minitorch


text:
from UpdatedImageProcessing.ShapeDetection import utils
from .settings import ShapeDetectionSettings
#from settings import ShapeDetectionSettings
#from utils import alpha_trace
#from utils import init_harris_corners_and_cluster
#from utils.polar_side_counter import PolarSideCounter
#from utils.simple_pca import SimplePCA
#from utils.hough_lines import HoughSideCounter
from PIL import Image
import random
import numpy

class ShapeClassification(object):
    def __init__(self, path_to_cropped_img_in):
        self.load_images(path_to_cropped_img_in)
        self.load_polar_side_counter()
        self.determine_shape()

    def load_images(self, path_to_cropped_img_in):
        self.pil_img = Image.open(path_to_cropped_img_in)
        self.monochrome_pil_img = Image.open(path_to_cropped_img_in).convert("L")
        self.canny_img = utils.alpha_trace(self.pil_img)

    def load_polar_side_counter(self):
        self.polar_side_counter = utils.PolarSideCounter(self.canny_img)
        self.polar_side_maximums = self.polar_side_counter.get_polar_side_maximums()
        self.num_polar_side_maximums = len(self.polar_side_maximums)
        self.polar_side_minimums = self.polar_side_counter.get_polar_side_minimums()
        self.num_polar_side_minimums = len(self.polar_side_counter.get_polar_side_minimums())
        self.circle_score = self.polar_side_counter.get_circle_score()
        self.noise_score = self.polar_side_counter.get_noise_score()
        self.origin = self.polar_side_counter.get_origin()

    def determine_shape(self):
        if self.circle_score >= ShapeDetectionSettings.CIRCLE_SCORE_THRESHOLD:
            self.shape_type = "circle"
        elif self.noise_score >= ShapeDetectionSettings.NOISE_SCORE_THRESHOLD:
            self.shape_type = "NOISE"
        elif len(utils.HoughSideCounter(self.pil_img, self.canny_img, ShapeDetectionSettings.QUARTER_CIRCLE_HOUGH_THRESHOLD).get_sides()) == 2:
            self.shape_type = "quarter_circle"
        elif self.num_polar_side_maximums not in range(2,8):
            #panic -- chose random shape
            self.shape_type = random.choice(ShapeDetectionSettings.SHAPE_CHOICES)
            self.shape_type = "random"
        elif self.num_polar_side_maximums == 2:
            self.shape_type = "semicircle"
        elif self.num_polar_side_maximums == 3:
            self.shape_type = "triangle"
        elif self.num_polar_side_maximums == 6:
            self.shape_type = "hexagon"
        elif self.num_polar_side_maximums == 7:
            self.shape_type = "heptagon"
        else:
            if self.num_polar_side_maximums == 4:

                bounding_box = utils.BoundingBox(self.pil_img)

                if bounding_box.get_side_length_difference() < ShapeDetectionSettings.SQUARE_SIDE_LENGTH_THRESHOLD:
                    self.shape = "square"

                else:
                    if bounding_box.get_area_difference() > ShapeDetectionSettings.TRAPEZOID_AREA_THRESHOLD:
                        self.shape_type = "trapezoid"
                    else:
                        self.shape_type = "rectangle"

            else:
                convex_corners = utils.convex_corners(self.pil_img)

                if self.num_polar_side_maximums == 5:
                    if convex_corners == 5:
                        self.shape_type = "star"
                    else:
                        self.shape_type = "pentagon"
                elif self.num_polar_side_maximums == 8:
                    num_harris_clusters = init_harris_corners_and_cluster(self.monochrome_pil_img, self.polar_side_maximums, self.polar_side_minimums, self.origin)
                    if num_harris_clusters < ShapeDetectionSettings.OCTAGON_CROSS_CLUSTER_THRESHOLD:
                        self.shape_type = "octagon"
                    else:
                        self.shape_type = "cross"

    def get_shape_type(self):
        return self.shape_type


text:
"""Extensions to dataset classes from TAPE Repository: https://github.com/songlab-cal/tape
Date         Change
----------   ---------------------
05/01/2020   Added Binding Site dataset class
             Added one-vs-all Secondary structure dataset class

"""

from pathlib import Path
from typing import Union, List, Tuple, Sequence, Dict, Any

import numpy as np
import torch
from scipy.spatial.distance import pdist, squareform
from tape.datasets import dataset_factory
from tape.tokenizers import TAPETokenizer
from torch.utils.data import Dataset
from transformers import BertTokenizer, XLNetTokenizer, AlbertTokenizer

ss8_cds = ['G', 'H', 'I', 'B', 'E', 'S', 'T', ' ']
ss8_to_idx = {cd: i for i, cd in enumerate(ss8_cds)}

ss8_blank_index = 7
ss4_blank_index = 3

class SecondaryStructureOneVsAllDataset(Dataset):

    def __init__(self,
                 data_path: Union[str, Path],
                 split: str,
                 label_scheme: str,
                 label: str,
                 tokenizer: Union[str, TAPETokenizer, BertTokenizer, XLNetTokenizer, AlbertTokenizer] = 'iupac',
                 in_memory: bool = False,
                 max_seqlen: int = 512):

        if label_scheme != 'ss8' and label_scheme != 'ss4':
            raise NotImplementedError

        if split not in ('train', 'valid', 'casp12', 'ts115', 'cb513'):
            raise ValueError(f"Unrecognized split: {split}. Must be one of "
                             f"['train', 'valid', 'casp12', "
                             f"'ts115', 'cb513']")

        if isinstance(tokenizer, str):
            tokenizer = TAPETokenizer(vocab=tokenizer)
        self.tokenizer = tokenizer

        data_path = Path(data_path)
        data_file = f'secondary_structure/secondary_structure_{split}.lmdb'
        self.data = dataset_factory(data_path / data_file, in_memory)
        if label_scheme == 'ss8':
            self.label = ss8_to_idx[label]
        elif label_scheme == 'ss4':
            self.label = label
        else:
            raise NotImplementedError
        self.label_scheme = label_scheme
        self.max_seqlen = max_seqlen

    def __len__(self) -> int:
        return len(self.data)

    def __getitem__(self, index: int):
        item = self.data[index]
        sequence = item['primary']
        if self.max_seqlen:
            sequence = sequence[:self.max_seqlen]
        if isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, XLNetTokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            token_ids = np.array(self.tokenizer.encode(list(sequence)), np.int64)
        elif isinstance(self.tokenizer, TAPETokenizer):
            token_ids = self.tokenizer.encode(sequence)
        else:
            raise NotImplementedError
        input_mask = np.ones_like(token_ids)

        if self.label_scheme == 'ss4':
            # ss8 code 7 is for blank label. 3 is used to represent blank in ss4
            ss_labels = [ss4_blank_index if ss8 == ss8_blank_index else ss3 for ss3, ss8 in
                         zip(item['ss3'], item['ss8'])]
        else:
            ss_labels = item['ss8']
        if self.max_seqlen:
            ss_labels = ss_labels[:self.max_seqlen]
        labels = np.asarray([label == self.label for label in ss_labels], np.int64)
        if isinstance(self.tokenizer, XLNetTokenizer):
            # pad with two -1s at end because of sep/cls tokens
            labels = np.pad(labels, (0, 2), 'constant', constant_values=-1)
        elif isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, TAPETokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            # pad with -1s at beginning and end because of cls/sep tokens
            labels = np.pad(labels, (1, 1), 'constant', constant_values=-1)
        else:
            raise NotImplementedError

        return token_ids, input_mask, labels

    def collate_fn(self, batch: List[Tuple[Any, ...]]) -> Dict[str, torch.Tensor]:
        input_ids, input_mask, ss_label = tuple(zip(*batch))
        input_ids = torch.from_numpy(pad_sequences(input_ids, 0))
        input_mask = torch.from_numpy(pad_sequences(input_mask, 0))
        ss_label = torch.from_numpy(pad_sequences(ss_label, -1))

        output = {'input_ids': input_ids,
                  'input_mask': input_mask,
                  'targets': ss_label}

        return output

class BindingSiteDataset(Dataset):

    def __init__(self,
                 data_path: Union[str, Path],
                 split: str,
                 tokenizer: Union[str, TAPETokenizer, BertTokenizer] = 'iupac',
                 in_memory: bool = False,
                 max_seqlen: int = 512):

        allowed_splits = ('train', 'valid')
        if split not in allowed_splits:
            raise ValueError(f"Unrecognized split: {split}. Must be one of: {', '.join(allowed_splits)}")

        if isinstance(tokenizer, str):
            tokenizer = TAPETokenizer(vocab=tokenizer)
        self.tokenizer = tokenizer

        data_path = Path(data_path)
        data_file = f'binding_sites/binding_site_{split}.lmdb'
        self.data = dataset_factory(data_path / data_file, in_memory)
        self.max_seqlen = max_seqlen

    def __len__(self) -> int:
        return len(self.data)

    def __getitem__(self, index: int):
        item = self.data[index]
        sequence = item['primary']
        positions = item['positions']
        if self.max_seqlen:
            sequence = sequence[:self.max_seqlen]
            positions = positions[:self.max_seqlen]

        if isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, XLNetTokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            token_ids = np.array(self.tokenizer.encode(list(sequence)), np.int64)
        elif isinstance(self.tokenizer, TAPETokenizer):
            token_ids = self.tokenizer.encode(sequence)
        else:
            raise NotImplementedError

        input_mask = np.ones_like(token_ids)

        labels = [1 if seq_pos in item['sites'] else 0 for seq_pos in positions]

        if isinstance(self.tokenizer, XLNetTokenizer):
            # pad with two -1s at end because of sep/cls tokens
            labels = np.pad(labels, (0, 2), 'constant', constant_values=-1)
        elif isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, TAPETokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            # pad with -1s at beginning and end because of cls/sep tokens
            labels = np.pad(labels, (1, 1), 'constant', constant_values=-1)
        else:
            raise NotImplementedError

        return token_ids, input_mask, labels

    def collate_fn(self, batch: List[Tuple[Any, ...]]) -> Dict[str, torch.Tensor]:
        input_ids, input_mask, label = tuple(zip(*batch))
        input_ids = torch.from_numpy(pad_sequences(input_ids, 0))
        input_mask = torch.from_numpy(pad_sequences(input_mask, 0))
        label = torch.from_numpy(pad_sequences(label, -1))

        output = {'input_ids': input_ids,
                  'input_mask': input_mask,
                  'targets': label}

        return output

class ProteinModificationDataset(Dataset):

    def __init__(self,
                 data_path: Union[str, Path],
                 split: str,
                 tokenizer: Union[str, TAPETokenizer, BertTokenizer, XLNetTokenizer] = 'iupac',
                 in_memory: bool = False,
                 max_seqlen: int = 512):

        allowed_splits = ('train', 'valid')
        if split not in allowed_splits:
            raise ValueError(f"Unrecognized split: {split}. Must be one of: {', '.join(allowed_splits)}")

        if isinstance(tokenizer, str):
            tokenizer = TAPETokenizer(vocab=tokenizer)
        self.tokenizer = tokenizer

        data_path = Path(data_path)
        data_file = f'protein_modifications/protein_modification_{split}.lmdb'
        self.data = dataset_factory(data_path / data_file, in_memory)
        self.max_seqlen = max_seqlen

    def __len__(self) -> int:
        return len(self.data)

    def __getitem__(self, index: int):
        item = self.data[index]
        sequence = item['primary']
        positions = item['positions']
        if self.max_seqlen:
            sequence = sequence[:self.max_seqlen]
            positions = positions[:self.max_seqlen]

        if isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, XLNetTokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            token_ids = np.array(self.tokenizer.encode(list(sequence)), np.int64)
        elif isinstance(self.tokenizer, TAPETokenizer):
            token_ids = self.tokenizer.encode(sequence)
        else:
            raise NotImplementedError

        input_mask = np.ones_like(token_ids)

        labels = [1 if seq_pos in item['modifications'] else 0 for seq_pos in positions]

        if isinstance(self.tokenizer, XLNetTokenizer):
            # pad with two -1s at end because of sep/cls tokens
            labels = np.pad(labels, (0, 2), 'constant', constant_values=-1)
        elif isinstance(self.tokenizer, BertTokenizer) or isinstance(self.tokenizer, TAPETokenizer) or \
                isinstance(self.tokenizer, AlbertTokenizer):
            # pad with -1s at beginning and end because of cls/sep tokens
            labels = np.pad(labels, (1, 1), 'constant', constant_values=-1)
        else:
            raise NotImplementedError

        return token_ids, input_mask, labels

    def collate_fn(self, batch: List[Tuple[Any, ...]]) -> Dict[str, torch.Tensor]:
        input_ids, input_mask, label = tuple(zip(*batch))
        input_ids = torch.from_numpy(pad_sequences(input_ids, 0))
        input_mask = torch.from_numpy(pad_sequences(input_mask, 0))
        label = torch.from_numpy(pad_sequences(label, -1))

        output = {'input_ids': input_ids,
                  'input_mask': input_mask,
                  'targets': label}

        return output

class ProteinnetDataset(Dataset):

    def __init__(self,
                 data_path: Union[str, Path],
                 split: str,
                 tokenizer: Union[str, TAPETokenizer] = 'iupac',
                 in_memory: bool = False,
                 max_seq_len=None):

        if split not in ('train', 'train_unfiltered', 'valid', 'test'):
            raise ValueError(f"Unrecognized split: {split}. Must be one of "
                             f"['train', 'train_unfiltered', 'valid', 'test']")

        if isinstance(tokenizer, str):
            tokenizer = TAPETokenizer(vocab=tokenizer)
        self.tokenizer = tokenizer

        data_path = Path(data_path)
        data_file = f'proteinnet/proteinnet_{split}.lmdb'
        self.data = dataset_factory(data_path / data_file, in_memory)
        self.max_seq_len = max_seq_len

    def __len__(self) -> int:
        return len(self.data)

    def __getitem__(self, index: int):
        item = self.data[index]
        primary = item['primary']
        tertiary = item['tertiary']
        valid_mask = item['valid_mask']
        if self.max_seq_len:
            primary = primary[:self.max_seq_len]
            tertiary = tertiary[:self.max_seq_len]
            valid_mask = valid_mask[:self.max_seq_len]

        protein_length = len(primary)
        token_ids = self.tokenizer.encode(primary)
        input_mask = np.ones_like(token_ids)

        contact_map = np.less(squareform(pdist(tertiary)), 8.0).astype(np.int64)

        yind, xind = np.indices(contact_map.shape)
        invalid_mask = ~(valid_mask[:, None] & valid_mask[None, :])
        invalid_mask |= np.abs(yind - xind) < 6
        contact_map[invalid_mask] = -1

        return token_ids, input_mask, contact_map, protein_length

    def collate_fn(self, batch: List[Tuple[Any, ...]]) -> Dict[str, torch.Tensor]:
        input_ids, input_mask, contact_labels, protein_length = tuple(zip(*batch))
        input_ids = torch.from_numpy(pad_sequences(input_ids, 0))
        input_mask = torch.from_numpy(pad_sequences(input_mask, 0))
        contact_labels = torch.from_numpy(pad_sequences(contact_labels, -1))
        protein_length = torch.LongTensor(protein_length)  # type: ignore

        return {'input_ids': input_ids,
                'input_mask': input_mask,
                'targets': contact_labels,
                'protein_length': protein_length}

def pad_sequences(sequences: Sequence, constant_value=0, dtype=None) -> np.ndarray:
    batch_size = len(sequences)
    shape = [batch_size] + np.max([seq.shape for seq in sequences], 0).tolist()

    if dtype is None:
        dtype = sequences[0].dtype

    if isinstance(sequences[0], np.ndarray):
        array = np.full(shape, constant_value, dtype=dtype)
    elif isinstance(sequences[0], torch.Tensor):
        array = torch.full(shape, constant_value, dtype=dtype)

    for arr, seq in zip(array, sequences):
        arrslice = tuple(slice(dim) for dim in seq.shape)
        arr[arrslice] = seq

    return array

max_stars_repo_name: salesforce/provis

max_issues_repo_name: salesforce/provis

max_forks_repo_name: salesforce/provis

reason: import numpy,from scipy


text:
#
#   Generates input for phenomenological calculation in bigstick
#   J Fox 2019
#

import numpy as np

nl = "\n"
sp = " "

Acore = 16

def make_input_dict():
    input_dict = {}
    input_dict['run_opt'] = None
    input_dict['run_name'] = None
    input_dict['sps_file'] = None
    input_dict['Zv'] = None
    input_dict['Nv'] = None
    input_dict['twoJz'] = None
    input_dict['frag_opt'] = None
    input_dict['lanc_opt'] = None
    input_dict['nkeep'] = None
    input_dict['lanc_iters'] = None
#     for i in range(n_ints)
#         input_dict['int_file_'+i] = None
#         input_dict['int_scale_'+i] = None
    return input_dict

def write_out_bigstick_input(fn,d,n_ints):
    with open(fn,'w') as fh:
        fh.write(d['run_opt']+nl)
        fh.write(d['run_name']+nl)
        fh.write(d['sps_file']+nl)
        fh.write(d['Zv'] + sp +d['Nv']+nl)
        fh.write(d['twoJz']+nl)
        if d['frag_opt'] is not None:
            fh.write(d['frag_opt']+nl)
        for i in range(n_ints):
            fh.write(d['int_file_'+str(i)]+nl)
            fh.write(d['int_scale_'+str(i)]+nl)
        fh.write('end'+nl)
        fh.write(d['lanc_opt']+nl)
        fh.write(d['nkeep'] + sp + d['lanc_iters']+nl)

def write_out_genstrength_input(fn,d,n_ints):
    with open(fn,'w') as fh:
        fh.write(d['opme']+nl)
        fh.write(d['oscb']+nl)
        fh.write(d['run_name']+nl)
        fh.write('0'+nl)
        fh.write(d['run_name']+nl)
        fh.write('0'+nl)
        fh.write(d['run_name']+nl)
        fh.write('n'+nl)
        fh.write(d['run_name']+d['opme']+nl)

if __name__=="__main__":

    input_dict = make_input_dict()

#  MPI?
    x = input("Do you want MPI? (y/n)"+nl)
    if 'y' in x:
        input_dict['frag_opt'] = '0'

# option
    opt_list = ['n','d','dp']
    x = input("Enter bigstick option"+nl+str(opt_list)+nl)
    while x not in opt_list:
        x = input("Option not available. Try again.")
    input_dict['run_opt'] = x

# name
    x = input("Enter name for run"+nl)
    input_dict['run_name'] = x

# sps
    x = input("Enter name for .sps file"+nl)
    input_dict['sps_file'] = x

# Zv Nv
    x = input("Enter Zv Nv"+nl).split()
    input_dict['Zv'] = x[0]
    input_dict['Nv'] = x[1]

# twoJz
    x = input("Enter 2*Jz"+nl)
    input_dict['twoJz'] = x

# number of ints
    n_ints = int(input("Enter number of interactions"+nl))
    for i in range(n_ints):
        input_dict['int_file_'+str(i)] = input("Enter name of .int"+nl)
        input_dict['int_scale_'+str(i)] = \
                input("Enter scaling for .int   (i.e. 1 Acore+2 A 0.33) "+nl)

# lanczos
    x = input("Enter # of states"+nl)
    input_dict["lanc_opt"] = "ld"
    input_dict['nkeep'] = x
    input_dict["lanc_iters"] = str(20*int(x))

# out
    #fn_out = input("Enter input filename:"+nl)
    fn_out = input_dict['run_name']+'.input'
    write_out_bigstick_input(fn_out,input_dict,n_ints)

#   genstrength
    x = input("Do you want a genstrength input?"+nl)
    if 'y' in x or 'Y' in x:
        pass
    else:
        exit()

#opme
    x = input('Enter name of .opme file'+nl)
    input_dict['opme'] = x

#b
    x = input('Enter scaling (enter -1 to use formula for b^2)'+nl)
    if x == -1:
        hbarc = 197.327
        mp = 938.272
        mn = 939.565
        mass = 0.5*(mp+mn)
        A = Acore + int(input_dict['Zv']) + int(input_dict['Nv'])
        hw = 45.0*A**(-0.3) - 25.0*A**(-0.6)
        #hw = 45.0*A**(-1/3) - 25.0*A**(-2/3)
        #hw = 41.0*A**(-1/3)

        b = hbarc / np.sqrt(hw*mass)
        scaling = b*b
    else:
        scaling = x

    input_dict['oscb'] = str(scaling)
# write gsin
    fn_out = input_dict['run_name']+input_dict['opme']+'.gsin'
    write_out_genstrength_input(fn_out,input_dict,n_ints)

max_stars_repo_name: cwjsdsu/scriptorium

max_issues_repo_name: cwjsdsu/scriptorium

max_forks_repo_name: cwjsdsu/scriptorium


text:
'''
    This project is done using only openCV.
    No Machine Learning or Deep Learning concepts/algos have been used.

'''
import cv2
import numpy as np

cam = cv2.VideoCapture(0)   #captures the video.

i = 0

while i<50:
    i+=1
    _, background = cam.read()
replace = cv2.resize(background,(1000,1000))

while True:
    _,frame = cam.read()
    frame = cv2.resize(frame,(1000,1000))

    hsv = cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)

    lower_blue = np.array([64,74,0])
    upper_blue = np.array([111,255,251])

    mask = cv2.inRange(hsv,lower_blue,upper_blue)
    mask_inv = cv2.bitwise_not(mask)

    IL = cv2.bitwise_and(frame, frame, mask=mask_inv)
    CL = cv2.bitwise_and(replace, replace, mask=mask)

    final_img = cv2.add(IL,CL)

    cv2.imshow('image', final_img)

    if cv2.waitKey(50) & 0xFF == ord('q'):
        break

cam.release()
cv2.destroyAllWindows()

max_stars_repo_path: Invisible cloak/inv_cloak.py

max_issues_repo_path: Invisible cloak/inv_cloak.py

max_forks_repo_path: Invisible cloak/inv_cloak.py


text:
import numpy as np
import typing

def make_release(config, fname=None):
    config = load_config(config)

    # Set seed if specified
    if 'seed' in config:
        np.random.seed(config['seed'])

    # Create release params
    import pandas as pd
    frames = [pd.DataFrame(make_single_release(c)) for c in config['groups']]
    frame = pd.concat(frames).fillna(0)

    # Sort data frame by release time
    frame = frame.sort_values('date')

    # Make column selection if specified
    if 'columns' in config:
        frame = frame[config['columns']]

    if fname:
        frame.to_csv(fname, sep="\t", header=False, index=False)

    return frame.to_dict(orient='list')

def load_config(config):
    # Is this a file name?
    config_str = None
    if isinstance(config, str):
        with open(config, encoding='utf8') as config_file:
            config_str = config_file.read()

    # Is this a stream?
    elif hasattr(config, 'read') and callable(config.read):
        config_str = config.read()

    if config_str is not None:
        import yaml
        try:
            config = yaml.safe_load(config_str)
        except yaml.YAMLError as e:
            raise ValueError(f'Error parsing yaml file: {e}') from e

    # Is this a list? If so, convert to group format
    if isinstance(config, list):
        config = dict(groups=config)

    # Is this not a dict? If not, it wrong format
    if not isinstance(config, dict):
        raise TypeError(f'Not a valid config format: {type(config)}')

    # Is this a flat format? If so, make a single group
    if 'groups' not in config:
        global_params = ['seed', 'columns']
        new_config = {k: v for k, v in config.items() if k in global_params}
        groups = [{k: v for k, v in config.items() if k not in global_params}]
        new_config['groups'] = groups
        config = new_config

    # Does any of the groups lack necessary parameters?
    necessary = ['date', 'location', 'num']
    not_present = [
        [k for k in necessary if k not in params]
        for params in config['groups']
    ]
    if any(m for m in not_present):
        messages = [
            ", ".join(p) + (
                f" in group {i}" if len(config['groups']) > 1 else '')
            for i, p in enumerate(not_present) if p
        ]
        msg = "\n  and ".join(messages)
        raise ValueError("Missing parameters: " + msg)

    # Are any of the date strings wrongly formatted?
    for g in config['groups']:
        np.array(g['date']).astype('datetime64')

    return config

def make_single_release(conf):
    # Enumerate explicit and implicit attributes
    special_keys = ['num', 'date', 'location', 'attrs']
    attrs_default = dict(depth=0.0)
    attrs_explicit = conf.get('attrs', dict())
    attrs_implicit = {k: v for k, v in conf.items() if k not in special_keys}
    attrs_all = {**attrs_default, **attrs_implicit, **attrs_explicit}

    # Compute parameter values
    num = conf['num']
    release_time = date_range(conf['date'], num)
    lon, lat = get_location(conf['location'], num)
    attrs = get_attrs(attrs_all, num)

    # Assemble return value
    r = dict(date=release_time, longitude=lon, latitude=lat)
    return {**r, **attrs}

def get_location(loc_conf, num):
    if isinstance(loc_conf, str):
        with open(loc_conf, 'r', encoding='utf-8') as file:
            return get_location_file(file, num)
    elif hasattr(loc_conf, 'read'):
        return get_location_file(loc_conf, num)

    lon, lat = loc_conf

    if isinstance(loc_conf, dict) and 'offset' in loc_conf:
        return get_location_offset(loc_conf, num)
    elif not hasattr(lon, '__len__'):
        return [lon] * num, [lat] * num
    else:
        slat, slon = latlon_from_poly(lat, lon, num)
        return slon.tolist(), slat.tolist()

def get_location_offset(loc_conf, num):
    clon, clat = loc_conf['center']
    olon, olat = loc_conf['offset']
    dlon, dlat = metric_diff_to_degrees(olon, olat, clat)
    plon = clon + np.array(dlon)
    plat = clat + np.array(dlat)
    slat, slon = latlon_from_poly(plat, plon, num)
    return slon.tolist(), slat.tolist()

def get_location_file(file, num):
    import json

    data = json.loads(file.read())
    if isinstance(data, dict):
        data = [data]

    def get_points_from_layer(layer):
        feats = layer['features']
        return [p for f in feats for p in get_points_from_feature(f)]

    def get_points_from_feature(feature):
        geom = feature['geometry']
        assert geom['type'].upper() == 'MULTIPOLYGON'
        return [np.array(p[0]) for p in geom['coordinates']]

    points = get_points_from_layer(data[0])
    plon = [p[:-1, 0] for p in points]
    plat = [p[:-1, 1] for p in points]

    slat, slon = latlon_from_poly(plat, plon, num)
    return slon.tolist(), slat.tolist()

def date_range(date_span, num):
    if isinstance(date_span, str) or not hasattr(date_span, '__len__'):
        date_span = [date_span] * 2

    start, stop = [np.datetime64(d) for d in date_span]
    dt = (stop - start).astype('timedelta64[s]')
    drange = start + (np.arange(num) * dt) / (num - 1)
    return drange.astype(str).tolist()

def get_attrs(attrs_conf, num):
    return {k: get_attr(v, num) for k, v in attrs_conf.items()}

def get_attr(v, num):
    if hasattr(v, '__len__') and len(v) == 2 and num != 2:
        v = dict(distribution='uniform', min=v[0], max=v[1])

    if isinstance(v, str):
        import importlib
        mod_name, fn_name = v.rsplit('.', 1)
        mod = importlib.import_module(mod_name)
        v = getattr(mod, fn_name)

    if callable(v):
        v = v(num)

    if not hasattr(v, '__len__'):
        v = [v] * num

    if isinstance(v, dict) and 'distribution' in v:
        v = get_distribution(v, num)

    return list(v)

def get_distribution(v, num):
    if v['distribution'] == 'uniform':
        return np.random.uniform(v['min'], v['max'], num)

    elif v['distribution'] == 'gaussian':
        r = np.random.normal(v['mean'], v['std'], num)
        minimum = v.get('min', -np.inf)
        maximum = v.get('max', np.inf)
        return np.clip(minimum, maximum, r)
    elif v['distribution'] == 'exponential':
        return np.random.exponential(v['mean'], num)
    elif v['distribution'] == 'piecewise':
        from scipy.interpolate import InterpolatedUnivariateSpline
        knots = np.array(v['knots'])
        cdf = np.array(v['cdf'])
        fn = InterpolatedUnivariateSpline(cdf, knots, k=1)
        return fn(np.random.rand(num))
    else:
        raise ValueError(f'Unknown distribution: {v["distribution"]}')

def get_depth(depth_span, num):
    if not hasattr(depth_span, '__len__'):
        depth_span = [depth_span] * 2
    depth = np.linspace(*depth_span, num=num).tolist()  # type: typing.Any
    np.random.shuffle(depth)
    return depth

# --- Triangulation procedures ---

def _unit_triangle_sample(num):
    xy = np.random.rand(num*2).reshape((2, -1))
    is_upper_triangle = np.sum(xy, axis=0) > 1
    xy[:, is_upper_triangle] = 1 - xy[:, is_upper_triangle]
    return xy

def get_polygon_sample(coords, num):
    if is_convex(coords):
        return get_polygon_sample_convex(coords, num)
    else:
        return get_polygon_sample_nonconvex(coords, num)

def triangulate(coords):
    triangles = []
    for i in range(len(coords) - 2):
        idx = [0, i + 1, i + 2]
        triangles.append(coords[idx])
    return np.array(triangles)

def triangulate_nonconvex(coords):
    import triangle as tr

    # Triangulate the polygon
    sequence = list(range(len(coords)))
    trpoly = dict(vertices=coords,
                  segments=np.array((sequence, sequence[1:] + [0])).T)
    trdata = tr.triangulate(trpoly, 'p')
    coords = [trdata['vertices'][tidx] for tidx in trdata['triangles']]
    return np.array(coords)

def triangulate_nonconvex_multi(coords):
    import triangle as tr

    # Build flat list of coordinates
    coords_flat = np.concatenate(coords)

    # Build list of segments for multipolygons
    # Multipolygon [[0, 1, 2, 3], [4, 5, 6]] is encoded as
    # segments = [[0, 1], [1, 2], [2, 3], [3, 0], [4, 5], [5, 6], [6, 4]]
    start = np.arange(len(coords_flat))
    stop = start + 1
    coordpos = np.cumsum([0] + [len(c) for c in coords])
    stop[coordpos[1:] - 1] = coordpos[:-1]
    segments = np.stack((start, stop)).T

    # Triangulate and parse output
    trpoly = dict(vertices=coords_flat, segments=segments)
    trdata = tr.triangulate(trpoly, 'p')
    coords = [trdata['vertices'][tidx] for tidx in trdata['triangles']]
    return np.array(coords)

def triangle_areas(triangles):
    a = triangles[..., 1, :] - triangles[..., 0, :]
    b = triangles[..., 2, :] - triangles[..., 0, :]
    return 0.5 * np.abs(a[..., 0] * b[..., 1] - a[..., 1] * b[..., 0])

def get_polygon_sample_convex(coords, num):
    triangles = triangulate(coords)
    return get_polygon_sample_triangles(triangles, num)

def get_polygon_sample_nonconvex(coords, num):
    triangles = triangulate_nonconvex(coords)
    return get_polygon_sample_triangles(triangles, num)

def get_polygon_sample_triangles(triangles, num):
    # Triangulate the polygon
    areas = triangle_areas(triangles)

    # Distribute the points proportionally among the different triangles
    cumarea = np.cumsum(areas)
    triangle_num = np.searchsorted(cumarea / cumarea[-1], np.random.rand(num))

    # Sample within the triangles
    s, t = _unit_triangle_sample(num)
    (x1, x2, x3), (y1, y2, y3) = triangles[triangle_num].T
    x = (x2 - x1) * s + (x3 - x1) * t + x1
    y = (y2 - y1) * s + (y3 - y1) * t + y1
    return x, y

def is_convex(coords):
    # Compute coord differences
    c = np.concatenate([coords, coords[0:2]])
    v = c[:-1, :] - c[1:, :]

    # Compute sign of the cross product
    sgn = v[:-1, 0] * v[1:, 1] > v[:-1, 1] * v[1:, 0]
    # noinspection PyUnresolvedReferences
    return np.all(sgn == sgn[0])

# --- lat / lon procedures ---

def metric_diff_to_degrees(dx, dy, reference_latitude):
    a = 6378137.0
    b = 6356752.314245
    reflat_rad = reference_latitude * np.pi / 180
    lat_cos = np.cos(reflat_rad)
    lat_sin = np.sin(reflat_rad)

    phi_diff = dy / np.sqrt((a*lat_sin)**2 + (b*lat_cos)**2)
    theta_diff = dx / (a * lat_cos)

    lat_diff = phi_diff * 180 / np.pi
    lon_diff = theta_diff * 180 / np.pi

    return lon_diff, lat_diff

def degree_diff_to_metric(lon_diff, lat_diff, reference_latitude):
    a = 6378137.0
    b = 6356752.314245
    reflat_rad = reference_latitude * np.pi / 180
    lat_cos = np.cos(reflat_rad)
    lat_sin = np.sin(reflat_rad)

    phi_diff = lat_diff * np.pi / 180
    theta_diff = lon_diff * np.pi / 180

    dy = np.sqrt((a * lat_sin) ** 2 + (b * lat_cos) ** 2) * phi_diff
    dx = (a * lat_cos) * theta_diff

    return dx, dy

def latlon_from_poly(lat, lon, n):
    # Make multipolygon if given a single polygon
    if np.shape(lat[0]) == ():
        lat = [lat]
        lon = [lon]

    coords = [np.stack((lat_e, lon_e)).T for lat_e, lon_e in zip(lat, lon)]
    triangles = triangulate_nonconvex_multi(coords)
    return get_polygon_sample_triangles(np.array(triangles), n)

def main():
    import sys

    if len(sys.argv) < 2:
        print("""
    makrel: MAKe RELease files

    Usage: makrel <makrel.yaml> <particles.rls>

    Sample makrel.yaml file:

    # Required attributes
    num: 5                                      # Number of particles
    date: [2000-01-01 01:00, 2000-02-01 01:00]  # Start and stop dates
    location: [5, 60]                           # Release location (lon, lat)
    depth: [0, 10]                              # Release depth range

    # Additional attributes
    region: 0                                   # Constant-valued attribute
    age: [0, 0, 0, 3, 3]                        # Vector-valued attribute
    id: numpy.arange                            # Function-valued attribute
    weight:                                     # Gaussian distribution
      distribution: gaussian
      mean: 40
      std: 10
    length:                                     # Exponential distribution
      distribution: exponential
      mean: 10

    # Alternative: Release polygon
    # location: [[5, 6, 6, 5], [60, 60, 61, 61]]

    # Alternative: Release polygon from .geojson file
    # location: area.geojson

    # Alternative: Metric offset from center location
    # location:
    #   center: [5, 60]
    #   offset: [[-50, 50, 50, -50], [-50, -50, 50, 50]]  # 100m x 100m square

    """)
    elif len(sys.argv) == 2:
        import pandas as pd
        out = make_release(sys.argv[1])
        print(pd.DataFrame(out))
    else:
        make_release(sys.argv[1], sys.argv[2])


text:
import tensorflow as tf
from DL_Tools.My_Data.TF_Reader import TF_Reader
from DL_Tools.My_Data.Data_Argument import *
from DL_Tools.My_Trainer import Trainer,Train_op
from DL_Tools.My_Trainer.Trainer_utils import *
from Attention_Model import Attention_Model
from tensorflow.python.platform import flags
import common_flags
import os
import cv2
import numpy as np

os.environ['CUDA_VISIBLE_DEVICES']='0'
common_flags.define()
Flags = flags.FLAGS

arg_ops = [
    base_norm,
    distorting_color,
]

# tmp_path = "/media/night/0002FCA800053168/data/fsns_tf_train/"
# tmp_files_name = os.listdir(tmp_path)
# train_files_name = []
# for name in tmp_files_name:
#     train_files_name.append(os.path.join(tmp_path,name))
#
# tmp_path = "/media/night/0002FCA800053168/data/fsns_tf_validation/"
# tmp_files_name = os.listdir(tmp_path)
# val_files_name = []
# for name in tmp_files_name:
#     val_files_name.append(os.path.join(tmp_path,name))

with tf.variable_scope("train_tf_reader"):
    train_tf_reader = TF_Reader(["/home/hp/data/syn_eng/tfrecord_new/train_1.records","/home/hp/data/syn_eng/tfrecord_new/train_2.records","/home/hp/data/syn_eng/tfrecord_new/train_3.records","/home/hp/data/syn_eng/tfrecord_new/train_4.records","/home/hp/data/syn_eng/tfrecord_new/train_5.records"],True)
    train_tf_reader.data_argument(arg_ops)
    train_images,train_labels = train_tf_reader.shuffle_batch()
with tf.variable_scope("val_tf_reader"):
    val_tf_reader = TF_Reader(["/home/hp/data/syn_eng/tfrecord_new/val.records"],True)
    val_tf_reader.data_argument(arg_ops)
    val_images,val_labels = val_tf_reader.shuffle_batch()
# with tf.variable_scope("visual_reader"):
#     visual_net_images = tf.placeholder(tf.float32, [Flags.batch_size, Flags.net_image_height, Flags.net_image_width,
#                                                     Flags.net_image_channel])
    # for ops in arg_ops:
    #     visual_net_images = ops(visual_net_images)
train_model = Attention_Model(train_images,train_labels)
train_model.build_model("Attention_Model")
train_model_pred_res = train_model.pred_res

val_model = Attention_Model(val_images)
val_model.is_training = False
val_model.build_model("Attention_Model",True)
val_model_pred_res = val_model.pred_res

# visual_model = Attention_Model(visual_net_images)
# visual_model.is_training = False
# visual_model.build_model("Attention_Model", True)
# visual_model_pred_res = visual_model.pred_res

train_loss = train_model.get_loss(train_labels)
val_loss = val_model.get_loss(val_labels)

train_single_acc = single_accuracy(train_model_pred_res,train_labels)
train_seq_acc = seq_accuracy(train_model_pred_res,train_labels)
val_single_acc = single_accuracy(val_model_pred_res,val_labels)
val_seq_acc = seq_accuracy(val_model_pred_res,val_labels)

epoch = Flags.epoch
iteration_in_epoch  = Flags.train_iter_num
iteration_in_val = Flags.val_iter_num

train_op = Train_op.Train_op("MomentumOptimizer", Flags.learning_rate, clip_value=Flags.clip_gradient_norm).build(train_loss)

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
tf.train.start_queue_runners(sess)
# tf.summary.image("train_images",train_images,10)
# tf.summary.scalar("train_loss",train_loss)
# tf.summary.scalar("train_single_acc",train_single_acc)
# tf.summary.scalar("train_seq_acc",train_seq_acc)
# merge_summary = tf.summary.merge_all()
# summary_writer = tf.summary.FileWriter("train_log/",sess.graph)

my_trainer = Trainer.Trainer(sess = sess,
                             train_op = train_op,
                             epoch = Flags.epoch,
                             train_iter_num = Flags.train_iter_num,
                             val_iter_num = Flags.val_iter_num
                             )
my_trainer.add_train_info_ops(
    Trainer.Info_Ops(train_loss, "train_loss", 1))
my_trainer.add_train_info_ops(
    Trainer.Info_Ops(train_single_acc,"train_single_acc",1))
my_trainer.add_train_info_ops(
    Trainer.Info_Ops(train_seq_acc, "train_seq_acc", 1))

my_trainer.add_val_info_ops(
    Trainer.Info_Ops(val_loss, "val_loss", 1))
my_trainer.add_val_info_ops(
    Trainer.Info_Ops(val_single_acc, "val_single_acc", 1))
my_trainer.add_val_info_ops(
    Trainer.Info_Ops(val_seq_acc, "val_seq_acc", 1))

# visual_weights:
all_variables = tf.global_variables()
tmp_count = 0
for v in all_variables:
    if "Momentum" not in v.name:
        tmp_count += 1
        my_trainer.add_train_info_ops(Trainer.Info_Ops(v,v.name.replace("/","%"),1000,visible=False,need_to_be_saved=True))
# graph = tf.get_default_graph()
# for key in graph.get_all_collection_keys():
#     if key != 'variables'and key!='trainable_variables' \
#             and key != 'regularization_losses'and key!='queue_runners' \
#             and key != 'summaries'and key!='global_step' \
#             and key != 'cond_context'and key!='train_op' \
#             and key != 'update_ops'and key!='model_variables':
#         print key
#         my_trainer.add_train_info_ops(Trainer.Info_Ops(tf.get_collection(key), key, 10,visible=False))
#         my_trainer.add_visual_info(key)

my_trainer.set_model_saving_info("models/","AN_cap2")
# load value
def cmp_value(bef_value, now_value):
    if bef_value < now_value:
        return True
    else:
        return False

my_trainer.set_best_saving_info("val_seq_acc",cmp_fuc=cmp_value)

# train_model.restore_model(sess, "models/AN_cap2_new")

# visual_real_images = []
# visual_data_path = "visual_data/"
# visual_names = os.listdir(visual_data_path)
# for n in visual_names[:Flags.batch_size]:
#     tmp_img = cv2.imread(os.path.join(visual_data_path,n))
#     tmp_img = cv2.resize(tmp_img,(Flags.net_image_width, Flags.net_image_height))
#     tmp_img = np.asarray(tmp_img,np.float32)
#     tmp_img = tmp_img / 255. - 0.5
#     tmp_img = tmp_img*1.5
#     visual_real_images.append(tmp_img)
# visual_real_images = np.asarray(visual_real_images)
# def visual_save_fuc(res_data,e):
#     if os.path.exists("visual_save_path/"):
#         os.mkdir("visual_save_path")
#     np.save("visual_save_path/e_{0}_.npy".format(e),res_data)
#
# my_trainer.add_visual_info_ops(visual_model_pred_res,visual_net_images,visual_real_images,visual_save_fuc)

my_trainer.run()


text:
import torch.utils.data as data
import os
import skimage.io as io
import numpy as np
import torch

class SuperSloMo(data.Dataset):
    def __init__(self, root, transform=None, train=True, frm_num = 0):

        # Populate the list with image paths for all the frame in `root`.

        indexSum = 0
        framesPath = []
        sketchPath = []

        if train:
            clips = os.listdir(os.path.join(root, 'frame'))[0:-100]
        else:
            clips = os.listdir(os.path.join(root, 'frame'))[-100:]

        for folder in clips:

            clipsPath = os.path.join(root, 'frame', folder)
            sktchPath = os.path.join(root, 'sketch', folder)

            frameList = sorted(os.listdir(clipsPath))
            sketcList = sorted(os.listdir(sktchPath))

            indexNum  = int((len(frameList) - 1) / (frm_num - 1))

            for i in range(indexNum):
                framesPath.append([])
                sketchPath.append([])

                for j in range(frm_num):
                    framesPath[indexSum].append(os.path.join(clipsPath, frameList[i * (frm_num - 1) + j]))

                sketchPath[indexSum].append(os.path.join(sktchPath, frameList[int(i * (frm_num - 1) + (frm_num - 1) / 2)]))

                indexSum = indexSum + 1

        self.root = root
        self.transform = transform
        self.framesPath = framesPath
        self.sketchPath = sketchPath

    def __getitem__(self, index):
        imgs = []
        for i in range(len(self.framesPath[index])):
            img = io.imread(self.framesPath[index][i]).astype(np.float32)/255.0
            if self.transform is not None:
                img = self.transform(img)
            imgs.append(img)

        if np.random.randint(2):
            imgs.reverse()

        sktt = torch.from_numpy(io.imread(self.sketchPath[index][0])[np.newaxis, :, :].astype(np.float32)/255.0)

        return imgs, sktt

    def __len__(self):
        return len(self.framesPath)