text:
import numpy as np
import math
import operator

def getNeighbors(trainingSet, testInstance, k):
    distances = []
    length = len(testInstance)-1

    for x in range(len(trainingSet)):
        dist = euclideanDistance(testInstance, trainingSet[x], length)
        distances.append((trainingSet[x], dist))

    distances.sort(key=operator.itemgetter(1))

    neighbors = []
    for x in range(k):
        neighbors.append(distances[x])
    return neighbors

def euclideanDistance(instance1, instance2, length):
    distance = 0
    for x in range(length):
        distance += pow((instance1[x] - instance2[x]), 2)
    return math.sqrt(distance)

def centroid(neighborcoor,k,length):
    centro=[]
    for x in range(length):
        mean_coor=0
        for y in range(k):
            mean_coor += neighborcoor[y][x]
        centro.append(mean_coor/k)
    return centro

def meanDistance(neighborcoor,testInstance,k):
    meandist = []
    length = len(testInstance)-1
    center = centroid(neighborcoor,k,length)
    print('center:',center)
    dist = euclideanDistance(testInstance,center,length)
    meandist.append(dist)
    return meandist

def main():

    # prepare data
    #trainingSet = [[1,2,'-'],[2,2,'-'],[3,2,'-'],[4,2,'-'],[1,1,'-'],[2,1,'-'],[3,1,'-'],[4,1,'-']]
    #testSet = [[2.5,2.5,'-'],[0.7,1,'-']]
    trainingSet = [[1,1,'-'],[2,1,'-'],[3,1,'-'],[1.5,2,'-'],[2.5,2,'-']]
    testSet = [[1.15,1.2,'-'],[2,2.2,'-']]

    k = 4

    neighbors_inform=[]
    knn_dist_mean=[]
    for x in range(len(testSet)):
        neighbors = getNeighbors(trainingSet, testSet[x], k)
        neighbors=np.array(neighbors)
        neighbors_inform.append(neighbors)

        neighborcoordinate = np.array(neighbors)[:,0]
        meanDist = meanDistance(neighborcoordinate,testSet[x],k)
        knn_dist_mean.append(meanDist)

    print('knn_dist_mean:',knn_dist_mean)

    #print('neighbors_inform:',neighbors_inform)
    #neighbors_dist=np.array(neighbors_inform)[:,:,1]
    #knn_dist_max=np.max(neighbors_dist,axis=1)
    #knn_dist_avg=np.mean(neighbors_dist,axis=1)
    #print('knn_dist_max:',knn_dist_max)
    #print('knn_dist_avg:',knn_dist_avg)

main()

max_stars_repo_path: 03 Novelty Detection/Tutorial 09 - Distance and reconstruction-based novelty detection/KNN/knn_mean_dist.py

max_stars_repo_name: KateYeon/Business-Anlaytics

max_issues_repo_name: KateYeon/Business-Anlaytics

max_forks_repo_name: KateYeon/Business-Anlaytics


text:
# 回路のグラフ化をできます。量子状態の棒グラフ、縮約した後の玉表示
import matplotlib.pyplot as plt
import numpy as np
from qulacs import Observable

def show_blochsphere(state, bit):
    n_qubit = state.get_qubit_count()
    observableX = Observable(n_qubit)
    observableX.add_operator(1.0, f"X {bit}")  # オブザーバブルを設定
    observableY = Observable(n_qubit)
    observableY.add_operator(1.0, f"Y {bit}")  # オブザーバブルを設定
    observableZ = Observable(n_qubit)
    observableZ.add_operator(1.0, f"Z {bit}")  # オブザーバブルを設定

    X = observableX.get_expectation_value(state)
    Y = observableY.get_expectation_value(state)
    Z = observableZ.get_expectation_value(state)
    print(X, Y, Z)
    fig = plt.figure()

    ax = fig.add_subplot(111, projection="3d")
    ax.set_box_aspect((1, 1, 1))
    # sphere
    u, v = np.mgrid[0 : (2 * np.pi) : 8j, 0 : np.pi : 8j]
    x = np.cos(u) * np.sin(v)
    y = np.sin(u) * np.sin(v)
    z = np.cos(v)
    ax.plot_wireframe(x, y, z, color="lightskyblue", linewidth=0.5)
    ax.quiver(0, 0, 0, X, Y, Z, color="red")
    ax.scatter(X, Y, Z, color="red")
    ax.set_xlabel("X-axis")
    ax.set_ylabel("Y-axis")
    ax.set_zlabel("Z-axis")
    plt.show()

ext: py

max_stars_repo_name: Qulacs-Osaka/scikit-qulacs

max_issues_repo_name: Qulacs-Osaka/scikit-qulacs

max_forks_repo_name: Qulacs-Osaka/scikit-qulacs

reason: import numpy


text:
import logging
import os
import random

import numpy as np
import pickle
import torch
from torch import nn
from torch.utils import data
from transformers import BertTokenizer, BertConfig, AdamW, get_linear_schedule_with_warmup

from architectures.BERT_model_multidata import BERT_model_multidata
from architectures.architectures_utils_multidata import train_bert_model_multidata, predict_multidata
from eval_metrics.evaluateNER import EvaluateNER
from utils.BatcherBERT_multidata import BatcherBERT_multidata
from utils.ReadNERMultiData import MultiDataset
import argparse

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
					datefmt='%m/%d/%Y %H:%M:%S',
					level=logging.INFO)
logger = logging.getLogger(__name__)

def parse_arguments(parser):
	parser.add_argument("--file_extension", type=str, default="txt")
	parser.add_argument("--experiment", type=str, default="baseline")
	parser.add_argument("--model_saving_path", type=str, default="")
	parser.add_argument("--train_file", type=str, default="train")
	parser.add_argument("--data_path", type=str, default="train")
	parser.add_argument("--train", action="store_true")
	parser.add_argument("--epochs", type=int, default=5)
	parser.add_argument("--force_size", action="store_true")
	parser.add_argument("--special_labels", action="store_true")
	parser.add_argument("--masking_percentage", type=float, default=0.0)
	parser.add_argument("--seed", type=int, default=12)  # 12 seems to be a good seed
	parser.add_argument("--early_stop", type=int, default=0)
	parser.add_argument("--tags2use", type=str, default="NER_IOB2")
	parser.add_argument("--separator", type=str, default=" ")
	parser.add_argument("--test_file", type=str, default="test")
	parser.add_argument("--dev_file", type=str, default="valid")
	parser.add_argument("--predict", type=str, default="None",
						choices=["None", "Train", "Test", "Dev", "All", "DevTest"])
	parser.add_argument("--crf", action="store_true")
	parser.add_argument("--uppercase", action="store_true")
	parser.add_argument("--lr", type=float, default=2e-5)
	parser.add_argument("--epsilon", type=float, default=1e-8)
	parser.add_argument("--bert_model", type=str, default="bert-base-cased")
	parser.add_argument("--token_col", type=int, default=0)
	parser.add_argument("--ner_col", type=int, default=3)
	parser.add_argument("--evaluate", action="store_true", help="It will be activated if training is activated")
	parser.add_argument("--mask_entities", action="store_true",
						help="If activated, instead of masking based on tokens, it will mask based on entities.")
	parser.add_argument("--train_batch_size", type=int, default=32)
	parser.add_argument("--comment_line", type=str, default="-DOCSTART")
	parser.add_argument("--sequence_size", type=int, default=128)
	parser.add_argument("--multi_gpu", action="store_true")
	parser.add_argument("--fullwords_mask", action="store_false", default=None)
	parser.add_argument("--redundant_uppercase", action="store_false", default=None)
	parser.add_argument("--bert_hidden_size", type=int, default=768)
	parser.add_argument("--no_dev", action="store_true")
	parser.add_argument("--multidata_model", type=str, default="None", choices=["None", "Daume", "All"])
	parser.add_argument("--annotate_as_dataset", type=int, help="Indicate the ID of the dataset that will be used for annoting.", default=-1)
	parser.add_argument("--output_saving_path", type=str, default=None, help="If not set, the path will be the same where the model is saved")
	parser.add_argument("--predict_boundaries", action="store_true")
	parser.add_argument("--uppercase_percentage", type=float, default=0.0)
	parser.add_argument("--no_test", action="store_true")

	args = parser.parse_args()
	for k in args.__dict__:
		print(k + ": " + str(args.__dict__[k]))

	return args

parser = argparse.ArgumentParser()
opt = parse_arguments(parser)

seed = opt.seed

random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)

extension = opt.file_extension
columns = None
masking = False
mask_entities = False
fullwords_mask = False
do_predictions_on = []
if opt.predict != "None":
	if opt.predict == "All":
		do_predictions_on = ["Dev", "Test", "Train"]
		if opt.extra_test_file:
			do_predictions_on.append("ExtraTest")
	elif opt.predict == "DevTest":
		do_predictions_on = ["Dev", "Test"]
	else:
		do_predictions_on = [opt.predict]

if opt.train:
	opt.evaluate = True

if opt.masking_percentage > 0:
	masking = True
	mask_entities = opt.mask_entities
	fullwords_mask = opt.fullwords_mask

columns = {
	opt.token_col: 'tokens',
	opt.ner_col: 'NER_IOB'
}

experiment_name = opt.experiment
tags2use = opt.tags2use
model_saving_path = opt.model_saving_path
train_file = opt.train_file
data_path = opt.data_path
if opt.separator == "\\t":
	opt.separator = "\t"

if opt.no_dev:
	opt.early_stop = 0

if not os.path.exists(f"{model_saving_path}/{experiment_name}/"):
	os.makedirs(f"{model_saving_path}/{experiment_name}/")

if opt.output_saving_path is not None:
	output_saving_path = f"{opt.output_saving_path}/"
	if not os.path.exists(output_saving_path):
		os.makedirs(output_saving_path)
else:
	output_saving_path = f"{model_saving_path}/{experiment_name}/"

dataset_config = {
	'columns': columns,
	'columnsSeparator': opt.separator,
	'basePath': data_path,
	'dataType': {},
	'commentSymbol': [opt.comment_line],
	'specialLabels': opt.special_labels
}

if not opt.no_test:
	dataset_config["dataType"]["Test"] = {
		'prefix': opt.test_file,
		'extension': extension,
		'labelsAsTraining': False
	}

if not opt.no_dev:
	dataset_config["dataType"]["Dev"] = {
		'prefix': opt.dev_file,
		'extension': extension,
		'labelsAsTraining': False
	}

if opt.train:

	dataset_config["dataType"]["Train"] = {
		'prefix': train_file,
		'extension': extension,
		'labelsAsTraining': True
	}

	bert_base_model = opt.bert_model
	tokenizer = BertTokenizer.from_pretrained(bert_base_model, do_lower_case=False)

	data_processor = MultiDataset(dataset_config, tokenizer=tokenizer)
	mapping = data_processor.getMapping()

	add_tokens = data_processor.getTokensToAdd()

	resize_vocab_size = None

	num_added_tokens = 0
	if len(add_tokens) > 0:
		print(f"Adding {len(add_tokens)} tokens to Tokenizer vocabulary")
		num_added_tokens = tokenizer.add_tokens(add_tokens)
		#We need to add this to the mappings as the tokenizer of Transformers 2.11.0 do not store the added tokens
		mapping["new_tokens"] = add_tokens
	if opt.uppercase:
		special_tokens_dict = {'additional_special_tokens': ['[UP]', '[up]']}

		num_added_tokens += tokenizer.add_special_tokens(special_tokens_dict)
		#We need to add this to the mappings as the tokenizer of Transformers 2.11.0 do not store the added tokens
		mapping["new_special_tokens"] = special_tokens_dict

	if num_added_tokens > 0:
		resize_vocab_size = len(tokenizer)

	tokenizer.save_vocabulary(f"{model_saving_path}/{experiment_name}/")

	with open(f"{model_saving_path}/{experiment_name}/mapping.pkl", 'wb') as mapping_file:
		pickle.dump(mapping, mapping_file, -1)

	dataset = data_processor.getProcessedData()
	del data_processor

	with open(f"{model_saving_path}/{experiment_name}/params-{experiment_name}.txt", "w") as output_file:
		output_file.write(f"seed: {seed}\n")
		output_file.write(f"file_extension: {extension}\n")
		output_file.write(f"experiment_name: {experiment_name}\n")
		output_file.write(f"tags2use: {tags2use}\n")
		output_file.write(f"special_labesl: {opt.special_labels}\n")
		output_file.write(f"force_size: {opt.force_size}\n")
		output_file.write(f"train_file: {train_file}\n")
		output_file.write(f"masking_percentage: {opt.masking_percentage}\n")
		output_file.write(f"seed: {seed}\n")
		output_file.write(f"epochs: {opt.epochs}\n")
		output_file.write(f"early_stop: {opt.early_stop}\n")
		output_file.write(f"separator: {opt.separator}\n")
		output_file.write(f"test_file: {opt.test_file}\n")
		output_file.write(f"dev_file: {opt.dev_file}\n")
		output_file.write(f"crf: {opt.crf}\n")
		output_file.write(f"uppercase: {opt.uppercase}\n")
		output_file.write(f"lr: {opt.lr}\n")
		output_file.write(f"epsilon: {opt.epsilon}\n")
		output_file.write(f"bert_model: {opt.bert_model}\n")
		output_file.write(f"mask_entities: {mask_entities}\n")
		output_file.write(f"train_batch_size: {opt.train_batch_size}\n")
		output_file.write(f"comment_line: {opt.comment_line}\n")
		output_file.write(f"sequence_size: {opt.sequence_size}\n")
		output_file.write(f"multi_gpu: {opt.multi_gpu}\n")
		output_file.write(f"fullwords_mask: {opt.fullwords_mask}\n")
		output_file.write(f"redundant_uppercase: {opt.redundant_uppercase}\n")
		output_file.write(f"bert_hidden_size: {opt.bert_hidden_size}\n")
		output_file.write(f"no_dev: {opt.no_dev}\n")
		output_file.write(f"multidata_model: {opt.multidata_model}\n")
		output_file.write(f"training_datasets: {dataset['Train_files']}\n")
		output_file.write(f"predict_boundaries: {opt.predict_boundaries}\n")
		output_file.write(f"uppercase_percentage: {opt.uppercase_percentage}\n")
else:
	with open(f"{model_saving_path}/{experiment_name}/mapping.pkl", 'rb') as mapping_file:
		mapping = pickle.load(mapping_file)
	data_processor = MultiDataset(dataset_config, predifined_mapping=mapping)
	tokenizer = BertTokenizer.from_pretrained(f"{model_saving_path}/{experiment_name}/", do_lower_case=False)
	#It is mandatory to add the tokens in the same order than before
	if "new_tokens" in mapping:
		tokenizer.add_tokens(mapping["new_tokens"])
	if "new_special_tokens" in mapping:
		tokenizer.add_special_tokens(mapping["new_special_tokens"])

	dataset = data_processor.getProcessedData()
	del data_processor

inverse_tagset = dict(map(reversed, mapping[tags2use].items()))
evaluateNER = EvaluateNER(inverse_tagset)

def Train():
	torch.backends.cudnn.deterministic = True
	torch.backends.cudnn.benchmark = False
	type_crf_constraints = None
	if opt.crf:
		if tags2use in ["NER_IOB2", "NER_IOBA"]:
			type_crf_constraints = "BIO"
		elif tags2use in ["NER_IOBES", "NER_IOBESA"]:
			type_crf_constraints = "BIOES"

	md_model = False
	md_da = False
	md_number = 0
	if opt.multidata_model in ["Daume", "All"]:
		md_model = True
		if opt.multidata_model == "Daume":
			md_number = len(dataset["Train"])
			md_da = True

	boundaries_labels = None
	if opt.predict_boundaries:
		if tags2use == "NER_IOB2":
			boundaries_labels = mapping["NER_IOBA"]
		if tags2use == "NER_IOBES":
			boundaries_labels = mapping["NER_IOBESA"]

	labels_to_use = mapping[tags2use]

	bert_config = BertConfig.from_pretrained(bert_base_model, num_labels=len(labels_to_use) + 1,
											 finetuning_task=experiment_name,
											 task_specific_params={"crf": opt.crf,
																   "type_crf_constraints": type_crf_constraints,
																   "predict_masked": masking,
																   "bert_hidden_size": opt.bert_hidden_size,
																   "predict_boundaries": opt.predict_boundaries,
																   "boundaries_labels": boundaries_labels,
											 					   "md_model": md_model,
																   "md_number": md_number},
											 label2id=labels_to_use)

	params_train = {'batch_size': opt.train_batch_size,
					'shuffle': True}

	print("Processing train")
	training_batcher = BatcherBERT_multidata(dataset["Train"], tokenizer,
											 	tags_field=tags2use,
												max_length=opt.sequence_size,
												tagset=mapping[tags2use],
												force_size=opt.force_size,
												special_labels=opt.special_labels,
												mask_percentage=opt.masking_percentage,
												uppercase=opt.uppercase,
												fullwords_mask=fullwords_mask,
												redundant_uppercase=opt.redundant_uppercase,
												boundaries_dict=boundaries_labels,
												predict_boundaries=opt.predict_boundaries,
												as_da=md_da,
											 	uppercase_percentge=opt.uppercase_percentage
											 	)

	training_batcher.createBatches()

	if opt.no_dev:
		dev_generator = None
		dev_aligner = None
	else:
		params_dev = {'batch_size': 8,
					  'shuffle': False}

		print("Processing dev")
		dev_set = BatcherBERT_multidata(dataset["Dev"], tokenizer, tags_field=tags2use,
							  max_length=opt.sequence_size, test=True, tagset=mapping[tags2use], force_size=opt.force_size,
							  special_labels=opt.special_labels, uppercase=opt.uppercase,
							  redundant_uppercase=opt.redundant_uppercase, as_da=md_da)

		dev_generator = data.DataLoader(dev_set, **params_dev, collate_fn=dev_set.collate_fn)

		dev_aligner = dev_set.getAligner()

	model = BERT_model_multidata.from_pretrained(bert_base_model,
												   from_tf=False,
												   config=bert_config)

	if resize_vocab_size is not None:
		model.resize_embedding_and_fc(resize_vocab_size)

	if opt.multi_gpu and torch.cuda.device_count() > 1:
		print(f"Using {torch.cuda.device_count()} GPUs")
		model = nn.DataParallel(model)
		#model = DataParallelModel(model)
		opt.multi_gpu = True
	else:
		opt.multi_gpu = False

	num_train_epochs = opt.epochs
	param_optimizer = list(model.named_parameters())
	no_decay = ['bias', 'LayerNorm.weight']
	weight_decay = 0.01
	optimizer_grouped_parameters = [
		{'params': [p for n, p in param_optimizer if not any(
			nd in n for nd in no_decay)], 'weight_decay': weight_decay},
		{'params': [p for n, p in param_optimizer if any(
			nd in n for nd in no_decay)], 'weight_decay': 0.0}
	]

	warmup_proportion = 0.1
	gradient_accumulation_steps = 1
	learning_rate = opt.lr
	num_train_optimization_steps = int(
		len(training_batcher) / params_train['batch_size'] / gradient_accumulation_steps) * num_train_epochs
	warmup_steps = int(warmup_proportion * num_train_optimization_steps)
	optimizer = AdamW(optimizer_grouped_parameters,
					  lr=learning_rate, eps=opt.epsilon,
					  correct_bias=True)  # If we would like to replicate BERT, we need to set the compensate_bias as false
	scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=warmup_steps,
												num_training_steps=num_train_optimization_steps)

	train_bert_model_multidata(model, experiment_name, num_train_epochs,
						 optimizer, scheduler, training_batcher, params_train, dev_generator, evaluateNER.calculate,
						 model_saving_path, use_gpu=True, masking=masking, early_stop=opt.early_stop, bert_hidden_size=opt.bert_hidden_size,
						 dev_aligner=dev_aligner, multi_gpu=opt.multi_gpu, uppercase_percentage=opt.uppercase_percentage)

def printCommentLines(output_file, lines_array):
	for line in lines_array:
		output_file.write(f"{line}\n")

def testAndPredict(model, data_split, data_id, tagged=False, md_da=False):
	params_test = {'batch_size': 8,
				   'shuffle': False}

	override_da_index = data_id
	if opt.annotate_as_dataset >= 0:
		override_da_index = opt.annotate_as_dataset

	override_da_index += 1

	tagset = mapping[tags2use]

	output_file_name = dataset[f"{data_split}_files"][data_id]

	print(f"Batching {output_file_name}")

	test_set = BatcherBERT_multidata([dataset[data_split][data_id]], tokenizer, tags_field=tags2use,
						   max_length=opt.sequence_size, test=True, tagset=tagset, force_size=opt.force_size,
						   special_labels=opt.special_labels, uppercase=opt.uppercase,
						   redundant_uppercase=opt.redundant_uppercase, as_da=md_da, override_da_index=override_da_index)

	test_generator = data.DataLoader(test_set, **params_test, collate_fn=test_set.collate_fn)

	print(f"Processing model on {output_file_name}")

	if tagged:
		predictions, _, report = predict_multidata(model, test_generator, tagged=True,
														evaluation_function=evaluateNER.calculate, use_gpu=True,
														test_aligner=test_set.getAligner(), multi_gpu=opt.multi_gpu,
														bert_hidden_size=opt.bert_hidden_size)
		with open(f"{output_saving_path}/{output_file_name}_{experiment_name}-results.txt", "w") as output_file:
			output_file.write(report)
			output_file.write("\n")
	else:
		predictions = predict_multidata(model, test_generator, tagged=False,
										evaluation_function=evaluateNER.calculate,
										use_gpu=True,
										test_aligner=test_set.getAligner(),
										multi_gpu=opt.multi_gpu,
										bert_hidden_size=opt.bert_hidden_size)

	with open(f"{output_saving_path}/{output_file_name}_{experiment_name}-predictions.txt", "w") as output_file:
		if -1 in dataset[f"{data_split}_comments"][data_id]:
			printCommentLines(output_file, dataset[f"{data_split}_comments"][data_id][-1])
			output_file.write("\n")

		for sentence_id, sentence in enumerate(dataset[data_split][data_id]):
			assert (len(predictions[sentence_id]) == len(sentence["tokens"]))
			for (before, token, middle, predicted_tag, after) in zip(sentence["colsBefore"], sentence["tokens"],
																	 sentence["colsMiddle"],
																	 predictions[sentence_id],
																	 sentence["colsAfter"]):
				if not tagged:
					if predicted_tag not in inverse_tagset:
						predicted_tag = "O"
					else:
						predicted_tag = inverse_tagset[predicted_tag]

				if before is not None:
					output_file.write(f"{before}{opt.separator}")
				output_file.write(f"{token}{opt.separator}")
				if middle is not None:
					output_file.write(f"{middle}{opt.separator}")
				output_file.write(f"{predicted_tag}")
				if after is not None:
					output_file.write(f"{opt.separator}{after}")
				output_file.write("\n")
			output_file.write(f"\n")
			if sentence_id in dataset[f"{data_split}_comments"][data_id]:
				printCommentLines(output_file, dataset[f"{data_split}_comments"][data_id][sentence_id])
				output_file.write("\n")

if opt.train:
	Train()

if opt.evaluate or len(do_predictions_on) > 0:

	model = BERT_model_multidata.from_pretrained(f"{model_saving_path}/{experiment_name}/", from_tf=False)

	if opt.multi_gpu and torch.cuda.device_count() > 1:
		print(f"Using {torch.cuda.device_count()} GPUs")
		model = nn.DataParallel(model)
		opt.multi_gpu = True
	else:
		opt.multi_gpu = False

	md_da = False
	if opt.multidata_model == "Daume":
		md_da = True

	for data_split in do_predictions_on:
		if data_split == "Train":
			continue
		for i in range(len(dataset[data_split])):
			testAndPredict(model, data_split, i, tagged=opt.evaluate, md_da=md_da)


text:
# AUTOGENERATED! DO NOT EDIT! File to edit: 33_EM.ipynb (unless otherwise specified).

__all__ = ['KMeansInit', 'EM', 'MultiEM', 'generateBags']

# Cell
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import scipy.stats as ss
from easydict import EasyDict
from scipy.spatial.distance import cdist
from sklearn.metrics import roc_auc_score,adjusted_mutual_info_score, adjusted_rand_score
from tqdm.notebook import tqdm

# Cell

# export
def KMeansInit(X, k):
    means = [X[np.random.choice(np.arange(X.shape[0]))]]
    for iteration in range(k-1):
        dists = cdist(X, np.stack(means)).min(1)
        probs = dists / dists.sum()
        nextIndex = np.random.choice(np.arange(X.shape[0]),p=probs)
        means.append(X[nextIndex])
    return np.stack(means)

class EM:
    def __init__(self, X, k):
        self.X = X
        self.k = k
        self.N,self.dim = self.X.shape
        self.initializeComponents()

    def initializeComponents(self):
        mins = self.X.min(0)
        maxs = self.X.max(0)
        # USE K-MEANS++
#         self.mus = KMeansInit(self.X, self.k)
        self.mus = self.X[np.random.choice(np.arange(self.X.shape[0]),size=self.k)]
        self.covs = np.stack([np.eye(self.dim) for _ in range(self.k)])
        self.pi = np.random.dirichlet(np.ones(self.k))

    def E_Step(self):
        gammas = np.zeros((self.N,self.k))
        for n in range(self.N):
            for k in range(self.k):
                gammas[n,k] = self.pi[k] * ss.multivariate_normal.pdf(self.X[n],
                                                                      mean=self.mus[k],
                                                                      cov=self.covs[k])
            gammas[n] = gammas[n] / gammas[n].sum()
        self.gammas = gammas

    def M_Step(self):
        Nk = np.sum(self.gammas, axis=0)[:,np.newaxis]
        self.mus = np.dot(self.gammas.T, self.X) / Nk
        self.pi = np.mean(self.gammas,axis=0)
        for k in range(self.k):
            x = np.matrix(self.X - self.mus[k,:])
            gamma_diag = np.matrix(np.diag(self.gammas[:,k]))
            sigma_k = x.T * gamma_diag * x
            self.covs[k,:,:] = sigma_k / Nk[k]

    def log_likelihood(self):
        ll = 0
        for xn in self.X:
            ll += np.log(np.sum([pik * ss.multivariate_normal.pdf(xn,
                                                                  mean=muk,
                                                                  cov=covk) for pik,muk,covk in zip(self.pi,
                                                                                self.mus,
                                                                                self.covs)]))
        return ll

# Cell
class MultiEM:
    def __init__(self, bags, n_components):
        self.em_instances = []
        self.labels = []
        self.bags = bags
        # Initialize each EM instance
        for bag in self.bags:
            self.em_instances.append(EM(bag.X_pos, n_components))
        # Override the initialization

    def run_positive_em(self, n_iters, multi=True, KMeansPPInit=True):
        if KMeansPPInit:
            if multi:
                for inst in range(len(self.em_instances)):
                    self.em_instances[inst].mus = KMeansInit(np.concatenate([e.X for e in self.em_instances]),
                                                      self.em_instances[0].k)
            else:
                self.em_instances[0].mus = KMeansInit(np.concatenate([e.X for e in self.em_instances]),
                                                      self.em_instances[0].k)
        self.logLikelihoods = np.zeros((len(self.em_instances),
                                        n_iters+2))
        for inst in range(len(self.em_instances)):
            self.logLikelihoods[inst,0] = self.em_instances[inst].log_likelihood()
        for iteration in tqdm(range(1, n_iters+1),total=n_iters,leave=False):
            for inst in range(len(self.em_instances)):
                self.em_instances[inst].E_Step()
                self.em_instances[inst].M_Step()
                if multi:
                    self.em_instances[(inst+1) % len(self.em_instances)].mus = self.em_instances[inst].mus
                    self.em_instances[(inst+1) % len(self.em_instances)].covs = self.em_instances[inst].covs
                self.logLikelihoods[inst,iteration] = self.em_instances[inst].log_likelihood()
        # After final iteration, pass mu and cov to all instances and update gammas through the E_Step
        for j in range(len(self.em_instances)):
            if multi:
                self.em_instances[j].mus = self.em_instances[0].mus
                self.em_instances[j].covs = self.em_instances[0].covs
                # update gamma with new mus and covs values
                self.em_instances[j].E_Step()
            self.logLikelihoods[j, -1] = self.em_instances[j].log_likelihood()
        self.compute_positive_clustering_performance()

    def compute_positive_clustering_performance(self):
        adjusted_rand_scores = []
        for em,bag in zip(self.em_instances, self.bags):
            adjusted_rand_scores.append(adjusted_rand_score(bag.positive_component_labels,
                                                            np.argmax(em.gammas,axis=1)))
        self.score = np.mean(adjusted_rand_scores)

    def cluster_unlabeled(self):
        for bagnum, (bag, em) in enumerate(zip(self.bags, self.em_instances)):
            assignments = np.zeros(bag.x_unlabeled.shape[0]).astype(int)
            for instnum,inst in enumerate(bag.x_unlabeled):
                lls = np.zeros(em.k)
                for i,(mu,cov) in enumerate(zip(em.mus, em.covs)):
                    lls[i] = ss.multivariate_normal.logpdf(inst,mean=mu,cov=cov)
                assignments[instnum] = np.argmax(lls)
            self.bags[bagnum].unlabeled_assignments = assignments

# Cell
def generateBags(NBags,
                 NPos=200,
                 NUnlabeled=1000,
                 pos_means=[[2,2], [-2,-2], [2,-2]],
                 neg_means=[[0,0]],
                 pos_covs=[np.eye(2)]*3,
                 neg_covs=[np.eye(2)]):
    bags = []
    K = len(pos_means)
    D = len(pos_means[0])
    for _ in tqdm(range(NBags)):
        bag = EasyDict()
        bag.alpha = np.random.uniform()
        bag.pi = np.random.dirichlet(np.ones(K)*1.5)
        bag.rho = np.random.dirichlet(np.ones(K)*1.5)
        bag.gamma = bag.alpha * bag.pi + (1 - bag.alpha) * bag.rho
        bag.eta = bag.alpha * bag.pi / bag.gamma
        bag.posClusterAssignment = np.random.choice(np.arange(K), p=bag.pi.ravel(),size=NPos)
        bag.unlabeledPosClusterAssignment = np.random.choice(np.arange(K),
                                                         p=bag.pi.ravel(),
                                                         size=np.round(NUnlabeled * bag.alpha).astype(int))
        upLabels, upCounts = np.unique(bag.unlabeledPosClusterAssignment,return_counts=True)
        bag.empiricalPi = np.zeros(K)
        for label,count in zip(upLabels,upCounts):
            bag.empiricalPi[label] = count / upCounts.sum()
        bag.unlabeledNegClusterAssignment = np.random.choice(np.arange(K),
                                                         p=bag.rho.ravel(),
                                                         size=np.round(NUnlabeled * (1 - bag.alpha)).astype(int))
        unLabels, unCounts = np.unique(bag.unlabeledNegClusterAssignment,return_counts=True)
        bag.empiricalRho = np.zeros(K)
        for label,count in zip(unLabels,unCounts):
            bag.empiricalRho[label] = count / unCounts.sum()
        bag.empiricalGamma = bag.alpha * bag.empiricalPi + (1-bag.alpha) * bag.empiricalRho
        bag.empiricalEta = bag.alpha * bag.empiricalPi / bag.empiricalGamma
        bag.X_pos = np.zeros((0,D))
        bag.hiddenLabels = np.concatenate((np.ones_like(bag.unlabeledPosClusterAssignment),
                                           np.zeros_like(bag.unlabeledNegClusterAssignment)))
        bag.x_unlabeled = np.zeros((0,D))
        # sample positive data
        for ci in tqdm(bag.posClusterAssignment,leave=False):
            xi = ss.multivariate_normal(pos_means[ci], pos_covs[ci]).rvs()
            if D==1:
                xi = xi[None,None]
            else:
                xi = xi[None]
            bag.X_pos = np.concatenate((bag.X_pos,xi))

        # sample unlabeled pos data
        for ci in tqdm(bag.unlabeledPosClusterAssignment,leave=False):
            xi = ss.multivariate_normal(pos_means[ci], pos_covs[ci]).rvs()
            if D==1:
                xi = xi[None,None]
            else:
                xi = xi[None]
            bag.x_unlabeled = np.concatenate((bag.x_unlabeled, xi))
        # sample unlabeled neg data
        for ci in tqdm(bag.unlabeledNegClusterAssignment,leave=False):
            xi = ss.multivariate_normal(neg_means[ci], neg_covs[ci]).rvs()
            if D==1:
                xi = xi[None,None]
            else:
                xi = xi[None]
            bag.x_unlabeled = np.concatenate((bag.x_unlabeled, xi))
        bags.append(bag)
    return bags

max_stars_repo_name: Dzeiberg/multiinstance

max_issues_repo_name: Dzeiberg/multiinstance

max_forks_repo_name: Dzeiberg/multiinstance

reason: import numpy,import scipy,from scipy


text:
"""
Methods and functions to perform dimensionality reduction
"""
import numpy as np
import json
from copy import copy
import umap
from sklearn.manifold import TSNE
from sklearn.preprocessing import StandardScaler

from .sparse_kpca import SPARSE_KPCA
from .ml_pca import PCA
from ..io import NpEncoder

class Dimension_Reducers:
    def __init__(self, dreduce_spec_dict={}):
        """
        Object handing the dimensionality reduction of design matrices
        Parameters
        ----------
        dreduce_spec_dict: dictionaries that specify which dimensionality reduction algorithm to use
        we can perform a list of D-reduction sequentially.
        e.g.
        dreduce_spec_dict = {
        "preprocessing": {"type": 'SCALE', 'parameter': None},
        "reduce2_umap": {"type": 'UMAP', 'parameter': {"n_components": 10, "n_neighbors": 10, "metric":'euclidean'}}
        }
        e.g.
        dreduce_spec_dict = {
        "reduce1_pca": {"type": 'PCA', 'parameter':{"n_components": 50, "scalecenter":True}},
        "reduce2_tsne": {"type": 'TSNE', 'parameter': {"n_components": 10, "perplexity":20}}
        }
        """
        self.dreduce_spec_dict = dreduce_spec_dict
        # list of Atomic_Descriptor objections
        self.engines = {}
        #self.acronym = ""
        self._fitted = False
        self.bind()

    def add(self, dreduce_spec, tag):
        """
        adding the specifications of a new dimensionality reducer
        Parameters
        ----------
        dreduce_spec: a dictionary that specify which dimensionality reducer to use
        """
        self.dreduce_spec_dict[tag] = dreduce_spec

    def pack(self):
        return json.dumps(self.dreduce_spec_dict, sort_keys=True, cls=NpEncoder)
    """
    def get_acronym(self):
        if self.acronym == "":
            for element in self.dreduce_spec_dict.keys():
                self.acronym += self.engines[element].get_acronym()
        return self.acronym
    """
    def bind(self):
        """
        binds the objects that actually performs dimension reduction
        these objects need to have .fit(desc)/.transform(desc)/.fit_transform(desc) methods to perform reduction on a design matrix desc
        """
        # clear up the objects
        self.engines = {}
        for element in self.dreduce_spec_dict.keys():
            self.engines[element] = self._call(self.dreduce_spec_dict[element])
            #self.dreduce_spec_dict[element]['acronym'] = self.engines[element].get_acronym()

    def _call(self, dreduce_spec):
        """
        call the specific descriptor objects
        """
        if "type" not in dreduce_spec.keys():
            raise ValueError("Did not specify the type of the dimensionality reducer.")
        elif dreduce_spec["type"] == "SCALE":
            print ("Using standard scaling of the data ...")
            return StandardScaler()
        elif dreduce_spec["type"] == "PCA":
            print ("Using PCA ...")
            return PCA(**dreduce_spec['parameter'])
        elif dreduce_spec["type"] == "SPARSE_KPCA":
            print ("Using kernel PCA (sparsified) ...")
            return SPARSE_KPCA(**dreduce_spec['parameter'])
        elif dreduce_spec["type"] == "UMAP":
            """https://umap-learn.readthedocs.io/en/latest/api.html#umap.umap_.UMAP.fit_transform"""
            print ("Using UMAP ...")
            return umap.UMAP(**dreduce_spec['parameter'])
        elif dreduce_spec["type"] == "TSNE":
            print ("Using t-sne ...")
            """https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html"""
            return TSNE(**dreduce_spec['parameter'])
        else:
            raise NotImplementedError

    def fit(self, X):
        """
        compute the global descriptor vector for a frame from atomic contributions
        Parameters
        ----------
        X : array-like, shape=[n_samples,n_dim_high]
            Input points.
        """
        for element in self.dreduce_spec_dict.keys():
            X = self.engines[element].fit(X)
        self._fitted = True

    def fit_transform(self, X):
        """
        compute the global descriptor vector for a frame from atomic contributions
        Parameters
        ----------
        X : array-like, shape=[n_samples,n_dim_high]
            Input points.

        Returns
        -------
        x: array-like, shape=[n_samples,n_dim_low]
        """
        X = X.copy()
        for element in self.dreduce_spec_dict.keys():
            X = self.engines[element].fit_transform(X)
        self._fitted = True
        return X

    def transform(self, X):
        """
        compute the global descriptor vector for a frame from atomic contributions
        Parameters
        ----------
        X : array-like, shape=[n_samples,n_dim_high]
            Input points.

        Returns
        -------
        x: array-like, shape=[n_samples,n_dim_low]
        """
        if self._fitted == False:
            raise ValueError("Haven't fit the dimensionality reducer. Use .fit or .fit_transform first")
        X = X.copy()
        for element in self.dreduce_spec_dict.keys():
            if self.dreduce_spec_dict[element]['type'] == 'TSNE':
                raise ValueError("TSNE does not allow out-of-sampling embedding")
            X = self.engines[element].transform(X)
        return X

max_stars_repo_name: BingqingCheng/ASAP

max_issues_repo_name: FelixWodaczek/ASAP

max_forks_repo_name: FelixWodaczek/ASAP


text:
from typing import Dict, Optional, Union

import numpy as np

from ..utils.type_hints import IntegerScalar, NPType
from .base import IReader
from .functional import get_one_hot

class ScalarReader(IReader):
    """Numeric loader readers abstraction.  Reads a single float, int, str or other from loader."""

    def __init__(
        self,
        input_key: str,
        output_key: str,
        dtype: NPType = np.float32,
        default_value: float = -1.0,
        one_hot_classes: Optional[int] = None,
        smoothing: Optional[float] = None,
    ) -> None:
        """Create a new instance of ScalarReader.

        Args:
            input_key: input key to use from annotation dict
            output_key: output key to use to store the result
            dtype: datatype of scalar values to use
            default_value: default value to use if something goes wrong
            one_hot_classes: number of one-hot classes
            smoothing: if specified applies label smoothing to one_hot classes
        """
        super().__init__(input_key, output_key)
        self.dtype = dtype
        self.default_value = default_value
        self.one_hot_classes = one_hot_classes
        self.smoothing = smoothing
        if one_hot_classes is not None and smoothing is not None:
            assert 0.0 < smoothing < 1.0, f"If smoothing is specified it must be in (0; 1), " f"got {smoothing}"

    def __call__(self, element: Dict[str, Union[int, float, IntegerScalar]]) -> Dict[str, IntegerScalar]:
        """Read a row from your annotations dict and transfer it to a single value.

        Args:
            element: elem in your datasets

        Returns:
            dtype: Scalar value
        """
        scalar = self.dtype(element.get(self.input_key, self.default_value))
        if self.one_hot_classes is not None:
            scalar = get_one_hot(scalar, self.one_hot_classes, smoothing=self.smoothing)
        output = {self.output_key: scalar}
        return output

__all__ = ["ScalarReader"]

max_stars_repo_name: jexio/fulmo

max_issues_repo_name: jexio/fulmo

max_forks_repo_name: jexio/fulmo


text:
import os
import shutil
import sys

import numpy as np
import torch
from torch.autograd import Variable
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.transforms import Compose
from tqdm import tqdm

import preprocessing.normal_staining as ns
from dataset.tcga_dataset import TCGADataset
from models.alexnet import alexnet
from models.utils import to_scalar
from postprocessing.evaluator import Evaluator
from samplers.fast_wr_sampler import FastWeightedRandomSampler
from utils import abs_path, argmax

def train_a_epoch(name, data, model, optimizer, criterion, res_file, lr, batch_size, label2id, en):
    model = model.train(True)

    evaluator = Evaluator(name, label2id)
    print("evaluator loaded")
    i = 0
    pbar = tqdm(data, desc=name, total=np.math.ceil(len(data)))
    for images, labels, _ in pbar:
        # zero the parameter gradients
        sys.stdout.flush()
        optimizer.zero_grad()
        if i == 0:
            pbar.total = np.math.ceil(len(data))
        inp = Variable(images.cuda())
        seq_out = model(inp)
        pred = argmax(seq_out)

        loss = criterion(seq_out, Variable(torch.cuda.LongTensor(labels)))
        pbar.set_description("{0} loss: {1}".format(name, to_scalar(loss)))
        evaluator.append_data(to_scalar(loss), pred, labels)

        loss.backward()
        optimizer.step()
        i += 1

    print("Training Done")
    evaluator.gen_results()
    evaluator.print_results()
    evaluator.write_results(res_file,
                            "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(lr,
                                                                                batch_size,
                                                                                en))

    return model

def test_a_epoch(name, data, model, result_file, label2id, lr, batch_size, en):
    model = model.eval()

    full_img_paths = []
    evaluator = Evaluator(name, label2id)

    for images, labels, img_paths in tqdm(data, desc=name, total=np.math.ceil(len(data))):
        inp = Variable(images.cuda())

        seq_out = model(inp)

        pred = argmax(seq_out)
        evaluator.append_data(0, pred, labels)

        # print(predicted)
        full_img_paths.extend(img_paths)

    evaluator.gen_results()
    evaluator.print_results()
    evaluator.write_results(result_file,
                            "epoch = {2}; GOOGLE NET; lr={0}; batch_size={1}".format(lr,
                                                                                batch_size,
                                                                                en))

    return evaluator, full_img_paths

def resume(resume_file, model, optimizer):
    best_res_val = 0.0
    best_epoch = 0
    start_epoch = 0

    if os.path.isfile(resume_file):
        print("=> loading checkpoint '{}'".format(resume_file))
        checkpoint = torch.load(resume_file)
        start_epoch = checkpoint.get('epoch', 0)
        best_epoch = checkpoint.get('best_epoch', 0)
        best_res_val = checkpoint.get('best_f1', 0)
        try:
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '{}' (epoch {})"
                  .format(resume_file, checkpoint.get('epoch', 0)))
        except (KeyError, RuntimeError):
            print("Unable to load checkpoint {} (epoch {}) correctly. "
                  "Starting from scratch".format(resume_file, checkpoint.get('epoch', None)))
            best_res_val = 0.0
            best_epoch = 0
            start_epoch = 0

    else:
        print("=> no checkpoint found at '{}'".format(resume_file))

    return model, optimizer, start_epoch, best_epoch, best_res_val

def build_model(dataset,
                label2id,
                no_labels=2,
                batch_size=16,
                no_workers=8,
                train_res_file="train_results.txt",
                test_res_file="test_results.txt",
                dev_res_file="dev_results.txt",
                model_save_file="save.m",
                train_till_epoch=20,
                pretrain=True,
                lr=0.003,
                weight_decay=0.002,
                balance_train=False,
                max_epochs=100,
                resume_file="checkpoint.dict"):
    # init model
    model = alexnet(pretrained=pretrain, num_classes=no_labels)
    optimizer = torch.optim.Adadelta(model.parameters(), lr=lr)
    print("Model Loaded")

    # Turn on cuda
    model = model.cuda()
    print("Model loaded in cuda memory")

    optimizer.zero_grad()
    model.zero_grad()

    # init loss criteria
    criterion = torch.nn.CrossEntropyLoss()

    model, optimizer, start_epoch, best_epoch, best_res_val = resume(resume_file, model, optimizer)

    for epoch in range(start_epoch, max_epochs):
        print('-' * 40)
        print("EPOCH = {0}".format(epoch))
        print('-' * 40)
        if balance_train:
            class_weights, sample_labels = dataset.train.get_sample_weights(uniform=True)
            sampler = FastWeightedRandomSampler(class_weights=class_weights,
                                                sample_labels=sample_labels,
                                                max_nsamples=len(sample_labels),
                                                replacement=False,
                                                get_all=False)
            print("Using weighted random sampler. n_classes = {0}".format(len(class_weights)))
            shuffle = False
        else:
            sampler = None
            shuffle = True

        train_loader = DataLoader(dataset=dataset.train,
                                  batch_size=batch_size,
                                  num_workers=no_workers,
                                  shuffle=shuffle,
                                  sampler=sampler,
                                  collate_fn=dataset.batchify)

        print("train_loader ready")
        model = train_a_epoch(name="train",
                              data=train_loader,
                              model=model,
                              optimizer=optimizer,
                              criterion=criterion,
                              res_file=abs_path(train_res_file),
                              lr=lr,
                              batch_size=batch_size,
                              label2id=label2id,
                              en=epoch)

        dev_loader = DataLoader(dataset=dataset.dev,
                                batch_size=batch_size,
                                num_workers=no_workers,
                                shuffle=True,
                                collate_fn=dataset.batchify)

        dev_eval, _ = test_a_epoch(name="dev",
                                   data=dev_loader,
                                   model=model,
                                   result_file=abs_path(dev_res_file),
                                   label2id=label2id,
                                   lr=lr,
                                   batch_size=batch_size,
                                   en=epoch)

        dev_eval.verify_results()

        is_best = dev_eval.f > best_res_val
        save_checkpoint({
            'epoch': epoch + 1,
            'model': "alexnet",
            'best_epoch': best_epoch,
            'state_dict': model.state_dict(),
            'best_f1': best_res_val,
            'optimizer': optimizer.state_dict(),
        },
            is_best,
            resume_file,
            model_save_file)

        if epoch == 0 or dev_eval.f > best_res_val:
            best_epoch = epoch
            best_res_val = dev_eval.f

        print("current dev score: {0}".format(dev_eval.f))
        print("best dev score: {0}".format(best_res_val))
        print("best_epoch: {0}".format(str(best_epoch)))

        if 0 < train_till_epoch <= (epoch - best_epoch):
            break

    # print("Loading Best Model ...")
    best_model_checkpoint = torch.load(model_save_file)
    model.load_state_dict(best_model_checkpoint['state_dict'])

    return model, best_epoch

def save_checkpoint(state, is_best, cp_file, model_save_file):
    torch.save(state, cp_file)
    if is_best:
        shutil.copyfile(cp_file, model_save_file)

def single_run(cfg):

    print("Prepping Dataset ...")
    transform = Compose([
        ns.normalize_staining,
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406],
                             std=[0.229, 0.224, 0.225])])

    if cfg['MITOSES_MODEL_FILE']:
        filter_model = cfg['MITOSES_MODEL_FILE']
    else:
        filter_model = None

    db_man = TCGADataset(class_type=cfg['CLASS_TYPE'],
                         image_dir=cfg['IMG_DIR'],
                         label_filepath=cfg['LABEL_FILE'],
                         split=cfg['SPLIT'],
                         label2id=cfg['PURE_LABELS'],
                         transform=transform,
                         filter_model=filter_model,
                         filter_percent=cfg['MITOSES_FILTER_PERCENT'])

    test_loader = DataLoader(dataset=db_man.test,
                             batch_size=cfg['BATCH_SIZE'],
                             num_workers=cfg['NUM_WORKERS'],
                             collate_fn=db_man.batchify)

    the_model, best_en = build_model(db_man,
                                     label2id=db_man.label2id,
                                     no_labels=db_man.no_labels,
                                     batch_size=cfg['BATCH_SIZE'],
                                     no_workers=cfg['NUM_WORKERS'],
                                     train_res_file=cfg['TRAIN_RES_FILE'],
                                     test_res_file=cfg['TEST_RES_FILE'],
                                     dev_res_file=cfg['DEV_RES_FILE'],
                                     model_save_file=cfg['MODEL_SAVE_FILE'],
                                     train_till_epoch=cfg["TRAIN_TILL_EPOCH"],
                                     pretrain=cfg['PRETRAIN'],
                                     lr=cfg['LEARNING_RATE'],
                                     weight_decay=cfg['L2_REG'],
                                     balance_train=cfg['BALANCE_TRAIN'],
                                     max_epochs=cfg['MAX_EPOCH'],
                                     resume_file=cfg['CHECKPOINT_FILE'])

    print("running test")
    test_eval, img_paths = test_a_epoch(name="test",
                                        data=test_loader,
                                        model=the_model,
                                        result_file=cfg['TEST_RES_FILE'],
                                        label2id=db_man.label2id,
                                        lr=cfg['LEARNING_RATE'],
                                        batch_size=cfg['BATCH_SIZE'],
                                        en=best_en)

    test_eval.print_results()


text:
import json
import time
import numpy as np
import os
import sys

import client_config

sys.path.append('../../client_tools')
sys.path.append('../../arduino/clients')
sys.path.append('../../conductor/clients')

from PyQt4.QtGui import *
from PyQt4.QtCore import *
from twisted.internet.defer import inlineCallbacks

SEP = os.path.sep

from status_client import StatusClient
from arduino_client import ArduinoClient
from abort_button import AbortButton
from parameter_values_control import ParameterControl, ControlConfig
from refresh_button import RefreshButton

# Runs the indicator button and communicates with Labrad servers
class MainClient(QMainWindow):
    RAD = client_config.widget['rad']

    def __init__(self, reactor, parent=None):
        super(MainClient, self).__init__(parent)
        self.reactor = reactor
        self.setGeometry(0, 0, 5*self.RAD, 5*self.RAD)

        # layout the GUI
        self.setupLayout()

    # Lays out the widget
    def setupLayout(self):
        self.setWindowTitle('Auxiliary Widgets')

        self.status_widget = StatusClient(self.reactor)

        self.arduino_widget = QDockWidget()
        arduino_title = QLabel()
        arduino_title.setText('Arduino')
        arduino_title.setAlignment(Qt.AlignHCenter | Qt.AlignBottom)
        self.arduino_widget.setTitleBarWidget(arduino_title)
        self.arduino_widget.setWidget(ArduinoClient(self.reactor))

        pvc_title = QLabel()
        pvc_title.setText('Variables Control')
        pvc_title.setAlignment(Qt.AlignHCenter | Qt.AlignBottom)
        self.parameter_widget = QDockWidget()
        self.parameter_widget.setTitleBarWidget(pvc_title)

        self.scroll = QScrollArea()
        self.scroll.setWidget(ParameterControl(ControlConfig(), self.reactor))
        self.scroll.setWidgetResizable(False)
        self.parameter_widget.setWidget(self.scroll)

        self.setCentralWidget(self.status_widget)

        self.addDockWidget(Qt.RightDockWidgetArea, self.arduino_widget)
        self.addDockWidget(Qt.RightDockWidgetArea, self.parameter_widget)

        self.arduino_widget.setMaximumHeight(client_config.arduino['y'])
        #self.setFixedSize(client_config.main['x'], client_config.main['y'])

    def closeEvent(self, x):
        #stop the reactor when closing the widget
        self.alive = False
        self.reactor.stop()

# Starts the widget
if  __name__ == '__main__':
    a = QApplication( [] )
    import qt4reactor
    qt4reactor.install()
    from twisted.internet import reactor
    widget = MainClient(reactor)
    widget.setWindowTitle('Experiment Status')
    widget.setWindowIcon(QIcon('./circle.png'))
    widget.show()
    reactor.run()


text:
# -*- coding: utf-8 -*-
import numpy as np

from .. import solution

class InterfaceSolutionResult(solution.SolutionResult):
    def __init__(self, equilibrium_system, x, TK,
                 x_bulk, transport_vector, reaction_imp, relative_diffusion_vector,
                 *args, **kwargs):
        super().__init__(equilibrium_system, x, TK, **kwargs)
        if relative_diffusion_vector is None:
            self._transport_fluxes = transport_vector*(x_bulk - x)
        else:
            self._transport_fluxes = transport_vector*(x_bulk - x*relative_diffusion_vector)
        self._interface_indexes = equilibrium_system._explicit_interface_indexes + \
                                  equilibrium_system._implicit_interface_indexes
        if reaction_imp is None:
            self._reaction_fluxes = self._make_reaction_fluxes_1(equilibrium_system)
        else:
            self._reaction_fluxes = self._make_reaction_fluxes_2(equilibrium_system, reaction_imp)

    def _make_reaction_fluxes_1(self, equilibrium_system):
        #First calculate J^R, whose order will be self._interface_indexes controled
        reaction_stoich_matrix = self.solid_stoich_matrix[self._interface_indexes, :]
        balance_matrix = self.formula_matrix[2:, :] #Reduced formula matrix
        reduced_balance_matrix = balance_matrix[:, 1:] #Only solutes
        b = reduced_balance_matrix@self._transport_fluxes
        A = balance_matrix@reaction_stoich_matrix.transpose()
        jr, _, _, _ = np.linalg.lstsq(A, b, rcond=None)
        reaction_fluxes = np.zeros(len(self.solid_phase_names))
        for i, phase in enumerate(equilibrium_system.interface_phases):
            reaction_fluxes[equilibrium_system.interface_indexes_dict[phase]] = jr[i]
        return reaction_fluxes

    def _make_reaction_fluxes_2(self, equilibrium_system, jr_imp):
        #First calculate J^R, whose order will be self._interface_indexes controled
        reaction_stoich_matrix_exp = \
            self.solid_stoich_matrix[equilibrium_system._explicit_interface_indexes, :]
        reaction_stoich_matrix_imp = \
            self.solid_stoich_matrix[equilibrium_system._implicit_interface_indexes, :]
        balance_matrix = self.formula_matrix[2:, :] #Reduced formula matrix
        reduced_balance_matrix = balance_matrix[:, 1:] #Only solutes
        b1 = reduced_balance_matrix@self._transport_fluxes
        b2 = -balance_matrix@reaction_stoich_matrix_imp.transpose()@jr_imp
        b = b1 + b2
        A = balance_matrix@reaction_stoich_matrix_exp.transpose()
        if A.shape[1] == 0:
            jr_exp = np.zeros(0)
        else:
            jr_exp, _, _, _ = np.linalg.lstsq(A, b, rcond=None)
        reaction_fluxes = np.zeros(len(self.solid_phase_names))
        for i, phase in enumerate(equilibrium_system.explicit_interface_phases):
            reaction_fluxes[equilibrium_system._explicit_interface_indexes_dict[phase]] = jr_exp[i]
        for i, phase in enumerate(equilibrium_system.implicit_interface_phases):
            reaction_fluxes[equilibrium_system._implicit_interface_indexes_dict[phase]] = jr_imp[i]
        return reaction_fluxes

    @property
    def reaction_fluxes(self): #mol/m^2 s
        return {phase: self._reaction_fluxes[i] for i, phase in enumerate(self.solid_phase_names)}
                #Next make reaction fluxes including non-reacting species

    @property
    def transport_fluxes(self): #mol/m^2 s
        return {solute: self._transport_fluxes[i] for i, solute in enumerate(self.solutes)}

    @property
    def elements_reaction_fluxes(self):
        return {element: self._elements_reaction_fluxes[i] for i, element in enumerate(self.elements)}

    @property
    def _elements_reaction_fluxes(self):
        return self.formula_matrix@(self.solid_stoich_matrix.T@self._reaction_fluxes)

max_stars_repo_name: pyequion/pyequion

max_issues_repo_name: pyequion/pyequion

max_forks_repo_name: pyequion/pyequion


text:
import cv2
import numpy as np
from http.server import BaseHTTPRequestHandler, HTTPServer

class CamHandler(BaseHTTPRequestHandler):
	def do_GET(self):
		if self.path.endswith('.mjpg'):
			self.send_response(200)
			self.send_header('Content-type','multipart/x-mixed-replace; boundary=--jpgboundary')
			self.end_headers()
			while True:
				try:
					__img = self.capture_func()
					if not __img is None:
						success, __encoded = cv2.imencode(".jpg", __img)
						if success:
							self.wfile.write(b"--jpgboundary")
							self.send_header('Content-type', b'image/jpeg')
							self.send_header('Content-length', len(__encoded))
							self.end_headers()
							self.wfile.write(__encoded.tobytes())
						else:
							break
					else:
						break
				except KeyboardInterrupt:
					break

class CamServer(HTTPServer):
	def serve_forever(self, capture_func):
		self.RequestHandlerClass.capture_func = capture_func
		HTTPServer.serve_forever(self)

def serve(config, capture_func, server_class=CamServer, handler_class=CamHandler):
	server = server_class((config["host"], config["port"]), handler_class)
	server.serve_forever(capture_func)

max_stars_repo_name: SkyZH/stronghold-vision-opencv

max_issues_repo_name: SkyZH/stronghold-vision-opencv

max_forks_repo_name: SkyZH/stronghold-vision-opencv


text:
import numpy as np #the Numpy library
import matplotlib.pyplot as plt    #Matplotlibs pyplot

import sys    #gives access to a c-like sys library
import os     #ives access to operating system

print(sys.argv)    #prints any command line arguments, incl program name
print(os.getcwd())  #prints the current working direction


text:
import numpy as np

from quasinet import qnet

class Forecaster:
    """Forecast the data week by week by sequantially generating qnet predictions for the next timestamp and using the filled timestamp to update qnet predictions
    """

    def __init__(self, qnet_orchestrator):
        """Initialization

        Args:
            qnet_orchestrator (qbiome.QnetOrchestrator): an instance with a trained qnet model
        """
        self.qnet_orchestrator = qnet_orchestrator
        self.quantizer = qnet_orchestrator.quantizer

    def forecast_data(self,
                      data,
                      start_week,
                      end_week=None,
                      n_samples=100,
                      subject_id=None):
        """Forecast the data matrix from `start_week` to `end_week`

        Output format:

        |   subject_id | variable         |   week |    value |
        |-------------:|:-----------------|-------:|---------:|
        |            1 | Actinobacteriota |     27 | 0.36665  |
        |            1 | Bacteroidota     |     27 | 0.507248 |
        |            1 | Campilobacterota |     27 | 0.002032 |

        Args:
            data (numpy.ndarray): 2D array of label strings, produced by `self.get_qnet_inputs`
            start_week (int): start predicting from this week
            end_week (int, optional): end predicting after this week
            n_samples (int, optional): the number of times to sample from qnet predictions for one masked entry. Defaults to 100.
            subject_id (list[str], optional): subject id for each row of data. If None, we assume data has as many rows as quantized originally. This must be set for new patient data.

        Returns:
            pandas.DataFrame: see format above
        """
        if end_week is None:
            end_week = self.qnet_orchestrator.get_max_timestamp()
        forecasted_matrix = np.empty(data.shape)
        for idx, seq in enumerate(data):
            forecasted_seq = self.qnet_orchestrator.predict_sequentially_by_week(
                seq, start_week, end_week, n_samples=n_samples
            )
            forecasted_matrix[idx] = forecasted_seq

        if subject_id is None:
            df = self.quantizer.add_meta_to_matrix(forecasted_matrix)
        else:
            df = self.quantizer.add_meta_to_matrix(forecasted_matrix, add_subject_id=False)
            if len(subject_id) != df.index.size:
                raise('Subject_id list must match in length to number of rows in input data matrix in forecaster')
            df['subject_id']=subject_id
        # convert to plottable format
        plot_df = self.quantizer.melt_into_plot_format(df)
        return plot_df

max_stars_repo_name: zeroknowledgediscovery/qbiome

max_issues_repo_name: zeroknowledgediscovery/qbiome

max_forks_repo_name: zeroknowledgediscovery/qbiome


text:
#  from numba import jit, njit, float64, int32

from enum import IntEnum
from collections import defaultdict
import torch
import itertools

from flatland.envs.rail_env import RailEnv
from flatland.envs.agent_utils import RailAgentStatus

from flatland.envs.rail_generators import sparse_rail_generator
from flatland.envs.schedule_generators import sparse_schedule_generator
from flatland.envs.malfunction_generators import malfunction_from_params, MalfunctionParameters

from flatland.utils.rendertools import RenderTool, AgentRenderVariant

MOVEMENT_ARRAY = ((-1, 0), (0, 1), (1, 0), (0, -1))
#  @njit
def get_new_position(position, movement):
    return (position[0] + MOVEMENT_ARRAY[movement][0], position[1] + MOVEMENT_ARRAY[movement][1])

class TrainAction(IntEnum):
    NOTHING = 0
    LEFT = 1
    FORWARD = 2
    RIGHT = 3
    STOP = 4

class FlatlandWrapper():
    def __init__(self, env, reward_shaper):
        self.env = env
        self.reward_shaper = reward_shaper
        self.width = self.env.width
        self.height = self.env.height
        self._max_episode_steps = env._max_episode_steps

        #  self.n_cities = self.env.max_num_cities
        self.n_cities = 3 # TODO fix?
        self.n_agents = env.number_of_agents # check if works with remote
        self.action_sz = 3
        self.state_sz = self.env.obs_builder.state_sz
        self.steps = 0
        self.total_reward = 0
        self.max_time_steps = int(8 * (self.env.width + self.env.height + self.n_agents / self.n_cities))

        self.agents = DelegatedAttribute(self.env, "agents")
        self.rail = DelegatedAttribute(self.env, "rail")
        self.obs_builder = DelegatedAttribute(self.env, "obs_builder")
        self.distance_map = DelegatedAttribute(self.env, "distance_map")

        self.cur_env = 0

    def step(self, action_dict):
        transformed_action_dict = dict()
        for handle, value in action_dict.items():
            action = self.transform_action(handle, value)
            if action != -1:
                transformed_action_dict[handle] = action

        obs, reward, done, info = self.env.step(transformed_action_dict)
        for key in set(obs.keys()):
            if obs[key] is None:
                del obs[key]

        for handle in set(obs.keys()):
            if self.obs_builder.deadlock_checker.is_deadlocked(handle):
                done[handle] = 1 # the end

        real_reward = sum(reward.values())
        reward = dict(zip(reward.keys(), itertools.repeat(0.)))
        reward = self.reward_shaper(self, obs, action_dict, reward, done)

        self.total_reward += real_reward
        self.steps += 1

        return obs, reward, done, info, real_reward

    def reset(self):
        self.steps, self.total_reward = 0, 0
        obs, info = self.env.reset()
        self.n_agents = len(self.env.agents) # check if works with remote
        for key in set(obs.keys()):
            if obs[key] is None:
                del obs[key]
        self.reward_shaper.reset(self)
        return obs

    def greedy_position(self, handle):
        return False

    def greedy_action(self, handle):
        return None

    def reinit_greedy(self):
        pass

    def render(self):
        self.env.render()

    def get_steps(self):
        return self.steps

    def get_total_reward(self):
        return self.total_reward

    def get_available_actions(self, handle):
        agent = self.env.agents[handle]
        position = agent.position
        direction = agent.direction
        if agent.status == RailAgentStatus.READY_TO_DEPART:
            position = agent.initial_position
            direction = agent.initial_direction

        transitions = self.env.rail.get_transitions(*position, direction)
        available_actions = []
        for i in range(-1, 2): # 'L', 'F', 'R'
            new_dir = (direction + i + 4) % 4
            if transitions[new_dir]:
                available_actions.append(i + 2);
        return available_actions

    # agent chooses on of three actions:
    #   0 -- move in the leftmost direction
    #   1 -- move in the rightmost direction
    #   2 -- stop moving
    def transform_action(self, handle, action):
        self.env.obs_builder.last_action[handle] = action
        if action == 2:
            if self.env.agents[handle].status == RailAgentStatus.READY_TO_DEPART:
                return -1
            return TrainAction.STOP

        if self.env.agents[handle].status == RailAgentStatus.DONE_REMOVED:
            return -1

        available_actions = self.get_available_actions(handle)

        if len(available_actions) == 0: # DEAD-END, LET'S TURN
            return TrainAction.FORWARD
        if len(available_actions) == 1 and action == 1: # INVALID CHOICE
            return available_actions[0]
        return available_actions[action]

    def get_env_actions_new_pos(self, pos, dir, action):
        if action == 0 or action == 4: return pos, dir

        transitions = self.env.rail.get_transitions(*pos, dir)
        new_dir = (dir + action - 2 + 4) % 4
        if transitions[new_dir]:
            return get_new_position(pos, new_dir), new_dir
        return pos, dir

# Just creates env and env_renderer
class Flatland():
    def __init__(
            self,
            width,
            height,
            n_agents,
            n_cities,
            grid_distribution_of_cities,
            max_rails_between_cities,
            max_rail_in_cities,
            observation_builder,
            random_seed,
            malfunction_rate
        ):

        self.width = width
        self.height = height
        self.number_of_agents = n_agents
        self.max_num_cities = n_cities
        self.obs_builder = observation_builder

        rail_generator = sparse_rail_generator(
            max_num_cities=n_cities,
            seed=random_seed,
            grid_mode=grid_distribution_of_cities,
            max_rails_between_cities=max_rails_between_cities,
            max_rails_in_city=max_rail_in_cities,
        )

        speed_ration_map = {1. : 1.}
        schedule_generator = sparse_schedule_generator(speed_ration_map)
        stochastic_data = MalfunctionParameters(malfunction_rate, min_duration=20, max_duration=50)

        self.env = RailEnv(
            width=width,
            height=height,
            rail_generator=rail_generator,
            schedule_generator=schedule_generator,
            number_of_agents=n_agents,
            obs_builder_object=observation_builder,
            malfunction_generator_and_process_data=malfunction_from_params(stochastic_data),
            remove_agents_at_target=True,
            random_seed=random_seed
        )
        self._max_episode_steps = self.env._max_episode_steps

        self._renderer_inited = False

        self.agents = DelegatedAttribute(self.env, "agents")
        self.rail = DelegatedAttribute(self.env, "rail")
        self.obs_builder = DelegatedAttribute(self.env, "obs_builder")
        self.distance_map = DelegatedAttribute(self.env, "distance_map")

    def step(self, action_dict):
        return self.env.step(action_dict)

    def reset(self):
        if self._renderer_inited:
            self.env_renderer.reset()
        obs, info = self.env.reset()
        return obs, info

    def render(self):
        if not self._renderer_inited:
            self._renderer_inited = True
            self.env_renderer = RenderTool(
                self.env,
                agent_render_variant=AgentRenderVariant.ONE_STEP_BEHIND,
                show_debug=True,
                screen_height=600,
                screen_width=800
            )
        self.env_renderer.render_env(show=True, show_observations=False, show_predictions=False)

class DelegatedAttribute:
    def __init__(self, owner, name):
        self.owner = owner
        self.name = name

    def __getattr__(self, name):
        return getattr(getattr(self.owner, self.name), name)

    def __get__(self, instance, owner):
        return getattr(self.owner, self.name)

    def __getitem__(self, handle):
        return getattr(self.owner, self.name)[handle]

    def __len__(self):
        return len(getattr(self.owner, self.name))

max_stars_repo_name: mahkons/jbr-flatland

max_issues_repo_name: mahkons/jbr-flatland

max_forks_repo_name: mahkons/jbr-flatland

reason: from numba


text:
import numpy as np

class Actuator_model_ekv(object):

    """
        Thruster model for spacecraft computes force and torque in the body frame and converts
        to inertial frame
        Commanded thrust is clipped to lie between zero and one, and then scaled based off of
        thrust capability
        NOTE: maximum thrust is achieved with:
            tcmd = np.zeros(12)
            tcmd[0:2]=np.ones(2)
            tcmd[4:6]=np.ones(2)
            tcmd[8:10]=np.ones(2)
        and is norm([2,2,2])=3.46

    """
    def __init__(self,  pulsed=False, max_thrust_trans=2.0, max_thrust_att=0.2, debug_fail=False, fail_idx_override=None, Isp=210.0,
                 com=np.zeros(3), com_mag_max=0.0,  com_dir=None, com_fuel_scale=25):
        #                   dvec                      body position
        config = [
                      [ 0.0, -1.0,    0.0,    0.00,  -0.25,    0.0 ],  # upper -x face,
                      [ 0.0,  1.0,    0.0,    0.00,   0.25,    0.0 ],  # upper +x face,

                      [ 0.0,  0.0,    1.0,    0.0,    0.00,   0.25 ],  # left -y face,
                      [ 0.0,  0.0,   -1.0,    0.0,    0.00,  -0.25 ],  # left +y face,

                      [ 0.0,  0.0,    1.0,    0.00,  -0.25,    0.0 ],  # rotate around x      4
                      [ 0.0,  0.0,    1.0,    0.00,   0.25,    0.0 ],  # rotate around x      5

                      [ 0.0,  0.0,    1.0,    0.50,   0.00,   0.25 ],  # rotate around y      6
                      [ 0.0,  0.0,   -1.0,    0.50,   0.00,  -0.25 ],  # rotate around y      7

                      [ 0.0, -1.0,    0.0,    0.50,  -0.25,   0.00 ],  # rotate around z      8
                      [ 0.0,  1.0,    0.0,    0.50,   0.25,   0.00 ]   # rotate around z      9

                    ]

        config = np.asarray(config)

        self.dvec = config[:,0:3]

        self.position = config[:,3:6]

        self.num_actuators = 10

        trans_thrust = max_thrust_trans*np.ones(4)
        rot_thrust = max_thrust_att*np.ones(6)
        self.max_thrust1 = np.hstack((trans_thrust,rot_thrust))

        self.reward_scale_att = np.sum(rot_thrust)

        self.com = com
        self.com_dir = com_dir
        self.com_fuel_scale = com_fuel_scale
        self.com_mag_max = com_mag_max

        self.fuel_reward_scale = np.sum(self.max_thrust1)
        self.fuel_reward_scale_att = np.sum(rot_thrust)
        self.pulsed = pulsed

        self.Isp = Isp
        self.g_o = 9.81

        self.eps = 1e-8

        self.mdot = None

        self.debug_fail = debug_fail

        self.fail_idx_override = fail_idx_override

        self.fail_scale = None

        self.fail_idx = 0

        self.fail = False

        print('thruster model x( fixed ): ',max_thrust_trans, max_thrust_att, max_thrust_att / max_thrust_trans)

    def thrust_map(self,commanded_thrust):
        thrust = np.zeros(7)
        thrust = commanded_thrust
        if thrust[0] > self.eps and thrust[1] > self.eps:
            thrust[0] = thrust[1] = -1
        if thrust[2] > self.eps and thrust[3] > self.eps:
            thrust[2] = thrust[3] = -1
        if thrust[4] > self.eps and thrust[5] > self.eps:
            thrust[4] = thrust[5] = -1
        if thrust[6] > self.eps and thrust[7] > self.eps:
            thrust[6] = thrust[7] = -1
        if thrust[8] > self.eps and thrust[9] > self.eps:
            thrust[9] = thrust[9] = -1

        return thrust

    def set_action(self,commanded_thrust):
        #print('1: ', commanded_thrust)

        assert commanded_thrust.shape[0] == self.num_actuators

        if self.pulsed:
            commanded_thrust = commanded_thrust > self.eps
            commanded_thrust = self.thrust_map(commanded_thrust)
        commanded_thrust = np.clip(commanded_thrust, 0, 1.0) * self.max_thrust1

        if self.fail:
            if self.fail_idx_override is None:
                fail_idx = self.fail_idx
            else:
                fail_idx = self.fail_idx_override

            if self.debug_fail:
                orig_commanded_thrust = commanded_thrust.copy()
            commanded_thrust[fail_idx] = commanded_thrust[self.fail_idx] * self.fail_scale
            if self.debug_fail:
                if not np.all(orig_commanded_thrust ==  commanded_thrust):
                    print('orig: ', orig_commanded_thrust)
                    print('mod:  ', commanded_thrust)

        #print('2: ', commanded_thrust)

        force = np.expand_dims(commanded_thrust,axis=1) * self.dvec

        torque = np.cross(self.position + self.com, force)

        force = np.sum(force,axis=0)
        torque = np.sum(torque,axis=0)

        mdot = -np.sum(np.abs(commanded_thrust)) / (self.Isp * self.g_o)
        #print(mdot, np.sum(np.abs(commanded_thrust)), self.g_o)
        self.mdot = mdot # for rewards
        self.force = force
        self.torque = torque
        self.commanded_thrust = commanded_thrust

    def get_action(self):
        return self.commanded_thrust, self.force, self.torque, self.mdot


text:
import numpy as np

class NodeEdge:

    def __init__(self, vertex, weight):
        self.vertex = vertex
        self.weight = weight

class NodeV:

    def __init__(self, vertex, color):
        self.vertex = vertex
        self.color = color
        self.known_parents = 0
        self.AdjList = []
        self.distance = np.inf
        self.distance2 = np.inf

class Graph:

    def __init__(self, filename):

        self.vertices = []
        self.n_vertices = 0

        #read edges for undirected graph
        def readEdges(string):
            for line in string:
                vertices_data = line.split()
                if (len(vertices_data) < 3):
                    weight = 1
                else:
                    weight = int(vertices_data[2])
                self.insertEdge(int(vertices_data[0])-1, int(vertices_data[1])-1, weight)
                self.insertEdge(int(vertices_data[1])-1, int(vertices_data[0])-1, weight)

        #read edges for directed graph
        def readArcs(string):
            for line in string:
                vertices_data = line.split()
                if (len(vertices_data) < 3):
                    weight = 1
                else:
                    weight = int(vertices_data[2])
                self.insertEdge(int(vertices_data[0])-1, int(vertices_data[1])-1, weight)

        with open(filename, "r") as file:
            string = file.readlines()

            #Reading number of vertices
            line = string[0].split()
            self.n_vertices = int(line[1])

            for i in range (self.n_vertices):
                n = NodeV(i, 'W')
                self.vertices.append(n)

            line = string[1]
            if("Edges" in line):
                readEdges(string[2:])
            elif("Arcs" in line):
                readArcs(string[2:])

    def __str__(self):
        string = ""

        for i in range(0, self.n_vertices):
            string = string + "vertex:"+str(i+1) + "\n"
            for edge in self.vertices[i].AdjList:
                string = string + "  |-> "+str(edge.vertex+1)+"  "+str(edge.weight)
                string = string + "\n"

        return string

    """
    Checks whether vertex g_dest already has an edge with vertex g_src
        If it doesn't, insert new edge between them
    """
    def insertEdge(self, g_src, g_dest, weight):

        newEdge = NodeEdge(g_dest, weight)
        self.vertices[g_src].AdjList.append(newEdge)
        return

    """
    Searches in vertex g_src edge between it and g_dest and removes it
    """
    def removeEdge(self, g_src, g_dest):

        for edge in self.vertices[g_src].AdjList:
            if (edge.vertex == g_dest):
                self.vertices[g_src].AdjList.remove(edge)
                return
        return

    """
    Depth First Search
        Returns:
            Number of source vertices
            Number of sink vertices
            List with all sink vertices whose both parents descend from vertex 1
    """
    def DFS(self):

        def recursiveDFS(cur_index, S_Black, L_White, n_sink_vertices):

            cur_vertex = self.vertices[cur_index]

            if (cur_vertex.color != 'W'):
                if (len(cur_vertex.AdjList) == 0 and cur_vertex.known_parents > 1):
                    S_Black.append(cur_index)
                return n_sink_vertices

            L_White.remove(cur_index)

            if(len(cur_vertex.AdjList) > 0):

                cur_vertex.color = 'G'

                for node in cur_vertex.AdjList:
                    self.vertices[node.vertex].known_parents += 1
                    n_sink_vertices = recursiveDFS(node.vertex, S_Black, L_White, n_sink_vertices)
            else:
                n_sink_vertices += 1

            cur_vertex.color = 'B'

            return n_sink_vertices

        L_White = [] #List with non visited vertices
        S_Black = [] #List (stack) with visited vertices. Used only for sink vertices

        for i in range(0, self.n_vertices):
            self.vertices[i].color = 'W'
            L_White.append(i)

        n_source_vertices = 0
        n_sink_vertices = 0
        L_DescendFirstVertex = [] #List with sink vertices whose both parents descend from vertex 1

        #Get all necessary data from vertex 1 before checking other lineages
        root = L_White[0]
        n_sink_vertices = recursiveDFS(root, L_DescendFirstVertex, L_White, n_sink_vertices)
        n_source_vertices += 1
        L_DescendFirstVertex = sorted(L_DescendFirstVertex)

        while(len(L_White) > 0):
            root = L_White[0]
            n_sink_vertices = recursiveDFS(root, S_Black, L_White, n_sink_vertices)
            n_source_vertices += 1

        return n_source_vertices, n_sink_vertices, L_DescendFirstVertex

    """
    Breadth First Search
        Returns:
            List with all elements in L_DescendFirstVertex sorted by their distance to vertex 1
    """
    def BFS (self, L_DescendFirstVertex):
        L_White = []
        Q_Gray = []
        L_Black = [] #List with visited vertices
                        #Used only to store vertices also in L_DescendFirstVertex

        for i in range(1, self.n_vertices):
            self.vertices[i].color = 'W'
            L_White.append(self.vertices[i])

        Q_Gray.append(self.vertices[0])
        self.vertices[0].color = 'G'
        self.vertices[0].distance = 0

        while(len(Q_Gray) != 0):
            q_vertex = Q_Gray.pop(0)

            for node in q_vertex.AdjList:
                cur_vertex = self.vertices[node.vertex]

                #if minimum distance is already found, stores other distance in distance2
                if (cur_vertex.distance == np.inf):
                    cur_vertex.distance = q_vertex.distance + 1
                else:
                    cur_vertex.distance2 = q_vertex.distance + 1

                if(cur_vertex.color == 'W'):
                    cur_vertex.color = 'G'
                    Q_Gray.append(cur_vertex)
                    L_White.remove(cur_vertex)

            #append to L_Black only if q_vertex is descendant from vertex 1
            if (q_vertex.vertex in L_DescendFirstVertex):
                L_Black.append(q_vertex)
            q_vertex.color = 'B'

        #Sum the distances from both parents to each vertex in L_Black
        for v in L_Black:
            v.distance += v.distance2

        #sort according to their distance to vertex 1
        L_Black = sorted(L_Black, key=lambda vertex: vertex.distance)

        return L_Black

class main:

    filename = str(input())

    if (filename == "case.pajek"):
        filename = "case3.pajek"

    G = Graph(filename)

    n_source_vertices, n_sink_vertices, L_DescendFirstVertex = G.DFS()
    L_Black = G.BFS(L_DescendFirstVertex)

    #PART1: Number of source vertices:
    print("PART1")
    print(n_source_vertices)
    print("----------------")

    #PART2: Number of sink vertices:
    print("PART2")
    print(n_sink_vertices)
    print("----------------")

    #PART3: Number of sink vertices descendent from vertex 1:
    print("PART3")
    for i in L_DescendFirstVertex:
        print(i+1)
    print("----------------")

    #PART4: Shortest distance from L_DescendFirstVertex
    #Since L_Black has all elements from L_DescendFirstVertex sorted by their distance,
        #the shortest distance is the first element's
    shortest_dist = L_Black[0].distance
    output_shortest_dist = []
    for v in L_Black:
        if v.distance != shortest_dist:
            break
        output_shortest_dist.append(v.vertex+1)

    print("PART4")
    #sorting them by vertex number before printing
    output_shortest_dist = sorted(output_shortest_dist)
    for i in output_shortest_dist:
        print(i)
    print("----------------")


text:
import os

import numpy as np

def mkdir(paths):
    if not isinstance(paths, (list, tuple)):
        paths = [paths]
    for path in paths:
        if not os.path.isdir(path):
            os.makedirs(path, mode=0o777)

def position_encode(X, m=3, axis=1):
    x_p_list = [X]
    for i in range(m):
        x_p_list.append(np.sin((2**(i+1)) * X))
        x_p_list.append(np.cos((2**(i+1)) * X))
    return np.concatenate(x_p_list, axis=axis)


text:
# This file classify the normalized data based on SVM
#  Nose – 0, Neck – 1, Right Shoulder – 2, Right Elbow – 3, Right Wrist – 4, Left Shoulder – 5, Left Elbow – 6,
#  Left Wrist – 7, Right Hip – 8, Right Knee – 9, Right Ankle – 10, Left Hip – 11, Left Knee – 12, LAnkle – 13,
import pandas as pd
import numpy as np
from sklearn import svm
from sklearn import linear_model
from sklearn.model_selection import cross_val_score
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc

import matplotlib.pyplot as plt

def SVC_classifier(ADLDataFile, FallDataFile, features=None, k='rbf', C=1.0):
    """

    :param ADLDataFile: the normalized data of ADL
    :param FallDataFile: the normalized data of fall
    :param features: the key points chosen to train model
    :param k: kernel of svc
    :param C: the normalize index of svc
    :return: cross validation score
    """
    # default: nose, neck, shoulders, hips, knees, elbows
    if features is None:
        features = ['0x', '0y', '1x', '1y', '2x', '2y', '3x', '3y', '5x', '5y', '6x', '6y', '8x', '8y', '9x', '9y',
                    '11x', '11y', '12x', '12y']

    # generate data set
    ADL_data = pd.read_csv(ADLDataFile, index_col=0)
    ADL_training = ADL_data
    ADL_training = np.array(ADL_training)
    ADL_training = ADL_training.tolist()
    false_num = len(ADL_training)

    Fall_data = pd.read_csv(FallDataFile, index_col=0)
    Fall_training = Fall_data
    Fall_training = np.array(Fall_training)
    Fall_training = Fall_training.tolist()
    true_num = len(Fall_training)

    trainingSet = ADL_training + Fall_training

    # generate labels
    trainingLabel = []
    for i in range(false_num):
        trainingLabel.append(0)
    for i in range(true_num):
        trainingLabel.append(1)

    featureDf = pd.DataFrame(trainingSet, columns=['0x', '0y', '1x', '1y', '2x', '2y', '3x', '3y', '4x', '4y', '5x',
                                                   '5y', '6x', '6y', '7x', '7y', '8x', '8y', '9x', '9y', '10x', '10y',
                                                   '11x', '11y', '12x', '12y', '13x', '13y'])
    featureDf['Label'] = trainingLabel
    # shuffle
    shuffledTrainingDf = shuffle(featureDf)

    # select features
    trainingSet = shuffledTrainingDf[features]

    # convert label vector to one-dimension array
    trainingLabelVec = np.array(shuffledTrainingDf[['Label']]).tolist()
    trainingLabel = []
    for line in trainingLabelVec:
        trainingLabel.append(line[0])

    # SVM with rbf kernel
    svc_classifier = svm.SVC(C=C, kernel='rbf')
    svc_classifier.fit(trainingSet, trainingLabel)
    scores_svc_classifier = cross_val_score(svc_classifier, trainingSet, trainingLabel, cv=10)

    return scores_svc_classifier

scores_thigh = SVC_classifier("../data/normalized/ADLNormThigh.csv", "../data/normalized/FallNormThigh.csv")
print(scores_thigh)
scores_torso = SVC_classifier("../data/normalized/ADLNormTorso.csv", "../data/normalized/FallNormTorso.csv")
print(scores_torso)
scores_none = SVC_classifier("../data/normalized/ADLNormNone.csv", "../data/normalized/FallNormNone.csv")
print(scores_none)

# svc_thigh = svm.SVC()
#
#
# thigh_train, thigh_test, L_train, L_test = train_test_split(trainingSet, trainingLabel, test_size=.5)
# L_score = svc_thigh.fit(thigh_train, L_train).decision_function(thigh_test)
#
#
#
# fpr, tpr, thresholds = roc_curve(L_test, L_score)
#
# plt.plot(fpr, tpr)
# plt.xlim([0.0, 1.0])
# plt.ylim([0.0, 1.0])
# plt.title('ROC curve for diabetes classifier')
# plt.xlabel('False Positive Rate (1 - Specificity)')
# plt.ylabel('True Positive Rate (Sensitivity)')
# plt.grid(True)
# plt.show()


text:
"""
Random Correlation matrix (LKJ 2009)
ported R code from @rmcelreath, original see
https://github.com/rmcelreath/rethinking/blob/master/R/distributions.r#L165-L184

Created on Wed Aug  2 09:09:02 2017

@author: junpenglao
"""
import numpy as np
from scipy import stats

def lkj_random(n, eta, size=None):
    size = size if isinstance(size, tuple) else (size,)
    beta = eta - 1 + n/2
    r12 = 2 * stats.beta.rvs(a=beta, b=beta, size=size) - 1
    P = np.eye(n)[:,:,np.newaxis] * np.ones(size)
    P = np.transpose(P, (2, 0 ,1))
    P[:, 0, 1] = r12
    P[:, 1, 1] = np.sqrt(1 - r12**2)
    if n > 2:
        for m in range(1, n-1):
            beta -= 0.5
            y = stats.beta.rvs(a=(m+1) / 2., b=beta, size=size)
            z = stats.norm.rvs(loc=0, scale=1, size=(m+1, ) + size)
            z = z/np.sqrt(np.einsum('ij,ij->j', z, z))
            P[:, 0:m+1, m+1] = np.transpose(np.sqrt(y) * z)
            P[:, m+1, m+1] = np.sqrt(1-y)

    C = np.einsum('...ji,...jk->...ik', P, P)
    return C

reason: import numpy,from scipy


text:
import numpy as np

import sqlalchemy
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func

from flask import Flask, jsonify
import datetime as dt

engine = create_engine("sqlite:///Resources/hawaii.sqlite")

Base = automap_base()
Base.prepare(engine, reflect=True)
# Save references to each table
Measurement = Base.classes.measurement
Station = Base.classes.station

# Create an app, being sure to pass __name__
app = Flask(__name__)

# 3. Define what to do when a user hits the index route
@app.route("/")
def home():
    return (

        f"Welcome to the Hawaii Weather API!
"
        f"Available Routes:
"
        f"/api/v1.0/precipitation
"
        f"/api/v1.0/stations
"
        f"/api/v1.0/temperatures
"
        f"/api/v1.0/start
"
        f"/api/v1.0/start/end"
    )

# 4. Define what to do when a user hits the /about route
@app.route("/api/v1.0/precipitation/")
def precipitation():

    session = Session(engine)

    # Query results to a dictionary using `date` as the key and `prcp` as the value.
    date = dt.datetime(2016, 8, 23)
    results = session.query(Measurement.prcp, Measurement.date).filter(Measurement.date > date).order_by(Measurement.date).all()

    session.close()

    # Create a list of dictionaries
    precip_results = []
    for prcp, date in results:
        precip_dict = {}
        precip_dict["precip"] = prcp
        precip_dict["date"] = date

        precip_results.append(precip_dict)

    return jsonify(precip_results)

@app.route("/api/v1.0/stations")
def stations():
    session = Session(engine)
    station_resulst = session.query(Station.name).distinct().all()
    session.close()
    return jsonify(station_resulst)

@app.route("/api/v1.0/temperatures")
def temperatures():
    session = Session(engine)
    date = dt.datetime(2016, 8, 23)
    temp_results = session.query(Measurement.tobs, Measurement.date).\
    filter(Measurement.station == "USC00519281").\
    filter(Measurement.date > date).\
    order_by(Measurement.date).all()
    session.close()

    # Create a list of dictionaries
    temp_results_list = []
    for tobs, date in temp_results:
        temp_dict = {}
        temp_dict["temps"] = tobs
        temp_dict["date"] = date

        temp_results_list.append(temp_dict)
    return jsonify(temp_results_list)

@app.route("/api/v1.0/<start>")
def start_date(start):
    session = Session(engine)

    sel = [func.avg(Measurement.tobs),
       func.min(Measurement.tobs),
       func.max(Measurement.tobs)]

    temp_averages = session.query(*sel).filter(Measurement.date >= start).all()
    session.close()

    return jsonify(f"Here is the average temperature: {temp_averages[0][0]}, minimum temperature: {temp_averages[0][1]}, and maximum temperature: {temp_averages[0][2]} from {start} to 2017-08-23.")

@app.route("/api/v1.0/<start>/<end>")
def start_and_end_date(start, end):
    session = Session(engine)
    sel = [func.avg(Measurement.tobs),
       func.min(Measurement.tobs),
       func.max(Measurement.tobs)]

    temp_averages = session.query(*sel).\
        filter(Measurement.date >= start).\
        filter(Measurement.date <= end).all()
    session.close()

    return jsonify(f"Here is the average temperature: {temp_averages[0][0]}, minimum temperature: {temp_averages[0][1]}, and maximum temperature: {temp_averages[0][2]} from {start} to {end}.")

if __name__ == "__main__":
    app.run(debug=True)

max_stars_repo_name: cynthiakellogg/sqlalchemy-challenge

max_issues_repo_name: cynthiakellogg/sqlalchemy-challenge

max_forks_repo_name: cynthiakellogg/sqlalchemy-challenge


text:
## Contains functions to generate trajectories. Not specific to any robot.
## Author: Avadesh Meduri
## Date: 9/12/2019

import numpy as np

class TrajGenerator:
    def __init__(self, robot):

        self.robot = robot

    def get_frame_location(self, q, dq, frame_idx):
        """
        returns the global location of the frame by computing forward kinematics
        Input:
            q: current joint configuration of the robot
            dq: current joint velocity of the robot
        """

        self.robot.framesForwardKinematics(q)
        return np.reshape(
            np.array(self.robot.model.oMf[frame_idx].translation), (3,)
        )

    def generate_traj(self, start, end, traj_time, t):
        """
        returns desired location at a given time t,
        Generates a straight line start to end point(uniform velocity motion)
        Input:
            start: Start point of the trajectory.
            end: End point of the trajecory.
            traj_time: Total trajectory time
            t: Current time step
        """

        slope = np.divide(np.subtract(end, start), traj_time)

        return np.add(start, slope * t)

    def generate_sin_traj(self, start, end, via_point, traj_time, t):
        """
        returns desired location at a given time t,
        psi = sse.return_dcm_location(q, dq, step_planner.omega)
        Generates sine trajectory from start to end point through a via point
        The trajectory passes through the via point at mid duration
        Input:
            start: Start point of the trajectory.
            end: End point of the trajecory.
            via_point: The point through which trajectory will go through mid trajecory
            traj_time: Total trajectory time
            t: Current time step
        """
        assert np.shape(start) == np.shape(end)
        assert np.shape(start) == np.shape(via_point)

        if t < traj_time / 2.0:
            amplitude = np.subtract(via_point, start)
            omega = (np.pi) / traj_time
            return np.add(start, amplitude * np.sin(omega * t))
        elif t == traj_time / 2.0:
            return via_point
        else:
            amplitude = np.subtract(via_point, end)
            omega = (np.pi) / traj_time
            return np.add(end, amplitude * np.sin(omega * t))

    def generate_foot_traj(self, start, end, via_point, traj_time, t):
        """
        Generates foot trajectory for walking. A uniform velocity is tracked in x,y direction.
        A sine trajectory is generated in the z direction.
        Input:
            Start: Current location of the foot 3d
            end: desired location of the of the foot at the end of the step
            via_point: the hieght in the z axis the leg has to rise
            traj_time: step time
            t: current time
        """

        assert np.shape(start) == (3,)
        assert np.shape(end) == (3,)

        x_des = np.zeros(3)
        x_des[0:2] = self.generate_traj(start[0:2], end[0:2], traj_time, t)
        x_des[2] = self.generate_sin_traj(
            start[2], end[2], via_point[2], traj_time, t
        )

        return x_des