text:
"""This module contains functions and classes to read and check input files."""

import json
import math
from typing import Dict, NamedTuple, List

import networkx as nx
import pandas as pd

# Type aliases used to make the types for the functions below easier to read
Age = str
Compartment = str
NodeName = str

# CurrentlyInUse
def _checkAgeParameters(agesDictionary):
    """Check the consistency of data within the ages dictionary.

    :param agesDictionary:
    :type agesDictionary: dictionary
    :return: agesDictionary
    """

    all_compartments = None
    for age, compartments in agesDictionary.items():
        if all_compartments is None:
            all_compartments = list(compartments.keys())
        else:
            assert all_compartments == list(compartments.keys()), f"compartments mismatch in {age}"
        for compartment, transitions in compartments.items():
            assert compartment in transitions.keys(), f"{age},{compartment} does not have self referencing key"
            assert math.isclose(sum(transitions.values()), 1.0), f"{age},{compartment} transitions do not add up to 1.0"
            for new_name, prob in transitions.items():
                assert 0.0 <= prob <= 1.0, f"{age},{compartment},{new_name},{prob} not a valid probability"

    return agesDictionary

# CurrentlyInUse
def readCompartmentRatesByAge(table: pd.DataFrame,) -> Dict[Age, Dict[Compartment, Dict[Compartment, float]]]:
    """Read a file containing a list of age-specific epidemiological parameters.

    Epidemiological parameters are transition rates between or out of epidemiological compartments,
    here they differ by age group.

    :param table: Age transition data
    :return: nested dictionary with progression rates
    """
    agesDictionary = {}
    for row in table.to_dict(orient="row"):
        compartments = agesDictionary.setdefault(row["age"], {})
        transitions = compartments.setdefault(row["src"], {})
        transitions[row["dst"]] = float(row["rate"])
    return _checkAgeParameters(agesDictionary)

# CurrentlyInUse
def readPopulationAgeStructured(table: pd.DataFrame) -> Dict[NodeName, Dict[Age, int]]:
    """Read a file containing population data.

    Population is labelled by node ID, sex and age. Sex is currently ignored.

    :param table: Population data
    :return: Nested dict with age-stratified population in each node
    """
    dictOfPops = {}

    for row in table.to_dict(orient="row"):
        board = dictOfPops.setdefault(row["Health_Board"], {})
        total = int(row["Total"])
        if total < 0:
            raise ValueError(f"invalid total {total}")
        # We are ignoring the Sex column here. The same age group will appear multiple times (once per age) and we will
        # end up just grouping them all together
        board.setdefault(row["Age"], 0)
        board[row["Age"]] += total

    return dictOfPops

# CurrentlyInUse
# making this general to include arbitrary future attributes.  Location is the primary one for right now
# keeps them in a dictionary and returns that.  Keys are
def readNodeAttributesJSON(filename):
    """Read a file containing node (health board) attributes.

    This might include health board name and geographic co-ordinates.

    :param filename: Health board attributes in json format.
    :type filename: file-like object
    :return: Dictionary of health board attributes.
    """
    f = open(filename,)
    node_data = json.load(f)
    return node_data

# CurrentlyInUse
# at the moment this uses vanilla networkx edge list reading - needs weights
#  I've set it apart as its own function in case we want to do anything fancier with edge files
# in future - e.g. sampling, generating movements, whatever
# it should return a networkx graph, ideally with weighted edges
# eventual replacement with HDF5 reading code?
def genGraphFromContactFile(commutes: pd.DataFrame) -> nx.DiGraph:
    """Read a file containing edge weights between nodes.

    Pairs of nodes are listed in the file by source, destination, weight and adjustment.

    :param commutes: Weighted edge list.
    :return: `networkx.classes.digraph.DiGraph` object representing the graph.
    """
    G = nx.convert_matrix.from_pandas_edgelist(commutes, edge_attr=True, create_using=nx.DiGraph)
    for edge in G.edges.data():
        assert edge[2]["weight"] >= 0.0
        assert edge[2]["delta_adjustment"] >= 0.0
    return G

def readInitialInfections(df: pd.DataFrame) -> Dict[NodeName, Dict[Age, float]]:
    """Read initial numbers of infected individuals by health board and age.

    :param df: raw data to be loaded
    :return: nested dict with number of infected per age per node
    """
    infections: Dict[str, Dict[str, float]] = {}
    for row in df.to_dict(orient="row"):
        if float(row["Infected"]) >= 0.0 and row["Infected"] != math.inf:
            infections.setdefault(row["Health_Board"], {})[row["Age"]] = row["Infected"]
        else:
            raise ValueError(f"Invalid infected value: {row['Infected']}")
    return infections

class Multiplier(NamedTuple):
    movement: float
    contact: float

def _assertPositiveNumber(value: float):
    if value < 0.0 or math.isinf(value) or math.isnan(value):
        raise ValueError(f"{value} must be a positive number")

def readMovementMultipliers(table: pd.DataFrame) -> Dict[int, Multiplier]:
    """Read file containing movement multipliers by time.

    :param table: pandas DataFrame containing movement multipliers
    :return: A dict of ints (time) pointing to a (named)tuple (contact=float, movement=float). Contact will alter how
             the disease spreads inside of a node, whereas movement will change how it spread across nodes
    """
    multipliers = {}
    for row in table.to_dict(orient="row"):
        time = int(row["Time"])
        if time < 0:
            raise ValueError("can't have negative time")

        movement = float(row["Movement_Multiplier"])
        _assertPositiveNumber(movement)

        contact = float(row["Contact_Multiplier"])
        _assertPositiveNumber(contact)

        multipliers[time] = Multiplier(movement=movement, contact=contact)

    return multipliers

def readInfectiousStates(infectious_states: pd.DataFrame) -> List[Compartment]:
    """
    Transforms the API output of infectious_states into the internal representation: a list of strings
    :param infectious_states: pandas DataFrame with the raw data from the API
    :return: a list of strings of infectious compartments
    """
    if infectious_states.size == 0:
        return []
    return list(infectious_states.Compartment)

def readInfectionProbability(df: pd.DataFrame) -> Dict[int, float]:
    """
    Transforms the dataframe from the data API into a dict usable inside the model

    :param df: a timeseries of infection probabilities
    :return: a timeseries of infection probabilities in the dict format
    """
    probs: Dict[int, float] = {}
    has_time_zero = False
    for row in df.to_dict(orient="row"):
        time = int(row["Time"])
        if time == 0:
            has_time_zero = True
        elif time < 0:
            raise ValueError("can't have negative time")
        value = float(row["Value"])
        if value < 0.0 or value > 1.0 or math.isnan(value):
            raise ValueError("infection probability must be between 0 and 1")
        probs[time] = value

    if not has_time_zero:
        raise ValueError("the infection probability needs to be present since the time 0")

    return probs

def readRandomSeed(df: pd.DataFrame) -> int:
    """
    Transforms the dataframe from the data API into a bool usable inside the model

    :param df: a dataframe containing the random seed
    :return: a value of using the random seed as an int
    """
    if df is None:
        return 0

    assert len(df) == 1
    assert df.columns == ["Value"]

    for row in df.to_dict(orient="row"):
        seed = row["Value"]

        if not isinstance(seed, int):
            raise ValueError("Seed must be an int")

        if seed < 0:
            raise ValueError("Seed must be positive")

        return seed

def readTrials(df: pd.DataFrame) -> int:
    """
    Transforms the dataframe from the data API into a bool int inside the model

    :param df: The dataframe containing the number of trials
    :return: the number of trials to run
    """
    assert len(df) == 1
    assert df.columns == ["Value"]

    for row in df.to_dict(orient="row"):
        trials = row["Value"]

        if not isinstance(trials, int):
            raise ValueError("trials must be an int")

        if trials < 1:
            raise ValueError("trials must be > 0")

        return trials

def readStochasticMode(df: pd.DataFrame) -> bool:
    """
    Transforms the dataframe from the data API into a bool usable inside the model

    :param df: a dataframe with the stochastic mode
    :return: a value of using the stochastic mode as a bool
    """
    if df is None:
        return False

    assert len(df) == 1
    assert df.columns == ["Value"]

    for row in df.to_dict(orient="row"):
        stochastic_mode = row["Value"]

        if not isinstance(stochastic_mode, bool):
            raise ValueError("stochastic_mode must be bool")

        return bool(stochastic_mode)

def _check_overlap(one, two):
    """Check two AgeRange objects to see if they overlap.

    If they do, raise an Exception.
    :param one:
    :type one: simple_network_sim.loaders.AgeRange
    :param two:
    :type two: simple_network_sim.loaders.AgeRange
    """
    assert one._upper <= two._lower or two._upper <= one._lower, f"Overlap in age ranges with {one} and {two}"

class AgeRange:
    """A helper class for an age range.

    The age_group parameter can be any string, but it is usually in the format [a,b) or 70+
    """

    def __init__(self, age_group: str):
        """Initialise."""
        self.age_group = age_group

    def __str__(self):
        """Return a string representation of the current AgeRange instance."""
        return self.age_group

    def __eq__(self, other: "AgeRange"):
        """Return true if "other" is the same as the current AgeRange instance."""
        return self.age_group == other.age_group

    def __neq__(self, other: "AgeRange"):
        """Return true if "other" is not the same as the current AgeRange instance."""
        return not self == other

    def __hash__(self):
        """Return a hash of the current AgeRange instance."""
        return hash(self.age_group)

class MixingRow:
    """One row of a mixing table.

    This is a helper class. A row represents a given population,
    and can return the expected number of interactions (per unit time)
    a member of this population will have with some other target.

    :param ages: Ages
    :type ages: list
    :param interactions: Interactions / person / day
    :type interactions: list

    Both lists must be the same length.

    Upon initialization, the property `self._entries` is set to a dictionary mapping
    AgeRange objects to numbers of interactions / person / day.
    """

    def __init__(self, ages: List[str], interactions: List[str]):
        """Initialise."""
        self._entries = {}
        for age, interact in zip(ages, interactions):
            self._entries[age] = float(interact)

    def __getitem__(self, age: str) -> float:
        """Return expected number of interactions.

        Return the expected number of interactions (per day) that someone from
        this MixingRow would have with someone with the given age, or age
        range.
        """
        return self._entries[age]

    def __str__(self):
        """Return a string representing the current MixingRow instance."""
        return "[" + ", ".join(f"{str(key)}: {str(val)}"
                               for key, val in self._entries.items()) + "]"

    def __iter__(self):
        """Iterate through age-interactions dictionary."""
        return iter(age_range for age_range in self._entries)

class MixingMatrix:
    """Stores expected numbers of interactions between people of different ages.

    Stores expected number of interactions per day a given person will
    have with some other person, based on the ages of the two people, or given
    age-ranges.

    Examples:
    `mm = MixingMatrix(api.read_table("human/mixing-matrix")`
    `print(mm[28][57])` Prints the expected number of interactions a 28 year old
    would have with a 57 year old in a day
    `print(mm["[30,40)"]["70+"])` or `print(mm[(30,40)]["70+"])` Prints the expected number of interactions someone in
    the age range [30-40) would have with someone aged 70 or older
    in any given day.

    :param mixing_table: Raw DataFrame from the data API. The expected columns are: source, target and mixing (value).
    """

    def __init__(self, mixing_table: pd.DataFrame):
        """Initialise."""
        self._matrix = {
            group_name: MixingRow([target for target in group["target"]], list(group["mixing"]))
            for group_name, group in mixing_table.groupby("source")
        }

    def __getitem__(self, age: str) -> MixingRow:
        """Return MixingRow for given age.

        Gets a MixingRow for the given age, which in turn can give the
        expected number of interactions. Most often, you will probably want to
        just use MixingMatrix[age1][age2] to get the expected number of
        interactions that a person of age1 will have with a person of age2.
        Note that either age1 or age2 can be numbers, or age ranges.
        """
        return self._matrix[age]

    def __str__(self):
        """Return a string representing the current MixingMatrix instance."""
        return "\n".join(f"{str(key)}: {str(row)}" for key, row in self._matrix.items())

    def __iter__(self):
        """Iterate by row.

        Iterator that iterates over the matrix keys (a key points to a row). The values returned by the iterator will
        all be strings, since that's how the public interface when indexing the matrix.
        """
        return iter(age_range for age_range in self._matrix)

reason: import networkx


text:
import io

import cv2
import numpy as np
import torch
from PyQt5 import QtWidgets
from PyQt5.QtCore import QSize
from PyQt5.QtGui import QImage, QPixmap

from backend.tab_widget.item import CustomItem

class InfoTab(QtWidgets.QWidget):
    def __init__(self, parent, bucket, database, user_id):
        super(InfoTab, self).__init__(parent)

        # Set up database
        self.bucket = bucket
        self.database = database
        self.user_id = user_id
        self.optimizer = torch.optim.Adam(self.parent().video_container.model.parameters(), lr=3e-4)

        self.monitor = QtWidgets.QLabel()
        self.monitor.setFixedSize(500, 500)

        self.reload_btn = QtWidgets.QPushButton()
        self.reload_btn.setStyleSheet(
            """
            QPushButton {
                padding: 5px;
                border: 1px solid gray;
                border-radius: 5px;
                background-color: white;
            }
            QPushButton::hover {
                background-color: black;
                color: white;
            }
            """
        )
        self.reload_btn.setText("reload list")
        self.reload_btn.clicked.connect(self.reload_list)
        self.delete_info_btn = QtWidgets.QPushButton()
        self.delete_info_btn.setStyleSheet(
            """
            QPushButton {
                padding: 5px;
                border: 1px solid gray;
                border-radius: 5px;
                background-color: white;
            }
            QPushButton::hover {
                background-color: black;
                color: white;
            }
            """
        )
        self.delete_info_btn.setText("delete")
        self.delete_info_btn.clicked.connect(self.delete_selected_info)
        self.list_view = QtWidgets.QListWidget()
        self.list_view.itemClicked.connect(self.list_item_selected)
        self.list_view.setStyleSheet(
            """
                QListWidget::item {
                    border: 0.5px solid gray;
                }
                QListWidget::item:selected {
                    border: 2px solid black;
                }
            """
        )
        secure_info = self.database.get(f"/users/{self.user_id}", "secure")

        if secure_info is not None:
            for keys, values in secure_info.items():
                custom_widget = CustomItem(self)
                custom_widget_item = QtWidgets.QListWidgetItem(self.list_view)
                custom_widget_item.setSizeHint(QSize(100, 100))
                custom_widget.set_pass_info(values["pass"], values["name"])
                custom_widget.set_time_info(keys)

                self.list_view.addItem(custom_widget_item)
                self.list_view.setItemWidget(custom_widget_item, custom_widget)

        side_h_box = QtWidgets.QHBoxLayout()
        side_h_box.addWidget(self.reload_btn)
        side_h_box.addWidget(self.delete_info_btn)
        side_v_box = QtWidgets.QVBoxLayout()
        side_v_box.addLayout(side_h_box)
        side_v_box.addWidget(self.list_view)
        main_h_box = QtWidgets.QHBoxLayout()
        main_h_box.addWidget(self.monitor, 1)
        main_h_box.addLayout(side_v_box)

        self.setLayout(main_h_box)

    def reload_list(self):
        self.list_view.clear()

        secure_info = self.database.get(f"/users/{self.user_id}", "secure")

        if secure_info is not None:
            for keys, values in secure_info.items():
                custom_widget = CustomItem()
                custom_widget_item = QtWidgets.QListWidgetItem(self.list_view)
                custom_widget_item.setSizeHint(QSize(100, 100))
                custom_widget.set_pass_info(values["pass"], values["name"])
                custom_widget.set_time_info(keys)

                self.list_view.addItem(custom_widget_item)
                self.list_view.setItemWidget(custom_widget_item, custom_widget)

        self.monitor.clear()

    def delete_selected_info(self):
        if len(self.list_view.selectedItems()) != 0:
            file_name = self.list_view.itemWidget(self.list_view.selectedItems()[0]).time_info.text()
            self.database.delete(f"/users/{self.user_id}/secure", file_name)
            blob = self.bucket.blob(file_name)
            try:
                blob.delete()
            except Exception:
                pass
            self.list_view.takeItem(self.list_view.selectionModel().selectedRows()[0].row())
            self.monitor.clear()

    def list_item_selected(self, item):
        try:
            file_name = item.listWidget().itemWidget(item).time_info.text()
            blob = self.bucket.blob(file_name)

            # Create temp_file_bytes
            temp_file_bytes = io.BytesIO()
            blob.download_to_file(temp_file_bytes)

            image = np.frombuffer(temp_file_bytes.getbuffer(), dtype="uint8")
            image = cv2.imdecode(image, flags=1)
            height, width, channels = image.shape
            bytes_per_line = width * channels
            converted_Qt_image = QImage(image, width, height, bytes_per_line, QImage.Format_RGB888)

            self.monitor.setPixmap(
                QPixmap.fromImage(converted_Qt_image).scaled(
                    self.monitor.width(), self.monitor.height())
            )
        except Exception as e:
            self.monitor.setText("No image")

ext: py

reason: import numpy


text:
import numpy as np
import yaml
import torch
from abc import ABC, abstractmethod
import os

from os.path import join, exists, dirname, abspath

# use relative import for being compatible with Open3d main repo
from ...utils import Config, make_dir

class BasePipeline(ABC):
    """Base pipeline class."""

    def __init__(self,
                 model,
                 dataset=None,
                 device='cuda',
                 distributed=False,
                 **kwargs):
        """Initialize.

        Args:
            model: A network model.
            dataset: A dataset, or None for inference model.
            device: 'gpu' or 'cpu'.
            distributed: Whether to use multiple gpus.
            kwargs:

        Returns:
            class: The corresponding class.
        """
        self.cfg = Config(kwargs)

        if kwargs['name'] is None:
            raise KeyError("Please give a name to the pipeline")
        self.name = self.cfg.name

        self.model = model
        self.dataset = dataset
        self.rng = np.random.default_rng(kwargs.get('seed', None))

        self.distributed = distributed
        if self.distributed and self.name == "SemanticSegmentation":
            raise NotImplementedError(
                "Distributed training not implemented for SemanticSegmentation!"
            )

        self.rank = kwargs.get('rank', 0)

        dataset_name = dataset.name if dataset is not None else ''
        self.cfg.logs_dir = join(
            self.cfg.main_log_dir,
            model.__class__.__name__ + '_' + dataset_name + '_torch')

        if self.rank == 0:
            make_dir(self.cfg.main_log_dir)
            make_dir(self.cfg.logs_dir)

        if device == 'cpu' or not torch.cuda.is_available():
            if distributed:
                raise ValueError(
                    "Distributed training is ON, but CUDA not available.")
            self.device = torch.device('cpu')
        else:
            if distributed:
                self.device = torch.device(device)
                print(f"Rank : {self.rank} using device : {self.device}")
                torch.cuda.set_device(self.device)
            else:
                self.device = torch.device('cuda')
        self.summary = {}
        self.cfg.setdefault('summary', {})

    @abstractmethod
    def run_inference(self, data):
        """Run inference on a given data.

        Args:
            data: A raw data.

        Returns:
            Returns the inference results.
        """
        return

    @abstractmethod
    def run_test(self):
        """Run testing on test sets."""
        return

    @abstractmethod
    def run_train(self):
        """Run training on train sets."""
        return


text:
import pdb  # noqa
import sys

import numpy as np
import rosbag
import yaml

# static parameters
CUBE_TOPIC_STRING = '/tagslam/odom/body_cube'
BOARD_TOPIC_STRING = '/tagslam/odom/body_surface'

if len(sys.argv) < 3:
    print("usage: python[2] apriltag_csv.py [ROSBAG] [CSV_OUT]")
    sys.exit()

inbag = sys.argv[1]
outcsv = sys.argv[2]

# open rosbag
bag = rosbag.Bag(inbag)

# get summary info from rosbag as a dictionary
info = yaml.load(bag._get_yaml_info())

# extract metadata from cube and board topics
cube_topic = [topic for topic in info['topics'] if topic['topic'] == CUBE_TOPIC_STRING][0]
board_topic = [topic for topic in info['topics'] if topic['topic'] == BOARD_TOPIC_STRING][0]

# ensure there are an equal number of cube and board odom messages
num_msg = cube_topic['messages']
if not num_msg == board_topic['messages']:
    raise Exception('Missing odom messages for board and/or cube!')

def extract_times(messages):
    t_ros = np.zeros(len(messages))
    for i, data in enumerate(list(messages)):
        (_, msg, _) = data
        tstamp = msg.header.stamp
        t_ros[i] = tstamp.secs + tstamp.nsecs * 1e-9

    return t_ros

def extract_poses(messages):
    poses = np.zeros((7, len(messages)))
    for i, data in enumerate(messages):
        (_, msg, _) = data
        pose = msg.pose.pose
        pose_pos = np.asarray([pose.position.x, pose.position.y, pose.position.z])
        pose_quat = np.asarray([pose.orientation.x, pose.orientation.y,
                                pose.orientation.z, pose.orientation.w])
        poses[:4, i] = pose_quat
        poses[4:7, i] = pose_pos

    return poses

t_ros = extract_times(list(bag.read_messages(topics=[CUBE_TOPIC_STRING])))
cube_ros = extract_poses(list(bag.read_messages(topics=[CUBE_TOPIC_STRING])))
board_ros = extract_poses(list(bag.read_messages(topics=[BOARD_TOPIC_STRING])))

bag.close()

# assemble CSV data and save
data = np.concatenate((np.expand_dims(t_ros, axis=0), cube_ros, board_ros), axis=0)
data = data.T
np.savetxt(outcsv, data, delimiter=',')

max_stars_repo_name: DAIRLab/contact-nets

max_issues_repo_name: DAIRLab/contact-nets

max_forks_repo_name: DAIRLab/contact-nets


text:
import logging
from math import gcd

from sage.all import ZZ

from shared import small_roots
from shared.polynomial import max_norm

def integer_bivariate(p, k, X, Y, roots_method="resultants"):
    """
    Computes small integer roots of a bivariate polynomial.
    More information: Coron J., "Finding Small Roots of Bivariate Polynomial Equations Revisited"
    Note: integer_bivariate in the coron_direct will probably be more efficient.
    :param p: the polynomial
    :param k: the amount of shifts to use
    :param X: an approximate bound on the x roots
    :param Y: an approximate bound on the y roots
    :param roots_method: the method to use to find roots (default: "resultants")
    :return: a generator generating small roots (tuples of x and y roots) of the polynomial
    """
    pr = p.parent()
    x, y = pr.gens()
    delta = max(p.degrees())

    _, W = max_norm(p(x * X, y * Y))

    p00 = int(p.constant_coefficient())
    assert p00 != 0 and gcd(p00, X * Y) == 1
    u = W + (1 - W) % abs(p00)
    n = u * (X * Y) ** k

    q = (pow(p00, -1, n) * p) % n
    q = q.change_ring(ZZ)

    logging.debug("Generating shifts...")

    shifts = set()
    monomials = set()
    for i in range(k + delta + 1):
        for j in range(k + delta + 1):
            if i <= k and j <= k:
                shift = x ** i * y ** j * X ** (k - i) * Y ** (k - j) * q
            else:
                shift = x ** i * y ** j * n
            shifts.add(shift)
            monomials.update(shift.monomials())

    L = small_roots.fill_lattice(shifts, monomials, [X, Y])
    L = small_roots.reduce(L)
    polynomials = small_roots.reconstruct_polynomials(L, monomials, [X, Y])
    for roots in small_roots.find_roots(p, polynomials, pr, method=roots_method):
        yield roots[x], roots[y]

max_stars_repo_name: jvdsn/crypto-attacks

max_issues_repo_name: jvdsn/crypto-attacks

max_forks_repo_name: jvdsn/crypto-attacks

reason: from sage


text:
from datetime import date, timedelta

import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score

from factory import DaoFactory
from publicdata.MarketState import MarketState
from utils import DateUtil

class StockBl(object):
    def __init__(self):
        self.stock_dao = DaoFactory.stockDao
        self.invest_dao = DaoFactory.investDao

    def calculate_max_back(self, input_list: []):
        max_back = 0
        now_max_back = 0
        is_now_max_back = False
        max_start_index = 0
        max_end_index = 0
        max_back_return_index = 0
        start_index = 0
        start_value = input_list[0]
        last_value = start_value
        for i in range(1, input_list.__len__()):
            if input_list[i] <= last_value:
                now_max_back = input_list[i] - start_value
                last_value = input_list[i]
                is_now_max_back = False
            else:
                if not is_now_max_back:
                    if max_back > is_now_max_back:
                        max_back = now_max_back
                        max_start_index = start_index
                        max_end_index = i - 1
                start_index = i
                start_value = input_list[i]
        for i in range(max_end_index, input_list.__len__()):
            if input_list[i] >= input_list[max_start_index]:
                max_back_return_index = i - max_end_index
                break
        max_back_start_date = date(2016, 9, 1) + timedelta(max_start_index)
        max_back_end_date = date(2016, 9, 1) + timedelta(max_end_index)
        return max_back, max_back_start_date, max_back_end_date, max_back_return_index

    def calculate_quarter_regression(self, big_growg, big_worthg, mid_growg, mid_worthg, small_growg,
                                     small_worthg, myfundg):
        style_ratio = []
        styleg = []
        r_2 = []

        for i in range(np.math.floor(big_growg.__len__() / 90)):
            index = i * 90
            big_growg_block = big_growg[index:index + 90]
            big_worthg_block = big_worthg[index:index + 90]
            mid_growg_block = mid_growg[index:index + 90]
            mid_worthg_block = mid_worthg[index:index + 90]
            small_growg_block = small_growg[index:index + 90]
            small_worthg_block = small_worthg[index:index + 90]
            myfundg_block = myfundg[index:index + 90]
            train_x = []
            train_x.append(big_growg_block)
            train_x.append(big_worthg_block)
            train_x.append(mid_growg_block)
            train_x.append(mid_worthg_block)
            train_x.append(small_growg_block)
            train_x.append(small_worthg_block)
            train_x = np.transpose(train_x)
            train_y = myfundg_block
            linreg = LinearRegression()
            linreg.fit(train_x, train_y)
            power = linreg.coef_
            power.__add__(linreg.intercept_)
            style_ratio.append(power)
            pred = linreg.predict(train_x)
            for day_data in pred:
                styleg.append(day_data)
            r_2.append(r2_score(train_y, pred))
        return style_ratio, styleg, r_2

    def transform_list_with_date(self, list_with_date: []):
        result = []
        for i in range(list_with_date.__len__()):
            result.append(list_with_date[i][1])
        return result

    def get_achievement(self, req_id):
        stocks, numbers = self.invest_dao.get_stock_position(req_id)
        ratios = np.array(numbers) / np.array(numbers).sum()
        start_date = date(2016, 9, 1)
        end_date = date(2018, 6, 1)
        stocks_price = []
        for stock in stocks:
            _, values = self.stock_dao.get_stock_period_price(stock, start_date, end_date)
            for i in range(values.__len__()):
                if np.isnan(values[i]):
                    values[i] = 0
            stocks_price.append(values)
        myfund = []
        for i in range(stocks_price[0].__len__()):
            fund = 0
            for j in range(stocks_price.__len__()):
                fund += stocks_price[j][i] * ratios[j]
            myfund.append(fund)
        base_300 = self.transform_list_with_date(self.stock_dao.get_csi_300(start_date, end_date))
        base_300g = [0]
        myfundg = [0]
        for i in range(base_300.__len__() - 1):
            base_300g.append((base_300[i + 1] - base_300[i]) / base_300[i])
        for i in range(myfund.__len__() - 1):
            myfundg.append((myfund[i + 1] - myfund[i]) / myfund[i])
        revenueg = []
        for i in range(myfundg.__len__()):
            revenueg.append(myfundg[i] - base_300g[i])
        revenuem = []
        for i in range(np.math.floor(revenueg.__len__() / 30)):
            index = i * 30
            month_revenueg = np.average(myfundg[index:index + 30])
            revenuem.append(month_revenueg)
        revenueq = []
        for i in range(np.math.floor(revenueg.__len__() / 90)):
            index = i * 90
            quarter_revenueg = np.average(myfundg[index:index + 90])
            revenueq.append(quarter_revenueg)
        revenue = []
        for i in range(myfund.__len__()):
            revenue.append(myfund[i] - base_300[i])
        back = []
        for i in range(revenue.__len__()):
            max_revenue = max(revenue[0:i + 1])
            value = (revenue[i] - max_revenue) / max_revenue
            if value < 0:
                value = 0
            back.append(value)

        # 开始设置返回值
        chart_data = []
        for i in range(revenueg.__len__()):
            chart_data.append({
                'date': DateUtil.get_format_date(start_date + timedelta(i)),
                'returnNum': "%.2f" % revenueg[i],
                'revenueNum': "%.2f" % back[i]
            })
        form_data = []
        for j in range(stocks.__len__()):
            stock_price = self.stock_dao.get_stock_period_price(stocks[j], start_date, end_date)[1]
            stock_priceg = [0]
            for i in range(myfund.__len__() - 1):
                stock_priceg.append((stock_price[i + 1] - stock_price[i]) / stock_price[i])
            stock_revenueg = []
            for i in range(stock_priceg.__len__()):
                stock_revenueg.append(stock_priceg[i] - base_300g[i])
            stock_revenuem = []
            for i in range(np.math.floor(stock_revenueg.__len__() / 30)):
                index = i * 30
                month_stock_revenueg = np.average(stock_revenueg[index:index + 30])
                stock_revenuem.append(month_stock_revenueg)
            stock_winm_num = 0
            for i in range(stock_revenuem.__len__()):
                if stock_revenuem[i] > 0:
                    stock_winm_num += 0
            stock_winm = stock_winm_num * 1.0 / stock_revenuem.__len__()
            stock_revenue = []
            for i in range(stock_price.__len__()):
                stock_revenue.append(stock_price[i] - base_300[i])
            stock_back = []
            for i in range(stock_revenue.__len__()):
                max_revenue = max(stock_revenue[0:i + 1])
                value = (stock_revenue[i] - max_revenue) / max_revenue
                if value < 0:
                    value = 0
                stock_back.append(value)
            stock_max_back, stock_max_back_start_date, stock_max_back_end_date, stock_max_back_return_time = self.calculate_max_back(
                stock_back)  # 超额最大回撤,超额最大回撤起开始日,超额最大回撤起终止日,超额最大回撤回补日
            form_data.append({
                'quotaName': stocks[j],  # 名称
                'period': 7,  # 周期数：月
                'totalRevenue': stock_max_back,  # 超额累计回撤率：百分比
                'monthly': stock_winm,  # 月胜率：百分比
                'maxReturn': "%.2f" % sum(stock_revenueg),  # 超额最大回撤率：百分比
                'maxReturnBegin': DateUtil.get_format_date(stock_max_back_start_date),  # 超额最大回撤起始日期：date
                'maxReturnEnd': DateUtil.get_format_date(stock_max_back_end_date),  # 超额最大回撤结束日期：date
                'maxReturnSupplement': stock_max_back_return_time  # 超额最大回撤补期：天
            })
        return {
            'chartData': chart_data,
            'formData': form_data
        }

    def get_reason(self, req_id):
        stocks, numbers = self.invest_dao.get_stock_position(req_id)
        stocks = ["002495.XSHE", "603189.XSHG"]
        numbers = [200, 100]
        ratios = np.array(numbers) / np.array(numbers).sum()
        start_date = date(2016, 9, 1)
        end_date = date(2018, 6, 1)
        base_300 = self.stock_dao.get_csi_300(start_date, end_date)
        big_grow = self.stock_dao.get_big_grow(start_date, end_date)
        big_worth = self.stock_dao.get_big_worth(start_date, end_date)
        mid_grow = self.stock_dao.get_mid_grow(start_date, end_date)
        mid_worth = self.stock_dao.get_mid_worth(start_date, end_date)
        small_grow = self.stock_dao.get_small_grow(start_date, end_date)
        small_worth = self.stock_dao.get_small_worth(start_date, end_date)
        stocks_price = []
        for stock in stocks:
            _, values = self.stock_dao.get_stock_period_price(stock, start_date, end_date)
            for i in range(values.__len__()):
                if np.isnan(values[i]):
                    values[i] = 0
            stocks_price.append(values)
        myfund = []
        for i in range(stocks_price[0].__len__()):
            fund = 0
            for j in range(stocks_price.__len__()):
                fund += stocks_price[j][i] * ratios[j]
            myfund.append(fund)
        base_300g = [0]
        myfundg = [0]
        big_growg = [0]
        big_worthg = [0]
        mid_growg = [0]
        mid_worthg = [0]
        small_growg = [0]
        small_worthg = [0]
        for i in range(base_300.__len__() - 1):
            base_300g.append((base_300[i + 1][1] - base_300[i][1]) / base_300[i][1])
        for i in range(myfund.__len__() - 1):
            myfundg.append((myfund[i + 1] - myfund[i]) / myfund[i])
        for i in range(big_grow.__len__() - 1):
            big_growg.append((big_grow[i + 1] - big_grow[i]) / big_grow[i])
        for i in range(big_worth.__len__() - 1):
            big_worthg.append((big_worth[i + 1] - big_worth[i]) / big_worth[i])
        for i in range(mid_grow.__len__() - 1):
            mid_growg.append((mid_grow[i + 1] - mid_grow[i]) / mid_grow[i])
        for i in range(mid_worth.__len__() - 1):
            mid_worthg.append((mid_worth[i + 1] - mid_worth[i]) / mid_worth[i])
        for i in range(small_grow.__len__() - 1):
            small_growg.append((small_grow[i + 1] - small_grow[i]) / small_grow[i])
        for i in range(small_worth.__len__() - 1):
            small_worthg.append((small_worth[i + 1] - small_worth[i]) / small_worth[i])
        style_ratio, styleg, r_2 = self.calculate_quarter_regression(big_growg, big_worthg, mid_growg, mid_worthg,
                                                                     small_growg,
                                                                     small_worthg, myfundg)
        style_timing = []
        for i in range(styleg.__len__()):
            style_timing.append(styleg[i] - base_300g[i])
        style_timingq = []
        for i in range(np.math.floor(style_timing.__len__() / 90)):
            index = i * 90
            quarter_sum = np.sum(style_timing[index:index + 90])
            style_timingq.append(quarter_sum)
        style_choose = []
        for i in range(styleg.__len__()):
            style_choose.append(myfundg[i] - styleg[i])
        style_chooseq = []
        for i in range(np.math.floor(style_choose.__len__() / 90)):
            index = i * 90
            quarter_sum = np.sum(style_choose[index:index + 90])
            style_chooseq.append(quarter_sum)
        fund_over = []
        for i in range(style_choose.__len__()):
            sum = 0
            for j in range(0, i):
                sum += style_choose[j]
            fund_over.append(sum)
        fund_baseover = []
        for i in range(style_timing.__len__()):
            sum = 0
            for j in range(0, i):
                sum += style_timing[j]
            fund_baseover.append(sum)
        plus_count = 0
        for i in range(style_timingq.__len__()):
            if style_timingq[i] > 0:
                plus_count += 1
        style_timingw = plus_count * 1.0 / style_timingq.__len__()
        plus_count = 0
        for i in range(style_chooseq.__len__()):
            if style_chooseq[i] > 0:
                plus_count += 1
        style_choosew = plus_count * 1.0 / style_chooseq.__len__()
        style_timingr = fund_baseover[fund_baseover.__len__() - 1] / 630 * 365 * 100
        style_chooser = fund_over[fund_over.__len__() - 1] / 630 * 365 * 100
        dateq = ['2016Q4', '2017Q1', '2017Q2', '2017Q3', '2017Q4', '2018Q1', '2018Q2', '2018Q3']
        base_ratio, _, _ = self.calculate_quarter_regression(big_growg, big_worthg, mid_growg, mid_worthg, small_growg,
                                                             small_worthg, base_300g)
        time_over = []
        time_below = []
        for i in range(base_ratio[0].__len__()):
            over_num = 0
            below_num = 0
            for j in range(base_ratio.__len__()):
                if base_ratio[j][i] < style_ratio[j][i]:
                    over_num += 1
                elif base_ratio[j][i] > style_ratio[j][i]:
                    below_num += 1
            time_over.append(over_num)
            time_below.append(below_num)

        # 开始设置返回值
        style_config_form = [{
            'style': "风格择时贡献",
            'winRate': style_timingw,
            'yearRevenue': "%.2f" % style_timingr,
        }, {
            'style': "风格择时贡献",
            'winRate': style_choosew,
            'yearRevenue': "%.2f" % style_chooser,
        }]
        style_explain_form = []
        for i in range(r_2.__len__()):
            style_explain_form.append({
                'season': dateq[i],
                'rSquare': "%.2f" % r_2[i],
            })
        prefer_and_contr_list = []
        sum_style_ratios = []
        for i in range(style_ratio[0].__len__()):
            temp_sum = 0
            for j in range(style_ratio.__len__()):
                temp_sum += style_ratio[j][i]
            sum_style_ratios.append(temp_sum * 1.0 / style_ratio.__len__())
        prefer_and_contr_list.append({
            'style': "平均持仓",
            'largeGrowth': "%.2f" % sum_style_ratios[0],
            'largeValue': "%.2f" % sum_style_ratios[1],
            'middleGrowth': "%.2f" % sum_style_ratios[2],
            'middleValue': "%.2f" % sum_style_ratios[3],
            'smallGrowth': "%.2f" % sum_style_ratios[4],
            'smallValue': "%.2f" % sum_style_ratios[5]
        })
        prefer_and_contr_list.append({
            'style': "平均超额收益率",
            'largeGrowth': "%.2f" % (np.average(big_growg) / 630 * 365),
            'largeValue': "%.2f" % (np.average(big_worthg) / 630 * 365),
            'middleGrowth': "%.2f" % (np.average(mid_growg) / 630 * 365),
            'middleValue': "%.2f" % (np.average(mid_worthg) / 630 * 365),
            'smallGrowth': "%.2f" % (np.average(small_growg) / 630 * 365),
            'smallValue': "%.2f" % (np.average(small_worthg) / 630 * 365)
        })
        prefer_and_contr_list.append({
            'style': "超配次数",
            'largeGrowth': time_over[0],
            'largeValue': time_over[1],
            'middleGrowth': time_over[2],
            'middleValue': time_over[3],
            'smallGrowth': time_over[4],
            'smallValue': time_over[5]
        })
        prefer_and_contr_list.append({
            'style': "低配次数",
            'largeGrowth': time_below[0],
            'largeValue': time_below[1],
            'middleGrowth': time_below[2],
            'middleValue': time_below[3],
            'smallGrowth': time_below[4],
            'smallValue': time_below[5]
        })

        style_config_chart = []
        for i in range(style_timingq.__len__()):
            style_config_chart.append({
                'date': dateq[i],
                'styleTiming': style_timingq[i] * 100,
                'styleChoosing': style_chooseq[i] * 100,
                'fundOver': fund_over[i] * 100,
                'fundBaseOver': fund_baseover[i] * 100
            })
        return {
            'styleConfigChart': style_config_chart,
            'styleConfigForm': style_config_form,
            'styleExplainForm': style_explain_form,
            'preferAndContributionChart': prefer_and_contr_list
        }

    def get_scene(self, req_id):
        stocks, numbers = self.invest_dao.get_stock_position(req_id)
        ratios = np.array(numbers) / np.array(numbers).sum()
        start_date = date(2016, 9, 1)
        end_date = date(2018, 6, 1)
        stocks_price = []
        for stock in stocks:
            _, values = self.stock_dao.get_stock_period_price(stock, start_date, end_date)
            for i in range(values.__len__()):
                if np.isnan(values[i]):
                    values[i] = 0
            stocks_price.append(values)
        if stocks_price.__len__() == 0:
            return []
        myfund = []
        for i in range(stocks_price[0].__len__()):
            fund = 0
            for j in range(stocks_price.__len__()):
                fund += stocks_price[j][i] * ratios[j]
            myfund.append(fund)
        base_300 = self.transform_list_with_date(self.stock_dao.get_csi_300(start_date, end_date))
        base_300g = [0]
        myfundg = [0]
        for i in range(base_300.__len__() - 1):
            base_300g.append((base_300[i + 1] - base_300[i]) / base_300[i])
        for i in range(myfund.__len__() - 1):
            myfundg.append((myfund[i + 1] - myfund[i]) / myfund[i])

        start_date = date(2016, 9, 1)
        end_date = date(2018, 6, 1)
        big = self.stock_dao.get_big_worth(start_date, end_date)
        small = self.stock_dao.get_small_worth(start_date, end_date)
        bigg = [0]
        smallg = [0]
        for i in range(big.__len__() - 1):
            bigg.append((big[i + 1] - big[i]) / big[i])
        for i in range(small.__len__() - 1):
            smallg.append((small[i + 1] - small[i]) / small[i])
        market = []
        for i in range(np.math.floor(base_300g.__len__() / 30)):
            index = i * 30
            month_sum = np.sum(base_300g[index:index + 30])
            market.append(month_sum)
        biggm = []
        for i in range(np.math.floor(bigg.__len__() / 30)):
            index = i * 30
            month_sum = np.sum(bigg[index:index + 30])
            biggm.append(month_sum)
        smallgm = []
        for i in range(np.math.floor(smallg.__len__() / 30)):
            index = i * 30
            month_sum = np.sum(smallg[index:index + 30])
            smallgm.append(month_sum)
        sign = []
        for i in range(biggm.__len__()):
            sign.append(biggm[i] - smallgm[i])
        myfundgm = []
        for i in range(np.math.floor(bigg.__len__() / 30)):
            index = i * 30
            month_sum = np.sum(myfundg[index:index + 30])
            myfundgm.append(month_sum)
        rf = self.stock_dao.get_risk_free_rate()
        market_up_and_big_up = 0
        market_up_and_big_up_num = 0
        market_up_and_big_up_list = []
        market_down_and_big_up = 0
        market_down_and_big_up_num = 0
        market_down_and_big_up_list = []
        market_up_and_small_up = 0
        market_up_and_small_up_num = 0
        market_up_and_small_up_list = []
        market_down_and_small_up = 0
        market_down_and_small_up_num = 0
        market_down_and_small_up_list = []
        myfundgm_sum = 0
        max_back_market_up_and_big_up = 0
        max_back_market_down_and_big_up = 0
        max_back_market_up_and_small_up = 0
        max_back_market_down_and_small_up = 0
        for i in range(myfundgm.__len__()):
            if market[i] > 0 and sign[i] > 0:
                market_up_and_big_up_num += 1
                market_up_and_big_up += myfundgm[i]
                market_up_and_big_up_list.append(myfundgm[i])
                for j in range(30 * (i - 1), 30 * i):
                    for k in range(0, j):
                        this_back = (myfund[k] - myfund[j]) / myfund[j]
                        if this_back > max_back_market_up_and_big_up:
                            max_back_market_up_and_big_up = this_back
            elif market[i] < 0 < sign[i]:
                market_down_and_big_up_num += 1
                market_down_and_big_up += myfundgm[i]
                market_down_and_big_up_list.append(myfundgm[i])
                for j in range(30 * (i - 1), 30 * i):
                    for k in range(0, j):
                        this_back = (myfund[k] - myfund[j]) / myfund[j]
                        if this_back > max_back_market_down_and_big_up:
                            max_back_market_down_and_big_up = this_back
            elif market[i] > 0 > sign[i]:
                market_up_and_small_up_num += 1
                market_up_and_small_up += myfundgm[i]
                market_up_and_small_up_list.append(myfundgm[i])
                for j in range(30 * (i - 1), 30 * i):
                    for k in range(0, j):
                        this_back = (myfund[k] - myfund[j]) / myfund[j]
                        if this_back > max_back_market_up_and_small_up:
                            max_back_market_up_and_small_up = this_back
            elif market[i] < 0 and sign[i] < 0:
                market_down_and_small_up_num += 1
                market_down_and_small_up += myfundgm[i]
                market_down_and_small_up_list.append(myfundgm[i])
                for j in range(30 * (i - 1), 30 * i):
                    for k in range(0, j):
                        this_back = (myfund[k] - myfund[j]) / myfund[j]
                        if this_back > max_back_market_down_and_small_up:
                            max_back_market_down_and_small_up = this_back
            myfundgm_sum += myfundgm[i]
        market_up_and_big_up_rate = market_up_and_big_up / myfundgm_sum
        market_down_and_big_up_rate = market_down_and_big_up / myfundgm_sum
        market_up_and_small_up_rate = market_up_and_small_up / myfundgm_sum
        market_down_and_small_up_rate = market_down_and_small_up / myfundgm_sum

        volatility_market_up_and_big_up = np.std(market_up_and_big_up_list)
        volatility_market_down_and_big_up = np.std(market_down_and_big_up_list)
        volatility_market_up_and_small_up = np.std(market_up_and_small_up_list)
        volatility_market_down_and_small_up = np.std(market_down_and_small_up_list)
        sharpe_ratio_market_up_and_big_up = (market_up_and_big_up - rf) / volatility_market_up_and_big_up
        sharpe_ratio_market_down_and_big_up = (market_down_and_big_up - rf) / volatility_market_down_and_big_up
        sharpe_ratio_market_up_and_small_up = (market_up_and_small_up - rf) / volatility_market_up_and_small_up
        sharpe_ratio_market_down_and_small_up = (market_down_and_small_up - rf) / volatility_market_down_and_small_up

        sortino_ratio_market_up_and_big_up = np.std(market_up_and_big_up_list, ddof=1)
        sortino_ratio_market_down_and_big_up = np.std(market_down_and_big_up_list, ddof=1)
        sortino_ratio_market_up_and_small_up = np.std(market_up_and_small_up_list, ddof=1)
        sortino_ratio_market_down_and_small_up = np.std(market_down_and_small_up_list, ddof=1)
        calmar_ratio_market_up_and_big_up = np.average(market_up_and_big_up) / 30 / 365 / max_back_market_up_and_big_up
        calmar_ratio_market_down_and_big_up = np.average(
            market_down_and_big_up) / 30 / 365 / max_back_market_down_and_big_up
        calmar_ratio_market_up_and_small_up = np.average(
            market_up_and_small_up) / 30 / 365 / max_back_market_up_and_small_up
        calmar_ratio_market_down_and_small_up = np.average(
            market_down_and_big_up) / 30 / 365 / max_back_market_down_and_big_up

        fluctuation = []
        for i in range(np.math.floor(myfund.__len__() / 30)):
            index = i * 30
            month_sum = np.std(myfund[index:index + 30])
            fluctuation.append(month_sum)
        get = []
        for i in range(np.math.floor(myfundg.__len__() / 30)):
            index = i * 30
            month_sum = np.average(myfundg[index:index + 30])
            get.append(month_sum)

        average_fluctuation = np.average(fluctuation)
        std_fluctuation = np.std(fluctuation)
        fluctuationf = []
        fluctuationf_2d = []
        for i in range(fluctuation.__len__()):
            fluctuationf.append((fluctuation[i] - average_fluctuation) / std_fluctuation)
            fluctuationf_2d.append([(fluctuation[i] - average_fluctuation) / std_fluctuation])
        average_get = np.average(get)
        std_get = np.std(get)
        getf = []
        getf_2d = []
        for i in range(get.__len__()):
            getf.append((get[i] - average_get) / std_get)
            getf_2d.append([(get[i] - average_get) / std_get])
        # 线性拟合
        line_regression = LinearRegression()
        line_regression.fit(fluctuationf_2d, getf_2d)
        line_pred = line_regression.predict(fluctuationf_2d)
        # 曲线拟合
        poly_fit = np.polyfit(fluctuationf, getf, 3)  # 用3次多项式拟合
        poly = np.poly1d(poly_fit)
        poly_pred = poly(fluctuationf)

        # 开始设置返回值
        market_route_chart = []
        for i in range(market.__len__()):
            if market[i] > 0 and sign[i] > 0:
                market_state = MarketState.OBVIOUS_UP
            elif market[i] < 0 < sign[i]:
                market_state = MarketState.OBVIOUS_DOWN
            elif market[i] > 0 > sign[i]:
                market_state = MarketState.WAVERY_UP
            else:
                market_state = MarketState.WAVERY_DOWN
            market_route_chart.append({
                'date': str(start_date + timedelta(i)),
                'myfund': "%.2f" % myfund[i],
                'marketState': market_state.value
            })
        market_route_form = [{
            'index': '周期数',
            'obviousUp': "%.2f" % market_up_and_big_up_num,
            'waveryUp': "%.2f" % market_down_and_big_up_num,
            'obviousDown': "%.2f" % market_up_and_small_up_num,
            'waveryDown': "%.2f" % market_down_and_small_up_num
        }, {
            'index': '场景收益占比',
            'obviousUp': "%.2f" % market_up_and_big_up_rate,
            'waveryUp': "%.2f" % market_down_and_big_up_rate,
            'obviousDown': "%.2f" % market_up_and_small_up_rate,
            'waveryDown': "%.2f" % market_down_and_small_up_rate
        }, {
            'index': '波动率',
            'obviousUp': "%.2f" % volatility_market_up_and_big_up,
            'waveryUp': "%.2f" % volatility_market_down_and_big_up,
            'obviousDown': "%.2f" % volatility_market_up_and_small_up,
            'waveryDown': "%.2f" % volatility_market_down_and_small_up
        }, {
            'index': '夏普比率',
            'obviousUp': "%.2f" % sharpe_ratio_market_up_and_big_up,
            'waveryUp': "%.2f" % sharpe_ratio_market_down_and_big_up,
            'obviousDown': "%.2f" % sharpe_ratio_market_up_and_small_up,
            'waveryDown': "%.2f" % sharpe_ratio_market_down_and_small_up
        }, {
            'index': '最大回撤率',
            'obviousUp': "%.2f" % max_back_market_up_and_big_up,
            'waveryUp': "%.2f" % max_back_market_down_and_big_up,
            'obviousDown': "%.2f" % max_back_market_up_and_small_up,
            'waveryDown': "%.2f" % max_back_market_down_and_big_up
        }, {
            'index': 'Sortino比率',
            'obviousUp': "%.2f" % sortino_ratio_market_up_and_big_up,
            'waveryUp': "%.2f" % sortino_ratio_market_down_and_big_up,
            'obviousDown': "%.2f" % sortino_ratio_market_up_and_small_up,
            'waveryDown': "%.2f" % sortino_ratio_market_down_and_small_up
        }, {
            'index': 'Calmar比率',
            'obviousUp': "%.2f" % calmar_ratio_market_up_and_big_up,
            'waveryUp': "%.2f" % calmar_ratio_market_down_and_big_up,
            'obviousDown': "%.2f" % calmar_ratio_market_up_and_small_up,
            'waveryDown': "%.2f" % calmar_ratio_market_down_and_small_up
        }]
        sensibility_chart = []
        for i in range(fluctuationf.__len__()):
            sensibility_chart.append({
                'fluctuation': round(fluctuationf[i], 2),
                'lineRevenue': round(line_pred[i][0], 2),
                'curveRevenue': round(poly_pred[i], 2)
            })
        return {
            "marketRouteChart": market_route_chart,
            "marketRouteForm": market_route_form,
            "sensibilityChart": sensibility_chart
        }

max_stars_repo_name: FinTechNJU/CIti-FinBrain

max_issues_repo_name: FinTechNJU/CIti-FinBrain

max_forks_repo_name: FinTechNJU/CIti-FinBrain


text:
"""
steps:
    - get data interval
    - preprocess
        - log, background estimation
    - setup and run optimization problem
    - threshold

Example:
```
# load data
tec, times, ssmlon, n = io.get_tec_data(start_time, end_time)
# preprocess
x, times = ttec.preprocess_interval(tec, times)
# setup optimization
args = rbf_inversion.get_optimization_args(x, times)
# run optimization
model_output = rbf_inversion.run_multiple(args)
# threshold
initial_trough = model_output >= 1
# postprocess
trough = ttec.postprocess(initial_trough)
```
"""
import numpy as np
import cvxpy as cp
import multiprocessing
from scipy import stats
import matplotlib.pyplot as plt
import os

from sklearn.metrics.pairwise import rbf_kernel
from scipy.sparse import csr_matrix

from ttools import tec as ttec, deminov, config, io, utils, plotting
from ttools.trough_labeling.labeler import TroughLabelJob

def get_rbf_matrix(shape, bandwidth=1):
    X, Y = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]))
    xy = np.column_stack((X.ravel(), Y.ravel()))
    gamma = np.log(2) / bandwidth ** 2
    basis = rbf_kernel(xy, xy, gamma)
    basis[basis < .01] = 0
    return csr_matrix(basis)

def get_tv_matrix(im_shape, hw=1, vw=1):
    size = im_shape[0] * im_shape[1]

    right = np.eye(size)
    cols = im_shape[1] * (np.arange(size) // im_shape[1]) + (np.arange(size) + 1) % im_shape[1]
    right[np.arange(size), cols] = -1

    left = np.eye(size)
    cols = im_shape[1] * (np.arange(size) // im_shape[1]) + (np.arange(size) - 1) % im_shape[1]
    left[np.arange(size), cols] = -1

    up = np.eye(size)
    cols = np.arange(size) + im_shape[1]
    mask = cols < size
    cols = cols[mask]
    up[np.arange(size)[mask], cols] = -1

    down = np.eye(size)
    cols = np.arange(size) - im_shape[1]
    mask = cols >= 0
    cols = cols[mask]
    down[np.arange(size)[mask], cols] = -1
    return csr_matrix(hw * (right + left) + vw * (up + down))

def get_optimization_args(x, times, model_mlat, model_weight_max, rbf_bw, tv_hw, tv_vw, l2_weight, tv_weight,
                          prior_order, mlat_grid=None):
    if mlat_grid is None:
        mlat_grid = config.mlat_grid
    all_args = []
    # get rbf basis matrix
    basis = get_rbf_matrix(x.shape[1:], rbf_bw)
    # get tv matrix
    tv = get_tv_matrix(x.shape[1:], tv_hw, tv_vw) * tv_weight

    for i in range(times.shape[0]):
        # l2 norm cost away from model
        if prior_order == 1:
            l2 = abs(mlat_grid - model_mlat[i, :])
        elif prior_order == 2:
            l2 = (mlat_grid - model_mlat[i, :]) ** 2
        else:
            raise Exception("Invalid prior order")
        l2 -= l2.min()
        l2 = (model_weight_max - 1) * l2 / l2.max() + 1
        l2 *= l2_weight
        fin_mask = np.isfinite(x[i].ravel())
        args = (cp.Variable(x.shape[1] * x.shape[2]), basis[fin_mask, :], x[i].ravel()[fin_mask], tv, l2.ravel(),
                times[i], mlat_grid.shape)
        all_args.append(args)
    return all_args

def run_single(u, basis, x, tv, l2, t, output_shape):
    print(t)
    if x.size == 0:
        return np.zeros(output_shape)
    main_cost = u.T @ basis.T @ x
    tv_cost = cp.norm1(tv @ u)
    l2_cost = l2 @ (u ** 2)
    total_cost = main_cost + tv_cost + l2_cost
    prob = cp.Problem(cp.Minimize(total_cost))
    try:
        prob.solve(solver=config.SOLVER)
    except Exception as e:
        print("FAILED, USING ECOS:", e)
        prob.solve(solver=cp.ECOS)
    return u.value.reshape(output_shape)

def run_multiple(args, parallel=True):
    if parallel:
        with multiprocessing.Pool(processes=4) as p:
            results = p.starmap(run_single, args)
    else:
        results = []
        for arg in args:
            results.append(run_single(*arg))
    return np.stack(results, axis=0)

PARAM_SAMPLING = {
    'tv_weight': stats.uniform(0, 1),
    'l2_weight': stats.uniform(0, 1),
    'bge_x_rad': stats.randint(6, 11),
    'bge_y_rad': stats.randint(6, 11),
    'bge_temporal_rad': stats.randint(0, 2),
    'rbf_bw': stats.randint(1, 3),
    'tv_hw': stats.randint(1, 4),
    'tv_vw': stats.randint(1, 4),
    'model_weight_max': stats.randint(1, 30),
    'perimeter_th': stats.randint(10, 100),
    'area_th': stats.randint(10, 100),
    'prior_order': [1, 2],
    'prior': ['empirical_model', 'auroral_boundary'],
    'prior_offset': stats.randint(-5, 1),
}

BG_EST_SHAPE = (1, 19, 15)
MODEL_WEIGHT_MAX = 15
RBF_BW = 1
TV_HW = 2
TV_VW = 1
L2_WEIGHT = .06
TV_WEIGHT = .15
PERIMETER_TH = 40
AREA_TH = 40
PRIOR_ORDER = 1
PRIOR = 'empirical_model'
PRIOR_OFFSET = -3
THRESHOLD = 0.7
CLOSING_RAD = 0

class RbfInversionLabelJob(TroughLabelJob):

    def __init__(self, date, bg_est_shape=BG_EST_SHAPE, model_weight_max=MODEL_WEIGHT_MAX, rbf_bw=RBF_BW, tv_hw=TV_HW,
                 tv_vw=TV_VW, l2_weight=L2_WEIGHT, tv_weight=TV_WEIGHT, prior_order=PRIOR_ORDER, prior=PRIOR,
                 prior_offset=PRIOR_OFFSET, perimeter_th=PERIMETER_TH, area_th=AREA_TH, closing_rad=0,
                 threshold=THRESHOLD):
        super().__init__(date, bg_est_shape=bg_est_shape)
        self.model_weight_max = model_weight_max
        self.rbf_bw = rbf_bw
        self.tv_vw = tv_vw
        self.tv_hw = tv_hw
        self.l2_weight = l2_weight
        self.tv_weight = tv_weight
        self.prior_order = prior_order
        self.prior = prior
        self.prior_offset = prior_offset
        self.perimeter_th = perimeter_th
        self.area_th = area_th
        self.threshold = threshold
        self.closing_rad = closing_rad

    def run(self):
        print("Setting up inversion optimization")
        if self.prior == 'empirical_model':
            # get deminov model
            ut = self.times.astype('datetime64[s]').astype(float)
            model_mlat = deminov.get_model(ut, config.mlt_vals)
        elif self.prior == 'auroral_boundary':
            model_mlat = self.arb + self.prior_offset
        else:
            raise Exception("Invalid prior name")

        args = get_optimization_args(self.x, self.times, model_mlat, self.model_weight_max, self.rbf_bw, self.tv_hw,
                                     self.tv_vw, self.l2_weight, self.tv_weight, self.prior_order)
        # run optimization
        print("Running inversion optimization")
        self.model_output = run_multiple(args, parallel=config.PARALLEL)
        if self.threshold is not None:
            # threshold
            initial_trough = self.model_output >= self.threshold
            # postprocess
            print("Postprocessing inversion results")
            self.trough = ttec.postprocess(initial_trough, self.perimeter_th, self.area_th, self.arb, self.closing_rad)

    @staticmethod
    def get_random_params():
        params = {}
        for p in PARAM_SAMPLING:
            if isinstance(PARAM_SAMPLING[p], list):
                val = PARAM_SAMPLING[p][np.random.randint(len(PARAM_SAMPLING[p]))]
            else:
                try:
                    val = PARAM_SAMPLING[p].rvs().item()
                except:
                    val = PARAM_SAMPLING[p].rvs()
            params[p] = val
        params['bg_est_shape'] = (
            params['bge_temporal_rad'] * 2 + 1,
            params['bge_x_rad'] * 2 + 1,
            params['bge_y_rad'] * 2 + 1,
        )
        params['threshold'] = None
        del params['bge_temporal_rad']
        del params['bge_x_rad']
        del params['bge_y_rad']
        return params

    def _plot_single(self, i, swarm_troughs, plot_dir):
        plots = {
            'tec': {'data': self.tec[i + self.bg_est_shape[0] // 2], 'kwargs': dict(vmin=0, vmax=16)},
            'x': {'data': self.x[i], 'kwargs': dict(vmin=-.5, vmax=.5, cmap='coolwarm')},
            'output': {'data': self.model_output[i], 'kwargs': dict(vmin=-1.5, vmax=1.5, cmap='coolwarm')},
            'rbf_trough': {'data': self.trough[i], 'kwargs': dict(cmap='Blues')},
        }
        for name, plot in plots.items():
            fig, ax = plt.subplots(1, 1, figsize=(10, 10), subplot_kw=dict(projection='polar'), tight_layout=True)
            pcm = plotting.polar_pcolormesh(ax, config.mlat_grid, config.mlt_grid, plot['data'], **plot['kwargs'])
            plotting.plot_swarm_troughs_polar(ax, swarm_troughs)
            plotting.plot_mlon_lines(ax, self.ssmlon[i])
            plotting.plot_arb(ax, config.mlt_vals, self.arb[i])
            plotting.format_polar_mag_ax(ax)
            ax.set_title(f"{self.times[i]} {name}")
            plt.colorbar(pcm)
            fig.savefig(os.path.join(plot_dir, f"{self.date.astype('datetime64[D]')}_{i}_{name}.png"))
            plt.close(fig)

    def plot(self, swarm_troughs, plot_dir):
        for i in range(self.times.shape[0]):
            st = swarm_troughs[i == (swarm_troughs['tec_ind'] % 24)]
            self._plot_single(i, st, plot_dir)

max_stars_repo_name: gregstarr/ttools

max_issues_repo_name: gregstarr/ttools

max_forks_repo_name: gregstarr/ttools

reason: import numpy,from scipy,import cvxpy


text:
from typing import Tuple
import numpy as np
import torch

def MSELoss(y_true, y_pred):
    return torch.mean((y_pred - y_true) ** 2)

class RouletteWheelSelector:
    def select(self, fitness: np.array, n=None) -> np.array:
        n = n if n is not None else len(fitness)
        total_fitness = np.sum(fitness)
        selection_prob = fitness / total_fitness
        return np.random.choice(n, size=fitness.shape, p=selection_prob)

class GreedySelector:
    def select(self, \
        prev_population: np.ndarray, \
        prev_fitness: np.array, \
        population: np.ndarray, \
        fitness: np.array
    ) -> Tuple[np.ndarray, np.array]:
        """
        """
        new_fit_x = np.where(fitness > prev_fitness, fitness, prev_fitness)
        new_x = np.where(fitness > prev_fitness, population, prev_population)

        prev_population[fitness > prev_fitness] = population[fitness > prev_fitness]

        return new_x, new_fit_x

max_stars_repo_name: trdvangraft/evoalg

max_issues_repo_name: trdvangraft/evoalg

max_forks_repo_name: trdvangraft/evoalg


text:
import cv2
import numpy as np
import math
import logging
from datetime import datetime
import torch.nn as nn
import torch
import os

def get_time():
    return (str(datetime.now())[:-10]).replace(' ', '-').replace(':', '-')

def init_weights(m):
    if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
        nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
        if m.bias is not None:
            m.bias.data.zero_()
    elif isinstance(m, nn.BatchNorm2d):
        m.weight.data.fill_(1)
        m.bias.data.zero_()

def save_checkpoint(cfg, model, step=None, extra=''):
    save_path = cfg.MODEL_PATH

    if extra:
        if step is not None:
            torch.save(model.state_dict(), os.path.join(save_path, ('{}_step_{}_{}.pth'.format(cfg.MODEL_TYPE.lower(), step, extra))))
        else:
            torch.save(model.state_dict(), os.path.join(save_path, ('{}_{}.pth'.format(cfg.MODEL_TYPE.lower(), extra))))
    else:
        if step is not None:
            torch.save(model.state_dict(), os.path.join(save_path, ('{}_step_{}.pth'.format(cfg.MODEL_TYPE.lower(), step))))
        else:
            torch.save(model.state_dict(), os.path.join(save_path, ('{}.pth'.format(cfg.MODEL_TYPE.lower()))))

def set_logger(log_path):
    """Set the logger to log info in terminal and file `log_path`.

    In general, it is useful to have a logger so that every output to the terminal is saved
    in a permanent file. Here we save it to `model_dir/train.log`.

    Example:
    ```
    logging.info("Starting training...")
    ```

    Args:
        log_path: (string) where to log
    """
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)  # CRITICAL > ERROR > WARNING > INFO > DEBUG

    if not logger.handlers:
        # Logging to a file
        file_handler = logging.FileHandler(log_path)
        file_handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s: %(message)s'))
        file_handler.setLevel(logging.INFO)
        logger.addHandler(file_handler)

        # Logging to console
        stream_handler = logging.StreamHandler()
        stream_handler.setFormatter(logging.Formatter('%(message)s'))
        stream_handler.setLevel(logging.WARNING)
        logger.addHandler(stream_handler)

def write_cfg(logging, cfg):
    for key_val in cfg:
        logging.info(str(key_val) + ': ' + str(cfg[key_val]))
    logging.info("\n")

def calculate_pitch_yaw_roll(landmarks_2D, cam_w=256, cam_h=256,
                             radians=False):
    """ Return the the pitch  yaw and roll angles associated with the input image.
    @param radians When True it returns the angle in radians, otherwise in degrees.
    """

    assert landmarks_2D is not None, 'landmarks_2D is None'

    # Estimated camera matrix values.
    c_x = cam_w / 2
    c_y = cam_h / 2
    f_x = c_x / np.tan(60 / 2 * np.pi / 180)
    f_y = f_x
    camera_matrix = np.float32([[f_x, 0.0, c_x], [0.0, f_y, c_y],
                                [0.0, 0.0, 1.0]])
    camera_distortion = np.float32([0.0, 0.0, 0.0, 0.0, 0.0])

    # dlib (68 landmark) trached points
    # TRACKED_POINTS = [17, 21, 22, 26, 36, 39, 42, 45, 31, 35, 48, 54, 57, 8]
    # wflw(98 landmark) trached points
    # TRACKED_POINTS = [33, 38, 50, 46, 60, 64, 68, 72, 55, 59, 76, 82, 85, 16]
    # X-Y-Z with X pointing forward and Y on the left and Z up.
    # The X-Y-Z coordinates used are like the standard coordinates of ROS (robotic operative system)
    # OpenCV uses the reference usually used in computer vision:
    # X points to the right, Y down, Z to the front
    landmarks_3D = np.float32([
        [6.825897, 6.760612, 4.402142],  # LEFT_EYEBROW_LEFT,
        [1.330353, 7.122144, 6.903745],  # LEFT_EYEBROW_RIGHT,
        [-1.330353, 7.122144, 6.903745],  # RIGHT_EYEBROW_LEFT,
        [-6.825897, 6.760612, 4.402142],  # RIGHT_EYEBROW_RIGHT,
        [5.311432, 5.485328, 3.987654],  # LEFT_EYE_LEFT,
        [1.789930, 5.393625, 4.413414],  # LEFT_EYE_RIGHT,
        [-1.789930, 5.393625, 4.413414],  # RIGHT_EYE_LEFT,
        [-5.311432, 5.485328, 3.987654],  # RIGHT_EYE_RIGHT,
        [-2.005628, 1.409845, 6.165652],  # NOSE_LEFT,
        [-2.005628, 1.409845, 6.165652],  # NOSE_RIGHT,
        [2.774015, -2.080775, 5.048531],  # MOUTH_LEFT,
        [-2.774015, -2.080775, 5.048531],  # MOUTH_RIGHT,
        [0.000000, -3.116408, 6.097667],  # LOWER_LIP,
        [0.000000, -7.415691, 4.070434],  # CHIN
    ])
    landmarks_2D = np.asarray(landmarks_2D, dtype=np.float32).reshape(-1, 2)

    # Applying the PnP solver to find the 3D pose of the head from the 2D position of the landmarks.
    # retval - bool
    # rvec - Output rotation vector that, together with tvec, brings points from the world coordinate system to the camera coordinate system.
    # tvec - Output translation vector. It is the position of the world origin (SELLION) in camera co-ords
    _, rvec, tvec = cv2.solvePnP(landmarks_3D, landmarks_2D,
                                 camera_matrix, camera_distortion)
    # Get as input the rotational vector, Return a rotational matrix

    # const double PI = 3.141592653;
    # double thetaz = atan2(r21, r11) / PI * 180;
    # double thetay = atan2(-1 * r31, sqrt(r32*r32 + r33*r33)) / PI * 180;
    # double thetax = atan2(r32, r33) / PI * 180;

    rmat, _ = cv2.Rodrigues(rvec)
    pose_mat = cv2.hconcat((rmat, tvec))
    _, _, _, _, _, _, euler_angles = cv2.decomposeProjectionMatrix(pose_mat)
    return map(lambda k: k[0], euler_angles)  # euler_angles contain (pitch, yaw, roll)

class AverageMeter(object):
    """Computes and stores the average and current value"""

    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def vis_landmark(img_path, annotation, norm, point_num):
    """
    line format: [img_name bbox_x1 bbox_y1  bbox_x2 bbox_y2 landmark_x1 landmark y1 ...]
    """
    # check point len
    assert len(line) == 1 + 4 + point_num * 2  # img_path + bbox + point_num*2

    img = cv2.imread(img_path)
    h, w = img.shape[:2]

    img_name = annotation[0]
    bbox_x1, bbox_y1, bbox_x2, bbox_y2 = annotation[1:5]
    landmark = annotation[5:]

    landmark_x = line[1 + 4::2]
    landmark_y = line[1 + 4 + 1::2]
    if norm:
        for i in range(len(landmark_x)):
            landmark_x[i] = landmark_x[i] * w
            landmark_y[i] = landmark_y[i] * h

    # draw bbox and face landmark
    cv2.rectangle(img, (int(bbox_x1), int(bbox_y1)), (int(bbox_x2), int(bbox_y2)), (0, 0, 255), 2)
    for i in range(len(landmark_x)):
        cv2.circle(img, (int(landmark_x[i]), int(landmark_y[i])), 2, (255, 0, 0), -1)

    cv2.imshow("image", img)
    cv2.waitKey(0)


text:
# -*- coding: utf-8 -*-
# @Author: gunjianpan
# @Date:   2019-09-14 14:47:48
# @Last Modified by:   gunjianpan
# @Last Modified time: 2020-03-21 20:56:38

import base64
import json
import os
import sys
import time
import urllib

import numpy as np
import regex
import rsa

sys.path.append(os.getcwd())
from util.util import can_retry, echo, encoder_cookie, send_email, time_stamp, get_accept, get_content_type

from bilibili.basicBilibili import BasicBilibili
from bilibili.geetestE import E, O, S

proxy_req = 0
one_day = 86400
root_dir = os.path.abspath('bilibili')
data_dir = os.path.join(root_dir, 'data/')
PUBLIC = '00C1E3934D1614465B33053E7F48EE4EC87B14B95EF88947713D25EECBFF7E74C7977D02DC1D9451F79DD5D1C10C29ACB6A9B4D6FB7D0A0279B6719E1772565F09AF627715919221AEF91899CAE08C0D686D748B20A3603BE2318CA6BC2B59706592A9219D0BF05C9F65023A21D2330807252AE0066D59CEEFA5F2748EA80BAB81'

class Login(BasicBilibili):
    ''' bilibili login module '''

    def __init__(self):
        super(Login, self).__init__()
        self.update_proxy(1)
        self.access_key = ''
        self.aes_key = ''
        self.T = E(list(PUBLIC), 16)

    def get_access_key(self):
        captcha, cookie = self.get_captcha()
        hash_salt, key, cookie = self.get_hash_salt(cookie)
        if captcha is None:
            return
        types, cookie = self.get_type(captcha['gt'], cookie)
        return {
            'captcha': captcha,
            'hash_salt': hash_salt,
            'types': types,
            'cookie': cookie
        }

    def get_aes_key(self):
        def wl():
            return hex(int(65536 * (1 + np.random.random())))[3:]
        return wl() + wl() + wl() + wl()

    def get_t(self, aes_key: str, t: int = 128):
        n = np.zeros(t).astype(np.int)
        for ii, jj in enumerate(aes_key):
            n[ii + 112] = ord(jj)
        i = S()
        for ii in range(t - 2, 1, -1):
            n[ii] = i()
        n[1] = 2
        return n

    def doPublic(self):
        n = self.get_t(self.get_aes_key())
        self.N = E(n, 256)
        n = self.N.modPowInt(65537, self.T)
        r = n.tostring(16)
        add = '' if not (1 & len(r)) else '0'
        return '{}{}'.format(add, r)

    def get_hash_salt(self, cookie: dict = {}):
        url = self.GET_KEY_URL % np.random.random()
        headers = self.get_login_headers(2, cookie)
        hash_salt, cookies = proxy_req(url, 1, header=headers,
                                       need_cookie=True)
        if hash_salt is None or list(hash_salt.keys()) != ['hash', 'key']:
            if can_retry(url):
                return self.get_hash_salt()
            else:
                return None, {}
        return hash_salt['hash'], hash_salt['key'], cookies

    def get_captcha(self, cookie: dict = {}):
        url = self.CAPTCHA_URL
        headers = self.get_login_headers(0, cookie)
        captcha, cookies = proxy_req(url, 1, header=headers, need_cookie=True)
        if captcha is None or list(captcha.keys()) != ['code', 'data']:
            if can_retry(url):
                return self.get_captcha()
            else:
                return None, {}
        return captcha['data']['result'], cookies

    def get_access_key_req(self, hash_salt: str, key: str, challenge: str, validate: str, cookie: dict = {}):
        data = {
            'captchaType': 11,
            'username': self.username,
            'password': self.encoder_login_info(hash_salt, key),
            'keep': True,
            'key': key,
            'goUrl': self.AV_URL,
            'challenge': challenge,
            'validate': validate,
            'seccode': f'{validate}|jordan'
        }
        headers = self.get_login_headers(1, cookie)
        login = proxy_req(self.LOGIN_V2_URL, 12, header=headers)

    def get_type(self, gt: str, cookies: dict = {}) -> dict:
        url = self.GETTYPE_URL % (gt, int(time_stamp() * 1000))
        headers = self.get_login_headers(3, cookies)
        req, cookie = proxy_req(url, 3, header=headers, need_cookie=True)
        j_begin = req.find('{')
        if req == '' or j_begin == -1:
            if can_retry(self.GETTYPE_URL):
                return self.get_type(gt, cookies)
            else:
                return None, {}
        type_json = json.loads(req[j_begin:-1])
        return type_json['data'], cookie

    def encode_login_info(self, hash_salt: str, key: str):
        public_key = rsa.PublicKey.load_pkcs1_openssl_pem(key.encode())
        concate = rsa.encrypt(hash_salt + self.password).encode('utf-8')
        s = base64.b64encode(concate, public_key)
        s = urllib.parse.quote_plus(s)
        return s

    def get_login_headers(self, mode: int = 0, cookie: dict = {}):
        headers = {
            'Referer': self.LOGIN_URL,
        }
        if mode != 3:
            headers['Accept'] = get_accept('*') if mode == 2 else get_accept('xhr')
        if mode == 1:
            headers['Content-Type'] = get_content_type('')
        elif mode == 2:
            headers['X-Requested-With'] = 'XMLHttpRequest'
        if len(cookie):
            headers['Cookie'] = encoder_cookie(cookie)
        return headers

    def update_proxy(self, mode: int = 0):
        global proxy_req
        if not mode:
            self.update_proxy_basic()
        proxy_req = self.proxy_req


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 15 22:53:49 2018

@author: Kazuki
"""
import numpy as np
import pandas as pd
import os
import utils

PREF = 'f007'

os.system(f'rm ../data/t*_{PREF}*')
os.system(f'rm ../feature/t*_{PREF}*')

usecols = ['object_id', 'passband']

def mk_feature(df, output_path):

    df['passband_last'] = df['passband'].shift()
    df.loc[df['object_id'] != df['object_id'].shift(), 'passband_last'] = np.nan

    df['passband'] = df['passband_last'].astype(str) + '->' + df['passband'].astype(str)
#    df.apply(lambda x: f"{x['passband_last']}->{x['passband']}", axis=1)

    ct = pd.crosstab(df['object_id'], df['passband'])
    ct.columns = [f'passband{c}_cnt' for c in ct.columns]

    ct_norm = pd.crosstab(df['object_id'], df['passband'], normalize='index')
    ct_norm.columns = [f'passband{c}_norm' for c in ct_norm.columns]

    df = pd.concat([ct, ct_norm], axis=1)

    df.reset_index(drop=True, inplace=True)
    df.add_prefix(PREF+'_').to_feather(output_path)

    return

# =============================================================================
# main
# =============================================================================
if __name__ == "__main__":
    utils.start(__file__)

    mk_feature(pd.read_feather('../data/train_log.f')[usecols], f'../data/train_{PREF}.f')
    mk_feature(pd.read_feather('../data/test_log.f')[usecols],  f'../data/test_{PREF}.f')

    utils.end(__file__)


text:
#!/usr/bin/env python

'''
this is the file containing basic class definitions and importing the libraries
required for the implementation of the source code

Author: Sourya Bhattacharyya
Dept of CSE, IIT Kharagpur
V1.0 - 06.02.2014 - basic implementation
'''

import dendropy
from dendropy import *
#from dendropy import treesim
#from dendropy import Tree, Taxon, TaxonSet, TreeList, Node
#from dendropy import treecalc
import itertools
import numpy
from numpy import *
import os
import re
import math
from math import *
#import matplotlib.pyplot as plt
#from pylab import *
import operator
from operator import itemgetter
import time
from optparse import OptionParser
import random
from random import gauss
from cStringIO import StringIO

# variable for debugging option set
DEBUG_LEVEL = 2

# executable path of neighbor joining code
# integrated in the PHYLIP package
location_neighbor_exec = '/home/sourya/SOURYA_ALL/PhD_Codes/GSP/REFERENCE_CODES/PHYLIP/phylip-3.695/exe/neighbor'

neighbor_batch_filename = 'NJ_exec_command_seq.txt'
out_NJ_exec_filename = 'out_NJ_file.phy'
out_NJ_exec_treename = 'out_NJ_tree.phy'

""" this is the list of taxon names used for the current execution
it is filled by manual reading of the input distance matrix file
the list is useful to find out the index of individual taxa labels in the distance matrix
since the NJ tree and original distance matrix may use different taxa ordering during creation of respective distance matrices """
LIST_OF_TAXON_NAMES = []

#------------------------------------
# parameters used for genetic algorithm
#------------------------------------
# mutation rate is 0.5 for measure based parameter control
# we can also tune this rate between 0 and 1 with a step size of 0.1
# for measure based parameter control, following adaptation is made -----
# 1) results are not so good for 10 generations then this rate is increased by 0.05
# 2) results are good for 2 generations then this rate is decreased by 0.05
# if the rate goes below 0.06 or above 0.95 then it is reset to 0.5
#mutation_rate = 0.5

# recombination rate is 0.5 for measure based parameter control
# we can also tune this rate between 0 and 1 with a step size of 0.1
# for measure based parameter control, following adaptation is made -----
# 1) results are not so good for 10 generations then this rate is increased by 0.05
# 2) results are good for 2 generations then this rate is decreased by 0.05
# if the rate goes below 0.06 or above 0.95 then it is reset to 0.5
#recombination_rate = 0.5

no_of_repeated_experiments = 5

# this variable signifies the different mutation operators currently implemented
# so far SWAP TAXA, NNI, and SPR are implemented
num_diff_mutation_operators = 3

# this is the subset size employed for tournament selection
tournament_select_subset_size = 2

#-------------------------------------------------------
""" this class represents one entry of a population
it constitutes one tree (in string newick format), crossover rate, and a mutation rate """
class Entity_Population(object):
  def __init__(self, inp_tree=None, inp_mut_rate=0, inp_cros_rate=0):
    self._SetTree(inp_tree)
    self._SetMutationRate(inp_mut_rate)
    self._SetCrossOverRate(inp_cros_rate)

  def _GetMutationRate(self):
    return self.Mutation_Rate

  def _GetCrossOverRate(self):
    return self.CrossOver_Rate

  def _SetMutationRate(self, inp_val):
    self.Mutation_Rate = inp_val

  def _SetCrossOverRate(self, inp_val):
    self.CrossOver_Rate = inp_val

  def _GetTree(self):
    return self.Inp_Tree

  def _SetTree(self, inp_tree):
    if (inp_tree is None):
      self.Inp_Tree = Tree()
    else:
      self.Inp_Tree = Tree(inp_tree)

max_stars_repo_name: yodebu/Interview-Street

max_issues_repo_name: yodebu/Interview-Street

max_forks_repo_name: yodebu/Interview-Street

reason: import numpy,from numpy


text:
from treys import evaluator, Card
from holdem_calc import holdem_calc
from poker.hand import Combo, Range
import numpy as np
import random

class Ivan():

    def __init__(self):

        self.positioning = ['BTN','CO', 'HJ', 'UTH', 'BB', 'SB']
        self.early_range = Range('ATs+, KTs+, 77+, QTs+, T9s+, KJ+')
        self.mid_range  = Range('55+, ATs+, K9s+, Q9s+, JTs+, KT+, QT+, JT+')
        self.late_range = Range('22+, A2s+, A8o+, K9s+, Q9s+, J9s+, JTo+, QTo+, KTo+, 98s, 87s, 76s, 65s, 54s')
        self.any_range = Range('22+, AJ+, KQ+, KJs, QJs+, ATs+')
        self.allin_range = Range('QQ+, AK+')

    def calc_odds_range(self, board, hero, rng):
        itms = [holdem_calc.calculate_odds_villan(board, False, 50, False, hero, x, False, True) for x in rng.combos]
        # print(itms)
        return {'win' : np.mean([r[0]['win'] for r in itms if r]), 'tie':np.mean([r[0]['tie'] for r in itms if r]), 'lose' : np.mean([r[0]['lose'] for r in itms if not r == None])}
        # return [odds.update({o: np.mean([r[0][o] for r in itms if r])}) for o in ['tie', 'win', 'lose']]

    def next_move(self, board, players, pot1, pot2):
        """
        INPUTS:
        board: list of strings representing table cards
        players: list of list objects where each list is a player, [card1, card2, button ("D" if they're under the button), bet amt, equity (if pl0)]
        pot1: totpot - the cumulative sum of the current hand's mainpots
        pot2: mainpot - the cumulative sum of the current bets
        """

        com = Combo(players[0][0] + players[0][1])
        print(com)
        simulationEarly = holdem_calc.calculate_odds_villan(board, False, 30, None, com, None, False, False)
        #holdem_calc.calculate_odds_villan(board, False, 30, None, com, ['', '', '', '?'])

        winp = str(simulationEarly[0]['win'])
        tiep = str(simulationEarly[0]['tie'])
        losep = str(simulationEarly[0]['lose'])

        print(winp)
        print(losep)
        print(tiep)

        # prebetting
        if(len(board) == 0):

            pass

        # flop
        if(len(board) == 3):
            pass

        # turn
        if(len(board) == 4):
            pass

        # river
        if(len(board) == 5):
            pass
        return

a = Ivan()
a.next_move(['As', 'Ad', '3s'], [['Ks', 'Kd']], [], [])

# simulationEarly = calc_odds_range(tablecards, Hero, Villan)
# # holdem_calc.calculate_odds_villan(tablecards, False, 50, None, Hero, Villan, False, True)

# winp = str(simulationEarly['win'])
# tiep = str(simulationEarly['tie'])
# losep = str(simulationEarly['lose'])

# print(winp)


text:
import numpy as np
import itertools
from scipy.spatial.distance import cdist
import h5py

def hamming_set(num_crops, num_permutations, selection, output_file_name):
    """
    generate and save the hamming set
    :param num_crops: number of tiles from each image
    :param num_permutations: Number of permutations to select (i.e. number of classes for the pretext task)
    :param selection: Sample selected per iteration based on hamming distance: [max] highest; [mean] average
    :param output_file_name: name of the output HDF5 file
    """
    P_hat = np.array(list(itertools.permutations(list(range(num_crops)), num_crops)))
    n = P_hat.shape[0]

    for i in range(num_permutations):
        if i == 0:
            j = np.random.randint(n)
            P = np.array(P_hat[j]).reshape([1, -1])
        else:
            P = np.concatenate([P, P_hat[j].reshape([1, -1])], axis=0)

        P_hat = np.delete(P_hat, j, axis=0)
        D = cdist(P, P_hat, metric='hamming').mean(axis=0).flatten()

        if selection == 'max':
            j = D.argmax()
        elif selection == 'mean':
            m = int(D.shape[0] / 2)
            S = D.argsort()
            j = S[np.random.randint(m - 10, m + 10)]

    h5f = h5py.File('./hamming_set/' + output_file_name + str(num_permutations) + '.h5', 'w')
    h5f.create_dataset('max_hamming_set', data=P)
    h5f.close()
    print('file created --> ' + output_file_name + str(num_permutations) + '.h5')

if __name__ == "__main__":
    hamming_set(num_crops=9,
                num_permutations=100,
                selection='max',
                output_file_name='max_hamming_set_')

max_stars_repo_name: mawanda-jun/NoLabels

max_issues_repo_name: mawanda-jun/NoLabels

max_forks_repo_name: mawanda-jun/NoLabels

reason: import numpy,from scipy


text:
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.3'
#       jupytext_version: 0.8.6
#   kernelspec:
#     display_name: Arjun
#     language: python
#     name: arjun
# ---

import argparse
import os
import csv
import random
import logging
from tqdm import tqdm, trange, tqdm_notebook as tqnb

import numpy as np
import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)

from pytorch_transformers import (OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer,
                                     AdamW, cached_path, WEIGHTS_NAME, CONFIG_NAME,
                                     WarmupLinearSchedule)

logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt = '%m/%d/%Y %H:%M:%S',
                    level = logging.INFO)
logger = logging.getLogger(__name__)
""
def accuracy(out, labels):
    outputs = np.argmax(out, axis=1)
    return np.sum(outputs == labels)

def pre_process_datasets_binclass(encoded_datasets, input_len, cap_length, start_token, delimiter_token, clf_token):
    """ Pre-process datasets containing lists of tuples(story, 1st continuation, 2nd continuation, label)

        To Transformer inputs of shape (n_batch, n_alternative, length) comprising for each batch, continuation:
        input_ids[batch, alternative, :] = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
    """
    print("clf_token",clf_token)
    tensor_datasets = []
    for dataset in encoded_datasets:
        #print(dataset)
        n_batch = len(dataset)*2
        input_ids = np.zeros((n_batch, 1, input_len), dtype=np.int64)
        mc_token_ids = np.zeros((n_batch, 1), dtype=np.int64)
        lm_labels = np.full((n_batch, 1, input_len), fill_value=-1, dtype=np.int64)
        mc_labels = np.zeros((n_batch), dtype=np.int64)
        i=0
        for story, cont1, cont2, mc_label in dataset:
            with_cont1 = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
            with_cont2 = [start_token] + story[:cap_length] + [delimiter_token] + cont2[:cap_length] + [clf_token]
            input_ids[i, 0, :len(with_cont1)] = with_cont1
            input_ids[i+1, 0, :len(with_cont2)] = with_cont2
            mc_token_ids[i, 0] = len(with_cont1) - 1
            mc_token_ids[i+1, 0] = len(with_cont2) - 1
            lm_labels[i, 0, :len(with_cont1)] = with_cont1
            lm_labels[i+1, 0, :len(with_cont2)] = with_cont2
            if mc_label==0:
                mc_labels[i] = 1
                mc_labels[i+1] = 0
            else:
                mc_labels[i] = 0
                mc_labels[i+1] = 1
            i+=2
        all_inputs = (input_ids, mc_token_ids, lm_labels, mc_labels)
        tensor_datasets.append(tuple(torch.tensor(t) for t in all_inputs))
    return tensor_datasets

def pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, delimiter_token, clf_token):
    """ Pre-process datasets containing lists of tuples(story, 1st continuation, 2nd continuation, label)

        To Transformer inputs of shape (n_batch, n_alternative, length) comprising for each batch, continuation:
        input_ids[batch, alternative, :] = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
    """
    print("clf_token",clf_token)
    tensor_datasets = []
    for dataset in encoded_datasets:
        #print(dataset)
        n_batch = len(dataset)
        input_ids = np.zeros((n_batch, 2, input_len), dtype=np.int64)
        mc_token_ids = np.zeros((n_batch, 2), dtype=np.int64)
        lm_labels = np.full((n_batch, 2, input_len), fill_value=-1, dtype=np.int64)
        mc_labels = np.zeros((n_batch), dtype=np.int64)
        for i, (story, cont1, cont2, mc_label), in enumerate(dataset):
            with_cont1 = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
            with_cont2 = [start_token] + story[:cap_length] + [delimiter_token] + cont2[:cap_length] + [clf_token]
            input_ids[i, 0, :len(with_cont1)] = with_cont1
            input_ids[i, 1, :len(with_cont2)] = with_cont2
            mc_token_ids[i, 0] = len(with_cont1) - 1
            mc_token_ids[i, 1] = len(with_cont2) - 1
            lm_labels[i, 0, :len(with_cont1)] = with_cont1
            lm_labels[i, 1, :len(with_cont2)] = with_cont2
            mc_labels[i] = mc_label
        all_inputs = (input_ids, mc_token_ids, lm_labels, mc_labels)
        tensor_datasets.append(tuple(torch.tensor(t) for t in all_inputs))
    return tensor_datasets

def load_rocstories_dataset(dataset_path):
    """ Output a list of tuples(story, 1st continuation, 2nd continuation, label) """
    with open(dataset_path, encoding='utf_8') as f:
        f = csv.reader(f)
        output = []
        next(f) # skip the first line
        for line in tqdm(f):
            output.append(('.'.join(line[0:-3]), line[-3], line[-2], int(line[-1])))
    return output

def tokenize_and_encode(obj):
    """ Tokenize and encode a nested object """
    if isinstance(obj, str):
        return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(obj))
    elif isinstance(obj, int):
        return obj
    return list(tokenize_and_encode(o) for o in obj)

# +
# Defining constants over here
seed = 42
# model_name = '/home/ether/hdd/ether/gpt_domain_minds/se/epoch3/'
model_name = 'openai-gpt'

train_dataset = '../data/customer_service_ROC_data.csv'
# train_dataset = "../data/data_original.csv"
# train_dataset = "../data/se_data/se_nsp_4+w_data.csv"

eval_dataset = train_dataset
do_train = True
output_dir = '/home/ether/hdd/arjun/models_bkp/gpt_models/cs_model/'
save_model_every_epoch = True
parallel_train = True
num_train_epochs = 3
train_batch_size = 4
eval_batch_size = 1
max_grad_norm = 1
learning_rate = 6.25e-5
warmup_proportion = 0.002
lr_schedule = 'warmup_linear'
weight_decay = 0.01
lm_coef = 0.9
mc_coef = 1
n_valid = 374
max_steps = -1
gradient_accumulation_steps = 1
adam_epsilon = 1e-8
warmup_steps = 0
try:
    os.mkdir(output_dir+"/")
except OSError as e:
    print(e)
    pass
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
# -

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
logger.info("device: {}, n_gpu {}".format(device, n_gpu))

"""
# Load tokenizer and model
# This loading functions also add new tokens and embeddings called `special tokens`
# These new embeddings will be fine-tuned on the RocStories dataset
special_tokens = ['_start_', '_delimiter_', '_classify_']
tokenizer = OpenAIGPTTokenizer.from_pretrained(model_name, special_tokens=special_tokens)
special_tokens_ids = list(tokenizer.convert_tokens_to_ids(token) for token in special_tokens)
model = OpenAIGPTDoubleHeadsModel.from_pretrained(model_name, num_special_tokens=len(special_tokens))
model.to(device)
"""
def save_model(output_dir):
    try:
        os.mkdir(output_dir+"/")
    except OSError:
        pass
    model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self

    # If we save using the predefined names, we can load using `from_pretrained`
    output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
    output_config_file = os.path.join(output_dir, CONFIG_NAME)

    torch.save(model_to_save.state_dict(), output_model_file)
    model_to_save.config.to_json_file(output_config_file)
    tokenizer.save_vocabulary(output_dir)

special_tokens = ['_start_', '_delimiter_', '_classify_']
tokenizer = OpenAIGPTTokenizer.from_pretrained(model_name)
tokenizer.add_tokens(special_tokens)
special_tokens_ids = list(tokenizer.convert_tokens_to_ids(token) for token in special_tokens)
model = OpenAIGPTDoubleHeadsModel.from_pretrained(model_name,num_labels=2)
model.resize_token_embeddings(len(tokenizer))
model.to(device)

logger.info("Encoding dataset...")

train_dataset = load_rocstories_dataset(train_dataset)
print('length dataset',len(train_dataset))
datasets = (train_dataset[:],)
print(len(datasets[0]))
encoded_datasets = tokenize_and_encode(datasets)
# Compute the max input length for the Transformer
max_length = model.config.n_positions // 2 - 2
input_length = max(len(story[:max_length]) + max(len(cont1[:max_length]), len(cont2[:max_length])) + 3  \
                        for dataset in encoded_datasets for story, cont1, cont2, _ in dataset)
input_length = min(input_length, model.config.n_positions)  # Max size of input for the pre-trained model

# Prepare inputs tensors and dataloaders
tensor_datasets = pre_process_datasets(encoded_datasets, input_length, max_length, *special_tokens_ids)
train_tensor_dataset = tensor_datasets[0]
train_data = TensorDataset(*train_tensor_dataset)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=train_batch_size)

if parallel_train:
    model = torch.nn.DataParallel(model,output_device=torch.device("cuda:1"))

# Prepare optimizer
if do_train:
    if max_steps > 0:
        t_total = max_steps
        num_train_epochs = max_steps //\
            (len(train_dataloader) // gradient_accumulation_steps) + 1
    else:
        t_total = len(train_dataloader)\
            // gradient_accumulation_steps * num_train_epochs
    param_optimizer = list(model.named_parameters())
    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': weight_decay},
        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon)
    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)

if do_train:
    nb_tr_steps, tr_loss, exp_average_loss = 0, 0, None
    model.train()
    for ep_no in trange(int(num_train_epochs), desc="Epoch"):
        tr_loss = 0
        nb_tr_steps = 0
        tqdm_bar = tqdm(train_dataloader, desc="Training")
        for step, batch in enumerate(tqdm_bar):
            batch = tuple(t.to(device) for t in batch)
            input_ids, mc_token_ids, lm_labels, mc_labels = batch
            losses = model(input_ids, mc_token_ids, lm_labels, mc_labels)
            lm_loss = losses[0].mean()
            mc_loss = losses[1].mean()
            loss = (lm_coef * lm_loss) + (mc_coef * mc_loss)

            loss.backward()
            scheduler.step()
            optimizer.step()
            optimizer.zero_grad()
            tr_loss += loss.item()
            exp_average_loss = loss.item() if exp_average_loss is None else 0.7*exp_average_loss+0.3*loss.item()
            nb_tr_steps += 1
            tqdm_bar.desc = "Training loss: {:.2e} lr: {:.2e}".format(exp_average_loss, scheduler.get_lr()[0])
            # Save a trained model, configuration and tokenizer every step steps
#             if step%((len(train_dataloader)-2)//3)==0:
#                 save_model(output_dir+"_ep"+str(1+ep_no)+"_step"+str(step))
        # Save a trained model, configuration and tokenizer every epoch
        if save_model_every_epoch:
            save_model(output_dir+"epoch_"+str(1+ep_no))

# Save a trained model
if do_train:
    # Save a trained model, configuration and tokenizer
    model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self

    # If we save using the predefined names, we can load using `from_pretrained`
    output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
    output_config_file = os.path.join(output_dir, CONFIG_NAME)

    torch.save(model_to_save.state_dict(), output_model_file)
    model_to_save.config.to_json_file(output_config_file)
    tokenizer.save_vocabulary(output_dir)

    # Load a trained model and vocabulary that you have fine-tuned
    model = OpenAIGPTDoubleHeadsModel.from_pretrained(output_dir)
    tokenizer = OpenAIGPTTokenizer.from_pretrained(output_dir)
    model.to(device)

max_stars_repo_name: etherlabsio/hinton

max_issues_repo_name: etherlabsio/hinton

max_forks_repo_name: etherlabsio/hinton