text:
import numpy as np
import matplotlib.pyplot as plt

def draw(x1, x2):
    ln = plt.plot(x1, x2)

def sigmoid(score):
    return 1/(1 + np.exp(-score))

def calculate_error(line_parameters, points, y):
    m = points.shape[0]
    p = sigmoid(points*line_parameters)
    cross_entropy = -(1/m)*(np.log(p).transpose() * y + np.log(1-p).transpose()*(1-y))
    return cross_entropy

def gradient_descent(line_parameters, points, y, alpha):
    m = points.shape[0]
    for i in range(5000):
        p = sigmoid(points*line_parameters)
        gradient = (1/m)*(points.T * (p - y))
        line_parameters = line_parameters - gradient
        w1 = line_parameters.item(0)
        w2 = line_parameters.item(1)
        b = line_parameters.item(2)
        x1 = np.array([points[:, 0].min(), points[:, 0].max()])
        x2 = -b / w2 + x1 * (-w1 / w2)
    draw(x1,x2)

n_pts = 100
np.random.seed(0)
bias = np.ones(n_pts)
top_region = np.array([np.random.normal(10, 2, n_pts), np.random.normal(12,2,n_pts), bias]).transpose()
bottom_region = np.array([np.random.normal(5, 2, n_pts), np.random.normal(6,2,n_pts), bias]).transpose()
all_points = np.vstack((top_region, bottom_region))

line_parameters = np.matrix([np.zeros(3)]).T
# x1 = np.array([bottom_region[:, 0].min(), top_region[:, 0].max()])
# w1x1 + w2x2 + b = 0
# x2 = -(b + w1x1) / w2
# x2 = -b / w2 + x1 * (-w1 / w2)
y = np.array([np.zeros(n_pts), np.ones(n_pts)]).reshape(n_pts*2, 1)

_, ax = plt.subplots(figsize=(4,4))
ax.scatter(top_region[:, 0], top_region[:, 1], color='r')
ax.scatter(bottom_region[:, 0], bottom_region[:, 1], color='b')
gradient_descent(line_parameters, all_points, y, 0.06)
plt.show()

max_stars_repo_name: vedanshdwivedi/SDC-Self-Driving-Car-

max_issues_repo_name: vedanshdwivedi/SDC-Self-Driving-Car-

max_forks_repo_name: vedanshdwivedi/SDC-Self-Driving-Car-


text:
from modulefinder import Module

import torch
import torch.nn as nn
import torch.nn.functional as F
import os
import sys
# from numpy import linspace
#
from data.data_loader import Dataset_ETT_hour
import time
import pandas as pd
from scipy.fftpack import dct, idct
import matplotlib.pyplot as plt
import numpy as np

#### FOURRIER
# # preprocessing
data = pd.read_csv('/Users/marc-antoine/Documents/S7/物理实验/gch/erhao.csv', sep=';')
temps = data['Temps'].to_numpy()[20:]
volt = data['EA1'].rolling(10).mean().to_numpy()[20:]
# volt *= 1.0293
volt -= volt.mean()
# temps = np.concatenate((temps, np.linspace(80, 90, 10000)))
# volt = np.concatenate((volt, volt[74980:79980]/2))
# volt = np.concatenate((volt, volt[74980:79980]/2))
font2 = {'weight': 'normal',
         'size': 14,
         }

# I(t)
plt.plot(temps, (volt+1.356)*1.0293-0.5508, linewidth=0.5, color='black')
plt.xlabel('T/s', font2)
plt.ylabel('I/cd', font2)
plt.grid()
plt.xlim(43, 80)
plt.ylim(0.5, 2)
plt.savefig('/Users/marc-antoine/Documents/S7/物理实验/gch/I(t).jpg')
plt.show()

ext: py

max_stars_repo_name: MarcAntoineAlex/informerdata

max_issues_repo_name: MarcAntoineAlex/informerdata

max_forks_repo_name: MarcAntoineAlex/informerdata

reason: import numpy,from numpy,from scipy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2011 yinhm

'''Imigu.com specific datafeed server implementation.
'''

import datetime
import logging
import time

import numpy as np

from datafeed.providers.dzh import DzhDividend, DzhSector
from datafeed.server import *
from datafeed.utils import *

__all__ = ['ImiguApplication', 'ImiguHandler']

class SnapshotIndexError(KeyError):
    pass

class ImiguApplication(Application):

    def __init__(self, datadir, exchange):
        self.archive_minute_time = 0
        self.archive_day_time = 0
        self.crontab_time = 0

        self._tasks = []

        super(ImiguApplication, self).__init__(datadir, exchange, handler=ImiguHandler)

        # last quote time reset to SH000001's timestamp
        try:
            r = self.dbm.get_report("SH000001")
            self.dbm.set_mtime(r['timestamp'])
        except KeyError:
            self.dbm.set_mtime(time.time())

    def periodic_job(self):
        today = datetime.datetime.today()

        if self.scheduled_archive_minute(today):
            request = Request(None, 'archive_minute')
            self.__call__(request)

        if self.scheduled_archive_day(today):
            request = Request(None, 'archive_day')
            self.__call__(request)

        if self.scheduled_crontab_daily(today):
            request = Request(None, 'crontab_daily')
            self.__call__(request)

        if len(self._tasks) > 0:
            logging.info("tasks left: %s" % len(self._tasks))
            request = Request(None, 'run_task')
            self.__call__(request)

    def scheduled_archive_minute(self, today):
        """Test is archive minute scheduled.
        """
        now = time.time()

        market_open_at = self.exchange.open_time(now=now)
        if now < market_open_at:
            # Should not archive any data if market not open yet.
            logging.debug("market not open yet")
            return False

        market_closing_at = self.exchange.close_time(now=now)
        if now > (market_closing_at + 60 * 5):
            # Do not archive if time passed 15:05.
            # Should be archived already. If not, something is broken.
            logging.debug("market closed more than 5 minutes")
            return False

        # quote_time = self.dbm.mtime
        # if (now - quote_time) > 60:
        #     return False

        # in session, we run it every 60 sec or greater
        if today.second == 0 or (now - self.archive_minute_time) > 60:
            return True

        return False

    def scheduled_archive_day(self, today):
        """Test is daily archive scheduled.
        """
        now = time.time()
        close_time = self.exchange.close_time(now=now)

        if now < close_time:
            logging.debug("market not closed yet.")
            return False

        if self.dbm.mtime < close_time:
            logging.debug("No market data: Weekday or holiday or datafeed receiver broken.")
            return False

        if self.dbm.mtime < self.archive_day_time:
            logging.debug("Already archived.")
            return False

        # skip 60 * 3 sec make sure we got the last data
        if now > (close_time + 60 * 3):
            return True

        return False

    def scheduled_crontab_daily(self, today):
        """Test is daily crontab scheduled.
        """
        if today.hour == 8:
            if today.minute == 0 and today.second == 0:
                return True

            now = time.time()
            if today.minute == 0 and (now - self.crontab_time) > 86400:
                # not runned before
                return True

        return False

    def task_add(self, task):
        self._tasks.append(task)

    def task_reserve(self):
        return self._tasks.pop(0)

class ImiguHandler(Handler):

    SUPPORTED_METHODS = Handler.SUPPORTED_METHODS + \
        ('archive_day',
         'archive_minute',
         'crontab_daily',
         'sync_dividend',
         'sync_sector',
         'run_task')

    ###### periodic jobs ######

    def archive_day(self, *args):
        """Archive daily data from report datastore.
        """
        dt = datetime.datetime.fromtimestamp(self.dbm.mtime).date()

        store = self.dbm.daystore
        reports = self.dbm.get_reports()
        for symbol, report in reports:
            if 'timestamp' not in report:
                continue

            d = datetime.datetime.fromtimestamp(report['timestamp'])

            if dt != d.date():
                # skip instruments which no recent report data
                continue

            t = int(time.mktime(d.date().timetuple()))

            row = (t, report['open'], report['high'], report['low'],
                   report['close'], report['volume'], report['amount'])

            data = np.array([row], dtype=store.DTYPE)
            store.update(symbol, data)

        self.application.archive_day_time = time.time()
        logging.info("daily data archived.")

        if self.request.connection:
            self.request.write("+OK\r\n")

    def archive_minute(self, *args):
        '''Archive minute data from report datastore.
        '''
        logging.info("starting archive minute...")
        self.application.archive_minute_time = time.time()

        dbm = self.dbm
        pre_open_time = dbm.exchange.pre_open_time(now=dbm.mtime)
        open_time = dbm.exchange.open_time(now=dbm.mtime)
        break_time = dbm.exchange.break_time(now=dbm.mtime)
        close_time = dbm.exchange.close_time(now=dbm.mtime)

        try:
            report = dbm.get_report('SH000001')
            rts = report['timestamp']
        except KeyError:
            logging.error("No SH000001 data.")
            if not self.request.connection:
                return
            return self.request.write("-ERR No data yet.\r\n")

        if rts < pre_open_time:
            logging.error("wrong report time: %s." % \
                              (datetime.datetime.fromtimestamp(rts), ))
            if not self.request.connection:
                return
            return self.request.write("-ERR No data yet.\r\n")

        mintime, index = ImiguHandler.get_snapshot_index(open_time, rts)

        if index < 0:
            raise SnapshotIndexError

        # Rotate when we sure there is new data coming in.
        dbm.rotate_minute_store()
        store = dbm.minutestore

        snapshot_time = mintime
        cleanup_callback = lambda r: r
        if index > 120 and index < 210:
            # sometimes we received report within 11:31 - 12:59
            # reset to 11:30
            snapshot_time = break_time
            def cleanup_callback(r):
                r['timestamp'] = break_time
                r['time'] = str(datetime.datetime.fromtimestamp(break_time))

            index = 120
        elif index >= 210 and index <= 330:
            index = index - 89  # subtract 11:31 - 12:59
        elif index > 330:
            # sometimes we received report after 15:00
            # reset to 15:00
            snapshot_time = close_time
            def cleanup_callback(r):
                r['timestamp'] = close_time
                r['time'] = str(datetime.datetime.fromtimestamp(close_time))

            index = 241

        reports = dbm.get_reports()
        for key, report in reports:
            if 'timestamp' not in report:
                # Wrong data
                continue

            if mintime - report['timestamp'] > 1800:
                # no new data in 30 mins, something broken
                # skip this symbol when unknown
                continue

            cleanup_callback(report)

            mindata = (snapshot_time, report['price'], report['volume'], report['amount'])
            y = np.array(mindata, dtype=store.DTYPE)

            store.set(key, index, y)

        #store.flush()

        logging.info("snapshot to %i (index of %i)." % (mintime, index))
        self.request.write_ok()

    @classmethod
    def get_snapshot_index(cls, open_time, report_time):
        ts = time.time()
        d = datetime.datetime.fromtimestamp(ts)
        mintime = time.mktime((d.year, d.month, d.day,
                               d.hour, d.minute,
                               0, 0, 0, 0))
        index = int((mintime - open_time) / 60)
        logging.info("minute data at %i (index of %i)." % (mintime, index))
        return (int(mintime), index)

    def crontab_daily(self, *args):
        self.application.crontab_time = time.time()
        self.sync_dividend()
        self.sync_sector()

    def sync_dividend(self, *args):
        io = DzhDividend()
        for symbol, data in io.read():
            self.dbm.update_dividend(symbol, data)
        self.dbm.divstore.flush()
        self.request.write_ok()

    def sync_sector(self, *args):
        io = DzhSector()
        for sector, options in io.read():
            self.dbm.sectorstore[sector] = options
        self.request.write_ok()

    def run_task(self):
        for i in xrange(300):
            try:
                task = self.application.task_reserve()
            except IndexError:
                break
            task.run()

class Task(object):
    __slots__ = ['store', 'key', 'index', 'data']

    def __init__(self, store, key, index, data):
        self.store = store
        self.key = key
        self.index = index
        self.data = data

    def run(self):
        self.store.set(self.key, self.index, self.data)

max_stars_repo_name: tigersean/datafeed

max_issues_repo_name: tigersean/datafeed

max_forks_repo_name: tigersean/datafeed

reason: import numpy


text:
import json
import os
import sys
import urllib.parse
from datetime import datetime
from string import Template

import numpy
import uvicorn
from fastapi import FastAPI, BackgroundTasks
from starlette.requests import Request
from starlette.responses import HTMLResponse, RedirectResponse, Response
from starlette.staticfiles import StaticFiles

from arguments import get_arguments
from web.backend import Backend
from web.templating import (
    generate_index_html,
    generate_index_html_validation,
)
from web.utils import (
    CommentBody,
    SourcingState,
    RequestBody,
    ValidationRequestBody,
    WorkingRequestBody,
)

app = FastAPI()
app.mount("/static", StaticFiles(directory="web/static"), name="static")

def get_prediction(args, example, models, processor, tokenizer):
    option_probs = []
    for model in models:
        pred_label, pred_probs = predict_example(
            args, model, example, processor, tokenizer
        )
        option_probs.append(pred_probs)
    option_average_probs = list(numpy.average(option_probs, axis=0))
    pred_label = int(numpy.argmax(option_average_probs))
    gold_label = int(example["gold_answer"][-1]) - 1
    if example["mode"] in ["adv", "plain"]:
        success = gold_label != pred_label
    else:
        success = False
    return {
        "model_prediction_probability": option_average_probs,
        "model_prediction_label": pred_label,
        "success": success,
        "gold_label": gold_label,
        "probabilities": option_probs,
    }

def make_prediction(input_body: RequestBody):
    example = input_body.convert_to_example()
    return get_prediction(
        app.state.args,
        example,
        app.state.model_kit["models"],
        app.state.model_kit["processor"],
        app.state.model_kit["tokenizer"],
    )

def call_template(template_dir, backend, uid, worker_id, run_mode, is_external):
    collection_mode = backend.user_assignments[uid].collection_mode
    template_name = backend.hit_config[collection_mode].get(
        "template_name", collection_mode
    )
    instruction_path = backend.hit_config[collection_mode].get(
        "instruction_template", f"instruction_{template_name}.html"
    )
    mini_instruction_path = backend.hit_config[collection_mode].get(
        "mini_instruction_template", f"mini_instruction_{template_name}.html"
    )
    faq_path = backend.hit_config[collection_mode].get(
        "faq_template", f"faq_{template_name}.html"
    )
    task_path = backend.hit_config[collection_mode].get(
        "task_template", f"task_{template_name}.html"
    )
    subconfig = backend.hit_config[collection_mode].get("subconfig", "")
    if collection_mode == "plain" and subconfig == "justification":
        instruction_path = "instruction_justification.html"
        faq_path = "faq_justification.html"

    index_title = backend.hit_config[collection_mode].get("index_title", "")

    # todo: migrate tutorial codes
    if collection_mode in ["plain", "adv"]:
        passages = backend.get_passages(uid)
        task_ids = backend.get_task_ids(uid)
        if not index_title:
            index_title = "Writing hard reading comprehension questions."
        return generate_index_html(
            collection_mode,
            index_title,
            os.path.join(template_dir, "index.html"),
            os.path.join(template_dir, instruction_path),
            os.path.join(template_dir, faq_path),
            os.path.join(template_dir, mini_instruction_path),
            os.path.join(template_dir, task_path),
            uid,
            task_ids,
            passages,
            backend.get_current_task_index(uid),
            backend.get_success_task_ids(uid),
            app.state.args.run_mode,
            "yes" if app.state.args.is_external else "no",
            subconfig,
        )
    elif collection_mode in ["validation"]:
        if not index_title:
            if template_name == "evaluation":
                index_title = "Evaluate reading comprehension questions!"
            else:
                index_title = "Solve reading comprehension questions!"
        validation_tasks = backend.get_user_validation_task(uid, worker_id)
        if validation_tasks is None:
            backend.logger.info(
                f"Cannot issue a new validation task for worker_id={worker_id}"
            )
            return open(os.path.join(template_dir, "error.html"), "r").read()
        return generate_index_html_validation(
            collection_mode,
            index_title,
            os.path.join(template_dir, "index.html"),
            os.path.join(template_dir, instruction_path),
            os.path.join(template_dir, faq_path),
            os.path.join(template_dir, mini_instruction_path),
            os.path.join(template_dir, task_path),
            uid,
            validation_tasks,
            app.state.args.run_mode,
            "yes" if app.state.args.is_external else "no",
            subconfig=subconfig,
        )

    else:
        raise ValueError(f"Invalid collection mode: {collection_mode}")

@app.get("/", response_class=HTMLResponse)
def read_root(
    request: Request,
    uid: str = None,
    status: str = None,
    worker_id: str = None,
    assignment_id: str = None,
    debug: str = None,
    assignmentId: str = None,
    workerId: str = None,
):
    # # emergency mode
    # return "We are sorry that no HITs are available. Please click 'Return' on the HIT page to avoid any impact on your approval rating."
    # return "We are sorry that an error has occured. Please click 'Return' on the HIT page to avoid any impact on your approval rating."

    if workerId is not None and worker_id is None:
        worker_id = workerId
    if assignmentId is not None and assignment_id is None:
        assignment_id = assignmentId

    template_dir = app.state.args.template_dir
    run_mode = app.state.args.run_mode

    backend = Backend.get_backend(app)
    # uid must be included in url when sandbox and release modes
    if not uid and run_mode in ["sandbox", "release"]:
        # backend.logger.info("Called error.html")
        return open(os.path.join(template_dir, "error.html"), "r").read()

    # if the link is clicked in preview
    if assignment_id == "ASSIGNMENT_ID_NOT_AVAILABLE" and run_mode != "debug":
        return open(os.path.join(template_dir, "not_accepted.html"), "r").read()

    datestr = datetime.now().isoformat()
    if not uid and run_mode == "debug":  # issue a uid if debug
        uid = backend.get_debug_uid()
        if debug:
            url = urllib.parse.urlencode({"uid": uid, "debug": debug})
            return RedirectResponse(url="?" + url)
        else:
            url = urllib.parse.urlencode({"uid": uid})
            return RedirectResponse(url="?" + url)

    # check how many HITs the worker did
    if app.state.args.worker_log_path and worker_id:
        if not backend.check_if_worker_can_accept_hit(worker_id, uid):
            backend.logger.info(f"GET: worker {worker_id} cannot accept hit")
            return open(os.path.join(template_dir, "known_worker.html"), "r").read()
        else:
            backend.logger.info(f"GET: worker {worker_id} can accept hit")

    # create user data
    backend.logger.info(f"GET / {datestr} / {uid} {worker_id} {assignment_id} {status}")
    uid = backend.check_user_data(uid, status, worker_id, assignment_id)
    if uid is not None and status == "new":
        redirect_url = urllib.parse.urlencode(
            {
                "uid": uid,
                "assignmentId": assignment_id,
                "workerId": worker_id,
                "status": "redirect",
            }
        )
        return RedirectResponse(url="?" + redirect_url)

    if uid == "" and run_mode != "debug":
        backend.logger.info("Called error.html")
        return open(os.path.join(template_dir, "error.html"), "r").read()

    backend.logging_current_status()
    state = backend.get_current_state(uid)
    if state == SourcingState.START or state == SourcingState.WORKING:
        return call_template(
            template_dir, backend, uid, worker_id, run_mode, app.state.args.is_external
        )
    elif state == SourcingState.FINISHED or state == SourcingState.QUIT:
        fpath = os.path.join(template_dir, "finished.html")
        template = Template(open(fpath, "r").read())
        return template.substitute(
            {"completion_code": backend.get_completion_code(uid)}
        )
    else:
        return open(os.path.join(template_dir, "error.html"), "r").read()

@app.post("/predict")
def predict(body: RequestBody):
    backend = Backend.get_backend(app)
    response = make_prediction(body)
    backend.save_submission(
        body.uid.strip(), body.passage_id, body.convert_to_example(), response
    )
    return response or {"success": True}

@app.post("/submit")
def submit(body: RequestBody):
    backend = Backend.get_backend(app)
    backend.save_submission(
        body.uid.strip(), body.passage_id, body.convert_to_example(), None
    )
    return {"success": True}

@app.post("/complete")  # complete writing HIT
async def complete(body: CommentBody, background_tasks: BackgroundTasks):
    backend = Backend.get_backend(app)
    backend.finalize_user_assignment(body.uid)
    backend.save_comment(body.uid, body.comment)
    backend.logger.info(f"Mturk submitted: {backend.get_current_state(body.uid)} {body.uid} {body.worker_id} {body.assignment_id}")
    backend.logging_current_status()
    background_tasks.add_task(
        backend.sleep_and_process_terminal_actions,
        body.uid,
        body.worker_id,
        body.assignment_id,
    )
    return {"completion_code": backend.get_completion_code(body.uid)}

@app.post("/validation_submit")  # complete validation HIT
async def validation_submit(
    body: ValidationRequestBody, background_tasks: BackgroundTasks
):
    backend = Backend.get_backend(app)
    uid = body.uid.strip()
    responses = body.responses
    mode = body.mode
    backend.save_validation_submission(body.uid.strip(), mode, responses)
    background_tasks.add_task(
        backend.sleep_and_process_terminal_actions,
        uid,
        body.worker_id,
        body.assignment_id,
    )
    backend.logging_current_status()
    backend.logger.info(f"Mturk submitted: {backend.get_current_state(body.uid)} {body.uid} {body.worker_id} {body.assignment_id}")
    return {"completion_code": backend.get_completion_code(uid)}

@app.get("/serialize")
def serialize_backend_data():
    """
    To stop the API server and resume it with the same data
    """
    backend = Backend.get_backend(app)
    backend.serialize_data()

@app.get("/show_log", include_in_schema=False)
def show_log_file(mode: str = "log", passcode: str = None):
    if app.state.args.passcode:
        backend = Backend.get_backend(app)
        res = backend.show_log(mode, passcode, app.state.args.passcode)
        return Response(content=res)
    else:
        return ""

@app.get("/set_value", include_in_schema=False)
def set_value(variable: str, value: int, passcode: str = None):
    if not app.state.args.passcode:
        return ""
    backend = Backend.get_backend(app)
    if not backend.check_passcode(passcode, app.state.args.passcode):
        return "incorrect passcode"
    if variable == "max_acceptable_hit_num":
        backend.max_acceptable_hit_num = value
        return f"set {variable}={value}"

@app.get("/get_value", include_in_schema=False)
def get_value(variable: str, passcode: str = None):
    if not app.state.args.passcode:
        return ""
    backend = Backend.get_backend(app)
    if not backend.check_passcode(passcode, app.state.args.passcode):
        return "incorrect passcode"
    if variable == "max_acceptable_hit_num":
        return f"{variable}={backend.max_acceptable_hit_num}"

@app.get("/enter_debug", include_in_schema=False)
def enter_debug(passcode: str = None):
    if not app.state.args.passcode:
        return ""
    backend = Backend.get_backend(app)
    if not backend.check_passcode(passcode, app.state.args.passcode):
        return "incorrect passcode"
    # backend.dataset["validation"]['gutenberg-Deephaven_and_Selected_Stories_&_Sketches-paragraph-0152_adv_3VW6495TLOJSA6R7QJX94RMER2TYYR_0'] = backend.dataset["validation"]['gutenberg-Deephaven_and_Selected_Stories_&_Sketches-paragraph-0152_adv_3VW6495TLOJSA6R7QJX94RMER2TYYR_0']
    breakpoint()

@app.post("/working")
def update_working_status(body: WorkingRequestBody):
    uid = body.uid
    backend = Backend.get_backend(app)
    backend.update_current_time(uid)
    return {"updated": 1}

def check_config_template_paths(config, template_dir):
    def check_exists(file_name):
        filepath = os.path.join(template_dir, file_name)
        if not os.path.exists(filepath):
            print(f"Not found: {filepath}")
            return False
        return True

    for temp_type in ["instruction", "mini_instruction", "faq", "task"]:
        if f"{temp_type}_template" in config:
            assert check_exists(config[f"{temp_type}_template"])
        elif "template_name" in config:
            # TODO
            template_name = config["template_name"]
            assert check_exists(f"{temp_type}_{template_name}.html")
        else:
            assert check_exists(f"{temp_type}_{config['collection_mode']}.html")

if __name__ == "__main__":
    if sys.version_info.minor <= 6:
        print("Run with Python >= 3.7")
        exit(1)
    # setup arguments
    args = get_arguments()

    # check arguments
    if args.max_acceptable_hit_num == 1 and args.run_mode == "release":
        msg = f"Proceed with args.max_acceptable_hit_num={args.max_acceptable_hit_num}? [Y/N] "
        if input(msg) != "Y":
            raise ValueError("Aborted.")

    hit_configs = json.load(open(args.hit_config, "r"))
    setattr(args, "hit_config", {})
    for config in hit_configs:
        check_config_template_paths(config, args.template_dir)
        if "hit_template" in config:
            config["hit_template"] = os.path.join(
                args.template_dir, config["hit_template"]
            )
        args.hit_config[config["collection_mode"]] = config

    if len(args.hit_config) == 1:
        collection_mode = list(args.hit_config.keys())[0]
    else:
        collection_mode = "multiple"

    # if not args.collected_passage_path and args.run_mode in ["release", "sandbox"]:
    #     msg = f"Proceed without args.collected_passage_path? [Y/N] "
    #     if input(msg) != "Y":
    #         raise ValueError("Aborted.")

    # set up path strings
    datestr = datetime.now().strftime("%Y%m%d%H%M%S")

    if not args.worker_log_path and args.run_mode != "debug":
        args.worker_log_path = os.path.join(
            args.worker_log_dir, f"{args.run_mode}{datestr}.json"
        )

    args.log_dir = os.path.join(args.log_dir, f"{args.run_mode}", f"{collection_mode}")
    os.makedirs(args.log_dir, exist_ok=True)
    if args.resume:
        annot_datestr = os.path.basename(os.path.normpath(args.save_dir))
        log_file = f"{args.log_dir}/{annot_datestr}.log"
        if os.path.exists(log_file):
            args.log_file = log_file
        else:
            args.log_file = f"{args.log_dir}/{args.annotation}{datestr}resume.log"
    else:
        args.save_dir = os.path.join(
            args.save_dir,
            f"{args.run_mode}",
            f"{collection_mode}",
            f"{args.annotation}{datestr}",
        )
        os.makedirs(args.save_dir, exist_ok=True)
        args.log_file = f"{args.log_dir}/{args.annotation}{datestr}.log"

    if args.run_mode == "debug":
        args.is_external = True

    print(json.dumps(args.__dict__))

    for k, v in json.load(open(args.mturk_config, "r")).items():
        setattr(args, k, v)

    # setup models
    models = []
    if (
        not args.no_model
        and len(set(args.hit_config.keys()) & set(["adv"])) > 0
    ):
        for k, v in json.load(open(args.model_config, "r")).items():
            setattr(args, k, v)
        from model.run_multiple_choice import load_model
        if args.model_type == "unifiedqa":
            from model.run_multiple_choice import predict_generation_example as predict_example
        else:
            from model.run_multiple_choice import predict_example

        if args.model_ensemble:
            for model_path in args.ensemble_models:
                args.model_name_or_path = model_path
                model, processor, tokenizer = load_model(args)
                models.append(model)
        else:
            model, processor, tokenizer = load_model(args)
            models.append(model)
    else:
        processor, tokenizer = None, None

    app.state.model_kit = {
        "models": models,
        "processor": processor,
        "tokenizer": tokenizer,
    }

    app.state.args = args

    if args.is_external:
        question_template_path = None
    else:
        question_template_path = os.path.join(
            args.template_dir, "hit_template_COLLECTION_MODE.html"
        )
    if not args.base_url.endswith(":8000"):
        if args.base_url.endswith("/"):
            args.base_url = args.base_url[:-1]
        args.base_url = args.base_url + f":{args.port}"
    mturk_config = {
        "access_id": args.aws_access_key_id,
        "secret_key": args.aws_secret_access_key,
        "base_url": args.base_url,
        "template_path": question_template_path,
        "issued_hits": "issued_hits",  # save directory of issued HITs
    }
    app.state.mturk_config = mturk_config

    Backend.get_backend(app)

    # start app
    if args.run_mode in ["sandbox", "release"] and "localhost" not in args.base_url:
        uvicorn.run(app, host="0.0.0.0", port=args.port, log_level="info")
    else:
        uvicorn.run(app, log_level="info")


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.1.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # s_info_processing_comparison [<img src="https://www.arpm.co/lab/icons/icon_permalink.png" width=30 height=30 style="display: inline;">](https://www.arpm.co/lab/redirect.php?code=s_info_processing_comparison&codeLang=Python)
# For details, see [here](https://www.arpm.co/lab/redirect.php?permalink=eb-sablepcomparison).

# +
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter

from arpym.estimation import exp_decay_fp
from arpym.statistics import meancov_sp
from arpym.tools import plot_ellipse, add_logo
from arpym.views import black_litterman, min_rel_entropy_normal
# -

# ## [Input parameters](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-parameters)

c = 0.82  # confidence level in the views
eta = np.array([1, -1])  # parameters for qualitative views
lam = 1.2  # average risk-aversion level
tau = 252  # uncertainty level in the reference model
tau_hl = 1386  # half life parameter
v = np.array([[1, - 1, 0], [0, 0, 1]])  # pick matrix
w = np.array([1/3, 1/3, 1/3])  # market-weighted portfolio

# ## [Step 0](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step00): Upload data

# +
path = '../../../databases/global-databases/equities/db_stocks_SP500/'

data = pd.read_csv(path + 'db_stocks_sp.csv', index_col=0, header=[0, 1])
# -

# ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step01): Compute time series of returns

n_ = len(w)  # market dimension
r_t = data.pct_change().iloc[1:, :n_].values  # returns

# ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step02): Compute the sample mean and the exponential decay sample covariance

t_ = len(r_t)
p_t_tau_hl = exp_decay_fp(t_, tau_hl)  # exponential decay probabilities
mu_hat_r, sig2_hat_r = meancov_sp(r_t, p_t_tau_hl)  # sample mean and cov.

# ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step03): Compute Black-Litterman prior parameters

# +
# expectation in terms of market equilibrium
mu_r_equil = 2 * lam * sig2_hat_r @ w

mu_m_pril = mu_r_equil
cv_pri_pred = (1 + 1 / tau) * sig2_hat_r
# -

# ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step04): Compute Black-Litterman posterior parameters

# +
# vectors quantifying the views
i = v @ mu_r_equil + eta * np.sqrt(np.diag(v @ sig2_hat_r@ v.T))
sig2_i_mu = ((1 - c) / (tau * c)) * (v @ sig2_hat_r @ v.T)

# Black-Litterman posterior parameters
mu_m_pos, cv_pos_pred = black_litterman(mu_r_equil, sig2_hat_r, tau, v, i,
                                        sig2_i_mu)
# -

# ## [Step 5](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step05): Compute Black-Litterman sure posterior parameters

mu_r_sure_bl = mu_r_equil + sig2_hat_r @ v.T @ \
             np.linalg.solve(v @ sig2_hat_r @ v.T, i - v @ mu_r_equil)
sig2_r_sure_bl = (1 + 1 / tau) * sig2_hat_r - (1 / tau) * sig2_hat_r @ v.T\
               @ np.linalg.solve(v @ sig2_hat_r @ v.T, v @ sig2_hat_r)

# ## [Step 6](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step06): Compute posterior parameters from distributional views (Minimum Relative Entropy)

# +
v_mre = v
v_sig_mre = np.eye(n_)
imre = i
sig2_i_mumre = sig2_hat_r

mu_r_mre, sig2_r_mre = min_rel_entropy_normal(mu_r_equil, sig2_hat_r, v_mre,
                                              imre, v_sig_mre, sig2_i_mumre)
# -

# ## [Step 7](https://www.arpm.co/lab/redirect.php?permalink=s_info_processing_comparison-implementation-step07): Compute posterior parameters from point views

# +
k_ = len(v)  # view variables dimension
v_point = v
z_point = i

mu_r_point, sig2_r_point = min_rel_entropy_normal(mu_r_equil, sig2_hat_r,
                                                  v_point, z_point, v_point,
                                                  np.zeros((k_)))
# -

# ## Plots

# +
col_darkgrey = [.6, .6, .6]
col_orange = [.9, .6, 0]
col_reddishpurple = [.8, .6, .7]
col_skyblue = [.35, .7, .9]
spot = [0, 1]

plt.style.use('arpm')
fig = plt.figure(figsize=(10, 8))

plot_ellipse(mu_hat_r[spot], sig2_hat_r[np.ix_(spot, spot)],
             color=col_darkgrey, line_width=1.3)
plot_ellipse(mu_m_pril[spot], cv_pri_pred[np.ix_(spot, spot)], color='k',
             line_width=1.3)
plot_ellipse(mu_m_pos[spot], cv_pos_pred[np.ix_(spot, spot)], color='b',
             line_width=1.3)
plot_ellipse(mu_r_sure_bl[spot], sig2_r_sure_bl[np.ix_(spot, spot)],
             color=col_skyblue, line_width=2)
plot_ellipse(mu_r_mre[spot], 0.98 * sig2_r_mre[np.ix_(spot, spot)],
             color=col_orange, line_width=1.5)
plot_ellipse(mu_r_point[spot], sig2_r_point[np.ix_(spot, spot)],
             color=col_reddishpurple, line_width=1.3)

plt.plot(mu_hat_r[spot[0]], sig2_hat_r[0, spot[1]], '.', color=col_darkgrey,
         markersize=20)
plt.plot(mu_m_pril[spot[0]], mu_m_pril[spot[1]], '*', color='k', markersize=15)
plt.annotate('equilibrium', weight="bold",
             xy=(mu_m_pril[spot[0]], mu_m_pril[spot[1]]),
             xytext=(0.006, 0.01),
             arrowprops=dict(facecolor="black", width=0.5,
                             headwidth=4, shrink=0.1))
plt.plot(mu_m_pos[spot[0]], mu_m_pos[spot[1]], '.', color='b', markersize=15)
plt.plot(mu_r_sure_bl[spot[0]], mu_r_sure_bl[spot[1]], 'o', color=col_orange,
         markersize=10)
plt.plot(mu_r_mre[spot[0]], mu_r_mre[spot[1]], '.', color=col_skyblue,
         markersize=15)
plt.plot(mu_r_point[spot[0]], mu_r_point[spot[1]], '.',
         color=col_reddishpurple, markersize=5)

plt.plot(r_t.T[0], r_t.T[1], '.', color=col_darkgrey, markersize=4)
plt.xticks(np.arange(-0.04,  0.071, step=0.01))

plt.xlim([-0.035, 0.07])
plt.ylim([-0.065, 0.07])
plt.gca().xaxis.set_major_formatter(FuncFormatter(lambda y, _:
                                                  '{:.0%}'.format(y)))
plt.gca().yaxis.set_major_formatter(FuncFormatter(lambda y, _:
                                                  '{:.0%}'.format(y)))
plt.xlabel('$R_{1}$')
plt.ylabel('$R_{2}$')
legend = plt.legend(['Returns( & sample mean/covariance)',
                     'Black-Litterman prior (equilibrium)',
                     'Black-Litterman posterior', 'Black-Litterman sure',
                     'Min. Rel. Entropy distributional view',
                     'regression/Min.Rel.Entropy point view'])

str1 = r'Confidence level in the prior: $\tau$ = %d ' % np.floor(tau)
str2 = 'Confidence level in the views: c = %d' % np.floor(100 * c)
plt.text(0.036, -0.034, str1)
plt.text(0.036, -0.038, str2 + '%')

plt.grid(True)
add_logo(fig, location=1)
plt.tight_layout()

max_stars_repo_name: dpopadic/arpmRes

max_issues_repo_name: dpopadic/arpmRes

max_forks_repo_name: dpopadic/arpmRes


text:
import re
from pathlib import PurePosixPath
from itertools import groupby, islice
import numpy as np

from typing import Dict, Union, Any
from cw.tree_node import TreeNode

# Regular expressions matching the naming scheme of ndarrays.
ndarray_re = re.compile(r"^\s*(\w+?)((?:_\d+)+)\s*$")

def tables_to_object_hierarchy(tables):
    """
    Transforms a dictionary of tables into a object hierarchy.

    List elements with the same name ending with trailing integers separated
    by underscores (example `name_0_1`) are combined into a single
    :class:`numpy.ndarray` with n+1 dimensions, where n is the number of trailing
    integers. The first dimension's size is the same as the length of the lists.
    The trailing integers then define the index of the higher dimensions where
    the data will be inserted to.

    Elements have be grouped in sub-dictionaries. If the key of the dictionary
    is a string it will be appended to the front of the child elements name.
    It can thus be used to define namespaces. Otherwise the key is ignored.

    Example
    .. code-block:: python

       inp = {
           4: {
               'foo.quux_0_0': np.array([9, 11, 14, 17]),
               'foo.quux_1_0': np.array([10, 12, 15, 18]),
               'foo.quux_2_0': np.array([11, 13, 16, 19])
           },
           "bar": {
               "bas": 123
           }
       }

       {
           'foo': {
               'quux': np.array([[[9.],  [10.], [11.]],
                                 [[11.], [12.], [13.]],
                                 [[14.], [15.], [16.]],
                                 [[17.], [18.], [19.]]])
           },
           "bar": {
               "bas": 123
           }
       }

    :param tables: Dictionary containing dictionaries whose key is the path
       to the element in the resulting object hierarchy.
    :return: Object hierarchy
    """
    data_tables = flatten_tables(tables)
    process_ndarrays(data_tables)
    root_node = TreeNode.from_path_value_pairs(data_tables)
    root_obj = root_node.object_hierarchy()
    return root_obj

def flatten_tables(tables: Dict[Union[int, str], Dict[str, Any]]):
    """
    Returns a list containing tuples with two elements, the first being a
    :class:`pathlib.PurePosixPath` with the path to the value in the final
    object hierarchy and second one being the value. Namespaces are resolved

    :param tables:
    :return:
    """

    flat_tables = {}
    for namespace, local_tables in tables.items():
        # If the namespace is not a string, the element is placed on the root namespace.
        # Everything behind the hashtag is a comment.
        namespace = namespace.split("#")[0].strip() if isinstance(namespace, str) else ""

        for node_name, node_value in local_tables.items():
            path = PurePosixPath(namespace, *node_name.split("."))
            flat_tables[path] = node_value

    return flat_tables

def process_ndarrays(tables):
    # It is not possible to make the changes in the table inside of the
    # main loop because it's not possible to change the length of a
    # dictionary while iterating through it.
    table_changes = []

    for path, group in find_ndarrays(tables):
        # This list will contain the changes that are needed to be made
        # in the table.
        change = [path, [], None]

        # Find shape of ndarray
        # Initializes the size as 0 for all dimensions.
        # Iterates through all of the elements in the array to look for
        # the largest index and sets the size to the index plus 1.
        shape = [0] * len(group[0][1])
        for _, idx in group:
            for i, size in enumerate(idx):
                size += 1
                if size > shape[i]:
                    shape[i] = size

        # The first dimension of the final array should have the same
        # size as the length of the columns.
        shape = (len(tables[group[0][0]]), *shape)

        # Initialize new ndarray
        array = np.empty(shape)

        # Copy the data of the old column and put it in the new array.
        for col_path, idx in group:
            array[(slice(None), *idx)] = np.array(tables[col_path])
            change[1].append(col_path)

        change[2] = array
        table_changes.append(change)

    # Apply the changes to the table.
    for path, old_paths, value in table_changes:
        for old_path in old_paths:
            del tables[old_path]
        tables[path] = value

def find_ndarrays(tables):
    def ndarray_cols():
        for path, value in tables.items():
            match = ndarray_re.match(path.name)
            if match:
                yield path.with_name(match.group(1)), path, \
                      tuple(map(int, islice(match.group(2).split("_"), 1, None)))

    for k, g in groupby(ndarray_cols(), lambda x: x[0]):
        group = []
        for e in g:
            group.append((e[1], e[2]))
        yield k, group

max_stars_repo_name: aarondewindt/cw

max_issues_repo_name: aarondewindt/cw

max_forks_repo_name: aarondewindt/cw


text:
import os
import shutil
import tempfile
import unittest
from glob import glob

import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim

from easy_model_saving import model_saver

def ensure_path_tf(folder):
    if not tf.gfile.Exists(folder):
        tf.gfile.MakeDirs(folder)
    return folder

def ensure_join_path(*path_list):
    return ensure_path_tf(os.path.join(*path_list))

def cnn_block(net, features, param, scope):
    with tf.variable_scope(scope):
        net = slim.repeat(net, param.cnn_convolutions, slim.conv2d, features, param.cnn_convolution_kernel,
                          padding=param.cnn_convolution_padding,
                          activation_fn=param.cnn_convolution_activation,
                          scope='Convolution')
        net = slim.max_pool2d(net, param.cnn_max_pool_kernel, padding=param.cnn_max_pool_padding, scope='MaxPool')
    return net

class Parameters:
    image_x_len = 28
    image_y_len = 28
    output_features = 10

    cnn_blocks = 2
    cnn_features_initial = 64
    cnn_features_block_multiply = 2

    cnn_convolutions = 2
    cnn_convolution_kernel = (3, 3)
    cnn_convolution_padding = 'SAME'
    cnn_convolution_activation = tf.nn.relu
    cnn_max_pool_kernel = (2, 2)
    cnn_max_pool_padding = 'SAME'

def graph(batch_size=None, param=Parameters):
    x_ = tf.placeholder(tf.float32, [batch_size, param.image_x_len * param.image_y_len], name='Image')
    y_ = tf.placeholder(tf.float32, [batch_size, param.output_features], name='Label')

    x_image = tf.reshape(x_, [-1, param.image_x_len, param.image_y_len, 1], name='2DImage')

    cnn_features = param.cnn_features_initial
    cnn = x_image
    for i in range(param.cnn_blocks):
        cnn = cnn_block(cnn, cnn_features, param, 'Block_' + repr(i + 1))
        cnn_features *= param.cnn_features_block_multiply

    cnn_flattened = slim.flatten(cnn)
    unscaled_logit = slim.fully_connected(cnn_flattened, param.output_features,
                                          activation_fn=None)

    label_probability = tf.nn.softmax(unscaled_logit, name='Probability')
    tf.argmax(label_probability, 1, name='Prediction')

class TestSaver(unittest.TestCase):
    def test_model_saving(self):
        root_path = tempfile.gettempdir()
        checkpoint_path = ensure_join_path(root_path, 'checkpoint')
        try:
            shutil.rmtree(checkpoint_path)
        except FileNotFoundError:
            pass
        with tf.Session() as sess:
            graph()

            last_step = model_saver.restore_graph_variables(checkpoint_path)
            assert last_step == 0
            if last_step == 0:
                model_saver.initialize_graph_variables()

            stat_1 = np.sum(sess.run(tf.trainable_variables())[0].flatten())
            saver = model_saver.Saver(checkpoint_path)

            i = 1
            for global_step in range(last_step, last_step + 10):
                saver.save(global_step=global_step)
                assert len(glob(checkpoint_path + '/*.meta', recursive=True)) == i
                i += 1

        tf.reset_default_graph()
        with tf.Session() as sess:
            graph()
            last_step = model_saver.restore_graph_variables(checkpoint_path)
            assert last_step == 9
            if last_step == 0:
                model_saver.initialize_graph_variables()

            stat_2 = np.sum(sess.run(tf.trainable_variables())[0].flatten())

            assert stat_1 == stat_2

            saver = model_saver.Saver(checkpoint_path)

            i = last_step + 1
            for global_step in range(last_step, last_step + 10):
                saver.save(global_step=global_step)
                assert len(glob(checkpoint_path + '/*.meta', recursive=True)) == i
                i += 1

        try:
            shutil.rmtree(checkpoint_path)
        except FileNotFoundError:
            pass
        tf.reset_default_graph()
        with tf.Session() as sess:
            graph()
            last_step = model_saver.restore_graph_variables(checkpoint_path)
            assert last_step == 0
            if last_step == 0:
                model_saver.initialize_graph_variables()

            stat_3 = np.sum(sess.run(tf.trainable_variables())[0].flatten())

            assert stat_1 != stat_3

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: philipperemy/tf-easy-model-saving

max_issues_repo_name: philipperemy/tf-easy-model-saving

max_forks_repo_name: philipperemy/tf-easy-model-saving


text:
"""
Remove background from dataset

* Import data (.csv):
*     File > Load File
*         OR
*     Ctrl+L
* Set Order and Threshold parameters
* Calculate Background
*     File > Calculate Background
*         OR
*     Ctrl+B
*         OR
*     Click "Background"
* Export data
*     File > Export (.csv)
*         OR
*     Ctrl+E

Colin Brosseau (colin@erzatz.info)
License:
Last modified: 2016/11/29
"""

"""
TODO

grey calculate background, save image of export data if not possible
display error message if background absent while saving
add options for reading the csv file (header, etc)
add generic interface for custom cost function
"""

#import sys, os, random
import sys
from PyQt4.QtCore import *
from PyQt4.QtGui import *

import numpy as np

import matplotlib

from matplotlib.backends.backend_qt4agg import FigureCanvasQTAgg as FigureCanvas
#from matplotlib.backends.backend_qt4agg import NavigationToolbar2QTAgg as NavigationToolbar
from matplotlib.backends.backend_qt4agg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure

import backcor
from importCsvGui import importCsv

class AppForm(QMainWindow):
    def __init__(self, parent=None):
        QMainWindow.__init__(self, parent)
        self.showMaximized()
        self.x = 0
        self.y = 0
        self.setWindowTitle('Background removal')

        self.create_menu()
        self.create_main_frame()
        self.create_status_bar()

        self.order.setText('6')
        self.threshold.setText('50')
        self.has_background = False
        self.background_removed = False
        #self.plot_data()

    def export_csv(self):
        header = ''
        if self.background_removed:
            header = header + ' > Background removed'

        file_choices = "CSV (*.csv)|*.csv"

        path = str(QFileDialog.getSaveFileName(self,
                        'Save file', '',
                        file_choices))
        if path:
#            self.canvas.print_figure(path, dpi=self.dpi)
            self.statusBar().showMessage('Saved to %s' % path, 2000)

            np.savetxt(path, np.array([self.x, self.y-self.background]).transpose(), fmt='%1.7e', delimiter=',', header=header)

    def save_plot(self):
        file_choices = "PNG (*.png)|*.png"

        path = str(QFileDialog.getSaveFileName(self,
                        'Save file', '',
                        file_choices))
        if path:
            self.canvas.print_figure(path, dpi=self.dpi)
            self.statusBar().showMessage('Saved to %s' % path, 2000)

    def load_csv(self):
        self.x, self.y, parameters, ok = importCsv.getData()
        if ok:
            print(parameters)
            self.draw_button.setEnabled(True)
            self.has_background = False
            self.background_removed = False
            self.plot_data()

    def on_about(self):
        msg = """
         Remove background from dataset

         * Import data (.csv):
         *     File > Load File
         * Set Order and Threshold parameters
         * Calculate Background
         *     File > Calculate Background
         * Export data
         *     File > Export (.csv)
        """
        QMessageBox.about(self, "Basic usage", msg.strip())

    def plot_data(self):
        """ Redraws the figure
        """
        self.axes.clear()
        self.axes.grid(self.grid_cb.isChecked())

        self.axes.plot(self.x, self.y, '.', label='data')
        self.axes.set_xlim([np.min(self.x), np.max(self.x)])
        self.axes.set_ylim([np.min(self.y), np.max(self.y)])
        self.axes.set_title('Data')
        self.axes.set_xticklabels([])

        self.axes2.clear()
        if self.has_background:
            self.axes.set_title('Data & Background')
            self.axes.plot(self.x,self.background,'', label='background')

            z = self.y - self.background
            self.axes2.grid(self.grid_cb.isChecked())

            self.axes2.plot(self.x, z, '.', label='data - background')
            self.axes2.set_xlim([np.min(self.x), np.max(self.x)])
            self.axes2.set_ylim([np.min(z), np.max(z)])
            self.axes2.set_title('Difference')
            self.axes2.legend()

        self.axes.legend()
        self.canvas.draw()

    def calculate_background(self):
        """ Calculate the background
        """
        # fit method
        text = str(self.fit_method.currentText())
        if text == "symmetric Huber function":
            method = 'sh'
        elif text == "asymmetric Huber function":
            method = 'ah'
        elif text == "symmetric truncated quadratic":
            method = 'stq'
        elif text == "asymmetric truncated quadratic":
            method = 'atq'

        self.statusBar().showMessage('Calculating background...')  # problem here: it just make status bar to blink. Like it print this line after it make calculation
        order = int(self.order.text())
        threshold = float(self.threshold.text())
        z, a, it = backcor.backcor(self.x, self.y, order, threshold, method)
        self.statusBar().showMessage('')
        self.has_background = True
        self.background = z
        self.plot_data()

    def create_main_frame(self):
        self.main_frame = QWidget()

        self.dpi = 100
#        self.fig = Figure((5.0, 4.0), dpi=self.dpi)
        self.fig = Figure(dpi=self.dpi, tight_layout=True)
        self.canvas = FigureCanvas(self.fig)
        self.canvas.setParent(self.main_frame)

        self.axes = self.fig.add_subplot(211)
        self.axes2 = self.fig.add_subplot(212)

        # Create the navigation toolbar, tied to the canvas
        self.mpl_toolbar = NavigationToolbar(self.canvas, self.main_frame)

        # Other GUI controls

        # Order of the polynomial
        order_label = QLabel('Order')
        self.order = QLineEdit()
        self.order.setValidator(QIntValidator(0, 101, self))
        self.order.setMinimumWidth(3)
        self.order.setMaxLength(3)
        order = QVBoxLayout()
        order.addWidget(order_label)
        order.addWidget(self.order)

        # Threshold (peak detection)
        threshold_label = QLabel('Threshold')
        self.threshold = QLineEdit()
        self.threshold.setMinimumWidth(3)
        self.threshold.setValidator(QDoubleValidator(bottom=0))
        threshold = QVBoxLayout()
        threshold.addWidget(threshold_label)
        threshold.addWidget(self.threshold)

        # Fit method
        fit_method_label = QLabel('Fit Method')

        self.fit_method = QComboBox(self)
        self.fit_method.addItem("symmetric Huber function")
        self.fit_method.addItem("asymmetric Huber function")
        self.fit_method.addItem("symmetric truncated quadratic")
        self.fit_method.addItem("asymmetric truncated quadratic")
        index = self.fit_method.findText('asymmetric truncated quadratic')
        if index >= 0:
            self.fit_method.setCurrentIndex(index)
        fit_method = QVBoxLayout()
        fit_method.addWidget(fit_method_label)
        fit_method.addWidget(self.fit_method)

        self.draw_button = QPushButton("&Background")
        self.draw_button.setEnabled(False)
        self.connect(self.draw_button, SIGNAL('clicked()'), self.calculate_background)

        self.grid_cb = QCheckBox("Show &Grid")
        self.grid_cb.setChecked(False)
        self.connect(self.grid_cb, SIGNAL('stateChanged(int)'), self.plot_data)

        # Layout with box sizers
        hbox = QHBoxLayout()

        hbox.addLayout(order)
        hbox.setAlignment(order, Qt.AlignVCenter)
        hbox.addLayout(threshold)
        hbox.setAlignment(threshold, Qt.AlignVCenter)
        hbox.addLayout(fit_method)
        hbox.setAlignment(fit_method, Qt.AlignVCenter)
#        for w in [  self.order, self.threshold, self.draw_button, self.remove_button, self.grid_cb,
        for w in [ self.draw_button, self.grid_cb,]:
            hbox.addWidget(w)
            hbox.setAlignment(w, Qt.AlignVCenter)

        # Main window (everything together)
        vbox = QVBoxLayout()
        vbox.addWidget(self.canvas)
        vbox.addWidget(self.mpl_toolbar)
        vbox.addLayout(hbox)

        self.main_frame.setLayout(vbox)
        self.setCentralWidget(self.main_frame)

    def create_status_bar(self):
        self.status_text = QLabel("")
        self.statusBar().addWidget(self.status_text, 1)

    def create_menu(self):
        self.file_menu = self.menuBar().addMenu("&File")

        load_file_action = self.create_action("&Load file",
            shortcut="Ctrl+O", slot=self.load_csv,
            tip="Load data")
        calculate_action = self.create_action("Calculate &Background",
            shortcut="Ctrl+B", slot=self.calculate_background,
            tip="Calculate Background")
        save_file_action = self.create_action("&Save plot",
            shortcut="Ctrl+S", slot=self.save_plot,
            tip="Save the plot")
        export_csv_action = self.create_action("&Export (.csv)",
            shortcut="Ctrl+E", slot=self.export_csv,
            tip="Export to csv")
        quit_action = self.create_action("&Quit", slot=self.close,
            shortcut="Ctrl+Q", tip="Close the application")

        self.add_actions(self.file_menu,
            (load_file_action, calculate_action, save_file_action, export_csv_action, None, quit_action))

        self.help_menu = self.menuBar().addMenu("&Help")
        about_action = self.create_action("&About",
            shortcut='F1', slot=self.on_about,
            tip='About this program')

        self.add_actions(self.help_menu, (about_action,))

    def add_actions(self, target, actions):
        for action in actions:
            if action is None:
                target.addSeparator()
            else:
                target.addAction(action)

    def create_action(  self, text, slot=None, shortcut=None,
                        icon=None, tip=None, checkable=False,
                        signal="triggered()"):
        action = QAction(text, self)
        if icon is not None:
            action.setIcon(QIcon(":/%s.png" % icon))
        if shortcut is not None:
            action.setShortcut(shortcut)
        if tip is not None:
            action.setToolTip(tip)
            action.setStatusTip(tip)
        if slot is not None:
            self.connect(action, SIGNAL(signal), slot)
        if checkable:
            action.setCheckable(True)
        return action

def main():
    app = QApplication(sys.argv)
    form = AppForm()
    form.show()
    app.exec_()

if __name__ == "__main__":
    main()

max_stars_repo_name: ColinBrosseau/Background-Removal

max_issues_repo_name: ColinBrosseau/Background-Removal

max_forks_repo_name: ColinBrosseau/Background-Removal


text:
import setup
import json
import boto3
import os
import numpy as np
import PIL
from PIL import Image

MODEL_BUCKET = 'e6765'
cache = {}
s3 = boto3.resource('s3')
url = "https://s3.amazonaws.com/chat-bot-hanzhou2/source.jpg"
def lambda_handler(event, context):
    # TODO implement
    #print(os.listdir("/tmp/"))
    filename = event["Records"][0]["s3"]["object"]["key"];
    print(filename)
    sourceFile = get_image("gu.jpg")
    targetFile = get_image(filename)
    client=boto3.client('rekognition')

    imageSource=open(sourceFile,'rb')
    imageTarget=open(targetFile,'rb')

    response=client.compare_faces(SimilarityThreshold=90,
                                  SourceImage={'Bytes': imageSource.read()},
                                  TargetImage={'Bytes': imageTarget.read()})

    s3_client = boto3.client('s3')
    if len(response['FaceMatches']) == 0:
        message = "tresspassing found "
        s3_client.upload_file(targetFile, 'chat-bot-hanzhou2', 'source.jpg')
    else:
        faceMatch = response['FaceMatches'][0]
        similarity = faceMatch['Similarity']
        imageSource.close()
        imageTarget.close()
        #image = np.expand_dims(image, axis=0)
        #result = model.predict(image)
        if similarity>=0.9:
            message = "successfully entered "
            print("good")
        else:
            message = "tresspassing found "
            s3_client.upload_file(targetFile, 'chat-bot-hanzhou2', 'source.jpg')
            print("bad")
    sns = boto3.client('sns')
    response = sns.publish(
    TopicArn='arn:aws:sns:us-east-1:817280606254:FaceTopic',
    Message=message+url
    )

    return {
        'statusCode': 200,
        'body': json.dumps('Hello from Lambda!')
    }

def get_image(key):

    local_path = os.path.join('/tmp', key)
    bucket=s3.Bucket(MODEL_BUCKET)
    bucket.download_file(key, local_path)
    return  local_path


text:
# Create by Packetsss
# Personal use is allowed
# Commercial use is prohibited

from os import environ
environ['PYGAME_HIDE_SUPPORT_PROMPT'] = '1'
environ['SDL_VIDEO_WINDOW_POS'] = "1500, 200"

import numpy as np
import pygame

import collisions
import event
import gamestate
import graphics
import config

was_closed = False
while not was_closed:
    game = gamestate.GameState()
    button_pressed = graphics.draw_main_menu(game)

    if button_pressed == config.play_game_button:
        game.start_pool()
        events = event.events()
        while not (events["closed"] or game.is_game_over or events["quit_to_main_menu"]):
            events = event.events()
            collisions.resolve_all_collisions(game.balls, game.holes, game.table_sides)
            game.redraw_all()

            if game.all_not_moving():
                game.turned_over = False
                game.check_pool_rules()
                if not game.turned_over:
                    print("Good hit!")
                game.cue.make_visible(game.current_player)
                while not (
                    (events["closed"] or events["quit_to_main_menu"]) or game.is_game_over) and game.all_not_moving():
                    game.redraw_all()
                    events = event.events()
                    # operate cue
                    if game.cue.is_clicked(events):
                        game.cue.cue_is_active(game, events)
                    # move cue ball on foul
                    elif game.can_move_white_ball and game.white_ball.is_clicked(events):
                        game.white_ball.is_active(game, game.is_behind_line_break())
            if config.resolution[0] < game.white_ball.rect.center[0] or game.white_ball.rect.center[0] < 0\
                or config.resolution[1] < game.white_ball.rect.center[1] or game.white_ball.rect.center[1] < 0:
                game.check_potted(ball_outside_table=True)

        was_closed = events["closed"]

    if button_pressed == config.exit_button:
        was_closed = True

pygame.quit()

max_stars_repo_path: Python Tutorial Reinforcement Learning/12_stable_baseline3/pool_game_posvr/pool/main.py

max_issues_repo_name: packetsss/Python

max_forks_repo_name: packetsss/Python


text:
from collections import namedtuple
from ctypes import Structure, c_int, c_void_p, sizeof
from itertools import product
from math import ceil
from abc import ABC, abstractmethod
import atexit

from cached_property import cached_property
from cgen import Struct, Value

import numpy as np

from devito.data import Decomposition
from devito.parameters import configuration
from devito.types import LEFT, RIGHT, CompositeObject, Object
from devito.tools import EnrichedTuple, as_tuple, is_integer

# Do not prematurely initialize MPI
# This allows launching a Devito program from within another Python program
# that has *already* initialized MPI
import mpi4py
mpi4py.rc(initialize=False, finalize=False)
from mpi4py import MPI  # noqa

__all__ = ['Distributor', 'SparseDistributor', 'MPI']

class AbstractDistributor(ABC):

    """
    Decompose a set of :class:`Dimension`s over a set of MPI processes.

    .. note::

        This is an abstract class, which simply defines the interface that
        all subclasses are expected to implement.
    """

    def __init__(self, shape, dimensions):
        self._glb_shape = as_tuple(shape)
        self._dimensions = as_tuple(dimensions)

    def __repr__(self):
        return "%s(nprocs=%d)" % (self.__class__.__name__, self.nprocs)

    @abstractmethod
    def comm(self):
        """The MPI communicator."""
        return

    @abstractmethod
    def myrank(self):
        """The rank of the calling MPI process."""
        return

    @abstractmethod
    def mycoords(self):
        """The coordinates of the calling MPI rank in the Distributor topology."""
        return

    @abstractmethod
    def nprocs(self):
        """A shortcut for the number of processes in the MPI communicator."""
        return

    @property
    def is_parallel(self):
        return self.nprocs > 1

    @cached_property
    def glb_numb(self):
        """The global indices owned by the calling MPI rank."""
        assert len(self.mycoords) == len(self.decomposition)
        glb_numb = [i[j] for i, j in zip(self.decomposition, self.mycoords)]
        return EnrichedTuple(*glb_numb, getters=self.dimensions)

    @cached_property
    def glb_slices(self):
        """The global indices owned by the calling MPI rank, as a mapper from
        :class:`Dimension`s to slices."""
        return {d: slice(min(i), max(i) + 1)
                for d, i in zip(self.dimensions, self.glb_numb)}

    @property
    def glb_shape(self):
        """Shape of the decomposed domain."""
        return EnrichedTuple(*self._glb_shape, getters=self.dimensions)

    @property
    def shape(self):
        """The calling MPI rank's local shape."""
        return tuple(len(i) for i in self.glb_numb)

    @property
    def dimensions(self):
        """The decomposed :class:`Dimension`s."""
        return self._dimensions

    @cached_property
    def decomposition(self):
        """The :class:`Decomposition`s, one for each decomposed :class:`Dimension`."""
        return EnrichedTuple(*self._decomposition, getters=self.dimensions)

    @property
    def ndim(self):
        """Number of decomposed :class:`Dimension`s"""
        return len(self._glb_shape)

    def glb_to_loc(self, dim, *args, strict=True):
        """
        Convert a global index into a relative (default) or absolute local index.

        Parameters
        ----------
        dim : :class:`Dimension`
            The global index Dimension.
        *args
            There are several possibilities, documented in
            :meth:`Decomposition.convert_index`.
        strict : bool, optional
            If False, return args without raising an error if `dim` does not appear
            among the Distributor Dimensions.
        """
        if dim not in self.dimensions:
            if strict is True:
                raise ValueError("`%s` must be one of the Distributor dimensions" % dim)
            else:
                return args[0]
        return self.decomposition[dim].convert_index(*args)

class Distributor(AbstractDistributor):

    """
    Decompose a set of :class:`Dimension`s over a set of MPI processes.

    :param shape: The global shape of the domain to be decomposed.
    :param dimensions: The decomposed :class:`Dimension`s.
    :param comm: An MPI communicator.
    """

    def __init__(self, shape, dimensions, input_comm=None):
        super(Distributor, self).__init__(shape, dimensions)

        if configuration['mpi']:
            # First time we enter here, we make sure MPI is initialized
            if not MPI.Is_initialized():
                MPI.Init()

                # Make sure Finalize will be called upon exit
                def finalize_mpi():
                    MPI.Finalize()
                atexit.register(finalize_mpi)

            self._input_comm = (input_comm or MPI.COMM_WORLD).Clone()

            # `MPI.Compute_dims` sets the dimension sizes to be as close to each other
            # as possible, using an appropriate divisibility algorithm. Thus, in 3D:
            # * topology[0] >= topology[1] >= topology[2]
            # * topology[0] * topology[1] * topology[2] == self._input_comm.size
            # However, `MPI.Compute_dims` is distro-dependent, so we have to enforce
            # some properties through our own wrapper (e.g., OpenMPI v3 does not
            # guarantee that 9 ranks are arranged into a 3x3 grid when shape=(9, 9))
            topology = compute_dims(self._input_comm.size, len(shape))
            # At this point MPI's dimension 0 corresponds to the rightmost element
            # in `topology`. This is in reverse to `shape`'s ordering. Hence, we
            # now restore consistency
            self._topology = tuple(reversed(topology))

            if self._input_comm is not input_comm:
                # By default, Devito arranges processes into a cartesian topology.
                # MPI works with numbered dimensions and follows the C row-major
                # numbering of the ranks, i.e. in a 2x3 Cartesian topology (0,0)
                # maps to rank 0, (0,1) maps to rank 1, (0,2) maps to rank 2, (1,0)
                # maps to rank 3, and so on.
                self._comm = self._input_comm.Create_cart(self._topology)
            else:
                self._comm = input_comm
        else:
            self._input_comm = None
            self._comm = MPI.COMM_NULL
            self._topology = tuple(1 for _ in range(len(shape)))

        # The domain decomposition
        self._decomposition = [Decomposition(np.array_split(range(i), j), c)
                               for i, j, c in zip(shape, self.topology, self.mycoords)]

    def __del__(self):
        if self._input_comm is not None:
            self._input_comm.Free()

    @property
    def comm(self):
        return self._comm

    @property
    def myrank(self):
        if self.comm is not MPI.COMM_NULL:
            return self.comm.rank
        else:
            return 0

    @property
    def mycoords(self):
        if self.comm is not MPI.COMM_NULL:
            return tuple(self.comm.coords)
        else:
            return tuple(0 for _ in range(self.ndim))

    @property
    def nprocs(self):
        if self.comm is not MPI.COMM_NULL:
            return self.comm.size
        else:
            return 1

    @property
    def topology(self):
        return self._topology

    @cached_property
    def all_coords(self):
        """Return an iterable containing the coordinates of each MPI rank in
        the decomposed domain. The iterable is order based on the MPI rank."""
        ret = product(*[range(i) for i in self.topology])
        return tuple(sorted(ret, key=lambda i: self.comm.Get_cart_rank(i)))

    @cached_property
    def all_numb(self):
        """Return an iterable containing the global numbering of all MPI ranks."""
        ret = []
        for c in self.all_coords:
            glb_numb = [i[j] for i, j in zip(self.decomposition, c)]
            ret.append(EnrichedTuple(*glb_numb, getters=self.dimensions))
        return tuple(ret)

    @cached_property
    def all_ranges(self):
        """Return an iterable containing the global ranges of each MPI rank."""
        ret = []
        for i in self.all_numb:
            ret.append(EnrichedTuple(*[range(min(j), max(j) + 1) for j in i],
                                     getters=self.dimensions))
        return tuple(ret)

    @cached_property
    def glb_pos_map(self):
        """Return the mapper ``dimension -> side`` telling the position
        of the calling rank in the global grid."""
        ret = {}
        for d, i, s in zip(self.dimensions, self.mycoords, self.topology):
            v = []
            if i == 0:
                v.append(LEFT)
            if i == s - 1:
                v.append(RIGHT)
            ret[d] = tuple(v)
        return ret

    def glb_to_rank(self, index):
        """
        Return the rank owning a given global index.

        :param index: A single domain index, or a list of domain indices. In
                      the latter case, a list of corresponding ranks is returned.
        """
        if isinstance(index, (tuple, list)):
            if len(index) == 0:
                return None
            elif is_integer(index[0]):
                # `index` is a single point
                indices = [index]
            else:
                indices = index
        ret = []
        for i in indices:
            assert len(i) == self.ndim
            found = False
            for r, j in enumerate(self.all_ranges):
                if all(v in j[d] for v, d in zip(i, self.dimensions)):
                    ret.append(r)
                    found = True
                    break
            assert found
        return tuple(ret) if len(indices) > 1 else ret[0]

    @property
    def neighbours(self):
        """
        Return the mapper ``proc -> side``; ``proc`` is the rank of a
        neighboring process, while ``side`` tells whether ``proc`` is
        logically at right (value=1) or left (value=-1) of ``self``.
        """
        shifts = {d: self._comm.Shift(i, 1) for i, d in enumerate(self.dimensions)}
        ret = {}
        for d, (src, dest) in shifts.items():
            ret[d] = {}
            ret[d][LEFT] = src
            ret[d][RIGHT] = dest
        return ret

    @cached_property
    def _C_comm(self):
        """An :class:`Object` wrapping an MPI communicator."""
        return MPICommObject(self.comm)

    @cached_property
    def _C_neighbours(self):
        """A :class:`ctypes.Struct` to access the neighborhood of a given rank."""
        entries = list(product(self.dimensions, [LEFT, RIGHT]))
        fields = [('%s%s' % (d, i), c_int) for d, i in entries]
        obj = CompositeObject('nb', 'neighbours', Structure, fields)
        for d, i in entries:
            setattr(obj.value._obj, '%s%s' % (d, i), self.neighbours[d][i])
        cdef = Struct('neighbours', [Value('int', i) for i, _ in fields])
        CNeighbours = namedtuple('CNeighbours', 'ctype cdef obj')
        return CNeighbours(obj.dtype, cdef, obj)

class SparseDistributor(AbstractDistributor):

    """
    Decompose a :class:`Dimension` representing a set of data values
    arbitrarily spread over a cartesian grid.

    :param npoint: The number of sparse data values.
    :param dimension: The decomposed :class:`Dimension`.
    :param distributor: The :class:`Distributor` the SparseDistributor depends on.
    """

    def __init__(self, npoint, dimension, distributor):
        super(SparseDistributor, self).__init__(npoint, dimension)
        self._distributor = distributor

        # The dimension decomposition
        decomposition = SparseDistributor.decompose(npoint, distributor)
        offs = np.concatenate([[0], np.cumsum(decomposition)])
        self._decomposition = [Decomposition([np.arange(offs[i], offs[i+1])
                                              for i in range(self.nprocs)], self.myrank)]

    @classmethod
    def decompose(cls, npoint, distributor):
        """Distribute `npoint` points over `nprocs` MPI ranks."""
        nprocs = distributor.nprocs
        if isinstance(npoint, int):
            # `npoint` is a global count. The `npoint` are evenly distributed
            # across the various MPI ranks. Note that there is nothing smart
            # in the following -- it's entirely possible that the MPI rank 0,
            # which lives at the top-left of a 2D grid, gets some points even
            # though there physically are no points in the top-left region
            if npoint < 0:
                raise ValueError('`npoint` must be >= 0')
            glb_npoint = [npoint // nprocs]*(nprocs - 1)
            glb_npoint.append(npoint // nprocs + npoint % nprocs)
        elif isinstance(npoint, (tuple, list)):
            # The i-th entry in `npoint` tells how many sparse points the
            # i-th MPI rank has
            if len(npoint) != nprocs:
                raise ValueError('The `npoint` tuple must have as many entries as '
                                 'MPI ranks (got `%d`, need `%d`)' % (npoint, nprocs))
            elif any(i < 0 for i in npoint):
                raise ValueError('All entries in `npoint` must be >= 0')
            glb_npoint = npoint
        else:
            raise TypeError('Need `npoint` int or tuple argument')
        return tuple(glb_npoint)

    @property
    def distributor(self):
        return self._distributor

    @property
    def comm(self):
        return self.distributor._comm

    @property
    def myrank(self):
        return self.distributor.myrank

    @property
    def mycoords(self):
        return (self.myrank,)

    @property
    def nprocs(self):
        return self.distributor.nprocs

class MPICommObject(Object):

    name = 'comm'

    # See https://github.com/mpi4py/mpi4py/blob/master/demo/wrap-ctypes/helloworld.py
    if MPI._sizeof(MPI.Comm) == sizeof(c_int):
        dtype = type('MPI_Comm', (c_int,), {})
    else:
        dtype = type('MPI_Comm', (c_void_p,), {})

    def __init__(self, comm=None):
        if comm is None:
            # Should only end up here upon unpickling
            comm = MPI.COMM_WORLD
        comm_ptr = MPI._addressof(comm)
        comm_val = self.dtype.from_address(comm_ptr)
        self.value = comm_val

    # Pickling support
    _pickle_args = []

def compute_dims(nprocs, ndim):
    # We don't do anything clever here. In fact, we do something very basic --
    # we just try to distribute `nprocs` evenly over the number of dimensions,
    # and if we can't we fallback to whatever MPI.Compute_dims gives...
    v = pow(nprocs, 1/ndim)
    if not v.is_integer():
        # Since pow(64, 1/3) == 3.999..4
        v = int(ceil(v))
        if not v**ndim == nprocs:
            # Fallback
            return MPI.Compute_dims(nprocs, ndim)
    else:
        v = int(v)
    return tuple(v for _ in range(ndim))

max_stars_repo_name: LukasMosser/SNIST

max_issues_repo_name: LukasMosser/SNIST

max_forks_repo_name: LukasMosser/SNIST


text:
#!/usr/bin/python
#
# This is a quick script to show the effect of the wheel nonlinearity term on
# turning rate

from matplotlib import pylab
import numpy

if __name__ == '__main__':
  x = numpy.arange(-1, 1, 0.01)

  for nonlin in numpy.arange(0.2, 1.0, 0.1):
    angular_range = numpy.pi * nonlin / 2.0
    newx1 = numpy.sin(x * angular_range) / numpy.sin(angular_range)
    newx2 = numpy.sin(newx1 * angular_range) / numpy.sin(angular_range)

    pylab.plot(x, newx2, label="nonlin %f" % nonlin)

  pylab.legend()
  pylab.show()


text:
from flask import Flask, jsonify, request

import tensorflow as tf
import tensorflow_addons as tfa

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
from sklearn.model_selection import train_test_split

import unicodedata
import re
import numpy as np
import os
import io
import time

import pickle

# https://www.tutorialspoint.com/flask
import flask
app = Flask(__name__)

###################################################
os.chdir('C://Users//HP//Documents//applied_ai//Casestudy_2//Autocomplete//Autosuggest_deployment//')
with open('train.pickle', 'rb') as f:
    input_tensor_train, target_tensor_train,input_tensor_val, target_tensor_val,inp_lang_tokenizer, targ_lang_tokenizer,vocab_inp_size,vocab_tar_size,max_length_input,max_length_output, vocab_inp_size, vocab_tar_size = pickle.load(f)

BUFFER_SIZE = 32000
BATCH_SIZE = 64
# Let's limit the #training examples for faster training
embedding_dim = 256
units = 1024

def call(input_tensor_train, target_tensor_train,input_tensor_val, target_tensor_val,inp_lang_tokenizer, targ_lang_tokenizer, BATCH_SIZE):

        #input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)

        train_dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train))
        train_dataset = train_dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)

        val_dataset = tf.data.Dataset.from_tensor_slices((input_tensor_val, target_tensor_val))
        val_dataset = val_dataset.batch(BATCH_SIZE, drop_remainder=True)

        return train_dataset, val_dataset, inp_lang_tokenizer, targ_lang_tokenizer

train_dataset, val_dataset, inp_lang_tokenizer, targ_lang_tokenizer=call(input_tensor_train, target_tensor_train,input_tensor_val, target_tensor_val,inp_lang_tokenizer, targ_lang_tokenizer, BATCH_SIZE)

example_input_batch, example_target_batch = next(iter(train_dataset))
example_input_batch.shape, example_target_batch.shape

example_input_batch, example_target_batch = next(iter(train_dataset))
example_input_batch.shape, example_target_batch.shape

#print("max_length_spanish, max_length_english, vocab_size_spanish, vocab_size_english")
#max_length_input, max_length_output, vocab_inp_size, vocab_tar_size

"""# **Load weights and Define Models**"""

#####
class Encoder(tf.keras.Model):
  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):
    super(Encoder, self).__init__()
    self.batch_sz = batch_sz
    self.enc_units = enc_units
    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)

    ##-------- LSTM layer in Encoder ------- ##
    self.lstm_layer = tf.keras.layers.LSTM(self.enc_units,
                                   return_sequences=True,
                                   return_state=True,
                                   recurrent_initializer='glorot_uniform')

  def call(self, x):
    hidden=x[1]
    embeddin_op = self.embedding(x[0])
    output, h, c = self.lstm_layer(embeddin_op,initial_state = hidden)#
    return output, h, c

  def initialize_hidden_state(self):
    return [tf.zeros((self.batch_sz, self.enc_units)), tf.zeros((self.batch_sz, self.enc_units))]

## Test Encoder Stack

encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)

# sample input
sample_hidden = encoder.initialize_hidden_state()
sample_output, sample_h, sample_c = encoder([example_input_batch, sample_hidden])#

encoder.load_weights('10_Loss_0.0947_encoder.h5')

class Decoder(tf.keras.Model):
  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz, attention_type='luong'):
    super(Decoder, self).__init__()
    self.batch_sz = batch_sz
    self.dec_units = dec_units
    self.attention_type = attention_type

    # Embedding Layer
    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)

    #Final Dense layer on which softmax will be applied
    self.fc = tf.keras.layers.Dense(vocab_size)

    # Define the fundamental cell for decoder recurrent structure
    self.decoder_rnn_cell = tf.keras.layers.LSTMCell(self.dec_units)

    # Sampler
    self.sampler = tfa.seq2seq.sampler.TrainingSampler()

    # Create attention mechanism with memory = None
    self.attention_mechanism = self.build_attention_mechanism(self.dec_units,
                                                              None, self.batch_sz*[max_length_input], self.attention_type)

    # Wrap attention mechanism with the fundamental rnn cell of decoder
    self.rnn_cell = self.build_rnn_cell(batch_sz)

    # Define the decoder with respect to fundamental rnn cell
    self.decoder = tfa.seq2seq.BasicDecoder(self.rnn_cell, sampler=self.sampler, output_layer=self.fc)

  def build_rnn_cell(self, batch_sz):
    rnn_cell = tfa.seq2seq.AttentionWrapper(self.decoder_rnn_cell,
                                  self.attention_mechanism, attention_layer_size=self.dec_units)
    return rnn_cell

  def build_attention_mechanism(self, dec_units, memory, memory_sequence_length, attention_type='luong'):
    # ------------- #
    # typ: Which sort of attention (Bahdanau, Luong)
    # dec_units: final dimension of attention outputs
    # memory: encoder hidden states of shape (batch_size, max_length_input, enc_units)
    # memory_sequence_length: 1d array of shape (batch_size) with every element set to max_length_input (for masking purpose)

    if(attention_type=='bahdanau'):
      return tfa.seq2seq.BahdanauAttention(units=dec_units, memory=memory, memory_sequence_length=memory_sequence_length)
    else:
      return tfa.seq2seq.LuongAttention(units=dec_units, memory=memory, memory_sequence_length=memory_sequence_length)

  def build_initial_state(self, batch_sz, encoder_state, Dtype):
    decoder_initial_state = self.rnn_cell.get_initial_state(batch_size=batch_sz, dtype=Dtype)
    decoder_initial_state = decoder_initial_state.clone(cell_state=encoder_state)
    return decoder_initial_state

  def call(self, x):
    initial_state=x[1]
    embedding_op = self.embedding(x[0])
    outputs, _, _ = self.decoder(embedding_op, initial_state=initial_state, sequence_length=self.batch_sz*[max_length_output-1])
    return outputs

# Test decoder stack

decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE, 'luong')
sample_x = tf.random.uniform((BATCH_SIZE, max_length_output))
decoder.attention_mechanism.setup_memory(sample_output)
initial_state = decoder.build_initial_state(BATCH_SIZE, [sample_h, sample_c], tf.float32)
sample_decoder_outputs = decoder([sample_x, initial_state])

decoder.load_weights('10_Loss_0.0947_decoder.h5')

"""# **Evaluation**"""

## Original Funcion
def evaluate_sentence(sentence):
  sentence =sentence #dataset_creator.preprocess_sentence(sentence)

  inputs = [inp_lang_tokenizer.word_index[i] for i in sentence.split(' ')]#
  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],
                                                          maxlen=max_length_input,
                                                          padding='post')
  inputs = tf.convert_to_tensor(inputs)
  inference_batch_size = inputs.shape[0]
  result = ''

  enc_start_state = [tf.zeros((inference_batch_size, units)), tf.zeros((inference_batch_size,units))]
  enc_out, enc_h, enc_c = encoder([inputs, enc_start_state])

  dec_h = enc_h
  dec_c = enc_c

  start_tokens = tf.fill([inference_batch_size], targ_lang_tokenizer.word_index['cls'])
  end_token = targ_lang_tokenizer.word_index['sep']

  greedy_sampler = tfa.seq2seq.GreedyEmbeddingSampler()

  # Instantiate BasicDecoder object
  decoder_instance = tfa.seq2seq.BasicDecoder(cell=decoder.rnn_cell, sampler=greedy_sampler, output_layer=decoder.fc)
  # Setup Memory in decoder stack
  decoder.attention_mechanism.setup_memory(enc_out)

  # set decoder_initial_state
  decoder_initial_state = decoder.build_initial_state(inference_batch_size, [enc_h, enc_c], tf.float32)

  ### Since the BasicDecoder wraps around Decoder's rnn cell only, you have to ensure that the inputs to BasicDecoder
  ### decoding step is output of embedding layer. tfa.seq2seq.GreedyEmbeddingSampler() takes care of this.
  ### You only need to get the weights of embedding layer, which can be done by decoder.embedding.variables[0] and pass this callabble to BasicDecoder's call() function

  decoder_embedding_matrix = decoder.embedding.variables[0]

  outputs, _, _ = decoder_instance(decoder_embedding_matrix, start_tokens = start_tokens, end_token= end_token, initial_state=decoder_initial_state)
  return outputs.sample_id.numpy()

def translate(sentence):
  result = evaluate_sentence(sentence)
  result = targ_lang_tokenizer.sequences_to_texts(result)
  result=str(result).replace(" sep","").replace("]","").replace("[","").replace("'","")
  return result

def give_suggestion(input):
	print(input)
	try:
	   return translate(input)
	except Exception:
	   return

###################################################

@app.route('/')
def hello_world():
    return 'Hello World!'

@app.route('/index')
def index():
    return flask.render_template('index.html')

@app.route('/predict', methods=['POST'])
def predict():
	to_predict = request.form.to_dict()
	prediction=give_suggestion(to_predict['intial_text'])#jsonify(request.json) request.get_json(force=True)
	return jsonify({'suggestion': prediction})

if __name__ == '__main__':
	app.run(host='127.0.0.1', port=8080)


text:
"""

shallow water system simulated by vectorial solver D2Q[4,4,4]

"""
import numpy as np
import sympy as sp
import pylbm
import sys

def printProgress (iteration, total, prefix = '', suffix = '', decimals = 1, barLength = 100):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        barLength   - Optional  : character length of bar (Int)
    """
    formatStr       = "{0:." + str(decimals) + "f}"
    percents        = formatStr.format(100 * (iteration / float(total)))
    filledLength    = int(round(barLength * iteration / float(total)))
    bar             = '*' * filledLength + '-' * (barLength - filledLength)
    sys.stdout.write('\r%s |%s| %s%s %s' % (prefix, bar, percents, '%', suffix)),
    sys.stdout.flush()
    if iteration == total:
        sys.stdout.write('\n')
        sys.stdout.flush()

h5_save = True

X, Y, LA = sp.symbols('X, Y, LA')
rho, qx, qy = sp.symbols('rho, qx, qy')

def initialization_rho(x, y):
    return rhoo * np.ones((x.size, y.size)) + deltarho * ((x-0.5*(xmin+xmax))**2+(y-0.5*(ymin+ymax))**2 < 0.25**2)

def save(sol, filename, path, num):
    h5 = pylbm.H5File(sol.domain.mpi_topo, filename, path, num)
    h5.set_grid(sol.domain.x, sol.domain.y)
    h5.add_scalar('rho', sol.m[rho])
    h5.save()

# parameters
rhoo = 1.
deltarho = 1.
Taille = 2.
xmin, xmax, ymin, ymax = -0.5*Taille, 0.5*Taille, -0.5*Taille, 0.5*Taille
la = 4 # velocity of the scheme
g = 1.
sigma = 1.e-4
s_0qx = 2.#1./(0.5+sigma)
s_0xy = 1.5
s_1qx = 1.5
s_1xy = 1.2
s0  = [0., s_0qx, s_0qx, s_0xy]
s1  = [0., s_1qx, s_1qx, s_1xy]
if h5_save:
    dx = 1./512 # spatial step
else:
    dx = 1./128

vitesse = list(range(1, 5))
polynomes = [1, LA*X, LA*Y, X**2-Y**2]

dico   = {
    'box': {
        'x': [xmin, xmax],
        'y': [ymin, ymax],
        'label':-1
    },
    'space_step': dx,
    'scheme_velocity': la,
    'parameters': {LA: la},
    'schemes': [
        {
            'velocities': vitesse,
            'conserved_moments': rho,
            'polynomials': polynomes,
            'relaxation_parameters': s0,
            'equilibrium': [rho, qx, qy, 0.],
        },
        {
            'velocities': vitesse,
            'conserved_moments': qx,
            'polynomials': polynomes,
            'relaxation_parameters': s1,
            'equilibrium': [qx, qx**2/rho + 0.5*g*rho**2, qx*qy/rho, 0.],
        },
        {
            'velocities': vitesse,
            'conserved_moments': qy,
            'polynomials': polynomes,
            'relaxation_parameters': s1,
            'equilibrium': [qy, qy*qx/rho, qy**2/rho + 0.5*g*rho**2, 0.],
        },
    ],
    'init': {rho: initialization_rho,
             qx: 0.,
             qy: 0.
    },
    'generator': 'cython',
}

sol = pylbm.Simulation(dico)

x, y = sol.domain.x, sol.domain.y

if h5_save:
    Tf = 20.
    im = 0
    l = Tf / sol.dt / 32
    printProgress(im, l, prefix='Progress:', suffix='Complete', barLength=50)
    filename = 'shallow_water'
    path = './data_' + filename
    save(sol, filename, path, im)
    while sol.t < Tf:
        for k in range(32):
            sol.one_time_step()
        im += 1
        printProgress(im, l, prefix='Progress:', suffix='Complete', barLength=50)
        save(sol, filename, path, im)
else:
    viewer = pylbm.viewer.matplotlib_viewer
    fig = viewer.Fig()
    ax = fig[0]
    im = ax.image(sol.m[rho].transpose(), clim=[rhoo-.5*deltarho, rhoo+1.5*deltarho])
    ax.title = 'solution at t = {0:f}'.format(sol.t)

    def update(iframe):
        for k in range(32):
            sol.one_time_step()      # increment the solution of one time step
        im.set_data(sol.m[rho].transpose())
        ax.title = 'solution at t = {0:f}'.format(sol.t)

    # run the simulation
    fig.animate(update, interval=1)
    fig.show()

reason: import numpy,import sympy