text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
This experiment was created using PsychoPy2 Experiment Builder (v1.90.1),
    on Tue May  8 15:02:31 2018

Created 5/8/18 by DJ.
Updated 5/8/18 by DJ - added NetStationEEG code from https://github.com/imnotamember/PyNetstation (per Pete's instructions)
"""

from __future__ import absolute_import, division
from psychopy import locale_setup, sound, gui, visual, core, data, event, logging, clock
from psychopy.constants import (NOT_STARTED, STARTED, PLAYING, PAUSED,
                                STOPPED, FINISHED, PRESSED, RELEASED, FOREVER)
import numpy as np  # whole numpy lib is available, prepend 'np.'
from numpy import (sin, cos, tan, log, log10, pi, average,
                   sqrt, std, deg2rad, rad2deg, linspace, asarray)
from numpy.random import random, randint, normal, shuffle
import os  # handy system and path functions
import sys  # to get file system encoding

# Declare movie params
movieFile = u'/Users/jangrawdc/Documents/Python/PsychoPyParadigms/BasicExperiments/Movies/Paperman.mp4'
movieDur = 301.0

# === EEG === #
# === Set up
isEegConnected = True
tcpipAddress = '10.10.10.42'
tcpipPort = 55513
movieSize = (640.0*3,360.0*3) # for Boldscreen

# === Initialize
if isEegConnected == False:
    # # This will import the debugging version of the PyNetStation module,
    # #  which will not actually attempt a connection but will check to make sure
    # #  your code is properly functioning.
    import egi.fake as egi  # FOR TESTING WITHOUT CONNECTION TO NETSTATION COMPUTER
else:
    # # This will import the single-threaded version of the PyNetStation module
    import egi.simple as egi # FOR RUNNING CONNECTED TO NETSTATION COMPUTER -- USE THIS IN A REAL EXPERIMENT

# === Timing Obj
# # Create a proper timing object to reference. To retrieve the time you want later,
# #  call this method using ms_localtime(), it returns the time in a millisecond format
# #  appropriate for the NetStation TCP/IP protocol.
# # This is only necessary if you are in need of direct contact with the clock object that NetStation is utilizing,
# #  which you don't actually need since it's working behind the scenes in the egi module.
# ms_localtime = egi.ms_localtime

# === Netstation Obj
# # Create the NetStation event-sending object. After this you can call
# #  the methods via the object instance, in this case 'ns'.
ns = egi.Netstation()

# === Establish Cxn
# # The next line is for connecting the actual, single-threaded module version to the computer.
if isEegConnected:
    ns.connect(tcpipAddress, tcpipPort)  # sample address and port -- change according to your network settings

# === Link Expt to Session
# # This sends some initialization info to NetStation for recording events.
ns.BeginSession()
# # This synchronizes the clocks of the stim computer and the NetStation computer.
ns.sync()

# === END EEG === #

# Ensure that relative paths start from the same directory as this script
_thisDir = os.path.dirname(os.path.abspath(__file__)).decode(sys.getfilesystemencoding())
os.chdir(_thisDir)

# Store info about the experiment session
expName = u'MovieTask_Builder_d2'  # Name of experiment
expInfo = {'session': '001', 'participant': ''}
dlg = gui.DlgFromDict(dictionary=expInfo, title=expName)
if dlg.OK == False:
    core.quit()  # user pressed cancel
expInfo['date'] = data.getDateStr()  # add a simple timestamp
expInfo['expName'] = expName

# Data file name stem = absolute path + name; later add .psyexp, .csv, .log, etc
filename = _thisDir + os.sep + u'data/%s_%s_%s' % (expInfo['participant'], expName, expInfo['date'])

# An ExperimentHandler isn't essential but helps with data saving
thisExp = data.ExperimentHandler(name=expName, version='',
    extraInfo=expInfo, runtimeInfo=None,
    originPath=u'/Users/jangrawdc/Documents/Python/PsychoPyParadigms/BasicExperiments/TEST.psyexp',
    savePickle=True, saveWideText=True,
    dataFileName=filename)
# save a log file for detail verbose info
logFile = logging.LogFile(filename+'.log', level=logging.EXP)
logging.console.setLevel(logging.WARNING)  # this outputs to the screen, not a file

endExpNow = False  # flag for 'escape' or other condition => quit the exp

# Start Code - component code to be run before the window creation

# Setup the Window
win = visual.Window(
    size=(1024, 768), fullscr=True, screen=0,
    allowGUI=False, allowStencil=False,
    monitor='testMonitor', color=[0,0,0], colorSpace='rgb',
    blendMode='avg', useFBO=True)
# store frame rate of monitor if we can measure it
expInfo['frameRate'] = win.getActualFrameRate()
if expInfo['frameRate'] != None:
    frameDur = 1.0 / round(expInfo['frameRate'])
else:
    frameDur = 1.0 / 60.0  # could not measure, so guess

# Initialize components for Routine "Trigger"
TriggerClock = core.Clock()
TriggerText = visual.TextStim(win=win, name='TriggerText',
    text=u'Waiting for trigger...\n\n(Experimenter: press t to override.)',
    font=u'Arial',
    pos=(0, 0), height=0.1, wrapWidth=None, ori=0,
    color=u'white', colorSpace='rgb', opacity=1,
    depth=0.0);

# Initialize components for Routine "Movie"
MovieClock = core.Clock()
movie = visual.MovieStim3(
    win=win, name='movie',
    noAudio = False,
    filename=movieFile,
    ori=0, pos=(0, 0), size=movieSize, opacity=1,
    depth=0.0,
    )

# Initialize components for Routine "CoolDown"
CoolDownClock = core.Clock()
CoolDownText = visual.TextStim(win=win, name='CoolDownText',
    text=u'Please stay still until\nthe scanner noise stops.',
    font=u'Arial',
    pos=(0, 0), height=0.1, wrapWidth=None, ori=0,
    color=u'white', colorSpace='rgb', opacity=1,
    depth=0.0);

# Create some handy timers
globalClock = core.Clock()  # to track the time since experiment started
routineTimer = core.CountdownTimer()  # to track time remaining of each (non-slip) routine

# === EEG === #
# # This starts the recording in NetStation acquisition. Equivalent to pressing the Record button.
# # If at some point you pause the experiment using the "StopRecording()" method,
# #  just call this method again to restart the recording.
ns.StartRecording()

# === END EEG === #

# ------Prepare to start Routine "Trigger"-------
t = 0
TriggerClock.reset()  # clock
frameN = -1
continueRoutine = True
# update component parameters for each repeat
TriggerKey = event.BuilderKeyResponse()
# keep track of which components have finished
TriggerComponents = [TriggerText, TriggerKey]
for thisComponent in TriggerComponents:
    if hasattr(thisComponent, 'status'):
        thisComponent.status = NOT_STARTED

# === EEG === #
# # This re-aligns the clocks between the stim computer and the NetStation computer.
# # Best to put at the start of each trial for maximal timing accuracy.
ns.sync()
# Send Message to EEG
win.callOnFlip(ns.send_event, key='WAIT', timestamp=None, label="WaitForTrigger", description="Waiting for Trigger from fMRI", pad=False)

# === END EEG === #

# -------Start Routine "Trigger"-------
while continueRoutine:
    # get current time
    t = TriggerClock.getTime()
    frameN = frameN + 1  # number of completed frames (so 0 is the first frame)
    # update/draw components on each frame

    # *TriggerText* updates
    if t >= 0.0 and TriggerText.status == NOT_STARTED:
        # keep track of start time/frame for later
        TriggerText.tStart = t
        TriggerText.frameNStart = frameN  # exact frame index
        TriggerText.setAutoDraw(True)

    # *TriggerKey* updates
    if t >= 0.0 and TriggerKey.status == NOT_STARTED:
        # keep track of start time/frame for later
        TriggerKey.tStart = t
        TriggerKey.frameNStart = frameN  # exact frame index
        TriggerKey.status = STARTED
        # keyboard checking is just starting
        win.callOnFlip(TriggerKey.clock.reset)  # t=0 on next screen flip
        event.clearEvents(eventType='keyboard')
    if TriggerKey.status == STARTED:
        theseKeys = event.getKeys(keyList=['t'])

        # check for quit:
        if "escape" in theseKeys:
            endExpNow = True
        if len(theseKeys) > 0:  # at least one key was pressed
            TriggerKey.keys = theseKeys[-1]  # just the last key pressed
            TriggerKey.rt = TriggerKey.clock.getTime()
            # a response ends the routine
            continueRoutine = False

    # check if all components have finished
    if not continueRoutine:  # a component has requested a forced-end of Routine
        break
    continueRoutine = False  # will revert to True if at least one component still running
    for thisComponent in TriggerComponents:
        if hasattr(thisComponent, "status") and thisComponent.status != FINISHED:
            continueRoutine = True
            break  # at least one component has not yet finished

    # check for quit (the Esc key)
    if endExpNow or event.getKeys(keyList=["escape"]):
        core.quit()

    # refresh the screen
    if continueRoutine:  # don't flip if this routine is over or we'll get a blank screen
        win.flip()

# -------Ending Routine "Trigger"-------
for thisComponent in TriggerComponents:
    if hasattr(thisComponent, "setAutoDraw"):
        thisComponent.setAutoDraw(False)
# check responses
if TriggerKey.keys in ['', [], None]:  # No response was made
    TriggerKey.keys=None
thisExp.addData('TriggerKey.keys',TriggerKey.keys)
if TriggerKey.keys != None:  # we had a response
    thisExp.addData('TriggerKey.rt', TriggerKey.rt)
thisExp.nextEntry()
# the Routine "Trigger" was not non-slip safe, so reset the non-slip timer
routineTimer.reset()

# ------Prepare to start Routine "Movie"-------
t = 0
MovieClock.reset()  # clock
frameN = -1
continueRoutine = True
routineTimer.add(movieDur)
# update component parameters for each repeat
# keep track of which components have finished
MovieComponents = [movie]
for thisComponent in MovieComponents:
    if hasattr(thisComponent, 'status'):
        thisComponent.status = NOT_STARTED

# === EEG === #
# # This re-aligns the clocks between the stim computer and the NetStation computer.
# # Best to put at the start of each trial for maximal timing accuracy.
ns.sync()
# Send Message to EEG
win.callOnFlip(ns.send_event, key='MOVI', timestamp=None, label="StartMovie", description="Started Movie", pad=False)

# === END EEG === #

# -------Start Routine "Movie"-------
while continueRoutine and routineTimer.getTime() > 0:
    # get current time
    t = MovieClock.getTime()
    frameN = frameN + 1  # number of completed frames (so 0 is the first frame)
    # update/draw components on each frame

    # *movie* updates
    if t >= 0.0 and movie.status == NOT_STARTED:
        # keep track of start time/frame for later
        movie.tStart = t
        movie.frameNStart = frameN  # exact frame index
        movie.setAutoDraw(True)
    frameRemains = 0.0 + movieDur- win.monitorFramePeriod * 0.75  # most of one frame period left
    if movie.status == STARTED and t >= frameRemains:
        movie.setAutoDraw(False)

    # check if all components have finished
    if not continueRoutine:  # a component has requested a forced-end of Routine
        break
    continueRoutine = False  # will revert to True if at least one component still running
    for thisComponent in MovieComponents:
        if hasattr(thisComponent, "status") and thisComponent.status != FINISHED:
            continueRoutine = True
            break  # at least one component has not yet finished

    # check for quit (the Esc key)
    if endExpNow or event.getKeys(keyList=["escape"]):
        core.quit()

    # refresh the screen
    if continueRoutine:  # don't flip if this routine is over or we'll get a blank screen
        win.flip()

# -------Ending Routine "Movie"-------
for thisComponent in MovieComponents:
    if hasattr(thisComponent, "setAutoDraw"):
        thisComponent.setAutoDraw(False)

# ------Prepare to start Routine "CoolDown"-------
t = 0
CoolDownClock.reset()  # clock
frameN = -1
continueRoutine = True
# update component parameters for each repeat
EndKey = event.BuilderKeyResponse()
# keep track of which components have finished
CoolDownComponents = [CoolDownText, EndKey]
for thisComponent in CoolDownComponents:
    if hasattr(thisComponent, 'status'):
        thisComponent.status = NOT_STARTED

# === EEG === #
# # This re-aligns the clocks between the stim computer and the NetStation computer.
# # Best to put at the start of each trial for maximal timing accuracy.
ns.sync()
# Send Message to EEG
win.callOnFlip(ns.send_event, key='COOL', timestamp=None, label="CoolDown", description="Waiting for End of Scan", pad=False)

# === END EEG === #

# -------Start Routine "CoolDown"-------
while continueRoutine:
    # get current time
    t = CoolDownClock.getTime()
    frameN = frameN + 1  # number of completed frames (so 0 is the first frame)
    # update/draw components on each frame

    # *CoolDownText* updates
    if t >= 0.0 and CoolDownText.status == NOT_STARTED:
        # keep track of start time/frame for later
        CoolDownText.tStart = t
        CoolDownText.frameNStart = frameN  # exact frame index
        CoolDownText.setAutoDraw(True)

    # *EndKey* updates
    if t >= 0.0 and EndKey.status == NOT_STARTED:
        # keep track of start time/frame for later
        EndKey.tStart = t
        EndKey.frameNStart = frameN  # exact frame index
        EndKey.status = STARTED
        # keyboard checking is just starting
        win.callOnFlip(EndKey.clock.reset)  # t=0 on next screen flip
        event.clearEvents(eventType='keyboard')
    if EndKey.status == STARTED:
        theseKeys = event.getKeys(keyList=['q', 'escape'])

        # check for quit:
        if "escape" in theseKeys:
            endExpNow = True
        if len(theseKeys) > 0:  # at least one key was pressed
            EndKey.keys = theseKeys[-1]  # just the last key pressed
            EndKey.rt = EndKey.clock.getTime()
            # a response ends the routine
            continueRoutine = False

    # check if all components have finished
    if not continueRoutine:  # a component has requested a forced-end of Routine
        break
    continueRoutine = False  # will revert to True if at least one component still running
    for thisComponent in CoolDownComponents:
        if hasattr(thisComponent, "status") and thisComponent.status != FINISHED:
            continueRoutine = True
            break  # at least one component has not yet finished

    # check for quit (the Esc key)
    if endExpNow or event.getKeys(keyList=["escape"]):
        core.quit()

    # refresh the screen
    if continueRoutine:  # don't flip if this routine is over or we'll get a blank screen
        win.flip()

# -------Ending Routine "CoolDown"-------
for thisComponent in CoolDownComponents:
    if hasattr(thisComponent, "setAutoDraw"):
        thisComponent.setAutoDraw(False)
# check responses
if EndKey.keys in ['', [], None]:  # No response was made
    EndKey.keys=None
thisExp.addData('EndKey.keys',EndKey.keys)
if EndKey.keys != None:  # we had a response
    thisExp.addData('EndKey.rt', EndKey.rt)
thisExp.nextEntry()
# the Routine "CoolDown" was not non-slip safe, so reset the non-slip timer
routineTimer.reset()

# === EEG === #
# # This re-aligns the clocks between the stim computer and the NetStation computer.
# # Best to put at the start of each trial for maximal timing accuracy.
ns.sync()
# Send Message to EEG
win.callOnFlip(ns.send_event, key='DONE', timestamp=None, label="ExperimentDone", description="End of Scan", pad=False)

# === End Session
# # This method is misleading, as it merely pauses the recording in NetStation. Equivalent to the pause button.
# # It is not actually stopping the recording session. That is done by the 'EndSession()' method.
ns.StopRecording()

# # I don't typically use this, as it is closes the current "Session" in NetStation.
# # I find it easier to just pause the recording using "StopRecording()" and then
# # get ending impedance measurements before manually closing NetStation.
ns.EndSession()

# # This line ends the connection via the ns object, and should then be destroying the object itself.
# # It is good practice to use so as not to waste memory or leave TCP/IP links open, which could lead to being
# # unable to reconnect without restarting the computer running the experiment.
if isEegConnected:
    ns.disconnect()

# === END EEG === #

# ----------Finishing Experiment----------
# these shouldn't be strictly necessary (should auto-save)
thisExp.saveAsWideText(filename+'.csv')
thisExp.saveAsPickle(filename)
logging.flush()
# make sure everything is closed down
thisExp.abort()  # or data files will save again on exit
win.close()
core.quit()

max_stars_repo_name: djangraw/PsychoPyParadigms

max_issues_repo_name: caofeizhen/PsychoPyParadigms

max_forks_repo_name: caofeizhen/PsychoPyParadigms

reason: import numpy,from numpy


text:
import cv2
import os
import sys
import unittest
import numpy as np

if __name__ == '__main__':
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

import leapvision.scanner

class HumanScannerTest(unittest.TestCase):
    def setUp(self):
        self.scanner = leapvision.scanner.HumanScanner()

    def test_initial_state(self):
        self.assertEqual(self.scanner.scoreThreshold, 0.6)
        self.assertEqual(self.scanner.winStride, (8, 8))
        self.assertEqual(self.scanner.padding, (8, 8))
        self.assertEqual(self.scanner.scale, 1.05)
        self.assertIsNotNone(self.scanner.hog)

    def test_single_person(self):
        test_image_path = os.path.join(
            os.path.dirname(__file__), 'data', 'person_view.jpg')
        test_image = cv2.imread(test_image_path)
        result = self.scanner.scan(test_image)
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['label'], 'person')
        self.assertAlmostEqual(result[0]['score'], 0.89, delta=1)
        self.assertEqual(result[0]['box'], (2138, 437, 1319, 2638))

    def test_demo(self):
        enabled = bool(os.environ.get('SCANNER_PERSON_DEMO_ENABLED'))
        if not enabled:
            return
        self.scanner = leapvision.scanner.HumanScanner(scoreThreshold=0.75)
        video_path = os.path.join(
            os.path.dirname(__file__),
            'data',
            'people_walking.mp4'
        )
        key = 0
        while key & 0xff != ord('q'):
            camera = cv2.VideoCapture(video_path)
            cv2.namedWindow('camera')
            cv2.moveWindow('camera', 0, 0)
            while camera.isOpened():
                ok, frame = camera.read()
                if not ok:
                    break
                tick_count = cv2.getTickCount()
                result = self.scanner.scan(frame)
                fps = cv2.getTickFrequency() / (cv2.getTickCount() - tick_count)
                cv2.putText(
                    frame,
                    "fps: {}".format(int(fps)),
                    (12, 24),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    0.75,
                    (0, 255, 0),
                    2
                )
                for entry in result:
                    box = entry['box']
                    pt1 = (box[0], box[1])
                    pt2 = (pt1[0]+box[2], pt1[1]+box[3])
                    cv2.rectangle(frame, pt1, pt2, (0, 255, 0), 2)
                    text = "Score : {}".format(entry['score'])
                    cv2.putText(frame, text, pt1,
                                cv2.FONT_HERSHEY_SIMPLEX, 0.75, (50, 170, 50), 2)
                fps = cv2.getTickFrequency() / (cv2.getTickCount() - tick_count)
                cv2.putText(
                    frame,
                    "fps: {}".format(int(fps)),
                    (12, 24),
                    cv2.FONT_HERSHEY_SIMPLEX,
                    0.75,
                    (0, 255, 0),
                    2
                )
                cv2.imshow('camera', frame)
                key = cv2.waitKey(1)

class FaceScannerTest(unittest.TestCase):
    def setUp(self):
        self.scanner = leapvision.scanner.FaceScanner()

    def test_first_frame(self):
        test_video_path = os.path.join(
            os.path.dirname(__file__),
            'data',
            'faces.mp4'
        )
        camera = cv2.VideoCapture(test_video_path)
        (ok, frame) = camera.read()
        self.assertTrue(ok)
        faces = self.scanner.scan(frame)
        self.assertEqual(len(faces), 7)
        self.assertEqual(faces[0]['box'], (157, 357, 44, 44))

    def test_demo(self):
        enabled = bool(os.environ.get('SCANNER_FACE_DEMO_ENABLED'))
        if not enabled:
            return
        cv2.namedWindow('camera')
        cv2.moveWindow('camera', 0, 0)
        test_video_path = os.path.join(
            os.path.dirname(__file__),
            'data',
            'faces.mp4'
        )
        camera = cv2.VideoCapture(test_video_path)
        while camera.isOpened():
            (ok, frame) = camera.read()
            if not ok:
                break
            tick_count = cv2.getTickCount()
            faces = self.scanner.scan(frame)
            foreground_mask = np.zeros(frame.shape[:2], np.uint8)
            for entry in faces:
                box = entry['box']
                pt1 = (box[0], box[1])
                pt2 = (pt1[0]+box[2], pt1[1]+box[3])
                cv2.rectangle(foreground_mask, pt1, pt2, (255), cv2.FILLED)

            # TODO: Optimize all this - it's likely making things slow.
            pixelated = cv2.resize(frame, (0,0), fx=0.05, fy=0.05, interpolation=cv2.INTER_NEAREST)
            pixelated = cv2.resize(pixelated, (0,0), fx=20.0, fy=20.0, interpolation=cv2.INTER_NEAREST)
            foreground = cv2.bitwise_and(pixelated, pixelated, mask=foreground_mask)
            background_mask = cv2.bitwise_not(foreground_mask)
            background = cv2.bitwise_and(frame, frame, mask=background_mask)
            masked_frame = cv2.add(background, foreground)

            fps = cv2.getTickFrequency() / (cv2.getTickCount() - tick_count)
            cv2.putText(
                masked_frame,
                "fps: {}".format(int(fps)),
                (12, 24),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.75,
                (0, 255, 0),
                2
            )
            cv2.imshow('camera', masked_frame)
            key = cv2.waitKey(1)

if __name__ == '__main__':
    unittest.main()

ext: py

reason: import numpy


text:
# -*- coding: UTF-8 -*-

"""
Python version: 2.7.6
Version: 1.0 at Sep 01 2014
Author: Chenglong Chen < yr@Kaggle >
Email: c.chenglong@gmail.com
"""

import argparse
import numpy as np
import pandas as pd
import pickle as pkl
from csv import DictReader
from datetime import datetime

def getRelevantIDs(tsvFile, verbose=True):
    """ Get all relevant IDs
    You only need to run this once
    """

    start = datetime.now()
    relevantIDs = []
    with open(tsvFile, "rb") as tsvReader:
        itemReader = DictReader(tsvReader, delimiter='\t', quotechar='"')
        for i, item in enumerate(itemReader):
            item = {featureName:featureValue.decode('utf-8') \
                    for featureName,featureValue in item.items() \
                    if featureValue is not None}

            if item["is_blocked"] == "1":
                relevantIDs.append( int(item["itemid"]) )

            if (i+1)%10000 == 0 and verbose:
                print(( "%s\t%s"%((i+1),str(datetime.now() - start)) ))

    return relevantIDs

def computeAPatK ( dfSub ):
    """Calculates AP@k given a file with IDs
    sorted in order of relevance
    """

    # load all relevant IDs stored in pkl format
    dataPath = "../Data/"
    try:
        with open(dataPath + "relevantIDs.pkl", "rb") as f:
            relevantIDs = pkl.load(f)
    except:
        tsvFileTrain = dataPath + "avito_train.tsv"
        relevantIDs = getRelevantIDs(tsvFileTrain, verbose=False)
        with open(dataPath + "relevantIDs.pkl", "wb") as f:
            pkl.dump(relevantIDs, f, -1)

    # read in submission file
    if type(dfSub) == str:
        dfSub = pd.read_csv(dfSub)
    numSample = dfSub.shape[0]
    # get the value of K
    ratio = 32500/(1351243*0.5)
    K = int(np.floor(numSample*ratio))

    # compute AP@k
    relevant = np.asarray(dfSub[:K].isin(relevantIDs), dtype=float)
    APatK_ = np.mean( np.cumsum(relevant)/np.arange(1,K+1) )

    return APatK_

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="compute AP@k")
    parser.add_argument("input")
    args = parser.parse_args()
    #print "--------------------"
    APatK_ = computeAPatK( args.input )
    print("%s" % np.round(APatK_, 8))
    #print "--------------------"

max_stars_repo_name: Keesiu/meta-kaggle

max_issues_repo_name: Keesiu/meta-kaggle

max_forks_repo_name: Keesiu/meta-kaggle


text:
"""
    Open Orchestrator Cloud Radio Access Network

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.

"""

from math import radians, cos, sin, asin, sqrt
import numpy as np
import random
import ast
import yaml
from vnfs.models import Vnf
from scenarios.models import RRH

keys = ["name", "ip", "vnf", "bw_dl", "bw_ul", "pt", "type"]

def jsontoheat(code):
    elements = {}
    connection = {}

    operators = code['operators']
    for key, value in operators.items():
        elements[key] = value['properties']['title']

    links = code['links']
    for key, value in links.items():
        connection[value['fromOperator']] = value['toOperator']

    return elements, connection

def check_content(parameters):
    if not set(keys) <= set(parameters):
        return False

def check_parameters(parameters):
    for k, v in parameters.items():
        if v is None:
            return False
    return {k: parameters[k] for k in set(keys) & set(parameters.keys())}

def check_vnf(data, operator):
    try:
        vnf = Vnf.objects.get(operator=operator, name=data)
    except:
        vnf = False
    return vnf

def read_yaml(file, operator):
    doc = yaml.load(file)

    list = []
    try:
        for rrh, parameters in doc.items():
            data = check_parameters(parameters)
            vnf = check_vnf(data['vnf'], operator)
            if check_content(parameters) is not False and data is not False and vnf is not False:
                data['name'] = data['name'] + '-' + data['ip']
                data['vnf'] = vnf
                data['operator'] = operator
                data['rrh'] = RRH.objects.get(ip=data['ip'])
                data.pop('ip')
                list.append(data)
            else:
                return False
        return list
    except:
        return True

def distance(lon1, lat1, lon2, lat2):
    """
    Calculate distance between terminal and bts
    """
    # convert decimal degrees to radians
    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
    # haversine formula
    dlon = lon2 - lon1
    dlat = lat2 - lat1
    a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
    c = 2 * asin(sqrt(a))
    km = 6367 * c
    return km * 1000

def price(nvf, spectre):
    price_spec = 0
    if spectre == 1400000:
        price_spec = 1.74
    if spectre == 3000000:
        price_spec = 200.0
    if spectre == 5000000:
        price_spec = 300.0
    if spectre == 10000000:
        price_spec = 400.0

    # t = int(deploy.stop.strftime("%H")) - int(deploy.start.strftime("%H"))
    price_comp = (price_spec + nvf.vnf.ram * 0.005 + nvf.vnf.cpu * 0.005)
    price = price_spec + price_comp

    return float(price)

def rb_offer(rb, total, start, stop, operation):
    value = total[1:-1].split(',')

    for i in range(stop - start + 1):
        if operation == "suma":
            value[start + i] = str(int(value[start + i]) + int(rb))
        if operation == "resta":
            value[start + i] = str(int(value[start + i]) - int(rb))

    valores = ','.join(value)
    return "[" + valores + "]"

def mcs(value, operator):
    mcs = operator.mcs
    i = 0
    for row in mcs:
        row = row.split('\n')[0]
        if row.split(',')[0] == value:
            return row.split(',')[1]
        i += 1

def optim(file):
    """
    Parser user characteristics
    """
    lista = []
    users = np.genfromtxt(file, dtype='str')

    for user in users:
        vm = {}
        vm['name'] = user.split(',')[0]
        vm['lat'] = user.split(',')[1]
        vm['long'] = user.split(',')[2]
        vm['bts'] = user.split(',')[3]
        vm['vnf'] = user.split(',')[4]
        vm['rb'] = user.split(',')[5]
        vm['mcs'] = user.split(',')[6]
        lista.append(vm)

    return lista

def rand_color():
    """
    Random color generator
    """
    r = lambda: random.randint(0, 255)
    color = ('#%02X%02X%02X' % (r(), r(), r()))
    return color

def planification_DL(nvf, colors_op, frequencies):
    start = nvf.rrh.start()
    colors = ast.literal_eval(colors_op.colors)

    if nvf.bw_dl == 1400000:
        nvf.rb = 18000000
    if nvf.bw_dl == 3000000:
        nvf.rb = 36000000
    if nvf.bw_dl == 5000000:
        nvf.rb = 72000000
    if nvf.bw_dl == 10000000:
        nvf.rb = 150000000

    assigned = [str(x) for x in nvf.rrh.freCs.split('/')]

    while True:
        if not str(start) + '-' + str(int(start) + int(nvf.bw_dl)) in frequencies:
            if not str(start) + '-' + str(int(start) + int(nvf.bw_dl)) in assigned:
                nvf.rrh.freCs = str(nvf.rrh.freCs) + '/' + str(start) + '-' + str(int(start) + int(nvf.bw_dl))
                nvf.freC_DL = int(start) + int(nvf.bw_dl) / 2
                break
            else:
                start = int(start) + int(nvf.bw_dl)
        else:
            start = int(start) + int(nvf.bw_dl)

    if colors.has_key(nvf.freC_DL):
        nvf.color_DL = colors[nvf.freC_DL]
    else:
        nvf.color_DL = rand_color()
        colors[nvf.freC_DL] = nvf.color_DL

    nvf.save()
    nvf.rrh.save()
    return nvf.rrh.freCs

def planification_UL(nvf, frequencies):
    start = int(nvf.rrh.start()) + 20000000

    nvf.color_UL = rand_color()

    assigned = [str(x) for x in nvf.rrh.freCs.split('/')]

    while (True):
        if not str(start) + '-' + str(int(start) + int(nvf.bw_ul)) in frequencies:
            if not str(start) + '-' + str(int(start) + int(nvf.bw_ul)) in assigned:
                nvf.rrh.freCs = str(nvf.rrh.freCs) + '/' + str(start) + '-' + str(int(start) + int(nvf.bw_ul))
                nvf.freC_UL = int(start) + int(nvf.bw_ul) / 2
                break
            else:
                start = int(start) + int(nvf.bw_ul)
        else:
            start = int(start) + int(nvf.bw_ul)
    nvf.save()
    nvf.rrh.save()
    return nvf.rrh.freCs


text:
# coding:utf-8

import sys

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
import jieba
from jieba import analyse
import pymysql
import numpy as np
from sklearn import metrics
from sklearn.svm import SVC

conn = pymysql.connect(host="127.0.0.1", user="root", passwd="123456", db="hot", charset="utf8")

def get_segment(all=False):
    cursor = conn.cursor()

    if True == all:
        # 找到全部文章的标题和内容
        sql = "select id, word, content from hotword"
    else:
        # 找到尚未切词的文章的标题和内容
        sql = "select id,word, content from hotword where segment is null"
    cursor.execute(sql)

    for row in cursor.fetchall():

        print("cutting %s" % row[1])
        # 对标题和内容切词
        seg_list = jieba.cut(row[1] + ' ' + row[2])
        line = ""
        for str in seg_list:
            line = line + " " + str
        line = line.replace('\'', ' ')

        # 把切词按空格分隔并去特殊字符后重新写到数据库的segment字段里
        sql = "update hotword set segment='%s' where id=%d" % (line, row[0])
        try:
            cursor.execute(sql)
            conn.commit()
        except Exception as e:
            print(e)
            sys.exit(-1)

def classify():
    cursor = conn.cursor()
    # 一共分成5类，并且类别的标识定为0，1，3，4，5
    category_ids = range(0, 5)
    category = {}
    category[0] = 'isTec'
    category[1] = 'isSoup'
    category[2] = 'isMR'
    category[3] = 'isMath'
    category[4] = 'isNews'

    corpus = []
    for category_id in category_ids:
        # 找到这一类的所有已分类的文章并把所有切词拼成一行，加到语料库中
        sql = "select segment from hotword where " + category[category_id] + "=1"
        cursor.execute(sql)
        line = ""
        for result in cursor.fetchall():
            segment = result[0]
            line = line + " " + segment
        corpus.append(line)
    print("success")
    # 把所有未分类的文章追加到语料库末尾行
    sql = "select id, word, segment from hotword where isTec=0 and isSoup=0 and isMR=0 and isMath=0 and isNews=0"
    cursor.execute(sql)
    line = ""
    update_ids = []
    update_titles = []
    need_predict = False
    for result in cursor.fetchall():
        id = result[0]
        print(id)
        update_ids.append(id)
        title = result[1]
        update_titles.append(title)
        segment = result[2]
        corpus.append(segment)
        need_predict = True
    print("12success")
    if False == need_predict:
        print("dfsf")
        return

    # 计算tf-idf
    vectorizer = CountVectorizer()
    csr_mat = vectorizer.fit_transform(corpus)
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(csr_mat)
    y = np.array(category_ids)
    print("5success")
    # 用前5行已标分类的数据做模型训练
    model = SVC()
    model.fit(tfidf[0:5], y)

    # 对5行以后未标注分类的数据做分类预测
    predicted = model.predict(tfidf[5:])

    # 把机器学习得出的分类信息写到数据库
    for index in range(0, len(update_ids)):
        update_id = update_ids[index]
        predicted_category = category[predicted[index]]

        print("predict title: %s <==============> category: %s" % (update_titles[index], predicted_category))
        sql = "update hotword set %s=1 where id=%d" % (predicted_category, update_id)
        cursor.execute(sql)

    conn.commit()

if __name__ == '__main__':
    get_segment()
    # 分类
    classify()
    conn.close()


text:
################################################################################
#             CLASS FOR RVE VISUALIZATION USING PYVISTA AND VTK                #
################################################################################
# This file provides the class definition for the mesh visualization of a Gmsh
# model. It is based on the pyvista and vtk libraries und implements additional
# features for the mesh visualization.

############################
# Load required libraries: #
############################
# Standard Python libraries
import numpy as np                                                              # numpy for fast array computations
import copy as cp                                                               # copy for deepcopies
import tempfile as tf                                                           # tempfile for temporary files and folders
import logging                                                                  # logging for warning messages
logger=logging.getLogger(__name__)                                              # -> set logger

# additional program libraries
import pyvista as pv                                                            # -> load pyVista visualization module
import vtk                                                                      # -> load vtk since pyvista depends on it, i.e. it must be available

##################################
# Define MeshVisualization class #
##################################
class MeshVisualization():
    """Class for Gmsh model mesh visualization

    This class visualizes the mesh of a given GmshModel using the pyvista and
    vtk libraries. It provides a simple method to visualize the model geometry
    by storing the mesh into a temporary ".vtu" file which is read by pyvista.
    With additional slider widgets for a threshold algorithm based on physical
    groups of the Gmsh model and a box widget to facilitate an extraction of
    areas within the mesh, the visualization tool allows to get an impression
    of the generated mesh.

    Attributes:
    -----------
    model: GmshModel object instance
        model instance for which the geometry has to be displayed

    defaultSettings: dict
        dictionary holding the default widget visualization options to allow
        restoring those options

    currentSettings: dict
        dictionary holding the current widget visualization options

    plotterObj: pyvista plotter object instance
        current plotter/renderer object

    mesh: mesh object instance
        object holding the current mesh information

    thresholdAlgorithm: vtk Threshold Algorithm instance
        object instance with information for the applied vtkThreshold algorithm

    thresholdAlgorithm: vtk Extraction Algorithm instance
        object instance with information for the applied vtkExtractGeometry
        algorithm

    extractionBox: vtkBox object instance
        object instance storing information of the extraction box

    extractionBoxBounds: pyvista PolyData object instance
        object instance storing information about the extraction box boundaries

    active widgets: list
        list of active widgets within the rendering object
    """
    #########################
    # Initialization method #
    #########################
    def __init__(self,model=None):
        """Initilaization method for the MeshVisualization class

        This method initializes a new instance of the MeshVisualization
        class and visualizes the mesh of the passed GmshModel instance.
        Within the method, basic visualization settings are defined, before the
        visualization method is called.

        Parameters:
        model: GmshModel object instance
            model for which the mesh has to be visualized
        """
        # plausibility checks for input arguments
        if model is None:
            raise TypeError("No model instance to visualize passed. For the visualization of a model, that model has to be known. Check your input data.")
        self.model=model

        # set plotting theme
        pv.set_plot_theme("paraview")                                           # use Paraview style for plotting

        # get necessary model information for the default configuration
        physicalIDs=model.getIDsFromTags(model.gmshAPI.getPhysicalGroups(dim=model.dimension))  # get group IDs for physical groups of the highest dimension
        modelBBox=model._getGmshModelBoundingBox()                              # get the overall bounding box of the Gmsh model

        # define settings
        self.defaultSettings={
                "lowerThreshold": min(physicalIDs),                             # set lower threshold value to minimum value of active scalar field
                "upperThreshold": max(physicalIDs),                             # set upper threshold value to minimum value of active scalar field
                "boxBounds": modelBBox.T.reshape(6)                             # define bounds of extraction box to match bounds of the model (factor 1.25 will be applied automatically)
        }
        self.currentSettings=cp.deepcopy(self.defaultSettings)                  # copy default settings as initial version of current settings

        # initialize arrays required for proper mesh visualization
        self.plotterObj=pv.Plotter(title="GmshModel Mesh Visualization")        # initialize plotterObj from pyvista and set title
        self.mesh=None                                                          # initialize mesh
        self.thresholdAlgorithm=None                                            # initialize threshold algorithm
        self.extractionBox=vtk.vtkBox()                                         # initiliaze dummy extraction box to update information of boxWidget
        self.extractionBoxBounds=pv.PolyData()                                  # initialize dummy extraction box bounds to store information of boxWidget bounds
        self.extractionAlgorithm=None                                           # initialize extraction algorithm
        self.activeWidgets=[]                                                   # initialize list of active widgets

        # visualize the model mesh
        self.visualizeMesh()

################################################################################
#                 MAIN METHODS FOR GEOMETRY VISUALIZATION                      #
################################################################################

    ################################
    # Method to visualize the mesh #
    ################################
    def visualizeMesh(self):
        """Main method for the mesh visualization

        Within this method, the mesh data of the model is stored into a temporary
        ".vtu"-file which is read pyvista. After setting up widgets and additional
        key-press-events, the mesh is visualized.
        """

        with tf.TemporaryDirectory() as tmpDir:                                 # create temporary directory
            tmpFile=tmpDir+"/"+self.model.modelName+".vtu"                      # set name of file in temorary directory
            self.model.saveMesh(tmpFile)                                        # create temporary file

            self.mesh=pv.UnstructuredGrid(tmpFile)                              # load mesh from temporary file with pyvista
            self.mesh.set_active_scalars("gmsh:physical")                       # set gmsh:physical to be the active scalar
            self.scalars = self.mesh.active_scalars_info                        # get field ID and name of active scalar field

            # add widgets and key events
            self.addSliderWidgets()                                             # add slider widgets for threshold filter
            self.addBoxWidget()                                                 # add box widget for extraction filter
            self.addKeyPressEvents()                                            # add defined key press events

            # add mesh to plotterObj
            self.plotterObj.add_mesh(self.mesh,scalars=self.scalars[1],name="mesh",reset_camera=False,clim=[self.currentSettings["lowerThreshold"],self.currentSettings["upperThreshold"]],show_edges=True)

            # update settings of the rendering scene and show plot
            self.plotterObj.remove_scalar_bar()                                 # remove scalar bar (since no results will be shown but just the mesh)
            self.plotterObj.show_axes()                                         # show axes orientation
            self.plotterObj.show()                                              # show resulting plot

    #######################################
    # Method to apply filters to the mesh #
    #######################################
    def filterPipeline(self):
        """Filter pipeline method for mesh visualization

        For every confirmed change of the widget settings, the mesh has to be
        updated. To this end, the filter pipeline consisting of the implemented
        threshold and extraction algorithms has to be executed. It is done
        within this method.
        """

        # apply threshold filter
        self.thresholdAlgorithm.ThresholdBetween(self.currentSettings["lowerThreshold"],self.currentSettings["upperThreshold"]) # set threshold values from slider widgets
        self.thresholdAlgorithm.Update()                                        # update information stored in thresold algorithm
        self.mesh.shallow_copy(self.thresholdAlgorithm.GetOutput())             # store resulting information in the mesh -> update mesh to visualize

        # apply extraction filter
        self.extractionBox.SetBounds(*self.currentSettings["boxBounds"])        # set box bounds from box widget
        self.extractionAlgorithm.Update()                                       # update information stored in extraction algorithm
        self.mesh.shallow_copy(self.extractionAlgorithm.GetOutput())            # store resulting information in the mesh -> update mesh to visualize

        # force update of the rendering scene
        self.plotterObj.render()                                                # if not done, the renderer might update only after the next interaction with the plot

################################################################################
#                      INTERFACING METHODS FOR PYVISTA                         #
################################################################################

    ############################################
    # Method to set up the threshold algorithm #
    ############################################
    def setupThresholdAlgorithm(self):
        """Threshold algorithm method for mesh visualization

        This method implements the threshold algorithm used within the filter
        pipeline. Based on the settings of the slider widgets, the visibility
        of physical groups is enabled oder disabled.
        """
        self.thresholdAlgorithm = vtk.vtkThreshold()                            # use vtkThreshold filter class of vtk library
        self.thresholdAlgorithm.SetInputDataObject(self.mesh)                   # set mesh to plot as input data object
        self.thresholdAlgorithm.SetInputArrayToProcess(0, 0, 0, self.scalars[0].value, self.scalars[1]) # args: (idx, port, connection, field, name)
        self.thresholdAlgorithm.Update()                                        # update threshold algorithm
        self.mesh=pv.wrap(self.thresholdAlgorithm.GetOutput())                  # update mesh once and connect it with with the algorithm output

    #############################################
    # Method to set up the extraction algorithm #
    #############################################
    def setupExtractionAlgorithm(self):
        """Extraction algorithm method for mesh visualization

        This method implements the extraction algorithm used within the filter
        pipeline. Based on the settings of the box widget, the mesh elements
        within the extraction box bounds are extracted.
        """
        self.extractionAlgorithm = vtk.vtkExtractGeometry()                     # use vtkExtractGeometry filter class of vtk library
        self.extractionAlgorithm.SetInputDataObject(self.mesh)                  # use (potentially updated) mesh to plot as input data object
        self.extractionAlgorithm.SetImplicitFunction(self.extractionBox)        # describe extraction box by implicit function with the information stored in extractionBox
        self.extractionAlgorithm.Update()                                       # update extraction algorithm

    ################################
    # Method to add slider widgets #
    ################################
    def addSliderWidgets(self):
        """Method to configure a slider widget and add it to the rendering scene"""

        # set up corresponding algorithm
        self.setupThresholdAlgorithm()                                          # connect threshold algorithm to slider widgets

        # set up widgets
        pointsA=np.array([[0.01, 0.92],[0.01, 0.79]])                           # define starting points of slider widgets (normalized coordinates)
        pointsB=np.array([[0.11, 0.92],[0.11, 0.79]])                           # define end points of slider widgets (normalized coordinates)
        initialValues=[self.currentSettings["lowerThreshold"],self.currentSettings["upperThreshold"]] # define initial slider values according to current settings
        titles=["Min", "Max"]                                                   # define titles of slider widgets
        callbacks=[self._callbackLowerThreshold, self._callbackUpperThreshold]  # define callback methods of the slider widgets
        for iSlider in [0,1]:                                                   # loop over both sliders
            slider=self.plotterObj.add_slider_widget(callback=callbacks[iSlider], value=initialValues[iSlider], rng=initialValues, title=titles[iSlider], color=[1,1,1]) # use add_slider_widget of pyvista to define the widgets
            slider.GetRepresentation().GetPoint1Coordinate().SetValue(pointsA[iSlider,0], pointsA[iSlider,1]) # properly set slider widget starting point (error in pyvista implementation (divide by int))
            slider.GetRepresentation().GetPoint2Coordinate().SetValue(pointsB[iSlider,0], pointsB[iSlider,1]) # properly set slider widget end point (error in pyvista implementation (divide by int))
            slider.GetRepresentation().SetSliderWidth(0.02)                     # set width of the slider (normalized coordinates)
            slider.GetRepresentation().SetSliderLength(0.015)                   # set length of the slider (normalized coordinates)
            slider.GetRepresentation().SetEndCapLength(0.005)                   # set length of the bars at beginning and end of the slider widget (normalized coordinates)
            slider.GetRepresentation().SetEndCapWidth(0.033)                    # set width of the bars at beginning and end of the slider widget (normalized coordinates)
            slider.GetRepresentation().SetTubeWidth(0.005)                      # set width of the line connecting both slider ends (normalized coordinates)
            slider.GetRepresentation().SetTitleHeight(0.025)                    # set height of the title (normalized coordinates)
            slider.GetRepresentation().SetLabelHeight(0.025)                    # set height of the label (normalized coordinates)
            slider.GetRepresentation().GetSelectedProperty().SetColor([0.9, 0.9, 0.95]) # set color of selected slider
            slider.Off()                                                        # initially make slider widget invisible
            self.activeWidgets.append(slider)                                   # add slider widget to the list of active widgets

    ############################
    # Method to add box widget #
    ############################
    def addBoxWidget(self):
        """Method to configure a box widget and add it to the rendering scene"""

        # set up corresponding algorithm
        self.setupExtractionAlgorithm()                                         # connect extraction algorithm to box widget

        # set up widget
        box=self.plotterObj.add_box_widget(callback=self._callbackExtraction, bounds=self.defaultSettings["boxBounds"], rotation_enabled=False) # use add_box_widget of pyvista to define the widget
        box.Off()                                                               # initially make box widget invisible
        self.activeWidgets.append(box)                                          # add box widget to the list of active widgets

    ##################################
    # Method to add key press events #
    ##################################
    def addKeyPressEvents(self):
        """Method to add all user-defined key-press events"""
        activeKeys=["h", "m", "d", "space", "x", "y", "z"]                      # define list of keys with active (user-defined) key press events
        for key in activeKeys:                                                  # loop over all active keys
            self.plotterObj.add_key_event(key,self._keyPressEvents)             # add corresponding key press events

    ####################################
    # Method to toggle menu visibility #
    ####################################
    def toggleMenu(self):
        """Method to toggle the menu/widget visibility"""
        # loop over all active widgets
        for widget in self.activeWidgets:                                       # get current widget
            toggleFuncs=[widget.On, widget.Off]                                 # define toggle functions depending on state [0/1]
            toggleStatus=widget.GetEnabled()                                    # get widget state
            toggleFuncs[toggleStatus]()                                         # use toggle function according to current state

    ######################################
    # Method to restore default settings #
    ######################################
    def restoreDefaultSettings(self):
        """Method to restore default visualization options"""
        self.currentSettings.update(self.defaultSettings)                       # restore default settings
        self.plotterObj.slider_widgets[0].GetRepresentation().SetValue(self.currentSettings["lowerThreshold"]) # set value of lower threshold slider according to settings
        self.plotterObj.slider_widgets[1].GetRepresentation().SetValue(self.currentSettings["upperThreshold"]) # set value of upper threshold slider according to settings
        self.plotterObj.box_widgets[0].PlaceWidget(*self.currentSettings["boxBounds"])  # set bounds of extraction box according to settings
        self._callbackExtraction(self.extractionBoxBounds)                      # also call callback method of extraction box to complete the update
        self.filterPipeline()                                                   # update filter pipeline with restored settings

    ####################################
    # Method to show command line help #
    ####################################
    def showCommandLineHelp(self):
        """Internal method to show command line help after the rendering window
        is displayed"""
        infoText=("\nUse one of the following key events to control the plot:\n"
                  "\n"
                  "\ts \tactivate surface representation of objects\n"
                  "\tw \tactivate wireframe representation if objects\n"
                  "\tv \tenable isometric view\n"
                  "\tx \tset view to y-z-plane\n"
                  "\ty \tset view to z-x-plane\n"
                  "\tz \tset view to x-y-plane\n"
                  "\tm \ttoggle menu\n"
                  "\tspace \tconfirm menu settings\n"
                  "\td \trestore default settings\n"
                  "\tq \tclose rendering window\n")
        print(infoText)

################################################################################
#           ADDITIONAL PRIVATE/HIDDEN METHODS FOR INTERNAL USE ONLY            #
################################################################################

    #########################################################
    # Internal callback method tofor lower threshold slider #
    #########################################################
    def _callbackLowerThreshold(self,sliderThresholdValue):
        """Internal callback method for the lower threshold widget"""
        self.currentSettings["lowerThreshold"]=sliderThresholdValue             # update lower threshold value with current slider position

        # prevent mixed up threshold sliders (lower > upper)
        try:                                                                    # use try-catch environment to prevent error in plot initialization (other slider is not defined yet)
            if sliderThresholdValue > self.plotterObj.slider_widgets[1].GetRepresentation().GetValue(): # check if value of lower threshold exceeds current upper threshold value
                self.plotterObj.slider_widgets[1].GetRepresentation().SetValue(sliderThresholdValue)    # -> update upper threshold value
                self.currentSettings["upperThreshold"]=sliderThresholdValue     # also update upper threshold value in current settings
        except:                                                                 # other slider does not exists yet
            pass                                                                # do nothing

    #######################################################
    # Internal callback method for upper threshold slider #
    #######################################################
    def _callbackUpperThreshold(self,sliderThresholdValue):
        """Internal callback method for the upper threshold widget"""
        self.currentSettings["upperThreshold"]=sliderThresholdValue             # update upper threshold value with current slider position

        # prevent mixed up threshold sliders (upper < lower)
        try:                                                                    # use try-catch environment to prevent error during plot initialization (other slider is not defined yet)
            if sliderThresholdValue < self.plotterObj.slider_widgets[0].GetRepresentation().GetValue(): # check if value of upper threshold is below current lower threshold value
                self.plotterObj.slider_widgets[0].GetRepresentation().SetValue(sliderThresholdValue)    # -> update lower threshold value
                self.currentSettings["lowerThreshold"]=sliderThresholdValue     # also update lower threshold value in current settings
        except:                                                                 # other slider does not exist yet
            pass                                                                # do nothing

    ##################################################
    # Internal callback method for extraction widget #
    ##################################################
    def _callbackExtraction(self,boundsObj):
        """Internal callback method for the extraction box widget"""
        # update bounds of extraction box with current box bounds
        self.extractionBoxBounds.shallow_copy(boundsObj)                        # copy information of passed bounds object fom widget
        self.currentSettings["boxBounds"]=boundsObj.bounds                      # update current settings

    ##############################################
    # Internal method to define key press events #
    ##############################################
    def _keyPressEvents(self):
        """Internal method setting up user-defined key-press events"""
        if self.plotterObj.iren.GetKeySym() == "h":                             # check if "h"-key was pressed
            self.showCommandLineHelp()                                          # -> show help

        elif self.plotterObj.iren.GetKeySym() == "m":                           # check if "m"-key was pressed
            self.toggleMenu()                                                   # -> toggle menu

        elif self.plotterObj.iren.GetKeySym() == "d":                           # check if "d"-key was pressed
            self.restoreDefaultSettings()                                       # restore defaults

        elif self.plotterObj.iren.GetKeySym() == "space":                       # check if "space"-key was pressed
            self.filterPipeline()                                               # update filter pipeline with confirmed settings

        elif self.plotterObj.iren.GetKeySym() == "x":                           # check if "x"-key was pressed
            self.plotterObj.view_yz()                                           # set view to y-z-plane

        elif self.plotterObj.iren.GetKeySym() == "y":                           # check if "y"-key was pressed
            self.plotterObj.view_zx()                                           # set view to z-x-plane

        elif self.plotterObj.iren.GetKeySym() == "z":                           # check if "z"-key was pressed
            self.plotterObj.view_xy()                                           # set view to x-y-plane


text:
# Copyright 2018 The Shabda Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Iterator that creates features using madmom
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse

import os
import time
import glob
import librosa
import numpy as np
import tensorflow as tf
from overrides import overrides
import matplotlib.pyplot as plt

from madmom.processors import Processor

from shabda.data.dataset.freesound_dataset import FreeSoundAudioDataset
from shabda.data.iterators.internal.free_sound_data_iterator_base import FreeSoundDataIteratorBase

class LibrosaProcessor(Processor):

    def __init__(self):
        pass

    def process(self, file_path, **kwargs):
        n_fft = 1024
        sr = 32000
        mono = True
        log_spec = False
        n_mels = 128

        hop_length = 192
        fmax = None

        if mono:
            sig, sr = librosa.load(file_path, sr=sr, mono=True)
            sig = sig[np.newaxis]
        else:
            sig, sr = librosa.load(file_path, sr=sr, mono=False)
            # sig, sf_sr = sf.read(file_path)
            # sig = np.transpose(sig, (1, 0))
            # sig = np.asarray([librosa.resample(s, sf_sr, sr) for s in sig])

        spectrograms = []
        for y in sig:

            # compute stft
            stft = librosa.stft(y, n_fft=n_fft, hop_length=hop_length, win_length=None, window='hann', center=True,
                                pad_mode='reflect')

            # keep only amplitures
            stft = np.abs(stft)

            # spectrogram weighting
            if log_spec:
                stft = np.log10(stft + 1)
            else:
                freqs = librosa.core.fft_frequencies(sr=sr, n_fft=n_fft)
                stft = librosa.perceptual_weighting(stft ** 2, freqs, ref=1.0, amin=1e-10, top_db=99.0)

            # apply mel filterbank
            spectrogram = librosa.feature.melspectrogram(S=stft, sr=sr, n_mels=n_mels, fmax=fmax)

            # keep spectrogram
            spectrograms.append(np.asarray(spectrogram))

        spectrograms = np.asarray(spectrograms)

        return spectrograms

class MadmomFeatureIteratorV1(FreeSoundDataIteratorBase):
    """
    Custom melspectrogram feature extraction using Madmom library pipepline
    using librosa APIs
    Reference: https://github.com/CPJKU/dcase_task2/blob/master/dcase_task2/prepare_spectrograms.py
    """

    def __init__(self, hparams, dataset: FreeSoundAudioDataset):
        super(MadmomFeatureIteratorV1, self).__init__(hparams, dataset)

        if not isinstance(dataset, FreeSoundAudioDataset):
            raise AssertionError("dataset should be FreeSoundAudioDataset")

        self.processor_version1 = LibrosaProcessor()

    @overrides
    def _user_map_func(self, file_path, label):
        """
        Function that maps the audio files into features and labels as on-hot vector
        :param file_path:
        :param label:
        :return:
        """
        data = self.processor_version1.process(file_path)

        label = self._dataset.get_one_hot_encoded(label)
        return data, label

    @overrides
    def _user_resize_func(self, data, label):
        """
        Function that sets up the sizes of the tensor, after execution of `tf.py_func` call
        :param data:
        :param label:
        :return:
        """
        data = tf.reshape(data, shape=[128, 33])
        label = tf.reshape(label, shape=[42])
        return data, label

max_stars_repo_name: dhiraa/sabdha

max_issues_repo_name: dhiraa/sabdha

max_forks_repo_name: dhiraa/sabdha


text:
from __future__ import absolute_import, division, print_function

import functools
import sys
import time

import numpy as np

from cctbx import factor_ev_angstrom
from libtbx.phil import parse

from dxtbx import IncorrectFormatError
from dxtbx.format.Format import Format, abstract
from dxtbx.format.FormatMultiImage import Reader
from dxtbx.format.FormatMultiImageLazy import FormatMultiImageLazy
from dxtbx.format.FormatStill import FormatStill

try:
    import psana

    from xfel.cxi.cspad_ana import cspad_tbx
except ImportError:
    psana = None
    cspad_tbx = None

locator_str = """
  experiment = None
    .type = str
    .help = Experiment identifier, e.g. mfxo1916
  run = None
    .type = ints
    .help = Run number or a list of runs to process
  mode = idx
    .type = str
    .help = Mode for reading the xtc data (see LCLS documentation)
  data_source = None
    .type = str
    .help = Complete LCLS data source.  Overrides experiment and run.  Example: \
            exp=mfxo1916:run=20:smd \
            More info at https://confluence.slac.stanford.edu/display/PSDM/Manual#Manual-Datasetspecification
  detector_address = None
    .type = str
    .multiple = True
    .help = detector used for collecting the data at LCLS
  calib_dir = None
    .type = str
    .help = Specifiy path to custom calib directory if needed
  use_ffb = False
    .type = bool
    .help = Run on the ffb if possible. Only for active users!
  wavelength_offset = None
    .type = float
    .help = Optional constant shift to apply to each wavelength
  spectrum_eV_per_pixel = None
    .type = float
    .help = If not None, use the FEE spectrometer to determine the wavelength. \
            spectrum_eV_offset should be also specified. A weighted average of \
            the horizontal projection of the per-shot FEE spectrometer is used.\
            The equation for each pixel eV is \
            eV = (spectrum_eV_per_pixel * pixel_number) + spectrum_eV_offset
  spectrum_eV_offset = None
    .type = float
    .help = See spectrum_eV_per_pixel
"""
locator_scope = parse(locator_str)

class XtcReader(Reader):
    def nullify_format_instance(self):
        """No-op for XTC streams. No issue with multiprocessing."""
        pass

@abstract
class FormatXTC(FormatMultiImageLazy, FormatStill, Format):
    def __init__(self, image_file, **kwargs):

        if not self.understand(image_file):
            raise IncorrectFormatError(self, image_file)
        FormatMultiImageLazy.__init__(self, **kwargs)
        FormatStill.__init__(self, image_file, **kwargs)
        Format.__init__(self, image_file, **kwargs)
        self.current_index = None
        self.current_event = None
        self._psana_runs = {}  # empty container, to prevent breaking other formats
        if "locator_scope" in kwargs:
            self.params = FormatXTC.params_from_phil(
                master_phil=kwargs["locator_scope"], user_phil=image_file, strict=True
            )
        else:
            self.params = FormatXTC.params_from_phil(
                master_phil=locator_scope, user_phil=image_file, strict=True
            )
        assert self.params.mode == "idx", "idx mode should be used for analysis"

        self._ds = FormatXTC._get_datasource(image_file, self.params)
        self.populate_events()
        self.n_images = len(self.times)

        self._cached_psana_detectors = {}
        self._beam_index = None
        self._beam_cache = None
        self._initialized = True
        self._fee = None

    @staticmethod
    def understand(image_file):
        """Extracts the datasource and detector_address from the image_file and then feeds it to PSANA
        If PSANA fails to read it, then input may not be an xtc/smd file. If success, then OK.
        If detector_address is not provided, a command line promp will try to get the address
        from the user"""
        if not psana or not cspad_tbx:
            return False
        try:
            params = FormatXTC.params_from_phil(locator_scope, image_file)
        except Exception:
            return False
        if params is None:
            return False

        try:
            FormatXTC._get_datasource(image_file, params)
        except Exception:
            return False
        return True

    @staticmethod
    def params_from_phil(master_phil, user_phil, strict=False):
        """Read the locator file"""
        try:
            user_input = parse(file_name=user_phil)
            working_phil, unused = master_phil.fetch(
                sources=[user_input], track_unused_definitions=True
            )
            unused_args = ["%s=%s" % (u.path, u.object.words[0].value) for u in unused]
            if len(unused_args) > 0 and strict:
                for unused_arg in unused_args:
                    print(unused_arg)
                print(
                    "Incorrect of unused parameter in locator file. Please check and retry"
                )
                return None
            params = working_phil.extract()
            return params
        except Exception:
            return None

    @classmethod
    def get_reader(cls):
        """
        Return a reader class
        """
        return functools.partial(XtcReader, cls)

    def populate_events(self):
        """Read the timestamps from the XTC stream.  Assumes the psana idx mode of reading data.
        Handles multiple LCLS runs by concatenating the timestamps from multiple runs together
        in a single list and creating a mapping."""
        if hasattr(self, "times") and len(self.times) > 0:
            return

        if not self._psana_runs:
            self._psana_runs = self._get_psana_runs(self._ds)

        self.times = []
        self.run_mapping = {}
        for run in self._psana_runs.values():
            times = run.times()
            self.run_mapping[run.run()] = (
                len(self.times),
                len(self.times) + len(times),
                run,
            )
            self.times.extend(times)

    def get_run_from_index(self, index=None):
        """Look up the run number given an index"""
        if index is None:
            index = 0
        for run_number in self.run_mapping:
            start, stop, run = self.run_mapping[run_number]
            if index >= start and index < stop:
                return run
        raise IndexError("Index is not within bounds")

    def _get_event(self, index):
        """Retrieve a psana event given and index. This is the slow step for reading XTC streams,
        so implement a cache for the last read event."""
        if index == self.current_index:
            return self.current_event
        else:
            self.current_index = index
            self.current_event = self.get_run_from_index(index).event(self.times[index])
            return self.current_event

    @staticmethod
    def _get_datasource(image_file, params):
        """Construct a psana data source object given the locator parameters"""
        if params.calib_dir is not None:
            psana.setOption("psana.calib-dir", params.calib_dir)
        if params.data_source is None:
            if (
                params.experiment is None
                or params.run is None
                or params.mode is None
                or len(params.run) == 0
            ):
                return False
            img = "exp=%s:run=%s:%s" % (
                params.experiment,
                ",".join(["%d" % r for r in params.run]),
                params.mode,
            )

            if params.use_ffb:
                # as ffb is only at SLAC, ok to hardcode /reg/d here
                img += ":dir=/reg/d/ffb/%s/%s/xtc" % (
                    params.experiment[0:3],
                    params.experiment,
                )
        else:
            img = params.data_source
        return psana.DataSource(img)

    @staticmethod
    def _get_psana_runs(datasource):
        """
        Extracts the runs,
        These can only be extracted once,
        only call this method after datasource is set
        """
        # this is key,value = run_integer, psana.Run, e.g. {62: <psana.Run(@0x7fbd0e23c990)>}
        psana_runs = {r.run(): r for r in datasource.runs()}
        return psana_runs

    def _get_psana_detector(self, run):
        """ Returns the psana detector for the given run """
        if run.run() not in self._cached_psana_detectors:
            assert len(self.params.detector_address) == 1
            self._cached_psana_detectors[run.run()] = psana.Detector(
                self.params.detector_address[0], run.env()
            )
        return self._cached_psana_detectors[run.run()]

    def get_psana_timestamp(self, index):
        """Get the cctbx.xfel style event timestamp given an index"""
        evt = self._get_event(index)
        time = evt.get(psana.EventId).time()
        # fid = evt.get(psana.EventId).fiducials()

        sec = time[0]
        nsec = time[1]

        return cspad_tbx.evt_timestamp((sec, nsec / 1e6))

    def get_num_images(self):
        return self.n_images

    def get_beam(self, index=None):
        return self._beam(index)

    def _beam(self, index=None):
        """Returns a simple model for the beam"""
        if index is None:
            index = 0
        if self._beam_index != index:
            self._beam_index = index
            evt = self._get_event(index)
            if self.params.spectrum_eV_per_pixel is not None:
                if self._fee is None:
                    self._fee = psana.Detector("FEE-SPEC0")
                fee = self._fee.get(evt)
                if fee is None:
                    wavelength = cspad_tbx.evt_wavelength(evt)
                else:
                    x = (
                        self.params.spectrum_eV_per_pixel
                        * np.array(range(len(fee.hproj())))
                    ) + self.params.spectrum_eV_offset
                    wavelength = factor_ev_angstrom / np.average(x, weights=fee.hproj())
            else:
                wavelength = cspad_tbx.evt_wavelength(evt)
            if wavelength is None:
                self._beam_cache = None
            else:
                if self.params.wavelength_offset is not None:
                    wavelength += self.params.wavelength_offset
                self._beam_cache = self._beam_factory.simple(wavelength)
            s, nsec = evt.get(psana.EventId).time()
            evttime = time.gmtime(s)
            if (
                evttime.tm_year == 2020 and evttime.tm_mon >= 7
            ) or evttime.tm_year > 2020:
                self._beam_cache.set_polarization_normal((1, 0, 0))

        return self._beam_cache

    def get_goniometer(self, index=None):
        return None

    def get_scan(self, index=None):
        return None

if __name__ == "__main__":
    for arg in sys.argv[1:]:
        print(FormatXTC.understand(arg))

max_stars_repo_name: TiankunZhou/dxtbx

max_issues_repo_name: TiankunZhou/dxtbx

max_forks_repo_name: TiankunZhou/dxtbx


text:
# -*- coding: utf-8 -*-

import nose
import numpy as np

from pandas.types.dtypes import DatetimeTZDtype, CategoricalDtype
from pandas.types.common import pandas_dtype

_multiprocess_can_split_ = True

def test_pandas_dtype():

    assert pandas_dtype('datetime64[ns, US/Eastern]') == DatetimeTZDtype(
        'datetime64[ns, US/Eastern]')
    assert pandas_dtype('category') == CategoricalDtype()
    for dtype in ['M8[ns]', 'm8[ns]', 'object', 'float64', 'int64']:
        assert pandas_dtype(dtype) == np.dtype(dtype)

if __name__ == '__main__':
    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                   exit=False)


text:
import numpy as np
def makeNewSequenceWithNoise(X_i:np.ndarray, n_frames = 5, noise_frac = 0.3):
    # X_i is shape 60, 25

    X = X_i.transpose()

    X_new = np.zeros_like(X)

    for i, seq in enumerate(X):
        noise = np.std(seq[:n_frames])
        if noise is 0: # from all nan slices being converted to 0
            seq_new = seq
        else:
            seq_new = seq + np.random.normal(loc=0.0, scale=noise * noise_frac, size=len(seq))

        X_new[i] = seq_new

    return X_new.transpose()

if __name__ == '__main__':
    from pathlib import Path
    from reading_data import *
    from sklearn.utils import resample
    import matplotlib.pyplot as plt
    inp = Path('./input/npz/')
    filenames = inp.glob('fold?Training.npz')

    filenames = [f for f in filenames if '3' not in str(f)] # use fold 3 for validation

    fpo = inp.joinpath('fold1-2Training_balanced.npz')

    Xs = []
    ys = []

    for fp in filenames:
        print('Now loading file: ')
        print(fp.name)
        X, y = load_npz_file(fp)
        Xs.append(X)
        ys.append(y)

    X = np.concatenate(Xs)
    y = np.concatenate(ys)
    print('concatenated all files')
    # print(newX.shape)

    ## Split dataset by positive or negative
    X_0 = X[np.argwhere(y == 0)] # type: np.ndarray
    X_1 = X[np.argwhere(y == 1)] # type: np.ndarray

    X_0 = X_0.reshape((-1, 60, 25))
    X_1 = X_1.reshape((-1, 60, 25))

    print(X_0.shape)
    print(X_1.shape)

    # TODO make this again so that we have more data to train on
    # for x in X_1:
    #     plt.plot(x[:,0])
    #     print(x[:,0])
    #     x = makeNewSequenceWithNoise(x)
    #     print(x[:,0])
    #     plt.plot(x[:,0])
    #     plt.show()
    #     exit()

    n_0 = X_0.shape[0]
    n_1 = X_1.shape[0]

    n   = min(n_0, n_1)

    ## Make the dataset 50/50 split

    X_0_new = resample(X_0, n_samples=n, replace=False)
    X_1_new = resample(X_1, n_samples=n, replace=False) # this is to make it bigger

    y_0_new = np.zeros(n, dtype=int)
    y_1_new = np.ones(n, dtype=int)

    X_new = np.concatenate([X_0_new, X_1_new])# type: np.ndarray
    y_new = np.concatenate([y_0_new, y_1_new])# type: np.ndarray

    # mix them up so we don't have any issues with indexing (although it shouldn't happen)
    X, y = resample(X_new, y_new, replace=False) # type: np.ndarray

    print(X.shape)
    print(y.shape)

    ## Save and profit
    ids = np.arange(1, 2*n + 1)

    assert len(ids ) == len(y)

    np.savez(fpo, data=X, labels=y, index=ids)
    print('Saved as: ')
    print(fpo.absolute())


text:
import collections
import logging
import json
import math
import os
import random
import six
import numpy as np
import tokenization

def _check_is_max_context(doc_spans, cur_span_index, position):
    """Check if this is the 'max context' doc span for the token."""

    # Because of the sliding window approach taken to scoring documents, a single
    # token can appear in multiple documents. E.g.
    #  Doc: the man went to the store and bought a gallon of milk
    #  Span A: the man went to the
    #  Span B: to the store and bought
    #  Span C: and bought a gallon of
    #  ...
    #
    # Now the word 'bought' will have two scores from spans B and C. We only
    # want to consider the score with "maximum context", which we define as
    # the *minimum* of its left and right context (the *sum* of left and
    # right context will always be the same, of course).
    #
    # In the example the maximum context for 'bought' would be span C since
    # it has 1 left context and 3 right context, while span B has 4 left context
    # and 0 right context.
    best_score = None
    best_span_index = None
    for (span_index, doc_span) in enumerate(doc_spans):
        end = doc_span.start + doc_span.length - 1
        if position < doc_span.start:
            continue
        if position > end:
            continue
        num_left_context = position - doc_span.start
        num_right_context = end - position
        score = min(num_left_context, num_right_context) + 0.01 * doc_span.length
        if best_score is None or score > best_score:
            best_score = score
            best_span_index = span_index

    return cur_span_index == best_span_index

def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False):
    """Project the tokenized prediction back to the original text."""

    # When we created the data, we kept track of the alignment between original
    # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So
    # now `orig_text` contains the span of our original text corresponding to the
    # span that we predicted.
    #
    # However, `orig_text` may contain extra characters that we don't want in
    # our prediction.
    #
    # For example, let's say:
    #   pred_text = steve smith
    #   orig_text = Steve Smith's
    #
    # We don't want to return `orig_text` because it contains the extra "'s".
    #
    # We don't want to return `pred_text` because it's already been normalized
    # (the SQuAD eval script also does punctuation stripping/lower casing but
    # our tokenizer does additional normalization like stripping accent
    # characters).
    #
    # What we really want to return is "Steve Smith".
    #
    # Therefore, we have to apply a semi-complicated alignment heruistic between
    # `pred_text` and `orig_text` to get a character-to-charcter alignment. This
    # can fail in certain cases in which case we just return `orig_text`.

    def _strip_spaces(text):
        ns_chars = []
        ns_to_s_map = collections.OrderedDict()
        for (i, c) in enumerate(text):
            if c == " ":
                continue
            ns_to_s_map[len(ns_chars)] = i
            ns_chars.append(c)
        ns_text = "".join(ns_chars)
        return (ns_text, ns_to_s_map)

    # We first tokenize `orig_text`, strip whitespace from the result
    # and `pred_text`, and check if they are the same length. If they are
    # NOT the same length, the heuristic has failed. If they are the same
    # length, we assume the characters are one-to-one aligned.
    tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case)

    tok_text = " ".join(tokenizer.tokenize(orig_text))

    start_position = tok_text.find(pred_text)
    if start_position == -1:
        return orig_text
    end_position = start_position + len(pred_text) - 1

    (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text)
    (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text)

    if len(orig_ns_text) != len(tok_ns_text):
        return orig_text

    # We then project the characters in `pred_text` back to `orig_text` using
    # the character-to-character alignment.
    tok_s_to_ns_map = {}
    for (i, tok_index) in six.iteritems(tok_ns_to_s_map):
        tok_s_to_ns_map[tok_index] = i

    orig_start_position = None
    if start_position in tok_s_to_ns_map:
        ns_start_position = tok_s_to_ns_map[start_position]
        if ns_start_position in orig_ns_to_s_map:
            orig_start_position = orig_ns_to_s_map[ns_start_position]

    if orig_start_position is None:
        return orig_text

    orig_end_position = None
    if end_position in tok_s_to_ns_map:
        ns_end_position = tok_s_to_ns_map[end_position]
        if ns_end_position in orig_ns_to_s_map:
            orig_end_position = orig_ns_to_s_map[ns_end_position]

    if orig_end_position is None:
        return orig_text

    output_text = orig_text[orig_start_position:(orig_end_position + 1)]
    return output_text


text:
import image
import numpy as np
import matplotlib.pyplot as plt

class Graph:
    """Class manages drawing of a graph.

    Class manages drawing of a graph; it draws customers, depots,
    connections between them and a legend. It also draws text
    information about generation, instance and fitness value on
    the graph. It is responsible for drawing a graph that shows
    fitness values through generations.

    Attributes:
        customers: An array of Customer objects, indicating all the
        customers.
        depots: An array of Depot objects, indicating all the depots.
    """

    def __init__(self, customers, depots):
        """Inits Graph with customers and depots."""
        self.customers = customers
        self.depots = depots

    @staticmethod
    def draw_fitness_graph(fitness_list):
        """Draws fitness values graph and saves image.

        Draws graph that shows fitness values through
        generations and then calls saving of it.

        Args:
            fitness_list: An array of Fitness objects, indicating
            final results of instances in evaluation.

        Returns:
            Method does not return anything.
        """

        generations = fitness_list[-1].generation
        instances = fitness_list[-1].instance

        data = Graph.get_fitness_data(fitness_list)
        values = [i[0] for i in data]
        titles = [i[1] for i in data]

        plt.figure(figsize=(20, 15), dpi=120)
        plt.rcParams.update({'font.size': 16})

        counter = 0
        for i in range(generations):
            color = np.random.random(3)
            plt.bar(titles[counter:(counter + instances)], values[counter:(counter + instances)],
                    0.8, alpha=0.5, label='Generation ' + str(i + 1), color=color)
            counter += instances

        plt.xticks([])
        plt.ylabel('Fitness')
        plt.xlabel('Instances')
        Graph.draw_legend()
        plt.suptitle('Fitness values through generations', fontsize=22)

        image.Image.save_fitness_image(plt)
        plt.close()

    @staticmethod
    def draw_legend():
        """Draws legend.

        Draws a legend, which is made of a color circle and
        belonging label.

        Args:
            Method does not have any arguments.

        Returns:
            Method does not return anything.
        """

        plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), fancybox=True, shadow=True, ncol=5)

    @staticmethod
    def draw_text(fitness):
        """Draws text.

        Draws text with information about generation, instance and
        fitness value of graph.

        Args:
            fitness: A Fitness object, indicating final result of instance
            in evaluation.

        Returns:
            Method does not return anything.
        """

        plt.suptitle('Generation: ' + str(fitness.generation) + ', Instance: ' + str(fitness.instance), fontsize=16)
        plt.title('Fitness: ' + str(round(fitness.value, 2)), fontsize=14)

    def draw(self, results, fitness):
        """Draws a graph.

        Main method, responsible for drawing a graph. It creates new figure and
        draws customers and depots. After that, it draws a legend and text with
        information about generation, instance and fitness value of graph.
        Lastly it creates connections between customers and depots and closes
        the figure.

        Args:
            results: An array of Result objects, indicating results from
            evaluation.
            fitness: A Fitness object, indicating final result of instance
            in evaluation.

        Returns:
            Method does not return anything.
        """

        plt.figure(figsize=(10, 10), dpi=90)
        self.draw_depots()
        self.draw_customers()
        self.draw_legend()
        self.draw_text(fitness)
        self.draw_connections(results, fitness)
        plt.close()

    def draw_depots(self):
        """Draws depots.

        Draws each depot on a graph in form of a point. Position of a
        point is determined from depots x and y coordinates.

        Args:
            Method does not have any arguments.

        Returns:
            Method does not return anything.
        """

        coordinates = self.get_coordinates(self.depots)
        sorted(coordinates)
        for index, i in enumerate(coordinates):
            if index == 0:
                plt.scatter(i[0], i[1], alpha=0.8, c='C0', edgecolors='C0', s=30, label='Depot', zorder=2)
            plt.scatter(i[0], i[1], alpha=0.8, c='C0', edgecolors='C0', s=30, zorder=2)

    def draw_customers(self):
        """Draws customers.

        Draws each customer on a graph in form of a point. Position of a
        point is determined from customers x and y coordinates.

        Args:
            Method does not have any arguments.

        Returns:
            Method does not return anything.
        """

        coordinates = self.get_coordinates(self.customers)
        sorted(coordinates)
        for index, i in enumerate(coordinates):
            if index == 0:
                plt.scatter(i[0], i[1], alpha=0.8, c='C1', edgecolors='C1', s=30, label='Customer', zorder=2)
            plt.scatter(i[0], i[1], alpha=0.8, c='C1', edgecolors='C1', s=30, zorder=2)

    def draw_connections(self, results, fitness):
        """Draws connections and saves images.

        Draws connections between already drawn customers and depots on
        a graph and calls saving its each intermediate state to an image.
        Connections are obtained from array of results.

        Args:
            results: An array of Result objects, indicating results from
            evaluation.
            fitness: A Fitness object, indicating final result of instance
            in evaluation.

        Returns:
            Method does not return anything.
        """

        img = image.Image(str(fitness.generation), str(fitness.instance))
        img.save(plt)

        for i in results:
            color = np.random.random(3)
            coordinates = self.get_connection_coordinates(i)

            for index, j in enumerate(coordinates):
                if index == len(coordinates)-1:
                    break
                plt.plot([j[0], coordinates[index + 1][0]], [j[1], coordinates[index + 1][1]],
                         alpha=0.5, c=color, zorder=1)
                img.save(plt)
        img.create_instance_gif()

    @staticmethod
    def get_coordinates(data):
        """Gets coordinates.

        Gets x and y coordinates from multiple objects and stores them
        separately to an array, consisting of arrays of two float values.

        Args:
            data: An array of objects, indicating customers or depots.

        Returns:
            An array consisting of arrays of two float values, x-coordinate
            and y-coordinate.
        """

        coordinates = []
        for i in data:
            coordinates.append([i.x, i.y])
        return coordinates

    @staticmethod
    def get_fitness_data(fitness_list):
        """Gets fitness data.

        Gets fitness value, generation and instance from
        fitness results.

        Args:
            fitness_list: An array of Fitness objects, indicating
            final results of instances in evaluation.

        Returns:
            An array consisting of arrays of float and string values.
        """

        coordinates = []
        for i in fitness_list:
            coordinates.append([i.value, str(i.generation) + '.' + str(i.instance)])
        return coordinates

    def get_connection_coordinates(self, results):
        """Gets connection coordinates.

        Gets x and y coordinates of customers and depots from results.

        Args:
            results: An array of Result objects, indicating results from
            evaluation.

        Returns:
            An array consisting of arrays of two float values, x-coordinate
            and y-coordinate.
        """

        coordinates = self.get_coordinates(results.customers)
        coordinates.insert(0, [results.depot.x, results.depot.y])
        coordinates.append([results.depot.x, results.depot.y])
        return coordinates

max_stars_repo_name: karakatic/evovrp

max_issues_repo_name: karakatic/evovrp

max_forks_repo_name: karakatic/evovrp


text:
#!/usr/bin/env python

######
# 2015 by Donghan Lee
########

from sys import argv, stdout
from numpy import array
from estp.est_model_noex import est_model

import json

nrun = int(argv[2])

m2=est_model()
m2.verbose = True

print('**************')
stdout.write(m2.programName + '\n')
print('**************')

# define datasets

configFile = open(argv[1])
conf = json.load(configFile)
configFile.close()

projectName = conf['Project Name']
datasetsNames = conf['datasets']

for dataset in datasetsNames:
    m2.dataset.addData(dataset)

residues = conf['residues']

for r in residues:
    resid = r['name']
    active = r['flag']
    print( 'Residue: ' + resid + ' ' + active)
    for r in m2.dataset.res:
        if r.label == resid:
            if active == 'on':
                r.active = True
            elif active == 'off':
                r.active = False
            else:
                print('Error: wrong flag for the residue' + resid + ' ' + active)
                exit()

m2.datapdf(projectName + '_data.pdf')

p0=m2.initGuessAll(conf['init'])

out=m2.fit(p0)

pp = out[0]

m2.pdf(out,projectName + '.pdf')

allps = []
for i in range(0, nrun):
    mmc =est_model()
    mmc.verbose = True

    projectName = conf['Project Name']
    datasetsNames = conf['datasets']
    for dataset in datasetsNames:
        mmc.dataset.addDataWithError(dataset)

    residues = conf['residues']

    for r in residues:
        resid = r['name']
        active = r['flag']
        print('Residue: ' + resid + ' ' + active)
        for r in mmc.dataset.res:
            if r.label == resid:
	            if active == 'on':
	                r.active = True
	            elif active == 'off':
	                r.active = False
	            else:
	                print('Error: wrong flag for the residue' + resid + ' ' + active)
	                exit()

    mmc.selMethod(conf['init'])
    outmc=mmc.fit(pp)

    allps.append(outmc[0])

logBuf = m2.getLogBufferMC(pp,allps)

file1 = open(projectName + '_mc.txt', 'w')
print(logBuf[0])
file1.write(logBuf[0])
file1.close()

m2.pdf(logBuf,projectName + '_mcmean.pdf')

print(logBuf[1])

print('########\n')

max_stars_repo_name: deepnmr/ESTP

max_issues_repo_name: deepnmr/ESTP

max_forks_repo_name: deepnmr/ESTP

reason: from numpy


text:
from flask import Flask, jsonify
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.orm import Session
from sqlalchemy import create_engine, func, desc

import pandas as pd
import numpy as np
import datetime as dt
import sqlalchemy

engine = create_engine("sqlite:///Resources/hawaii.sqlite")
Base = automap_base()
Base.prepare(engine, reflect=True)
Base.classes.keys()
Measurement = Base.classes.measurement
Station = Base.classes.station
session = Session(engine)

#Flask
app = Flask(__name__)

#Home Route
@app.route("/")

def home():

	return "Possible routes:
/api/v1.0/precipitation
/api/v1.0/stations
/api/v1.0/tobs
/api/v1.0/STARTDATE
/api/v1.0/STARTDATE/ENDDATE

Format dates as YYYY-MM-DD"

#Precipitation Route
@app.route("/api/v1.0/precipitation")

def prcp():
	latest = dt.date.fromisoformat(session.query(Measurement).order_by(desc(Measurement.date)).first().date)
	earliest = latest-dt.timedelta(days=365)
	prcp_data = session.query(Measurement.date, Measurement.prcp).filter(earliest<Measurement.date)
	prcp_df = pd.read_sql(prcp_data.statement, session.bind)
	prcp_df['date']=pd.to_datetime(prcp_df.date)
	prcp_df.set_index('date', inplace=True)
	prcp_df.sort_index(inplace=True)

    #Convert to jsonify
	prcp_df.index = prcp_df.index.astype('str')
	return jsonify(prcp_df.to_dict())

#station route
@app.route("/api/v1.0/stations")

def stations():

	#Stations
	stations = session.query(Measurement.station).group_by(Measurement.station)
	stations_df = prcp_df = pd.read_sql(stations.statement, session.bind)
	return jsonify(stations_df.to_dict())

# Tobs Route
@app.route("/api/v1.0/tobs")

def tobs():

	latest = dt.date.fromisoformat(session.query(Measurement).order_by(desc(Measurement.date)).first().date)
	earliest = latest-dt.timedelta(days=365)
	tobs_data = session.query(Measurement.date, Measurement.tobs).filter(earliest<Measurement.date)
	tobs_df = pd.read_sql(tobs_data.statement, session.bind)
	tobs_df['date']=pd.to_datetime(tobs_df.date)
	tobs_df.set_index('date', inplace=True)
	tobs_df.sort_index(inplace=True)

	tobs_df.index = tobs_df.index.astype('str')
	return jsonify(tobs_df.to_dict())

#start Route
@app.route("/api/v1.0/<start>")

def start_metrics(start):

	date_list = session.query(Measurement.date)
	date_df = pd.read_sql(date_list.statement, session.bind)
	date_dict = date_df.to_dict()
	for date in date_dict["date"].values():

		if date == start:

			return jsonify(session.query(func.min(Measurement.tobs), func.avg(Measurement.tobs), func.max(Measurement.tobs)).\

        		filter(Measurement.date >= start).all())

	return jsonify({"error":"date not found"}),404

# End Route
@app.route("/api/v1.0/<start>/<end>")

def start_end_metrics(start,end):

	found_start = False
	found_end = False

	if start > end:

		return " Wrong order"

	date_list = session.query(Measurement.date)
	date_df = pd.read_sql(date_list.statement, session.bind)
	date_dict = date_df.to_dict()

	for date in date_dict["date"].values():

		if date == start:

			found_start = True

			if (found_start == True) and (found_end == True):

				return jsonify(session.query(func.min(Measurement.tobs), func.avg(Measurement.tobs), func.max(Measurement.tobs)).\

        			filter(Measurement.date >= start).filter(Measurement.date <= end).all())

		if date == end:

			found_end = True

			if (found_start == True) and (found_end == True):

				return jsonify(session.query(func.min(Measurement.tobs), func.avg(Measurement.tobs), func.max(Measurement.tobs)).\

        			filter(Measurement.date >= start).filter(Measurement.date <= end).all())

	return jsonify({"error":"date not found"}),404

if __name__ == "__main__":

    app.run(debug=True)


text:
import numpy as np

def convolucion(A,B):
    C=np.zeros((len(A)-2,len(A[0])-2))
    for i1 in range(len(C)):
        for j1 in range(len(C[0])):
            suma=0
            for i in range(len(B)):
                for j in range(len(B[0])):
                    suma+=A[i+i1][j+j1]*B[i][j]
            C[i1][j1]=suma
    return C