text:
import dash
from dash.dependencies import Input, Output, State, Event
import dash_core_components as dcc
import dash_html_components as html
import dash_table_experiments as dt
import plotly.plotly as py
from plotly import graph_objs as go
from plotly.graph_objs import *
from flask import Flask, send_from_directory
import pandas as pd
import numpy as np
import os
import datetime
import sys
import preprocessor

preprocessor.set_options(preprocessor.OPT.URL, preprocessor.OPT.RESERVED)
reload(sys)
sys.setdefaultencoding('utf8')

server = Flask('my app')
server.secret_key = os.environ.get('secret_key', 'secret')

app = dash.Dash('SavitrApp', server=server, url_base_pathname='/', csrf_protect=False)

if 'DYNO' in os.environ:
	app.scripts.append_script({
		'external_url': 'https://cdn.rawgit.com/chriddyp/ca0d8f02a1659981a0ea7f013a378bbd/raw/e79f3f789517deec58f41251f7dbb6bee72c44ab/plotly_ga.js'
	})

curate = ['harvey', 'irma', 'sudan', 'nigeria', 'uganda', 'kampala', 'croatia', 'niger', 'manila']
# google analytics with tracking ID
external_js = ['https://codepen.io/kaustubhhiware/pen/wmWBeQ.js']

mapbox_access_token = 'pk.eyJ1IjoiYWxpc2hvYmVpcmkiLCJhIjoiY2ozYnM3YTUxMDAxeDMzcGNjbmZyMmplZiJ9.ZjmQ0C2MNs1AzEBC_Syadg'

df = pd.read_csv('data/tweets.csv')
nan = np.nan

df.rename(columns={'cr/$date':'date', 'tlt':'Lat', 'tln':'Lon'}, inplace = True)
df["date"] = pd.to_datetime(df["date"], format="%Y-%m-%d %H:%M:%S")
df.index = df["date"]
untaggedTweets = df.query("plt != plt").query("date == date").copy() # get NaN tweets
untaggedTweets.drop([ '_id', 'plt', 'pln', 'acr/$date', 'uid', 'p', 'f', 'lang', 'flrs', 'cc', 'loc','Lat', 'Lon'], axis=1, inplace=True)
untaggedTweets = untaggedTweets[['t','date']]
# untaggedTweets["date"] = untaggedTweets["date"].apply(lambda x: x.strftime('%I:%M:%S %p'))
untaggedTweets['t'] = untaggedTweets['t'].apply(preprocessor.clean)
untaggedTweets.rename(columns={'t':'Tweet Text', 'date':'Date'}, inplace=True)

df.drop(['_id', 'plt','date', 'pln', 'acr/$date', 'uid', 'p', 'f'], axis=1, inplace=True)
totalList = []
for month in df.groupby(df.index.month):
	dailyList = []
	for day in month[1].groupby(month[1].index.day):
		dailyList.append(day[1])
	totalList.append(dailyList)

app.title = 'Savitr - Disaster Mapping using Twitter'

app.layout = html.Div([
	html.Link(
		rel='stylesheet',
		href='/styles.css'
	),
	html.Div([
		html.Div([
			html.P(id='total-rides', className="totalRides",style={"color": "black", "top":"140px"}),
			html.Div([html.P(id='total-rides-selection', className="totalRideSelection"),
			html.P(id='date-value', className="dateValue",style={"color": "black"})],style={"display": "hidden"}),
			html.Div([
				html.Div([
					html.H2("Savitr", style={'font-family': 'Dosis'}),
					html.P("Disaster mapping using Twitter", className="explanationParagraph twelve columns"),
				],style={"margin-bottom": "10px"}),

			html.Div([
				dcc.Input(
					id='search-bar',
					type='text',
					placeholder="Choose Situation(s), comma separated",
					className="bars",
					style={'width': '100%'}
				),
				],style={"margin-top": "10px","margin-bottom": "5px"}),

			dcc.Graph(id='map-graph'),

			html.H5('Select Date',style={"margin-top":"20px"}),

			html.Div([

				dcc.Dropdown(
				id='my-dropdown',
				options=[
					{'label': 'Sept 2017', 'value': 'Sept'},
					{'label': 'Oct 2017', 'value': 'Oct'}
				],
				value="Sept",
				placeholder="Please choose a month",
				className="month-picker"

			)],style={"margin-top": "10px","margin-bottom": "10px"}),

			html.Div([
				dcc.Dropdown(
					id='bar-selector',
					options=[
						{'label': '0:00', 'value': '0'},
						{'label': '1:00', 'value': '1'},
						{'label': '2:00', 'value': '2'},
						{'label': '3:00', 'value': '3'},
						{'label': '4:00', 'value': '4'},
						{'label': '5:00', 'value': '5'},
						{'label': '6:00', 'value': '6'},
						{'label': '7:00', 'value': '7'},
						{'label': '8:00', 'value': '8'},
						{'label': '9:00', 'value': '9'},
						{'label': '10:00', 'value': '10'},
						{'label': '11:00', 'value': '11'},
						{'label': '12:00', 'value': '12'},
						{'label': '13:00', 'value': '13'},
						{'label': '14:00', 'value': '14'},
						{'label': '15:00', 'value': '15'},
						{'label': '16:00', 'value': '16'},
						{'label': '17:00', 'value': '17'},
						{'label': '18:00', 'value': '18'},
						{'label': '19:00', 'value': '19'},
						{'label': '20:00', 'value': '20'},
						{'label': '21:00', 'value': '21'},
						{'label': '22:00', 'value': '22'},
						{'label': '23:00', 'value': '23'}
					],
					multi=True,
					placeholder="Select certain hours using \
								 the box-select/lasso tool or \
								 using the dropdown menu",
					className="bars"
				)],style={"display": "none"}),

			   dcc.Graph(id="histogram"),

			   html.Div([
				dcc.Slider(
					id="my-slider",
					min=1,
					step=1,
					value=1

				)],style={"padding": "20px"}),

				html.P("", id="popupAnnotation", className="popupAnnotation"),
				html.H4('Untagged tweets'),

				html.Div([
				dt.DataTable(
					rows=untaggedTweets.to_dict('records'),
					# optional - sets the order of columns
					columns=untaggedTweets.columns,
					filterable=True,
					sortable=True,
					id='datatable'
			   )],style={"margin-top": "10px","margin-bottom": "10px"}),

				html.Div([
					html.Hr(),
					html.P("Savitr - Disaster Mapping using Twitter", style={'text-align':'center'}),
					html.A('Scroll to top', href='/#'),
					html.Div([
						html.A('Source code and documentation', target='_blank', href='https://github.com/AvijitGhosh82/Savitr-Python'),
					], style={"float": "right"}),
					html.Hr(),
                ]),

			], className="graph twelve coluns"),
		], style={'margin': 'auto auto'}),
	], className="graphSlider ten columns offset-by-one"),
], style={"padding-top": "20px"})

def getValue(value):
	val = {
		'Sept': 30,
		'Oct': 13
	}[value]
	return val

def getValue_start(value):
	val = {
		'Sept': 12,
		'Oct': 1
	}[value]
	return val

def getIndex(value):
	if(value==None):
		return 0
	val = {
		'Sept': 0,
		'Oct': 1
	}[value]
	return val

def getClickIndex(value):
	if(value==None):
		return 0
	return value['points'][0]['x']

@app.callback(Output("datatable", "rows"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value')])
def update_datatable(value, slider_value):
	if value == 'Sept':
		month = 9
	else:
		month = 10
	slider_value = slider_value+getValue_start(value)-1
	return untaggedTweets[(untaggedTweets['Date'].dt.day == slider_value) & (untaggedTweets['Date'].dt.month == month)].to_dict('records')

@app.callback(Output("my-slider", "marks"),
			  [Input("my-dropdown", "value")])
def update_slider_ticks(value):
	marks = {}
	for i in range(getValue_start(value), getValue(value)+1, 1):
		if(i is getValue_start(value) or i is getValue(value)):
			marks.update({i-getValue_start(value)+1: '{} {}'.format(value, i)})
		else:
			marks.update({i-getValue_start(value)+1: '{}'.format(i)})
	return marks

@app.callback(Output("my-slider", "max"),
			  [Input("my-dropdown", "value")])
def update_slider_max(value):
	return getValue(value) - getValue_start(value) + 1

@app.callback(Output("bar-selector", "value"),
			  [Input("histogram", "selectedData")])
def update_bar_selector(value):
	holder = []
	if(value is None or len(value) is 0):
		return holder
	for x in value['points']:
		holder.append(str(int(x['x'])))
	return holder

@app.callback(Output("total-rides", "children"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value')])
def update_total_rides(value, slider_value):
	return ("Total # of tweets: {:,d}"
			.format(len(totalList[getIndex(value)][slider_value-1])))

@app.callback(Output("total-rides-selection", "children"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value'),
			   Input('bar-selector', 'value')])
def update_total_rides_selection(value, slider_value, selection):
	if(selection is None or len(selection) is 0):
		return ""
	totalInSelction = 0
	for x in selection:
		totalInSelction += len(totalList[getIndex(value)]
										[slider_value-1]
										[totalList[getIndex(value)]
												[slider_value-1].index.hour == int(x)])
	return ("Total tweets in selection: {:,d}"
			.format(totalInSelction))

@app.callback(Output("date-value", "children"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value'),
			   Input("bar-selector", "value")])
def update_date(value, slider_value, selection):
	holder = []
	if(value is None or selection is None or len(selection) is 24
	   or len(selection) is 0):
		slider_value = slider_value+getValue_start(value)-1
		return (value, " ", slider_value, "")

	for x in selection:
		holder.append(int(x))
	holder.sort()

	if(holder[len(holder)-1]-holder[0]+2 == len(holder)+1 and len(holder) > 2):
		return (value, " ", slider_value, " - showing hour(s): ",
				holder[0], "-", holder[len(holder)-1])

	x = ""
	for h in holder:
		if(holder.index(h) == (len(holder)-1)):
			x += str(h)
		else:
			x += str(h) + ", "
	return (value, " ", slider_value, " - showing hour(s): ", x)

@app.callback(Output("histogram", "selectedData"),
			  [Input("my-dropdown", "value")])
def clear_selection(value):
	if(value is None or len(value) is 0):
		return None

@app.callback(Output("popupAnnotation", "children"),
			  [Input("bar-selector", "value")])
def clear_selection(value):
	if(value is None or len(value) is 0):
		# return "Select any of the bars to section data by time"
		return ""
	else:
		return ""

def get_selection(value, slider_value, selection):
	xVal = []
	yVal = []
	xSelected = []

	colorVal = ["#F4EC15", "#DAF017", "#BBEC19", "#9DE81B", "#80E41D", "#66E01F",
				"#4CDC20", "#34D822", "#24D249", "#25D042", "#26CC58", "#28C86D",
				"#29C481", "#2AC093", "#2BBCA4", "#2BB5B8", "#2C99B4", "#2D7EB0",
				"#2D65AC", "#2E4EA4", "#2E38A4", "#3B2FA0", "#4E2F9C", "#603099"]

	if(selection is not None):
		for x in selection:
			xSelected.append(int(x))
	for i in range(0, 24, 1):
		if i in xSelected and len(xSelected) < 24:
			colorVal[i] = ('#FFFFFF')
		xVal.append(i)
		yVal.append(len(totalList[getIndex(value)][slider_value-1]
					[totalList[getIndex(value)][slider_value-1].index.hour == i]))

	return [np.array(xVal), np.array(yVal), np.array(xSelected),
			np.array(colorVal)]

@app.callback(Output("histogram", "figure"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value'),
			   Input("bar-selector", "value")])
def update_histogram(value, slider_value, selection):

	[xVal, yVal, xSelected, colorVal] = get_selection(value, slider_value,
													  selection)

	layout = go.Layout(
		bargap=0.01,
		bargroupgap=0,
		barmode='group',
		margin=Margin(l=10, r=0, t=0, b=30),
		showlegend=False,
		plot_bgcolor='#ffffff',
		paper_bgcolor='rgb(66, 134, 244, 0)',
		height=250,
		dragmode="select",
		xaxis=dict(
			range=[-0.5, 23.5],
			showgrid=False,
			nticks=25,
			fixedrange=True,
			ticksuffix=":00"
		),
		yaxis=dict(
			range=[0, max(yVal)+max(yVal)/4],
			showticklabels=False,
			showgrid=False,
			fixedrange=True,
			rangemode='nonnegative',
			zeroline='hidden'
		),
		annotations=[
			dict(x=xi, y=yi,
				 text=str(yi),
				 xanchor='center',
				 yanchor='bottom',
				 showarrow=False,
				 font=dict(
					color='black'
				 ),
				 ) for xi, yi in zip(xVal, yVal)],
	)

	return go.Figure(
		   data=Data([
				go.Bar(
					x=xVal,
					y=yVal,
					marker=dict(
						color=colorVal
					),
					hoverinfo="x"
				),
				go.Scatter(
					opacity=0,
					x=xVal,
					y=yVal/2,
					hoverinfo="none",
					mode='markers',
					marker=Marker(
						color='rgb(66, 134, 244, 0)',
						symbol="square",
						size=40
					),
					visible=True
				)
			]), layout=layout)

def get_lat_lon_color(selectedData, value, slider_value):
	listStr = "totalList[getIndex(value)][slider_value-1]"
	if(selectedData is None or len(selectedData) is 0):
		return listStr
	elif(int(selectedData[len(selectedData)-1])-int(selectedData[0])+2 == len(selectedData)+1 and len(selectedData) > 2):
		listStr += "[(totalList[getIndex(value)][slider_value-1].index.hour>"+str(int(selectedData[0]))+") & \
					(totalList[getIndex(value)][slider_value-1].index.hour<" + str(int(selectedData[len(selectedData)-1]))+")]"
	else:
		listStr += "["
		for point in selectedData:
			if (selectedData.index(point) is not len(selectedData)-1):
				listStr += "(totalList[getIndex(value)][slider_value-1].index.hour==" + str(int(point)) + ") | "
			else:
				listStr += "(totalList[getIndex(value)][slider_value-1].index.hour==" + str(int(point)) + ")]"

	return listStr

@app.callback(Output("map-graph", "figure"),
			  [Input("my-dropdown", "value"), Input('my-slider', 'value'),
			  Input("bar-selector", "value"), Input("search-bar", "value")],
			  [State('map-graph', 'relayoutData')])
def update_graph(value, slider_value, selectedData, searchBarInput, prevLayout):
	mapControls = None
	zoom = 4.0
	latInitial = 21.146633
	lonInitial = 79.088860
	bearing = 0
	listStr = get_lat_lon_color(selectedData, value, slider_value)
	listsOfStr = eval(listStr)['t'].tolist()
	new_list = []
	listStr_lat = eval(listStr)['Lat'].tolist()
	new_lat = []
	listStr_lon = eval(listStr)['Lon'].tolist()
	new_lon = []
	listStr_index = eval(listStr).index.hour
	to_del = []
	if (not searchBarInput):
		# new_list = listsOfStr
		# new_lon = listStr_lon
		# new_lat = listStr_lat
		for i in range(len(listsOfStr)):
			a = listsOfStr[i]
			a = preprocessor.clean(a)
			fl = True
			for c in curate:
				if c in a.lower():
					fl = False
			if fl==True:
				new_list.append(a)
				new_lat.append(listStr_lat[i])
				new_lon.append(listStr_lon[i])
			else:
				to_del.append(i)
	else:
		searchBarInput = searchBarInput.decode('utf-8')
		searchBarInput = searchBarInput.lower()
		inputs = searchBarInput.split(',')
		for i in range(len(listsOfStr)):
			a = listsOfStr[i]
			if any(situation in a.lower() for situation in inputs):
				# clean out tweets: remove urls, reserved words
				a = preprocessor.clean(a)
				fl = True
				for c in curate:
					if c in a.lower():
						fl = False
				if fl==True:
					new_list.append(a)
					new_lat.append(listStr_lat[i])
					new_lon.append(listStr_lon[i])
			else:
				to_del.append(i)
	print len(new_list), 'tweets found! for', searchBarInput
	# print discarded tweets per character of search
	for i in to_del:
		print listsOfStr[i]
	if(len(to_del) > 0):
		np.delete(listStr_index, to_del, axis=0)
	if(prevLayout is not None and mapControls is not None and
	   'lock' in mapControls):
		zoom = float(prevLayout['mapbox']['zoom'])
		latInitial = float(prevLayout['mapbox']['center']['lat'])
		lonInitial = float(prevLayout['mapbox']['center']['lon'])
		bearing = float(prevLayout['mapbox']['bearing'])
	return go.Figure(
		data=Data([
			Scattermapbox(
				lat=new_lat,
				lon=new_lon,
				mode='markers',
				hoverinfo="text",
				text=new_list,
				marker=Marker(
					color=np.append(np.insert(listStr_index, 0, 0), 23),
					colorscale=[[0, "#F4EC15"], [0.04167, "#DAF017"],
								[0.0833, "#BBEC19"], [0.125, "9DE81B"],
								[0.1667, "#80E41D"], [0.2083, "#66E01F"],
								[0.25, "#4CDC20"], [0.292, "#34D822"],
								[0.333, "#24D249"], [0.375, "#25D042"],
								[0.4167, "#26CC58"], [0.4583, "#28C86D"],
								[0.50, "#29C481"], [0.54167, "#2AC093"],
								[0.5833, "#2BBCA4"],
								[1.0, "#613099"]],
					opacity=0.5,
					size=15,
					colorbar=dict(
						thicknessmode="fraction",
						title="Time of<br>Day",
						x=0.935,
						xpad=0,
						nticks=24,
						tickfont=dict(
							color='black'
						),
						titlefont=dict(
							color='black'
						),
						titleside='left'
						)
				),
			),
		]),
		layout=Layout(
			autosize=True,
			height=750,
			margin=Margin(l=0, r=0, t=0, b=0),
			showlegend=False,
			mapbox=dict(
				accesstoken=mapbox_access_token,
				center=dict(
					lat=latInitial, # 40.7272
					lon=lonInitial # -73.991251
				),
				style='light',
				bearing=bearing,
				zoom=zoom
			),
			updatemenus=[
				dict(
					buttons=([
						dict(
							args=[{
									'mapbox.zoom': 4,
									'mapbox.center.lon': '79.088860',
									'mapbox.center.lat': '21.146633',
									'mapbox.bearing': 0,
									'mapbox.style': 'light'
								}],
							label='Reset Zoom',
							method='relayout'
						)
					]),
					direction='left',
					pad={'r': 0, 't': 0, 'b': 0, 'l': 0},
					showactive=False,
					type='buttons',
					x=0.45,
					xanchor='left',
					yanchor='bottom',
					bgcolor='white',
					borderwidth=1,
					bordercolor="#6d6d6d",
					font=dict(
						color="black"
					),
					y=0.02
				)
			]
		)
	)

external_css = ["https://cdnjs.cloudflare.com/ajax/libs/skeleton/2.0.4/skeleton.min.css",
				"//fonts.googleapis.com/css?family=Raleway:400,300,600",
				"//fonts.googleapis.com/css?family=Dosis:Medium",
				"https://cdn.rawgit.com/plotly/dash-app-stylesheets/62f0eb4f1fadbefea64b2404493079bf848974e8/dash-uber-ride-demo.css",
				"https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css"]

for css in external_css:
	app.css.append_css({"external_url": css})

for js in external_js:
    app.scripts.append_script({'external_url': js})

@server.route('/favicon.ico')
def favicon():
	return send_from_directory(os.path.join(server.root_path, '/'),
									 'favicon.ico')

if __name__ == '__main__':
	app.run_server(debug=True)

max_issues_repo_name: evijit/Savitr-Python

max_forks_repo_name: evijit/Savitr-Python


text:
import warnings
import numpy as np
from operator import itemgetter
from scipy.linalg import cholesky, cho_solve, solve_triangular

from sklearn.base import clone
from sklearn.utils.validation import check_X_y, check_array
from sklearn.utils import check_random_state
import sklearn.gaussian_process as gp

class GaussianProcessRegressorWithCategorical(gp.GaussianProcessRegressor):
    """
    This is exactly the same as scikit-learn's GaussianProcessRegressor, but with
    two checks removed. Since the input may contain string values check_array and
    check_X_y would throw exceptions. These are commented in lines 30 and 132
    """
    def fit(self, X, y):
        """Fit Gaussian process regression model.

        Args:
            X: Training data
            y: Target values

        Returns:
            self : returns an instance of self.
        """
        self.kernel_ = clone(self.kernel)
        self._rng = check_random_state(self.random_state)

        #X, y = check_X_y(X, y, multi_output=True, y_numeric=True)

        # Normalize target value
        if self.normalize_y:
            self._y_train_mean = np.mean(y, axis=0)
            # demean y
            y = y - self._y_train_mean
        else:
            self._y_train_mean = np.zeros(1)

        if np.iterable(self.alpha) \
           and self.alpha.shape[0] != y.shape[0]:
            if self.alpha.shape[0] == 1:
                self.alpha = self.alpha[0]
            else:
                raise ValueError("alpha must be a scalar or an array"
                                 " with same number of entries as y.(%d != %d)"
                                 % (self.alpha.shape[0], y.shape[0]))

        self.X_train_ = np.copy(X) if self.copy_X_train else X
        self.y_train_ = np.copy(y) if self.copy_X_train else y

        if self.optimizer is not None and self.kernel_.n_dims > 0:
            # Choose hyperparameters based on maximizing the log-marginal
            # likelihood (potentially starting from several initial values)
            def obj_func(theta, eval_gradient=True):
                if eval_gradient:
                    lml, grad = self.log_marginal_likelihood(
                        theta, eval_gradient=True)
                    return -lml, -grad
                else:
                    return -self.log_marginal_likelihood(theta)

            # First optimize starting from theta specified in kernel
            optima = [(self._constrained_optimization(obj_func,
                                                      self.kernel_.theta,
                                                      self.kernel_.bounds))]

            # Additional runs are performed from log-uniform chosen initial
            # theta
            if self.n_restarts_optimizer > 0:
                if not np.isfinite(self.kernel_.bounds).all():
                    raise ValueError(
                        "Multiple optimizer restarts (n_restarts_optimizer>0) "
                        "requires that all bounds are finite.")
                bounds = self.kernel_.bounds
                for iteration in range(self.n_restarts_optimizer):
                    theta_initial = \
                        self._rng.uniform(bounds[:, 0], bounds[:, 1])
                    optima.append(
                        self._constrained_optimization(obj_func, theta_initial,
                                                       bounds))
            # Select result from run with minimal (negative) log-marginal
            # likelihood
            lml_values = list(map(itemgetter(1), optima))
            self.kernel_.theta = optima[np.argmin(lml_values)][0]
            self.log_marginal_likelihood_value_ = -np.min(lml_values)
        else:
            self.log_marginal_likelihood_value_ = \
                self.log_marginal_likelihood(self.kernel_.theta)

        # Precompute quantities required for predictions which are independent
        # of actual query points
        K = self.kernel_(self.X_train_)
        K[np.diag_indices_from(K)] += self.alpha
        try:
            self.L_ = cholesky(K, lower=True)  # Line 2
        except np.linalg.LinAlgError as exc:
            exc.args = ("The kernel, %s, is not returning a "
                        "positive definite matrix. Try gradually "
                        "increasing the 'alpha' parameter of your "
                        "GaussianProcessRegressor estimator."
                        % self.kernel_,) + exc.args
            raise
        self.alpha_ = cho_solve((self.L_, True), self.y_train_)  # Line 3
        return self

    def predict(self, X, return_std=False, return_cov=False):
        """Predict using the Gaussian process regression model
        We can also predict based on an unfitted model by using the GP prior.
        In addition to the mean of the predictive distribution, also its
        standard deviation (return_std=True) or covariance (return_cov=True).
        Note that at most one of the two can be requested.
        Args:
            X : Query points where the GP is evaluated
            return_std : If True, the standard-deviation of the predictive distribution at
                the query points is returned along with the mean. default is False
            return_cov : If True, the covariance of the joint predictive distribution at
                the query points is returned along with the mean. default is False

        Returns:
            y_mean: Mean of predictive distribution a query points
            y_std: Standard deviation of predictive distribution at query points.
                   Only returned when return_std is True.
            y_cov: Covariance of joint predictive distribution a query points.
                   Only returned when return_cov is True.
        """
        if return_std and return_cov:
            raise RuntimeError(
                "Not returning standard deviation of predictions when "
                "returning full covariance.")

        #X = check_array(X)

        if not hasattr(self, "X_train_"):  # Unfitted;predict based on GP prior
            kernel = self.kernel
            y_mean = np.zeros(X.shape[0])
            if return_cov:
                y_cov = kernel(X)
                return y_mean, y_cov
            elif return_std:
                y_var = kernel.diag(X)
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean
        else:  # Predict based on GP posterior
            K_trans = self.kernel_(X, self.X_train_)
            y_mean = K_trans.dot(self.alpha_)  # Line 4 (y_mean = f_star)
            y_mean = self._y_train_mean + y_mean  # undo normal.
            if return_cov:
                v = cho_solve((self.L_, True), K_trans.T)  # Line 5
                y_cov = self.kernel_(X) - K_trans.dot(v)  # Line 6
                return y_mean, y_cov
            elif return_std:
                # compute inverse K_inv of K based on its Cholesky
                # decomposition L and its inverse L_inv
                L_inv = solve_triangular(self.L_.T, np.eye(self.L_.shape[0]))
                K_inv = L_inv.dot(L_inv.T)
                # Compute variance of predictive distribution
                y_var = self.kernel_.diag(X)
                y_var -= np.einsum("ij,ij->i", np.dot(K_trans, K_inv), K_trans)

                # Check if any of the variances is negative because of
                # numerical issues. If yes: set the variance to 0.
                y_var_negative = y_var < 0
                if np.any(y_var_negative):
                    warnings.warn("Predicted variances smaller than 0. "
                                  "Setting those variances to 0.")
                    y_var[y_var_negative] = 0.0
                return y_mean, np.sqrt(y_var)
            else:
                return y_mean

ext: py

max_stars_repo_name: johannespetrat/OptML

reason: import numpy,from scipy


text:
import numpy as np

import torch
from torch import nn
from torch.utils.data import TensorDataset, DataLoader
from torch.optim import Adam
#from torch.distributions import half_normal, normal

from numpy import random
rs = random.RandomState(0)
import pandas as pd
#from distributed_model_flows import init_flow_model

import matplotlib.pyplot as plt
import matplotlib as mpl
from scipy.optimize import minimize

from obj import PyTorchObjective
from tqdm import tqdm
from sklearn.preprocessing import StandardScaler

from hessian import hessian
import pickle
#import pyro
#from pyro.distributions import Beta, Binomial, HalfCauchy, Normal, Pareto, Uniform, LogNormal, HalfNormal, MultivariateNormal
#from pyro.distributions.util import scalar_like
#from pyro.infer import MCMC, NUTS, Predictive
#from pyro.infer.mcmc.util import initialize_model, summary
#from pyro.util import ignore_experimental_warning

from torch.multiprocessing import Pool, set_start_method, freeze_support

from torch.distributions import constraints, normal, half_normal
from torch.distributions.utils import broadcast_all
#from pyro.distributions import TorchDistribution
#
#import arviz as az
#import make_cmdlogp_plot

import corner

#from pyro_torch_model_5D_linearA import gen_model, ObjectiveOpt, plot_marginal, plot_truth, plot_approx_posterior, add_cmd, optimize_model
from pyro_numerical_derivative import gen_model, ObjectiveOpt, plot_marginal, plot_truth, plot_approx_posterior, add_cmd, optimize_model, numerical_hessian

def plot1d():
    fig, ax = plt.subplots(2,2)
    ax = ax.flatten()
    ax[0].axvline(teststars['AV_0'][ind], lw=2, label='{:.2f}'.format(np.exp(theta_hat[0])))
    lnA = np.linspace(-3, 0, 100)
    ax[0].plot(np.exp(lnA), norm.pdf(lnA, theta_hat[0], np.sqrt(sigma_hat[0,0])))
    ax[0].axvline(np.exp(theta_hat[0]), lw=2, c='green')
    ax[0].set_xlabel('A')

    ax[1].axvline(teststars['distance_0'][ind]/1e3 , lw=2, label='{:.2f}'.format(np.exp(theta_hat[-1])))
    lnd = np.linspace(-3, 2, 100)
    ax[1].plot(np.exp(lnd), norm.pdf(lnd, theta_hat[3], np.sqrt(sigma_hat[-1,-1])))
    ax[1].axvline(np.exp(theta_hat[-1]), lw=2, c='green')
    ax[1].set_xlabel('d')

    m_hat = theta_hat[1+len(c_true):-1]
    m_sig = sigma_hat[1+len(c_true):-1,1+len(c_true):-1]
    for i, mag in enumerate(m_hat):
        ax[2].axvline(M_true[i], lw=2, label='{:.2f}'.format(m_hat[i]))
        M = np.linspace(-5, 10, 100)
        #M = np.linspace(theta_hat[1] - np.sqrt(sigma_hat[1,1])*3.,
        #                  theta_hat[1] + np.sqrt(sigma_hat[1,1])*3, 100)
        ax[2].axvline(m_hat[i], lw=2, c='green')
        ax[2].plot(M, norm.pdf(M, m_hat[i], np.sqrt(m_sig[i,i])))
        ax[2].set_xlabel('M')

    c_hat = theta_hat[1:1+len(c_true)]
    c_sig = sigma_hat[1:1+len(c_true)]
    for i, col in enumerate(c_hat):
        ax[3].axvline(c_true[i], lw=2, label='{:.2f}'.format(c_hat[i]))
        ax[3].axvline(c_hat[i], lw=2, c='green')
        c = np.linspace(-0.5, 2, 100)
        #c = np.linspace(theta_hat[2] - np.sqrt(sigma_hat[2,2])*3.,
        #                  theta_hat[2] + np.sqrt(sigma_hat[2,2])*3, 100)
        ax[3].plot(c, norm.pdf(c, c_hat[i], np.sqrt(c_sig[i, i])))
        ax[3].set_xlabel('c')
        #A_hat, A_sigma = meansig_lognorm(theta_hat, sigma_hat, 0)
        #d_hat, d_sigma = meansig_lognorm(theta_hat, sigma_hat, 3)
    for a in ax: a.legend()
    phot = samples['Mc']*ss.scale_ + ss.mean_
    ax[0].hist(samples['A'], bins=20, histtype='step', density=True)
    ax[1].hist(samples['d'], bins=20, histtype='step', density=True)
    ax[2].hist(phot[:,0, -1], bins=20, histtype='step', density=True)
    for i in range(len(c_true)):
        ax[3].hist(phot[:,0,i], bins=20, histtype='step', density=True)

    plt.savefig('mcmc_{}.pdf'.format(ind))
    #plt.show()
    plt.close(fig)

def plot2d(theta_true, theta_hat, sigma_hat):
    fig, axes = plt.subplots(7,7, figsize=(10,10))
    names = ['A', 'G-H', 'G-J', 'J-K', 'K-W1', 'G', 'd']
    plot_truth(theta_true, axes)
    plot_approx_posterior(theta_hat, sigma_hat, axes)
    axcmd = fig.add_axes([0.6, 0.6, 0.3, 0.3])
    axcmd.pcolormesh(xx_cmd, yy_cmd, cmd_logp, cmap=plt.get_cmap('Blues'))
    axcmd.axhline(M_true[0], c='red')
    axcmd.axvline(c_true[0], c='red')
    axcmd.scatter(c[0], M[0], c='black')
    axcmd.scatter(theta_hat[1], theta_hat[-2], c='green')
    axcmd.invert_yaxis()
    for i in range(7):
        for j in range(7):
            axes[i,j].set_xlim(theta_true[j]*0.5, theta_true[j]*1.5)
            axes[i,j].set_ylim(theta_true[i]*0.5, theta_true[i]*1.5)
            if j == 6: axes[i,j].set_xlabel(names[i])
            if i == 0: axes[i,j].set_ylabel(names[i])
            if j > i: axes[i,j].remove()
    plt.tight_layout()
    fig.savefig('mcmc2d_optimized_{}.pdf'.format(ind))

device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
torch.set_default_dtype(torch.float64)
scale=10
fraction=0.1
model = gen_model(scale=scale, fraction=fraction)
model.eval()
model.requires_grad_(False)
#hack to get sample working, log_probs needs to be called first
#0foo = torch.zeros(1, 5, device=device)

#model.log_probs(foo)

#from pyro.nn.module import to_pyro_module_
#to_pyro_module_(model)
model.requires_grad_(False)
numdatasamples = 1000000
ss = pickle.load(open(f'transform_nsamples{numdatasamples}.pkl','rb'))

#ss = pickle.load(open(f'transform_cycle_gauss_scale{scale}_frac{fraction}.pkl', 'rb')) #'transform.pkl','rb'))
#cmd_logp, xx_cmd, yy_cmd = make_cmdlogp_plot.make()
#
if __name__ == '__main__':
    from astropy.table import Table
    from astropy.io import ascii
    np.random.seed(222)
    torch.random
    freeze_support()

    from scipy.stats import norm
    device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
    torch.set_default_dtype(torch.float64)

    #samples = model.sample(num_samples=10000).detach().numpy()
    #corner.corner(samples)

    import time
    t = time.time()
    teststars = pd.read_csv('teststars_335.csv')
    teststars_nodust = pd.read_csv('teststars_335_nodust.csv')
    bands = ['G_mag', 'J_mag', 'H_mag', 'K_mag', 'W1_mag']

    dust_coeff = {}

    for b in bands:
        coeff_band = (teststars[b] - teststars_nodust[b])/teststars['AV_0']
        dust_coeff[b] = np.median(coeff_band)

    for pop in [teststars, teststars_nodust]:
        for band1, band2 in zip(bands[:-1], bands[1:]):
            b1 = band1.split('_')[0]
            b2 = band2.split('_')[0]
            pop['{0}-{1}'.format(b1, b2)] = pop[band1] - pop[band2]
            dust_coeff['{0}-{1}'.format(b1, b2)] = dust_coeff[band1] - dust_coeff[band2]

    use_bands = ['G_mag', 'J_mag', 'H_mag', 'K_mag', 'W1_mag']
    color_bands = ['G_mag', 'J_mag', 'H_mag', 'K_mag', 'W1_mag']
    color_keys = ['G-J', 'J-H', 'H-K', 'K-W1']
    absmag_keys = ['G_mag']

    use_cols = ['G-J', 'J-H', 'H-K', 'K-W1', 'G_mag']
    #use_cols = ['G_mag', 'bp_rp']#, 'BP', 'RP']
    cond_cols = [] # ['age', 'feh'] #, 'alpha', 'fB', 'gamma']

    dustco_c = [dust_coeff[b] for b in color_keys]
    dustco_m = [dust_coeff[b] for b in absmag_keys]
    print(dustco_c, dustco_m)

    fullpop = pd.read_pickle('fullpop.pkl')
    c0 = np.median(fullpop[color_keys])
    M0 = np.median(fullpop[absmag_keys])

    nstars = 335
    #nstars = 50
    arr = {'A_true': np.zeros(nstars),
           'd_true': np.zeros(nstars),
           'M_true': np.zeros(nstars),
           'c_true': np.zeros((nstars, len(color_keys))),
           'varpi_obs': np.zeros(nstars),
           'm_obs':np.zeros(nstars),
           'c_obs':np.zeros((nstars, len(color_keys))),
           'varpi_sig':np.zeros(nstars),
           'm_sig':np.zeros(nstars),
           'c_sig': np.zeros((nstars, len(color_keys))),
           'A': np.zeros(nstars),
           'd': np.zeros(nstars),
           'M': np.zeros(nstars),
           'c': np.zeros((nstars, len(color_keys))),
           'cov':np.zeros((nstars, len(color_keys) + len(absmag_keys) + 2, len(color_keys) + len(absmag_keys) + 2)),
           'hes':np.zeros((nstars, len(color_keys) + len(absmag_keys) + 2, len(color_keys) + len(absmag_keys) + 2)),
           'res':[]}

    for ind in range(nstars): #[3,  5,   7,   8,  11,  12,  13]: #4 range(nstars): # range(len(teststars['AV'])):
        #ind = 3
        t = time.time()
        distance = teststars['distance_0'][ind]/1e3
        A = teststars['AV_0'][ind]

        M_true = [teststars_nodust[m][ind] - 5.*np.log10(distance*1e3/10) for m in absmag_keys]
        c_true = [teststars_nodust[m][ind] for m in color_keys]

        M = [teststars[m][ind] - 5.*np.log10(distance*1e3/10) for m in absmag_keys]
        c = [teststars[c][ind] for c in color_keys]

        #M = [teststars_nodust[m][ind] - 5.*np.log10(distance*1e3/10) + A*dust_coeff[m] for m in absmag_keys]
        #c = [teststars_nodust[m][ind] + A*dust_coeff[m] for m in color_keys]

        sigmac = [0.01, 0.01, 0.001, 0.001]
        sigmam = [0.01] * len(M)
        sigmavarpi = 0.05

        chat = [color + rs.normal()*sigmacobs for color, sigmacobs in zip(c, sigmac)] #(scale=sigmac)
        varpihat = 1/distance + rs.normal()*sigmavarpi #(scale=sigmavarpi)
        mhat = [absm + 5*np.log10(distance*1e3/10.) + rs.normal()*sigmamobs for absm, sigmamobs in zip(M, sigmam)] #(scale=sigmam)

        arr['A_true'][ind] = A
        arr['d_true'][ind] = distance
        arr['M_true'][ind] = M_true[0]
        arr['c_true'][ind,:] = c_true
        arr['varpi_obs'][ind] = varpihat
        arr['m_obs'][ind] = mhat[0]
        arr['c_obs'][ind] = chat
        arr['varpi_sig'][ind]=sigmavarpi
        arr['m_sig'][ind] = sigmam[0]
        arr['c_sig'][ind,:] = sigmac

        #theta_0 for optimization [lnA, c, M, lnd]
        """
        theta_0 = [torch.log(torch.from_numpy(np.array(0.1) + 0.1 * np.random.randn(1))),
                             torch.from_numpy(np.array(chat)+ 0.1 * np.random.randn(len(color_keys))),
                             torch.from_numpy(np.array(mhat - 5*np.log10(1e2/varpi))+ 0.1 * np.random.randn(len(absmag_keys))),
                   torch.log(torch.from_numpy(np.array(1/varpihat)+ 0.1 * np.random.randn(1)))]
        """
        A0 = 0.1 #A
        theta_0 = [          torch.from_numpy(np.array(A0)), #torch.log()
                             torch.from_numpy(np.array(chat) - A0*np.array(dustco_c)),
                             torch.from_numpy(np.array(mhat - 5*np.log10(1e2/varpihat) - A0*np.array(dustco_m))),
                             torch.from_numpy(np.array(1/varpihat))]

        """
        theta_0 = [          torch.from_numpy(np.array(A) + 1e-1 * np.random.randn(1)), #torch.log()
                             torch.from_numpy(np.array(c_true)+ 1e-2 * np.random.randn(len(color_keys))),
                             torch.from_numpy(np.array(M_true)+ 1e-2 * np.random.randn(len(absmag_keys))),
                             torch.from_numpy(np.array(distance)+ 1e-1 * np.random.randn(1))]
        """
        print('###############################################################')
        print('###############################################################')
        print('###############################################################')
        print(f'theta_0 is: {theta_0}')
        print(f'A true is: {A}')

        #hessian = numerical_hessian(np.hstack([t.cpu().detach().numpy().astype(np.float64) for t in theta_hat]), chat, mhat, varpihat, sigmac, sigmam, sigmavarpi, dustco_c, dustco_m, ss)
        res, sigma_hat = optimize_model(theta_0, chat, mhat, varpihat, sigmac, sigmam, sigmavarpi, dustco_c, dustco_m, ss, ind)
        theta_hat = res.x
        hessian = numerical_hessian(np.hstack(theta_hat), chat, mhat, varpihat, sigmac, sigmam, sigmavarpi, dustco_c, dustco_m, ss)
        sigma_hat_num = np.linalg.inv(-1.*hessian)
        print(f'A true is {A:.4f}, A infered is {theta_hat[0]:.4f}, difference is {A - theta_hat[0]:.4f}')
        print(f'sigma A pytorch is {np.sqrt(sigma_hat[0,0]):.3f} and sigma A numerical is {np.sqrt(sigma_hat_num[0,0]):.3f}')
        #hessian = (theta_hat, chat, mhat, varpihat, sigmac, sigmam, sigmavarpi, dustco_c, dustco_m, ss)
        arr['res'].append(res)
        #import pdb; pdb.set_trace()
        arr['A'][ind] = theta_hat[0] #np.exp(theta_hat[0])
        arr['d'][ind] = theta_hat[-1] #np.exp(theta_hat[-1])
        arr['c'][ind,:] = theta_hat[1:1+len(color_keys)]
        arr['M'][ind] = theta_hat[-2]
        arr['cov'][ind, :,:] = sigma_hat_num
        arr['hes'][ind, :, :] = hessian

        theta_true = [A] + list(c_true) + list(M_true) + [distance]
        #theta_hat[0] = np.exp(theta_hat[0])
        #theta_hat[-1] = np.exp(theta_hat[-1])
        #plot1d()
        #print([np.sqrt(sigma_hat[i,i]) for i in range(7)])
        #plot2d(theta_true, theta_hat, sigma_hat)

    t = Table(arr)
    np.save('optvalues_{}_linearA_nograd.npy'.format(nstars), t)
    #import pdb; pdb.set_trace()
    #ascii.write(t, 'optvalues.dat', format='ascii')
        #import pdb; pdb.set_trace()
        #import pdb; pdb.set_trace()

    #if __name__ == '__main__': main()

max_stars_repo_name: andersdot/extinctions

max_issues_repo_name: andersdot/extinctions

max_forks_repo_name: andersdot/extinctions

reason: import numpy,from numpy,from scipy,from astropy


text:
import numpy as np
import PIL.Image

from os import listdir
from pickle import dump, load

from numpy import array
from numpy import argmax

from keras.applications.vgg16 import VGG16, preprocess_input
from keras.preprocessing.image import load_img, img_to_array
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.layers.merge import add
from keras.models import Model, load_model
from keras.layers import Input, Dense, LSTM, Embedding, Dropout
from keras.callbacks import ModelCheckpoint

from nltk.translate.bleu_score import corpus_bleu

# Importing the model
model= load_model("model_19.h5")
path_to_images= "#"

# Function for generating the features of the imahes
def extract_features(filename):
    model = VGG16()
    model.layers.pop()
    model = Model(inputs=model.inputs, outputs=model.layers[-1].output)
    image = load_img(filename, target_size=(224, 224))
    image = img_to_array(image)
    image = image.reshape((1, image.shape[0], image.shape[1], image.shape[2]))
    image = preprocess_input(image)
    feature = model.predict(image, verbose=0)
    return feature

# Function for generating descriptions
def generate_desc(model, tokenizer, photo, max_length):
    in_text = 'startseq'
    for i in range(max_length):
        sequence = tokenizer.texts_to_sequences([in_text])[0]
        sequence = pad_sequences([sequence], maxlen=max_length)
        yhat = model.predict([photo,sequence], verbose=0)
        yhat = argmax(yhat)
        word = word_for_id(yhat, tokenizer)
        if word is None:
            break
        in_text += ' ' + word
        if word == 'endseq':
            break
    return in_text

photo = extract_features(path)

description= generate_desc(model, tokenizer, photo, max_length)
print(description)

reason: import numpy,from numpy


text:
from tests import import_source_modules
import_source_modules()

import datetime as dt
import pandas as pd
import numpy as np
from pathlib import Path
from resources import OPERATION_BUY, OPERATION_SELL, OPERATION_REVERSE_SPLIT,\
    OPERATION_SPLIT, Portfolio, Order, Account, build_account_history,\
    create_new_account, generate_orders, load_account, set_account_path
'''
=============================================
Portfolio class tests
=============================================
'''

def test_create_empty():
    portfolio = Portfolio.create_empty()
    assert len(portfolio._data) == 0
    assert portfolio.get_position('A') == 0

def test_create_from_vectors():
    portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, 2])
    assert len(portfolio._data) == 2
    assert portfolio.get_position('A') == 1
    assert portfolio.get_position('B') == 2
    assert portfolio.assets == set(['A', 'B'])

def test_create_from_vectors_is_none_for_negative_qty():
    portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, -2])
    assert portfolio is None

def test_create_from_transactions():
    initial_portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, 2])
    transactions = [
        Order('C', 10, 1.5, OPERATION_BUY),
        Order('D', 5, 0.5, OPERATION_BUY),
    ]
    new_portfolio = Portfolio.create_from_transaction(initial_portfolio,
                                                      transactions)
    assert len(new_portfolio._data) == 4
    assert new_portfolio.assets == set(['A', 'B', 'C', 'D'])
    assert new_portfolio.get_position('A') == 1
    assert new_portfolio.get_position('B') == 2
    assert new_portfolio.get_position('C') == 10
    assert new_portfolio.get_position('D') == 5

    transactions = [
        Order('A', 1, 1.5, OPERATION_SELL),
        Order('D', 5, 0.5, OPERATION_SELL),
    ]
    new_portfolio = Portfolio.create_from_transaction(new_portfolio,
                                                      transactions)
    assert len(new_portfolio._data) == 2
    assert new_portfolio.assets == set(['B', 'C'])
    assert new_portfolio.get_position('A') == 0
    assert new_portfolio.get_position('B') == 2
    assert new_portfolio.get_position('C') == 10
    assert new_portfolio.get_position('D') == 0

    # Selling more than the current position
    new_portfolio = Portfolio.create_from_transaction(
        new_portfolio, [Order('B', 3, 0.1, OPERATION_SELL)])
    assert new_portfolio is None

    # Selling a stock that is not in the portfolio
    new_portfolio = Portfolio.create_from_transaction(
        Portfolio.create_empty(), [Order('A', 3, 0.1, OPERATION_SELL)])
    assert new_portfolio is None

def test_get_postion():
    initial_portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, 2])
    assert initial_portfolio.get_position('A') == 1
    assert initial_portfolio.get_position('B') == 2
    assert initial_portfolio.get_position('C') == 0

def test_fractional_postions():
    initial_portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, 2.5])
    assert not initial_portfolio.position_is_fractional('A')
    assert initial_portfolio.position_is_fractional('B')
    assert not initial_portfolio.position_is_fractional('C')
    print(initial_portfolio)

def test_modify_position():
    initial_portfolio = Portfolio.create_from_vectors(['A', 'B'], [1, 2.5])
    initial_portfolio.modify_position('A', 12)
    initial_portfolio.modify_position('C', 1)
    assert initial_portfolio.get_position('A') == 12
    assert initial_portfolio.get_position('B') == 2.5
    assert initial_portfolio.get_position('C') == 1

def test_add_portfolios():
    portfolio_a = Portfolio.create_from_vectors(['A', 'B'], [1, 2.5])
    portfolio_b = Portfolio.create_from_vectors(['A', 'C'], [1, 3])
    portfolio_c = Portfolio.create_from_vectors(['C'], [10])
    final_portfolio = (portfolio_a + portfolio_b) + portfolio_c
    assert final_portfolio.get_position('A') == 2
    assert final_portfolio.get_position('B') == 2.5
    assert final_portfolio.get_position('C') == 13

def test_add_none_portfolios():
    portfolio_a = Portfolio.create_from_vectors(['A', 'B'], [0, 2.5])
    assert portfolio_a + None is None

def test_add_invalid_portfolios():
    portfolio_a = Portfolio.create_from_vectors(['A', 'B'], [0, 2.5])
    portfolio_a._data['A']['qty'] = -1
    portfolio_b = Portfolio.create_from_vectors(['A', 'B'], [1, 2.5])
    assert portfolio_a + portfolio_b is None

def test_str_repr():
    portfolio_a = Portfolio.create_from_vectors(['A', 'B'], [0, 2.5])
    assert str(portfolio_a) == f'{"A":8s}{0}\n{"B":8s}{2.5}\n'
    assert portfolio_a.__str__() == portfolio_a.__repr__()

'''
=============================================
Order class tests
=============================================
'''

def test_order_str():
    order = Order('A', 1, 1.0, OPERATION_BUY)
    assert f"{order}" == "Buy A : 1 : 1.0000"
    order = Order('A', 1, 1.0001, OPERATION_BUY)
    assert f"{order}" == "Buy A : 1 : 1.0001"
    order = Order('A', 1, 1.00005, OPERATION_BUY)
    assert f"{order}" == "Buy A : 1 : 1.0001"
    assert order.__str__() == order.__repr__()

'''
=============================================
Account class tests
=============================================
'''

def test_new_account():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    assert account.last_transaction <= dt.datetime(2021, 10, 31, 9, 30)
    assert len(account.portfolio.assets) == 0

def test_account_deposit():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    deposit_date = dt.datetime(2021, 12, 12, 10, 10)
    assert account.deposit(deposit_date, 1_000)
    assert account.cash_onhand == 1_000

def test_account_withdraw():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    deposit_date = dt.datetime(2021, 12, 12, 10, 10)
    account.deposit(deposit_date, 1_000)
    withdraw_date = dt.datetime(2021, 12, 12, 10, 14)

    assert not account.withdraw(withdraw_date, 10_000)
    assert account.cash_onhand == 1_000

    assert account.withdraw(withdraw_date, 100)
    assert account.cash_onhand == 900

def test_account_update():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    account.deposit(dt.datetime(2021, 12, 13, 10, 10), 1_000)

    update_status = account.update_account(dt.datetime(2021, 12, 17, 9, 35),
                                           orders=[
                                               Order('ABC', 100, 1.2,
                                                     OPERATION_BUY),
                                               Order('XYZ', 1, 453.2,
                                                     OPERATION_BUY)
                                           ])

    assert update_status
    assert account.portfolio.get_position('ABC') == 100
    assert account.portfolio.get_position('XYZ') == 1
    assert account.cash_onhand == 1_000 - 1.2 * 100 - 453.2

def test_account_update_sell():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    account.deposit(dt.datetime(2021, 12, 12, 10, 10), 1_000)
    account.update_account(dt.datetime(2021, 12, 13, 9, 30),
                           orders=[
                               Order('ABC', 100, 1.2, OPERATION_BUY),
                               Order('XYZ', 1000, 0.2, OPERATION_BUY)
                           ])
    initial_cash_onhand = account.cash_onhand

    update_status = account.update_account(dt.datetime(2021, 12, 17, 9, 30),
                                           orders=[
                                               Order('ABC', 10, 2.2,
                                                     OPERATION_BUY),
                                               Order('XYZ', 10, 1.0,
                                                     OPERATION_SELL)
                                           ])
    assert update_status
    assert account.portfolio.get_position('ABC') == 110
    assert account.portfolio.get_position('XYZ') == 990
    assert account.cash_onhand == initial_cash_onhand - 10 * 2.2 + 10 * 1.0

def test_account_update_sells_missing_asset():
    account = Account('holder', dt.datetime(2021, 10, 31, 9, 30))
    account.deposit(dt.datetime(2021, 12, 12, 10, 10), 1_000)
    account.update_account(dt.datetime(2021, 12, 13, 9, 30),
                           orders=[Order('ABC', 100, 1.2, OPERATION_BUY)])
    initial_cash_onhand = account.cash_onhand

    update_status = account.update_account(dt.datetime(2021, 12, 17, 9, 30),
                                           orders=[
                                               Order('ABC', 10, 2.2,
                                                     OPERATION_BUY),
                                               Order('XYZ', 10, 2.2,
                                                     OPERATION_SELL)
                                           ])
    assert not update_status
    assert account.cash_onhand == initial_cash_onhand
    assert account.portfolio.get_position('ABC') == 100
    assert account.portfolio.get_position('XYZ') == 0

def test_account_operations_history():
    set_account_path(Path(__file__).parent / 'test_data/')
    account = create_new_account('holder_a', dt.datetime(2021, 12, 12))
    account.update_account(dt.datetime(2021, 12, 13, 9, 30),
                           orders=[Order('ABC', 1, 123, OPERATION_BUY)])
    account.update_account(dt.datetime(2021, 12, 15, 9, 30),
                           orders=[
                               Order('ABC', 0.5, 123, OPERATION_SELL),
                               Order('XYZ', 2, 456, OPERATION_BUY),
                           ])
    account.update_account(dt.datetime(2021, 12, 16, 9, 30),
                           orders=[Order('XYZ', 1, 450, OPERATION_SELL)])

    history = account.operations_history()
    assert len(history) == 4
    assert history[0][1] == 123
    assert history[1][1] == 2 * 456
    assert history[2][1] == -0.5 * 123
    assert history[3][1] == -450

    account = load_account('holder_a')
    assert account.holder == 'holder_a'

    account = load_account('holder_b')
    assert account is None

def test_load_account():
    set_account_path(Path(__file__).parent / 'test_data/')
    create_new_account('holder_a', dt.datetime.now)

    account = load_account('holder_a')
    assert account.holder == 'holder_a'

    account = load_account('holder_b')
    assert account is None

def test_generate_orders():
    old_portfolio = Portfolio.create_from_vectors(['A', 'B', 'D'], [1, 2, 4])
    new_portfolio = Portfolio.create_from_vectors(['A', 'C', 'D'], [10, 2, 4])

    orders = generate_orders(old_portfolio,
                             new_portfolio,
                             prices={
                                 "A": 10.3,
                                 "B": 31.4,
                                 "C": 0.341,
                                 "D": 1.7
                             })
    assert len(orders) == 3

    assert orders[0].ticker == "A"
    assert orders[0].qty == 9
    assert orders[0].price == 10.3
    assert orders[0].operation_type == OPERATION_BUY

    assert orders[1].ticker == "B"
    assert orders[1].qty == 2
    assert orders[1].price == 31.4
    assert orders[1].operation_type == OPERATION_SELL

    assert orders[2].ticker == "C"
    assert orders[2].qty == 2
    assert orders[2].price == 0.341
    assert orders[2].operation_type == OPERATION_BUY

def test_generate_orders_missing_prices():
    old_portfolio = Portfolio.create_from_vectors(['A', 'B', 'D'], [1, 2, 4])
    new_portfolio = Portfolio.create_from_vectors(['A', 'C', 'D'], [10, 2, 3])

    orders = generate_orders(old_portfolio,
                             new_portfolio,
                             prices={
                                 "A": 10.3,
                                 "B": 31.4,
                                 "C": 0.341
                             })
    # Asset D is missing from the price dictionary
    assert orders is None

def test_build_history():
    class DataManagerMock:
        def get_prices(_, assets):
            df = pd.DataFrame(data={
                'A': [1, 1, 1, 10, 11],
                'B': [3, 4, 5, 2, 3],
                'C': [100, 100, 110, 100, 100],
            },
                              index=[
                                  dt.datetime(2021, 12, 13),
                                  dt.datetime(2021, 12, 14),
                                  dt.datetime(2021, 12, 15),
                                  dt.datetime(2021, 12, 16),
                                  dt.datetime(2021, 12, 17),
                              ])
            return df[assets]

    data_mocker = DataManagerMock()
    portfolios = {
        dt.datetime(2021, 12, 13):
        Portfolio.create_from_vectors(['A'], [10]),
        dt.datetime(2021, 12, 15):
        Portfolio.create_from_vectors(['A', 'C'], [10, 1]),
    }
    history_dates, account_history = build_account_history(
        portfolios, data_mocker)

    assert history_dates[0].day == 13
    assert history_dates[1].day == 14
    assert history_dates[2].day == 15
    assert history_dates[3].day == 15
    assert history_dates[4].day == 16

    assert account_history == [10, 10, 10, 10 + 110, 100 + 100, 110 + 100]

def test_split():
    account = Account('holder', dt.datetime(2021, 11, 10, 9, 30))
    account.deposit(dt.datetime(2021, 11, 11, 10, 10), 1_000)
    account.update_account(dt.datetime(2021, 11, 12, 9, 30),
                           orders=[
                               Order('ABC', 100, 0.2, OPERATION_BUY),
                               Order('XYZ', 1000, 0.2, OPERATION_BUY)
                           ])
    account.update_account(dt.datetime(2021, 11, 13, 9, 30),
                           orders=[
                               Order('ABC', 10, 1.2, OPERATION_BUY),
                           ])
    initial_cash_onhand = account.cash_onhand

    account.split('ABC', 2, 220, 0.6, dt.datetime(2021, 11, 14, 9, 30))

    portfolio_1 = account.portfolios[dt.datetime(2021, 11, 12, 9, 30)]
    assert portfolio_1.get_position('ABC') == 200
    assert portfolio_1.get_position('XYZ') == 1000
    portfolio_2 = account.portfolios[dt.datetime(2021, 11, 13, 9, 30)]
    assert portfolio_2.get_position('ABC') == 220
    assert portfolio_2.get_position('XYZ') == 1000
    portfolio_3 = account.portfolios[dt.datetime(2021, 11, 14, 9, 30)]
    assert portfolio_3.get_position('ABC') == 220
    assert portfolio_3.get_position('XYZ') == 1000
    assert account.transactions.iloc[-1]['operation'] == OPERATION_SPLIT
    assert account.cash_onhand == initial_cash_onhand

def test_reverse_split():
    account = Account('holder', dt.datetime(2021, 11, 10, 9, 30))
    account.deposit(dt.datetime(2021, 11, 11, 10, 10), 1_000)
    account.update_account(dt.datetime(2021, 11, 12, 9, 30),
                           orders=[
                               Order('ABC', 100, 0.2, OPERATION_BUY),
                               Order('XYZ', 1000, 0.2, OPERATION_BUY)
                           ])
    account.update_account(dt.datetime(2021, 11, 13, 9, 30),
                           orders=[
                               Order('ABC', 10, 1.2, OPERATION_BUY),
                           ])
    initial_cash_onhand = account.cash_onhand

    account.split('ABC', 1 / 3, 36, 3.6, dt.datetime(2021, 11, 14, 9, 30))

    portfolio_1 = account.portfolios[dt.datetime(2021, 11, 12, 9, 30)]
    assert np.abs(portfolio_1.get_position('ABC') - 100 / 3) < 1e-6
    assert portfolio_1.get_position('XYZ') == 1000
    portfolio_2 = account.portfolios[dt.datetime(2021, 11, 13, 9, 30)]
    assert np.abs(portfolio_2.get_position('ABC') - 110 / 3) < 1e-6
    assert portfolio_2.get_position('XYZ') == 1000
    portfolio_3 = account.portfolios[dt.datetime(2021, 11, 14, 9, 30)]
    assert portfolio_3.get_position('ABC') == 36
    assert portfolio_3.get_position('XYZ') == 1000
    assert account.transactions.iloc[-1]['operation'] == OPERATION_REVERSE_SPLIT
    assert np.abs(account.cash_onhand - (initial_cash_onhand +
                                         (110 / 3 - 36) * 3.6)) < 1e-6

def test_double_split():
    account = Account('holder', dt.datetime(2021, 11, 10, 9, 30))
    account.deposit(dt.datetime(2021, 11, 11, 10, 10), 1_000)
    account.update_account(dt.datetime(2021, 11, 12, 9, 30),
                           orders=[
                               Order('ABC', 100, 0.2, OPERATION_BUY),
                               Order('XYZ', 1000, 0.2, OPERATION_BUY)
                           ])
    account.update_account(dt.datetime(2021, 11, 13, 9, 30),
                           orders=[
                               Order('ABC', 10, 1.2, OPERATION_BUY),
                           ])
    initial_cash_onhand = account.cash_onhand

    account.split('ABC', 2, 220, 0.6, dt.datetime(2021, 11, 14, 9, 30))
    account.split('ABC', 3, 660, 0.2, dt.datetime(2021, 11, 15, 9, 30))

    portfolio_1 = account.portfolios[dt.datetime(2021, 11, 12, 9, 30)]
    assert np.abs(portfolio_1.get_position('ABC') - 600) < 1e-6
    assert portfolio_1.get_position('XYZ') == 1000
    portfolio_2 = account.portfolios[dt.datetime(2021, 11, 13, 9, 30)]
    assert np.abs(portfolio_2.get_position('ABC') - 660) < 1e-6
    assert portfolio_2.get_position('XYZ') == 1000
    portfolio_3 = account.portfolios[dt.datetime(2021, 11, 14, 9, 30)]
    assert portfolio_3.get_position('ABC') == 660
    assert portfolio_3.get_position('XYZ') == 1000
    portfolio_4 = account.portfolios[dt.datetime(2021, 11, 15, 9, 30)]
    assert portfolio_4.get_position('ABC') == 660
    assert portfolio_4.get_position('XYZ') == 1000
    assert account.transactions.iloc[-1]['operation'] == OPERATION_SPLIT
    assert account.transactions.iloc[-1]['operation'] == OPERATION_SPLIT
    assert np.abs(account.cash_onhand - initial_cash_onhand) < 1e-6

max_stars_repo_name: dukduque/PortfolioManager

max_issues_repo_name: dukduque/PortfolioManager

max_forks_repo_name: dukduque/PortfolioManager

reason: import numpy


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.patches as mpatches
from pathlib import Path

sns.set_context("talk")

folder = Path('E:/ALBERTO/expe20-21/repetition_def/results')
file = 'dataT_T.csv'
filename = folder / file
data = pd.read_csv(filename)#read datafile

#%% CALCULATE PROBABILITY FIRST SEEN AS REPETITION

long = pd.melt(data, id_vars=['suje','DISTR'],
             value_vars=['2','3','4','5','6','7','8','9','10'],
        var_name='REPE', value_name='acc')#long format
long['acc'] = long['acc']/100# 0 to 1 values

long[['REPE','DISTR']] = long[['REPE','DISTR']].astype(int)
long.reset_index(inplace=True)
long = long.sort_values(['suje','DISTR','REPE'],axis=0)

# p-first= ((accuracy at repetition x) - (accuracy at repetition x-1)) /(1-accuracy at repetition x-1)
long['p_first'] = (long.acc-long.acc.shift(1)) / (1-long.acc.shift(1))
long['p_first'] = np.where(long['REPE']==2,long['acc'],long['p_first'])#p_first at Repetition 2 = accuracy at Repetition 2
long = long.replace([np.inf, -np.inf], np.nan)

#%% Figure
single = long[(long['DISTR']==1)&(long['REPE']<5)]
double = long[(long['DISTR']==2)&(long['REPE']<7)]
fig,ax = plt.subplots()
g = sns.pointplot(x = 'REPE', y = "p_first", ci = 68, color = 'slateblue',
                  ax = ax, data = single)

sns.pointplot(x = 'REPE', y = "p_first", color = 'r', linestyles = '--',
              ci=68, ax = ax, data = double)

sns.despine()
g.set(ylim = (0,1), xlim = (-0.5,4.2),
      xlabel = 'Number of Presentations',
      ylabel = 'Probability')
red_patch = mpatches.Patch(color='r', label='Two')
blue_patch = mpatches.Patch(color='slateblue', label='One')
plt.legend(handles = [blue_patch,red_patch],
            loc = 'upper right',
           frameon = False, title = 'No. of distractors')
fig.set_size_inches(7,6)
#fig.savefig('p_first.png',dpi = 450)


text:
import sympy

from einsteinpy.symbolic.tensor import Tensor

class MetricTensor(Tensor):
    """
    Class to define a metric tensor for a space-time
    """

    def __init__(self, arr, syms, config="ll"):
        """
        Constructor and Initializer

        Parameters
        ----------
        arr : ~sympy.tensor.array.dense_ndim_array.ImmutableDenseNDimArray or list
            Sympy Array or multi-dimensional list containing Sympy Expressions
        syms : tuple or list
            Tuple of crucial symbols dentoting time-axis, 1st, 2nd, and 3rd axis (t,x1,x2,x3)
        config : str
            Configuration of contravariant and covariant indices in tensor. 'u' for upper and 'l' for lower indices. Defaults to 'll'.

        Raises
        ------
        TypeError
            Raised when arr is not a list or sympy Array
        TypeError
            syms is not a list or tuple
        ValueError
            config has more or less than 2 indices

        """
        super(MetricTensor, self).__init__(arr, config=config)
        self._order = 2
        self._invmetric = None
        if isinstance(syms, (list, tuple)):
            self.syms = syms
            self.dims = len(self.syms)
        else:
            raise TypeError("syms should be a list or tuple")
        if not len(config) == self._order:
            raise ValueError("config should be of length {}".format(self._order))

    def change_config(self, newconfig="uu"):
        """
        Changes the index configuration(contravariant/covariant)

        Parameters
        ----------
        newconfig : str
            Specify the new configuration. Defaults to 'uu'

        Returns
        -------
        ~einsteinpy.symbolic.metric.MetricTensor
            New Metric with new configuration. Defaults to 'uu'

        Raises
        ------
        ValueError
            Raised when new configuration is not 'll' or 'uu'.
            This constraint is in place because we are dealing with Metric Tensor.

        """
        if newconfig == self.config:
            return self
        elif newconfig == "uu" or newconfig == "ll":
            inv_met = MetricTensor(
                sympy.simplify(sympy.Matrix(self.arr.tolist()).inv()).tolist(),
                self.syms,
                config=newconfig,
            )
            inv_met._invmetric = self
            return inv_met
        else:
            raise ValueError(
                "Configuration can't have one upper and one lower index in Metric Tensor."
            )

    def inv(self):
        """
        Returns the inverse of the Metric.
        Returns contravariant Metric if it is originally covariant or vice-versa.

        Returns
        -------
        ~einsteinpy.symbolic.metric.MetricTensor
            New Metric which is the inverse of original Metric.

        """
        if self._invmetric is None:
            if self.config == "ll":
                self._invmetric = self.change_config("uu")
            else:
                self._invmetric = self.change_config("ll")
        return self._invmetric

    def symbols(self):
        """
        Returns the symbols used for defining the time & spacial axis

        Returns
        -------
        tuple
            tuple containing (t,x1,x2,x3)

        """
        return self.syms

reason: import sympy


text:
import os
import numpy as np
import time

from tensorflow_privacy.privacy.analysis.rdp_accountant import compute_rdp, get_privacy_spent

from dph.core.modeling.model_factory import ModelFactory
from dph.core.parameters.parameters import Parameters, DPParameters
from dph.core.training.model_checkpoint import ModelCheckpoint
from dph.core.training.train_loop import TrainLoop
from dph.utils.saver import Saver
import tensorflow as tf
import pandas as pd

class Trainer:

    @staticmethod
    def train(train_dataset: tf.data.Dataset,
              val_dataset: tf.data.Dataset,
              p: Parameters,
              saver: Saver):
        """
        Training method.
        """
        strategy, model, optimizer, loss_dict, metrics = ModelFactory().build_model(p)

        if hasattr(p, "steps_per_epoch") and hasattr(p, "val_steps") and hasattr(p, "train_size"):
            train_dataset = train_dataset \
                .shuffle(p.train_size, seed=p.seed).batch(int(p.batch_size_train), drop_remainder=True)
            val_dataset = val_dataset.batch(p.batch_size_val)

            callbacks = []
            if p.patience > 0:
                callbacks.append(tf.keras.callbacks.EarlyStopping(patience=p.patience, restore_best_weights=True))
            callbacks.append(tf.keras.callbacks.TensorBoard(log_dir=saver.log_dir, histogram_freq=1))
            callbacks.append(tf.keras.callbacks.CSVLogger(filename=os.path.join(saver.log_dir, "log.csv")))
            if hasattr(p, "save_model_freq"):
                callbacks.append(ModelCheckpoint(freq=p.save_model_freq, saver=saver))

            # Save parameters
            tf.print(p.get_parameters())
            saver.save_dict(p.get_parameters(), "parameters")

            # Train and evaluate using tf.keras.Model.fit()
            start = time.time()

            grad_file_path = (saver.log_dir + "/gradient_norms.txt") if hasattr(p, "write_grads") else None
            train_loop = TrainLoop(strategy,
                                   model, optimizer, loss_dict,
                                   metrics,
                                   train_dataset, p.num_epochs, p.steps_per_epoch, p.batch_size_train,
                                   val_dataset,
                                   p.val_steps,
                                   callbacks=callbacks,
                                   grad_file_path=grad_file_path)

            train_loop.start()

            history = train_loop.history

            end = time.time()
            print("\n")

            if len(history.history) > 0:
                best_epoch = int(np.argmin(history.history['val_loss'])) + 1
                training = {"time": int(end - start),
                            "n_epochs": int(len(history.history['loss'])),
                            "best_epoch": best_epoch}

                if grad_file_path is not None:
                    gradients = pd.read_csv(grad_file_path)
                    gradients_median = float(gradients.median())
                    training['gradients_median'] = gradients_median

                if isinstance(p, DPParameters) and p.sigma > 0:
                    epsilon, delta = get_dp_guarantee(p.train_size, p.batch_size_train, best_epoch,
                                                      p.sigma / p.clipnorm)
                    training['epsilon'] = epsilon
                    training['delta'] = delta

                # Save training metrics
                saver.save_dict(training, "training")

            return model, history

def get_dp_guarantee(training_size, batch_size, n_epochs, noise_multiplier):
    orders = [1 + x / 10. for x in range(1, 100)] + list(range(12, 64))

    sampling_probability = batch_size / training_size
    steps = n_epochs * training_size // batch_size

    rdp = compute_rdp(q=sampling_probability,
                      noise_multiplier=noise_multiplier,
                      steps=steps,
                      orders=orders)

    delta = 1 / training_size
    epsilon = get_privacy_spent(orders, rdp, target_delta=delta)[0]

    return epsilon, delta


text:
#!flask/bin/python
from flask import Flask, request, jsonify
import json
import os
from flask_cors import CORS
import numpy as np
import cv2
import datetime
import time
import logging
from logging.handlers import RotatingFileHandler
from threading import Thread

app = Flask(__name__)
app.config['JSON_SORT_KEYS'] = False

cors = CORS(app, supports_credentials=True, resources={r"/api/*": {"origins": "*"}})

"""
    Date: 2-June-2019
    Build:0.1
    Developer: rajat.chaudhari@accenture.com
    Functionality: reads analog gauge images and gives current reading
    Reviewer:
"""

class VideoGet:
    """
    Class that continuously gets frames from a VideoCapture object
    with a dedicated thread. Used for live feed
    """

    def __init__(self, src=0):
        self.stream = cv2.VideoCapture(src)
        (self.grabbed, self.frame) = self.stream.read()
        self.stopped = False
    def start(self):
        Thread(target=self.get, args=()).start()
        #self.get()
        return self

    def get(self):
        while not self.stopped:
            if not self.grabbed:
                self.stop()
            else:
                (self.grabbed, self.frame) = self.stream.read()

    def stop(self):
        self.stopped = True
        self.stream.release()
        cv2.destroyAllWindows()

def dist_2_pts(x1, y1, x2, y2):
    #print np.sqrt((x2-x1)^2+(y2-y1)^2)
    return np.sqrt((x2 - x1)**2 + (y2 - y1)**2)

def avg_circles(circles, b):
    avg_x=0
    avg_y=0
    avg_r=0
    for i in range(b):
        #optional - average for multiple circles (can happen when a gauge is at a slight angle)
        avg_x = avg_x + circles[0][i][0]
        avg_y = avg_y + circles[0][i][1]
        avg_r = avg_r + circles[0][i][2]
    avg_x = int(avg_x/(b))
    avg_y = int(avg_y/(b))
    avg_r = int(avg_r/(b))
    return avg_x, avg_y, avg_r

def crop_cricle(image):
    """
    This is used to get the region of interest in cases of heavily tilted images, obsolete as it takes very long
    """
    """(h, w) = image.shape[:2]
    if (w>h) :
        #M = cv2.getRotationMatrix2D((h/2,w/2), 270, 1.0)
        #image = cv2.warpAffine(image, M, (w,w))
        image = imutils.rotate_bound(image, 90)"""

    height, width = image.shape[:2]
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)  #convert to gray

    circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1, 20, np.array([]), 80, 50, int(height*0.20), int(height*0.45))
    a, b, c = circles.shape
    x,y,r = avg_circles(circles, b)
    #draw center and circle
    #cv2.circle(image, (x, y), r, (0, 0, 255), 3, cv2.LINE_AA)  # draw circle
    #cv2.circle(image, (x, y), 2, (0, 255, 0), 3, cv2.LINE_AA)  # draw center of circle
    icircle=circles[0,:][0]
    cropSize = (r*2, r*2)
    nx=x-int(icircle[0]-cropSize[1]//2)
    ny=y-int(icircle[1]-cropSize[0]//2)

    cropCoords = (max(0, icircle[1]-cropSize[0]//2),min(image.shape[0], icircle[1]+cropSize[0]//2),
                      max(0, icircle[0]-cropSize[1]//2),min(image.shape[1], icircle[0]+cropSize[1]//2))
    crop_cimg = image[int(cropCoords[0]):int(cropCoords[1]),int(cropCoords[2]):int(cropCoords[3])]

    return crop_cimg,nx,ny,r

def alignImages(app,img1, img2,gauge_number, file_type,log_dir,debug):

    MAX_FEATURES = 1000#500 #
    GOOD_MATCH_PERCENT = 0.10#0.15 #
    #print("ALIGNING")
    grey1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
    grey2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)

    # Detect ORB features and compute descriptors.
    orb = cv2.ORB_create(MAX_FEATURES)
    impts1, d1 = orb.detectAndCompute(grey1, None)
    impts2, d2 = orb.detectAndCompute(grey2, None)

    # Match features.
    featurematcheObj = cv2.DescriptorMatcher_create(cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING)
    matchedfeatures = featurematcheObj.match(d1, d2, None)

    # Sort matches by score
    matchedfeatures.sort(key=lambda x: x.distance, reverse=False)

    # Remove not so good matches
    GoodMatches = int(len(matchedfeatures) * GOOD_MATCH_PERCENT)
    matchedfeatures = matchedfeatures[:GoodMatches]

    # Extract location of good matches
    points1 = np.zeros((len(matchedfeatures), 2), dtype=np.float32)
    points2 = np.zeros((len(matchedfeatures), 2), dtype=np.float32)

    for i, feature in enumerate(matchedfeatures):
      points1[i, :] = impts1[feature.queryIdx].pt
      points2[i, :] = impts2[feature.trainIdx].pt

    # Find homography
    h, mask = cv2.findHomography(points1, points2, cv2.RANSAC)

    # Use homography
    height, width, channels = img2.shape
    transformed = cv2.warpPerspective(img1, h, (width, height))
    if debug=="True":
        imgMatches = cv2.drawMatches(img1, impts1, img2, impts2, matchedfeatures, None)
        cv2.imwrite('%s/frame%s-align.%s' % (log_dir,gauge_number, file_type), transformed)
        cv2.imwrite('%s/frame%s-matches.%s' % (log_dir,gauge_number, file_type), imgMatches)
    return transformed, h

def calibrate_gauge(app,img,gauge_number, file_type,log_dir,debug):
    '''
        This function calibrates the range available to the dial as well as the
        units.  It works by first finding the center point and radius of the gauge.  Then it draws lines at intervals
        (separation) in degrees from values defined a config file. This assumes that the gauge is linear (as most probably are).
    '''
    height, width = img.shape[:2]
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)  #convert to gray
    #detect circles
    #restricting the search from 35-48% of the possible radii gives fairly good results across different samples.  Remember that
    #these are pixel values which correspond to the possible radii search range.
    circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1, 20, np.array([]), 200, 80, int(height*0.35), int(height*0.48))
    # average found circles, found it to be more accurate than trying to tune HoughCircles parameters to get just the right one
    a, b, c = circles.shape
    x,y,r = avg_circles(circles, b)
    #draw center and circle
    cv2.circle(img, (x, y), r, (0, 0, 255), 3, cv2.LINE_AA)
    cv2.circle(img, (x, y), 2, (0, 255, 0), 3, cv2.LINE_AA)
    if debug =="True":
        #for calibration, plot lines from center going out at every 10 degrees and add marker
        #for i from 0 to 36 (every 10 deg)
        '''
        goes through the motion of a circle and sets x and y values based on the set separation spacing.  Also adds text to each
        line.  These lines and text labels serve as the reference point for the user to enter
        NOTE: by default this approach sets 0/360 to be the +x axis (if the image has a cartesian grid in the middle), the addition
        (i+9) in the text offset rotates the labels by 90 degrees so 0/360 is at the bottom (-y in cartesian).  So this assumes the
        gauge is aligned in the image, but it can be adjusted by changing the value of 9 to something else.
        '''

        separation = 10.0 #in degrees
        interval = int(360 / separation)
        p1 = np.zeros((interval,2))  #set empty arrays
        p2 = np.zeros((interval,2))
        p_text = np.zeros((interval,2))
        for i in range(0,interval):
            for j in range(0,2):
                if (j%2==0):
                    p1[i][j] = x + 0.9 * r * np.cos(separation * i * 3.14 / 180) #point for lines
                else:
                    p1[i][j] = y + 0.9 * r * np.sin(separation * i * 3.14 / 180)
        text_offset_x = 10
        text_offset_y = 5
        for i in range(0, interval):
            for j in range(0, 2):
                if (j % 2 == 0):
                    p2[i][j] = x + r * np.cos(separation * i * 3.14 / 180)
                    p_text[i][j] = x - text_offset_x + 1.2 * r * np.cos((separation) * (i+27) * 3.14 / 180) #point for text labels, i+9 rotates the labels by 90 degrees
                else:
                    p2[i][j] = y + r * np.sin(separation * i * 3.14 / 180)
                    p_text[i][j] = y + text_offset_y + 1.2* r * np.sin((separation) * (i+27) * 3.14 / 180)  # point for text labels, i+9 rotates the labels by 90 degrees

        #add the lines and labels to the image
        for i in range(0,interval):
            cv2.line(img, (int(p1[i][0]), int(p1[i][1])), (int(p2[i][0]), int(p2[i][1])),(0, 255, 0), 3)
            cv2.putText(img, '%s' %(int(i*separation)), (int(p_text[i][0]), int(p_text[i][1])), cv2.FONT_HERSHEY_SIMPLEX, 0.3,(0,0,0),2,cv2.LINE_AA)
        #saving calibrated frames for logs to validate, debug and troubleshoot
        cv2.imwrite('%s/funcframe%s-calibration.%s' % (log_dir,gauge_number, file_type), img)
    return img, x, y, r

def get_current_value(app,img, min_angle, max_angle, min_value, max_value, x, y, r, gauge_number, file_type,log_dir,outputFolder,debug):
    new_value=0
    # preprocessing the image for finding lines
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    gray2 = cv2.bitwise_not(gray)
    binaryImg = cv2.adaptiveThreshold(gray2, 255, cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY, 15, -2)

    kernel = np.ones((5,5),np.uint8)
    opening = cv2.morphologyEx(binaryImg, cv2.MORPH_OPEN, kernel)
    medianBlur = cv2.medianBlur(binaryImg, 5)
    guassianBlur = cv2.GaussianBlur(medianBlur, (5, 5), 0)

    if debug=="True":
        #saving processed frames for logs to validate, debug and troubleshoot
        cv2.imwrite('%s/funcframe%s-tempdst1.%s' % (log_dir,gauge_number, file_type), guassianBlur)
    # find lines
    minLineLength = 200 # this is critical, can be taken from config file if AI people are handling this
    maxLineGap = 0

    lines = cv2.HoughLinesP(image=guassianBlur, rho=1, theta=np.pi / 180, threshold=100,minLineLength=minLineLength, maxLineGap=0)  # rho is set to 3 to detect more lines, easier to get more then filter them out later

    # define a range for lines
    final_line_list = []
    line_length=[]

    if lines is not None:
        for i in range(0, len(lines)):
            for x1, y1, x2, y2 in lines[i]:
                diff1 = dist_2_pts(x, y, x1, y1)  # x, y is center of circle
                diff2 = dist_2_pts(x, y, x2, y2)  # x, y is center of circle
                #set diff1 to be the smaller (closest to the center) of the two), makes the math easier
                if (diff1 > diff2):
                    temp = diff1
                    diff1 = diff2
                    diff2 = temp
                line_length.append(dist_2_pts(x1, y1, x2, y2))
                    # add to final list
                final_line_list.append([x1, y1, x2, y2])
                #cv2.line(img, (x1, y1), (x2, y2), (0, 255, 0), 2)
        longest=line_length.index(max(line_length))    #get the longest line
    # assuming the longest line is the best one
    if(len(final_line_list)>0):

        x1 = final_line_list[longest][0]
        y1 = final_line_list[longest][1]
        x2 = final_line_list[longest][2]
        y2 = final_line_list[longest][3]

        #find the farthest point from the center to be what is used to determine the angle
        dist_pt_0 = dist_2_pts(x, y, x1, y1)
        dist_pt_1 = dist_2_pts(x, y, x2, y2)
        if (dist_pt_0 > dist_pt_1):
            x_angle = x1 - x
            y_angle = y - y1
            cv2.line(img, (x, y), (x1, y1), (0, 255, 0), 2)
        else:
            x_angle = x2 - x
            y_angle = y - y2
            cv2.line(img, (x, y), (x2, y2), (0, 255, 0), 2)

        # take the arc tan of y/x to find the radian angle
        res = np.arctan(np.divide(float(y_angle), float(x_angle)))
        res = np.rad2deg(abs(res))

        if x_angle > 0 and y_angle > 0:  #in quadrant I
            final_angle = 90-res
        if x_angle > 0 and y_angle < 0:  #in quadrant II
            final_angle = 90+res
        if x_angle < 0 and y_angle < 0:  #in quadrant III
            final_angle = 270-res
        if x_angle < 0 and y_angle > 0:  #in quadrant IV
            final_angle = 270+res
        # calculates the reading based on the gauge range and calibrated values
        old_min = float(min_angle)
        old_max = float(max_angle)

        new_min = float(min_value)
        new_max = float(max_value)

        old_value = final_angle

        old_range = (old_max - old_min)
        new_range = (new_max - new_min)
        new_value = (((old_value - old_min) * new_range) / old_range) + new_min

        cv2.putText(img, str(round(new_value,2)), (0, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 0), lineType=cv2.LINE_AA)
        if debug =="True":
            cv2.imwrite('%s/frame%s-lines.%s' % (log_dir,gauge_number, 'jpg'), img) #get path from config

        #cv2.imwrite('%s/Output_%s.%s' % (outputFolder,gauge_number, 'jpg'), img) #prints all frames for video
        imgName=('Output_%s.%s' % (gauge_number, 'jpg'))
        return img,round(new_value,2),imgName #change response image between frame and transformed image here
    else:
        app.logger.info("No line was detected on the gauge")
        return 0,int(-1),0

def GetReadings(app,file,path,frames_to_process,refimg,min_angle,max_angle,min_value,max_value,units,typeoffile,logFolder,outputFolder,currTime,debug,camera):
    GaugeReadings=[]
    frameNumber=0
    #to make sure only 1st image and reading is written to output
    found=0
    iName="empty"

    try:
        #classifying input as looping pattern changes depending on type of input file
        if (file.split('.')[-1] in ['jpg','jpeg','png']):
            try:
                frame=cv2.imread(file)
                frameNumber+=1
                currTime = datetime.datetime.now().strftime("%d.%m.%y-%H.%M.%S")
                image_Id=str(frameNumber)+'_'+currTime
                image,h=alignImages(app,frame,refimg,frameNumber, typeoffile,logFolder,debug)
                fimg, xf, yf, rf = calibrate_gauge(app,image,frameNumber, typeoffile,logFolder,debug)
                resulting_frame,currentValue,imgName = get_current_value(app,fimg, min_angle, max_angle, min_value, max_value, xf, yf, rf, image_Id, typeoffile,logFolder,outputFolder,debug)
                if imgName !=str(0) and found==0:
                    cv2.imwrite('%s/Output_%s.%s' % (outputFolder,image_Id, 'jpg'), resulting_frame)
                    iName=imgName
                    found=1
                if currentValue > -1:
                        GaugeReadings.append(currentValue)
                        app.logger.info("Latest frame gauge reading : "+ str(currentValue)+" "+units)
            except Exception as e :
                app.logger.error("Failure in reading gauge value")
                app.logger.error(str(e))
        elif (file.split('.')[-1] in ['avi','wmv','mp4']):
            cap=cv2.VideoCapture(file)
            if cap.isOpened():
                ret, frame = cap.read()
            else:
                ret=False
            #if user gives frames to process more than the video length take minimum of the two
            length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
            while ret==True and frameNumber<int(min(int(frames_to_process),length)):
                try:
                    currTime = datetime.datetime.now().strftime("%d.%m.%y-%H.%M.%S")
                    frameNumber+=1
                    image_Id=str(frameNumber)+'_'+currTime
                    ret, frame = cap.read()

                    image,h=alignImages(app,frame,refimg,frameNumber, typeoffile,logFolder,debug)
                    fimg, xf, yf, rf = calibrate_gauge(app,image,frameNumber, typeoffile,logFolder,debug)
                    resulting_frame,currentValue,imgName = get_current_value(app,fimg, min_angle, max_angle, min_value, max_value, xf, yf, rf, image_Id, typeoffile,logFolder,outputFolder,debug)
                    #write only 1 reading to output folder
                    if imgName !=str(0) and found==0:
                        iName=imgName
                        cv2.imwrite('%s/Output_%s.%s' % (outputFolder,image_Id, 'jpg'), resulting_frame)
                        found=1

                    #dont consider bad frames as it interferes with average and min value calculations
                    if currentValue > -1:
                        GaugeReadings.append(currentValue)
                        app.logger.info("Latest frame gauge reading : "+ str(currentValue)+" "+units)
                except Exception as e :
                    app.logger.error("Failure in reading gauge value")
                    app.logger.error(str(e))
                    pass
            cap.release()
        elif file.split('\\')[-1]== "Live_Feed":
            record=0
            video_getter = VideoGet(camera).start()
            while (True) and frameNumber<int(frames_to_process):
                try:
                    frame = video_getter.frame
                    #special functionality to show input video feed to help user to adjust the position of gauge
                    #needs extra while loop to enable recording frames on press of 's' key
                    if debug == "True":
                        cv2.namedWindow("Input Frame")
                        cv2.imshow("Input Frame",frame)
                        if (cv2.waitKey(1) == ord("s")):
                            record=1
                        while (record==1) and frameNumber<int(frames_to_process):
                            frame = video_getter.frame
                            currTime = datetime.datetime.now().strftime("%d.%m.%y-%H.%M.%S")
                            frameNumber+=1
                            image_Id=str(frameNumber)+'_'+currTime
                            if debug == "True":
                                cv2.imwrite('%s/Input%s.%s' % (outputFolder,image_Id, typeoffile), frame)
                            image,h=alignImages(app,frame,refimg,frameNumber, typeoffile,logFolder,debug)
                            fimg, xf, yf, rf = calibrate_gauge(app,image,frameNumber, typeoffile,logFolder,debug)
                            resulting_frame,currentValue,imgName = get_current_value(app,fimg, min_angle, max_angle, min_value, max_value, xf, yf, rf, image_Id, typeoffile,logFolder,outputFolder,debug)
                            if imgName !=str(0) and found==0:
                                iName=imgName
                                found=1

                            if currentValue > -1:
                                GaugeReadings.append(currentValue)
                                app.logger.info("Latest frame gauge reading : "+ str(currentValue)+" "+units)
                        record=0
                    else:
                        currTime = datetime.datetime.now().strftime("%d.%m.%y-%H.%M.%S")
                        frameNumber+=1
                        image_Id=str(frameNumber)+'_'+currTime
                        if debug == "True":
                            cv2.imwrite('%s/Input%s.%s' % (outputFolder,image_Id, typeoffile), frame)
                        image,h=alignImages(app,frame,refimg,frameNumber, typeoffile,logFolder,debug)
                        fimg, xf, yf, rf = calibrate_gauge(app,image,frameNumber, typeoffile,logFolder,debug)
                        resulting_frame,currentValue,imgName = get_current_value(app,fimg, min_angle, max_angle, min_value, max_value, xf, yf, rf, image_Id, typeoffile,logFolder,outputFolder,debug)
                        if imgName !=str(0) and found==0:
                            iName=imgName
                            cv2.imwrite('%s/Output_%s.%s' % (outputFolder,image_Id, 'jpg'), resulting_frame)
                            found=1

                        if currentValue > -1:
                            GaugeReadings.append(currentValue)
                            app.logger.info("Latest frame gauge reading : "+ str(currentValue)+" "+units)
                except Exception as e :
                    app.logger.info("ERROR:Failure in reading gauge value")
                    app.logger.info(str(e))
                    pass
            video_getter.stop()
            del video_getter
    except Exception as e :
        app.logger.info("ERROR:Failure in reading gauge value")
        app.logger.info(str(e))

    return GaugeReadings,frameNumber,iName

def RemoveOutliers(readingList):
    GR=sorted(readingList)
    q1, q3= np.percentile(GR,[25,75])
    iqr = q3 - q1
    lower_bound = q1 -(1.5 * iqr)
    upper_bound = q3 +(1.5 * iqr)

    GaugeReadings=[item for item in readingList if item > lower_bound and item <upper_bound]
    return GaugeReadings

@app.route('/', methods=['GET','POST'])
def readDial():
    App_start_time = time.time()
    try:
        with open(os.getcwd()+'\\gaugeconfig.json','r') as json_config_file:
            gaugeconfig = json.load(json_config_file)
        with open(os.getcwd()+'\\code_view_mapping.json','r') as json_data_file:
            dataconfig = json.load(json_data_file)
    except Exception as e:
        print("failed to load config file, error is - %s" % e)
    try:
        if not (os.path.exists(os.getcwd()+gaugeconfig['log_dir'])):
            os.mkdir(os.getcwd()+gaugeconfig['log_dir'])
    except OSError as e:
        print ("Creation of the directory %s failed" % e)

    currTime = datetime.datetime.now().strftime("%d.%m.%y-%H.%M.%S")
    if not (os.path.exists(os.getcwd()+gaugeconfig['log_dir']+currTime)):
        os.mkdir(os.getcwd()+gaugeconfig['log_dir']+currTime)
    loc_code = request.args.get("functional_loc_code")
    path = request.args.get("path")
    frames_to_process = request.args.get("frames_to_process")
    if frames_to_process is None:
        frames_to_process=gaugeconfig['frames_to_process']

    GaugeReadings=[]

    min_angle = gaugeconfig['calibration']['min_angle']
    max_angle = gaugeconfig['calibration']['max_angle']
    min_value = gaugeconfig['calibration']['min_value']
    max_value = gaugeconfig['calibration']['max_value']
    units = gaugeconfig['calibration']['units']
    typeoffile=gaugeconfig["type_of_image"]
    camera=gaugeconfig["camera"]
    debug=gaugeconfig['debug']
    outliers=gaugeconfig['remove_outliers']
    if (loc_code and path and frames_to_process and loc_code in dataconfig):

        try:
            if not (os.path.exists(path)):
                os.mkdir(path)
        except OSError:
            print ("Creation of the directory %s failed" % path)

        logFolder=os.getcwd()+gaugeconfig['log_dir']+currTime
        outputFolder=path
        refimg=cv2.imread(os.getcwd()+gaugeconfig['reference_image'])
        file=os.getcwd()+dataconfig[loc_code]

        log_file= logFolder+'\\GR_'+currTime+'.log'
        open(log_file, 'w').close()
        handler = RotatingFileHandler(log_file, maxBytes=1024*1024*100, backupCount=1)
        handler.setLevel(logging.INFO)
        formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
        handler.setFormatter(formatter)
        app.logger.setLevel(logging.INFO)
        app.logger.addHandler(handler)

        GaugeReadings,framesProcessed,imgName = GetReadings(app,file,path,frames_to_process,refimg,min_angle,max_angle,min_value,max_value,units,typeoffile,logFolder,outputFolder,currTime,debug,camera)
        now = datetime.datetime.now()
        OverallTime=time.time()-App_start_time
        if len(GaugeReadings)>0:

            if outliers=="True":
                GR=RemoveOutliers(GaugeReadings)
                Average=round(sum(GR)/len(GR),2)
            else:
                Average=round(sum(GaugeReadings)/len(GaugeReadings),2)

            minimum = min(GaugeReadings)
            maximum = max(GaugeReadings)
            reading=GaugeReadings[0]
            readingStatus=1
        else:
            Average=0
            minimum=0
            maximum=0
            reading="No Readings Captured"
            readingStatus=0
        date_time = now.strftime("%d/%m/%Y, %H:%M:%S")
        response = {
            "Date" : date_time,
            "Reading_Status" : readingStatus,
            "Measure" : {
                "First Frame Reading":reading,
                "Average" : round(Average,2),
                "Min" : round(minimum,2),
                "Max" :round(maximum,2),
            },
            "Frame_Details":
            {
            "Frames_Processed" : framesProcessed,
            "Frames_Success" : len(GaugeReadings),
            "Total Time":OverallTime
            },
            "Analysis_image_path" : str(os.getcwd())+"\\"+path+"\\"+imgName,
            }
    else:
        response = {"ERROR":"Inavlid Payload or invalid parameter values. Please pass the query parameters functional_loc_code, path and frames_to_process and their expected values."}
        app.logger.info("ERROR:Invalid Payload or invalid parameter values")
    return jsonify(response)

if __name__ == '__main__':
    app.run()

max_stars_repo_name: RajatChaudhari/AnalogReader

max_issues_repo_name: RajatChaudhari/AnalogReader

max_forks_repo_name: RajatChaudhari/AnalogReader


text:
#!/usr/bin/python3

import os, pickle
import pandas as pd
import numpy as np

from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import ParameterSampler

import sys
sys.path.append("../..")
import utilities

def optimize_hyperparameters(param_list, train_set, val_set, max_iter=500):

	op_score_val, op_fit = 0, 0

	for params in param_list:

		print("-" * 75)
		print("   ".join(["{} {}".format(k.upper(), v) for k, v in params.items()]))
		print("-" * 75 + "\n")

		# Specify the classifier with the current hyperparameter combination
		classifier = OneVsRestClassifier(LogisticRegression(penalty=params["penalty"], C=params["C"], fit_intercept=params["fit_intercept"],
														max_iter=max_iter, tol=1e-10, solver="liblinear", random_state=42))

		# Fit the classifier on the training set
		classifier.fit(train_set[0], train_set[1])

		# Evaluate on the validation set
		preds_val = classifier.predict_proba(val_set[0])
		score_val = roc_auc_score(val_set[1], preds_val, average="macro")
		print("\n   Validation Set ROC-AUC {:6.4f}\n".format(score_val))

		# Update outputs if this model is the best so far
		if score_val > op_score_val:
			print("   Best so far!\n")
			op_score_val = score_val
			op_fit = classifier

	return op_fit

def train_classifier(framework, direction, suffix="", clf="", dtm_version=190325, in_path="../", out_path=""):

	fit_file = "{}fits/{}_{}.p".format(out_path, framework, direction)
	if not os.path.isfile(fit_file):

		# Load the data splits
		splits = {}
		for split in ["train", "validation"]:
			splits[split] = [int(pmid.strip()) for pmid in open("{}../data/splits/{}.txt".format(in_path, split), "r").readlines()]

		# Load the activation coordinate and text data
		act_bin = utilities.load_coordinates(path="{}../data".format(in_path))
		dtm_bin = utilities.load_doc_term_matrix(version=dtm_version, binarize=True, path="{}../data".format(in_path))

		# Score the texts using the framework
		lists, circuits = utilities.load_framework(framework, suffix=suffix, clf=clf, path="{}../ontology".format(in_path))
		scores = utilities.score_lists(lists, dtm_bin)

		# Specify the hyperparameters for the randomized grid search
		param_grid = {"penalty": ["l1", "l2"],
				  	  "C": [0.001, 0.01, 0.1, 1, 10, 100, 1000],
					  "fit_intercept": [True, False]}
		param_list = list(ParameterSampler(param_grid, n_iter=28, random_state=42))
		max_iter = 1000

		# Split the train set into batches and load the validation set as a batch
		if direction == "forward":
			train_set = [scores.loc[splits["train"]], act_bin.loc[splits["train"]]]
			val_set = [scores.loc[splits["validation"]], act_bin.loc[splits["validation"]]]

		elif direction == "reverse":
			train_set = [act_bin.loc[splits["train"]], scores.loc[splits["train"]]]
			val_set = [act_bin.loc[splits["validation"]], scores.loc[splits["validation"]]]

		# Search for the optimal hyperparameter combination
		op_fit = optimize_hyperparameters(param_list, train_set, val_set, max_iter=max_iter)

		# Export the optimized results
		pickle.dump(op_fit, open(fit_file, "wb"), protocol=2)

# train_classifier("data-driven", "forward", clf="_lr", dtm_version=190325)
# train_classifier("data-driven", "reverse", clf="_lr", dtm_version=190325)
# train_classifier("rdoc", "forward", suffix="_opsim", dtm_version=190325)
# train_classifier("rdoc", "reverse", suffix="_opsim", dtm_version=190325)
# train_classifier("dsm", "forward", suffix="_opsim", dtm_version=190325)
# train_classifier("dsm", "reverse", suffix="_opsim", dtm_version=190325)

max_stars_repo_name: ehbeam/neuro-knowledge-engine

max_issues_repo_name: YifeiCAO/neuro-knowledge-engine

max_forks_repo_name: YifeiCAO/neuro-knowledge-engine


text:
# -*- coding: utf-8 -*-
"""
computes the ACF coefficients of a time domain signal

  Args:
    x: audio signal
    iBlockLength: block length in samples
    iHopLength: hop length in samples
    f_s: sample rate of audio data (unused)
    eta: index (or vector of indices) of coeff result

  Returns:
    vacf autocorrelation coefficient
    t time stamp
"""

import numpy as np
import pyACA

def FeatureTimeAcfCoeff(x, iBlockLength, iHopLength, f_s, eta=19):

    # create blocks
    xBlocks = pyACA.ToolBlockAudio(x, iBlockLength, iHopLength)

    # number of results
    iNumOfBlocks = xBlocks.shape[0]
    if (np.isscalar(eta)):
        iNumOfResultsPerBlock = 1
    else:
        iNumOfResultsPerBlock = eta.size

    # compute time stamps
    t = (np.arange(0, iNumOfBlocks) * iHopLength + (iBlockLength / 2)) / f_s

    # allocate memory
    vacf = np.zeros([iNumOfResultsPerBlock, iNumOfBlocks])

    for n, block in enumerate(xBlocks):
        # calculate the acf
        if not block.sum():
            vacf[np.arange(0, iNumOfResultsPerBlock), n] = np.zeros(iNumOfResultsPerBlock)
            continue
        else:
            afCorr = np.correlate(block, block, "full") / np.dot(block, block)

        # find the coefficients specified in eta
        vacf[np.arange(0, iNumOfResultsPerBlock), n] = afCorr[iBlockLength + eta]

    return vacf, t

max_stars_repo_name: ruohoruotsi/pyACA

max_issues_repo_name: ruohoruotsi/pyACA

max_forks_repo_name: ruohoruotsi/pyACA