text:
# AUTOGENERATED! DO NOT EDIT! File to edit: notebooks/03_Map_Basics_Intake_and_Operations.ipynb (unless otherwise specified).

__all__ = ['workWithGeometryData', 'readInGeometryData', 'map_points']

# Cell
# @title Run: Import Modules

# These imports will handle everything
import os
import sys
import csv
import numpy as np
import pandas as pd
import pyproj
from pyproj import Proj, transform
# conda install -c conda-forge proj4
from shapely.geometry import LineString
# from shapely import wkb
# https://pypi.org/project/geopy/
import folium

# In case file is KML, enable support
import fiona
fiona.drvsupport.supported_drivers['kml'] = 'rw'
fiona.drvsupport.supported_drivers['KML'] = 'rw'

import psycopg2

# Cell
import pandas as pd
import geopandas as gpd
from geopandas import GeoDataFrame
from shapely.geometry import Point
from shapely.wkt import loads
from geopy.geocoders import Nominatim
from IPython.display import clear_output
from folium import plugins
from folium.plugins import TimeSliderChoropleth
from folium.plugins import MarkerCluster

# Cell
from dataplay import intaker
from dataplay import merge

# Cell
#
# Work With Geometry Data
# Description: geomSummary, getPointsInPolygons, getPolygonOnPoints, mapPointsInPolygons, getCentroids
#
def workWithGeometryData(method=False, df=False, polys=False, ptsCoordCol=False, polygonsCoordCol=False, polyColorCol=False, polygonsLabel='polyOnPoint', pntsClr='red', polysClr='white', interactive=False):
  def geomSummary(df): return type(df), df.crs, df.columns;
  def getCentroid(df, col): return df[col].representative_point() # df['geometry'].centroid

  # To 'import' a script you wrote, map its filepath into the sys
  def getPolygonOnPoints(pts, polygons, ptsCoordCol, polygonsCoordCol, polygonsLabel, interactive):
      count = 0
      # We're going to keep a list of how many points we find.
      boundaries = []

      # Loop over polygons with index i.
      for i, pt in pts.iterrows():
          # print('Searching for point within Geom:', pt )
          # Only one Label is accepted.
          poly_on_this_point = []
          # Now loop over all polygons with index j.
          for j, poly in polygons.iterrows():
              if poly[polygonsCoordCol].contains(pt[ptsCoordCol]):
                  # Then it's a hit! Add it to the list
                  poly_on_this_point.append(poly[polygonsLabel])
                  count = count + 1
                  # pts = pts.drop([j])

          # We could do all sorts, like grab a property of the
          # points, but let's just append the number of them.
          boundaries.append(poly_on_this_point)
          clear_output(wait=True)

      # Add the number of points for each poly to the dataframe.
      pts = pts.assign(CSA2010 = boundaries)
      if (interactive):
        print( 'Total Points: ', (pts.size / len(pts.columns) ) )
        print( 'Total Points in Polygons: ', count )
        print( 'Prcnt Points in Polygons: ', count / (pts.size / len(pts.columns) ) )
      return pts

  # To 'import' a script you wrote, map its filepath into the sys
  def getPointsInPolygons(pts, polygons, ptsCoordCol, polygonsCoordCol, interactive):
    count = 0
    total = pts.size / len(pts.columns)
    # We're going to keep a list of how many points we find.
    pts_in_polys = []

    # Loop over polygons with index i.
    for i, poly in polygons.iterrows():
        # print('Searching for point within Geom:', poly )
        # Keep a list of points in this poly
        pts_in_this_poly = 0

        # Now loop over all points with index j.
        for j, pt in pts.iterrows():
            if poly[polygonsCoordCol].contains(pt[ptsCoordCol]):
                # Then it's a hit! Add it to the list,
                pts_in_this_poly += 1
                # and drop it so we have less hunting. # pts = pts.drop([j])

        # We could do all sorts, like grab a property of the
        # points, but let's just append the number of them.
        pts_in_polys.append(pts_in_this_poly)
        if (interactive): print('Found this many points within the Geom:', pts_in_this_poly )
        count += pts_in_this_poly
        clear_output(wait=True)

    # Add the number of points for each poly to the dataframe.
    polygons['pointsinpolygon'] = gpd.GeoSeries(pts_in_polys)
    if (interactive):
      print( 'Total Points: ', total )
      print( 'Total Points in Polygons: ', count )
      print( 'Prcnt Points in Polygons: ', count / total )
    return polygons

  def mapPointsandPolygons(pnts, polys, pntsCl, polysClr, polyColorCol):
    print('mapPointsandPolygons');
    # We restrict to South America.
    ax = 1
    if polyColorCol:
      ax = polys.plot( column=polyColorCol, legend=True)
    else:
      ax = polys.plot( color=polysClr, edgecolor='black')

    # We can now plot our ``GeoDataFrame``.
    pnts.plot(ax=ax, color=pntsClr)

    return plt.show()

  if method=='summary': return geomSummary(df);
  if method=='ponp': return getPolygonOnPoints(df, polys, ptsCoordCol, polygonsCoordCol, polygonsLabel, interactive);
  if method=='pinp': return getPointsInPolygons(df, polys, ptsCoordCol, polygonsCoordCol, interactive);
  if method=='pandp': return mapPointsandPolygons(df, polys, pntsClr, polysClr, polyColorCol);
  if method=='centroid': return getCentroid(df, col);

# Cell

# reverseGeoCode, readFile, getGeoParams, main
def readInGeometryData(url=False, porg=False, geom=False, lat=False, lng=False, revgeocode=False,  save=False, in_crs=4326, out_crs=False):

  def reverseGeoCode(df, lat ):
    # STREET	CITY	STATE ZIP NAME
    # , format_string="%s, BALTIMORE MD"
    geometry = []
    geolocator = Nominatim(user_agent="my-application")
    for index, row in df.iterrows():
      try:
          geol = geolocator.geocode(row[lat], timeout=None)
          pnt = Point(geol.longitude, geol.latitude)
          geometry.append(pnt)
      except:
          geometry.append(Point(-76, 39) )
          print(row[lat]);
    return geometry

  def readFile(url, geom, lat, lng, revgeocode, in_crs, out_crs):
    df = False
    gdf = False
    ext = isinstance(url, pd.DataFrame)
    if ext: ext='csv'
    else: ext = url[-3:]

    #XLS
    # b16 = pd.read_excel('Jones.BirthsbyCensus2016.XLS', sheetname='Births')

    # The file extension is used to determine the appropriate import method.
    if ext in ['son', 'kml', 'shp', 'pgeojson']: gdf = gpd.read_file(url)
    if ext == 'csv':
      df = url if isinstance(url, pd.DataFrame) else pd.read_csv(url)
      # Read using Geom, Lat, Lat/Lng, revGeoCode
      if revgeocode=='y': df['geometry'] = reverseGeoCode(df, lat)
      elif geom: df['geometry'] = df[geom].apply(lambda x: loads( str(x) ))
      elif lat==lng: df['geometry'] = df[lat].apply(lambda x: loads( str(x) ))
      elif lat!=lng: df['geometry'] = gpd.points_from_xy(df[lng], df[lat]);

      gdf = GeoDataFrame(df, crs=in_crs, geometry='geometry') #crs=2248
      if not out_crs == in_crs: gdf = gdf.to_crs(epsg=out_crs)
    return gdf

  def getGeoParams(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs):
    addr=False

    if not url: url = input("Please enter the location of your dataset: " )
    # if url[-3:] == 'csv' :
    #   df = pd.read_csv(url,index_col=0,nrows=1)
    #   print(df.columns)

    # Geometries inside
    if geom and not (lat and lng): porg = 'g'
    # Point data inside
    elif not geom and lat or lng:
      porg = 'p';
      if not lat: lat = lng
      if not lng: lng = lat

    # If the P/G could not be infered...
    if not (porg in ['p', 'g']):
      if not revgeocode in ['y', 'n']: revgeocode = input("Do your records need reverse geocoding: (Enter: y/n') " )
      if revgeocode == 'y': porg = 'p'; lng = lat = input("Please enter the column name where the address is stored: " );
      elif revgeocode == 'n': porg = input("""Do the records in this dataset use (P)oints or (g)eometric polygons?: (Enter: 'p' or 'g') """ );
      else: return getGeoParams(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs);

      if porg=='p':
          if not lat: lat = input("Please enter the column name where the latitude coordinate is stored: " );
          if not lng: lng = input("Please enter the column name where the longitude cooridnate is stored: (Could be same as the lat) " );
      elif porg=='g':
        if not geom: geom = input("Please enter column name where the geometry data is stored: (*optional, skip if unkown)" );
      else: return getGeoParams(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs)

    if not out_crs: out_crs=in_crs

    return url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs

  # This function uses all the other functions
  def main(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs):

    # Check for missing values. retrieve them
    if (isinstance(url, pd.DataFrame)): print('Converting DF to GDF')
    elif (not (url and porg) ) or (
        not (porg == 'p' or porg == 'g') ) or (
        porg == 'g' and not geom) or (
        porg == 'p' and (not (lat and lng) ) ):
      return readInGeometryData( *getGeoParams(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs) );

    # print(f"RECIEVED url: {url}, \r\n porg: {porg}, \r\n geom: {geom}, \r\n lat: {lat}, \r\n lng: {lng}, \r\n revgeocode: {revgeocode}, \r\n in_crs: {in_crs}, \r\n out_crs: {out_crs}")

    # Quit if the Columns dont exist -> CSV Only
    # status = checkColumns(url, geom, lat, lng)
    # if status == False: print('A specified column does not exist'); return False;

    # Perform operation
    gdf = readFile(url, geom, lat, lng, revgeocode, in_crs, out_crs)

    # Tidy up

    # Save
    # if save: saveGeoData(gdf, url, fileName, driver='esri')

    return gdf

  return main(url, porg, geom, lat, lng, revgeocode, save, in_crs, out_crs)

# Cell
# draw_heatmap, cluster_points, plot_points,
def map_points(data, lat_col='POINT_Y', lon_col='POINT_X', zoom_start=11, plot_points=True, cluster_points=False,
               pt_radius=15, draw_heatmap=False, heat_map_weights_col=None, heat_map_weights_normalize=True,
               heat_map_radius=15, popup=False):
    """Creates a map given a dataframe of points. Can also produce a heatmap overlay

    Arg:
        df: dataframe containing points to maps
        lat_col: Column containing latitude (string)
        lon_col: Column containing longitude (string)
        zoom_start: Integer representing the initial zoom of the map
        plot_points: Add points to map (boolean)
        pt_radius: Size of each point
        draw_heatmap: Add heatmap to map (boolean)
        heat_map_weights_col: Column containing heatmap weights
        heat_map_weights_normalize: Normalize heatmap weights (boolean)
        heat_map_radius: Size of heatmap point

    Returns:
        folium map object
    """
    df = data.copy()
    ## center map in the middle of points center in
    middle_lat = df[lat_col].median()
    middle_lon = df[lon_col].median()

    curr_map = folium.Map(location=[middle_lat, middle_lon], zoom_start=zoom_start)

    # add points to map
    if plot_points:
      for _, row in df.iterrows():
        # print([row[lat_col], row[lon_col]], row[popup])
        folium.CircleMarker([row[lat_col], row[lon_col]],
          radius=pt_radius,
          popup=row[popup],
          fill_color="#3db7e4", # divvy color
        ).add_to(curr_map)
    if cluster_points:
      marker_cluster = MarkerCluster().add_to(curr_map)
      for index, row in df.iterrows():
        folium.Marker( location=[row[lat_col],row[lon_col]], popup=row[popup], icon=None ).add_to(marker_cluster)

    # add heatmap
    if draw_heatmap:
        # convert to (n, 2) or (n, 3) matrix format
        if heat_map_weights_col is None:
            stations = zip(df[lat_col], df[lon_col])
        else:
            # if we have to normalize
            if heat_map_weights_normalize:
                df[heat_map_weights_col] = \
                    df[heat_map_weights_col] / df[heat_map_weights_col].sum()

            stations = zip(df[lat_col], df[lon_col], df[heat_map_weights_col])

        curr_map.add_child(plugins.HeatMap(stations, radius=heat_map_radius))

    return curr_map

max_stars_repo_name: BNIA/dataplay

max_issues_repo_name: BNIA/dataplay

max_forks_repo_name: BNIA/dataplay


text:
#!/Users/tkirke/anaconda/bin/python
# -*- coding: utf-8 -*-
import re,sys,os,codecs
from time import sleep
from math import sqrt,log,pi,sin,atan2
import cmath
from scipy import signal,fft
import numpy, matplotlib
from lame  import *
matplotlib.use('qt4agg')
import matplotlib.pyplot as plt
from tone_est import *

show_plot = False
debug = False
count = 0
points = 44100/50

eps = 1e-3 # -60 dB for plots

sr = 44100
samples = 44100
audio_in = numpy.zeros(samples)

show_t = False
save_files = False
f0 = 50/44100.0
phase1 = 1; # radians
phase2 = 2; # radians
phase3 = 3; # radians
phase4 = 4; # radians
phase5 = 5; # radians

for i in xrange(samples):
    audio_in[i] = sin(2*pi*f0*i+phase1) - 0.000005*sin(2*pi*2*f0*i+phase2)
    #+ 0.25*sin(2*pi*3*f0*i+phase3) - 0.125*sin(2*pi*4*f0*i+phase4) + 0.0625*sin(2*pi*5*f0*i+phase5);

(a,f,p) = tone_est(audio_in,sr)
print "(a,f,p) = ",a,f,p
f = 50.0
#a = 1.0
adata = est_tone_phase_and_remove(audio_in,a,f,sr)

#plt.xlim([0,4000]);
plt.plot(audio_in, 'r')
plt.hold(True)
plt.grid(True)
plt.plot(adata, 'b')
plt.show()

ext: py

reason: import numpy,from scipy


text:
"""
.. module:: experiment
   :synopsis: Holding all experiment classes!
.. moduleauthor:: Klaus Kades
"""

import os
import time
from typing import List, Dict, Union
from abc import ABC, abstractmethod
from datetime import datetime
import json
from deprecated import deprecated
from sklearn.preprocessing import StandardScaler

import pickle
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split

from mtc.core.sentence import Sentence
from mtc.core.preprocessor import Preprocessor
from mtc.core.evaluator import Evaluator
from mtc.core.similarity_measures import SimilarityMeasures
from mtc.core.embeddings import DocumentEmbeddings
from mtc.core.embeddings import TokenEmbeddings
from mtc.settings import NLP_EXPERIMENT_PATH
from mtc.helpers.decorators import timeit
from mtc.helpers.file_management import load_sts_data

class Measuring:

    @staticmethod
    def preprocess_sentences(preprocessor, raw_texts: List[str]) -> List[Sentence]:
        preprocessed_texts = preprocessor.preprocess(raw_texts)
        return [Sentence(sen) for sen in preprocessed_texts]

    def __init__(self, measures: Union[SimilarityMeasures, List[SimilarityMeasures]], pickle_folder):

        self.measures = measures
        self.pickle_folder = pickle_folder

        self.X = []

        self.sentences_dict = dict()
        self.time_log = []

        if type(self.measures) is SimilarityMeasures:
            self.measures = [self.measures]

    def __call__(self):
        return self.X, self.sentences_dict['y'], np.array(self.sentences_dict['raw_sentences_a']), \
               np.array(self.sentences_dict['raw_sentences_b'])

    @timeit
    def create_features(self):
        self.X = np.array([], dtype=np.int64).reshape(len(self.sentences_dict['raw_sentences_a']), 0)
        if self.measures is not None:
            for measure in self.measures:
                sentences_a, sentences_b = self._get_prerpocessed_sentences(measure.pa_preprocessor)
                measure_values = []
                for sentence_a in sentences_a:
                    feature_to_add = sentence_a.get_sentence_property(measure.key_name)
                    if type(feature_to_add) is not list:
                        feature_to_add = [feature_to_add]
                    measure_values.append(feature_to_add)
                    #measure_values.append(sentence_a.get_sentence_property(measure.key_name))
                self.X = np.append(self.X, np.array(measure_values), axis=1)

        if self.X.size != 0:
            scaler = StandardScaler()
            self.X = scaler.fit_transform(self.X)

    @timeit
    def save_feature_matrix(self):
        pickle.dump(np.nan_to_num(self.X), open(os.path.join(self.pickle_folder, f'feature_matrix.p'), 'wb'))

    @timeit
    def load_feature_matrix(self):
        self.X = pickle.load(open(os.path.join(self.pickle_folder, f'feature_matrix.p'), 'rb'))

    @timeit
    def save_sentences_objects(self):
        pickle.dump(self.sentences_dict, open(os.path.join(self.pickle_folder, f'sentences_dict.p'), 'wb'))

    @timeit
    def load_sentences_objects(self):
        self.sentences_dict = pickle.load(open(os.path.join(self.pickle_folder, f'sentences_dict.p'), 'rb'))

    @timeit
    def set_sts_data_dict(self, sts_data_path):

        try:
            self.load_sentences_objects()
        except FileNotFoundError:
            print('No file was located')
            sts_data = load_sts_data(sts_data_path)
            if 'similarity_score' in sts_data:
                self.sentences_dict.update({
                    'raw_sentences_a': sts_data['raw_sentences_a'],
                    'raw_sentences_b': sts_data['raw_sentences_b'],
                    'y': np.array(sts_data['similarity_score'])
                })
            else:
                self.sentences_dict.update({
                    'raw_sentences_a': sts_data['raw_sentences_a'],
                    'raw_sentences_b': sts_data['raw_sentences_b'],
                    'y': np.arange(0, len(sts_data['raw_sentences_a']))
                })
            self.save_sentences_objects()

    @timeit
    def _get_prerpocessed_sentences(self, pa_preprocessor):
        name = str(pa_preprocessor)
        if name not in self.sentences_dict.keys():
            preprocessor = Preprocessor(*pa_preprocessor['args'], **pa_preprocessor['kwargs'])
            sentences_combined = self.preprocess_sentences(preprocessor, self.sentences_dict['raw_sentences_a'] +
                                                           self.sentences_dict['raw_sentences_b'])
            sentences_a = sentences_combined[:len(sentences_combined) // 2]
            sentences_b = sentences_combined[len(sentences_combined) // 2:]
            # sentences_a = self.preprocess_sentences(preprocessor, self.raw_sentences_a)
            # sentences_b = self.preprocess_sentences(preprocessor, self.raw_sentences_b)
            self.sentences_dict.update({
                name: {'sentences_a': sentences_a, 'sentences_b': sentences_b}
            })
        return self.sentences_dict[name]['sentences_a'], self.sentences_dict[name]['sentences_b']

    @timeit
    def measure(self, train_or_test):
        if self.measures is not None:
            for measure in self.measures:
                sentences_a, sentences_b = self._get_prerpocessed_sentences(measure.pa_preprocessor)
                measure.measure(sentences_a, sentences_b, train_or_test)

    def plot_correlation_matrix(self):
        import matplotlib.pyplot as plt
        import matplotlib
        key_names = [measure.key_name for measure in self.measures]

        for idx, key_name in enumerate(key_names):
            print(idx+2, key_name)

        features = np.concatenate((np.array([self.sentences_dict['y']]).reshape(-1, 1), self.X), axis=1)
        df = pd.DataFrame(features)
        #df.columns = ['ground truth'] + key_names
        plt.matshow(df.corr(method='pearson').abs())
        plt.colorbar()
        plt.show()
        plt.savefig('correlation.png')

    @timeit
    def save_measuring_config(self):

        # experiment_params
        experiment_params = dict()
        if self.measures is not None:
            experiment_params['measures'] = [measure.key_name for measure in self.measures]

        experiment_params['timings'] = self.time_log
        with open(os.path.join(self.pickle_folder, f'experiment_parameters.json'), 'w+') as fp:
            json.dump(experiment_params, fp)

class Training:

    def __init__(self,  estimator):

        self.estimator = estimator

    def __call__(self, X_train, y_train):
        self.estimator.fit(X_train, y_train)

    def save_training_properties(self, experiment_path,  pearson_training, pearson_test):

        # experiment_params
        training_params = dict()
        if hasattr(self.estimator, 'cv_results_'):
            training_params['estimator'] = {
                'cv_results': pd.DataFrame(self.estimator.cv_results_).to_dict(),
                'best_estimator': self.estimator.best_estimator_.get_params(),
                'best_score': self.estimator.best_score_,
                'best_params': self.estimator.best_params_
            }
        else:
            training_params['estimator'] = self.estimator.get_params()

        training_params['estimator'].update({'estimator_name': self.estimator.__class__.__name__})

        with open(os.path.join(f'{experiment_path}_training_parameters.json'), 'w+') as fp:
            json.dump(training_params, fp)

        if hasattr(self.estimator, 'cv_results_'):
            print(training_params['estimator']['best_params'])
            best_results = dict()
            best_results.update(training_params['estimator']['best_estimator'])
            best_results.update({'best_score': [training_params['estimator']['best_score']]})
            best_results.update(training_params['estimator']['best_params'])
            best_results.update({'pearson_training': [pearson_training],
                                 'pearson_test': [pearson_test]})

            best_results_name = os.path.join(NLP_EXPERIMENT_PATH, f'{str(self.estimator.estimator.__class__.__name__)}_estimator_list.csv')
            df_best_results = pd.DataFrame(best_results)
            if not os.path.isfile(best_results_name):
                df_best_results.to_csv(best_results_name, header=list(df_best_results.columns), index=False)
            else:  # else it exists so append without writing the header
                df_best_results.to_csv(best_results_name, mode='a', header=False, index=False)

class Predicting:

    def __init__(self, estimator):
        self.estimator = estimator

    def __call__(self, X_test):
        return self.estimator.predict(X_test)

class Evaluating:

    def __init__(self, evaluators: Union[Evaluator, List[Evaluator]]):

        self.evaluators = evaluators

        if type(self.evaluators) is Evaluator:
            self.evaluators = [self.evaluators]

    def __call__(self, y_test, y_test_predicted, test_index, rsa_a_test, rsa_b_test):
        if self.evaluators is not None:
            for evaluator in self.evaluators:
                evaluator.evaluate(y_test, y_test_predicted, test_index, rsa_a_test, rsa_b_test)

if __name__ == '__main__':
    pass

reason: import numpy


text:
# NLU Project
# Main file
# Description: The script creates a class for the model seq2seq

# Import site-packages
import tensorflow as tf
from models.basic_model import BasicModel
import numpy as np
import datetime
import os
import pandas as pd

class RNN_Model(BasicModel):
    # Build the graph
    def _build_graph(self):
        # Set random seed
        tf.set_random_seed(13)

        # Allocate to GPU
        with tf.device('/gpu:0'):
            # Load inputs
            self.input = tf.placeholder(tf.int32, [None, None], name="input")

            # The length of the sentences
            self.input_length = tf.placeholder(tf.int32, [None], name="input_length")

            # Load labels
            self.labels = tf.placeholder(tf.int32, [None], name="labels")

            # Define additional parameters
            self.global_step = tf.Variable(0, name="global_step", trainable=False)
            self.learning_rate = tf.Variable(self.learning_rate, trainable=False, dtype=tf.float32)
            self.learning_rate_decay_op = self.learning_rate.assign(self.learning_rate * self.learning_rate_decay_factor)

            # Add drouput
            self.keep_prob = tf.placeholder(tf.float32, name="keep_prob")

            # Load labels to compute loss
            self.max_sequence = tf.reduce_max(self.input_length, name = "max_sequence")

            # Load embedding
            emb_initial = tf.constant(self.emb, dtype=tf.float32)
            emb_pretrained = tf.Variable(emb_initial[4:, :], trainable=False)
            emb_train = tf.Variable(tf.random_uniform([4, self.embedding_dim], -0.1, 0.1))
            emb = tf.concat([emb_train, emb_pretrained], axis=0, name="emb")
            print('Printing embeddings tensor size: {}'.format(str(emb.get_shape())))

            # Embedding layer
            with tf.name_scope("embedding"):
                emb_words = tf.nn.embedding_lookup(emb, self.input)
            print('Printing input tensor size: {}'.format(str(self.input.get_shape())))
            print('Printing embeddings tensor size: {}'.format(str(emb_words.get_shape())))

            # Rnn layer - encoder
            with tf.name_scope("rnn-encoder"):
                lstm_encoder = tf.contrib.rnn.DropoutWrapper(tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(self.rnn_size) for _ in range(4)]),
                                                             output_keep_prob = self.keep_prob)
                outputs, final_state = tf.nn.dynamic_rnn(lstm_encoder, emb_words,
                                                             sequence_length = self.input_length, dtype=tf.float32)
                final_state = tf.squeeze(tf.concat(final_state, 0)[-1,:,:])
                print('Printing outputs tensor size: {}'.format(str(outputs.get_shape())))
                print('Printing final state tensor size: {}'.format(str(final_state.get_shape())))

            # Fully connected layer
            with tf.name_scope("rnn_layer"):
                # Parameters to calibrate
                W = tf.get_variable("W", [self.rnn_size, self.output_size], tf.float32,initializer=tf.contrib.layers.xavier_initializer())
                b = tf.get_variable("b", [self.output_size], tf.float32, initializer=tf.zeros_initializer())

                # Calculate logits and predictions
                logits = tf.nn.xw_plus_b(final_state, W, b, name ="mul_logits")
                self.predictions = tf.argmax(logits, 1, name ="predictions")
                print('Printing logits tensor size: {}'.format(str(logits.get_shape())))
                print('Printing predictions tensor size: {}'.format(str(self.predictions.get_shape())))

            # Soft-max layer
            with tf.name_scope("softmax"):
                self.losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.labels, logits=logits)
                self.loss = tf.reduce_mean(self.losses, name="mean_loss")

            # Calculate accuracy
            with tf.name_scope("accuracy"):
                correct_predictions = tf.equal(tf.reshape(tf.cast(self.predictions, tf.int32) , [-1]),
                                               self.labels, name="equal_predictions")
                self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32), name="sum_accuracy")

            # Optimizer
            tvars = tf.trainable_variables()
            with tf.name_scope('optimizer'):
                optimizer = tf.train.AdamOptimizer(self.learning_rate)
            grads, tvars = zip(*optimizer.compute_gradients(self.loss, tvars))
            grads, _ = tf.clip_by_global_norm(grads, self.grad_clip)
            self.train_op = optimizer.apply_gradients(zip(grads, tvars), global_step=self.global_step)

        # Call summary builder
        self._build_summary()

        # Call checkpoint builder
        self._save()

        # Initialize all variables
        self.session.run(tf.global_variables_initializer())
        self.session.graph.finalize()

        # Calculate number of parameters
        total_parameters = 0
        for variable in tf.trainable_variables():
            shape = variable.get_shape()
            variable_parametes = 1
            for dim in shape:
                variable_parametes *= dim.value
            total_parameters += variable_parametes
        print("Total training params: %.5fM" % (total_parameters / 1e6))

    def train(self):
        # Set up numpy seed
        np.random.seed(13)
        self.cum_loss = []

        # Train the model
        for n_epoch in range(self.num_epoch):
            # Create batches and shuffle
            self.avg_loss = []
            train_input, train_length_input, train_labels = \
                self.batchify(self.train_input, self.train_length_input, self.train_labels)

            # Train each batch
            for idx in range(len(train_input)):
                feed_dict = {self.input: train_input[idx],
                             self.input_length: train_length_input[idx],
                             self.labels: train_labels[idx,:],
                             self.keep_prob: self.keep_prob_dropout}
                _, step, summaries, loss_batch, accuracy_batch, = self.session.run(
                    [self.train_op, self.global_step, self.train_summary_op, self.loss, self.accuracy], feed_dict)
                if step % self.summary_every == 0:
                    time_str = datetime.datetime.now().isoformat()
                    print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, loss_batch, accuracy_batch))
                    self.train_summary_writer.add_summary(summaries, step)
                if step % self.save_every == 0:
                    path = self.saver.save(self.session, self.checkpoint_prefix, global_step=step)
                    print("Saved model checkpoint to {}\n".format(path))
                if step % self.evaluate_every == 0:
                    self.avg_loss.append(np.mean(np.array(self.cum_loss).flatten()))
                    print("Average loss: {}".format(self.avg_loss[-1]))
                    if len(self.avg_loss) > 3 and n_epoch > 1 and max(self.avg_loss[-3:-1]) < self.avg_loss[-1]:
                        self.session.run(self.learning_rate_decay_op)
                        print("Changing the learning rate")
                    self.cum_loss = []
                self.cum_loss.append(loss_batch)

            # Save per epoch
            path = self.saver.save(self.session, self.checkpoint_prefix, global_step=step)
            print("Saved model checkpoint to {}\n".format(path))

    def infer(self):
        # Set up the saver
        saver = tf.train.import_meta_graph("{}.meta".format(self.file_dir))

        # Load the saved meta graph and restore variables
        saver.restore(self.session, self.file_dir)
        print(self.graph.get_operations())
        print(self.graph.get_all_collection_keys())
        for v in tf.global_variables():
            print(v.name)
            print(v.get_shape())
        print("Graph restored and calculating perplexity:")

        # Load operations
        self.input = self.graph.get_operation_by_name("input").outputs[0]
        self.input_length = self.graph.get_operation_by_name("input_length").outputs[0]
        self.labels = self.graph.get_operation_by_name("labels").outputs[0]
        self.keep_prob = self.graph.get_operation_by_name("keep_prob").outputs[0]
        self.predictions = self.graph.get_operation_by_name("rnn_layer/predictions").outputs[0]

        # Compute predictions
        pred = []
        for idx in range(len(self.test_input)):
            # Setup the feed dictionary
            feed_dict = {self.input: self.test_input[idx],
                         self.input_length: self.test_length_input[idx].flatten(),
                         self.labels: np.ones((self.test_input[idx].shape[0])),
                         self.keep_prob: 1.0}

            # Run operation
            pred.extend(self.session.run(self.predictions, feed_dict))

        # Prepare data for submission
        pred = np.array([np.arange(1,len(pred)+1).tolist(), pred]).T
        pred = pd.DataFrame(pred).replace(to_replace=[0, "0"], value=-1)
        filename = "submission.csv"
        header = ["Id", "Prediction"]
        pd.DataFrame(pred).to_csv(filename, header = header, index = False)

max_stars_repo_name: oliverdelacruz/sentiment-analysis

max_issues_repo_name: oliverdelacruz/sentiment-analysis

max_forks_repo_name: oliverdelacruz/sentiment-analysis


text:
import torch
import torch.nn as nn
import torch.nn.functional as F

class BasicBlock(nn.Module):
    def __init__(self, c_in, c_out,is_downsample=False):
        super(BasicBlock,self).__init__()
        self.is_downsample = is_downsample
        if is_downsample:
            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=2, padding=1, bias=False)
        else:
            self.conv1 = nn.Conv2d(c_in, c_out, 3, stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(c_out)
        self.relu = nn.ReLU(True)
        self.conv2 = nn.Conv2d(c_out,c_out,3,stride=1,padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(c_out)
        if is_downsample:
            self.downsample = nn.Sequential(
                nn.Conv2d(c_in, c_out, 1, stride=2, bias=False),
                nn.BatchNorm2d(c_out)
            )
        elif c_in != c_out:
            self.downsample = nn.Sequential(
                nn.Conv2d(c_in, c_out, 1, stride=1, bias=False),
                nn.BatchNorm2d(c_out)
            )
            self.is_downsample = True

    def forward(self,x):
        y = self.conv1(x)
        y = self.bn1(y)
        y = self.relu(y)
        y = self.conv2(y)
        y = self.bn2(y)
        if self.is_downsample:
            x = self.downsample(x)
        return F.relu(x.add(y),True)

def make_layers(c_in,c_out,repeat_times, is_downsample=False):
    blocks = []
    for i in range(repeat_times):
        if i ==0:
            blocks += [BasicBlock(c_in,c_out, is_downsample=is_downsample),]
        else:
            blocks += [BasicBlock(c_out,c_out),]
    return nn.Sequential(*blocks)

class Net(nn.Module):
    def __init__(self, num_classes=751 ,reid=False): #num class sayısı market için 751
        super(Net,self).__init__()
        # 3 128 64
        self.conv = nn.Sequential(
            nn.Conv2d(3,64,3,stride=1,padding=1), #no stride, 1px padding
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            # nn.Conv2d(32,32,3,stride=1,padding=1),
            # nn.BatchNorm2d(32),
            # nn.ReLU(inplace=True),
            nn.MaxPool2d(3,2,padding=1),
        )
        # 32 64 32
        self.layer1 = make_layers(64,64,2,False)
        # 32 64 32
        self.layer2 = make_layers(64,128,2,True)
        # 64 32 16
        self.layer3 = make_layers(128,256,2,True)
        # 128 16 8
        self.layer4 = make_layers(256,512,2,True)
        # 256 8 4
        self.avgpool = nn.AvgPool2d((8,4),1)
        # 256 1 1
        self.reid = reid
        self.classifier = nn.Sequential(
            nn.Linear(512, 256),
            nn.BatchNorm1d(256),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(256, num_classes),
        )

    def forward(self, x):
        #print(x.shape)
        #show_image(x,'featureOrj')

        x = self.conv(x)
        #print(x.shape) #1,64,64,32
        #show_feature(x,4,'cov1')

        x = self.layer1(x)
        #print(x.shape) #1,64,64,32
        #show_feature(x,5,'cov2')

        x = self.layer2(x)
        #print(x.shape) #1,128,32,16
        #show_feature(x,5,'cov3')

        x = self.layer3(x)
        #print(x.shape) #1,256,16,8
        #show_feature(x,5,'cov3')

        x = self.layer4(x)
        #print(x.shape) #1,521,8,4
        #show_feature(x,5,'conv5')

        x = self.avgpool(x)
        #print(x.shape) #1,512,1,1
        #show_feature(x,512,'avgpool')

        x = x.view(x.size(0),-1)
        #print(x.shape) #1,512

        # B x 128
        if self.reid:
            x = x.div(x.norm(p=2,dim=1,keepdim=True)) #L2 normalization by dividing its length.
            #print(x.shape) #1,512
            #show_last_feature(x)
            return x
        # classifier
        x = self.classifier(x)
        print(x.shape)
        return x

#random tensorden img yazdırmak için, eğer girdi olarak resim değil rand sayı verildiyse
def show_image(inp,saveName):
    inp = inp.view(3,128,64) #img id siliniyor
    import matplotlib.pyplot as plt
    import numpy as np
    inp = inp.numpy().transpose((1, 2, 0))
    plt.imshow(inp)
    #plt.savefig(saveName+'.png')
    plt.show()
    plt.pause(.001)
    return

#son feature hariç feature map'leri kaydetmek için
def show_feature(inp,numberOfFeature,saveName):
    import matplotlib.pyplot as plt
    import numpy as np
    inp = inp.view(list(inp.size())[1:]) #img id siliniyor.
    inp = inp.detach().cpu()
    dataDir = dataPath+'/'+saveName
    if not os.path.isdir(dataDir):
        os.mkdir(dataDir)
    for i in range(numberOfFeature):
        feature = inp[i,:,:] #sıfırıncı feature
        #print(feature.size())

        plt.gray()
        plt.imshow(feature,extent=(0, feature.size(1), 0, feature.size(0))) #orjin noktası extent ile ayarlanıyor.
        plt.savefig(dataDir+'/'+str(i)+'.png')
    print(saveName," was saved into ",dataDir)
    return

def show_last_feature(feature):
    #print(feature)
    feature = torch.reshape(feature, (32, 16))
    feature = feature.detach().cpu()
    import matplotlib.pyplot as plt
    import numpy as np
    #print(feature.shape) #32,16

    dataDir = dataPath+'/'+'lastFeature'
    if not os.path.isdir(dataDir):
        os.mkdir(dataDir)

    plt.jet()
    plt.imshow(feature)
    plt.imshow(feature,extent=(0, feature.size(1), 0, feature.size(0))) #orjin noktası extent ile ayarlanıyor.
    plt.savefig(dataDir+'/feature.png')

if __name__ == '__main__':
    #tek bir resim için model çalıştırılıp feature'lar kaydediliyor.

    import os
    from PIL import Image
    import torchvision.transforms.functional as TF

    dataPath = 'features' #kaydedilecek feature'lar için klasör
    if not os.path.isdir(dataPath):
        os.mkdir(dataPath)

    #random sayılar ya da bir resim yüklenmeli
    #inputs = torch.randn(64,3,128,64)

    #tek bir resim
    image = Image.open('demo.jpg')
    inputs = TF.to_tensor(image)
    inputs.unsqueeze_(0) #3,x,y 'den 1,3,x,y 'ye
    #print(inputs.shape)

    #model bir kez çalıştırılıyor
    net = Net(reid=True)
    assert os.path.isfile("./checkpoint/ckpt.t7"), "Error: no checkpoint file found!"
    checkpoint = torch.load("./checkpoint/ckpt.t7", map_location=torch.device('cpu'))
    net_dict = checkpoint['net_dict']
    net.load_state_dict(net_dict, strict=False)
    net.eval() #dropout gibi optimizasyonları kapatmak için, net.train() ile geri açılıyor.
    device = "cuda:{}".format(args.gpu_id) if torch.cuda.is_available() and not args.no_cuda else "cpu"
    net.to(device)
    inputs = inputs.to(device)
    features = net(inputs).cpu()
    print(features.shape)
    print(features.norm())

    layers = [layer for layer in net.children()]
    print('Layers: {}'.format(len(layers)))
    print('Layers[0]: {}'.format(len(layers[0]))) #4
    print('Layers[1]: {}'.format(len(layers[1]))) #2
    print('Layers[2]: {}'.format(len(layers[2]))) #2
    print('Layers[3]: {}'.format(len(layers[3]))) #2
    print('Layers[4]: {}'.format(len(layers[4]))) #2
    #print('Layers[5]: {}'.format(len(layers[5])))#no length
    print('Layers[6]: {}'.format(len(layers[6]))) #4

    #net = Net()
    #inputs = torch.randn(5,3,128,64)
    y = net(inputs)
    #import ipdb; ipdb.set_trace()
    print(net) #alternatif bir model görme şekli torchsummarye göre
    from torchsummary import summary
    summary(net, (3,128,64))


text:
#!/usr/bin/env python

import georasters as gr
import numpy as np
import matplotlib.pyplot as plt

T1 = r"./output/inyo_bcets/20141108_mojave_bcet/20141108_mojave_bcet_B1.tif"
T2 = r"./output/inyo_bcets/20140225_mojave_bcet/20140225_mojave_bcet_B1.tif"
T3 = r"./output/inyo_bcets/20151026_mojave_bcet/20151026_mojave_bcet_B1.tif"
T4 = r"./output/inyo_bcets/20160302_mojave_bcet/20160302_mojave_bcet_B1.tif"
T5 = r"./output/inyo_bcets/20160926_mojave_bcet/20160926_mojave_bcet_B1.tif"
T6 = r"./output/inyo_bcets/20170727_mojave_bcet/20170727_mojave_bcet_B1.tif"
T7 = r"./output/inyo_bcets/20170913_mojave_bcet/20170913_mojave_bcet_B1.tif"

T1_gr = gr.from_file(T1)
T2_gr = gr.from_file(T2)
T3_gr = gr.from_file(T3)
T4_gr = gr.from_file(T4)
T5_gr = gr.from_file(T5)
T6_gr = gr.from_file(T6)
T7_gr = gr.from_file(T7)

ndv, xsize, ysize, geot, projection, datatype = gr.get_geo_info(T1) # Raster information

Tstack = np.dstack((T1_gr.raster,T2_gr.raster,T3_gr.raster,T4_gr.raster,T5_gr.raster,T6_gr.raster,T7_gr.raster))

def difference(x):
	return np.max(x)-np.min(x)

Rdiff = np.apply_along_axis(difference,axis=2, arr=Tstack)

output_gr = gr.GeoRaster(Rdiff,
     T1_gr.geot,
     nodata_value=ndv,
     projection=T1_gr.projection,
     datatype=T1_gr.datatype)

output_gr.to_tiff('./output/diff')

max_stars_repo_name: hijinks/python-bcet

max_issues_repo_name: hijinks/python-bcet

max_forks_repo_name: hijinks/python-bcet


text:
#! /usr/bin/python3
######################################################################################################################################################################

import numpy as np
from numpy import linalg as LA
import math
from scipy.integrate import odeint
import matplotlib.pyplot as plt
import random
import warnings

###############################################################################

def sort(S,t,num):

     tindex = np.where(S == num)[0]
     times = np.ones(len(tindex))

     for i in range(len(tindex)):
         times[i] = t[tindex[i]]

     return times

def sortDRIVER(S,t,s0):

    j=s0

    Atimes = np.ones(s0+1)

    for i in range(len(Atimes)):
        Stimes=sort(S,t,j)
        Atimes[i] = np.mean(Stimes)
        j = j-1

    return Atimes

#S = np.array([2, 1, 2, 3, 2, 5, 6, 2, 8, 2])

#print(len(S))
#t = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0])
#num =2

#y = sort(S,t,num)
#print(y)

reason: import numpy,from numpy,from scipy


text:
# Polynomials
# Problem Link: https://www.hackerrank.com/challenges/np-polynomials/problem

import numpy

n = list(map(float, input().split()))
m = int(input())
print(numpy.polyval(n, m))


text:
import csv
import time
import pprint as pp
import networkx as nx

import Network_Based_Recommendation_System_FUNCTIONS as homework_2

print()
print("Current time: " + str(time.asctime(time.localtime())))
print()
print()

all_groups = [
    {1701: 1, 1703: 1, 1705: 1, 1707: 1, 1709: 1},  ### Movie night with friends.
    {
        1701: 1,
        1702: 4,
    },  ### First appointment scenario: the preferences of the girl are 4 times more important than those of the man.
    {
        1701: 1,
        1702: 2,
        1703: 1,
        1704: 2,
    },  ### Two couples scenario: the preferences of girls are still more important than those of the men...
    {1701: 1, 1702: 1, 1703: 1, 1704: 1, 1705: 1, 1720: 10},  ### Movie night with a special guest.
    {1701: 1, 1702: 1, 1703: 1, 1704: 1, 1705: 1, 1720: 10, 1721: 10, 1722: 10},  ### Movie night with 3 special guests.
]
print()
pp.pprint(all_groups)
print()

graph_file = "./input_data/u_data_homework_format.txt"

pp.pprint("Load Graph.")
print("Current time: " + str(time.asctime(time.localtime())))
graph_users_items = homework_2.create_graph_set_of_users_set_of_items(graph_file)
print(" #Users in Graph= " + str(len(graph_users_items["users"])))
print(" #Items in Graph= " + str(len(graph_users_items["items"])))
print(" #Nodes in Graph= " + str(len(graph_users_items["graph"])))
print(" #Edges in Graph= " + str(graph_users_items["graph"].number_of_edges()))
print("Current time: " + str(time.asctime(time.localtime())))
print()
print()

pp.pprint("Create Item-Item-Weighted Graph.")
print("Current time: " + str(time.asctime(time.localtime())))
item_item_graph = homework_2.create_item_item_graph(graph_users_items)
print(" #Nodes in Item-Item Graph= " + str(len(item_item_graph)))
print(" #Edges in Item-Item Graph= " + str(item_item_graph.number_of_edges()))
print("Current time: " + str(time.asctime(time.localtime())))
print()
print()

### Conversion of the 'Item-Item-Graph' to a scipy sparse matrix representation.
### This reduces a lot the PageRank running time ;)
print()
print(" Conversion of the 'Item-Item-Graph' to a scipy sparse matrix representation.")
N = len(item_item_graph)
nodelist = item_item_graph.nodes()
M = nx.to_scipy_sparse_matrix(item_item_graph, nodelist=nodelist, weight="weight", dtype=float)
print(" Done.")
print()
#################################################################################################

output_file = open("./Output_Recommendation_for_Group.tsv", "w")
output_file_csv_writer = csv.writer(output_file, delimiter="\t", quotechar='"', quoting=csv.QUOTE_NONE)
print()
for current_group in all_groups:
    print("Current group: ")
    pp.pprint(current_group)
    print("Current time: " + str(time.asctime(time.localtime())))

    ################################################################################################################################################
    ################################################################################################################################################

    # in this part of the homework we have to build a recommendation engine for a GROUP of users, where any user in the group is characterized by
    # a constant that we decided to consider as a measure of relative importance within his group and we assume to be always a positive integer
    # bigger than or equal to one.
    # So, given these assumption, we decided to modify the preferences (used to compute the PageRank on the item-item graph)
    # weighting the ratings given by a user with the corresponding relative importance measure in this way:
    # if the rating given by user to item is smaller of equal to 2, we multiply the rating for the inverse of the relative importance measure;
    # else if the rating given by user to item is bigger or equal to 3, we multipy the rating directly for the relative importance measure.
    # For example, if the relative importance measure for user_i is 3 and his rating for item_j is 4, the final weighted score will be 12;
    # instead if the rating for item_(j+1) is 2, the final weighted score will be 2/3 ( rating * 1/relative importance measure).
    # With this method the relative importance measure works in positive and in negative direction.
    # After that, the final preference vector for the group is normalized to obtain a discrete probability distribution;
    # the personalized PageRank is computed; and the items are sorted by decreasing personalized PageRank score.
    # Finally we return an ordered list of recommendation for the group and all the process is repeated for every group.

    # P.S. we decided to remove (we know that it is not a constrain but the second part is open ended) the yet rated items from
    # the final recommendation list because we think that, with a personalized PageRank built in this way, the final recommendation order
    # could be biased if we consider the yet rated items; and in general we shoul order a longer list of final
    # scores also if we yet know the user score for these items(we could use different aggregation techniques).
    # However it is possible to include these yet rated items in the final recommendation list simply commenting the line 141 in
    # the code (where you see ## COMMMENT .... ##): this could be the case when your girlfriend wants to see for the 30-th Titanic or
    # your brother for the 100-th The gladiator. ':()

    ################################################################################################################################################
    ################################################################################################################################################

    sorted_list_of_recommended_items_for_current_group = []
    preference_for_current_group = {k: 0 for k in nodelist}

    rated_items_for_current_group = set()

    for user in current_group:

        # rated items by user
        rated_by_user = set(graph_users_items["graph"][user].keys())

        # update rated items by this group
        rated_items_for_current_group.update(rated_by_user)

        # update weighted preference in the preference group for this user respect this particular item
        for item in graph_users_items["graph"][user]:

            rating = graph_users_items["graph"][user][item]["weight"]

            # if this user has given a bad rating to this item ------> decrease the positive impact on the overall preference vector
            if rating < 3:
                preference_for_current_group[item] += rating * (1 / current_group[user])

            # if this user has given a good rating to this item -----> increase the positive impact on the overall preference vector
            else:
                preference_for_current_group[item] += rating * current_group[user]

    # normalization constant preference for the group
    normalization = sum(preference_for_current_group.values())

    # normalization preference for this group
    preference_for_current_group = {
        item: preference_for_current_group[item] / normalization for item in preference_for_current_group
    }

    # compute pageRank for group using preference for current group
    page_rank_for_current_group = homework_2.pagerank(
        M, N, nodelist, alpha=0.85, personalization=preference_for_current_group
    )

    # select only recommended items not yet rated by a member of this group
    recommendation_for_current_group = [
        [item, page_rank_for_current_group[item]]
        for item in page_rank_for_current_group
        if item not in rated_items_for_current_group
    ]  ## COMMENT this line if you want to consider yet rated items ##

    # sorted [item,score]
    recommendation_for_current_group = sorted(recommendation_for_current_group, key=lambda r: r[1], reverse=True)

    # final result [item]
    sorted_list_of_recommended_items_for_current_group = [rec[0] for rec in recommendation_for_current_group]

    ###############################################################################################################################################
    ###############################################################################################################################################

    print("Recommended Sorted List of Items:")
    print((str(sorted_list_of_recommended_items_for_current_group[:30])))
    print()
    output_file_csv_writer.writerow(sorted_list_of_recommended_items_for_current_group)

output_file.close()

print()
print()
print("Current time: " + str(time.asctime(time.localtime())))
print("Done ;)")
print()

reason: import networkx


text:
# -*- coding: utf-8 -*-

# This code is part of Qiskit.
#
# (C) Copyright IBM 2017, 2019.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

"""Test terra custom fields."""

import numpy
import sympy

from qiskit.circuit import Parameter

from qiskit.test import QiskitTestCase
from qiskit.validation import fields
from qiskit.validation.base import BaseModel, BaseSchema, bind_schema
from qiskit.validation.exceptions import ModelValidationError

class DogSchema(BaseSchema):
    """Example Dog schema."""
    barking = fields.InstructionParameter(required=True)

@bind_schema(DogSchema)
class Dog(BaseModel):
    """Example Dog model."""
    pass

class TestFields(QiskitTestCase):
    """Tests for Fields."""

    def test_parameter_field_float(self):
        """Test InstructionParameter with types equivalent to float."""
        dog = Dog(barking=0.1)
        dog_numpy = Dog(barking=numpy.dtype('float').type(0.1))
        dog_sympy = Dog(barking=sympy.Float(0.1))

        self.assertEqual(dog_numpy.barking,
                         numpy.dtype('float').type(0.1))
        self.assertEqual(dog.to_dict(), dog_numpy.to_dict())
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())

    def test_parameter_field_complex(self):
        """Test InstructionParameter with types equivalent to complex."""
        dog = Dog(barking=(0.1+0.2j))
        dog_numpy = Dog(barking=numpy.dtype('complex').type(0.1+0.2j))
        dog_sympy = Dog(barking=sympy.sympify(0.1+0.2j))

        self.assertEqual(dog_numpy.barking,
                         numpy.dtype('complex').type(0.1+0.2j))
        self.assertEqual(dog.to_dict(), dog_numpy.to_dict())
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())
        self.assertEqual(dog.to_dict()['barking'], [0.1, 0.2])

    def test_parameter_field_int(self):
        """Test InstructionParameter with types equivalent to int."""
        dog = Dog(barking=1)
        dog_numpy = Dog(barking=numpy.dtype('int').type(1))
        dog_sympy = Dog(barking=sympy.Integer(1))

        self.assertEqual(dog_numpy.barking,
                         numpy.dtype('int').type(1))
        self.assertEqual(dog.to_dict(), dog_numpy.to_dict())
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())

    def test_parameter_field_str(self):
        """Test InstructionParameter with types equivalent to str."""
        dog = Dog(barking='woof')
        dog_sympy = Dog(barking=sympy.Symbol('woof'))

        self.assertEqual(dog_sympy.barking, sympy.Symbol('woof'))
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())

    def test_parameter_field_container(self):
        """Test InstructionParameter with container types."""
        dog = Dog(barking=[0.1, 2.3])
        dog_numpy = Dog(barking=numpy.array([0.1, 2.3]))
        dog_sympy = Dog(barking=[sympy.Float(0.1), sympy.Float(2.3)])
        dog_complex = Dog(barking=complex(0.1, 2.3))

        self.assertEqual(dog.to_dict(), dog_numpy.to_dict())
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())
        self.assertEqual(dog.to_dict(), dog_complex.to_dict())

    def test_parameter_field_container_nested(self):
        """Test InstructionParameter with nested container types."""
        dog = Dog(barking=[[1, 2]])
        dog_numpy = Dog(barking=numpy.array([[1, 2]]))
        dog_sympy = Dog(barking=[[sympy.Integer(1),
                                  sympy.Integer(2)]])

        self.assertEqual(dog.to_dict(), dog_numpy.to_dict())
        self.assertEqual(dog.to_dict(), dog_sympy.to_dict())

    def test_parameter_field_from_dict(self):
        """Test InstructionParameter from_dict."""
        dog = Dog.from_dict({'barking': [0.1, 2]})
        self.assertEqual(dog.barking, [0.1, 2])

    def test_parameter_field_invalid(self):
        """Test InstructionParameter invalid values."""
        with self.assertRaises(ModelValidationError) as context_manager:
            _ = Dog(barking={})
        self.assertIn('barking', str(context_manager.exception))

        with self.assertRaises(ModelValidationError) as context_manager:
            # Invalid types inside containers are also not allowed.
            _ = Dog(barking=[{}])
        self.assertIn('barking', str(context_manager.exception))

    def test_parameter_field_from_dict_invalid(self):
        """Test InstructionParameter from_dict."""
        with self.assertRaises(ModelValidationError) as context_manager:
            _ = Dog.from_dict({'barking': {}})
        self.assertIn('barking', str(context_manager.exception))

        with self.assertRaises(ModelValidationError) as context_manager:
            # Invalid types inside containers are also not allowed.
            _ = Dog.from_dict({'barking': [{}]})
        self.assertIn('barking', str(context_manager.exception))

        with self.assertRaises(ModelValidationError) as context_manager:
            # Non-basic types cannot be used in from_dict, even if they are
            # accepted in the constructor. This is by design, as the serialized
            # form should only contain Python/json-schema basic types.
            _ = Dog.from_dict({'barking': sympy.Symbol('woof')})
        self.assertIn('barking', str(context_manager.exception))

    def test_parameter_expression_fully_bound(self):
        """Test ParameterExpressions valid after fully bound."""
        x = Parameter('x')
        y = Parameter('y')

        expr = x
        bound_expr = expr.bind({x: 2})

        dog_expr = Dog(barking=bound_expr)
        dog_float = Dog(barking=2)
        self.assertEqual(dog_expr.barking, bound_expr)
        self.assertEqual(dog_expr.to_dict(), dog_float.to_dict())

        expr = x + y
        bound_expr = expr.bind({x: 2, y: 3})

        dog_expr = Dog(barking=bound_expr)
        dog_float = Dog(barking=5)
        self.assertEqual(dog_expr.barking, bound_expr)
        self.assertEqual(dog_expr.to_dict(), dog_float.to_dict())

    def test_parameter_expression_partially_bound(self):
        """Test ParameterExpressions invalid if partially bound."""
        x = Parameter('x')
        y = Parameter('y')

        with self.assertRaises(ModelValidationError) as context_manager:
            _ = Dog(barking=x).to_dict()
        self.assertIn('barking', str(context_manager.exception))

        expr = x + y

        with self.assertRaises(ModelValidationError) as context_manager:
            _ = Dog(barking=expr).to_dict()
        self.assertIn('barking', str(context_manager.exception))

        partially_bound_expr = expr.bind({x: 2})

        with self.assertRaises(ModelValidationError) as context_manager:
            _ = Dog(barking=partially_bound_expr).to_dict()
        self.assertIn('barking', str(context_manager.exception))

max_stars_repo_name: lerongil/qiskit-terra

max_issues_repo_name: lerongil/qiskit-terra

max_forks_repo_name: lerongil/qiskit-terra

reason: import numpy,import sympy


text:
'''
Problem 45:

Triangle, pentagonal, and hexagonal numbers are generated by the following formulae:

Triangle	 	Tn=n(n+1)/2	 	1, 3, 6, 10, 15, ...
Pentagonal	 	Pn=n(3n−1)/2	 	1, 5, 12, 22, 35, ...
Hexagonal	 	Hn=n(2n−1)	 	1, 6, 15, 28, 45, ...
It can be verified that T285 = P165 = H143 = 40755.

Find the next triangle number that is also pentagonal and hexagonal.
'''

import numpy as np

def triangular_pentagonal_hexagonal():

    res = [] #append solutions
    i = 1
    while True:
        hexn = i*(2*i - 1)
        j = i + 1
        while True:
            pentn = j*(3*j - 1)/2
            if hexn == pentn:
                k = j + 1
                while True:
                    trian = k*(k + 1)/2
                    if pentn == trian:
                        res.append(trian)
                        if len(res) > 1:
                            return res
                        else:
                            break
                    elif trian > pentn:
                        break
                    k += 1
            elif pentn > hexn:
                break
            j += 1
        i += 1
        if i > 100000:
            return res

def main():
    print('number that is triangular, pentagonal and hexagonal: ', triangular_pentagonal_hexagonal())

if __name__=='__main__':
    main()


text:
#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt

x = np.array([0.,0.,1.,1.,1.,1.,1.,1.,1.,1.])
print(x)

fig1 = plt.figure(1)
plt.stem(x)
plt.pause(0.0001)

k = np.arange(-2.,8.,1.)

fig2 = plt.figure(2)
plt.stem(k,x)
plt.show()

N = 10
M = 20

x = np.hstack([np.zeros(shape=(N)), np.ones(shape=(M+1))])

k = np.arange(-10.,21.,1.)

x1 = np.hstack([np.zeros(shape=(N-5)), np.ones(shape=(M-4))])

fig1 = plt.figure(1)
plt.stem(k,x)
plt.title("u[n]")
plt.pause(0.0001)

k = np.arange(0.,21.,1.)
fig2 = plt.figure(2)
plt.stem(k,x1)
plt.title("u[n-5]")

k = np.arange(-40.,81.,1.)

x = np.hstack([(7*np.cos(0.1*k)) + np.cos(0.95*k)])

fig1 = plt.figure(1)
plt.stem(k,x)
plt.title("u[n]")

x1 = np.hstack([(7*np.cos(0.1*(k-20))) + np.cos(0.95*(k-20))])

fig1 = plt.figure(2)
plt.stem(k,x1)
plt.title("u[n-20]")

h = 1./5.*np.array([1.,1.,1.,1.,1.])

x = np.hstack([np.ones(shape=20)])

y = np.convolve(h,x, mode = 'full')

fig1 = plt.figure(1)
plt.stem(y)
plt.title("y[n]")

k =  np.arange(-40.,81.,1.)
x = np.hstack([np.cos(0.1*k)])

y = np.convolve(h,x, mode = 'full')

fig1 = plt.figure(1)
plt.stem(y)
plt.title("y[n]")

x = np.hstack([np.cos(0.95*k)])

y = np.convolve(h,x, mode = 'full')

fig1 = plt.figure(1)
plt.stem(y)
plt.title("y[n]")

h = np.array([1.,-1.])

x = np.hstack([np.ones(shape=20)])

y = np.convolve(h,x, mode = 'full')

fig1 = plt.figure(1)
plt.stem(y)
plt.title("y[n]")

h = np.array([1.,-1.])
k =  np.arange(-40.,81.,1.)
x = np.hstack([(7*np.cos(0.1*k)) + np.cos(0.95*k)])

y = np.convolve(h,x, mode = 'full')

fig1 = plt.figure(1)
plt.stem(y)
plt.title("y[n]")

max_stars_repo_path: 1-D convolutions in python.py

max_issues_repo_path: 1-D convolutions in python.py

max_forks_repo_path: 1-D convolutions in python.py


text:
"""
Returns the array of probability mass function values
for the Binomial probability distribution
using the given arrays for the abscissa or template
values and each of the parameters values.
"""
import numpy
import pyferret
import pyferret.stats

DISTRIB_NAME = "Binomial"
FUNC_NAME = "pmf"

def ferret_init(id):
    """
    Initialization for the stats_binom_pmf Ferret PyEF
    """
    return pyferret.stats.getinitdict(DISTRIB_NAME, FUNC_NAME)

def ferret_compute(id, result, resbdf, inputs, inpbdfs):
    """
    Result array assignment for the stats_binom_pmf Ferret PyEF
    """
    pyferret.stats.assignresultsarray(DISTRIB_NAME, FUNC_NAME,
                                      result, resbdf, inputs, inpbdfs)

max_stars_repo_name: josborne-noaa/PyFerret

max_issues_repo_name: josborne-noaa/PyFerret

max_forks_repo_name: josborne-noaa/PyFerret


text:
from pyproj import Proj
import time, cPickle, array
import numpy as N

params = {}
params["proj"] = "lcc"
params["R"] = 6371200
params["lat_1"] = 50
params["lat_2"] = 50
params["lon_0"] = -107
nx = 349
ny = 277
dx = 32463.41
dy = dx
# can either use a dict
# awips221 = Proj(params)
# or keyword args
awips221 = Proj(proj="lcc", R=6371200, lat_1=50, lat_2=50, lon_0=-107)
print("proj4 library version = ", awips221.proj_version)
# AWIPS grid 221 parameters
# (from http://www.nco.ncep.noaa.gov/pmb/docs/on388/tableb.html)
llcrnrx, llcrnry = awips221(-145.5, 1.0)
params["x_0"] = -llcrnrx
params["y_0"] = -llcrnry
awips221 = Proj(params)
llcrnrx, llcrnry = awips221(-145.5, 1.0)
# find 4 lon/lat crnrs of AWIPS grid 221.
llcrnrx = 0.0
llcrnry = 0.0
lrcrnrx = dx * (nx - 1)
lrcrnry = 0.0
ulcrnrx = 0.0
ulcrnry = dy * (ny - 1)
urcrnrx = dx * (nx - 1)
urcrnry = dy * (ny - 1)
llcrnrlon, llcrnrlat = awips221(llcrnrx, llcrnry, inverse=True)
lrcrnrlon, lrcrnrlat = awips221(lrcrnrx, lrcrnry, inverse=True)
urcrnrlon, urcrnrlat = awips221(urcrnrx, urcrnry, inverse=True)
ulcrnrlon, ulcrnrlat = awips221(ulcrnrx, ulcrnry, inverse=True)
print("4 crnrs of AWIPS grid 221:")
print(llcrnrlon, llcrnrlat)
print(lrcrnrlon, lrcrnrlat)
print(urcrnrlon, urcrnrlat)
print(ulcrnrlon, ulcrnrlat)
print("from GRIB docs")
print("(see http://www.nco.ncep.noaa.gov/pmb/docs/on388/tableb.html)")
print("   -145.5  1.0")
print("   -68.318 0.897")
print("   -2.566 46.352")
print("   148.639 46.635")
# compute lons and lats for the whole AWIPS grid 221 (377x249).
dx = (urcrnrx - llcrnrx) / (nx - 1)
dy = (urcrnry - llcrnry) / (ny - 1)
x = llcrnrx + dx * N.indices((ny, nx), "f")[1, :, :]
y = llcrnry + dy * N.indices((ny, nx), "f")[0, :, :]
t1 = time.clock()
lons, lats = awips221(N.ravel(x).tolist(), N.ravel(y).tolist(), inverse=True)
t2 = time.clock()
print("data in lists:")
print("compute lats/lons for all points on AWIPS 221 grid (%sx%s)" % (nx, ny))
print("max/min lons")
print(min(lons), max(lons))
print("max/min lats")
print(min(lats), max(lats))
print("took", t2 - t1, "secs")
xa = array.array("f", N.ravel(x).tolist())
ya = array.array("f", N.ravel(y).tolist())
t1 = time.clock()
lons, lats = awips221(xa, ya, inverse=True)
t2 = time.clock()
print("data in python arrays:")
print("compute lats/lons for all points on AWIPS 221 grid (%sx%s)" % (nx, ny))
print("max/min lons")
print(min(lons), max(lons))
print("max/min lats")
print(min(lats), max(lats))
print("took", t2 - t1, "secs")
t1 = time.clock()
lons, lats = awips221(x, y, inverse=True)
t2 = time.clock()
print("data in a numpy array:")
print("compute lats/lons for all points on AWIPS 221 grid (%sx%s)" % (nx, ny))
print("max/min lons")
print(N.minimum.reduce(N.ravel(lons)), N.maximum.reduce(N.ravel(lons)))
print("max/min lats")
print(N.minimum.reduce(N.ravel(lats)), N.maximum.reduce(N.ravel(lats)))
print("took", t2 - t1, "secs")
# reverse transformation.
t1 = time.clock()
xx, yy = awips221(lons, lats)
t2 = time.clock()
print("max abs error for x")
print(N.maximum.reduce(N.fabs(N.ravel(x - xx))))
print("max abs error for y")
print(N.maximum.reduce(N.fabs(N.ravel(x - xx))))
print("took", t2 - t1, "secs")
cPickle.dump(awips221, open("test.pickle", "wb"), -1)
print("compare output with sample.out")
print("now run test2.py to test pickling")

max_stars_repo_name: matthew-brett/pyproj

max_issues_repo_name: matthew-brett/pyproj

max_forks_repo_name: matthew-brett/pyproj


text:
import sys
sys.path.append('../')
import argparse
import time
import os
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
from torch.optim import Adam
from models.utils import create_model
from configs.defaults import get_cfg_defaults
from data.slp_weight_dataset import SLPWeightDataset
from torch.optim.lr_scheduler import MultiStepLR

def train(cfg, output_directory):
    train_set = SLPWeightDataset(cfg, phase='train')
    train_loader = DataLoader(train_set, batch_size=cfg.SOLVER.BATCH_SIZE_TRAIN, num_workers=cfg.INPUT.NUM_WORKERS,
                              shuffle=True)
    val_set = SLPWeightDataset(cfg, phase='val')
    val_loader = DataLoader(val_set, batch_size=cfg.SOLVER.BATCH_SIZE_TEST, num_workers=cfg.INPUT.NUM_WORKERS,
                            shuffle=False)

    # model
    model = create_model(cfg).to(cfg.MODEL.DEVICE)

    # optimizer and scheduler
    use_amp = cfg.SOLVER.USE_AMP # whether to use mixed precision
    optimizer = Adam(model.parameters(), lr=cfg.SOLVER.BASE_LR, weight_decay=cfg.SOLVER.WEIGHT_DECAY)
    scaler = torch.cuda.amp.GradScaler(enabled=use_amp)
    lr_scheduler = MultiStepLR(optimizer, cfg.SOLVER.LR_MILESTONES, cfg.SOLVER.LR_GAMMA)

    # loss criterion
    criterion = nn.MSELoss(reduction='mean')

    # logging: save training and validation accuracy after each epoch to numpy array
    validation_log = np.zeros([cfg.SOLVER.NUM_EPOCHS, 2])

    # Training
    for e in range(cfg.SOLVER.NUM_EPOCHS):
        model.train()
        start_time = time.time()
        for it, data in enumerate(train_loader, 1):
            inputs, labels, idx = data
            inputs = inputs.to(cfg.MODEL.DEVICE)
            labels = labels.to(cfg.MODEL.DEVICE)

            with torch.cuda.amp.autocast(enabled=use_amp):
                pred = model(inputs)
                loss = criterion(pred.view(-1), labels.view(-1))

            optimizer.zero_grad()
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()

        lr_scheduler.step()

        # Validation
        model.eval()

        # run evaluation on training set
        train_loss = 0.
        for it, data in enumerate(train_loader, 1):
            inputs, labels, idx = data
            inputs = inputs.to(cfg.MODEL.DEVICE)
            labels = labels.to(cfg.MODEL.DEVICE)

            with torch.cuda.amp.autocast(enabled=use_amp):
                with torch.no_grad():
                    pred = model(inputs)

            # remap network output in range [0, 1] to weight in kg
            if cfg.INPUT.NORMALIZE_OUTPUT:
                pred = pred * (train_set.max_weight_train - train_set.min_weight_train) + train_set.min_weight_train
                labels = labels * (train_set.max_weight_train - train_set.min_weight_train) + train_set.min_weight_train

            train_loss += torch.sum(torch.abs(pred.view(-1) - labels.view(-1))).item()

        train_loss = train_loss / len(train_set)

        # run evaluation on validation set
        val_loss = 0.
        for it, data in enumerate(val_loader, 1):
            inputs, labels, idx = data
            inputs = inputs.to(cfg.MODEL.DEVICE)
            labels = labels.to(cfg.MODEL.DEVICE)

            with torch.cuda.amp.autocast(enabled=use_amp):
                with torch.no_grad():
                    pred = model(inputs)

            # remap network output in range [0, 1] to weight in kg
            if cfg.INPUT.NORMALIZE_OUTPUT:
                pred = pred * (train_set.max_weight_train - train_set.min_weight_train) + train_set.min_weight_train
                labels = labels * (val_set.max_weight_train - val_set.min_weight_train) + val_set.min_weight_train

            val_loss += torch.sum(torch.abs(pred.view(-1) - labels.view(-1))).item()

        val_loss = val_loss / len(val_set)
        end_time = time.time()
        print('epoch', e, 'duration', '%0.3f' % ((end_time - start_time) / 60.), 'train_mae', '%0.3f' % train_loss,
              'val_mae', '%0.3f' % val_loss)

        validation_log[e, :] = [train_loss, val_loss]
        np.save(os.path.join(output_directory, "validation_history"), validation_log)
        torch.save(model.state_dict(), os.path.join(output_directory, 'model.pth'))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
    parser.add_argument(
        "--config-file",
        default="",
        metavar="FILE",
        help="path to config file",
        type=str,
    )
    parser.add_argument(
        "--gpu",
        default="0",
        metavar="FILE",
        help="gpu to train on",
        type=str,
    )
    args = parser.parse_args()
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu

    cfg = get_cfg_defaults()
    cfg.merge_from_file(args.config_file)
    cfg.freeze()

    output_directory = os.path.join(cfg.BASE_DIRECTORY, cfg.EXPERIMENT_NAME)
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)

    train(cfg, output_directory)