text:
# -*- coding: utf-8 -*-
"""Smart stick

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1hk3pL1Gm1TjFZdiC3pPfwfrI_9ucfIqg

# YOLOv4 Object Detection on Webcam In Google Colab
This notebook will walkthrough all the steps for performing YOLOv4 object detections on your webcam while in Google Colab. We will be using scaled-YOLOv4 (yolov4-csp) for this tutorial, the fastest and most accurate object detector there currently is.

![image.png](data:image/png;base64,[encoded data, 156564 characters])
"""

# Commented out IPython magic to ensure Python compatibility.
# import dependencies
from IPython.display import display, Javascript, Image
from google.colab.output import eval_js
from google.colab.patches import cv2_imshow
from base64 import b64decode, b64encode
import cv2
import numpy as np
import PIL
import io
import html
import time
import matplotlib.pyplot as plt
# %matplotlib inline

"""# FEATURES"""

pip install twilio

pip install num2words

import os
from os.path import exists, join, expanduser

project_name = "pytorch-dc-tts"
if not exists(project_name):
  ! git clone --quiet https://github.com/tugstugi/{project_name}
  ! cd {project_name} && pip install -q -r requirements.txt

# download text2mel
if not exists("ljspeech-text2mel.pth"):
  ! wget -q -O ljspeech-text2mel.pth https://www.dropbox.com/s/4t13ugxzzgnocbj/step-300K.pth

# download SSRN
if not exists("ljspeech-ssrn.pth"):
  ! wget -q -O ljspeech-ssrn.pth https://www.dropbox.com/s/gw4aqrgcvccmg0g/step-100K.pth

import sys
sys.path.append(project_name)

import warnings
warnings.filterwarnings("ignore")  # ignore warnings in this notebook

import numpy as np
import torch

from tqdm import *
import IPython
from IPython.display import Audio

from hparams import HParams as hp
from audio import save_to_wav
from models import Text2Mel, SSRN
from datasets.lj_speech import vocab, idx2char, get_test_data

class TTS:

  def tts(self,sentence):

    torch.set_grad_enabled(False)
    text2mel = Text2Mel(vocab)
    text2mel.load_state_dict(torch.load("ljspeech-text2mel.pth").state_dict())
    text2mel = text2mel.eval()
    ssrn = SSRN()
    ssrn.load_state_dict(torch.load("ljspeech-ssrn.pth").state_dict())
    ssrn = ssrn.eval()

    normalized_sentence = "".join([c if c.lower() in vocab else str(c) if c.isdigit() else '' for c in sentence])
    print(normalized_sentence)

    sentences = [normalized_sentence]
    max_N = len(normalized_sentence)
    L = torch.from_numpy(get_test_data(sentences, max_N))
    zeros = torch.from_numpy(np.zeros((1, hp.n_mels, 1), np.float32))
    Y = zeros
    A = None

    for t in range(hp.max_T):
      _, Y_t, A = text2mel(L, Y, monotonic_attention=True)
      Y = torch.cat((zeros, Y_t), -1)
      _, attention = torch.max(A[0, :, -1], 0)
      attention = attention.item()
      if L[0, attention] == vocab.index('E'):  # EOS
          break

    _, Z = ssrn(Y)

    Z = Z.cpu().detach().numpy()
    save_to_wav(Z[0, :, :].T, '%d.wav' % (1))
    IPython.display.display(Audio('%d.wav' % (1), rate=hp.sr, autoplay=True))

import requests
from geopy import Nominatim

class Loc:
  def loca(self):
    req = requests.get('https://get.geojs.io/')

    ip_request = requests.get('https://get.geojs.io/v1/ip.json')

    ipAdd = ip_request.json()['ip']

    #print(ipAdd)

    url='https://get.geojs.io/v1/ip/geo/'+ipAdd+'.json'

    geo_req=requests.get(url)

    geo_data = geo_req.json()

    lat=geo_data['latitude']
    lat=19.227225

    lon=geo_data['longitude']
    lon=72.846055

    #print(geo_data)
    #print(lat)
    #print(lon)

    geolocator=Nominatim(user_agent='test/1')
    location= geolocator.reverse(str(lat)+','+str(lon))
    #location= geolocator.reverse(str(19.227225)+','+str(72.846055))
    print("The user's location is "+str(location))
    locs="The user's location is "+str(location)
    #return "The user's location is "+str(location)

    return lat,lon,locs

import requests
import os
from datetime import datetime
from num2words import num2words

class Weather:
  def give_wu(self):
    user_api='32efefaf435edc96b5c402ce8dd53711'
    #location="Mumbai"

    l=Loc()
    locy=l.loca()
    lat=str(locy[0])
    lon=str(locy[1])
    print(lat,lon)
    api_link="https://api.openweathermap.org/data/2.5/weather?lat="+lat+"&lon="+lon+"&appid="+user_api

    #api_link="https://api.openweathermap.org/data/2.5/weather?q="+location+"&appid="+user_api

    api_link=requests.get(api_link)
    api_data= api_link.json()
    #print(type(api_data))
    #print(api_data)

    temp=int((api_data['main']['temp'])-273.15)
    weather_desc=api_data['weather'][0]['description']
    humid=api_data['main']['humidity']
    wind_speed=api_data['wind']['speed']
    wind_speed=wind_speed*3.6
    date_time=datetime.now().strftime("%d %b %Y | %I:%M:%S %p")

    #print(api_data)
    #print(lat, lon)

    #temp1=str(round(temp))
    #humid1=str(humid)
    #wind_speed1=str(round(wind_speed,2))
    #print(type(temp1))

    temp1=num2words(temp)
    humid1=num2words(humid)
    wind_speed1=num2words(round(wind_speed,2))
    #print(type(temp1))

    w1="The weather at this location is "+weather_desc
    w2="The temperature is "+temp1+" degree celsius"
    w3="The humidity is "+humid1+" percentage"
    w4="The wind speed is "+wind_speed1+" kilometer per hour"

    print(w1)
    print(w2)
    print(w3)
    print(w4)

    t=TTS()
    t.tts(w1)
    t.tts(w2)
    t.tts(w3)
    t.tts(w4)

import smtplib
#import voicent
from twilio.rest import Client

class Alert:
  def sendalert(self):
    sender='be35.project@gmail.com'
    receiver='be35.project@gmail.com'
    password='BE1234567'

    smtp_server=smtplib.SMTP_SSL('smtp.gmail.com',465)
    smtp_server.login(sender,password)

    l=Loc()
    locy=l.loca()

    subject="Emergency Alert"
    body="Emergency alert:\n "+"The user's location is "+locy[2]
    msg=f'Subject: {subject}\n\n{body}'
    #smtp_server.sendmail(sender,receiver,msg)

    #v=Voicent()ko
    #v.callText("8879214174", "Hello, how are you doing", "1")

    account_sid = 'AC91480ca18dc61c7615ddef0b34b67597'
    auth_token = 'be58a2d8ffe40f836d76a82c2e09736d'
    client = Client(account_sid, auth_token)

    """
    call = client.calls.create(
                            twiml='<Response><Say>Ahoy, World!</Say></Response>',
                            to='+918879214174',
                            from_='+17343445867'
                      )

    print(call.sid)
    """

    """
    message = client.messages \
                    .create(
                        body="Emergency alert.\n "+"The user's location is "+locy[2],
                        to='+918879214174',
                        from_='+17343445867'
                    )

    print(message.sid)
    """
    print("Alert sent")

"""## Cloning and Setting Up Darknet for YOLOv4
We will be using the famous AlexeyAB's darknet repository in this tutorial to perform YOLOv4 detections.
"""

# clone darknet repo
!git clone https://github.com/AlexeyAB/darknet

# Commented out IPython magic to ensure Python compatibility.
# change makefile to have GPU, OPENCV and LIBSO enabled
# %cd darknet
!sed -i 's/OPENCV=0/OPENCV=1/' Makefile
!sed -i 's/GPU=0/GPU=1/' Makefile
!sed -i 's/CUDNN=0/CUDNN=1/' Makefile
!sed -i 's/CUDNN_HALF=0/CUDNN_HALF=1/' Makefile
!sed -i 's/LIBSO=0/LIBSO=1/' Makefile

# make darknet (builds darknet so that you can then use the darknet.py file and have its dependencies)
!make

# get bthe scaled yolov4 weights file that is pre-trained to detect 80 classes (objects) from shared google drive
!wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1V3vsIaxAlGWvK4Aar9bAiK5U0QFttKwq" -O yolov4-csp.weights && rm -rf /tmp/cookies.txt

"""## Darknet for Python
In order to utilize YOLOv4 with Python code we will use some of the pre-built functions found within darknet.py by importing the functions into our workstation. Feel free to checkout the darknet.py file to see the function definitions in detail!
"""

# import darknet functions to perform object detections
from darknet import *
# load in our YOLOv4 architecture network
network, class_names, class_colors = load_network("cfg/yolov4-csp.cfg", "cfg/coco.data", "yolov4-csp.weights")
width = network_width(network)
height = network_height(network)

# darknet helper function to run detection on image
def darknet_helper(img, width, height):
  darknet_image = make_image(width, height, 3)
  img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
  img_resized = cv2.resize(img_rgb, (width, height),
                              interpolation=cv2.INTER_LINEAR)

  # get image ratios to convert bounding boxes to proper size
  img_height, img_width, _ = img.shape
  width_ratio = img_width/width
  height_ratio = img_height/height

  # run model on darknet style image to get detections
  copy_image_from_bytes(darknet_image, img_resized.tobytes())
  detections = detect_image(network, class_names, darknet_image)
  free_image(darknet_image)
  return detections, width_ratio, height_ratio

"""## YOLOv4 Example on Test Image
Let's make sure our model has successfully been loaded and that we can make detections properly on a test image.
"""

# run test on person.jpg image that comes with repository
image = cv2.imread("data/person.jpg")
detections, width_ratio, height_ratio = darknet_helper(image, width, height)

for label, confidence, bbox in detections:
  left, top, right, bottom = bbox2points(bbox)
  left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)
  cv2.rectangle(image, (left, top), (right, bottom), class_colors[label], 2)
  cv2.putText(image, "{} [{:.2f}]".format(label, float(confidence)),
                    (left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
                    class_colors[label], 2)
cv2_imshow(image)

"""## Helper Functions
Here are a few helper functions defined that will be used to easily convert between different image types within our later steps.
"""

# function to convert the JavaScript object into an OpenCV image
def js_to_image(js_reply):
  """
  Params:
          js_reply: JavaScript object containing image from webcam
  Returns:
          img: OpenCV BGR image
  """
  # decode base64 image
  image_bytes = b64decode(js_reply.split(',')[1])
  # convert bytes to numpy array
  jpg_as_np = np.frombuffer(image_bytes, dtype=np.uint8)
  # decode numpy array into OpenCV BGR image
  img = cv2.imdecode(jpg_as_np, flags=1)

  return img

# function to convert OpenCV Rectangle bounding box image into base64 byte string to be overlayed on video stream
def bbox_to_bytes(bbox_array):
  """
  Params:
          bbox_array: Numpy array (pixels) containing rectangle to overlay on video stream.
  Returns:
        bytes: Base64 image byte string
  """
  # convert array into PIL image
  bbox_PIL = PIL.Image.fromarray(bbox_array, 'RGBA')
  iobuf = io.BytesIO()
  # format bbox into png for return
  bbox_PIL.save(iobuf, format='png')
  # format return string
  bbox_bytes = 'data:image/png;base64,{}'.format((str(b64encode(iobuf.getvalue()), 'utf-8')))

  return bbox_bytes

"""## YOLOv4 on Wecam Images
Running YOLOv4 on images taken from webcam is fairly straight-forward. We will utilize code within Google Colab's **Code Snippets** that has a variety of useful code functions to perform various tasks.

We will be using the code snippet for **Camera Capture** which runs JavaScript code to utilize your computer's webcam. The code snippet will take a webcam photo, which we will then pass into our YOLOv4 model for object detection.

Below is a function to take the webcam picture using JavaScript and then run YOLOv4 on it.
"""

def take_photo(filename='photo.jpg', quality=0.8):
  js = Javascript('''
    async function takePhoto(quality) {
      const div = document.createElement('div');
      const capture = document.createElement('button');
      capture.textContent = 'Capture';
      div.appendChild(capture);

      const video = document.createElement('video');
      video.style.display = 'block';
      const stream = await navigator.mediaDevices.getUserMedia({video: true});

      document.body.appendChild(div);
      div.appendChild(video);
      video.srcObject = stream;
      await video.play();

      // Resize the output to fit the video element.
      google.colab.output.setIframeHeight(document.documentElement.scrollHeight, true);

      // Wait for Capture to be clicked.
      await new Promise((resolve) => capture.onclick = resolve);

      const canvas = document.createElement('canvas');
      canvas.width = video.videoWidth;
      canvas.height = video.videoHeight;
      canvas.getContext('2d').drawImage(video, 0, 0);
      stream.getVideoTracks()[0].stop();
      div.remove();
      return canvas.toDataURL('image/jpeg', quality);
    }
    ''')
  display(js)

  # get photo data
  data = eval_js('takePhoto({})'.format(quality))
  # get OpenCV format image
  img = js_to_image(data)

  # call our darknet helper on webcam image
  detections, width_ratio, height_ratio = darknet_helper(img, width, height)

  # loop through detections and draw them on webcam image
  for label, confidence, bbox in detections:
    left, top, right, bottom = bbox2points(bbox)
    left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)
    cv2.rectangle(img, (left, top), (right, bottom), class_colors[label], 2)
    cv2.putText(img, "{} [{:.2f}]".format(label, float(confidence)),
                      (left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
                      class_colors[label], 2)
  # save image
  cv2.imwrite(filename, img)

  return filename

def capture_image():
  try:
    filename = take_photo('photo.jpg')
    print('Saved to {}'.format(filename))

    # Show the image which was just taken.
    display(Image(filename))
  except Exception as err:
    # Errors will be thrown if the user does not have a webcam or if they do not
    # grant the page permission to access it.
    print(str(err))

"""## YOLOv4 on Webcam Videos
Running YOLOv4 on webcam video is a little more complex than images. We need to start a video stream using our webcam as input. Then we run each frame through our YOLOv4 model and create an overlay image that contains bounding box of detection(s). We then overlay the bounding box image back onto the next frame of our video stream.

YOLOv4 is so fast that it can run the detections in real-time!

![image.png](data:image/png;base64,[encoded data, 48240 characters])

Below is a function to start up the video stream using similar JavaScript as was used for images. The video stream frames are fed as input to YOLOv4.
"""

# JavaScript to properly create our live video stream using our webcam as input
def video_stream():
  js = Javascript('''
    var video;
    var div = null;
    var stream;
    var captureCanvas;
    var imgElement;
    var labelElement;

    var pendingResolve = null;
    var shutdown = false;

    function removeDom() {
       stream.getVideoTracks()[0].stop();
       video.remove();
       div.remove();
       video = null;
       div = null;
       stream = null;
       imgElement = null;
       captureCanvas = null;
       labelElement = null;
    }

    function onAnimationFrame() {
      if (!shutdown) {
        window.requestAnimationFrame(onAnimationFrame);
      }
      if (pendingResolve) {
        var result = "";
        if (!shutdown) {
          captureCanvas.getContext('2d').drawImage(video, 0, 0, 640, 480);
          result = captureCanvas.toDataURL('image/jpeg', 0.8)
        }
        var lp = pendingResolve;
        pendingResolve = null;
        lp(result);
      }
    }

    async function createDom() {
      if (div !== null) {
        return stream;
      }

      div = document.createElement('div');
      div.style.border = '2px solid black';
      div.style.padding = '3px';
      div.style.width = '100%';
      div.style.maxWidth = '600px';
      document.body.appendChild(div);

      const modelOut = document.createElement('div');
      modelOut.innerHTML = "<span>Status:</span>";
      labelElement = document.createElement('span');
      labelElement.innerText = 'No data';
      labelElement.style.fontWeight = 'bold';
      modelOut.appendChild(labelElement);
      div.appendChild(modelOut);

      video = document.createElement('video');
      video.style.display = 'block';
      video.width = div.clientWidth - 6;
      video.setAttribute('playsinline', '');
      video.onclick = () => { shutdown = true; };
      stream = await navigator.mediaDevices.getUserMedia(
          {video: { facingMode: "environment"}});
      div.appendChild(video);

      imgElement = document.createElement('img');
      imgElement.style.position = 'absolute';
      imgElement.style.zIndex = 1;
      imgElement.onclick = () => { shutdown = true; };
      div.appendChild(imgElement);

      const instruction = document.createElement('div');
      instruction.innerHTML =
          '<span style="color: red; font-weight: bold;">' +
          'When finished, click here or on the video to stop this demo</span>';
      div.appendChild(instruction);
      instruction.onclick = () => { shutdown = true; };

      video.srcObject = stream;
      await video.play();

      captureCanvas = document.createElement('canvas');
      captureCanvas.width = 640; //video.videoWidth;
      captureCanvas.height = 480; //video.videoHeight;
      window.requestAnimationFrame(onAnimationFrame);

      return stream;
    }
    async function stream_frame(label, imgData) {
      if (shutdown) {
        removeDom();
        shutdown = false;
        return '';
      }

      var preCreate = Date.now();
      stream = await createDom();

      var preShow = Date.now();
      if (label != "") {
        labelElement.innerHTML = label;
      }

      if (imgData != "") {
        var videoRect = video.getClientRects()[0];
        imgElement.style.top = videoRect.top + "px";
        imgElement.style.left = videoRect.left + "px";
        imgElement.style.width = videoRect.width + "px";
        imgElement.style.height = videoRect.height + "px";
        imgElement.src = imgData;
      }

      var preCapture = Date.now();
      var result = await new Promise(function(resolve, reject) {
        pendingResolve = resolve;
      });
      shutdown = false;

      return {'create': preShow - preCreate,
              'show': preCapture - preShow,
              'capture': Date.now() - preCapture,
              'img': result};
    }
    ''')

  display(js)

def video_frame(label, bbox):
  data = eval_js('stream_frame("{}", "{}")'.format(label, bbox))
  return data

"""### Running on Webcam Video"""

def capture_video(t):
  # start streaming video from webcam
  video_stream()
  # label for video
  label_html = 'Capturing...'
  # initialze bounding box to empty
  bbox = ''
  count = 0
  while True:
      js_reply = video_frame(label_html, bbox)
      if not js_reply:
          break

      # convert JS response to OpenCV Image
      frame = js_to_image(js_reply["img"])

      # create transparent overlay for bounding box
      bbox_array = np.zeros([480,640,4], dtype=np.uint8)

      # call our darknet helper on video frame
      detections, width_ratio, height_ratio = darknet_helper(frame, width, height)

      # loop through detections and draw them on transparent overlay image
      for label, confidence, bbox in detections:
        left, top, right, bottom = bbox2points(bbox)
        left, top, right, bottom = int(left * width_ratio), int(top * height_ratio), int(right * width_ratio), int(bottom * height_ratio)
        bbox_array = cv2.rectangle(bbox_array, (left, top), (right, bottom), class_colors[label], 2)
        bbox_array = cv2.putText(bbox_array, "{} [{:.2f}]".format(label, float(confidence)),
                          (left, top - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
                          class_colors[label], 2)

        t.tts(label+' detected')
        print(label+' detected')
        time.sleep(5)

      bbox_array[:,:,3] = (bbox_array.max(axis = 2) > 0 ).astype(int) * 255
      # convert overlay of bbox into bytes
      bbox_bytes = bbox_to_bytes(bbox_array)
      # update bbox so next frame gets new overlay
      bbox = bbox_bytes

import IPython
import time
from google.colab import output

RECORD = """
console.log('HERe')
console.log('Here');
console.log(window);
var synth = window.speechSynthesis;
console.log(synth)
window.SpeechRecognition = window.webkitSpeechRecognition || window.SpeechRecognition;
var recognition = new window.SpeechRecognition();
console.log(recognition)
speechToText = ""
started = false
stopped = false
recognition.onresult = function(event) {
  console.log('Event aya kya??');
  console.log(event);
  speechToText = event.results[0][0].transcript;
  window.som = speechToText
  console.log(window)

};
recognition.onspeechend = () => {
  recognition.stop();
  console.log('Stopped');
};

recognition.onstart = () => {
  console.log('Started');

};
 var display=(speechToText)=>{
   return speechToText;
};

console.log(recognition.start)
window.som = "Our current setter value"
recognition.start();
"""

def speak():
  display(IPython.display.Javascript(data=RECORD))
  time.sleep(10)
  data = output.eval_js('som')

  print(data)
  return data

speak()

t=TTS()
data=''
while 'exit' not in data:
  t.tts('Start Speaking')
  time.sleep(2)
  data=speak()
  t.tts('Speech Recorded')
  if 'capture video' in data:
    t.tts('Video Started Capturing')
    capture_video(t)
    t.tts('Process Completed')
  elif "tell me current location" in data:
    t.tts('Fetching current Location')
    l=Loc()
    ls=l.loca()
    t.tts(ls[2])
    print(ls[2])
    print("Loc")
    time.sleep(5)
    t.tts('Process Completed')
  elif "give me weather update" in data:
    t.tts('Fetching Weather Updates')
    w=Weather()
    w.give_wu()
    print("WU")
    t.tts('Process Completed')
  elif "send alert" in data:
    t.tts('Sending Alert')
    a=Alert()
    a.sendalert()
    t.tts("Alert sent")
    print("Alert")
    t.tts('Process Completed')
  elif "disable sound" in data:
    t.tts('Sound Disabled')
    time.sleep(5)
    t.tts('To Enable Sound, speak enable sound')
    time.sleep(4)
    while 'enable sound' not in data:
      data=speak()
      time.sleep(5)
    t.tts('Sound Enabled')
    continue
  elif 'exit' in data:
    t.tts('Exiting')
    continue
  else:
    t.tts('No Matches Found, You need to start speaking Again')
    time.sleep(5)

t.tts('Thank You For Your Time')


text:
import numpy as np

from hypothesis import given, settings
import hypothesis.strategies as st
import unittest

from caffe2.python import core
import caffe2.python.hypothesis_test_util as hu
import caffe2.python.serialized_test.serialized_test_util as serial

class TestNumpyTile(serial.SerializedTestCase):
    @given(ndim=st.integers(min_value=1, max_value=4),
           seed=st.integers(min_value=0, max_value=65536),
           **hu.gcs_cpu_only)
    @settings(deadline=10000)
    def test_numpy_tile(self, ndim, seed, gc, dc):
        np.random.seed(seed)

        input_dims = np.random.randint(1, 4, size=ndim)
        input = np.random.randn(*input_dims)
        repeats = np.random.randint(1, 5, size=ndim)

        op = core.CreateOperator(
            'NumpyTile', ['input', 'repeats'], 'out',
        )

        def tile_ref(input, repeats):
            tiled_data = np.tile(input, repeats)
            return (tiled_data,)

        # Check against numpy reference
        self.assertReferenceChecks(gc, op, [input, repeats],
                                   tile_ref)

    @given(ndim=st.integers(min_value=1, max_value=4),
           seed=st.integers(min_value=0, max_value=65536),
           **hu.gcs_cpu_only)
    @settings(deadline=10000)
    def test_numpy_tile_zero_dim(self, ndim, seed, gc, dc):
        np.random.seed(seed)

        input_dims = np.random.randint(0, 4, size=ndim)
        input = np.random.randn(*input_dims)
        repeats = np.random.randint(0, 5, size=ndim)

        op = core.CreateOperator(
            'NumpyTile', ['input', 'repeats'], 'out',
        )

        def tile_ref(input, repeats):
            tiled_data = np.tile(input, repeats)
            return (tiled_data,)

        # Check against numpy reference
        self.assertReferenceChecks(gc, op, [input, repeats],
                                   tile_ref)

if __name__ == "__main__":
    unittest.main()

ext: py

max_stars_repo_name: Hacky-DH/pytorch

max_issues_repo_name: Hacky-DH/pytorch

max_forks_repo_name: Hacky-DH/pytorch

reason: import numpy


text:
# -*- coding: utf-8 -*-
"""
Created on Mon May 30 11:45:37 2016

@author: root
"""

import numpy as np
import matplotlib.pyplot as plt

with open("pesmoc_0/hypervolume_solution.txt") as f:
    solution = f.read()
solution = float(solution)
data = np.array([])
data_random = np.array([])
iterations=100
number_executions=100
for i in range(1,number_executions,1):
    with open("pesmoc/hypervolumes_"+str(i)+".txt") as f:
        data_iteration = f.read()
    data_iteration = data_iteration.split('\n')[:-1]
    data = np.append(data,data_iteration)
    with open("random/hypervolumes_"+str(i)+".txt") as f:
        data_random_iteration = f.read()
    data_iteration_random= data_iteration_random.split('\n')[:-1]
    data_random = np.append(data_random,data_iteration_random)

data= data.reshape((iterations,number_executions))
data_random= data_random.reshape((iterations,number_executions))
data = np.mean(data,axis=0)
data_random = np.mean(data_random,axis=0)
data_var = np.var(data,axis=0)
data_random_var = np.var(data_random,axis=0)
data = [np.log(solution-float(i)) for i in data]
data_random = [np.log(solution-float(i)) for i in data_random]

fig = plt.figure()

ax1 = fig.add_subplot(111)

ax1.set_title("Hypervolumes")
ax1.set_xlabel('Iterations')
ax1.set_ylabel('Hypervolumes')

ax1.plot([i for i in range(0,len(data),1)],data)
ax1.plot([i for i in range(0,len(data_random),1)],data_random)

leg = ax1.legend(['PESMOC', 'RANDOM'], loc='upper right')

plt.show()
print len(data)

max_stars_repo_name: fernandezdaniel/Spearmint

max_issues_repo_name: fernandezdaniel/Spearmint

max_forks_repo_name: fernandezdaniel/Spearmint


text:
#!/bin/python
"""
Runs on-policy optimization of the degradable battery problem. States are aggregated
and the importance weights are derived from something that amounts to a limiting
distribution.
"""
import numpy as np
import raam
from raam import examples
from raam import crobust

## Configuration

horizon = 2000                  # maximal simulation horizon
runs = 1000                     # maximal number of runs
sample_count = 10000            # total maximal number of samples for constructing models
eval_sample_count = 10000       # total maximal number of samples for constructing models

discretization = 3
discount = 0.8

iterations = 5                  # number of iterations of sampling and optimization

print('Simulation parameter bounds: horizon %d, runs %d, samples %d, eval samples %d' \
        % (horizon,runs,sample_count,eval_sample_count))

## Solve on-policy with limiting weights

# construct an initial random policy
sim, create_srmdp, create_policy = examples.inventory.makesimulator(discretization,discount)

policy = sim.random_policy()

for i in range(iterations):
    print('Sampling ...')
    # transition limit makes sure that the simulation is fair in terms of the number of samples
    samples = sim.simulate(horizon, policy, runs, transitionlimit=sample_count)
    print('Return from samples:', samples.statistics(sim.discount)['mean_return'])

    srmdp = create_srmdp(samples)
    rmdp = srmdp.rmdp

    print('Solving aggregated MDP...')
    valfun,policy_vec,residual,iterations = rmdp.mpi_jac(300,maxresidual=1e-4,stype=0)

    print('Constructing policies ...')
    policy = create_policy(srmdp,policy_vec)

    #TODO: Why is this consistently lower than the first value?
    print('Evaluating ...')
    samples = sim.simulate(horizon, policy, runs, probterm=1-sim.discount, \
                            transitionlimit=eval_sample_count)
    print('Evaluation:', samples.statistics(1)['mean_return'])

max_stars_repo_name: marekpetrik/RAAM

max_issues_repo_name: rlsquared/RAAM

max_forks_repo_name: rlsquared/RAAM


text:
"""
EM example: Gaussian Mixture Models
-----------------------------------
Figure 6.6

A two-dimensional mixture of Gaussians for the stellar metallicity data. The
left panel shows the number density of stars as a function of two measures of
their chemical composition: metallicity ([Fe/H]) and alpha-element abundance
([alpha/Fe]). The right panel shows the density estimated using mixtures of
Gaussians together with the positions and covariances (2-sigma levels) of
those Gaussians. The center panel compares the information criteria AIC
and BIC (see Sections 4.3.2 and 5.4.3).
"""
# Author: Jake VanderPlas
# License: BSD
#   The figure produced by this code is published in the textbook
#   "Statistics, Data Mining, and Machine Learning in Astronomy" (2013)
#   For more information, see http://astroML.github.com
#   To report a bug or issue, use the following forum:
#    https://groups.google.com/forum/#!forum/astroml-general

from __future__ import print_function

import numpy as np
from matplotlib import pyplot as plt

from sklearn.mixture import GaussianMixture

from astroML.datasets import fetch_sdss_sspp
from astroML.utils.decorators import pickle_results
from astroML.plotting.tools import draw_ellipse

#----------------------------------------------------------------------
# This function adjusts matplotlib settings for a uniform feel in the textbook.
# Note that with usetex=True, fonts are rendered with LaTeX.  This may
# result in an error if LaTeX is not installed on your system.  In that case,
# you can set usetex to False.
if "setup_text_plots" not in globals():
    from astroML.plotting import setup_text_plots
setup_text_plots(fontsize=8, usetex=True)

#------------------------------------------------------------
# Get the Segue Stellar Parameters Pipeline data
data = fetch_sdss_sspp(cleaned=True)
X = np.vstack([data['FeH'], data['alphFe']]).T

# truncate dataset for speed
X = X[::5]

#------------------------------------------------------------
# Compute GaussianMixture models & AIC/BIC
N = np.arange(1, 14)

@pickle_results("GMM_metallicity.pkl")
def compute_GaussianMixture(N, covariance_type='full', max_iter=1000):
    models = [None for n in N]
    for i in range(len(N)):
        print(N[i])
        models[i] = GaussianMixture(n_components=N[i], max_iter=max_iter,
                                    covariance_type=covariance_type)
        models[i].fit(X)
    return models

models = compute_GaussianMixture(N)

AIC = [m.aic(X) for m in models]
BIC = [m.bic(X) for m in models]

i_best = np.argmin(BIC)
gmm_best = models[i_best]
print("best fit converged:", gmm_best.converged_)
print("BIC: n_components =  %i" % N[i_best])

#------------------------------------------------------------
# compute 2D density
FeH_bins = 51
alphFe_bins = 51
H, FeH_bins, alphFe_bins = np.histogram2d(data['FeH'], data['alphFe'],
                                          (FeH_bins, alphFe_bins))

Xgrid = np.array(list(map(np.ravel,
                          np.meshgrid(0.5 * (FeH_bins[:-1]
                                             + FeH_bins[1:]),
                                      0.5 * (alphFe_bins[:-1]
                                             + alphFe_bins[1:]))))).T
log_dens = gmm_best.score_samples(Xgrid).reshape((51, 51))

#------------------------------------------------------------
# Plot the results
fig = plt.figure(figsize=(5, 1.66))
fig.subplots_adjust(wspace=0.45,
                    bottom=0.25, top=0.9,
                    left=0.1, right=0.97)

# plot density
ax = fig.add_subplot(131)
ax.imshow(H.T, origin='lower', interpolation='nearest', aspect='auto',
          extent=[FeH_bins[0], FeH_bins[-1],
                  alphFe_bins[0], alphFe_bins[-1]],
          cmap=plt.cm.binary)
ax.set_xlabel(r'$\rm [Fe/H]$')
ax.set_ylabel(r'$\rm [\alpha/Fe]$')
ax.xaxis.set_major_locator(plt.MultipleLocator(0.3))
ax.set_xlim(-1.101, 0.101)
ax.text(0.93, 0.93, "Input",
        va='top', ha='right', transform=ax.transAxes)

# plot AIC/BIC
ax = fig.add_subplot(132)
ax.plot(N, AIC, '-k', label='AIC')
ax.plot(N, BIC, ':k', label='BIC')
ax.legend(loc=1)
ax.set_xlabel('N components')
plt.setp(ax.get_yticklabels(), fontsize=7)

# plot best configurations for AIC and BIC
ax = fig.add_subplot(133)
ax.imshow(np.exp(log_dens),
          origin='lower', interpolation='nearest', aspect='auto',
          extent=[FeH_bins[0], FeH_bins[-1],
                  alphFe_bins[0], alphFe_bins[-1]],
          cmap=plt.cm.binary)

ax.scatter(gmm_best.means_[:, 0], gmm_best.means_[:, 1], c='w')
for mu, C, w in zip(gmm_best.means_, gmm_best.covariances_, gmm_best.weights_):
    draw_ellipse(mu, C, scales=[1.5], ax=ax, fc='none', ec='k')

ax.text(0.93, 0.93, "Converged",
        va='top', ha='right', transform=ax.transAxes)

ax.set_xlim(-1.101, 0.101)
ax.set_ylim(alphFe_bins[0], alphFe_bins[-1])
ax.xaxis.set_major_locator(plt.MultipleLocator(0.3))
ax.set_xlabel(r'$\rm [Fe/H]$')
ax.set_ylabel(r'$\rm [\alpha/Fe]$')

plt.show()


text:
from __future__ import unicode_literals

from ncephes import ffi
from numpy.testing import assert_almost_equal

from ncephes import airy, hyp2f1

def test_airy():
    ai = ffi.new("double *")
    aip = ffi.new("double *")
    bi = ffi.new("double *")
    bip = ffi.new("double *")
    airy(0.2, ai, aip, bi, bip)
    assert_almost_equal(
        [ai[0], aip[0], bi[0], bip[0]],
        [
            0.30370315428638206,
            -0.25240547028561955,
            0.7054642029186613,
            0.46178928436215094,
        ],
    )

def test_hyp2f1():
    assert_almost_equal(hyp2f1(0.2, 1.1, 0.3, -1), 0.6248283119898905)

max_stars_repo_name: glimix/ncephes

max_issues_repo_name: limix/ncephes-py

max_forks_repo_name: glimix/ncephes

reason: from numpy


text:
import torch
from torch.utils import data
import json
import os
import numpy as np
import random
import soundfile as sf
import random
import torchaudio

class MedleydbDataset(data.Dataset):
    """MedleyDB: a dataset of annotated, royaltyfree multitrack recordings.

    It provides a stereo mix and both dry and processed multitrack stems for
    each song in the dataset. The dataset covers a wide distribution of genres
    and primarily consists of full length songs with professional
    or nearprofessional audio quality.

    The dataset consists of 254 full lengths music tracks of
    different genres. It provides a stereo mix and both dry and processed
    multitrack stems for each song.

    This dataset contains a .yml object for each multitrack containining
    metadata for each audio file in the multitrack. This dataloader utilises
    this metadata to select audio files and downmix them to create mixtures
    of instruments belonging to a list of instrument tags with specified
    polyphony.

    Args:
        json_dir (str): Path containing .json file with list of audio files
        n_src (int): Number of separate instrument classes (yet to be implemented)
        n_poly (int): Number of instances of each class to be present in mixture.
        segment (float, optional): Duration of segments in seconds,
            defaults to ``None`` which loads the full-length audio tracks.
        sample_rate (int, optional): Samplerate of files in dataset.
        threshold (float): activity confidence threshold to exclude segments with
            less than given fraction of time of activity in segment.

    References:
        "MedleyDB: A Multitrack Dataset for Annotation-Intensive MIR Research",
            R. Bittner et. al. ISMIR 2014

    """

    dataset_name = "MedleyDB"

    def __init__(self, json_dir, n_src=1, n_poly=2, sample_rate=44100, segment=5.0, threshold=0.1):
        super(MedleydbDataset, self).__init__()

        self.json_dir = json_dir
        self.sample_rate = sample_rate
        self.n_poly = n_poly
        self.threshold = threshold
        self.seg_len = int(segment)
        self.n_src = n_src
        self.like_test = self.seg_len is None
        # Load json files
        sources_json = [
            os.path.join(json_dir, source + ".json")
            for source in [f"inst{n+1}" for n in range(n_src)]
        ]
        sources_conf = []
        for src_json in sources_json:
            with open(src_json, "r") as f:
                sources_conf = np.array(json.load(f))

        # Filter out utterances with activity less than threshold
        drop_utt, drop_len, orig_len = 0, 0, 0
        sources_infos = []
        index_array = []

        for i in range(len(sources_conf)):
            conf = sources_conf[i][1]
            duration = sources_conf[i][1][-1][0]
            index_array.append(np.zeros(int(duration // segment) + 1))
            for timestamp, confidence in conf:
                j = int(timestamp // segment)
                index_array[i][j] = index_array[i][j] + confidence
            orig_len = orig_len + duration
            seg_dur = duration / len(index_array[i])
            #save list of segments with sufficient activity
            for k in range(len(index_array[i])):
                conf_thresh = threshold * float(len(sources_conf[i][0]))
                if index_array[i][k] < conf_thresh:
                    drop_utt += 1
                    drop_len += seg_dur
                    continue
                else:
                    sources_infos.append((sources_conf[i][0], k, sources_conf[i][2]))

        print(
            "Drop {} utts ({:.2f} h) from ({:.2f} h) with less than {} percent activity".format(
                drop_utt, drop_len / 3600, orig_len / 3600, threshold
            )
        )
        self.sources = sources_infos

    def __len__(self):
        return len(self.sources)

    def __getitem__(self, idx):
        """ Gets a mixture/sources pair.
        Returns:
            mixture, vstack([source_arrays])
        """
        # Load sources
        source_arrays = []

        for i in range(self.n_poly):
            if i:
                #idx = random.choice(range(len(self.sources)))
                idx = int((idx + ((len(self.sources)*i/self.n_poly))) % len(self.sources))

            start = self.sources[idx][1] * self.sources[idx][2]
            if self.like_test:
                stop = None
            else:
                stop = start + (self.seg_len * self.sources[idx][2])

            if self.sources[idx] is None:
                # Target is filled with zeros if n_src > default_nsrc
                s = np.zeros((self.seg_len * self.sources[idx][2],))
            else:
                s, sr = sf.read(self.sources[idx][0], start=start, stop=stop, dtype="float32")
            source_arrays.append(s)
        source = torch.from_numpy(np.vstack(source_arrays))
        if sr is not self.sample_rate:
            source = torchaudio.transforms.Resample(sr, self.sample_rate)(source)
        mix = torch.stack(list(source)).sum(0)
        return mix, source

    def get_infos(self):
        """ Get dataset infos (for publishing models).

        Returns:
            dict, dataset infos with keys `dataset`, `task` and `licences`.
        """
        infos = dict()
        infos["dataset"] = self.dataset_name
        infos["task"] = "harmony_sep"
        infos["licenses"] = [mdb_license]
        return infos

class SourceFolderDataset(data.Dataset):
    dataset_name = "SourceFolder"

    def __init__(self, json_dir, wav_dir, n_src=1, sample_rate=44100, batch_size=1, train=False):
        super(SourceFolderDataset, self).__init__()
        # Task setting
        self.json_dir = json_dir
        self.wav_dir = wav_dir
        self.sample_rate = sample_rate
        self.n_src = n_src
        self.like_test = True
        self.batch_size = batch_size
        self.train = train

        sources_infos = []
        # Make and load json files
        speaker_list = ["mix"] + [f"s{n+1}" for n in range(n_src)]
        for speaker in speaker_list:
            preprocess_one_dir(
                os.path.join(wav_dir, speaker), json_dir, speaker, sample_rate=sample_rate
            )

        sources_json = [
            os.path.join(json_dir, source + ".json") for source in [f"s{n+1}" for n in range(n_src)]
        ]
        #def sort(infos): return sorted(
        #    infos, key=lambda info: info[0], reverse=True)
        mix_json = os.path.join(json_dir, "mix.json")
        with open(mix_json, "r") as f:
            mix_infos = json.load(f)

        for src_json in sources_json:
            with open(src_json, "r") as f:
                temp = json.load(f)
                #if train:
                    #split = int(len(temp)/4)
                    #temp1 = temp[:-split]
                    #random.shuffle(temp1)
                    #temp[:-split] = temp1
                sources_infos.append(temp)

        sources_infos = np.array(sources_infos)
        self.mix = mix_infos
        self.sources = sources_infos

    def __len__(self):
        return len(self.sources[0])

    def __getitem__(self, idx):
        """ Gets a mixture/sources pair.
        Returns:
            mixture, vstack([source_arrays])
        """
        # Load mixture
        #x, sr = sf.read(self.mix[idx][0], dtype="float32")
        #seg_len = torch.as_tensor([len(x)])
        seg_len = 220500
        # Load sources
        source_arrays = []
        for src in self.sources:
            if src[idx] is None:
                # Target is filled with zeros if n_src > default_nsrc
                s = np.zeros((seg_len,))
            else:
                s, sr = sf.read(src[idx][0], dtype="float32")
                seg_len = torch.as_tensor([len(s)])
            source_arrays.append(s)
        source = torch.from_numpy(np.vstack(source_arrays))

        #mix = torch.from_numpy(x)
        #mix = mix.unsqueeze(0)
        if sr is not self.sample_rate:
            source = torchaudio.transforms.Resample(sr, self.sample_rate)(source)
            #mix = torchaudio.transforms.Resample(sr, self.sample_rate)(mix)
        mix = torch.stack(list(source)).sum(0)
        #mix = mix.unsqueeze(0)
        #source = source.unsqueeze(0)

        return mix, source

    def get_infos(self):
        """ Get dataset infos (for publishing models).

        Returns:
            dict, dataset infos with keys `dataset`, `task` and `licences`.
        """
        infos = dict()
        infos["dataset"] = self.dataset_name
        infos["task"] = "harmony_sep_eval"
        infos["licenses"] = [mdb_license]
        return infos

def preprocess_one_dir(in_dir, out_dir, out_filename, sample_rate=44100):
    file_infos = []
    in_dir = os.path.abspath(in_dir)
    wav_list = os.listdir(in_dir)
    for wav_file in wav_list:
        if not wav_file.endswith(".wav"):
            continue
        wav_path = os.path.join(in_dir, wav_file)
        samples, _ = sf.read(wav_path)
        file_infos.append((wav_path, len(samples)))
    def sort(infos): return sorted(
            infos, key=lambda info: info[0], reverse=False)

    new_infos = sort(file_infos)

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)
    with open(os.path.join(out_dir, out_filename + ".json"), "w") as f:
        json.dump(new_infos, f, indent=4)

mdb_license = dict(
    title="MedleyDB: A Multitrack Dataset for Annotation-Intensive MIR Research",
    title_link="https://medleydb.weebly.com/",
    author="R. Bittner, J. Salamon, M. Tierney, M. Mauch, C. Cannam and J. P. Bello",
    license="Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License",
    license_link="https://creativecommons.org/licenses/by-nc-sa/4.0/",
    non_commercial=True,
)

max_stars_repo_name: saurjya/asteroid

max_issues_repo_name: saurjya/asteroid

max_forks_repo_name: saurjya/asteroid


text:
import unittest
import os
from os import listdir
from os.path import join
from os import remove
import shutil
from numpy import squeeze
from pathlib import Path

from ridt.config import ConfigFileParser

from ridt.equation import EddyDiffusion

from ridt.container import Domain
from ridt.container.eddydiffusionrun import EddyDiffusionRun

from ridt.data import BatchDataStore

class ST17(unittest.TestCase):

    """System Test 17. test the system can use
       the information in the parsed JSON configuration
       file to evaluate the specified Eddy Diffusion model
       for all times, monitor locations, full domain, and
       points in parameter space (as defined by the ranges
       in the configuration file)."""

    def setUp(self) -> None:

        this_dir = os.path.dirname(os.path.abspath(__file__))
        self.output_dir = os.path.join(this_dir, "st17")
        Path(self.output_dir).mkdir(parents=True, exist_ok=True)
        with ConfigFileParser(os.path.join(this_dir, "st16/explicit.json")) as cfp:
            self.c = cfp

        self.bds = BatchDataStore()
        self.domain = Domain(self.c)
        self.edr = EddyDiffusionRun(self.c, self.output_dir)

    def tearDown(self) -> None:
        shutil.rmtree(self.output_dir)

    def test_verify(self):

        for val in self.edr.data_store.store.values():
            for point in val.points.values():
                self.assertEqual(len(point), self.c.time_samples)
            for line_name, line in val.lines.items():
                monitor = self.c.models.eddy_diffusion.monitor_locations.lines[line_name]
                axis = getattr(monitor, "parallel_axis")
                shape = (self.c.time_samples, getattr(self.c.spatial_samples, axis))
                self.assertEqual(line.shape, shape)
            for plane_name, plane in val.planes.items():
                monitor = self.c.models.eddy_diffusion.monitor_locations.planes[plane_name]
                axis = getattr(monitor, "axis")
                shape = [self.c.time_samples, 0, 0]
                for i, dim in enumerate(list(axis)):
                    shape[i + 1] = getattr(self.c.spatial_samples, dim)
                shape = tuple(shape)
                self.assertEqual(plane.shape, shape)

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: riskaware-ltd/ridt

max_issues_repo_name: riskaware-ltd/ridt

max_forks_repo_name: riskaware-ltd/ridt


text:
# ***************************************************************
# Copyright(c) 2019
#     Meng-Hao Guo <guomenghao1997@gmail.com>
#     Dun Liang <randonlang@gmail.com>.
# All Rights Reserved.
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.
# ***************************************************************

import numpy as np
import os
from PIL import Image
import matplotlib.pyplot as plt
from jittor.dataset.dataset import Dataset, dataset_root
import jittor as jt
import os
import os.path as osp

from PIL import Image, ImageOps, ImageFilter

import numpy as np
import scipy.io as sio

import random

def fetch(image_path, label_path=None):
    with open(image_path, 'rb') as fp:
        image = Image.open(fp).convert('RGB')

    with open(label_path, 'rb') as fp:
        label = Image.open(fp).convert('P')

    return image, label

def scale(image, label=None):
    SCALES = (0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0)
    ratio = np.random.choice(SCALES)
    w,h = image.size
    nw = (int)(w*ratio)
    nh = (int)(h*ratio)

    image = image.resize((nw, nh), Image.BILINEAR)
    label = label.resize((nw, nh), Image.NEAREST)

    return image, label

def pad(image, label=None):
    w,h = image.size

    crop_size = 513
    pad_h = max(crop_size - h, 0)
    pad_w = max(crop_size - w, 0)
    image = ImageOps.expand(image, border=(0, 0, pad_w, pad_h), fill=0)
    label = ImageOps.expand(label, border=(0, 0, pad_w, pad_h), fill=255)

    return image, label

def crop(image, label=None):
    w, h = image.size
    crop_size = 513
    x1 = random.randint(0, w - crop_size)
    y1 = random.randint(0, h - crop_size)
    image = image.crop((x1, y1, x1 + crop_size, y1 + crop_size))
    label = label.crop((x1, y1, x1 + crop_size, y1 + crop_size))

    return image, label

def normalize(image, label):
    mean = (0.485, 0.456, 0.40)
    std = (0.229, 0.224, 0.225)
    image = np.array(image).astype(np.float32)
    label = np.array(label).astype(np.float32)

    image /= 255.0
    image -= mean
    image /= std
    return image, label

def flip(image, label=None):
    if random.random() < 0.5:
        image = image.transpose(Image.FLIP_LEFT_RIGHT)
        label = label.transpose(Image.FLIP_LEFT_RIGHT)
    return image, label

class BaseDataset(Dataset):
    def __init__(self,  data_root=dataset_root+'/voc/', split='train', batch_size=1, shuffle=False):
        super().__init__()
        ''' total_len , batch_size, shuffle must be set '''
        self.data_root = data_root
        self.split = split
        self.batch_size = batch_size
        self.shuffle = shuffle

        self.image_root = os.path.join(data_root, 'images')
        self.label_root = os.path.join(data_root, 'annotations')

        self.data_list_path = os.path.join(self.data_root, self.split + '.txt')
        self.image_path = []
        self.label_path = []

        with open(self.data_list_path, "r") as f:
            lines = f.read().splitlines()

        for idx, line in enumerate(lines):
            _img_path = os.path.join(self.image_root, line + '.jpg')
            _label_path = os.path.join(self.label_root, line + '.png')

            assert os.path.isfile(_img_path)
            assert os.path.isfile(_label_path)
            self.image_path.append(_img_path)
            self.label_path.append(_label_path)
        self.total_len = len(self.image_path)

        # set_attr must be called to set batch size total len and shuffle like __len__ function in pytorch
        self.set_attr(batch_size = self.batch_size, total_len = self.total_len, shuffle = self.shuffle) # bs , total_len, shuffle

    def __getitem__(self, image_idimage_id):
        return NotImplementedError

class TrainDataset(BaseDataset):
    def __init__(self,  data_root=dataset_root+'/voc/', split='train', batch_size=1, shuffle=False):
        super(TrainDataset, self).__init__(data_root, split, batch_size, shuffle)

    def __getitem__(self, image_id):
        image_path = self.image_path[image_id]
        label_path = self.label_path[image_id]
        image, label = fetch(image_path, label_path)
        image, label = scale(image, label)
        image, label = pad(image, label)
        image, label = crop(image, label)
        image, label = flip(image, label)
        image, label = normalize(image, label)

        image = np.array(image).astype(np.float).transpose(2, 0, 1)
        image = jt.array(image)
        label = jt.array(np.array(label).astype(np.int))

        return image, label

class ValDataset(BaseDataset):
    def __init__(self,  data_root=dataset_root+'/voc/', split='train', batch_size=1, shuffle=False):
        super(ValDataset, self).__init__(data_root, split, batch_size, shuffle)

    def __getitem__(self, image_id):
        image_path = self.image_path[image_id]
        label_path = self.label_path[image_id]

        image, label = fetch(image_path, label_path)
        image, label = normalize(image, label)

        image = np.array(image).astype(np.float).transpose(2, 0, 1)
        image = jt.array(image)
        label = jt.array(np.array(label).astype(np.int))

        return image, label

def test_dt():

    train_loader = TrainDataset(data_root='/vocdata/', split='train', batch_size=4, shuffle=True)
    val_loader = ValDataset(data_root='/vocdata/', split='val', batch_size=1, shuffle=False)
    print('train', len(train_loader))
    for idx, (image, target) in enumerate(train_loader):
        if idx > 5 :
            break
        print (image.shape, target.shape)

    for idx, (image, target) in enumerate(val_loader):
        if idx > 5 :
            break
        print (image.shape, target.shape)

if __name__ == '__main__':
    test_dt()

max_stars_repo_name: qixuxiang/jittor-models

max_issues_repo_name: qixuxiang/jittor-models

max_forks_repo_name: qixuxiang/jittor-models

reason: import numpy,import scipy


text:
from drain.step import Step
from drain import util, data
from drain.data import FromSQL, Merge

import pandas as pd
import numpy as np
import logging

kid_addresses = FromSQL(table='output.kid_addresses')
kid_addresses.target = True

kids = FromSQL(table='output.kids')
kids.target = True

addresses = FromSQL(table='output.addresses')
addresses.target = True

class LeadLeft(Step):
    def __init__(self, month, day, year_min):
        Step.__init__(self, month=month, day=day, year_min=year_min)

        aux = Merge(on='kid_id', inputs=[kid_addresses, kids])
        self.inputs = [aux, addresses]

    def run(self, aux, addresses):
        min_date = util.timestamp(self.year_min, self.month, self.day)
        aux.dropna(subset=['date_of_birth'], inplace=True)
        aux.drop(aux.index[aux.date_of_birth < min_date], inplace=True)
        # Date stuff
        logging.info('dates')
        aux['date'] = aux.date_of_birth.apply(
                util.date_ceil(self.month, self.day))

        # if bll6 happens before dob.date_ceil() use date_floor instead
        bll6_before_date = aux.first_bll6_sample_date < aux.date
        aux.loc[bll6_before_date, 'date'] =  aux.loc[bll6_before_date,
                'first_bll6_sample_date'].apply(
                    util.date_floor(self.month, self.day))

        columns = aux.columns
        aux = aux.merge(addresses, on='address_id')

        left_columns = ['kid_id', 'date'] + list(addresses.columns)
        left_columns.remove('address')
        left = aux[left_columns]

        return {'left':left, 'aux':aux}


text:
import jax
import pytest

from jax_resnet import *  # noqa

# github.com/google/flax/blob/master/linen_examples/imagenet/models.py
# matches https://pytorch.org/hub/pytorch_vision_resnet/
RESNET_PARAM_COUNTS = {
    18: 11689512,
    34: 21797672,
    50: 25557032,
    101: 44549160,
    152: 60192808,
}

# https://pytorch.org/hub/pytorch_vision_wide_resnet/
WIDE_RESNET_PARAM_COUNTS = {50: 68883240, 101: 126886696}

# https://pytorch.org/hub/pytorch_vision_resnext/
RESNEXT_PARAM_COUNTS = {50: 25028904, 101: 88791336}

# https://docs.fast.ai/vision.models.xresnet.html
RESNETD_PARAM_COUNTS = {
    18: 11708744,
    34: 21816904,
    50: 25576264,
    101: 44568392,
    152: 60212040,
}

# github.com/zhanghang1989/ResNeSt (torch version)
RESNEST_PARAM_COUNTS = {50: 27483240, 101: 48275016, 200: 70201544, 269: 110929480}

def n_params(model, init_shape=(1, 224, 224, 3)):
    init_array = jnp.ones(init_shape, jnp.float32)
    params = model.init(jax.random.PRNGKey(0), init_array)['params']
    return sum(map(jnp.size, jax.tree_leaves(params)))

@pytest.mark.parametrize('size', [18, 34, 50, 101, 152])
def _test_resnet_param_count(size):
    model = eval(f'ResNet{size}')(n_classes=1000)
    assert n_params(model) == RESNET_PARAM_COUNTS[size]

@pytest.mark.parametrize('size', [18, 50])
def test_resnet_param_count(size):
    _test_resnet_param_count(size)

@pytest.mark.slow
@pytest.mark.parametrize('size', [34, 101, 152])
def test_resnet_param_count_slow(size):
    _test_resnet_param_count(size)

@pytest.mark.parametrize('size', [50])
def test_wide_resnet_param_count(size):
    model = eval(f'WideResNet{size}')(n_classes=1000)
    assert n_params(model) == WIDE_RESNET_PARAM_COUNTS[size]

@pytest.mark.slow
@pytest.mark.parametrize('size', [101])
def test_wide_resnet_param_count_slow(size):
    model = eval(f'WideResNet{size}')(n_classes=1000)
    assert n_params(model) == WIDE_RESNET_PARAM_COUNTS[size]

@pytest.mark.parametrize('size', [50])
def test_resnext_param_count(size):
    model = eval(f'ResNeXt{size}')(n_classes=1000)
    assert n_params(model) == RESNEXT_PARAM_COUNTS[size]

@pytest.mark.slow
@pytest.mark.parametrize('size', [101])
def test_resnext_param_count_slow(size):
    model = eval(f'ResNeXt{size}')(n_classes=1000)
    assert n_params(model) == RESNEXT_PARAM_COUNTS[size]

def _test_resnetd_param_count(size):
    model = eval(f'ResNetD{size}')(n_classes=1000)
    assert n_params(model) == RESNETD_PARAM_COUNTS[size]

@pytest.mark.parametrize('size', [18, 50])
def test_resnetd_param_count(size):
    _test_resnetd_param_count(size)

@pytest.mark.slow
@pytest.mark.parametrize('size', [34, 101, 152])
def test_resnetd_param_count_slow(size):
    _test_resnetd_param_count(size)

@pytest.mark.parametrize('size', [50, 101, 200, 269])
def _test_resnest_param_count(size):
    block_cls = partial(ResNeStBottleneckBlock,
                        splat_cls=partial(SplAtConv2d, match_reference=True))
    model = eval(f'ResNeSt{size}')(n_classes=1000, block_cls=block_cls)
    assert n_params(model) == RESNEST_PARAM_COUNTS[size]

@pytest.mark.parametrize('size', [50])
def test_resnest_param_count(size):
    _test_resnest_param_count(size)

@pytest.mark.slow
@pytest.mark.parametrize('size', [101, 200, 269])
def test_resnest_param_count_slow(size):
    _test_resnest_param_count(size)

def test_resnest_fast_param_count():
    block_cls = partial(ResNeStBottleneckBlock,
                        splat_cls=partial(SplAtConv2d, match_reference=True))
    model = ResNeSt50Fast(n_classes=1000, block_cls=block_cls)
    # From github.com/zhanghang1989/ResNeSt/blob/master/resnest/torch/ablation.py#L48
    assert n_params(model) == 27483240

def test_splatconv2d_param_count():
    splat = SplAtConv2d(128, (3, 3), padding=[(1, 1), (1, 1)], match_reference=True)
    assert n_params(splat, init_shape=(1, 28, 28, 128)) == 172992

@pytest.mark.parametrize('start,end', [(0, 5), (0, None), (0, -3), (4, -2), (3, -1),
                                       (2, None)])
def test_slice_variables(start, end):
    model = ResNet18(n_classes=10)
    key = jax.random.PRNGKey(0)

    variables = model.init(key, jnp.ones((1, 224, 224, 3)))
    sliced_vars = slice_variables(variables, start, end)
    sliced_model = Sequential(model.layers[start:end])

    # Need the correct number of input channels for slice:
    first = variables['params'][f'layers_{start}']['ConvBlock_0']['Conv_0']['kernel']
    slice_inp = jnp.ones((1, 224, 224, first.shape[2]))
    exp_sliced_vars = sliced_model.init(key, slice_inp)

    assert set(sliced_vars['params'].keys()) == set(exp_sliced_vars['params'].keys())
    assert set(sliced_vars['batch_stats'].keys()) == set(
        exp_sliced_vars['batch_stats'].keys())

    assert jax.tree_map(jnp.shape,
                        sliced_vars) == jax.tree_map(jnp.shape, exp_sliced_vars)

    sliced_model.apply(sliced_vars, slice_inp, mutable=False)

max_stars_repo_name: ruotianluo/jax-resnet

max_issues_repo_name: ruotianluo/jax-resnet

max_forks_repo_name: ruotianluo/jax-resnet

reason: import jax,from jax


text:
import numpy as np
import cv2
from pyzbar.pyzbar import decode

cap = cv2.VideoCapture(0)
cap.set(3, 640)
cap.set(4, 480)

while True:
    success, img = cap.read()

    for barcode in decode(img):
        myData = barcode.data.decode('utf-8')
        print(myData)
        pts = np.array([barcode.polygon], np.int32)
        pts = pts.reshape((-1, 1, 2))
        cv2.polylines(img, [pts], True, (255, 0, 255), 5)
        pts2 = barcode.rect
        cv2.putText(img, myData,(pts2[0],pts2[1]),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 0, 255), 2)

    cv2.imshow('frame', img)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        out = cv2.imwrite('../capture.jpg', img)
        break

cap.release()
cv2.destroyAllWindows()

max_stars_repo_name: ati-n/qr-code-scanner

max_issues_repo_name: ati-n/qr-code-scanner

max_forks_repo_name: ati-n/qr-code-scanner


text:
import sys
import os
import struct
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import PIL
import torch
import torchvision.transforms.functional as F
from torch.utils.data import Dataset, DataLoader
from sklearn import datasets

import geometry as geo

SUBJECTS = ['P{:d}'.format(i) for i in range(9)]
GESTURES = '1 2 3 4 5 6 7 8 9 I IP L MP RP T TIP Y'.split(' ')

def load_image_bin(f_name):
  f = open(f_name, 'rb')
  header = f.read(6*4)
  header = struct.unpack('6I', header)
  n_rows = header[5] - header[3]
  n_cols = header[4] - header[2]

  body = f.read(n_rows*n_cols*4)
  body = struct.unpack(str(n_rows*n_cols)+'f', body)
  img = np.array(body).reshape((n_rows, n_cols)).astype(np.float32)
  return img

def all_depth_and_joints(subjects, gestures):
  depth_files = []
  jts_list = []
  for subject in subjects:
    for gesture in gestures:
      joint_file = os.path.join('MSRA', subject, gesture, 'joint.txt')
      jts = load_joints(joint_file)
      jts_list += [jts]
      depth_files += [os.path.join('MSRA', subject, gesture, '{:06d}_depth.bin'.format(i)) for i in range(jts.shape[0])]
  return depth_files, np.concatenate(jts_list, axis=0)

def load_joints(f):
  jts = pd.read_csv(f, sep=' ', header=None, skiprows=[0])
  return np.float32(jts)

def get_hand(subject, gesture, idx=0):
  ds = MSRADataset(subjects=[subject], gestures=[gesture], max_buffer=4, image=True)
  dl = DataLoader(ds, batch_size=idx+1, shuffle=False)
  batch = dl.__iter__().__next__()
  pose = batch['jts'].detach().cpu().numpy()[idx]
  img = batch['img'].detach().cpu().numpy()[idx,0]
  return img, pose

class MSRADataset(Dataset):
  def __init__(self, subjects=None, gestures=None, image=False, angles=False, max_buffer=4, size=64, rotate=False):
    subjects = SUBJECTS if subjects is None else subjects
    gestures = GESTURES if gestures is None else gestures

    self.n_dim = 63
    self.image = image
    self.angles = angles
    self.max_buffer = max_buffer
    self.size = size
    self.rotate = rotate
    self.depth_files, self.joints = all_depth_and_joints(subjects, gestures)

  def __len__(self):
    return len(self.depth_files)

  def __getitem__(self, idx):

    jts = self.joints[idx].reshape((-1, 3))
    jts -= np.mean(jts, axis=0)[None,:]
    jts = jts/80

    if self.rotate:
      q = geo.random_quaternion()
      jts = geo.quaternion_rotation(q, jts)
      jts -= np.mean(jts, axis=0)[None,:]

    if self.angles:
      #azim, elev = geo.get_angles(jts)
      q = geo.get_quaternion(jts)
      q = torch.from_numpy(q.astype(np.float32))

    jts_tens = torch.from_numpy(jts.flatten())

    if self.image:
      img = load_image_bin(self.depth_files[idx])
      img = 255*img/img.max()
      pil_img = PIL.Image.fromarray(img)
      width, height = pil_img.size

      max_dim = np.max([width, height]) + np.random.randint(0, self.max_buffer+1)
      delta_width = max_dim - width
      delta_height = max_dim - height

      left = np.random.randint(0, delta_width+1)
      top = np.random.randint(0, delta_height+1)

      padded = F.pad(pil_img, (left, top, delta_width-left, delta_height-top))
      resized = F.resize(padded, self.size)
      gray = F.to_grayscale(resized)
      img_tens = F.to_tensor(gray)

      return {'img': img_tens, 'jts': jts_tens}

    else:
      if self.angles:
        return {'jts': jts_tens, 'q': q}

      else:
        return jts_tens

class Moon(Dataset):
  def __init__(self):
      self.x = datasets.make_moons(n_samples=30000, shuffle=True, noise=0.05)[0].astype(np.float32)
      self.n_dim = self.x.shape[1]

  def __len__(self):
      return self.x.shape[0]

  def __getitem__(self, idx):
      return torch.from_numpy(self.x[idx])

if __name__ == '__main__':

  ds = MSRADataset(image=True)

  dl = DataLoader(
    ds,
    num_workers=4,
    batch_size=1,
    shuffle=True)

  for d in dl:
    jts = d['jts'].detach().cpu().numpy()
    jt_mat = jts.reshape((-1, 3))
    print(jt_mat.std(axis=0))
    print(jt_mat.mean(axis=0))

max_stars_repo_name: ekrim/latent-tracking

max_issues_repo_name: ekrim/latent-tracking

max_forks_repo_name: ekrim/latent-tracking


text:
import atexit
import dataclasses
import csv
from typing import Dict, Iterable, Optional

import numpy as np

from molpal.objectives.base import Objective

import pyscreener as ps

class DockingObjective(Objective):
    """A DockingObjective calculates the objective function by calculating the
    docking score of a molecule

    Attributes
    ----------
    c : int
        the min/maximization constant, depending on the objective
    virtual_screen : pyscreener.docking.VirtualScreen
        the VirtualScreen object that calculated docking scores of molecules against a given
        receptor with specfied docking parameters

    Parameters
    ----------
    objective_config : str
        the path to a pyscreener config file containing the options for docking calculations
    path : str, default="."
        the path under which docking inputs/outputs should be collected
    verbose : int, default=0
        the verbosity of pyscreener
    minimize : bool, default=True
        whether this objective should be minimized
    **kwargs
        additional and unused keyword arguments
    """

    def __init__(
        self,
        objective_config: str,
        path: str = ".",
        verbose: int = 0,
        minimize: bool = True,
        **kwargs,
    ):

        args = ps.args.gen_args(f"--config {objective_config}")

        metadata_template = ps.build_metadata(args.screen_type, args.metadata_template)
        self.virtual_screen = ps.virtual_screen(
            args.screen_type,
            args.receptors,
            args.center,
            args.size,
            metadata_template,
            args.pdbids,
            args.docked_ligand_file,
            args.buffer,
            args.ncpu,
            args.base_name,
            path,
            args.score_mode,
            args.repeat_score_mode,
            args.ensemble_score_mode,
            args.repeats,
            args.k,
            verbose,
        )

        atexit.register(self.cleanup)
        super().__init__(minimize=minimize)

    def forward(self, smis: Iterable[str], **kwargs) -> Dict[str, Optional[float]]:
        """Calculate the docking scores for a list of SMILES strings

        Parameters
        ----------
        smis : List[str]
            the SMILES strings of the molecules to dock
        **kwargs
            additional and unused positional and keyword arguments

        Returns
        -------
        scores : Dict[str, Optional[float]]
            a map from SMILES string to docking score. Ligands that failed
            to dock will be scored as None
        """
        Y = self.c * self.virtual_screen(smis)
        Y = np.where(np.isnan(Y), None, Y)

        return dict(zip(smis, Y))

    def cleanup(self):
        results = self.virtual_screen.all_results()
        self.virtual_screen.collect_files()

        with open(self.virtual_screen.path / "extended.csv", "w") as fid:
            writer = csv.writer(fid)
            writer.writerow(field.name for field in dataclasses.fields(results[0]))
            writer.writerows(dataclasses.astuple(r) for r in results)

max_stars_repo_name: mchaker/lab-molpal

max_issues_repo_name: mchaker/lab-molpal

max_forks_repo_name: mchaker/lab-molpal


text:
import numpy as np

class Bandit:
    "Implements a online, learning strategy to solve the Multi-Armed Bandit problem."

    def __init__(self, p_array, choice_function):
        self.p_array   = p_array                  # Array of probabilities for each bandit
        self.n_bandits = len(p_array)
        self.optimal   = np.argmax(p_array)       # Index of the optimal bandit
        self.wins      = np.zeros(self.n_bandits)
        self.trials    = np.zeros(self.n_bandits)
        self.N         = 0                        # The cumulative number of samples
        self.choices   = []                       # The historical choices
        self.score     = []                       # The historical score
        self.choice_function = choice_function    # Should return an index, an int between 0 and n-1

    def pull(self, index):
        "Sample from a given bandit"
        return np.random.random() < self.p_array[index] # Boolean indicating whether the bandit returned a reward or not

    def sample_bandits(self, n=1):
        "Simulate n rounds of running the bandit process."
        score = np.zeros(n)
        choices = np.zeros(n)

        for k in range(n):
            # Sample from the bandits's priors, and select the largest sample
            choice = self.choice_function(self)

            # Sample the chosen bandit
            result = self.pull(choice)

            # Update priors and score
            self.wins[choice] += result
            self.trials[choice] += 1
            score[k] = result
            self.N += 1
            choices[k] = choice

        self.score = np.r_[self.score, score]
        self.choices = np.r_[self.choices, choices]

    def regret(self, probabilities):
        """Take an array of the true probabilities for each machine and
        an array of the indices of the machine played at each round.
        Return an array giving the total regret after each round."""
        p_opt = np.max(probabilities)
        return np.cumsum(p_opt - probabilities[self.choices.astype(int)]) # Total Regret

    def optimum_choices(self, probabilities):
        """Take an array of the true probabilities for each machine and
        an array of the indices of the machine played at each round.
        Return an array giving the total regret after each round."""
        p_opt = np.argmax(probabilities)
        return np.cumsum(np.select([self.choices==p_opt], [1]))

    def max_mean(self):
        "Pick the bandit with the current best observed proportion of winning."
        if len(self.trials.nonzero()[0]) < self.n_bandits: # Make sure to play each bandit at least once
            return np.random.randint(0, self.n_bandits)
        return np.argmax(self.wins / (self.trials + 1)) # Return the index of the winning bandit.

def print_results(bandit):
    assert len(bandit.p_array) == 4, "Expecting 4 options"
    print("Options: \t  {:>6} {:>6} {:>6} {:>6}".format(*"A B C D".split()))
    print("Probablities: \t  {:>6.2f} {:>6.2f} {:>6.2f} {:>6.2f}".format(*bandit.p_array))
    print("Number of wins:   {:>6} {:>6} {:>6} {:>6}".format(*[int(_) for _ in bandit.wins]))
    print("Number of trials: {:>6} {:>6} {:>6} {:>6}".format(*[int(_) for _ in bandit.trials]))
    rates = [bandit.wins / bandit.trials]
    print("Conversion rates: {:>6.2f} {:>6.2f} {:>6.2f} {:>6.2f}".format(*rates[0]))
    print()
    print(f"A total of {bandit.wins.sum():,.0f} wins out of {bandit.trials.sum():,.0f} trials.")

max_stars_repo_name: loftiskg/rl-course

max_issues_repo_name: loftiskg/rl-course

max_forks_repo_name: loftiskg/rl-course


text:
import os
import numpy as np
import pandas as pd
import argparse
import subprocess
from pathlib import Path
from itertools import product
import logging

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S',
                    level=logging.INFO)
logger = logging.getLogger(__name__)

def get_args_from_command_line():
    """Parse the command line arguments."""
    parser = argparse.ArgumentParser()
    parser.add_argument("--country_code", type=str, default='US')
    args = parser.parse_args()
    return args

if __name__ == '__main__':
    args = get_args_from_command_line()
    # Load tweets
    path_to_tweets = os.path.join(
        '/scratch/mt4493/twitter_labor/twitter-labor-data/data/random_samples/random_samples_splitted',
        args.country_code,
        'evaluation')  # Random set of tweets
    tweets = pd.concat([pd.read_parquet(path) for path in Path(path_to_tweets).glob('*.parquet')])
    tweets = tweets[['tweet_id', 'text']]

    # model_folder_list = ['iter_0-convbert-969622-evaluation', 'iter_1-convbert-3050798-evaluation',
    #                      'iter_2-convbert-3134867-evaluation',
    #                      'iter_3-convbert-3174249-evaluation', 'iter_4-convbert-3297962-evaluation',
    #                      'iter_0-convbert-969622-evaluation',
    #                      'iter_1-convbert_adaptive-5612019-evaluation', 'iter_2-convbert_adaptive-5972342-evaluation',
    #                      'iter_3-convbert_adaptive-5998181-evaluation', 'iter_4-convbert_adaptive-6057405-evaluation']
    # model_folder_list = ['iter_1-convbert_uncertainty-6200469-evaluation',
    #                      'iter_2-convbert_uncertainty-6253253-evaluation',
    #                      'iter_3-convbert_uncertainty-6318280-evaluation',
    #                      ]
    # model_folder_list = ['iter_4-convbert_uncertainty-6423646-evaluation']
    # model_folder_list = ['iter_1-convbert_uncertainty_uncalibrated-6480837-evaluation',
    #                      'iter_2-convbert_uncertainty_uncalibrated-6578026-evaluation',
    #                      'iter_3-convbert_uncertainty_uncalibrated-6596620-evaluation']
    # model_folder_list = ['iter_4-convbert_uncertainty_uncalibrated-6653849-evaluation']
    model_folder_dict = {'MX': ['iter_0-beto-3201262-evaluation', 'iter_1-beto-3741011-evaluation',
                         'iter_2-beto-4141605-evaluation',
                         'iter_3-beto-4379208-evaluation', 'iter_4-beto-4608158-evaluation'],
                        'BR': ['iter_0-bertimbau-2877651-evaluation', 'iter_1-bertimbau-3774133-evaluation',
                               'iter_2-bertimbau-4180985-evaluation', 'iter_3-bertimbau-4518774-evaluation',
                               'iter_4-bertimbau-4688729-evaluation']}
    model_folder_list = model_folder_dict[args.country_code]
    for model_folder in model_folder_list:
        logger.info(f'Folder: {model_folder}')
        path_to_evals = os.path.join(
            '/scratch/mt4493/twitter_labor/twitter-labor-data/data/active_learning/evaluation_inference',
            args.country_code, model_folder)  # Where to store the sampled tweets to be labeled
        if not os.path.exists(path_to_evals):
            os.makedirs(path_to_evals)
        for label in ['is_hired_1mo', 'lost_job_1mo', 'is_unemployed', 'job_search', 'job_offer']:
            logger.info(f'Class: {label}')
            path_to_scores = os.path.join('/scratch/mt4493/twitter_labor/twitter-labor-data/data/inference',
                                          args.country_code, model_folder, 'output',
                                          label)  # Prediction scores from classification
            scores = pd.concat([pd.read_parquet(path) for path in Path(path_to_scores).glob('*.parquet')]).reset_index()
            scores['rank'] = scores['score'].rank(method='first', ascending=False)
            scores = scores[scores['rank'].between(21, 50)]
            df = tweets.merge(scores, on=['tweet_id'])
            df = df.sort_values(by=['rank'], ascending=True).reset_index(drop=True)
            output_path = os.path.join(path_to_evals, f'extra_{label}.csv')
            df.to_csv(output_path, index=False)

max_stars_repo_name: worldbank/SDG-big-data

max_issues_repo_name: worldbank/SDG-big-data

max_forks_repo_name: worldbank/SDG-big-data


text:
# ==================================================================================================
# Copyright (c) 2021, Jennifer Williams and Yamagishi Laboratory, National Institute of Informatics
# Author: Jennifer Williams (j.williams@ed.ac.uk)
# All rights reserved.
# ==================================================================================================

import torch.nn as nn
import torch
import torch.nn.functional as F
import math
import utils.logger as logger
import numpy as np

class VectorQuant(nn.Module):
    """
        Input: (N, samples, n_channels, vec_len) numeric tensor
        Output: (N, samples, n_channels, vec_len) numeric tensor
    """
    def __init__(self, n_channels, n_classes, vec_len, normalize=False):
        # 1, 512, 128, True

        super().__init__()
        if normalize:
            target_scale = 0.06
            self.embedding_scale = target_scale
            self.normalize_scale = target_scale
        else:
            self.embedding_scale = 1e-3
            self.normalize_scale = None
        self.embedding0 = nn.Parameter(torch.randn(n_channels, n_classes, vec_len, requires_grad=True) * self.embedding_scale)
        self.offset = torch.arange(n_channels).cuda() * n_classes
        # self.offset: (n_channels) long tensor
        self.n_classes = n_classes
        self.after_update()

    def forward(self, x0):
        if self.normalize_scale:
            target_norm = self.normalize_scale * math.sqrt(x0.size(3))
            x = target_norm * x0 / x0.norm(dim=3, keepdim=True)
            embedding = target_norm * self.embedding0 / self.embedding0.norm(dim=2, keepdim=True)
        else:
            x = x0
            embedding = self.embedding0
        #logger.log(f'std[x] = {x.std()}')
        x1 = x.reshape(x.size(0) * x.size(1), x.size(2), 1, x.size(3))
        # x1: (N*samples, n_channels, 1, vec_len) numeric tensor

        # Perform chunking to avoid overflowing GPU RAM.
        index_chunks = []
        for x1_chunk in x1.split(512, dim=0):
            index_chunks.append((x1_chunk - embedding).norm(dim=3).argmin(dim=2))
        index = torch.cat(index_chunks, dim=0)
        # index: (N*samples, n_channels) long tensor
        if True: # compute the entropy
            hist = index.float().cpu().histc(bins=self.n_classes, min=-0.5, max=self.n_classes - 0.5)
            prob = hist.masked_select(hist > 0) / len(index)
            entropy = - (prob * prob.log()).sum().item()
            #logger.log(f'entrypy: {entropy:#.4}/{math.log(self.n_classes):#.4}')
        else:
            entropy = 0
        view_item = index.size(0) * index.size(1)
        view_ref = index + self.offset.cuda()
        index1 = view_ref.view(view_item)
        # index1: (N*samples*n_channels) long tensor
        output_flat = embedding.view(-1, embedding.size(2)).index_select(dim=0, index=index1)
        # output_flat: (N*samples*n_channels, vec_len) numeric tensor
        output = output_flat.view(x.size())

        out0 = (output - x).detach() + x
        out1 = (x.detach() - output).float().norm(dim=3).pow(2)
        out2 = (x - output.detach()).float().norm(dim=3).pow(2) + (x - x0).float().norm(dim=3).pow(2)
        #logger.log(f'std[embedding0] = {self.embedding0.view(-1, embedding.size(2)).index_select(dim=0, index=index1).std()}')
        return (out0, out1, out2, entropy, hist)

    def after_update(self):
        if self.normalize_scale:
            with torch.no_grad():
                target_norm = self.embedding_scale * math.sqrt(self.embedding0.size(2))
                self.embedding0.mul_(target_norm / self.embedding0.norm(dim=2, keepdim=True)
)


text:
#! /usr/local/bin/python

import pprint
import sqlite3

import matplotlib
matplotlib.use('Agg')

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

speaks_hist = []
conn = sqlite3.connect('mittab/pairing_db.sqlite3')
cursor = conn.cursor()
row_count = 0
raw_data = []

for row in cursor.execute('SELECT speaks, count(*) FROM tab_roundstats GROUP BY speaks ORDER BY speaks'):
    row_count += 1
    raw_data.append(row)
    speak, count = row
    speak, count = int(speak), int(count)
    speaks_hist.extend([speak] * count)

sns.distplot(speaks_hist, bins=row_count)
out_png = './dist.png'
plt.savefig(out_png, dpi=150)

print('Speaks output to ./dist.png')

def pearsonMedianSkew(dist):
    mean, med, s = np.mean(dist), np.median(dist), np.std(dist)
    return (3 * (mean - med)) / s

print('Speaks standard deviation: %.3f' % np.std(speaks_hist))
print('Speaks Pearson skewness: %.3f' % pearsonMedianSkew(speaks_hist))
print('Raw Data:')
pprint.pprint(raw_data)


text:
import numpy as np
from scipy import stats
import torch
import pytest

from nessai.flows.distributions import MultivariateNormal

@pytest.fixture
def dims():
    """Number of dimensions"""
    return 4

@pytest.fixture
def var():
    """Variance to test"""
    return 2

@pytest.fixture
def dist(dims, var):
    """Instance of MultivariateNormal"""
    return MultivariateNormal([dims], var=var)

@pytest.fixture
def scipy_dist(dims, var):
    mean = np.zeros(dims)
    var = var * np.eye(dims)
    return stats.multivariate_normal(mean=mean, cov=var)

def test_log_prob(dist, scipy_dist, dims):
    """Test the log probability of the multivariate normal"""
    x = np.random.rand(1000, dims)
    log_prob = dist._log_prob(torch.from_numpy(x).float(), None).numpy()
    np.testing.assert_array_almost_equal(log_prob, scipy_dist.logpdf(x))

def test_log_prob_invalid_shape(dist, dims):
    """
    Assert the the log prob raises an error for inputs that are the incorrect
    shape.
    """
    x = np.random.rand(10, dims + 2)
    with pytest.raises(ValueError) as excinfo:
        dist._log_prob(torch.from_numpy(x).float(), None)

    assert 'Expected input of shape' in str(excinfo.value)

@pytest.mark.flaky(run=5)
def test_sample(dist, scipy_dist):
    """
    Test the sample method and check if the resulting samples pass a
    KS test using the scipy distribution
    """
    samples = dist._sample(1000, None).numpy()
    p, _ = stats.kstest(samples, scipy_dist.cdf)
    assert p > 0.05

def test_sample_context(dist):
    """
    Test the sample method with a context that is not None.
    Should raise NotImplementedError
    """
    with pytest.raises(NotImplementedError):
        dist._sample(1000, True)

def test_mean(dist, dims):
    """Test the mean of the multivariate normal"""
    np.testing.assert_array_equal(dist._mean(None).numpy(), np.zeros(dims))

def test_mean_context(dist):
    """
    Test the mean of the multivariate normal with context raises
    a not implemneted error
    """
    with pytest.raises(NotImplementedError):
        dist._mean(True)

max_stars_repo_name: Rodrigo-Tenorio/nessai

max_issues_repo_name: Rodrigo-Tenorio/nessai

max_forks_repo_name: Rodrigo-Tenorio/nessai

reason: import numpy,from scipy


text:
# -*- coding: utf-8 -*-
"""
HISTORY:
    Created on Tue May 26 22:32:49 2020

Project: Vortex GUI

Author: DIVE-LINK (www.dive-link.net), divelink@mail.ru
        Shustov Aleksey (SemperAnte), semte@semte.ru

TODO:
    [ ] change to more model-view structure

DESCRIPTION:
    TransferWidget
        signals:
            imageLoaded
            transferStarted
            transferStopped
"""
from PyQt5 import QtWidgets as qtw
from PyQt5 import QtGui as qtg
from PyQt5 import QtCore as qtc

import matplotlib.image as mpimg
import numpy as np

class TransferWidget(qtw.QGroupBox):

    imageLoaded = qtc.pyqtSignal(np.ndarray)
    transferStarted = qtc.pyqtSignal(np.ndarray)
    transferStopped = qtc.pyqtSignal()

    WIDGET_TITLE = 'Modem options'

    def __init__(self, parent = None):
        super().__init__(parent)

        self.imageSize = 0
        self.imageSource = []

        self._setupUi()

    def _setupUi(self):
        self.setTitle(self.WIDGET_TITLE)

        # transfer type button group
        self.file = qtw.QRadioButton('File')
        self.image = qtw.QRadioButton('Image')
        self.voice = qtw.QRadioButton('Voice')
        self.message = qtw.QRadioButton('Message')
        self.file.click()
        self.typeGroup = qtw.QButtonGroup()
        self.typeGroup.addButton(self.file)
        self.typeGroup.addButton(self.image)
        self.typeGroup.addButton(self.voice)
        self.typeGroup.addButton(self.message)
        self.typeWidget = qtw.QWidget()
        self.typeWidget.setLayout(qtw.QHBoxLayout())
        self.typeWidget.layout().setContentsMargins(0, 0, 0, 0)
        self.typeWidget.layout().addWidget(self.file)
        self.typeWidget.layout().addWidget(self.image)
        self.typeWidget.layout().addWidget(self.voice)
        self.typeWidget.layout().addWidget(self.message)

        self.startButton = qtw.QPushButton('Start')
        self.startButton.setCheckable(True)
        self.stopButton = qtw.QPushButton('Stop')
        self.loadButton = qtw.QPushButton('Load')
        self.fileLabel = qtw.QLabel('No file')

        self.setLayout(qtw.QGridLayout())
        self.layout().addWidget(self.typeWidget, 0, 0, 1, 2)
        self.layout().addWidget(self.loadButton, 1, 0)
        self.layout().addWidget(self.fileLabel, 1, 1)
        self.layout().addWidget(self.startButton, 2, 0)
        self.layout().addWidget(self.stopButton, 2, 1)

        self.startButton.setEnabled(False)
        self.stopButton.setEnabled(False)

        # signal-slot connections
        self.loadButton.clicked.connect(self._openFile)
        self.startButton.clicked.connect(self._startTransfer)
        self.stopButton.clicked.connect(self._stopTransfer)

    @qtc.pyqtSlot()
    def _openFile(self):
        fileName, _ = qtw.QFileDialog.getOpenFileName(
            self,
            "Select image to transfer ...",
            qtc.QDir.currentPath(),
            'PNG Files (*.png) ;;All Files (*)',
            'PNG Files (*.png)',
            qtw.QFileDialog.DontUseNativeDialog |
            qtw.QFileDialog.DontResolveSymlinks
        )
        if fileName:
            image = mpimg.imread(fileName)
            if self._processImage(image):
                self.fileLabel.setText(f'Size: {self.imageSize} bytes')
                self.startButton.setEnabled(True)
                self.stopButton.setEnabled(False)
                self.imageLoaded.emit(image)
            else:
                self.fileLabel.setText('No file')
                self.startButton.setEnabled(False)
                self.stopButton.setEnabled(False)
        else:
            self.fileLabel.setText('No file')
            self.startButton.setEnabled(False)
            self.stopButton.setEnabled(False)

    def _processImage(self, image):
        # remove alpha channel
        if image.shape[2] == 4:
            image = image[:, :, 0:3]
        image = (image * np.iinfo(np.uint8).max).astype(np.uint8)
        self.imageSize = image.size

        dataType = 2 # type of image data
        dataType = np.atleast_1d(np.uint8(dataType))
        imageWidth = image.shape[0]
        if imageWidth > np.iinfo(np.uint16).max:
            self.showError('Too much width of image.')
            return False
        else:
            imageWidth = np.atleast_1d(np.uint16(imageWidth)).view(np.uint8)
        imageHeight = image.shape[1]
        if imageHeight > np.iinfo(np.uint16).max:
            self.showError('Too much height of image.')
            return False
        else:
            imageHeight = np.atleast_1d(np.uint16(imageHeight)).view(np.uint8)

        dataSizeByte = 1 + 2 + 2 + image.size
        if dataSizeByte > np.iinfo(np.uint32).max:
            self.showError('Too much size of image.')
            return False
        else:
            dataSizeByte = np.atleast_1d(np.uint32(dataSizeByte)).view(np.uint8)

        self.imageSource = np.concatenate((dataType, imageHeight, imageWidth, np.ravel(image)))
        return True

    @qtc.pyqtSlot(str)
    def showError(self, text):
        qtw.QMessageBox.warning(
            self,
            'Transfer Error',
            text)

    @qtc.pyqtSlot()
    def _startTransfer(self):
        self.typeWidget.setEnabled(False)
        self.startButton.setEnabled(False)
        self.stopButton.setEnabled(True)
        self.loadButton.setEnabled(False)

        self.transferStarted.emit(self.imageSource)

    @qtc.pyqtSlot()
    def _stopTransfer(self):
        self.typeWidget.setEnabled(True)
        self.startButton.setEnabled(True)
        self.stopButton.setEnabled(False)
        self.loadButton.setEnabled(True)

        self.transferStopped.emit()

# simple test
if __name__ == '__main__':
    import utility
    utility.runManualTest(TransferWidget)

max_stars_repo_name: SemperAnte/vortex-gui

max_issues_repo_name: SemperAnte/vortex-gui

max_forks_repo_name: SemperAnte/vortex-gui