text:
from io import BytesIO
from unittest import TestCase

from numpy import random
from requests_toolbelt import MultipartEncoder

from streaming_form_data import ParseFailedException, StreamingFormDataParser
from streaming_form_data.targets import BaseTarget, FileTarget, ValueTarget
from streaming_form_data.validators import MaxSizeValidator, ValidationError

def get_random_bytes(size, seed):
    random.seed(seed)
    return random.bytes(size)

def open_dataset(filename):
    if filename == 'file.txt':
        filedata = b'this is a txt file\r\n' * 10
    elif filename == 'image-600x400.png':
        filedata = get_random_bytes(1780, 600)
    elif filename == 'image-2560x1600.png':
        filedata = get_random_bytes(11742, 2560)
    elif filename == 'image-500k.png':
        filedata = get_random_bytes(437814, 500)
    elif filename == 'image-high-res.jpg':
        filedata = get_random_bytes(9450866, 945)
    elif filename == 'empty.html':
        filedata = b''
    elif filename == 'hyphen-hyphen.txt':
        filedata = b'--'
    elif filename == 'LF.txt':
        filedata = b'\n'
    elif filename == 'CRLF.txt':
        filedata = b'\r\n'
    elif filename == '1M.dat':
        filedata = get_random_bytes(1024 * 1024, 1024)
    elif filename == '1M-1.dat':
        filedata = get_random_bytes(1024 * 1024 - 1, 1024 - 1)
    elif filename == '1M+1.dat':
        filedata = get_random_bytes(1024 * 1024 + 1, 1024 + 1)
    else:
        raise Exception('Unknown file name: ' + filename)
    return BytesIO(filedata)

def encoded_dataset(filename):
    with open_dataset(filename) as dataset_:
        fields = {filename: (filename, dataset_, 'text/plain')}

        encoder = MultipartEncoder(fields=fields)

        return (encoder.content_type, encoder.to_string())

class StreamingFormDataParserTestCase(TestCase):
    def test_smoke(self):
        encoder = MultipartEncoder(fields={'name': 'hello'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.data_received(encoder.to_string())

    def test_basic_single(self):
        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': 'hello world'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('value', target)

        parser.data_received(encoder.to_string())

        self.assertEqual(target.value, b'hello world')
        self.assertTrue(target._started)
        self.assertTrue(target._finished)

    def test_case_insensitive_content_type(self):
        content_type_header = 'Content-Type'

        for header_key in (
            content_type_header,
            content_type_header.lower(),
            content_type_header.upper(),
            'cOnTeNt-tYPe',
        ):
            target = ValueTarget()

            encoder = MultipartEncoder(fields={'value': 'hello world'})

            parser = StreamingFormDataParser(
                headers={header_key: encoder.content_type}
            )
            parser.register('value', target)

            parser.data_received(encoder.to_string())

            self.assertEqual(target.value, b'hello world')

    def test_missing_content_type(self):
        self.assertRaises(ParseFailedException, StreamingFormDataParser, {})

        headers = {'someheader': 'somevalue'}
        self.assertRaises(
            ParseFailedException, StreamingFormDataParser, headers
        )

    def test_incorrect_content_type(self):
        headers = {'Content-Type': 'multipart/mixed; boundary=1234'}
        self.assertRaises(
            ParseFailedException, StreamingFormDataParser, headers
        )

        headers = {'Content-Type': 'multipart/form-data'}
        self.assertRaises(
            ParseFailedException, StreamingFormDataParser, headers
        )

        headers = {'Content-Type': 'multipart/form-data; delimiter=1234'}
        self.assertRaises(
            ParseFailedException, StreamingFormDataParser, headers
        )

    def test_basic_multiple(self):
        first = ValueTarget()
        second = ValueTarget()
        third = ValueTarget()

        encoder = MultipartEncoder(
            fields={'first': 'foo', 'second': 'bar', 'third': 'baz'}
        )

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.register('first', first)
        parser.register('second', second)
        parser.register('third', third)

        parser.data_received(encoder.to_string())

        self.assertEqual(first.value, b'foo')
        self.assertEqual(second.value, b'bar')
        self.assertEqual(third.value, b'baz')

    def test_chunked_single(self):
        expected_value = 'hello world'

        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': expected_value})

        body = encoder.to_string()

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('value', target)

        index = body.index(b'world')

        parser.data_received(body[:index])
        parser.data_received(body[index:])

        self.assertEqual(target.value, expected_value.encode('utf-8'))

    def test_chunked_multiple(self):
        expected_first_value = 'foo' * 1000
        expected_second_value = 'bar' * 1000
        expected_third_value = 'baz' * 1000

        first = ValueTarget()
        second = ValueTarget()
        third = ValueTarget()

        encoder = MultipartEncoder(
            fields={
                'first': expected_first_value,
                'second': expected_second_value,
                'third': expected_third_value,
            }
        )

        body = encoder.to_string()

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.register('first', first)
        parser.register('second', second)
        parser.register('third', third)

        chunks = []
        size = 100

        while len(body):
            chunks.append(body[:size])
            body = body[size:]

        for chunk in chunks:
            parser.data_received(chunk)

        self.assertEqual(first.value, expected_first_value.encode('utf-8'))
        self.assertEqual(second.value, expected_second_value.encode('utf-8'))
        self.assertEqual(third.value, expected_third_value.encode('utf-8'))

    def test_break_chunk_at_boundary(self):
        expected_first_value = 'hello' * 500
        expected_second_value = 'hello' * 500

        first = ValueTarget()
        second = ValueTarget()

        encoder = MultipartEncoder(
            fields={'first': 'hello' * 500, 'second': 'hello' * 500}
        )

        body = encoder.to_string()
        boundary = encoder.boundary.encode('utf-8')

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )

        parser.register('first', first)
        parser.register('second', second)

        index = body[50:].index(boundary) + 5

        parser.data_received(body[:index])
        parser.data_received(body[index:])

        self.assertEqual(first.value, expected_first_value.encode('utf-8'))
        self.assertEqual(second.value, expected_second_value.encode('utf-8'))

    def test_file_content_single(self):
        filenames = (
            'file.txt',
            'image-600x400.png',
            'image-2560x1600.png',
            'empty.html',
            'hyphen-hyphen.txt',
            'LF.txt',
            'CRLF.txt',
            '1M.dat',
            '1M-1.dat',
            '1M+1.dat',
        )

        for filename in filenames:
            with open_dataset(filename) as dataset_:
                expected_value = dataset_.read()

            content_type, body = encoded_dataset(filename)

            value = ValueTarget()

            parser = StreamingFormDataParser(
                headers={'Content-Type': content_type}
            )
            parser.register(filename, value)

            parser.data_received(body)

            self.assertEqual(value.value, expected_value)

    def test_file_content_multiple(self):
        with open_dataset('file.txt') as dataset_:
            expected_value = dataset_.read()

        content_type, body = encoded_dataset('file.txt')

        txt = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': content_type}
        )
        parser.register('file.txt', txt)

        size = 50
        chunks = []

        while body:
            chunks.append(body[:size])
            body = body[size:]

        for chunk in chunks:
            parser.data_received(chunk)

        self.assertEqual(txt.value, expected_value)

    def test_file_content_varying_chunk_size(self):
        with open_dataset('file.txt') as dataset_:
            expected_value = dataset_.read()

        content_type, body = encoded_dataset('file.txt')

        for index in range(len(body)):
            txt = ValueTarget()

            parser = StreamingFormDataParser(
                headers={'Content-Type': content_type}
            )
            parser.register('file.txt', txt)

            parser.data_received(body[:index])
            parser.data_received(body[index:])

            self.assertEqual(txt.value, expected_value)

    def test_mixed_content_varying_chunk_size(self):
        with open_dataset('file.txt') as dataset_:
            expected_value = dataset_.read()

        with open_dataset('file.txt') as dataset_:
            fields = {
                'name': 'hello world',
                'age': '10',
                'cv.txt': ('file.txt', dataset_, 'text/plain'),
            }

            encoder = MultipartEncoder(fields=fields)

            body = encoder.to_string()
            content_type = encoder.content_type

        for index in range(len(body)):
            name = ValueTarget()
            age = ValueTarget()
            cv = ValueTarget()

            parser = StreamingFormDataParser(
                headers={'Content-Type': content_type}
            )

            parser.register('name', name)
            parser.register('age', age)
            parser.register('cv.txt', cv)

            parser.data_received(body[:index])
            parser.data_received(body[index:])

            self.assertEqual(name.value, b'hello world')
            self.assertEqual(age.value, b'10')
            self.assertEqual(cv.value, expected_value)

    def test_parameter_contains_crlf(self):
        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': 'hello\r\nworld'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('value', target)
        parser.data_received(encoder.to_string())

        self.assertEqual(target.value, b'hello\r\nworld')

    def test_parameter_ends_with_crlf(self):
        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': 'hello\r\n'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('value', target)

        parser.data_received(encoder.to_string())

        self.assertEqual(target.value, b'hello\r\n')

    def test_parameter_starts_with_crlf(self):
        target = ValueTarget()

        encoder = MultipartEncoder(fields={'value': '\r\nworld'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.register('value', target)

        parser.data_received(encoder.to_string())

        self.assertEqual(target.value, b'\r\nworld')

    def test_parameter_contains_part_of_delimiter(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--123
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.multipart_filename, 'ab.txt')
        self.assertEqual(target.value, b'Foo\r\n--123')
        self.assertTrue(target._started)
        self.assertTrue(target._finished)

    def test_multiple_files(self):
        txt_filename = 'file.txt'
        png_filename = 'image-600x400.png'

        with open_dataset(txt_filename) as dataset_:
            expected_txt = dataset_.read()

        with open_dataset(png_filename) as dataset_:
            expected_png = dataset_.read()

        txt_target = ValueTarget()
        png_target = ValueTarget()

        with open_dataset(txt_filename) as txt_file, open_dataset(
            png_filename
        ) as png_file:
            encoder = MultipartEncoder(
                fields={
                    txt_filename: (
                        txt_filename,
                        txt_file,
                        'application/plain',
                    ),
                    png_filename: (png_filename, png_file, 'image/png'),
                }
            )

            parser = StreamingFormDataParser(
                headers={'Content-Type': encoder.content_type}
            )

            parser.register(txt_filename, txt_target)
            parser.register(png_filename, png_target)

            parser.data_received(encoder.to_string())

            self.assertEqual(txt_target.value, expected_txt)
            self.assertEqual(png_target.value, expected_png)

    def test_large_file(self):
        for filename in [
            'image-500k.png',
            'image-2560x1600.png',
            'image-600x400.png',
            'image-high-res.jpg',
        ]:
            with open_dataset(filename) as dataset_:
                expected_value = dataset_.read()

            content_type, body = encoded_dataset(filename)

            value = ValueTarget()

            parser = StreamingFormDataParser(
                headers={'Content-Type': content_type}
            )
            parser.register(filename, value)

            parser.data_received(body)

            self.assertEqual(value.value, expected_value)

    # The following tests have been added from tornado's
    # MultipartFormDataTestCase
    # https://github.com/tornadoweb/tornado/blob/master/tornado/test/httputil_test.py

    def test_file_upload(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.multipart_filename, 'ab.txt')
        self.assertEqual(target.value, b'Foo')
        self.assertTrue(target._started)
        self.assertTrue(target._finished)

    def test_unquoted_names(self):
        data = b'''\
--1234
Content-Disposition: form-data; name=files; filename=ab.txt

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'Foo')

    def test_special_filenames(self):
        filenames = [
            'a;b.txt',
            'a"b.txt',
            'a";b.txt',
            'a;"b.txt',
            'a";";.txt',
            'a\\"b.txt',
            'a\\b.txt',
        ]

        for filename in filenames:
            data = (
                '''\
--1234
Content-Disposition: form-data; name=files; filename={}

Foo
--1234--'''.format(
                    filename
                )
                .replace('\n', '\r\n')
                .encode('utf-8')
            )

            target = ValueTarget()

            parser = StreamingFormDataParser(
                headers={'Content-Type': 'multipart/form-data; boundary=1234'}
            )
            parser.register('files', target)

            parser.data_received(data)

            self.assertEqual(target.value, b'Foo')

    def test_boundary_starts_and_ends_with_quotes(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary="1234"'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.multipart_filename, 'ab.txt')
        self.assertEqual(target.value, b'Foo')

    def test_missing_headers(self):
        data = '''\
--1234

Foo
--1234--'''.replace(
            '\n', '\r\n'
        ).encode(
            'utf-8'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'')

    def test_invalid_content_disposition(self):
        data = b'''\
--1234
Content-Disposition: invalid; name="files"; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'')

    def test_without_name_parameter(self):
        data = b'''\
--1234
Content-Disposition: form-data; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'')

    def test_data_after_final_boundary(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--1234--
'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'Foo')

    def test_register_after_data_received(self):
        encoder = MultipartEncoder(fields={'name': 'hello'})

        parser = StreamingFormDataParser(
            headers={'Content-Type': encoder.content_type}
        )
        parser.data_received(encoder.to_string())

        self.assertRaises(
            ParseFailedException, parser.register, 'name', ValueTarget()
        )

    def test_missing_filename_directive(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"

Foo
--1234--
'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget()

        self.assertFalse(target.multipart_filename)

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        parser.data_received(data)

        self.assertEqual(target.value, b'Foo')
        self.assertFalse(target.multipart_filename)

    def test_filename_passed_to_target(self):
        filename = 'file.txt'

        content_type, body = encoded_dataset(filename)

        target = ValueTarget()

        self.assertFalse(target.multipart_filename)

        parser = StreamingFormDataParser(
            headers={'Content-Type': content_type}
        )
        parser.register(filename, target)
        parser.data_received(body)

        self.assertEqual(target.multipart_filename, filename)

    def test_target_raises_exception(self):
        filename = 'file.txt'

        content_type, body = encoded_dataset(filename)

        class BadTarget(BaseTarget):
            def data_received(self, data):
                raise ValueError()

        target = BadTarget()

        parser = StreamingFormDataParser(
            headers={'Content-Type': content_type}
        )
        parser.register(filename, target)

        self.assertRaises(ValueError, parser.data_received, body)

    def test_target_exceeds_max_size(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = ValueTarget(validator=MaxSizeValidator(1))

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        self.assertRaises(ValidationError, parser.data_received, data)

        self.assertTrue(target._started)
        self.assertTrue(target._finished)

    def test_file_target_exceeds_max_size(self):
        data = b'''\
--1234
Content-Disposition: form-data; name="files"; filename="ab.txt"

Foo
--1234--'''.replace(
            b'\n', b'\r\n'
        )

        target = FileTarget('/tmp/file.txt', validator=MaxSizeValidator(1))

        parser = StreamingFormDataParser(
            headers={'Content-Type': 'multipart/form-data; boundary=1234'}
        )
        parser.register('files', target)

        self.assertRaises(ValidationError, parser.data_received, data)

        self.assertTrue(target._started)
        self.assertTrue(target._finished)

max_stars_repo_name: trusttoken/streaming-form-data

max_issues_repo_name: trusttoken/streaming-form-data

max_forks_repo_name: trusttoken/streaming-form-data

reason: from numpy


text:
# coding: utf-8
# /*##########################################################################
#
# Copyright (C) 2016-2018 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/

from __future__ import absolute_import

__authors__ = ["V. Valls"]
__license__ = "MIT"
__date__ = "28/02/2019"

import numpy

from silx.gui import qt

from .DataModel import DataModel

class ImageModel(DataModel):

    def setValue(self, value):
        """Set the value of this image model."""
        if value is not None:
            if not isinstance(value, numpy.ndarray):
                raise TypeError("A numpy array is expected, but %s was found." % value.__class__.__name__)
            if len(value.shape) != 2:
                raise TypeError("A 2d array is expected, but %s was found." % value.shape)
            if value.dtype.kind not in "uif":
                raise TypeError("A numeric array is expected, but %s was found." % value.dtype.kind)
            previous = self.value()
            if previous is value:
                # Filter same images
                return
            if previous is not None and numpy.array_equal(value, previous):
                # Filter same images
                return
        super(ImageModel, self).setValue(value)

class ImageFilenameModel(DataModel):
    """Model storing an image using it's filename."""

    filenameChanged = DataModel.changed

    def hasFilename(self):
        """True if this model contains a filename.

        :rtype: bool
        """
        return self.value() is not None

    def filename(self):
        """Returns the filename associated with this model.

        :rtype: Union[None,str]
        """
        return self.value()

    def setFilename(self, filename):
        """
        Set a filename to this model

        :param str filename: The new filename
        """
        return self.setValue(filename)

class ImageFromFilenameModel(DataModel):
    """Model storing an image array which could come from a filename.

    This model deal with unsynchronized filename/data.
    """

    filenameChanged = qt.Signal()

    def __init__(self, parent=None):
        self.__filename = None
        self.__wasChanged = False
        DataModel.__init__(self, parent=parent)

    def setValue(self, value):
        """Set the value of this image model."""
        if value is not None:
            if not isinstance(value, numpy.ndarray):
                raise TypeError("A numpy array is expected, but %s was found." % value.__class__.__name__)
            if len(value.shape) != 2:
                raise TypeError("A 2d array is expected, but %s was found." % value.shape)
            if value.dtype.kind not in "uif":
                raise TypeError("A numeric array is expected, but %s was found." % value.dtype.kind)
            previous = self.value()
            if previous is value:
                # Filter same images
                return
            if previous is not None and numpy.array_equal(value, previous):
                # Filter same images
                return
        super(ImageFromFilenameModel, self).setValue(value)
        self.__isSynchronized = False

    def hasFilename(self):
        """True if this model contains a filename.

        :rtype: bool
        """
        return self.__filename is not None

    def filename(self):
        """Returns the filename associated with this model.

        :rtype: Union[None,str]
        """
        return self.__filename

    def unlockSignals(self):
        self.__filenameWasChanged()
        self.__wasChanged = False
        return DataModel.unlockSignals(self)

    def __filenameWasChanged(self):
        """Emit the change event in case of the model was not locked.

        :returns: True if the signal was emitted.
        """
        if self.isLocked():
            self.__wasChanged = True
            return False
        else:
            self.filenameChanged.emit()
            return True

    def setFilename(self, filename):
        """
        Set a filename to this model

        :param str filename: The new filename
        """
        self.__filename = filename
        self.__isSynchronized = False
        self.__filenameWasChanged()

    def setSynchronized(self, isSynchronized):
        """"
        Set if the filename and the data are synchronized.
        """
        self.__isSynchronized = isSynchronized

    def isSynchronized(self):
        """Returns True if the filename and the data are synchronized.

        Both contains the same data.
        """
        return self.__isSynchronized

ext: py

max_stars_repo_name: yugangzhang/pyFAI

max_issues_repo_name: yugangzhang/pyFAI

max_forks_repo_name: yugangzhang/pyFAI

reason: import numpy


text:
#import tensorflow as tf;
import torch.optim as optim
import os;
from distutils.util import strtobool;
import numpy as np;
import torch
import glob;
import skimage.io as io;

#from ..sa_net_train import CNNTrainer;
#from sa_net_arch import AbstractCNNArch;
#from sa_net_arch_utilities import CNNArchUtils;
from ..sa_net_optimizer_pytorch import OptimizerTypesPyTorch, CNNOptimizerPyTorch;
from ..sa_net_data_provider import AbstractDataProvider;
from ..sa_helpers import multiplex_utils;

class MultiplexAutoencoderTesterRGBExternalInput:
    def __init__(self, cnn_arch, cnn_arch_module, session_config, output_dir, device, output_ext, kwargs):
        # predefined list of arguments
        args = {'split_name':'test', 'batch_size':1, 'invert_out_img':'False', 'is_output_od':'False'};
        args.update(kwargs);

        self.cnn_arch = cnn_arch;
        self.cnn_arch_module = cnn_arch_module;

        #self.test_data_provider = test_data_provider;
        self.batch_size = int(args['batch_size']);

        #self.invert_out_img = bool(strtobool(args['invert_out_img']));
        #self.is_output_od = bool(strtobool(args['is_output_od']));
        self.pad_y = int(args['pad_y']);
        self.pad_x = int(args['pad_x']);
        self.pad_y1 = int(np.floor(self.pad_y / 2.0));
        self.pad_y2 = int(np.ceil(self.pad_y / 2.0));
        self.pad_x1 = int(np.floor(self.pad_y / 2.0));
        self.pad_x2 = int(np.ceil(self.pad_y / 2.0));

        self.device = device;
        self.output_dir = output_dir;
        self.output_ext = output_ext;

    def init_model(self, do_init, do_restore):
        if(do_restore):
            checkpoint = self.cnn_arch_module.restore_model(is_test=True);
            if(checkpoint is not None):
                print('restore succeed')

    def predict(self, inputs):
        with torch.no_grad():
            batch_x = inputs;
            if (batch_x is None):
                return None;

            batch_x = self.preprocess_input(inputs);

            if(self.device is not None):
                batch_x = batch_x.to(self.device);

            batch_y, batch_concentration, batch_concentration_sm, batch_stains = self.cnn_arch(batch_x);

            return batch_concentration.detach().cpu().numpy().astype(np.float16);

    def preprocess_input(self, inputs):
        np.clip(inputs, 0, 255, inputs);

        if(self.pad_y > 0 or self.pad_x > 0):
            inputs = np.pad(inputs, ((0,0),(0,0),(self.pad_y1, self.pad_y2),(self.pad_x1, self.pad_x2)),'constant', constant_values=128);

        inputs = torch.tensor(inputs, dtype = torch.float); # to avoid the error:  Input type (torch.cuda.DoubleTensor) and weight type (torch.cuda.FloatTensor) should be the same

        return inputs;


text:
# -*- coding: utf-8 -*-

# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Definition of a reward and successor network
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import six
import numpy as np
import tensorflow as tf
from seq2seq.graph_module import GraphModule
from seq2seq.configurable import Configurable

from tensorflow.python.ops import init_ops

class reward_network(GraphModule, Configurable):
"""
构建reward网络，简单的线性回归模型
"""

    def __init__(self, params, mode, name="attention"):
        GraphModule.__init__(self, name)
        Configurable.__init__(self, params, mode)

    def _build(self,outputs):
        outputs_size = tf.shape(outputs)[1]
        dtype = outputs.dtype.base_dtype
        # Set up the requested initialization.
        init_mean = 0.0
        init_stddev = 0.0

        reward = tf.contrib.layers.fully_connected(
            inputs=outputs,
            num_outputs= 1,
            weights_initializer=init_ops.random_normal_initializer(init_mean, init_stddev, dtype=dtype),dtype=dtype)
            biases_initializer = init_ops.random_normal_initializer(init_mean, init_stddev, dtype=dtype),\
                               dtype = dtype
            activation_fn=None,
            scope="reward")
        return reward


text:
import os
import sys
import random

import cv2
import numpy as np
from tqdm import tqdm

import tensorflow as tf
from settings import app

def _load_image(path):
    image = cv2.imread(path)
    if image is not None:
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        return image.astype(np.float32)
    return None

def _bytes_feature(value):
  return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def _build_examples_list(input_folder, seed):
    examples = []
    for classname in os.listdir(input_folder):
        class_dir = os.path.join(input_folder, classname)
        if (os.path.isdir(class_dir)):
            for filename in os.listdir(class_dir):
                filepath = os.path.join(class_dir, filename)
                example = {
                    'classname': classname,
                    'path': filepath
                }
                examples.append(example)

    random.seed(seed)
    random.shuffle(examples)
    return examples

def _split_list(alist, wanted_parts=1):
    length = len(alist)
    return [ alist[i*length // wanted_parts: (i+1)*length // wanted_parts]
             for i in range(wanted_parts) ]

def _get_examples_share(examples, training_split):
    examples_size = len(examples)
    len_training_examples = int(examples_size * training_split)

    return np.split(examples, [len_training_examples])

def _write_tfrecord(examples, output_filename):
    writer = tf.python_io.TFRecordWriter(output_filename)
    for example in tqdm(examples):
        try:
            image = _load_image(example['path'])
            if image is not None:
                encoded_image_string = cv2.imencode('.jpg', image)[1].tostring()
                feature = {
                    'train/label': _bytes_feature(tf.compat.as_bytes(example['classname'])),
                    'train/image': _bytes_feature(tf.compat.as_bytes(encoded_image_string))
                }

                tf_example = tf.train.Example(features = tf.train.Features(feature=feature))
                writer.write(tf_example.SerializeToString())
        except Exception as inst:
            print(inst)
            pass
    writer.close()

def _write_sharded_tfrecord(examples, number_of_shards, base_output_filename, is_training = True):
    sharded_examples = _split_list(examples, number_of_shards)
    for count, shard in tqdm(enumerate(sharded_examples, start = 1)):
        output_filename = '{0}_{1}_{2:02d}of{3:02d}.tfrecord'.format(
            base_output_filename,
            'training' if is_training else 'test',
            count,
            number_of_shards
        )
        _write_tfrecord(shard, output_filename)

examples = _build_examples_list(app['IMAGES_INPUT_FOLDER'], app['SEED'])
training_examples, test_examples = _get_examples_share(examples, app['TRAINING_EXAMPLES_SPLIT']) # pylint: disable=unbalanced-tuple-unpacking

print("Creating training shards", flush = True)
_write_sharded_tfrecord(training_examples, app['NUMBER_OF_SHARDS'], app['OUTPUT_FILENAME'])
print("\nCreating test shards", flush = True)
_write_sharded_tfrecord(test_examples, app['NUMBER_OF_SHARDS'], app['OUTPUT_FILENAME'], False)
print("\n", flush = True)

max_stars_repo_name: tinenbruno/tfrecord-image-builder

max_issues_repo_name: tinenbruno/tfrecord-image-builder

max_forks_repo_name: tinenbruno/tfrecord-image-builder


text:
from random import uniform
import numpy as np
import theano
from theano import config
import theano.tensor as tensor
from theano.tensor.signal import pool
from collections import OrderedDict, defaultdict
from itertools import tee
import time

def randi(N):
  """ get random integer in range [0, N) """
  return int(uniform(0, N))

def merge_init_structs(s0, s1):
  """ merge struct s1 into s0 """
  for k in s1['model']:
    assert (not k in s0['model']), 'Error: looks like parameter %s is trying to be initialized twice!' % (k, )
    s0['model'][k] = s1['model'][k] # copy over the pointer
  s0['update'].extend(s1['update'])
  s0['regularize'].extend(s1['regularize'])

def initw(n,d): # initialize matrix of this size
  magic_number = 0.1
  return (np.random.rand(n,d) * 2 - 1) * magic_number # U[-0.1, 0.1]

def initwTh(n,d,magic_number=0.1, inittype = 'norm'): # initialize matrix of this size
  if inittype == 'xavier':
     magic_number = np.sqrt(6.) / np.sqrt(n + d)
  return ((np.random.rand(n,d) * 2 - 1) * magic_number).astype(config.floatX) # U[-0.1, 0.1]

def initwThNd(shape, magic_number=0.1, inittype = 'norm'): # initialize matrix of this size
  if inittype == 'xavier':
     magic_number = np.sqrt(6.) / np.sqrt(shape.sum())
  return (np.random.randn(*shape) * magic_number).astype(config.floatX) # U[-0.1, 0.1]

def init_tparams(params):
  tparams = OrderedDict()
  for kk, pp in params.iteritems():
      tparams[kk] = theano.shared(params[kk], name=kk)
  return tparams

def _p(pp, name):
    return '%s_%s' % (pp, name)

def numpy_floatX(data):
    return np.asarray(data, dtype=config.floatX)

def accumNpDicts(d0, d1):
  """ forall k in d0, d0 += d1 . d's are dictionaries of key -> numpy array """
  for k in d1:
    if k in d0:
      d0[k] += d1[k]
    else:
      d0[k] = d1[k]

def zipp(params, tparams):
    """
    When we reload the model. Needed for the GPU stuff.
    """
    if type(tparams) == list:
        for i in xrange(len(params)):
            tparams[i].set_value(params[i])
    else:
        for kk in tparams:
            if kk in params:
                tparams[kk].set_value(params[kk])
            else:
                print '%s not found in cp. Skipping...'%(kk)

def unzip(zipped):
    """
    When we pickle the model. Needed for the GPU stuff.
    """
    if type(zipped) == list:
        new_params = []
        for vv in zipped:
            new_params.append(vv.get_value())
    else:
        new_params = OrderedDict()
        for kk, vv in zipped.iteritems():
            new_params[kk] = vv.get_value()
    return new_params

def forwardSubRoutine(Hin,Hout, X, WLSTM,IFOG,IFOGf,C,n,d):

    for t in xrange(n):

      prev = np.zeros(d) if t == 0 else Hout[t-1]
      #tanhC_version = 1
      Hin[t,0] = 1
      Hin[t,1:1+d] = X[t]
      Hin[t,1+d:] = prev

      # compute all gate activations. dots:
      IFOG[t] = Hin[t].dot(WLSTM)

      IFOGf[t,:3*d] = 1.0/(1.0+np.exp(-IFOG[t,:3*d])) # sigmoids; these are the gates
      IFOGf[t,3*d:] = np.tanh(IFOG[t, 3*d:]) # tanh

      C[t] = IFOGf[t,:d] * IFOGf[t, 3*d:]
      if t > 0: C[t] += IFOGf[t,d:2*d] * C[t-1]

      Hout[t] = IFOGf[t,2*d:3*d] * np.tanh(C[t])

      #  Hout[t] = IFOGf[t,2*d:3*d] * C[t]
    return Hin, Hout, IFOG,IFOGf,C

def softmax(x,axis = -1):
    xs = x.shape
    ndim = len(xs)
    if axis == -1:
        axis = ndim -1

    z = np.max(x,axis=axis)
    y = x - z[...,np.newaxis] # for numerical stability shift into good numerical range
    e1 = np.exp(y)
    p1 = e1 / np.sum(e1,axis=axis)[...,np.newaxis]

    return p1

def cosineSim(x,y):
    n1 = np.sqrt(np.sum(x**2))
    n2 = np.sqrt(np.sum(y**2))
    sim = x.T.dot(y)/(n1*n2) if n1 !=0.0 and n2!= 0.0 else 0.0
    return sim

def sliceT(_x, n, dim):
    if _x.ndim == 3:
        return _x[:, :, n * dim:(n + 1) * dim]
    return _x[:, n * dim:(n + 1) * dim]

#Theano functions
def ReLU(x, alpha = 0.):
    y = tensor.nnet.relu(x, alpha=alpha)
    return(y)
def Sigmoid(x):
    y = tensor.nnet.sigmoid(x)
    return(y)
def Tanh(x):
    y = tensor.tanh(x)
    return(y)
def Iden(x):
    y = x
    return(y)

def l2norm(X,axis=1):
    """
    Compute L2 norm, row-wise
    """
    norm = tensor.sqrt(tensor.pow(X, 2).sum(axis=axis)+1e-20)
    if axis == 1:
        X /= (norm[:, None] + 1e-10)
    else:
        X /= (norm[None, :] + 1e-10)
    return X

def nonLinLayer(x,layer_type='ReLU', alpha=0.01):
    if layer_type == 'relu':
        return ReLU(x, alpha=0.01)
    elif layer_type == 'sigm':
        return Sigmoid(x)
    elif layer_type == 'tanh':
        return Tanh(x)
    elif layer_type == 'iden':
        return Iden(x)
    else:
        raise ValueError('Unknown nonlinear layer type %s'%(layer_type))

def myMaxPool(x, ps=[],method='downsamp'):
    if method == 'downsamp':
        y = pool.pool_2d(input= x, ds=ps, ignore_border=True, mode= 'max')
    elif method == 'max':
        y = tensor.max(x, axis=3).max(axis=2)
    return(y)

def preProBuildWordVocab(sentence_iterator, word_count_threshold, options = None):
  # count up all word counts so that we can threshold
  # this shouldnt be too expensive of an operation

  st_iter1, st_iter2 = tee(sentence_iterator)

  print 'preprocessing word counts and creating vocab based on word count threshold %d' % (word_count_threshold, )
  t0 = time.time()
  word_counts = {}
  nsents = 0
  for sent in st_iter1:
    nsents += 1
    for w in sent['tokens']:
      word_counts[w] = word_counts.get(w, 0) + 1
  vocab = {w:word_counts[w] for w in word_counts if word_counts[w] >= word_count_threshold}
  print 'filtered words from %d to %d in %.2fs' % (len(word_counts), len(vocab), time.time() - t0)

  if (options != None) and (options['class_out_factoring'] == 1):
    print 'Clustering words into %d calsses for output factorization ' % (options['nClasses'])
    t0 = time.time()
    if options['class_inp_file'] == None:
        import os
        fInName = 'input' + options['dataset'] + 'TrainTok'
        fIn = open(os.path.join(options['clust_tool_dir'], fInName + '.txt'),'w')
        for st in st_iter2:
            fIn.write(' '.join([w if w in vocab else '' for w in st['tokens']]))
            fIn.write('\n')
        fIn.close()
        owd = os.getcwd()
        os.chdir(options['clust_tool_dir'])
        clust_cmd = './wcluster --text '+ fInName + '.txt --c ' + str(options['nClasses'])
        print ' Invoking the clustering tool now...'
        os.system(clust_cmd)
        os.chdir(owd)
        options['class_inp_file'] = os.path.join(options['clust_tool_dir'], fInName + '-c' + str(options['nClasses']) + '-p1.out/paths')
        print 'Clustering is done in %.2fs ... Now onto processing the outputs' % (time.time() - t0)

    clustOut = open(options['class_inp_file'],'r').readlines()
    classes = defaultdict(list)
    treetocix = {}

    for cls in clustOut:
        lineS = cls.split()
        if lineS[0] not in treetocix:
            treetocix[lineS[0]] = len(treetocix)
        cix = treetocix[lineS[0]]
        if int(lineS[2]) >= word_count_threshold and (lineS[1] != '#UNK' or options.get('use_unk_token',0)):
            classes[cix].append({'w':lineS[1],'c':word_counts[lineS[1]]})

    # Re-arrange the vocabulary by grouping into classes
    vocab = []
    clstoix = {}
    wordtocls= {}
    class_counts = defaultdict(int)
    for cls in classes:
        # +1 is to compensate for insertion of '.' later
        clstoix[cls] = {'strt':len(vocab)+1,'len':len(classes[cls])}
        for wSt in classes[cls]:
            class_counts[cls] += wSt['c']
            wordtocls[wSt['w']] = cls
            vocab.append(wSt['w'])

    # Adding special STOP class containing only '.' to the class list
    # #START# is not needed because it is force fed to the model and
    # model doesn't ever have to produce the class output #START#
    treetocix['STOP'] = len(treetocix)
    cls = treetocix['STOP']
    classes[cls] = [{'w':'.', 'c':nsents}]
    wordtocls['.'] = cls
    clstoix[cls] = {'strt':0,'len':1}
    class_counts[cls] = nsents

    cixtotree = {}
    for treeHash in treetocix:
        cixtotree[treetocix[treeHash]] = treeHash

    print 'Class based factorization of output done %.2fs' % (time.time() - t0)
    max_cls_len = np.max([clstoix[cls]['len'] for cls in clstoix])
    print 'Maximum Class size is  %d' % (max_cls_len)

  # with K distinct words:
  # - there are K+1 possible inputs (START token and all the words)
  # - there are K+1 possible outputs (END token and all the words)
  # we use ixtoword to take predicted indices and map them to words for output visualization
  # we use wordtoix to take raw words and get their index in word vector matrix
  ixtoword = {}
  ixtoword[0] = '.'  # period at the end of the sentence. make first dimension be end token
  wordtoix = {}
  wordtoix['#START#'] = 0 # make first vector be the start token
  ix = 1
  for w in vocab:
    wordtoix[w] = ix
    ixtoword[ix] = w
    ix += 1

  word_counts['.'] = nsents

  if (options == None) or (options['class_out_factoring'] == 0):
    ##############################################################################################
    # compute bias vector, which is related to the log probability of the distribution
    # of the labels (words) and how often they occur. We will use this vector to initialize
    # the decoder weights, so that the loss function doesnt show a huge increase in performance
    # very quickly (which is just the network learning this anyway, for the most part). This makes
    # the visualizations of the cost function nicer because it doesn't look like a hockey stick.
    # for example on Flickr8K, doing this brings down initial perplexity from ~2500 to ~170.
    ##############################################################################################
    bias_init_vector = np.array([1.0*word_counts[ixtoword[i]] for i in ixtoword])
    bias_init_vector /= np.sum(bias_init_vector) # normalize to frequencies
    bias_init_vector = np.log(bias_init_vector)
    bias_init_vector -= np.max(bias_init_vector) # shift to nice numeric range
    return wordtoix, ixtoword, bias_init_vector
  else:
    ##############################################################################################
    # Ths matrices below is used withing the lstm generator module inorder to correctly
    # factorize the output and compute softmax locally within a class.
    # This maps word id to its class id and class location info within the composite decode matrix
    # idx 0 is for class id. idx 1 is for class start location. idx 2 is for class end location
    # idx 3 is for word id within its class, needed to map class softmax to overall vocab
    ##############################################################################################
    ixtoclsinfo = np.zeros((len(ixtoword),4),dtype=np.int32)
    for cls in classes:
        for i,wSt in enumerate(classes[cls]):
            cS = clstoix[cls]['strt']
            cE = clstoix[cls]['strt'] +  clstoix[cls]['len']
            ixtoclsinfo[wordtoix.get(wSt['w'],0), :] = [cls, cS, cE, i]
    ##############################################################################################
    # For class based output clustering, we need to intialize two sets of biases. One is inter-class
    # bias reflecting the frequencies of each aggr lass. Next is the n intra-class biases, which is
    # only for items words within a class
    ##############################################################################################
    bias_init_inter_class = np.array([1.0*class_counts[cls] for cls in clstoix])
    bias_init_inter_class /= np.sum(bias_init_inter_class) # normalize to frequencies
    bias_init_inter_class = np.log(bias_init_inter_class)
    bias_init_inter_class -= np.max(bias_init_inter_class) # shift to nice numeric range

    bias_init_intra_class = -100*np.ones((1,len(classes),max_cls_len))
    for cls in classes:
        idx = np.arange(0, clstoix[cls]['len'])
        bias_init_intra_class[0, cls, idx] = np.array([1.0*word_counts.get(wrd['w'],0) for wrd in classes[cls]])
        bias_init_intra_class[0, cls, idx] /= np.sum(bias_init_intra_class[0,cls,idx]) # normalize to frequencies
        bias_init_intra_class[0, cls, idx] = np.log(bias_init_intra_class[0,cls,idx])
        bias_init_intra_class[0, cls, idx] -= np.max(bias_init_intra_class[0,cls,idx]) # shift to nice numeric range

    return [wordtoix, classes] , [ixtoword, cixtotree, ixtoclsinfo], [bias_init_intra_class, bias_init_inter_class]

# ========================================================================================
# LSTM LAYER DEFINITION
# This is a simple forward propogating lstm layer with no bells and whistles,
# This can be used to encode an input sequence or for training mode in image captioning
# Supports arbitrarily deep lstm layer, but only forward propogation and is only
# ========================================================================================

def basic_lstm_layer(tparams, state_below, aux_input, use_noise, options, prefix='lstm', sched_prob_mask = [], attn_nw = None):
  nsteps = state_below.shape[0]
  h_depth = options.get('hidden_depth',1)
  h_sz = options['hidden_size']

  if state_below.ndim == 3:
      n_samples = state_below.shape[1]
  else:
      n_samples = 1

  def _step(x_in, xp_m,  h_, c_, xwout_, xAux):
      preact = tensor.dot(sliceT(h_, 0, h_sz), tparams[_p(prefix, 'W_hid')])
      if options.get('sched_sampling_mode',None) == None:
        preact += x_in
      else:
        xy_emb = tensor.dot(xwout_, tparams[_p(prefix, 'W_inp')] + tparams[_p(prefix, 'b')])
        temp_container = tensor.concatenate([xy_emb.dimshuffle('x',0,1), x_in.dimshuffle('x', 0, 1)],axis=0)
        preact += temp_container[ xp_m, tensor.arange(n_samples),:]

      if options.get('en_aux_inp',0):
          if attn_nw == None:
            preact += tensor.dot(xAux,tparams[_p(prefix,'W_aux')])
          else:
             # Which hidden state should be used here!? Now using the bottom most one
            emb_xAux = attn_nw(xAux, sliceT(h_, 0, h_sz), use_noise, tparams['WIemb_aux'], tparams['b_Img_aux'])
            preact += tensor.dot(emb_xAux,tparams[_p(prefix,'W_aux')])

      #  preact += tparams[_p(prefix, 'b')]
      h = [[]]*h_depth
      c = [[]]*h_depth
      outp = [[]]*h_depth

      for di in xrange(h_depth):
          i = tensor.nnet.sigmoid(sliceT(preact, 0, h_sz))
          f = tensor.nnet.sigmoid(sliceT(preact, 1, h_sz))
          o = tensor.nnet.sigmoid(sliceT(preact, 2, h_sz))
          c[di] = tensor.tanh(sliceT(preact, 3, h_sz))
          c[di] = f * sliceT(c_, di, h_sz) + i * c[di]
          h[di] = o * tensor.tanh(c[di])
          outp[di] = h[di]
          if options.get('en_residual_conn',1):
              if (di > 0):
                outp[di] += outp[di-1]
                print "Connecting residual at %d"%(di)
          if di < (h_depth - 1):
              preact = tensor.dot(sliceT(h_, di+1, h_sz), tparams[_p(prefix, ('W_hid_' + str(di+1)))]) + \
                      tensor.dot(outp[di], tparams[_p(prefix, ('W_inp_' + str(di+1)))])

      c_out = tensor.concatenate(c,axis=1)
      h_out = tensor.concatenate(h+[outp[-1]],axis=1)
      if options.get('sched_sampling_mode',None) == None:
        xw_out = xwout_
      else:
        y = tensor.dot(h[-1],tparams['Wd']) + tparams['bd']
        xWIdx =  tensor.argmax(y, axis=-1,keepdims=True)
        xw_out = tparams['Wemb'][xWIdx.flatten()].reshape([n_samples,options['word_encoding_size']])

      return h_out, c_out, xw_out

  state_below = (tensor.dot(state_below, tparams[_p(prefix, 'W_inp')]) + tparams[_p(prefix, 'b')])

  if options.get('en_aux_inp',0) == 0:
     aux_input = []

  if options.get('sched_sampling_mode',None) == None:
    sched_prob_mask = tensor.alloc(1, nsteps, n_samples)
    xw_out = tensor.alloc(numpy_floatX(0.), 1, 1)
  else:
    xw_out = tensor.alloc(numpy_floatX(0.), n_samples, options['word_encoding_size'])

  rval, updates = theano.scan(_step,
                              sequences=[state_below, sched_prob_mask],
                              outputs_info=[tensor.alloc(numpy_floatX(0.),
                                                         n_samples,
                                                         (h_depth+1)*h_sz),
                                            tensor.alloc(numpy_floatX(0.),
                                                         n_samples,
                                                         h_depth*h_sz),
                                            xw_out
                                            ],
                              non_sequences = [aux_input] ,
                              name=_p(prefix, '_layers'),
                              n_steps=nsteps)
  return rval, updates

# ======================== Dropout Layer =================================================
# Implements a simple dropout layer. When droput is on it drops units according to speeci-
# -fied prob and scales the rest. NOP otherwise
# ========================================================================================
def dropout_layer(inp, use_noise, trng, prob, shp):
  scale = 1.0/(1.0-prob)
  proj = tensor.switch(use_noise,
                       (inp *
                        trng.binomial(shp,
                                      p=prob, n=1,
                                      dtype=inp.dtype)*scale),
                       inp)
  return proj

# ======================== Multimodal cosine sim =========================================
# Embeds Image feature vector and computes the cosine sim and softmax with a given sent emb
# ========================================================================================
def multimodal_cosine_sim_softmax(embImg, sent_emb, tparams, sm_f):
  sim_score = tensor.dot(embImg,sent_emb.T)/tensor.dot(embImg.norm(2,axis=1)[:,None],sent_emb.norm(2,axis=1)[None,:])
  # Now to implement the cost function!
  # We can use two kinds of cost, ranking hinge loss or negative log likelihod
  # Below we implement negetive log_likelihood
  smooth_factor = tensor.as_tensor_variable(numpy_floatX(sm_f), name='sm_f')
  probMatch = tensor.nnet.softmax(sim_score*smooth_factor)

  return probMatch,sim_score

# ======================== Multimodal cosine sim =========================================
# Embeds Image feature vector and computes the cosine sim and softmax with a given sent emb
# ========================================================================================
def multimodal_euc_dist_softmax(embImg, sent_emb, tparams, sm_f):
  euc_dist = ((embImg[:,None,:] - sent_emb[None,:,:])**2).sum(axis=-1) / (xI.shape[0] * xI.shape[1])
  # Now to implement the cost function!
  # We can use two kinds of cost, ranking hinge loss or negative log likelihod
  # Below we implement negetive log_likelihood
  smooth_factor = tensor.as_tensor_variable(numpy_floatX(sm_f), name='sm_f')
  probMatch = tensor.nnet.softmax(-euc_dist*smooth_factor)

  return probMatch, euc_dist

def normalizeSal(sal, normType = 'L1'):
    if normType[0] == 'L':
        return np.power(sal,float(normType[-1]))/np.power(sal,float(normType[-1])).sum()
    elif normType[0] == 'S':
        return (1 + np.power(sal,float(normType[-1]))/np.power(sal,float(normType[-1])).sum())

def applyFeatPool(pooltype, featInp, proj_mat = None):
  if pooltype == 'max':
    return featInp.max(axis=1, keepdims=True)
  elif pooltype == 'mean':
    return featInp.mean(axis=1, keepdims=True)
  elif pooltype == 'min':
    return featInp.min(axis=1, keepdims=True)
  elif pooltype == 'sqrmean':
    return (featInp**2).mean(axis=1, keepdims=True)
  elif pooltype == 'none':
    return featInp
  elif pooltype == 'randprojconcat':
    return np.dot(proj_mat,featInp).T.flatten()[:,None]
  elif 'sal_' in pooltype:
    poolConfig = pooltype.split('_')[1:]
    dims_f = map(int,poolConfig[0].split('x'))
    dims_sc = map(int,poolConfig[1].split('x')) if poolConfig[1] != 'I' else poolConfig[1]
    sal_norm = poolConfig[2]
    dims_pool = map(int,poolConfig[3].split('x'))
    pool_type = poolConfig[4]
    fullFeat = featInp[:reduce(lambda x,y: x*y, dims_f),:].reshape(dims_f)
    if dims_sc != 'I':
        fullSc   = featInp[reduce(lambda x,y: x*y, dims_f):,:].reshape(dims_sc)
        fullSc = normalizeSal(fullSc, sal_norm)
    red_dims = [dims_f[-2] - dims_pool[0]+1, dims_f[-1] - dims_pool[ 1]+1]
    red_feat = np.zeros([fullFeat.shape[0]*dims_pool[0]*dims_pool[1],red_dims[0]*red_dims[1]], dtype=config.floatX)
    red_sc = np.zeros([1,red_dims[0]*red_dims[1]], dtype=np.float32)
    for i in xrange(red_dims[0]):
        for j in xrange(red_dims[1]):
            red_feat[:,i*red_dims[0]+j] = fullFeat[:,i:i+dims_pool[0],j:j+dims_pool[1]].T.flatten()
            red_sc[:,i*red_dims[0]+j] = fullSc[:,i:i+dims_pool[0],j:j+dims_pool[1]].sum() if dims_sc !='I' else 1.
    red_sc = normalizeSal(red_sc, sal_norm)
    return applyFeatPool(pool_type, red_feat*red_sc, proj_mat = proj_mat)
  else:
    raise ValueError('Unknown pooling type %s'%(pooltype))

# Utilitites related to gumbel softmax:
def sample_gumbel(trng, shape, eps=np.float32(1e-8), U=None):
  """Sample from Gumbel(0, 1)"""
  if U == None:
    U = trng.uniform(shape,low=0.,high=1.)
  return -tensor.log(-tensor.log(U + eps) + eps)

# Utilitites related to gumbel softmax:
def gumbel_softmax_sample(trng, logits, tau, U=None, hard=False):
  """Sample from Gumbel(0, 1)"""
  ylog = logits + sample_gumbel(trng, logits.shape, U=U)
  y = tensor.nnet.softmax(ylog/tau)

  if hard:
      print 'Using hard gumbel'
      one_hot = tensor.cast(tensor.eq(y, y.max(axis=-1,keepdims=1)),dtype=config.floatX)
      y = theano.gradient.disconnected_grad(one_hot -y) + y
  return y

# Utilitites related to gumbel softmax:
def sample_gumbel_np(shape, eps=1e-20):
  """Sample from Gumbel(0, 1)"""
  U = np.random.uniform(0.0,1.0,size=shape)
  return -np.log(-np.log(U + eps) + eps)

# Utilitites related to gumbel softmax:
def gumbel_softmax_sample_np(logits, tau):
  """Sample from Gumbel(0, 1)"""
  y = logits + sample_gumbel_np(logits.shape)
  return softmax(y/tau)

# -----------------------------------------------
def find_ngrams(input_list, n):
    return zip(*[input_list[i:] for i in range(n)])

def compute_div_n(caps,n=1):
  aggr_div = []
  for k in caps:
      all_ngrams = set()
      lenT = 0.
      for c in caps[k]:
         tkns = c.split()
         lenT += len(tkns)
         ng = find_ngrams(tkns, n)
         all_ngrams.update(ng)
      aggr_div.append(float(len(all_ngrams))/ (1e-6 + float(lenT)))
  return np.array(aggr_div).mean(), np.array(aggr_div)

def compute_global_div_n(caps,n=1):
  aggr_div = []
  all_ngrams = set()
  lenT = 0.
  for k in caps:
      for c in caps[k]:
         tkns = c.split()
         lenT += len(tkns)
         ng = find_ngrams(tkns, n)
         all_ngrams.update(ng)
  if n == 1:
    aggr_div.append(float(len(all_ngrams)))
  else:
    aggr_div.append(float(len(all_ngrams))/ (1e-6 + float(lenT)))
  return aggr_div[0], np.repeat(np.array(aggr_div),len(caps))

max_stars_repo_name: rakshithShetty/captionGAN

reason: import numpy,import theano,from theano


text:
import itertools
import json
import os
from collections import defaultdict
from typing import List, Set

import numpy as np
import torch
from scipy import sparse as sp
from torch import nn
from tqdm import tqdm

import config as cfg
from nn_models import ItemModel, UserModel
from utils import (
    ProductEncoder,
    TrainingSample,
    coo_to_pytorch_sparse,
    get_shard_path,
    make_coo_row,
    normalized_average_precision,
)

MAX_COVERAGE = 0
MAX_AP = 0

def collect_train_data(jsons: List[str], product_encoder: ProductEncoder) -> List[TrainingSample]:
    samples = []
    for js_path in jsons:
        print("Load samples from {}".format(js_path))
        for js in tqdm((json.loads(s) for s in open(js_path))):
            samples.append(
                TrainingSample(
                    row=make_coo_row(js["transaction_history"], product_encoder),
                    target_items=set(product_encoder.toIdx(js["target"][0]["product_ids"])),
                    client_id=js["client_id"],
                )
            )
    return samples

def sample_aux_batch(batch: List[TrainingSample], neg_rate: int = 2, max_id: int = 43038):
    batch_indices = []
    batch_target = []
    batch_repeat_users = []
    for sample in batch:
        cur_repeat = 0
        if len(sample.target_items) > 0:
            positive_ids = sample.target_items
            num_positive = len(positive_ids)
            batch_indices.extend(positive_ids)
            batch_target.extend([1.0,] * num_positive)

            neg_candidates = np.random.choice(max_id, num_positive * neg_rate)
            negative_ids = [idx for idx in neg_candidates if idx not in positive_ids]
            num_negative = len(negative_ids)
            batch_indices.extend(negative_ids)
            batch_target.extend([-1.0,] * num_negative)
            cur_repeat = num_positive + num_negative
        batch_repeat_users.append(cur_repeat)

    return (
        torch.LongTensor(batch_repeat_users).cuda(),
        torch.LongTensor(batch_indices).cuda(),
        torch.FloatTensor(batch_target).cuda(),
    )

def evaluate(user_model: UserModel, item_model: ItemModel, valid_data: List[TrainingSample]):
    from sklearn.neighbors import NearestNeighbors

    user_vectors = (
        user_model(coo_to_pytorch_sparse(sp.vstack([sample.row for sample in valid_data])).cuda())
        .data.cpu()
        .numpy()
    )
    item_vectors = item_model._embeds.weight.data.cpu().numpy()
    knn = NearestNeighbors(n_neighbors=30, metric="cosine")
    knn.fit(item_vectors)
    preds = knn.kneighbors(user_vectors, n_neighbors=30, return_distance=False)

    coverage = []
    for i, recommended in enumerate(preds):
        gt = valid_data[i].target_items
        coverage.append(len(gt.intersection(recommended)) / len(gt))

    ap = []
    for i, recommended in enumerate(preds):
        gt = valid_data[i].target_items
        ap.append(normalized_average_precision(gt, recommended))

    return np.mean(coverage), np.mean(ap)

def eval_and_dump(
    batch_cnt: int,
    user_model: UserModel,
    item_model: ItemModel,
    valid_data: List[TrainingSample],
    output_root: str,
):
    global MAX_COVERAGE
    global MAX_AP
    coverage, ap = evaluate(user_model, item_model, valid_data)
    stats = {"batch_cnt": batch_cnt, "coverage": coverage, "ap": ap}
    print("[eval] {}".format(stats))
    if coverage > MAX_COVERAGE:
        MAX_COVERAGE = coverage
        print("\t This is a new COVERAGE leader!")
        output_dir = output_root + "/cov/"
        os.makedirs(output_dir, exist_ok=True)
        print("\t Save state to {}".format(output_dir))
        torch.save(user_model.state_dict(), output_dir + "/user_model.pth")
        torch.save(item_model.state_dict(), output_dir + "/item_model.pth")
        json.dump(stats, open(output_dir + "/stats.json", "w"))
    if ap > MAX_AP:
        MAX_AP = ap
        print("\t This is a new AP leader!")
        output_dir = output_root + "/ap/"
        os.makedirs(output_dir, exist_ok=True)
        print("\t Save state to {}".format(output_dir))
        torch.save(user_model.state_dict(), output_dir + "/user_model.pth")
        torch.save(item_model.state_dict(), output_dir + "/item_model.pth")
        json.dump(stats, open(output_dir + "/stats.json", "w"))

if __name__ == "__main__":
    np.random.seed(42)
    torch.manual_seed(43)

    product_encoder = ProductEncoder(cfg.PRODUCT_CSV_PATH)
    train_samples = collect_train_data([get_shard_path(i) for i in range(8)], product_encoder,)
    valid_samples = collect_train_data([get_shard_path(15)], product_encoder)

    dim = 512
    user_model = UserModel(product_encoder.num_products, dim).cuda()
    item_model = ItemModel(product_encoder.num_products, dim).cuda()

    criterion = nn.CosineEmbeddingLoss(margin=0.01).cuda()
    optimizer = torch.optim.Adam(list(user_model.parameters()) + list(item_model.parameters()), lr=0.01)

    epoches = (
        [{"num_batches": 512, "batch_size": 32, "neg_rate": 32}]
        + [
            {"num_batches": 256, "batch_size": 64, "neg_rate": 256},
            {"num_batches": 256, "batch_size": 128, "neg_rate": 128},
        ]
        * 30
        + [{"num_batches": 256, "batch_size": 64, "neg_rate": 256}] * 20
    )

    batch_cnt = 0
    for epoch in epoches:
        for _ in tqdm(range(epoch["num_batches"])):
            batch_cnt += 1

            optimizer.zero_grad()

            batch_samples = np.random.choice(train_samples, epoch["batch_size"], replace=False)
            _input = coo_to_pytorch_sparse(sp.vstack([sample.row for sample in batch_samples])).cuda()
            _repeat, _idx, _target = sample_aux_batch(
                batch=batch_samples, neg_rate=epoch["neg_rate"], max_id=product_encoder.num_products
            )

            raw_users = user_model.forward(_input)
            repeated_users = torch.repeat_interleave(raw_users, _repeat, dim=0)
            repeated_items = item_model.forward(_idx)

            loss = criterion(repeated_items, repeated_users, _target)
            loss.backward()
            optimizer.step()
        eval_and_dump(batch_cnt, user_model, item_model, valid_samples, "../tmp/embds_d{}/".format(dim))

reason: import numpy,from scipy


text:
""" get scores for F1 Threshold(rho) experiments for competitive approaches """
# requires:
#   - json file with predicted answers
#   - json data file

from __future__ import print_function
from collections import Counter
import string
import re
import argparse
import json
import sys

import numpy as np
from collections import OrderedDict

def normalize_answer(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        return re.sub(r'\b(a|an|the)\b', ' ', text)

    def white_space_fix(text):
        return ' '.join(text.split())

    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)

    def lower(text):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(s))))

def f1_score(prediction, ground_truth):
    prediction_tokens = normalize_answer(prediction).split()
    ground_truth_tokens = normalize_answer(ground_truth).split()
    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)
    num_same = sum(common.values())
    if num_same == 0:
        return 0
    precision = 1.0 * num_same / len(prediction_tokens)
    recall = 1.0 * num_same / len(ground_truth_tokens)
    f1 = (2 * precision * recall) / (precision + recall)
    return f1

def exact_match_score(prediction, ground_truth):
    return (normalize_answer(prediction) == normalize_answer(ground_truth))

def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):
    scores_for_ground_truths = []
    for ground_truth in ground_truths:
        score = metric_fn(prediction, ground_truth)
        scores_for_ground_truths.append(score)
    return max(scores_for_ground_truths)

def evaluate(dataset, predictions):
    f1 = exact_match = total = 0
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                total += 1
                if qa['id'] not in predictions:
                    message = 'Unanswered question ' + qa['id'] + \
                              ' will receive score 0.'
                    print(message, file=sys.stderr)
                    continue
                ground_truths = list(map(lambda x: x['text'], qa['answers']))
                prediction = predictions[qa['id']]
                exact_match += metric_max_over_ground_truths(
                    exact_match_score, prediction, ground_truths)
                f1 += metric_max_over_ground_truths(
                    f1_score, prediction, ground_truths)

    exact_match = 100.0 * exact_match / total
    f1 = 100.0 * f1 / total

    return {'EM': round(exact_match, 2), 'F1': round(f1, 2)}

def evaluate_dev_flat(dataset, predictions):
    f1 = exact_match = total = 0
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                if qa['id'] not in predictions:
                    message = 'Unanswered question ' + qa['id'] + \
                              ' will receive score 0.'
                    print(message, file=sys.stderr)
                    continue
                ground_truths = list(map(lambda x: x['text'], qa['answers']))
                ans_set = set(ground_truths)
                for a_i in range(len(set(ground_truths))):
                        if qa['answers'][a_i]['text'] in ans_set:
                                 ans_set.remove(qa['answers'][a_i]['text'])
                                 total += 1
                        else:
                                 continue
                        gt_ans = [qa['answers'][a_i]['text']]
                        prediction = predictions[qa['id']]
                        exact_match += metric_max_over_ground_truths(
                            exact_match_score, prediction, gt_ans)
                        f1 += metric_max_over_ground_truths(
                            f1_score, prediction, gt_ans)

    exact_match = 100.0 * exact_match / total
    f1 = 100.0 * f1 / total

    return {'EM': round(exact_match, 2), 'F1': round(f1, 2)}

def toscore(score, total):
    d = {}
    for p,s in score.items():
        d[p] = round(100.*s/total, 2)
    td = OrderedDict(sorted(d.items(), key=lambda t: t[0]))
    return td

def evaluate_dev_a(dataset, predictions, n_ans):
    f1 = exact_match = total = 0
    t_a = {0.1:0, 0.2:0, 0.3:0, 0.4:0, 0.5:0, 0.6:0, 0.7:0, 0.8:0, 0.9:0}
    f1_all = []
    for article in dataset:
        for paragraph in article['paragraphs']:
            for qa in paragraph['qas']:
                if qa['id'] not in predictions:
                    message = 'Unanswered question ' + qa['id'] + \
                              ' will receive score 0.'
                    print(message, file=sys.stderr)
                    continue
                ground_truths = list(map(lambda x: x['text'], qa['answers']))
                ans_set = set(ground_truths)
                if len(ans_set) < n_ans or (n_ans == 1 and len(ans_set)>1):
                    continue
                ans_set = list(ans_set)[:n_ans]
                f1s = []; total += 1
                prediction = predictions[qa['id']]
                for a in ans_set:
                    f1s += [metric_max_over_ground_truths(f1_score, prediction, [a])]

                f1_all += [max(f1s)]
                f1s = np.array(f1s)
                for p in t_a.keys():
                    t_a[p] = t_a[p] + int((f1s>p).sum() == n_ans)

    f1s_all = round(100. * sum(f1_all) / len(f1_all), 2)

    t_ans = toscore(t_a, total)

    return t_ans, total, f1s_all

if __name__ == '__main__':
    expected_version = '1.1'
    parser = argparse.ArgumentParser(
        description='Evaluation for SQuAD ' + expected_version)
    parser.add_argument('dataset_file', help='Dataset file')
    parser.add_argument('prediction_file', help='Prediction File')
    args = parser.parse_args()
    with open(args.dataset_file) as dataset_file:
        dataset_json = json.load(dataset_file)
        if (dataset_json['version'] != expected_version):
            print('Evaluation expects v-' + expected_version +
                  ', but got dataset with v-' + dataset_json['version'],
                  file=sys.stderr)
        dataset = dataset_json['data']
    with open(args.prediction_file) as prediction_file:
        predictions = json.load(prediction_file)

    print("Orig dev set", json.dumps(evaluate(dataset, predictions)))
    print("Flat dev set", json.dumps(evaluate_dev_flat(dataset, predictions)))

    print("Dev |a|=1", json.dumps(evaluate_dev_a(dataset, predictions, 1)))
    print("Dev |a|=2", json.dumps(evaluate_dev_a(dataset, predictions, 2)))
    print("Dev |a|=3", json.dumps(evaluate_dev_a(dataset, predictions, 3)))


text:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np

iris = pd.read_csv("iris.csv")

# violine plot combines benefit of PDF and boxplot
# by making densor regions of data fatter and sparser ones thinner in violin plot

sns.violinplot(x="species", y="petal_length", data=iris, size=8)
plt.savefig("1.3_violin_plot.png")
plt.show()

max_stars_repo_name: Akshaykumarcp/FUn-with-EDA

max_issues_repo_name: Akshaykumarcp/EDA

max_forks_repo_name: Akshaykumarcp/EDA


text:
""" Code used with permissision from Sylvia Zhu to calculate the range in
    frequency space that a signal occupies due to spindown and Doppler
    modulations
"""

import numpy as np
import logging

try:
    from astropy import units as u
    from astropy.coordinates import SkyCoord
    from astropy.time import Time
except ImportError:
    logging.warning("Python module astropy not installed")
import lal

# Assume Earth goes around Sun in a non-wobbling circle at constant speed;
# Still take the zero longitude to be the Earth's position during the March
# equinox, or March 20.
# Each day the Earth moves 2*pi/365 radians around its orbit.

def _eqToEcl(alpha, delta):
    source = SkyCoord(alpha * u.radian, delta * u.radian, frame="gcrs")
    out = source.transform_to("geocentrictrueecliptic")
    return np.array([out.lon.radian, out.lat.radian])

def _eclToEq(lon, lat):
    source = SkyCoord(lon * u.radian, lat * u.radian, frame="geocentrictrueecliptic")
    out = source.transform_to("gcrs")
    return np.array([out.ra.radian, out.dec.radian])

def _calcDopplerWings(s_freq, s_alpha, s_delta, lonStart, lonStop, numTimes=100):
    e_longitudes = np.linspace(lonStart, lonStop, numTimes)
    v_over_c = 2 * np.pi * lal.AU_SI / lal.YRSID_SI / lal.C_SI
    s_lon, s_lat = _eqToEcl(s_alpha, s_delta)

    vertical = s_lat
    horizontals = s_lon - e_longitudes

    dopplerShifts = s_freq * np.sin(horizontals) * np.cos(vertical) * v_over_c
    F0min, F0max = np.amin(dopplerShifts), np.amax(dopplerShifts)

    # Add twice the spin-modulation
    SpinModulationMax = 2 * np.pi * lal.REARTH_SI / lal.DAYSID_SI / lal.C_SI * s_freq
    return F0min - 2 * SpinModulationMax, F0max + 2 * SpinModulationMax

def _calcSpindownWings(freq, fdot, minStartTime, maxStartTime):
    timespan = maxStartTime - minStartTime
    return 0.5 * timespan * np.abs(fdot) * np.array([-1, 1])

def get_frequency_range_of_signal(F0, F1, Alpha, Delta, minStartTime, maxStartTime):
    """ Calculate the frequency range that a signal will occupy

    Parameters
    ----------
    F0, F1, Alpha, Delta: float
        Frequency, derivative, and sky position for the signal (all angles in
        radians)
    minStartTime, maxStartTime: float
        GPS time of the start and end of the data span

    Note: assumes tref is in the middle of the data span

    Note: See also `pyfstat.helper_functions.get_covering_band`

    Returns
    -------
    [Fmin, Fmax]: array
        The minimum and maximum frequency span
    """
    YEAR_IN_DAYS = lal.YRSID_SI / lal.DAYSID_SI
    tEquinox = 79

    minStartTime_t = Time(minStartTime, format="gps").to_datetime().timetuple()
    maxStartTime_t = Time(maxStartTime, format="gps").to_datetime().timetuple()
    tStart_days = minStartTime_t.tm_yday - tEquinox
    tStop_days = maxStartTime_t.tm_yday - tEquinox
    tStop_days += (maxStartTime_t.tm_year - minStartTime_t.tm_year) * YEAR_IN_DAYS

    lonStart = 2 * np.pi * tStart_days / YEAR_IN_DAYS - np.pi
    lonStop = 2 * np.pi * tStop_days / YEAR_IN_DAYS - np.pi

    dopplerWings = _calcDopplerWings(F0, Alpha, Delta, lonStart, lonStop)
    spindownWings = _calcSpindownWings(F0, F1, minStartTime, maxStartTime)
    return np.array([F0, F0]) + dopplerWings + spindownWings

max_stars_repo_name: pepCV/PyFstat

max_issues_repo_name: GregoryAshton/PyFstat

max_forks_repo_name: GregoryAshton/PyFstat

reason: import numpy,from astropy


text:
"""A basic encoder-decoder model."""
import itertools
import numpy
import theano
from theano import tensor as T
from theano.ifelse import ifelse
import sys

from encdecspec import EncoderDecoderSpec
from derivation import Derivation
from neural import NeuralModel, CLIP_THRESH
from vocabulary import Vocabulary

class EncoderDecoderModel(NeuralModel):
  """An encoder-decoder RNN model."""
  def setup(self):
    self.setup_encoder()
    self.setup_decoder_step()
    self.setup_decoder_write()
    self.setup_backprop()

  @classmethod
  def get_spec_class(cls):
    return EncoderDecoderSpec

  def setup_encoder(self):
    """Run the encoder.  Used at test time."""
    x = T.lvector('x_for_enc')
    def recurrence(x_t, h_prev, *params):
      return self.spec.f_enc(x_t, h_prev)
    results, _ = theano.scan(recurrence, sequences=[x],
                             outputs_info=[self.spec.get_init_state()],
                             non_sequences=self.spec.get_all_shared())
    h_last = results[-1]
    self._encode = theano.function(inputs=[x], outputs=h_last)

  def setup_decoder_step(self):
    """Advance the decoder by one step.  Used at test time."""
    y_t = T.lscalar('y_t_for_dec')
    h_prev = T.vector('h_prev_for_dec')
    h_t = self.spec.f_dec(y_t, h_prev)
    self._decoder_step = theano.function(inputs=[y_t, h_prev], outputs=h_t)

  def setup_decoder_write(self):
    """Get the write distribution of the decoder.  Used at test time."""
    h_prev = T.vector('h_prev_for_write')
    h_for_write = self.spec.decoder.get_h_for_write(h_prev)
    write_dist = self.spec.f_write(h_for_write)
    self._decoder_write = theano.function(inputs=[h_prev], outputs=write_dist)

  def setup_backprop(self):
    eta = T.scalar('eta_for_backprop')
    x = T.lvector('x_for_backprop')
    y = T.lvector('y_for_backprop')
    y_in_x_inds = T.lmatrix('y_in_x_inds_for_backprop')
    l2_reg = T.scalar('l2_reg_for_backprop')
    def enc_recurrence(x_t, h_prev, *params):
      return self.spec.f_enc(x_t, h_prev)
    enc_results, _ = theano.scan(fn=enc_recurrence,
                                 sequences=[x],
                                 outputs_info=[self.spec.get_init_state()],
                                 non_sequences=self.spec.get_all_shared())
    h_last = enc_results[-1]

    def decoder_recurrence(y_t, h_prev, *params):
      h_for_write = self.spec.decoder.get_h_for_write(h_prev)
      write_dist = self.spec.f_write(h_for_write)
      p_y_t = write_dist[y_t]
      h_t = self.spec.f_dec(y_t, h_prev)
      return (h_t, p_y_t)
    dec_results, _ = theano.scan(
        fn=decoder_recurrence, sequences=[y],
        outputs_info=[h_last, None],
        non_sequences=self.spec.get_all_shared())
    p_y_seq = dec_results[1]
    log_p_y = T.sum(T.log(p_y_seq))
    nll = -log_p_y
    # Add L2 regularization
    objective = nll
    for p in self.params:
      objective += l2_reg / 2 * T.sum(p**2)
    gradients = T.grad(objective, self.params)
    self._get_nll = theano.function(
        inputs=[x, y, y_in_x_inds], outputs=nll, on_unused_input='warn')

    # Do the updates here
    updates = []
    for p, g in zip(self.params, gradients):
      grad_norm = g.norm(2)
      clipped_grad = ifelse(grad_norm >= CLIP_THRESH,
                            g * CLIP_THRESH / grad_norm, g)
      updates.append((p, p - eta * clipped_grad))

    self._backprop = theano.function(
        inputs=[x, y, eta, y_in_x_inds, l2_reg],
        outputs=[p_y_seq, objective],
        updates=updates, on_unused_input='warn')

  def decode_greedy(self, ex, max_len=100):
    h_t = self._encode(ex.x_inds)
    y_tok_seq = []
    p_y_seq = []  # Should be handy for error analysis
    p = 1
    for i in range(max_len):
      write_dist = self._decoder_write(h_t)
      y_t = numpy.argmax(write_dist)
      p_y_t = write_dist[y_t]
      p_y_seq.append(p_y_t)
      p *= p_y_t
      if y_t == Vocabulary.END_OF_SENTENCE_INDEX:
        break
      y_tok = self.out_vocabulary.get_word(y_t)
      y_tok_seq.append(y_tok)
      h_t = self._decoder_step(y_t, h_t)
    return [Derivation(ex, p, y_tok_seq)]

  def decode_beam(self, ex, beam_size=1, max_len=100):
    h_t = self._encode(ex.x_inds)
    beam = [[Derivation(ex, 1, [], hidden_state=h_t,
                        attention_list=[], copy_list=[])]]
    finished = []
    for i in range(1, max_len):
      new_beam = []
      for deriv in beam[i-1]:
        cur_p = deriv.p
        h_t = deriv.hidden_state
        y_tok_seq = deriv.y_toks
        write_dist = self._decoder_write(h_t)
        sorted_dist = sorted([(p_y_t, y_t) for y_t, p_y_t in enumerate(write_dist)],
                             reverse=True)
        for j in range(beam_size):
          p_y_t, y_t = sorted_dist[j]
          new_p = cur_p * p_y_t
          if y_t == Vocabulary.END_OF_SENTENCE_INDEX:
            finished.append(Derivation(ex, new_p, y_tok_seq))
            continue
          y_tok = self.out_vocabulary.get_word(y_t)
          new_h_t = self._decoder_step(y_t, h_t)
          new_entry = Derivation(ex, new_p, y_tok_seq + [y_tok],
                                 hidden_state=new_h_t)
          new_beam.append(new_entry)
      new_beam.sort(key=lambda x: x.p, reverse=True)
      beam.append(new_beam[:beam_size])
      finished.sort(key=lambda x: x.p, reverse=True)
    return sorted(finished, key=lambda x: x.p, reverse=True)


text:
"""
Using OpenXBOW to extract Bag of Words feature from LLDs
https://github.com/openXBOW/openXBOW
IO handler:
db --> mfcc.csv -->(OpenXBow) --> bow.csv --> db
"""
import os
import traceback
import pandas as pd
import config
import numpy as np
from concurrent.futures import ProcessPoolExecutor, as_completed
from common.sql_handler import SqlHandler
from common.log_handler import get_logger
logger = get_logger()

def worker(sample_id):
    sql = f"select * from {config.tbl_mfcc} where name ={sample_id} "
    sql_handler = SqlHandler(type=config.db_type)
    res = sql_handler.get_df(sql=sql)
    sql_handler.disconnect()
    return res

def gen_bow(modal, llds,
            window_size=4,
            hop_size=0.5,
            size=100,
            a=1,
            attributes='nt1[39]',
            log=None,
            ):
    """generate bow (boaw or bovw) features from llds:
    Args:
        modal:                     video, audio or text
        llds:                      lld name , example: mfcc
        window_szie and hop_szie:  '-t' param in OpenXBOW
        size:                      codebook length, can be seen as cluster numbers
                                   and '-size' param in OpenXBOW
        a:                         the number of nearest nodes in clustering, 1 in defalt
        attributes:                split the feature into subsets to train cluster model
                                   it is the '--attributes' param on OpenXBOW
        log:                       the histgram representation of BoW vector
    """

    if modal == 'audio':
        if llds == 'mfcc':
            table_name = config.tbl_mfcc
            outpot_table = config.tbl_boaw_mfcc
        elif llds == 'egemaps':
            table_name = config.tbl_egemaps
            output_table = config.tbl_boaw_egemaps
    elif modal == 'video':
        if llds == 'pose_gaze_faus':
            table_name = config.tbl_pose_gaze_faus
            output_table = config.tbl_bovw_pose_gaze_faus
    # prepare llds data
    input_file = table_name + '_llds.csv'
    if not os.path.exists(input_file):
        # sql_handler = SqlHandler(config.db_type)
        # names = sql_handler.query("select distinct name from %s" % table_name)
        # sql_handler.disconnect()
        # with ProcessPoolExecutor(max_workers=19) as executor:
        #     tasks = [executor.submit(worker, name[0]) for name in names]

        # dfs = []
        # for name in names:
        #     df = worker(name[0])
        #     df = df.drop(['frameTime'], axis=1)
        #     dfs.append(df)
        # for future in as_completed(tasks):
        #     try:
        #         df = future.result().drop(['frameTime'], axis=1)
        #         dfs.append(df)
        #     except Exception:
        #         traceback.print_exc()
        #         continue
        # llds_df = pd.concat(dfs, axis=0)  # stack by row
        # llds_df.to_csv(input_file, header=None, sep=';')

        sql_handler = SqlHandler(type=config.db_type)
        llds_df = sql_handler.get_df(table_name)
        logger.info('[LLDS BOW get data from db over...]')
        llds_df = llds_df.drop(['frameTime'], axis=1)
        llds_df.to_csv(input_file, header=None, sep=';')

    openxbow = 'java -jar ' + config.jar_path + ' -i ' + input_file
    output_file = table_name + '_bow.csv'
    if os.path.exists(output_file): os.system(f'rm {output_file}')
    openxbow += f' -o {output_file}'
    openxbow += f' -attributes {attributes}'
    openxbow += f' -t {window_size} {hop_size} '
    openxbow += f' -size {size}'
    if a is not None:
        openxbow += f' -a {a}'
    if log is not None:
        openxbow += f' -log {log}'

    logger.info(openxbow)
    try:
        logger.info("[OpXBOW Starting....]")
        print(openxbow)
        os.system(openxbow)
    except:
        logger.info('Failing OpenXBOW....')
        return
    if os.path.exists(output_file):
        sql_handler = SqlHadnler()
        df = pd.read_csv(output_file, sep=';')
        sql_handler.df_to_db(df, output_table, if_exists='replace')
        logger.info(f'Store OpenXBOW...{output_table}')


text:
from study_lyte.detect import get_signal_event, get_acceleration_start, get_acceleration_stop, get_nir_surface, get_normal_neutral_acceleration
import pytest
import numpy as np
import pandas as pd

@pytest.mark.parametrize("data, threshold, direction, expected", [
    (np.array([0, 1, 0]), 0.5, 'forward', 1),
    (pd.Series([1, 0, 0]), 0.5, 'forward', 0),
    (np.array([0, 0, 0.1]), 0.01, 'backwards', 2),
    (np.array([20, 100, 100, 40]), 50, 'backwards', 2),
])
def test_get_signal_event(data, threshold, direction, expected):
    """
    Test the signal event is capable of return the correct index
    regardless of threshold, direction, and series or numpy array
    """
    idx = get_signal_event(data, threshold=threshold, search_direction=direction)
    assert idx == expected

@pytest.mark.parametrize("data, norm_value, expected", [
    # Test a predictable normal use
    ([-1, 3,-2,-1], -1, [0, 4, 1, 0]),
    # Test an unusual case of no motion
    ([-1, -1, -1, -1], -1, [0, 0, 0, 0]),
])
def test_get_normal_neutral_acceleration(data, norm_value, expected):
    """
    Test this function produces the expected normalized & biased adjusted acceleration
    """
    d = np.array(data)
    result = get_normal_neutral_acceleration(d, norm_value)
    np.testing.assert_equal(result, np.array(expected))

@pytest.mark.parametrize("data,  n_points_for_basis, threshold, expected", [
    # Test a typical acceleration signal
    ([-1, 0.3, -1.5, -1], 1, 0.1, 1),
    # Test the mean basis value
    ([-2, -6, -8, -12], 2, 2.1, 2),
    # No criteria met, return the first index
    ([-1, -1, -1, -1], 1, 10, 0),
])
def test_get_acceleration_start(data, n_points_for_basis, threshold, expected):
    d = np.array(data)
    idx = get_acceleration_start(d, n_points_for_basis=n_points_for_basis, threshold=threshold)
    assert idx == expected

@pytest.mark.parametrize("data,  n_points_for_basis, threshold, expected", [
    # Test normalization and abs
    ([0.1, -1.5, -1.0], 1, 1.0, 0),
    # Test a no detection returns the last index
    ([-1, -1, -1], 1, 10, 2),
])
def test_get_acceleration_stop(data, n_points_for_basis, threshold, expected):
    d = np.array(data)
    idx = get_acceleration_stop(d, n_points_for_basis=n_points_for_basis, threshold=threshold)
    assert idx == expected

@pytest.mark.parametrize("ambient, active, n_points_for_basis, threashold, expected", [
    # Typical bright->dark ambient
    ([3000, 3000, 1000, 100], [2500, 2500, 3000, 4000], 1, 0.1, 2),
    # no ambient change ( dark or super cloudy)
    ([100, 100, 100, 100], [1000, 1000, 1000, 2000], 2, 0.1, 3)

])
def test_get_nir_surface(ambient, active, n_points_for_basis, threashold, expected):
    amb = np.array(ambient)
    act = np.array(active)
    idx = get_nir_surface(amb, act, n_points_for_basis=n_points_for_basis, threshold=threashold)
    assert idx == expected