text:
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Copyright (c) 2014-2021 Megvii Inc. All rights reserved.

from typing import Optional

import numpy as np

import megengine.functional as F
from megengine import Tensor

__all__ = [
    "batched_nms",
    "filter_box",
    "postprocess",
    "bboxes_iou",
    "matrix_iou",
    "adjust_box_anns",
    "xyxy2xywh",
]

def batched_nms(
    boxes: Tensor, scores: Tensor, idxs: Tensor, iou_thresh: float, max_output: Optional[int] = None
) -> Tensor:
    r"""
    Performs non-maximum suppression (NMS) on the boxes according to
    their intersection-over-union (IoU).
    :param boxes: tensor of shape `(N, 4)`; the boxes to perform nms on;
        each box is expected to be in `(x1, y1, x2, y2)` format.
    :param iou_thresh: ``IoU`` threshold for overlapping.
    :param idxs: tensor of shape `(N,)`, the class indexs of boxes in the batch.
    :param scores: tensor of shape `(N,)`, the score of boxes.

    :return: indices of the elements that have been kept by NMS.

    Examples:
    .. testcode::
        import numpy as np
        from megengine import tensor
        x = np.zeros((100,4))
        np.random.seed(42)
        x[:,:2] = np.random.rand(100,2) * 20
        x[:,2:] = np.random.rand(100,2) * 20 + 100
        scores = tensor(np.random.rand(100))
        idxs = tensor(np.random.randint(0, 10, 100))
        inp = tensor(x)
        result = batched_nms(inp, scores, idxs, iou_thresh=0.6)
        print(result.numpy())

    Outputs:
    .. testoutput::
        [75 41 99 98 69 64 11 27 35 18]
    """
    assert (
        boxes.ndim == 2 and boxes.shape[1] == 4
    ), "the expected shape of boxes is (N, 4)"
    assert scores.ndim == 1, "the expected shape of scores is (N,)"
    assert idxs.ndim == 1, "the expected shape of idxs is (N,)"
    assert (
        boxes.shape[0] == scores.shape[0] == idxs.shape[0]
    ), "number of boxes, scores and idxs are not matched"

    idxs = idxs.detach()
    max_coordinate = boxes.max()
    offsets = idxs * (max_coordinate + 1)
    boxes = boxes + offsets.reshape(-1, 1)
    return F.vision.nms(boxes, scores, iou_thresh, max_output)

def filter_box(output, scale_range):
    """
    output: (N, 5+class) shape
    """
    min_scale, max_scale = scale_range
    w = output[:, 2] - output[:, 0]
    h = output[:, 3] - output[:, 1]
    keep = (w * h > min_scale * min_scale) & (w * h < max_scale * max_scale)
    return output[keep]

def postprocess(prediction, num_classes, conf_thre=0.7, nms_thre=0.45):
    box_corner = F.zeros_like(prediction)
    box_corner[:, :, 0] = prediction[:, :, 0] - prediction[:, :, 2] / 2
    box_corner[:, :, 1] = prediction[:, :, 1] - prediction[:, :, 3] / 2
    box_corner[:, :, 2] = prediction[:, :, 0] + prediction[:, :, 2] / 2
    box_corner[:, :, 3] = prediction[:, :, 1] + prediction[:, :, 3] / 2
    prediction[:, :, :4] = box_corner[:, :, :4]

    output = [None for _ in range(len(prediction))]
    for i, image_pred in enumerate(prediction):

        # If none are remaining => process next image
        if not image_pred.shape[0]:
            continue
        # Get score and class with highest confidence
        # class_conf, class_pred = torch.max(image_pred[:, 5: 5 + num_classes], 1, keepdim=True)
        class_conf = F.max(image_pred[:, 5: 5 + num_classes], axis=1, keepdims=True)
        class_pred = F.argmax(image_pred[:, 5: 5 + num_classes], axis=1, keepdims=True)

        conf_mask = (image_pred[:, 4] * class_conf.reshape(-1) >= conf_thre)
        # Detections ordered as (x1, y1, x2, y2, obj_conf, class_conf, class_pred)
        detections = F.concat((image_pred[:, :5], class_conf, class_pred), 1)
        detections = detections[conf_mask]
        if not detections.shape[0]:
            continue

        nms_out_index = batched_nms(
            detections[:, :4], detections[:, 4] * detections[:, 5], detections[:, 6], nms_thre,
        )
        detections = detections[nms_out_index]
        if output[i] is None:
            output[i] = detections
        else:
            output[i] = F.concat((output[i], detections))

    return output

def bboxes_iou(bboxes_a, bboxes_b, xyxy=True):
    if bboxes_a.shape[1] != 4 or bboxes_b.shape[1] != 4:
        raise IndexError

    if xyxy:
        tl = F.maximum(F.expand_dims(bboxes_a[:, :2], axis=1), bboxes_b[:, :2])
        br = F.minimum(F.expand_dims(bboxes_a[:, 2:], axis=1), bboxes_b[:, 2:])
        area_a = (bboxes_a[:, 2] - bboxes_a[:, 0]) * (bboxes_a[:, 3] - bboxes_a[:, 1])
        area_b = (bboxes_b[:, 2] - bboxes_b[:, 0]) * (bboxes_b[:, 3] - bboxes_b[:, 1])
    else:
        tl = F.maximum(
            F.expand_dims(bboxes_a[:, :2] - bboxes_a[:, 2:] / 2, axis=1),
            (bboxes_b[:, :2] - bboxes_b[:, 2:] / 2),
        )
        br = F.minimum(
            F.expand_dims(bboxes_a[:, :2] + bboxes_a[:, 2:] / 2, axis=1),
            (bboxes_b[:, :2] + bboxes_b[:, 2:] / 2),
        )

        area_a = bboxes_a[:, 2] * bboxes_a[:, 3]
        area_b = bboxes_b[:, 2] * bboxes_b[:, 3]
    mask = (tl < br).astype("int32")
    mask = mask[..., 0] * mask[..., 1]
    diff = br - tl
    area_i = diff[..., 0] * diff[..., 1] * mask
    del diff
    return area_i / (F.expand_dims(area_a, axis=1) + area_b - area_i)

def matrix_iou(a, b):
    """
    return iou of a and b, numpy version for data augenmentation
    """
    lt = np.maximum(a[:, np.newaxis, :2], b[:, :2])
    rb = np.minimum(a[:, np.newaxis, 2:], b[:, 2:])

    area_i = np.prod(rb - lt, axis=2) * (lt < rb).all(axis=2)
    area_a = np.prod(a[:, 2:] - a[:, :2], axis=1)
    area_b = np.prod(b[:, 2:] - b[:, :2], axis=1)
    return area_i / (area_a[:, np.newaxis] + area_b - area_i + 1e-12)

def adjust_box_anns(bbox, scale_ratio, padw, padh, w_max, h_max):
    bbox[:, 0::2] = np.clip(bbox[:, 0::2] * scale_ratio + padw, 0, w_max)
    bbox[:, 1::2] = np.clip(bbox[:, 1::2] * scale_ratio + padh, 0, h_max)
    return bbox

def xyxy2xywh(bboxes):
    bboxes[:, 2] = bboxes[:, 2] - bboxes[:, 0]
    bboxes[:, 3] = bboxes[:, 3] - bboxes[:, 1]
    return bboxes


text:
###############################################################################
#   SteadyLogSpiralPotential: a steady-state spiral potential
###############################################################################
import math
from galpy.util import bovy_conversion
from .planarPotential import planarPotential, _APY_LOADED
if _APY_LOADED:
    from astropy import units
_degtorad= math.pi/180.
class SteadyLogSpiralPotential(planarPotential):
    """Class that implements a steady-state spiral potential

    .. math::

        \\Phi(R,\\phi) = \\frac{\\mathrm{amp}\\times A}{\\alpha}\\,\\cos\\left(\\alpha\,\ln R - m\\,(\\phi-\\Omega_s\\,t-\\gamma)\\right)

    Can be grown in a similar way as the DehnenBarPotential, but using :math:`T_s = 2\pi/\Omega_s` to normalize :math:`t_{\mathrm{form}}` and :math:`T_{\mathrm{steady}}`.

    """
    def __init__(self,amp=1.,omegas=0.65,A=-0.035,
                 alpha=-7.,m=2,gamma=math.pi/4.,p=None,
                 tform=None,tsteady=None,ro=None,vo=None):
        """
        NAME:

           __init__

        PURPOSE:

           initialize a logarithmic spiral potential

        INPUT:

           amp - amplitude to be applied to the potential (default:
           1., A below)

           gamma - angle between sun-GC line and the line connecting the peak of the spiral pattern at the Solar radius (in rad; default=45 degree; or can be Quantity)

           A - amplitude (alpha*potential-amplitude; default=0.035; can be Quantity

           omegas= - pattern speed (default=0.65; can be Quantity)

           m= number of arms

           Either provide:

              a) alpha=

              b) p= pitch angle (rad; can be Quantity)

           tform - start of spiral growth / spiral period (default: -Infinity)

           tsteady - time from tform at which the spiral is fully grown / spiral period (default: 2 periods)

        OUTPUT:

           (none)

        HISTORY:

           2011-03-27 - Started - Bovy (NYU)

        """
        planarPotential.__init__(self,amp=amp,ro=ro,vo=vo)
        if _APY_LOADED and isinstance(gamma,units.Quantity):
            gamma= gamma.to(units.rad).value
        if _APY_LOADED and isinstance(p,units.Quantity):
            p= p.to(units.rad).value
        if _APY_LOADED and isinstance(A,units.Quantity):
            A= A.to(units.km**2/units.s**2).value/self._vo**2.
        if _APY_LOADED and isinstance(omegas,units.Quantity):
            omegas= omegas.to(units.km/units.s/units.kpc).value\
                /bovy_conversion.freq_in_kmskpc(self._vo,self._ro)
        self._omegas= omegas
        self._A= A
        self._m= m
        self._gamma= gamma
        if not p is None:
            self._alpha= self._m/math.tan(p)
        else:
            self._alpha= alpha
        self._ts= 2.*math.pi/self._omegas
        if not tform is None:
            self._tform= tform*self._ts
        else:
            self._tform= None
        if not tsteady is None:
            self._tsteady= self._tform+tsteady*self._ts
        else:
            if self._tform is None: self._tsteady= None
            else: self._tsteady= self._tform+2.*self._ts
        self.hasC= True

    def _evaluate(self,R,phi=0.,t=0.):
        """
        NAME:
           _evaluate
        PURPOSE:
           evaluate the potential at R,phi,t
        INPUT:
           R - Galactocentric cylindrical radius
           phi - azimuth
           t - time
        OUTPUT:
           Phi(R,phi,t)
        HISTORY:
           2011-03-27 - Started - Bovy (NYU)
        """
        if not self._tform is None:
            if t < self._tform:
                smooth= 0.
            elif t < self._tsteady:
                deltat= t-self._tform
                xi= 2.*deltat/(self._tsteady-self._tform)-1.
                smooth= (3./16.*xi**5.-5./8*xi**3.+15./16.*xi+.5)
            else: #spiral is fully on
                smooth= 1.
        else:
            smooth= 1.
        return smooth*self._A/self._alpha*math.cos(self._alpha*math.log(R)
                                                   -self._m*(phi-self._omegas*t
                                                             -self._gamma))

    def _Rforce(self,R,phi=0.,t=0.):
        """
        NAME:
           _Rforce
        PURPOSE:
           evaluate the radial force for this potential
        INPUT:
           R - Galactocentric cylindrical radius
           phi - azimuth
           t - time
        OUTPUT:
           the radial force
        HISTORY:
           2010-11-24 - Written - Bovy (NYU)
        """
        if not self._tform is None:
            if t < self._tform:
                smooth= 0.
            elif t < self._tsteady:
                deltat= t-self._tform
                xi= 2.*deltat/(self._tsteady-self._tform)-1.
                smooth= (3./16.*xi**5.-5./8*xi**3.+15./16.*xi+.5)
            else: #spiral is fully on
                smooth= 1.
        else:
            smooth= 1.
        return smooth*self._A/R*math.sin(self._alpha*math.log(R)
                                         -self._m*(phi-self._omegas*t
                                                   -self._gamma))

    def _phiforce(self,R,phi=0.,t=0.):
        """
        NAME:
           _phiforce
        PURPOSE:
           evaluate the azimuthal force for this potential
        INPUT:
           R - Galactocentric cylindrical radius
           phi - azimuth
           t - time
        OUTPUT:
           the azimuthal force
        HISTORY:
           2010-11-24 - Written - Bovy (NYU)
        """
        if not self._tform is None:
            if t < self._tform:
                smooth= 0.
            elif t < self._tsteady:
                deltat= t-self._tform
                xi= 2.*deltat/(self._tsteady-self._tform)-1.
                smooth= (3./16.*xi**5.-5./8*xi**3.+15./16.*xi+.5)
            else: #spiral is fully on
                smooth= 1.
        else:
            smooth= 1.
        return -smooth*self._A/self._alpha*self._m*math.sin(self._alpha*math.log(R)
                                                           -self._m*(phi
                                                                     -self._omegas*t
                                                                     -self._gamma))

    def wavenumber(self,R):
        """
        NAME:

           wavenumber

        PURPOSE:

           return the wavenumber at radius R (d f(R)/ d R in Phi_a(R) = F(R) e^[i f(R)]; see Binney & Tremaine 2008)

        INPUT:

           R - Cylindrical radius

        OUTPUT:

           wavenumber at R

        HISTORY:

           2014-08-23 - Written - Bovy (IAS)

        """
        return self._alpha/R

    def OmegaP(self):
        """
        NAME:

           OmegaP

        PURPOSE:

           return the pattern speed

        INPUT:

           (none)

        OUTPUT:

           pattern speed

        HISTORY:

           2011-10-10 - Written - Bovy (IAS)

        """
        return self._omegas

    def m(self):
        """
        NAME:

           m

        PURPOSE:

           return the number of arms

        INPUT:

           (none)

        OUTPUT:

           number of arms

        HISTORY:

           2014-08-23 - Written - Bovy (IAS)

        """
        return self._m

    def tform(self): #pragma: no cover
        """
        NAME:

           tform

        PURPOSE:

           return formation time of the bar

        INPUT:

           (none)

        OUTPUT:

           tform in normalized units

        HISTORY:

           2011-03-08 - Written - Bovy (NYU)

        """
        return self._tform

ext: py

max_stars_repo_name: turnergarrow/galpy

max_issues_repo_name: BurcuAkbulut/galpy

max_forks_repo_name: BurcuAkbulut/galpy

reason: from astropy


text:
from handyspark import *
import numpy.testing as npt
from pyspark.sql.types import IntegerType, StringType, ArrayType, MapType

def test_atomic_types():
    npt.assert_equal(IntegerType.ret('')[1], 'integer')
    npt.assert_equal(StringType.ret('')[1], 'string')

def test_composite_types():
    npt.assert_equal(ArrayType(IntegerType()).ret('')[1], 'array<int>')
    npt.assert_equal(MapType(StringType(), IntegerType()).ret('')[1], 'map<string,int>')

max_stars_repo_name: FoundryAI/handyspark

max_issues_repo_name: FoundryAI/handyspark

max_forks_repo_name: FoundryAI/handyspark

reason: import numpy


text:
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torch.utils.data import sampler

import torchvision.datasets as dset
import torchvision.transforms as T
import torch.nn.functional as F

import numpy as np

class ResNet(nn.Module):
    """
    Structure:
    conv - bn - 3x residualblocks - avgpool - fully connected
    """

    def __init__(self, n):
        super().__init__()
        self.n = n

        # layers
        self.initial_layer = nn.Conv2d(3, 16, kernel_size=3, padding=1)
        self.bn = nn.BatchNorm2d(16)
        self.layerBlock1 = ResidualBlocks(16, 16, n)
        self.layerBlock2 = ResidualBlocks(16, 32, n)
        self.layerBlock3 = ResidualBlocks(32, 64, n)
        self.avgpool = nn.AvgPool2d(kernel_size=8)
        self.fc = nn.Linear(64, 10)

        # initialization
        # TODO: Decide to collect initializations or in a modular fashion
        nn.init.kaiming_normal_(self.initial_layer.weight)
        nn.init.constant_(self.bn.weight, 1)
        nn.init.constant_(self.bn.bias, 0)
        nn.init.kaiming_normal_(self.fc.weight)
        nn.init.constant_(self.fc.bias, 0)

    def forward(self, x):
        initial_layer = self.initial_layer(x)
        initial_bn = self.bn(initial_layer)
        initial_relu = F.relu(initial_bn)
        resblocks_1 = self.layerBlock1(initial_relu)
        resblocks_2 = self.layerBlock2(resblocks_1)
        resblocks_3 = self.layerBlock3(resblocks_2)
        avg_pool = self.avgpool(resblocks_3)
        flat = flatten(avg_pool)
        scores = self.fc(flat)
        return scores

class ConvBlock(nn.Module):
    """
    Single Residual Block.

    Structure:
    conv - bn - relu - conv - bn - identity - relu
    """

    def __init__(self, input_channels, output_channels):
        super().__init__()
        self.sub_stride = 1
        # TODO: simplify/clarify construction

        # layers for subsampling
        if input_channels != output_channels:
            self.sub_stride = 2
            self.max = nn.MaxPool2d(1, stride=self.sub_stride)
        self.conv1 = nn.Conv2d(input_channels, output_channels, kernel_size=3, stride=self.sub_stride, padding=1)
        self.bn1 = nn.BatchNorm2d(output_channels)
        self.conv2 = nn.Conv2d(output_channels, output_channels, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(output_channels)

        # initialization
        nn.init.kaiming_normal_(self.conv1.weight, a=0, mode='fan_in', nonlinearity='relu')
        nn.init.kaiming_normal_(self.conv2.weight, a=0, mode='fan_in', nonlinearity='relu')
        nn.init.constant_(self.bn1.weight, 1)
        nn.init.constant_(self.bn1.bias, 0)
        nn.init.constant_(self.bn2.weight, 1)
        nn.init.constant_(self.bn2.bias, 0)

    def forward(self, x):
        conv1 = self.conv1(x)
        bn1 = self.bn1(conv1)
        relu1 = F.relu(bn1)
        residual = self.conv2(relu1)
        bn2 = self.bn2(residual)

        # resize x by zero padding and downsampling via maxpoling to match dimensions of feature maps
        if bn2.shape[1] != x.shape[1]:
            pad = int((bn2.shape[1] - x.shape[1])/2)
            x = F.pad(x, (0, 0, 0, 0, pad, pad, 0, 0))
            x = self.max(x)

        out = bn2 + x
        out = F.relu(out)
        return out

class ResidualBlocks(nn.Module):
    """
    Sequence of n Conv Blocks of same structure.

    """
    def __init__(self, input_channels, output_channels, n):
        super().__init__()
        #self.n = n
        blocks = [ConvBlock(input_channels, output_channels)]
        for _ in range(n-1):
            blocks.append(ConvBlock(output_channels, output_channels))
        self.blocks = nn.Sequential(*blocks)

    def forward(self, x):
        out = self.blocks(x)
        return out

def flatten(x):
    """
    Strech out feature map into vector for Linear Layers.

    Inputs:
    - x: (torch.tensor) feature map to be streched out

    Returns:
    - x: (torch.tensor) one dimensional tensor
    """
    N = x.shape[0]
    return x.view(N, -1)

max_stars_repo_name: kparch/ResNet

max_issues_repo_name: kparch/ResNet

max_forks_repo_name: kparch/ResNet


text:
# -*- coding: utf-8 -*-
"""
Created: 2020-11-02
Updated: 2021-02-11
Python: 3.7
@author: EddPineda
Version 1.0.3

The attributes and their purpose for the class are:
    casePara - the parameters dictionary for the "select case/switch" method - needs to
               stay in alignment case
    case     - the different data types dictionary and the method to use to process the data
    kc       - the indexes of the dataframe columns to be used as a list
    kct      - the column data type for the used data frame columns as a list
    featStrt - the column in the dataframe where the features start
    spell    - the spell checker

The methods for the class are:
    na              - do nothing with the data (needed for case/switch logic and takes the following parameters:
                      dta    - dataframe column data as a list or series
    removeColumns   - removes the unused columns in the dataframe and takes the following parameters:
                      df     - the dataframe
    cleanColumns    - cleans the column names (strips white space & changes to title format) and takes the following parameters:
                      df     - the dataframe
                      s      - optional (required if r != '') - the search string to search for a specific character/s to replace
                      r      - optional - the replacement for s
    cleanData       - strips the white space from the dataframe data and takes the following parameters:
                      df     - the dataframe
    dateChecker     - the "parent" method for dateFinder that handles date and time data it takes the following parameters:
                      dta    - dataframe column data as a list or series
                      f      - the format for the given date/time data ex: '%Y-%m-%d %H:%M:%S.%f'
    dateFinder      - converts given date time value to the desired format and takes the following parameters:
                      s      - date/time value as a string
                      f      - format for the date/time
    currencyCleaner - converts to a float regardless if US or International and takes the following parameters:
                      dta    - dataframe column data as a list or series
    spellChecker    - spell checks and updates text data and takes the following parameters:
                      dta    - dataframe column data as a list or series
    numberChecker   - the parent method for floatChecker and intChecker and takes the following parameters:
                      dta    - dataframe column data as a list or series
    floatChecker    - checks the given value presented as a string to see if it's a float and takes the following parameters:
                      s      - value as string
    intChecker      - checks the given value presented as a string to see if it's an integer and takes the following parameters:
                      s      - value as string
    dfDataClean     - the select case/switch method directs data to the correct data cleaner it takes the following parameters:
                      fun    - the name of the method to be used as string
                      para   - the parameters to be used for the given method
    cleanDFcols     - assesses the dataframe columns and validates the data - it takes the following parameters:
                      df     - the dataframe
    dataProcess     - performs the task to prep the dataframe data for visualization and takes the following parameters:
                      df     - the dataframe
"""
import numpy as np
#from autocorrect import Speller
from spellchecker import SpellChecker
from dateutil.parser import parse
from money_parser import price_str
import logging
import pandas as pd

dpca = {'casePara' : {'text' : {'dta': []},
                      'currency' : {'dta': []},
                      'date' : {'dta': [], 'f': '%Y-%m-%d'},
                      'time' : {'dta': [], 'f': '%H:%M:%S'},
                      'datetime' : {'dta': [], 'f': '%Y-%m-%d %H:%M:%S'},
                      'datetimestamp' : {'dta': [], 'f': '%Y-%m-%d %H:%M:%S.%f'},
                      'name' : {'dta': []},
                      'float' : {'dta': [], 'f': ',', 'typ': 'float'},
                      'intlfloat' : {'dta': [], 'f': '.', 'typ': 'float'},
                      'int' : {'dta': [], 'f': ',', 'typ': 'int'},
                      'intlint' : {'dta': [], 'f': '.', 'typ': 'int'},
                      'category' : {'dta': []},
                      'indcategory' : {'dta' : []},
                      'nestcategory' : {'dta' : []},
                      'subcategory' : {'dta' : []},
                      'adjcategory' : {'dta' : []}
                      }
        }

class DataProcess:
    def __init__(self,dataframe):
        self.dataframe = dataframe
        self.cleaned_dataframe = self.clean_normalise_data()
        self.data_types = self.get_datatypes_dict()
        self.data_types_strings = self.get_datatypes_strings()

    def clean_normalise_data(self):
        logging.info("clean and normalise data")
        cleaned_dataframe = self.dataframe
        return cleaned_dataframe

    def get_datatypes_dict(self):
        logging.info("get data types dict")
        dataTypes = dict(self.dataframe.dtypes)
        return dataTypes

    def get_datatypes_strings(self):
        logging.info("get data types string")

        data_types = []
        for col in self.dataframe.columns:
            data_type = self.dataframe.dtypes[col]
            data_types.append(data_type)

        return data_types

class DataDefinition:
    def __init__(self,data_location,selected_rows):
        self.data_location = data_location
        self.selected_rows = selected_rows

    def get_pandas_dataframe(self):
        logging.info("get pandas dataframe")
        dataframe = pd.read_csv(self.data_location)#,usecols=self.selected_rows)
        return dataframe

class DataPreProcessing:
    def __init__(self, cv):
        self.__dict__.update(**cv)
        self.__dict__.update(**dpca)
        self.case = {'text' : self.spellChecker,
                     'currency' : self.currencyCleaner,
                     'date' : self.dateChecker,
                     'time' : self.dateChecker,
                     'datetime' : self.dateChecker,
                     'datetimestamp' : self.dateChecker,
                     'name' : self.na,
                     'float' : self.numberChecker,
                     'intlfloat' : self.numberChecker,
                     'int' : self.numberChecker,
                     'intlint' : self.numberChecker,
                     'category' : self.na,
                     'indcategory' : self.na,
                     'nestcategory' : self.na,
                     'subcategory' : self.na,
                     'adjcategory' : self.na
                     }
        #self.spell = Speller(lang='en')
        self.spell = SpellChecker()
        print('data preprocessing class initiated')

    def __del__(self):
        print('data preprocessing class destroyed')

    def na(self, dta):
        return dta

    def removeColumns(self, df):
        df.drop(df.columns[[i for i in list(range(0, len(df.columns))) if i not in self.kc]], axis = 1, inplace = True)
        return df

    def cleanColumns(self, df, s = '', r = ''):
        if (r == ''):
            df.columns = [''.join(e for e in c.title() if e.isalnum()) for c in df.columns]
        else:
            df.columns = [c.title().replace(s, r) for c in df.columns.str.strip()]

        return df

    def cleanData(self, df):
        return df.apply(lambda x: x.str.strip() if x.dtype == 'object' else x)

    def dateChecker(self, dta, f):
        return [self.dateFinder(s, f) for s in dta]

    def dateFinder(self, s, f):
        try:
            return parse(s, fuzzy_with_tokens = False).strftime(f)
        except:
            return None

    def currencyCleaner(self, dta):
        return [price_str(e, default=None) for e in dta]

    def spellChecker(self, dta):
        return [self.spell.correction(e) if e != None else None for e in dta]

    def numberChecker(self, dta, f, typ):
        return [self.floatChecker(s) for s in dta.str.replace(f, '')] if typ == 'float' else [self.intChecker(s) for s in dta.str.replace(f, '')]

    def floatChecker(self, s):
        try:
            return float(s) if (~np.isnan(float(s)) and ~np.isinf(float(s))) else None
        except:
            return None

    def intChecker(self, s):
        try:
            return int(float(s))
        except:
            return None

    def dfDataClean(self, fun, para):
        func = self.case.get(fun, lambda: 'Invalid Type')

        return func(**para)

    def cleanDFcols(self, df):
        for i, c in enumerate(df.columns):
            td = self.casePara[self.kct[i]].copy()
            td['dta'] = df[c].astype('str')
            df[c] = self.dfDataClean(self.kct[i], td)

        return df

    def dataProcess(self, df):
        return self.cleanDFcols(self.cleanData(self.cleanColumns(self.removeColumns(df) if self.kc != list(range(0, len(df.columns))) else df)))

max_stars_repo_name: andrewpenrose/data-viz-python-sdk

max_issues_repo_name: andrewpenrose/data-viz-python-sdk

max_forks_repo_name: andrewpenrose/data-viz-python-sdk


text:
#!/usr/bin/env python

import os,sys
import numpy as np
from sklearn.mixture import GaussianMixture
from astropy.table import Table
from astropy.coordinates import SkyCoord
from astropy import units as u

def make_coreqso_table(dr14qso,ebosstarg):
    if isinstance(dr14qso,str):
        dr14qso = Table.read(dr14qso)
    if isinstance(ebosstarg,str):
        ebosstarg = Table.read(ebosstarg)
    #
    dr14coo = SkyCoord(dr14qso['RA'],dr14qso['DEC'],unit=u.deg)
    # restrict to CORE quasar targets
    ii = np.where(ebosstarg['EBOSS_TARGET1'] & (1<<10) > 0)[0]
    ebosstarg = ebosstarg[ii]
    ebosstargcoo = SkyCoord(ebosstarg['RA'],ebosstarg['DEC'],unit=u.deg)
    # now identify confirmed quasars from DR14 in the target list
    m1,m2,sep,_ = dr14coo.search_around_sky(ebosstargcoo,2*u.arcsec)
    # for some reason there is a repeated entry...
    _,ii = np.unique(m1,return_index=True)
    dr14qso = dr14qso[m2[ii]]
    # just a sanity check
    jj = np.where(dr14qso['EXTINCTION']>0)[0]
    assert np.allclose(dr14qso['EXTINCTION'][jj],
                       ebosstarg['EXTINCTION'][m1[ii[jj]]],atol=1e-3)
    # extract all the WISE columns from targeting
    wisecols = ['W1_MAG','W1_MAG_ERR',
                'W1_NANOMAGGIES','W1_NANOMAGGIES_IVAR',
                'W2_NANOMAGGIES','W2_NANOMAGGIES_IVAR',
                'HAS_WISE_PHOT']
    # overwriting the DR14Q flux fields because they have invalid entries
    for k in wisecols + ['EXTINCTION','PSFFLUX','PSFFLUX_IVAR']:
        dr14qso[k] = ebosstarg[k][m1[ii]]
    dr14qso.write('ebosscore_dr14q.fits',overwrite=True)

def get_column_ratio(a,j):
    col = a[:,[j]].copy()
    a /= col
    a = np.ma.hstack([a[:,:j],a[:,j+1:]])
    return col,a

class eBossQsos(object):
    def __init__(self,fileName='ebosscore_dr14q.fits',zrange=(0.9,4.0)):
        dat = Table.read(fileName)
        if zrange is not None:
            dat = dat[np.logical_and(dat['Z']>zrange[0],dat['Z']<zrange[1])]
        self.data = dat
        self.set_specz_col('Z')
        self._extract_fluxes()
    def set_specz_col(self,colname):
        self.speczName = colname
        self.specz = self.data[colname]
    def _extract_fluxes(self,ratios='byref',refNum=3):
        # SDSS
        sdssFluxes = np.array(self.data['PSFFLUX'])
        extCorr = np.array(self.data['EXTINCTION'])
        self.sdssFluxes = np.ma.array(sdssFluxes*10**(0.4*extCorr))
        # XXX need the galex ext values
        # Galex
        fuv = np.ma.array(self.data['FUV'],mask=self.data['FUV_IVAR']==0)
        nuv = np.ma.array(self.data['NUV'],mask=self.data['NUV_IVAR']==0)
        self.galexFluxes = np.ma.vstack([fuv,nuv]).transpose()
        # UKIDSS
        self.ukidssFluxes = np.ma.vstack(
                 [ np.ma.array(self.data[b+'FLUX'],
                               mask=self.data['UKIDSS_MATCHED']==0)
                      for b in 'YJHK' ] ).transpose()
        # units are W/m^2/Hz, 1e26 converts to Jy and 5.44 to nanomaggie
        self.ukidssFluxes *= 1e26*10**5.44
        # XXX the fluxes seem to be consistently offset when compared to
        #     catalog Vega magnitudes... why?
        self.ukidssFluxes *= 1.07
        # WISE (convert from Vega)
        # using extinction conversions from ebosstarget_qso_selection.pro
        w1 = np.ma.array(self.data['W1_NANOMAGGIES'],
                         mask=self.data['HAS_WISE_PHOT']!='T')
        w1 *= 10**(0.4*(0.184*extCorr[:,2]/2.285))
        w2 = np.ma.array(self.data['W2_NANOMAGGIES'],
                         mask=self.data['HAS_WISE_PHOT']!='T')
        w2 *= 10**(0.4*(0.113*extCorr[:,2]/2.285))
        self.wiseVegaFluxes = np.ma.vstack([w1,w2]).transpose()
        w1 *= 10**(-0.4*2.699)
        w2 *= 10**(-0.4*3.339)
        self.wiseFluxes = np.ma.vstack([w1,w2]).T
    def get_optwise(self):
        f_WISE = np.average(self.wiseVegaFluxes,axis=1,weights=[1,0.5])
        f_opt = np.average(self.sdssFluxes[:,1:4],axis=1,weights=[1,0.8,0.6])
        return f_opt,f_WISE
    def extract_features(self,featureset=['sdss','z'],
                         refband='i',ratios='byref'):
        fluxes = []
        names = []
        if 'galex' in featureset:
            fluxes.append(self.galexFluxes)
            names.extend(['f','n'])
        if 'sdss' in featureset:
            fluxes.append(self.sdssFluxes)
            names.extend(list('ugriz'))
        if 'ukidss' in featureset:
            fluxes.append(self.ukidssFluxes)
            names.extend(list('YJHK'))
        if 'wise' in featureset:
            fluxes.append(self.wiseFluxes)
            names.extend(['W1','W2'])
        #
        fluxes = np.ma.hstack(fluxes)
        j = names.index(refband)
        if ratios=='byref':
            refFlux,features = get_column_ratio(fluxes,j)
            names = [ b+'/'+refband for b in names
                                      if b is not refband ]
        elif ratios=='neighboring':
            refFlux = fluxes[:,[j]]
            features = fluxes[:,:-1]/fluxes[:,1:]
            names = [ b1+'/'+b2 for b1,b2 in zip(names[:-1],names[1:]) ]
        else:
            raise ValueError
        if 'z' in featureset:
            z = np.ma.array(self.specz)[:,None]
            z = np.ma.log10(1+z)
            features = np.ma.hstack([z,features])
            names = ['logz_q'] + names
        return features,names,refFlux

def prep_simqsos(simqsos,refband='i',sdssonly=False):
    j = 3 # XXX
    ii = np.where(simqsos['selected'])[0]
    fluxes = np.array(simqsos['obsFlux'][ii])
    if sdssonly:
        fluxes = fluxes[:,:5]
    print('WARNING: HACK to get around bad forest transmission vals')
    print(fluxes.max())
    jj = np.where(np.all(fluxes < 1e5,axis=1))[0]
    ii = ii[jj]
    fluxes = fluxes[jj]
    print(fluxes.max())
    refFlux,fratios = get_column_ratio(fluxes,j)
    logz = np.log10(1 + simqsos['z'][ii,None])
    X = np.ma.hstack([logz,fratios])
    return X,refFlux

def fit_mixtures(X,mag,mbins,binwidth=0.2,seed=None,
                 keepscore=False,keepbic=False,**kwargs):
    kwargs.setdefault('n_components',25)
    kwargs.setdefault('covariance_type','full')
    fits = []
    if keepscore:
        scores = []
    if keepbic:
        bics = []
    if seed:
        np.random.seed(seed)
    for bincenter in mbins:
        # this is not an efficient way to assign bins, but the time
        # is negligible compared to the GMM fitting anyway
        ii = np.where( np.abs(mag-bincenter) < binwidth )[0]
        if False:
            print('{:.2f}: {} qsos'.format(bincenter,len(ii)))
        gmm = GaussianMixture(**kwargs)
        gmm.fit(X[ii])
        fits.append(gmm)
        if keepscore:
            scores.append(gmm.score(X[ii]))
        if keepbic:
            bics.append(gmm.bic(X[ii]))
    rv = (fits,)
    if keepscore:
        rv += (scores,)
    if keepbic:
        rv += (bics,)
    return rv

def simqso_density_estimation(simqsos,refband='i',mbins=None,seed=None,
                              sdssonly=False):
    if mbins is None:
        mbins = np.arange(17.7,22.51,0.1)
    X,refFlux = prep_simqsos(simqsos,refband,sdssonly=sdssonly)
    mag = 22.5 - 2.5*np.log10(refFlux.clip(1e-10,np.inf))
    fits, = fit_mixtures(X,mag,mbins,seed=seed)
    return fits

def model_selection(simqsos,refband='i',mbin=20.):
    X,refFlux = prep_simqsos(simqsos,refband)
    mag = 22.5 - 2.5*np.log10(refFlux.clip(1e-10,np.inf))
    cv_types = ['full'] #['spherical', 'tied', 'diag', 'full']
    n_components = np.arange(5,51,5)
    bics = []
    for cv_type in cv_types:
        for ncomp in n_components:
            fits,bic = fit_mixtures(X,mag,np.array([mbin]),
                                    keepbic=True,
                                    n_components=ncomp,
                                    covariance_type=cv_type)
            bics.append(bic[0])
            print(cv_type,ncomp,bic[0])
    return np.array(bics)

def plot_model_selection(simqsos):
    import matplotlib.pyplot as plt
    n_components = np.arange(5,51,5)
    plt.figure()
    for mbin in [19.,20.,21.]:
        bics = model_selection(simqsos,mbin=mbin)
        plt.plot(n_components,bics-bics.min(),label=str(mbin))
    plt.legend()

def fit_ebossqsos(simqsos,qsos=None,seed=None,navg=1,
                  sdssonly=False,verbose=0):
    if isinstance(simqsos,str):
        simqsos = Table.read(simqsos)
    if qsos is None:
        qsos = eBossQsos()
    if sdssonly:
        featureset = ['sdss','z']
    else:
        featureset = ['sdss','ukidss','wise','z']
    features,names,refFlux = qsos.extract_features(featureset=featureset)
    mags = 22.5 - 2.5*np.log10(refFlux.clip(1e-10,np.inf))
    mbins = np.arange(17.7,22.51,0.1)
    binNums = np.digitize(mags,mbins-0.1/2)
    print('fitting features ',','.join(names))
    print('in {} magnitude bins'.format(len(mbins)))
    if verbose:
        score_zedgs = np.linspace(np.log10(1.9),np.log10(5.0),10)
        score_zvals = np.zeros((len(score_zedgs),2))
    allscore = []
    for iterNum in range(navg):
        fits = simqso_density_estimation(simqsos,seed=seed,sdssonly=sdssonly)
        n,score = 0,0
        for i,fit in enumerate(fits):
            ii = np.where(binNums==i)[0]
            if len(ii) > 0:
                s = fit.score_samples(features[ii])
                score += s.sum()
                n += len(ii)
                if verbose:
                    zi = np.digitize(features[ii,0],score_zedgs) - 1
                    assert np.all( (zi>=0) & (zi<len(score_zedgs)) )
                    np.add.at( score_zvals, (zi,0), s )
                    np.add.at( score_zvals, (zi,1), 1 )
        score /= n
        print('  iter {} score {:.3f}'.format(iterNum+1,score))
        allscore.append(score)
        # don't reuse the seed!
        seed = None
    if verbose:
        avzscore = score_zvals[:,0] / score_zvals[:,1]
        print('#{:3} {:5} {:5}'.format('z','score','n'))
        score_zbins = 10**score_zedgs-1
        score_zbins = score_zbins[:-1] + np.diff(score_zbins)/2
        for i,z in enumerate(score_zbins):
            print('{:4.2f} {:5.2f} {:5d}'.format(z,
                                         avzscore[i],int(score_zvals[i,1])))
    if navg > 1:
        allscore = np.array(allscore)
        print('final avg {:.3f} with rms {:.3f}'.format(
               allscore.mean(),allscore.std()))

if __name__=='__main__':
    import argparse
    parser = argparse.ArgumentParser(
                              description='fit simulated quasars to eboss')
    parser.add_argument('fitsfile',nargs='+',type=str,
        help='simulation output file name(s)')
    parser.add_argument('-n','--navg',type=int,default=1,
        help='number of GMM fits to average')
    parser.add_argument('-s','--seed',type=int,default=12345,
        help='random seed')
    parser.add_argument('--sdss',action='store_true',
        help='only use SDSS fluxes in fit')
    parser.add_argument('-v','--verbose',action='count',
        help='increase output verbosity')
    args = parser.parse_args()
    #make_coreqso_table(sys.argv[1],sys.argv[2])
    coreqsos = eBossQsos()
    for ff in args.fitsfile:
        print(ff)
        fit_ebossqsos(ff,coreqsos,args.seed,args.navg,
                      sdssonly=args.sdss,verbose=args.verbose)
        print()

max_stars_repo_name: desihub/simqso

max_issues_repo_name: desihub/simqso

max_forks_repo_name: desihub/simqso

reason: import numpy,from astropy


text:
import utils
import numpy as np
from skimage.metrics import structural_similarity as compare_ssim
from skimage.metrics import peak_signal_noise_ratio as compare_psnr

def calc_metric(hr, sr):
  psnr = []
  ssim = []
  for i in range(len(hr)):
    hr_, sr_ = hr[i,0], sr[i,0]
    psnr.append(compare_psnr(hr_, sr_))
    ssim.append(compare_ssim(hr_, sr_))

  psnr = np.array(psnr)
  ssim = np.array(ssim)

  return psnr.mean(), ssim.mean()
# --------------------------------------------------------------------------------------


text:
"""
Script goal,

Test out the google earth engine to see what i can do
	- find a landsat collection for a single point

"""
#==============================================================================

__title__ = "GEE Movie Fixer"
__author__ = "Arden Burrell"
__version__ = "v1.0(04.04.2019)"
__email__ = "arden.burrell@gmail.com"

#==============================================================================
# +++++ Check the paths and set ex path to fireflies folder +++++
import os
import sys
if not os.getcwd().endswith("fireflies"):
	if "fireflies" in os.getcwd():
		p1, p2, _ =  os.getcwd().partition("fireflies")
		os.chdir(p1+p2)
	else:
		raise OSError(
			"This script was called from an unknown path. CWD can not be set"
			)
sys.path.append(os.getcwd())

#==============================================================================
# Import packages
import numpy as np
import pandas as pd
import geopandas as gpd
import argparse
import datetime as dt
import warnings as warn
import xarray as xr
import bottleneck as bn
import scipy as sp
import glob

from collections import OrderedDict
from scipy import stats
from numba import jit

# Import the Earth Engine Python Package
import ee
import ee.mapclient
from ee import batch

# from netCDF4 import Dataset, num2date, date2num
# from scipy import stats
# import statsmodels.stats.multitest as smsM

# Import plotting and colorpackages
import matplotlib.pyplot as plt
import matplotlib.colors as mpc
import matplotlib as mpl
import palettable

# import fiona
# fiona.drvsupport.supported_drivers['kml'] = 'rw' # enable KML support which is disabled by default
# fiona.drvsupport.supported_drivers['KML'] = 'rw' # enable KML support which is disabled by default

import moviepy.editor as mpe
import skvideo.io     as skv
import skimage as ski
from moviepy.video.io.bindings import mplfig_to_npimage

# import seaborn as sns
import matplotlib as mpl
import cartopy.crs as ccrs
import cartopy.feature as cpf
import matplotlib.ticker as mticker
from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
from mpl_toolkits.axes_grid1 import make_axes_locatable
import socket

# import cartopy.feature as cpf
# from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER

# # Import debugging packages
import ipdb

print("numpy version  : ", np.__version__)
print("pandas version : ", pd.__version__)
print("xarray version : ", xr.__version__)

#==============================================================================

def main():
	# print(socket.gethostname())
	# fn     = "/home/ubuntu/Downloads/LANDSAT_5_7_8_TestBurn_RGB.mp4"
	# fng    = "/home/ubuntu/Downloads/LANDSAT_5_7_8_TestBurn_RGB_grid.tif"
	# da     = xr.open_rasterio(fng)
	# fn = "/home/ubuntu/Downloads/Site4_video_region_L8_time_v8_SR.mp4"

	# ========== setup the filename ==========
	# site = "TestBurn"
	site = "G5T1-50"
	# site="G10T1-50"
	test = False

	if site == "TestBurn":
		fnames = sorted(glob.glob("/home/ubuntu/Downloads/TestSite/*.tif"))
	elif site == "G10T1-50":
		fnames = sorted(glob.glob("/home/ubuntu/Downloads/Test2/*.tif"))
	elif site == "G5T1-50":
		fnames = sorted(glob.glob("/mnt/c/Users/user/Google Drive/FIREFLIES_geotifs_G5T1-50/*.tif"))
		# fnames = sorted(glob.glob("/home/ubuntu/Downloads/Test3/*.tif"))
	else:
		fnames = sorted(glob.glob("/home/ubuntu/Downloads/FIREFLIES_geotifs/*.tif"))
		# fnames = sorted(glob.glob("/mnt/c/Users/user/Google Drive/FIREFLIES_geotifs/*.tif"))
	SF     = 0.0001 # Scale Factor
	# ========== load the additional indomation ==========
	try:
		dft = pd.read_csv("./data/other/tmp/LANDSAT_5_7_8_%s_RGB_timeinfo.csv" % site, index_col=0, parse_dates=True)
		dfg = pd.read_csv("./data/other/tmp/LANDSAT_5_7_8_%s_RGB_gridinfo.csv" % site, index_col=0, parse_dates=True)

	except:
		dft = pd.read_csv("./data/other/tmp/LANDSAT_5_7_8_%s_NRGB_timeinfo.csv" % site, index_col=0, parse_dates=True)
		dfg = pd.read_csv("./data/other/tmp/LANDSAT_5_7_8_%s_NRGB_gridinfo.csv" % site, index_col=0, parse_dates=True)

	# fnn    = fnames[-1]

	# ========== Work out how many bands ==========
	bandcombo = OrderedDict()
	da_ts  = xr.open_rasterio(fnames[0]).transpose("y", "x", "band").rename({"x":"longitude", "y":"latitude"})
	if da_ts.shape[2] == 3:
		bandcombo["RGB"] = [1, 2, 3]
		bandlist = None
	elif da_ts.shape[2] == 4:
		bandcombo["NRG"] = [1, 2, 3]
		bandcombo["RGB"] = [2, 3, 4]
		bandlist = None
	elif da_ts.shape[2] == 7:
		# bandlist = ['B', 'G', 'R', 'NIR', 'SWIR1', 'SWIR2', 'pixel_qa']
		bandlist = ['pixel_qa', 'SWIR2','SWIR1', 'NIR', 'R', 'G', 'B']

		bandcombo["SNR"] = ['SWIR1', 'NIR', 'R']
		bandcombo["NRG"] = ['NIR'  , 'R'  , 'G']
		bandcombo["RGB"] = [  'R'  , 'G'  , 'B']

	else:
		warn.warn("unknown band structure")
		ipdb.set_trace()

	# ========== Loop over the band combinations ==========
	for bnd in bandcombo:
		maxvals = []
		raw  = []
		orig = []
		modi = []
		for fnn, date in zip(fnames, pd.to_datetime(dft.date)):

			# =========== Open the file and scale it ==========
			da_in  = xr.open_rasterio(fnn).transpose("y", "x", "band").rename({"x":"longitude", "y":"latitude"}) * SF
			if not bandlist is None:
				da_in["band"] = bandlist

			da_in  = da_in.sel(band=bandcombo[bnd])
			raw.append(da_in)

			# =========== build a date check ==========
			ymd = fnn.split("RGB_")[-1][:8]
			if not ( ymd  == date.strftime(format="%Y%m%d")):
				print("date is missing")
				warn.warn("date is missing")
				ipdb.set_trace()

			# =========== mask out dodgy values ==========
			# da_in = da_in.where(da_in <= 1.1)
			da_in  =  da_in.where(da_in <= 1)
			da_in  =  da_in.where(da_in >= 0)
			noNaN  = ~da_in.reduce(bn.anynan, dim='band')
			da_in  =  da_in.where(noNaN)
			noNaF =  noNaN.values.astype(float)
			noNaF[noNaF ==0] = np.NAN

			# =========== Add infomation so i can look it in a dataframe ==========
			# maxv = da_in.groupby("band").max().values.astype(float)
			mean = (da_in.groupby("band").mean().values).astype(float)
			gfrac = float((noNaN.sum().values)/noNaN.size)

			# =========== Pull out the values ==========
			img = da_in.values

			# =========== Convert them to a 1d array ==========
			try:
				first_test= False
				if first_test:
					# ========== Create a single greyscale image ==========
					img_gry = ski.color.rgb2gray(img)

					# ========== build a hsv version of the image ==========
					img_hsv = ski.color.rgb2hsv(img)

					img_bri = img_hsv[:, :, 2]

					pL, pH  = np.percentile(img[~np.isnan(img_bri)], (2, 98))
					# img_gam = ski.exposure.adjust_gamma(img, gamma_list[nx])
					img_ri  = ski.exposure.rescale_intensity(img_bri, in_range=(pL, pH))
					img_ae  = ski.exposure.equalize_adapthist(img_bri, clip_limit=0.03)
					img_eh  = ski.exposure.equalize_hist(img_bri)

					bright  = [img_bri, img_ri, img_ae, img_eh]
					bri_nm  = ["orig", "rescale_intensity", "equalize_adapthist", "equalize_hist"]
					for img_x, bri in zip(bright, bri_nm):
						print("The %s mean:" % bri, np.mean(img_x))
						print("The %s median:" % bri, np.median(img_x))

						plt.figure(1)
						plt.imshow(img_x * noNaF, vmin = 0, vmax=1)
						plt.colorbar()

						# ========== create the new rgb image ==========
						imgo = img_hsv.copy()
						imgo[:,:, 2] = img_x * noNaF
						imgo[:,:, 2] = imgo[:,:, 2] * 2
						imgo_hsv = ski.color.hsv2rgb(imgo)

						plt.figure(2)
						plt.imshow(imgo_hsv)

						plt.figure(3)
						plt.imshow(img)

						pL, pH = np.percentile(img[~np.isnan(img)], (1, 99))
						imgo_s = ski.exposure.rescale_intensity(img, in_range=(pL, pH))
						plt.figure(4)
						plt.imshow(imgo_s)
						plt.show()

					continue
					# img_gama = img.copy()
										# # gamma_list = [0.95, 1.1, 1]
					# # gamma_list = [0.95, 0.90, 1.1]
					# gamma_list = [0.90, 0.90, 0.95]
					# for nx in range(0, 3):
					# plt.figure(1)
					# plt.imshow(img_gama)

					# imgo = ski.exposure.adjust_sigmoid(img_gama, cutoff=0.15) # bn.nanmedian(img)
					# # plt.figure(2)
					# # plt.imshow(imgo2)
				else:
					hist = ski.exposure.histogram(img[~np.isnan(img)])
					pL, pH = np.percentile(img[~np.isnan(img)], (1, 99))
					imgo = ski.exposure.rescale_intensity(img, in_range=(pL, pH))
					# plt.figure(3)
					# plt.imshow(imgo)

					# plt.show()
					# ipdb.set_trace()

				histo = ski.exposure.histogram(imgo[~np.isnan(imgo)])

				# plt.figure(2)
				# plt.plot(histo[1], histo[0])

				# plt.show()

				imgA    = ski.color.gray2rgb(imgo)
				da_out  =  da_in.copy(data=imgA)
				meanMod = (da_out.groupby("band").mean().values).astype(float)
			except IndexError:
				meanMod = np.array([np.NAN, np.NAN, np.NAN])
				da_out  = da_in.copy()

			orig.append(da_in)

			# ========== convert to smart xarray dataset ==========
			da_out = da_out.expand_dims("time")
			da_out = da_out.assign_coords(time=[date])
			modi.append(da_out)

			# ========== Add the metadata ==========
			maxvals.append(np.hstack([mean, meanMod, gfrac]))

		# ========== Create a single dataarray for the raster images ===========
		da_mod = xr.concat(modi, dim="time")

		# ========== Add infomation to the dataframe ==========
		array = np.array(maxvals)
		keys = ["mean_R", "mean_G", "mean_B","mean_mR", "mean_mG", "mean_mB", "GoodFrac"]
		for nx in range(0, 7):	dft[keys[nx]] = array[:, nx]
		dft[ "Bright"] = 0.2125*dft.mean_R  + 0.7154*dft.mean_G  + 0.0721*dft.mean_B
		dft["BrightM"] = 0.2125*dft.mean_mR + 0.7154*dft.mean_mG + 0.0721*dft.mean_mB

		# ipdb.set_trace()

		if test:
			for ind in [-3, 0, dft.Bright.idxmax(), dft.Bright.idxmin(), -1]:

				plt.figure(0)
				raw[ind].plot.imshow(rgb="band")

				plt.figure(1)
				orig[ind].plot.imshow(rgb="band")

				plt.figure(2)
				modi[ind][0, :, :, :].plot.imshow(rgb="band")

				plt.show()
			ipdb.set_trace()

		# =========== Setup the annimation ===========
		fig, ax = plt.subplots(1, figsize=(11,10))#, dpi=400)#, subplot_kw={'projection': ccrs.PlateCarree()} )#, , dpi=400, )

		# ========== Loop over each frame of the video ==========
		nx = []

		def frame_maker(index):
			# ========== Pull the infomation from the pandas part of the loop ==========
			info  = dft.iloc[int(index)] #rowinfo[1]
			frame = da_mod.isel(time=int(index))
			# frame = videoin[int(index), :, :, :]

			# ========== Check the dates i'm exporting ==========
			nx.append(frame.time.values)

			# ========== setup the plot ==========
			ax.clear()
			frame.plot.imshow(ax=ax, rgb="band")#, transform=ccrs.PlateCarree())
			ax.set_title("%s %s" % (info.satellite, info.date.split(" ")[0]))
			# ax.imshow(frame, extent=bounds, transform=ccrs.PlateCarree())
			ax.scatter(dfg.lon[0], dfg.lat[0], 5, c='r', marker='+')#, transform=ccrs.PlateCarree())
			# ax.scatter(dfg.lon[0], dfg.lat[0], 5, c='r', marker='+')
			rect = mpl.patches.Rectangle(
				(dfg.lonb_min[0],dfg.latb_min[0]),
				dfg.lonb_max[0]-dfg.lonb_min[0],
				dfg.lonb_max[0]-dfg.lonb_min[0],linewidth=1,edgecolor='r',facecolor='none')
			ax.add_patch(rect)

			plt.axis('scaled')
			return mplfig_to_npimage(fig)

		# for frame, rowinfo in zip(videoin, dft.iterrows()):
		mov = mpe.VideoClip(frame_maker, duration=int(da_mod.shape[0]))

		fnout = "./results/movies/LANDSAT_5_7_8_%s_%s_updatedV3.mp4" % (site, bnd)

		print("Starting Write of the data at:", pd.Timestamp.now())
		mov.write_videofile(fnout, fps=1)

		# plt.show()
	ipdb.set_trace()

#==============================================================================

if __name__ == '__main__':
	main()

max_stars_repo_name: ArdenB/fireflies

max_issues_repo_name: ArdenB/fireflies

max_forks_repo_name: ArdenB/fireflies

reason: import numpy,import scipy,from scipy,from numba,import statsmodels