text:
##
# File: EntityInstanceExtractor.py
# Date: 19-Feb-2019  jdw
#
# Selected utilities to extract data from entity instance collections.
#
# PRELIMINARY VERSION -
#
# Updates:
#
#
##
__docformat__ = "restructuredtext en"
__author__ = "John Westbrook"
__email__ = "jwest@rcsb.rutgers.edu"
__license__ = "Apache 2.0"

import copy
import logging
from itertools import chain, groupby, islice
from operator import itemgetter
from statistics import mean, stdev

import numpy as np
import requests
from rcsb.db.mongo.Connection import Connection
from rcsb.db.mongo.MongoDbUtil import MongoDbUtil
from rcsb.utils.io.MarshalUtil import MarshalUtil

logger = logging.getLogger(__name__)

class EntityInstanceExtractor(object):
    """ Selected utilities to extract data from entity instance collections.

            >>> from operator import itemgetter
            >>>
            >>> seq2 = [1, 2, 4, 5, 6, 8, 9, 10]
            >>> list = []
            >>> for k, g in groupby(enumerate(seq2), lambda (i,x):i-x):
            ...     list.append(map(itemgetter(1), g))
            ...
            >>> print list
            [[1, 2], [4, 5, 6], [8, 9, 10]]
            Or as a list comprehension:

            >>> [map(itemgetter(1), g) for k, g in groupby(enumerate(seq2), lambda (i,x):i-x)]
            [[1, 2], [4, 5, 6], [8, 9, 10]]

            ##
            ##

            import numpy as np

            def main():
                # Generate some random data
                x = np.cumsum(np.random.random(1000) - 0.5)
                condition = np.abs(x) < 1

                # Print the start and stop indicies of each region where the absolute
                # values of x are below 1, and the min and max of each of these regions
                for start, stop in contiguous_regions(condition):
                    segment = x[start:stop]
                    print start, stop
                    print segment.min(), segment.max()

            import numpy as np

            Samples = np.array([[1, 2, 3],
                               [1, 2]])
            c = np.hstack(Samples)  # Will gives [1,2,3,1,2]
            mean, std = np.mean(c), np.std(c)
            newSamples = np.asarray([(np.array(xi)-mean)/std for xi in Samples])
            print newSamples

    """

    def __init__(self, cfgOb):
        self.__cfgOb = cfgOb
        self.__resourceName = "MONGO_DB"
        #
        self.__seqCache = {}
        self.__mU = MarshalUtil()
        #

    def getEntryInfo(self, **kwargs):
        """  Return a dictionary of PDB entries satifying the input conditions (e.g. method, resolution limit)
        """

        resLimit = kwargs.get("resLimit", 3.5)
        expMethod = kwargs.get("expMethod", "X-ray")
        #
        dbName = kwargs.get("dbName", "pdbx_core")
        collectionName = kwargs.get("collectionName", "pdbx_core_entry")
        #
        entryD = {}
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                if mg.collectionExists(dbName, collectionName):
                    logger.info("%s %s document count is %d", dbName, collectionName, mg.count(dbName, collectionName))
                    qD = {"rcsb_entry_info.experimental_method": expMethod, "refine.0.ls_d_res_high": {"$lte": resLimit}}
                    selectL = ["rcsb_entry_container_identifiers", "rcsb_entry_info", "refine"]
                    dL = mg.fetch(dbName, collectionName, selectL, queryD=qD)
                    logger.info("Selection %r fetch result count %d", selectL, len(dL))
                    #
                    for dV in dL:
                        if "rcsb_entry_container_identifiers" not in dV:
                            continue
                        entryId = dV["rcsb_entry_container_identifiers"]["entry_id"]
                        entryD[entryId] = {}
                        if "rcsb_entry_info" in dV and "polymer_composition" in dV["rcsb_entry_info"]:
                            entryD[entryId] = {
                                "polymer_composition": dV["rcsb_entry_info"]["polymer_composition"],
                                "experimental_method": dV["rcsb_entry_info"]["experimental_method"],
                            }
                        if "refine" in dV and dV["refine"] and "ls_d_res_high" in dV["refine"][0]:
                            entryD[entryId]["ls_d_res_high"] = dV["refine"][0]["ls_d_res_high"]
                            logger.debug("Got res %r", dV["refine"][0]["ls_d_res_high"])

        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return entryD
        #

    def getEntityIds(self, entryIdList):
        """
        """
        dbName = "pdbx_core"
        collectionName = "pdbx_core_polymer_entity"
        docD = {}
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                if mg.collectionExists(dbName, collectionName):
                    logger.info("%s %s document count is %d", dbName, collectionName, mg.count(dbName, collectionName))
                    for entryId in entryIdList:
                        qD = {"rcsb_polymer_entity_container_identifiers.entry_id": entryId}
                        selectL = ["rcsb_polymer_entity_container_identifiers"]
                        tL = mg.fetch(dbName, collectionName, selectL, queryD=qD)
                        #
                        logger.debug("Selection %r fetch result count %d", selectL, len(tL))
                        docD[entryId] = [vv["rcsb_polymer_entity_container_identifiers"] for vv in tL]
            logger.debug("docD is %r", docD)
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return docD

    def getPolymerEntities(self, entryD, **kwargs):
        """  Add 'selected_polymer_entities' satisfying the input contiditions and add this to the input entry dictionary.
        """
        dbName = kwargs.get("dbName", "pdbx_core")
        collectionName = kwargs.get("collectionName", "pdbx_core_polymer_entity")
        resultKey = kwargs.get("resultKey", "selected_polymer_entities")
        savePath = kwargs.get("savePath", "entry-data.pic")
        entryLimit = kwargs.get("entryLimit", None)
        saveKwargs = kwargs.get("saveKwargs", {"fmt": "pickle"})
        #
        try:
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                if mg.collectionExists(dbName, collectionName):
                    logger.info("%s %s document count is %d", dbName, collectionName, mg.count(dbName, collectionName))
                    selectL = [
                        "rcsb_polymer_entity_container_identifiers",
                        "entity_poly.type",
                        "entity_poly.pdbx_seq_one_letter_code_can",
                        "rcsb_entity_source_organism.ncbi_taxonomy_id",
                        "rcsb_entity_source_organism.ncbi_scientific_name",
                        "struct_ref.pdbx_seq_one_letter_code",
                        "struct_ref.pdbx_db_accession",
                        "struct_ref.db_name",
                        "struct_ref.entity_id",
                    ]
                    iCount = 0
                    for entryId in entryD:
                        #
                        if resultKey in entryD[entryId]:
                            continue
                        #
                        qD = {
                            "rcsb_polymer_entity_container_identifiers.entry_id": entryId,
                            "entity_poly.rcsb_entity_polymer_type": "Protein",
                            "entity.rcsb_multiple_source_flag": "N",
                        }
                        #
                        dL = mg.fetch(dbName, collectionName, selectL, queryD=qD)
                        logger.debug("%s query %r fetch result count %d", entryId, qD, len(dL))
                        eD = {}
                        for ii, dV in enumerate(dL, 1):
                            rD = {}
                            logger.debug("%s (%4d) d is %r", entryId, ii, dV)
                            if "rcsb_polymer_entity_container_identifiers" in dV and "asym_ids" in dV["rcsb_polymer_entity_container_identifiers"]:
                                rD["asym_ids"] = dV["rcsb_polymer_entity_container_identifiers"]["asym_ids"]
                                rD["entity_id"] = dV["rcsb_polymer_entity_container_identifiers"]["entity_id"]
                            if "entity_poly" in dV and "type" in dV["entity_poly"]:
                                rD["type"] = dV["entity_poly"]["type"]
                                rD["seq_one_letter_code_can"] = dV["entity_poly"]["pdbx_seq_one_letter_code_can"]

                            if "rcsb_entity_source_organism" in dV:
                                rD["ncbi_taxonomy_id"] = (
                                    dV["rcsb_entity_source_organism"][0]["ncbi_taxonomy_id"] if "ncbi_taxonomy_id" in dV["rcsb_entity_source_organism"][0] else None
                                )
                                rD["ncbi_scientific_name"] = (
                                    dV["rcsb_entity_source_organism"][0]["ncbi_scientific_name"] if "ncbi_scientific_name" in dV["rcsb_entity_source_organism"][0] else None
                                )

                            if "struct_ref" in dV and len(dV["struct_ref"]) == 1:
                                rD["seq_one_letter_code_ref"] = dV["struct_ref"][0]["pdbx_seq_one_letter_code"] if "pdbx_seq_one_letter_code" in dV["struct_ref"][0] else None
                                rD["db_accession"] = dV["struct_ref"][0]["pdbx_db_accession"] if "pdbx_db_accession" in dV["struct_ref"][0] else None
                                rD["db_name"] = dV["struct_ref"][0]["db_name"] if "db_name" in dV["struct_ref"][0] else None
                                #
                                refDbName = rD["db_name"]
                                dbAccession = rD["db_accession"]
                                dbRefSeq = self.__seqCache[dbAccession] if dbAccession in self.__seqCache else None

                                if refDbName in ["UNP"] and not dbRefSeq:
                                    dbRefSeq = self.__fetchUniprot(dbAccession)
                                    self.__seqCache[dbAccession] = dbRefSeq
                                    logger.debug("Fetch uniprot %r", dbRefSeq)
                                rD["ref_db_seq"] = dbRefSeq
                            else:
                                rD["seq_one_letter_code_ref"] = rD["db_accession"] = rD["db_name"] = None
                            #
                            if "entity_id" in rD:
                                eD[rD["entity_id"]] = copy.copy(rD)

                        entryD[entryId][resultKey] = copy.copy(eD)

                        iCount += 1
                        if iCount % 10 == 0:
                            logger.info("Completed polymer entities fetch %d/%d entries", iCount, len(entryD))
                        if iCount % 2000 == 0:
                            ok = self.__mU.doExport(savePath, entryD, **saveKwargs)
                            logger.info("Saved polymer entity results (%d) status %r in %s", iCount, ok, savePath)
                        if entryLimit and iCount >= entryLimit:
                            logger.info("Quitting after %d", iCount)
                            break
            #
            # for entryId in entryD:
            #    logger.debug(">>  %s docD  %r" % (entryId, entryD[entryId]))
            ok = self.__mU.doExport(savePath, entryD, **saveKwargs)
            logger.info("Saved polymer entity results (%d) entries %d status %r in %s", iCount, len(entryD), ok, savePath)
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return entryD

    def getEntityInstances(self, entryD, **kwargs):
        """ Get the selected validation data for the instances in the input entry dictionary.

        entryD[entryId]['selected_polymer_entities'][entityId]['validation'] = {}

        Add keys: 'pdbx_vrpt_instance_results'  and  'pdbx_unobs_or_zero_occ_residues' to the validation dictionary above.

        Args:
            resourceName (str):  resource name (e.g. DrugBank, CCDC)
            **kwargs: unused

        Returns:
            entryD: { }
        """
        dbName = kwargs.get("dbName", "pdbx_core")
        collectionName = kwargs.get("collectionName", "pdbx_core_polymer_entity_instance")
        savePath = kwargs.get("savePath", "entry-data.pic")
        saveKwargs = kwargs.get("saveKwargs", {"fmt": "pickle"})
        entryLimit = kwargs.get("entryLimit", None)
        #
        try:
            optF = False
            iCount = 0
            with Connection(cfgOb=self.__cfgOb, resourceName=self.__resourceName) as client:
                mg = MongoDbUtil(client)
                if mg.collectionExists(dbName, collectionName):
                    logger.info("%s %s total document count is %d", dbName, collectionName, mg.count(dbName, collectionName))
                    #
                    for entryId, dV in entryD.items():
                        for entityId, peD in dV["selected_polymer_entities"].items():
                            # if 'anal_instances' in peD:
                            #    continue
                            vD = {}
                            for asymId in peD["asym_ids"]:
                                qD = {
                                    "rcsb_polymer_entity_instance_container_identifiers.entry_id": entryId,
                                    "rcsb_polymer_entity_instance_container_identifiers.asym_id": asymId,
                                }
                                # qD = {'rcsb_entity_instance_container_validation_identifiers.entity_type': 'polymer'}
                                # selectL = ['pdbx_vrpt_instance_results', 'pdbx_unobs_or_zero_occ_residues']
                                selectL = ["pdbx_vrpt_instance_results"]
                                tL = mg.fetch(dbName, collectionName, selectL, queryD=qD)
                                dV = {}
                                if not tL:
                                    logger.info("No validation data for %s %s %s(%s)", dbName, collectionName, entryId, asymId)
                                    continue
                                #
                                logger.debug(">>> %s %s (%s) dict key length %d ", collectionName, entryId, asymId, len(tL[0]))

                                #
                                if optF:
                                    dV["pdbx_vrpt_instance_results"] = tL[0]["pdbx_vrpt_instance_results"] if "pdbx_vrpt_instance_results" in tL[0] else []
                                    dV["pdbx_unobs_or_zero_occ_residues"] = tL[0]["pdbx_unobs_or_zero_occ_residues"] if "pdbx_unobs_or_zero_occ_residues" in tL[0] else []
                                #
                                if optF:
                                    urdL = tL[0]["pdbx_unobs_or_zero_occ_residues"] if "pdbx_unobs_or_zero_occ_residues" in tL[0] else []
                                    oL = [{"label_seq_id": urd["label_seq_id"], "label_comp_id": urd["label_comp_id"]} for urd in urdL]
                                    dV["pdbx_unobs_or_zero_occ_residues"] = oL
                                #
                                try:
                                    irdL = tL[0]["pdbx_vrpt_instance_results"] if "pdbx_vrpt_instance_results" in tL[0] else []
                                    oL = [{"label_seq_id": ird["label_seq_id"], "label_comp_id": ird["label_comp_id"]} for ird in irdL]
                                    dV["pdbx_vrpt_instance_results_seq"] = oL
                                except Exception as e:
                                    logger.error("Failing with entryId %s entityId %s asymId %s bad validation data %s", entryId, entityId, asymId, str(e))

                                #
                                try:
                                    irdL = tL[0]["pdbx_vrpt_instance_results"] if "pdbx_vrpt_instance_results" in tL[0] else []
                                    oL = [{"OWAB": ird["OWAB"], "label_seq_id": ird["label_seq_id"], "label_comp_id": ird["label_comp_id"]} for ird in irdL]
                                    dV["pdbx_vrpt_instance_results_occ"] = oL
                                except Exception as e:
                                    logger.debug("Failing with entryId %s entityId %s asymId %s bad validation data %s", entryId, entityId, asymId, str(e))

                                vD[asymId] = copy.copy(dV)
                                #
                            analD = self.analEntity(entryId, peD, vD)
                            entryD[entryId]["selected_polymer_entities"][entityId]["anal_instances"] = copy.copy(analD)
                        iCount += 1
                        if iCount % 500 == 0:
                            logger.info("Completed %d/%d entries", iCount, len(entryD))
                        if iCount % 2000 == 0:
                            ok = self.__mU.doExport(savePath, entryD, **saveKwargs)
                            logger.info("Saved polymer entity instance results (%d) status %r in %s", iCount, ok, savePath)
                        if entryLimit and iCount >= entryLimit:
                            break
            ok = self.__mU.doExport(savePath, entryD, **saveKwargs)
            logger.info("Saved polymer instance results (%d) entries %d status %r in %s", iCount, len(entryD), ok, savePath)
        except Exception as e:
            logger.exception("Failing with %s", str(e))
        return entryD

    def analEntity(self, entryId, entityD, vD, **kwargs):
        """

        {'polymer_composition': 'protein/NA', 'experimental_method': 'X-ray',
        'selected_polymer_entities': {'1': {'asym_ids': ['D', 'C', 'E', 'A', 'B', 'F'],
                   'entity_id': '1', 'type': 'polypeptide(L)',
                   'seq_one_letter_code_can': 'MAKGQSLQDPFLNALRRERVPVSIYLVNGIKLQGQIESFDQFVILLKNTVSQMVYKHAISTVVPS',
                   'ncbi_taxonomy_id': 511693,
                    'ncbi_scientific_name': 'Escherichia coli BL21',
                    'seq_one_letter_code_ref': 'MAKGQSLQDPFLNALRRERVPVSIYLVNGIKLQGQIESFDQFVILLKNTVSQMVYKHAISTVVPS',
                    'db_accession': 'C5W5L7',
                    'db_name': 'UNP',
                    'validation': {'D': {'pdbx_vrpt_instance_results': [{'OWAB': 29.45, 'label_seq_id': 5, 'label_comp_id': 'GLN'},
                                                                            {'OWAB': 26.12, 'label_seq_id': 6, 'label_comp_id': 'SER'},
                                                                            {'OWAB': 22.72, 'label_seq_id': 7, 'label_comp_id': 'LEU'},
                                                                            {'OWAB': 14.56, 'label_seq_id': 8, 'label_comp_id': 'GLN'},
                                                                            {'OWAB': 19.18, 'label_seq_id': 9, 'label_comp_id': 'ASP'},
                                                                            {'OWAB': 16.56, 'label_seq_id': 10, 'label_comp_id': 'PRO'},
                                                                            {'OWAB': 14.78, 'label_seq_id': 11, 'label_comp_id': 'PHE'},
                                                                            {'OWAB': 11.2, 'label_seq_id': 12, 'label_comp_id': 'LEU'}, }}...]

                                        'pdbx_unobs_or_zero_occ_residues': [{'label_seq_id': 1, 'label_comp_id': 'MET'},
                                               {'label_seq_id': 2, 'label_comp_id': 'ALA'},
                                                {'label_seq_id': 3, 'label_comp_id': 'LYS'},
                                                 {'label_seq_id': 4, 'label_comp_id': 'GLY'}]}

        """
        _ = kwargs
        analD = {}
        try:
            entityId = entityD["entity_id"]
            asymIdL = entityD["asym_ids"]

            refSeq = entityD["seq_one_letter_code_ref"] if "seq_one_letter_code_ref" in entityD else None
            entitySeq = entityD["seq_one_letter_code_can"] if "seq_one_letter_code_can" in entityD else None
            # -------
            # Get UniProt
            #
            dbName = entityD["db_name"] if "db_name" in entityD else None
            dbAccession = entityD["db_accession"] if "db_accession" in entityD else None
            dbRefSeq = entityD["ref_db_seq"] if "ref_db_seq" in entityD else None
            # --
            if dbRefSeq:
                logger.debug("%s (%s) ref db %4d:  %r", dbAccession, dbName, len(dbRefSeq), dbRefSeq)
            if refSeq:
                logger.debug("%s (%s) seq ref pdb %4d:  %r", dbAccession, dbName, len(refSeq), refSeq)
            if entitySeq:
                logger.debug("%s (%s) entity sample %4d:  %r", dbAccession, dbName, len(entitySeq), entitySeq)
            #
            lenRefDbSeq = len(dbRefSeq) if dbRefSeq else None
            lenEntitySeq = len(entitySeq)
            # sampleSeqCov = 1.0 - float(lenRefDbSeq - lenEntitySeq) / float(lenRefDbSeq) if lenRefDbSeq else None
            #

            # -
            for asymId in asymIdL:
                if asymId not in vD:
                    logger.error("Missing validation data for %s %s %s", entryId, entityId, asymId)
                    continue
                #
                irDL = vD[asymId]["pdbx_vrpt_instance_results_seq"] if "pdbx_vrpt_instance_results_seq" in vD[asymId] else []
                lsL = list(set([dV["label_seq_id"] for dV in irDL]))
                lenInstanceSeq = len(lsL)

                instRefDbSeqCov = 1.0 - float(lenRefDbSeq - lenInstanceSeq) / float(lenRefDbSeq) if lenRefDbSeq else None
                instSampleSeqCov = 1.0 - float(lenEntitySeq - lenInstanceSeq) / float(lenEntitySeq)
                #
                occDL = vD[asymId]["pdbx_vrpt_instance_results_occ"] if "pdbx_vrpt_instance_results_occ" in vD[asymId] else []
                # average the
                owabRegD = {}
                if occDL:
                    owabD = {}
                    for dV in occDL:
                        owabD.setdefault(dV["label_seq_id"], []).append(dV["OWAB"])
                    #
                    # logger.info("owabD %r" % owabD)
                    meanOwabD = {k: mean(v) for k, v in owabD.items()}
                    meanOwab = mean(meanOwabD.values())
                    stdevOwab = stdev(meanOwabD.values())
                    #
                    logger.debug(">> Length of B values list %d mean %.3f stdev %.3f", len(meanOwabD), meanOwab, stdevOwab)
                    #
                    meanOwabA = np.array(list(meanOwabD.values()))
                    #
                    condition = meanOwabA > (meanOwab + meanOwab)
                    regL = self.__contiguousRegions(condition)
                    for ii, (start, stop) in enumerate(regL, 1):
                        segment = meanOwabA[start:stop]
                        logger.debug("B value range =  start %d stop %d min %.3f max %.3f", start, stop, segment.min(), segment.max())
                        owabRegD[ii] = {"length": stop - start + 1, "occ_min": segment.min(), "occ_max": segment.max()}

                #
                #
                # if False:
                #    uDL = vD[asymId]['pdbx_unobs_or_zero_occ_residues'] if 'pdbx_unobs_or_zero_occ_residues' in vD[asymId] else []
                #    unobsL = [d['label_seq_id'] for d in uDL]
                #
                # segL = []
                # for k, g in groupby(enumerate(lsL), lambda x: x[0] - x[1]):
                #    logger.info(" Segment entryId %s entityId %s asymId %s:  %r" % (entryId, entityId, asymId, list(map(itemgetter(1), g))))
                #
                # for k, g in groupby(enumerate(lsL), lambda(i, x): i - x):
                #    logger.info(" entryId %s entityId %s asymId %s:  %r" % (entryId, entityId, asymId, list(map(itemgetter(1), g)))

                segL = [list(map(itemgetter(1), g)) for _, g in groupby(enumerate(lsL), lambda x: x[0] - x[1])]
                logger.debug("Modeled sequence length %d segments %d", len(lsL), len(segL))
                #
                gapD = {}
                for ii in range(1, len(segL)):
                    bG = segL[ii - 1][-1]
                    eG = segL[ii][0]
                    gapD[ii] = eG - bG - 1
                    logger.debug("Gap %d length %d", ii, gapD[ii])
                #
                #
                if instRefDbSeqCov:
                    logger.debug(
                        "Summary %s %s %s refcov %.2f  sampleCov %.2f - gaps (%d) %r owabs seqments (%d) %r",
                        entryId,
                        entityId,
                        asymId,
                        instRefDbSeqCov,
                        instSampleSeqCov,
                        len(gapD),
                        list(gapD.values()),
                        len(owabRegD),
                        list(owabRegD.values()),
                    )
                else:
                    logger.debug(
                        "Summary %s %s %s sampleCov %.2f - gaps (%d) %r owabs seqments (%d) %r",
                        entryId,
                        entityId,
                        asymId,
                        instSampleSeqCov,
                        len(gapD),
                        list(gapD),
                        len(owabRegD),
                        list(owabRegD.values()),
                    )
                #
                analD[asymId] = {"coverage_inst_refdb": instRefDbSeqCov, "coverage_inst_entity": instSampleSeqCov, "gapD": copy.copy(gapD), "owabRegiond": copy.copy(owabRegD)}
                logger.debug("entry %s entity %s analD %r", entryId, entityId, analD)
        except Exception as e:
            logger.exception("%s failing with %s", entryId, str(e))
        #
        return analD

    def __getSegments(self, values):
        xV = np.asarray(values)
        # Generate some random data
        # x = np.cumsum(np.random.random(1000) - 0.5)
        #
        condition = np.abs(xV) < 1

        # Print the start and stop indicies of each region where the absolute
        # values of x are below 1, and the min and max of each of these regions
        for start, stop in self.__contiguousRegions(condition):
            segment = xV[start:stop]
            print(start, stop)
            print(segment.min(), segment.max())

    def __contiguousRegions(self, condition):
        """Finds contiguous True regions of the boolean array "condition.

        Returns a 2D array where the first column is the start index of the region and the
        second column is the end index.

        """

        # Find the indicies of changes in "condition"
        dV = np.diff(condition)
        (idx,) = dV.nonzero()

        # We need to start things after the change in "condition". Therefore,
        # we'll shift the index by 1 to the right.
        idx += 1

        if condition[0]:
            # If the start of condition is True prepend a 0
            idx = np.r_[0, idx]

        if condition[-1]:
            # If the end of condition is True, append the length of the array
            idx = np.r_[idx, condition.size]  # Edit

        # Reshape the result into two columns
        idx.shape = (-1, 2)
        return idx

    def __window(self, seq, num=2):
        """Returns a sliding window (of width n) over data from the iterable
           s -> (s0,s1,...s[n-1]), (s1,s2,...,sn), ...
        """
        it = iter(seq)
        result = tuple(islice(it, num))
        if len(result) == num:
            yield result
        for elem in it:
            result = result[1:] + (elem,)
            yield result

    def missingElements(self, lV):
        missing = chain.from_iterable(range(x + 1, y) for x, y in self.__window(lV) if (y - x) > 1)
        return list(missing)

    def __fetchUniprot(self, uniProtId):
        baseUrl = "http://www.uniprot.org"
        wsEndPoint = "/uniprot/"
        fS = ""
        try:
            fullUrl = baseUrl + wsEndPoint + uniProtId + ".fasta"
            result = requests.get(fullUrl)
            if result.ok:
                fL = result.text.split("\n")
                fS = "".join(fL[1:])
            else:
                logger.error("UniProt Fasta request for %s returns status %r", uniProtId, result.status_code)
        except Exception as e:
            logger.error("Failing request for %s with %s", uniProtId, str(e))
        return fS


text:
import logging
import numpy as np

class BasicSort:

    def __init__(self, arr, sortfn):
        logging.debug("Sorting class is initialized.")
        self.__arr = arr
        print(sortfn)
        self.__sortfn = sortfn.lower()

        if self.__sortfn == "insertsort":
            self.insertsort()
        elif self.__sortfn == "bubblesort":
            self.bubblesort()
        elif self.__sortfn == "mergesort":
            p = 0
            r = (len(self.__arr)-1)
            self.mergesort(p,r)
        elif self.__sortfn == "quicksort":
            p = 0
            r = (len(self.__arr)-1)
            self.quicksort(p,r)
        elif self.__sortfn == "pythonsort":
            self.pythonsort()
        elif self.__sortfn == "numpysort":
            self.numpysort()

    #Insert Sort
    def insertsort(self):
        logging.debug("Initializing InsertSort Function.")
        n = len(self.__arr)
        for i in range(1, n):
            key = self.__arr[i]
            j = i - 1
            while j >=0 and self.__arr[j] > key:
                self.__arr[j+1] = self.__arr[j]
                j = j - 1
            self.__arr[j+1] = key
        logging.debug("Sorted Array after InsertSort : {}".format(self.__arr))
        logging.debug("InsertSort Function completed successfully.")

    #Bubble Sort
    def bubblesort(self):
        logging.debug("Initializing BubbleSort Function.")
        n = len(self.__arr)
        for j in range(1, n):
            for i in range(0, n - 1):
                if self.__arr[i] > self.__arr[i+1]:
                    temp = self.__arr[i]
                    self.__arr[i] = self.__arr[i+1]
                    self.__arr[i+1] = temp
        logging.debug("Sorted Array after BubbleSort : {}".format(self.__arr))
        logging.debug("BubbleSort Function completed successfully.")

    # Merge Sort
    def mergesort(self, p, r):
        logging.debug("Initializing Merge Sort Function.")
        if p < r:
            q = (p + r) // 2  ##Double slash divides down to nearest whole number
            self.mergesort(p, q)
            self.mergesort(q + 1, r)
            self.merge(p, q, r)
        logging.debug("Sorted Array after MergeSort : {}".format(self.__arr))
        logging.debug("MergeSort Function completed successfully.")

    def merge(self, p, q, r):
        A = self.__arr
        n1 = q - p + 1
        n2 = r - q
        left = [0] * (n1 + 1)
        right = [0] * (n2 + 1)
        for i in range(0, n1):
            left[i] = A[p + i]
        for j in range(0, n2):
            right[j] = A[q + j + 1]
        left[n1] = float('inf')
        right[n2] = float('inf')
        i = 0
        j = 0

        for k in range(p, r + 1):
            if left[i] <= right[j]:
                A[k] = left[i]
                i = i + 1
            else:
                A[k] = right[j]
                j = j + 1

        # return A

    # Quick Sort
    def quicksort(self, p, r):
        if p < r:
            q = self.partition(p, r)
            self.quicksort(p, q - 1)
            self.quicksort(q + 1, r)
        logging.debug("Sorted Array after Quick Sort : {}".format(self.__arr))
        logging.debug("QuickSort Function completed successfully.")

    # Partition Function works with Quick Sort
    def partition(self, p, r):
        A = self.__arr
        x = A[r]
        i = p - 1
        for j in range(p, r):
            if A[j] <= x:
                i = i + 1
                A[i], A[j] = A[j], A[i]

        A[i + 1], A[r] = A[r], A[i + 1]

        return i + 1

ext: py

max_stars_repo_name: adiitya-dey/datastructures

max_issues_repo_name: adiitya-dey/datastructures

max_forks_repo_name: adiitya-dey/datastructures

reason: import numpy


text:
#                    __
#                  / *_)
#       _.----. _ /../
#     /............./
# __/..(...|.(...|
# /__.-|_|--|_|
#
# Christos Kyrkou, PhD
# 2019

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dropout, Dense, Add, Input, Concatenate, Layer,Permute
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Activation, Reshape, AveragePooling2D, \
    SeparableConv2D, GlobalAveragePooling2D, Multiply
from tensorflow.keras.layers import Conv2DTranspose, Cropping2D,DepthwiseConv2D
from tensorflow.keras.layers import Lambda, SpatialDropout2D, LeakyReLU, UpSampling2D, Dropout
from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2
from tensorflow.keras.applications.nasnet import NASNetMobile
from tensorflow.keras.applications.mobilenet import MobileNet
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.models import load_model, save_model

from tensorflow.keras.applications.resnet50 import ResNet50

import tensorflow.keras.initializers as initializers
import tensorflow.keras.regularizers as regularizers

from tensorflow.keras.activations import softmax

import tensorflow.keras.backend as K
import tensorflow as tf
import numpy as np

# from models.net_models import *
# from models.MiniGoogLeNet import *
# from models.sqeezenet import *
# from models.YOLOv2 import *

import sys

sys.path.append("..")
from gen_utils import change_model

ki = initializers.he_normal()

kid = initializers.RandomNormal(mean=0.0, stddev=0.02)

kr = regularizers.l2(5e-4)

def custom_preprocess(x):
    x = x/255.

    return x

def get_preprocess_method(net_name):
    preprocess_input = None

    if (net_name == 'resnet' or net_name == 'resnet25'):
        from tensorflow.keras.applications.resnet50 import preprocess_input

    if (net_name == 'vgg'):
        from tensorflow.keras.applications.vgg16 import preprocess_input

    if (net_name == 'efficient'):
        from tensorflow.keras.applications.EfficientNetB0 import preprocess_input
    if (net_name == 'nasmobile'):
        from tensorflow.keras.applications.nasnet import preprocess_input
    if (net_name == 'mobileV2'):
        from tensorflow.keras.applications.mobilenet_v2 import preprocess_input

    if (net_name == 'mobile'):
        from tensorflow.keras.applications.mobilenet import preprocess_input

    return preprocess_input

def add_SSD_Head(params, x, ks=1, stride=1):
    ki = initializers.RandomNormal(mean=0.0, stddev=0.01)

    kr = regularizers.l2(5e-4)

    x = Conv2D(params.FILTER_SSD, (ks, ks), kernel_initializer=ki,
               activation='linear', use_bias=False, strides=(stride, stride), name="last_layer")(x)

    x = BatchNormalization()(x)

    x = Reshape((params.GRID_H, params.GRID_W, params.BOX, params.CLASS + 4 + 1))(x)

    return x

def add_YOLO_Head(params, x, ks=1, stride=1, weight_decay=1e-4,name="",ki=initializers.RandomNormal(mean=0.0,stddev=0.02),kr=None):

    kr = regularizers.l2(weight_decay)

    x = Conv2D(params.FILTER, (ks, ks), padding='same', kernel_initializer=ki, kernel_regularizer=kr,
               activation='linear',  use_bias=True, strides=(stride, stride), name='last_layer'+name)(x)

    x = Reshape((params.GRID_H, params.GRID_W, params.BOX, params.CLASS + 4 + 1), name='yolo_head'+name)(x)

    return x

def conv_block(x, channels, kernel_size=3, stride=1, weight_decay=5e-4, dropout_rate=None,act='l'):
    kr = regularizers.l2(weight_decay)
    ki = initializers.RandomNormal(mean=0.0,stddev=0.02)

    x = Conv2D(channels, (kernel_size, kernel_size), kernel_initializer=ki, strides=(stride, stride),
               use_bias=False, padding='same', kernel_regularizer=kr)(x)
    x = BatchNormalization()(x)

    if(act == 'm'):
        x = Mish()(x)

    if (act == 'l'):
        x = LeakyReLU(alpha=0.1)(x)

    if (act == 'r'):
        x = Activation('relu')(x)

    if (act == 's'):
        x = Activation('sigmoid')(x)

    if dropout_rate != None and dropout_rate != 0.:
        x = Dropout(dropout_rate)(x)

    return x

def separable_conv_block(x, channels, kernel_size=3, stride=1, weight_decay=5e-4, dropout_rate=None,act='l'):
    ki = initializers.RandomNormal(mean=0.0, stddev=0.2)

    kr = regularizers.l2(weight_decay)

    x = SeparableConv2D(channels, (kernel_size, kernel_size), kernel_initializer=ki,
                        strides=(stride, stride), use_bias=False, padding='same',
                        kernel_regularizer=kr)(x)

    x = BatchNormalization()(x)

    if (act == 'm'):
        x = Mish()(x)

    if (act == 'l'):
        x = LeakyReLU(alpha=0.1)(x)

    if (act == 'r'):
        x = Activation('relu')(x)

    if (act == 's'):
        x = Activation('sigmoid')(x)

    if dropout_rate != None and dropout_rate != 0.:
        x = SpatialDropout2D(dropout_rate)(x)
    return x

def atrous_block(x, channels, kernel_size=3, stride=1, weight_decay=5e-4, dropout_rate=None):
    #From
    #Christos Kyrkou and Theocharis Theocharides, "EmergencyNet: Efficient Aerial Image Classification for Drone-Based Emergency Monitoring Using Atrous Convolutional Feature Fusion," in IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing, vol. 13, pp. 1687-1699, 2020, doi: 10.1109/JSTARS.2020.2969809.

    ki = initializers.he_normal()
    kr = regularizers.l2(weight_decay)

    x1 = DepthwiseConv2D(kernel_size=3, kernel_regularizer=kr, kernel_initializer=ki, strides=1, padding='same',
                        use_bias=False, dilation_rate=1)(x)
    x1 = BatchNormalization()(x1)
    x1 = Mish()(x1)
    x2 = DepthwiseConv2D(kernel_size=3, kernel_regularizer=kr, kernel_initializer=ki, strides=1, padding='same',
                         use_bias=False, dilation_rate=2)(x)
    x2 = BatchNormalization()(x2)
    x2 = Mish()(x2)
    x3 = DepthwiseConv2D(kernel_size=3, kernel_regularizer=kr, kernel_initializer=ki, strides=1, padding='same',
                         use_bias=False, dilation_rate=3)(x)
    x3 = BatchNormalization()(x3)
    x3 = Mish()(x3)

    x = tf.keras.layers.Maximum()([x1,x2,x3])

    x = Conv2D(channels, (1, 1), kernel_regularizer=kr, kernel_initializer=ki, strides=(1, 1), padding='same', use_bias=False)(x)

    if dropout_rate != None and dropout_rate != 0.:
        x = SpatialDropout2D(dropout_rate)(x)
    return x

def squeeze_excite_block(tensor, ratio=16):

    dim = K.int_shape(tensor)[-1]
    se_shape = (1, 1, dim)

    se = GlobalAveragePooling2D()(tensor)
    se = Reshape(se_shape)(se)
    se = Dense(dim // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se)
    se = Dense(dim, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se)
    x = Multiply()([tensor, se])
    return x

def spatial_attention_block(tensor):

    x1 = K.max(tensor,axis=-1,keepdims=True)
    x2 = K.mean(tensor,axis=-1,keepdims=True)

    x = Concatenate(axis=-1)([x1,x2])
    x = Conv2D(1, (3, 3), use_bias=True, padding='same')(x)
    x = Activation('sigmoid')(x)

    x = Multiply()([tensor, x])

    return x

def enc_resnet(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = ResNet50(input_shape=input_shape, weights='imagenet', include_top=False, pooling=None)
    base_model.summary()
    istrainable = True

    for ind, layer in enumerate(base_model.layers):
        print(ind, layer.name)
        if (layer.name == 'conv5_block3_out'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'conv4_block6_out'): #Freeze the whole backbone
            istrainable = True
        layer.trainable = istrainable

    x = BatchNormalization()(x)

    x = conv_block(x, 256, kernel_size=1, stride=1, dropout_rate=0.25,act='r')

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input

def enc_resnet25(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = ResNet50(input_shape=input_shape, weights='imagenet', include_top=False, pooling=None)
    istrainable = True

    base_model.summary()

    for ind, layer in enumerate(base_model.layers):
        print(ind, layer.name)
        if (layer.name == 'activation_25'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'activation_11'):
            istrainable = True
        layer.trainable = istrainable

    cls = x
    x = separable_conv_block(x, 256, kernel_size=2, stride=2, dropout_rate=0.)
    x = separable_conv_block(x, 256, kernel_size=1, stride=1, dropout_rate=0.)

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input, cls

def enc_vggnet(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = VGG16(input_shape=input_shape, weights='imagenet', include_top=False, pooling=None)
    base_model.summary()

    istrainable = False

    for ind, layer in enumerate(base_model.layers):
        print(ind, layer.name)
        if (layer.name == 'block5_pool'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'block5_conv3'):
            istrainable = True
        layer.trainable = istrainable

    cls = x

    x = conv_block(x, 256, kernel_size=3, stride=1, dropout_rate=0.25)

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input, cls

def enc_mobile(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]

    base_model = MobileNet(input_shape, weights='imagenet', include_top=False, pooling=None)
    base_model.summary()

    istrainable = True

    for ind, layer in enumerate(base_model.layers):
        if (layer.name == 'conv_pw_13_relu'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break

        if (layer.name == 'conv_pw_11_relu'):
            istrainable = True
        layer.trainable = istrainable

    x = SpatialDropout2D(0.25)(x)
    x = conv_block(x, 256, kernel_size=1, stride=1, dropout_rate=0.,weight_decay=0.)

    x = add_YOLO_Head(params, x, ks=1, stride=1,weight_decay=0., ki=initializers.he_normal())

    return x, base_model.input

def enc_mobileV2(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = MobileNetV2(input_shape=input_shape, weights='imagenet', include_top=False, pooling=None)

    base_model.summary()
    istrainable = True

    for ind, layer in enumerate(base_model.layers):
        if (layer.name == 'out_relu'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'block_14_add'):
            istrainable = True
        layer.trainable = istrainable

    x = Dropout(0.4)(x)

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input

def enc_EfficientNetB0(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = EfficientNetB0(input_shape=input_shape, weights='imagenet', include_top=False, pooling=None)

    base_model.summary()
    istrainable = True

    for ind, layer in enumerate(base_model.layers):
        if (layer.name == 'out_relu'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'block_12_add'):
            istrainable = True
        layer.trainable = istrainable

    x = SpatialDropout2D(0.25)(x)

    x = separable_conv_block(x, 512, kernel_size=1, stride=1, dropout_rate=0.25)

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input

def enc_nasmobile(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    base_model = NASNetMobile(input_shape=(224, 224, 3), weights='imagenet', include_top=False, pooling=None)
    base_model = change_model(base_model, new_input_shape=[None] + input_shape, custom_objects={})

    base_model.summary()
    istrainable = True

    for ind, layer in enumerate(base_model.layers):
        if (layer.name == 'out_relu'):
            layer.trainable = istrainable
            x = base_model.layers[ind].output
            break
        if (layer.name == 'block_12_add'):
            istrainable = True
        layer.trainable = istrainable

    x = base_model.layers[-1].output

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, base_model.input

def enc_yolopeds(params,name=''):

    input_shape = [params.NORM_H, params.NORM_W, 3]
    inp = Input(shape=input_shape)

    x = conv_block(inp, 16, 5, stride=2, dropout_rate=0.)#conv
    x = separable_conv_block(x, 32, 5, dropout_rate=0.)#conv1

    x = separable_conv_block(x, 32, 5, dropout_rate=0.)#conv2
    x = separable_conv_block(x, 32, 5, dropout_rate=0)#conv3
    x1 = AveragePooling2D(pool_size=(2, 2))(x)  # 80x80

    x2 = separable_conv_block(x1, 64, 3, dropout_rate=0.)#conv4
    x = separable_conv_block(x2, 128, 3, dropout_rate=0.)#conv5
    x = Concatenate()([x, x1])

    x3 = AveragePooling2D(pool_size=(2, 2))(x)  # 40x40

    x4 = separable_conv_block(x3, 128, 3, dropout_rate=0.)#conv6
    x = separable_conv_block(x4, 128, 3, dropout_rate=0.)#conv7
    x = Concatenate()([x, x3])

    x5 = AveragePooling2D(pool_size=(2, 2))(x)  # 20x20

    x = separable_conv_block(x5, 256, 3, dropout_rate=0.)#conv8
    x = separable_conv_block(x, 256, 3, dropout_rate=0.)#conv9
    x = Concatenate()([x, x5])

    x = AveragePooling2D(pool_size=(2, 2))(x)  # 10x10
    x = spatial_attention_block(x)

    xb = AveragePooling2D(pool_size=(8, 8))(x2)
    xb = spatial_attention_block(xb)

    xc = AveragePooling2D(pool_size=(4, 4))(x4)
    xc = spatial_attention_block(xc)

    x = Concatenate()([x,
                       xb,
                       xc])

    cls = x

    custom_objects = {'Mish': Mish}

    x = add_YOLO_Head(params, x, ks=1, stride=1,name="")

    return x, inp, cls

def enc_dronet(params,name=''):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    inp = Input(shape=input_shape)

    act='l'
    block = conv_block
    x = block(inp, 32, 5, stride=2, dropout_rate=0.,act=act)

    x = block(x, 32, 5, stride=1, dropout_rate=0.,act=act)
    x = block(x, 32, 1, stride=2, dropout_rate=0.,act=act)

    x = block(x, 64, 3, stride=1, dropout_rate=0.,act=act)
    x = block(x, 64, 1, stride=1, dropout_rate=0.,act=act)
    x = MaxPooling2D(pool_size=(2, 2))(x)

    x = block(x, 64, 3, stride=1, dropout_rate=0.,act=act)
    x = block(x, 64, 1, stride=1, dropout_rate=0.,act=act)
    x = MaxPooling2D(pool_size=(2, 2))(x)

    x4 = block(x, 64, 3, stride=2, dropout_rate=0.,act=act)
    x2 = block(x4, 128, 1, stride=1, dropout_rate=0.,act=act)

    x = block(x2, 128, 3, stride=1, dropout_rate=0., act=act)
    x = block(x, 256, 1, stride=1, dropout_rate=0., act=act)

    x = block(x, 128, 3, stride=1, dropout_rate=0.,act=act)
    x3 = block(x, 512, 1, stride=1, dropout_rate=0.,act=act)

    x=x3

    cls = x

    custom_objects = {'Mish': Mish}

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, inp, cls

def swish(x):
    def hard_swish(x):
        hrelu = tf.nn.relu6(tf.add(x, tf.constant(3.0)))
        tf.multiply(x, hrelu)
        return x

    return hard_swish(x)

def enc_tinyyolov2(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    inp = Input(shape=input_shape)

    kr = regularizers.l2(5e-4)
    ki = initializers.he_normal()#initializers.RandomNormal(mean=0.0,stddev=0.01)
    # Layer 1
    x = Conv2D(16, (3, 3), strides=(1, 1), padding='same', use_bias=False, input_shape=(416, 416, 3),
               kernel_regularizer=kr,
               kernel_initializer=ki)(inp)
    x = BatchNormalization()(x)
    x = LeakyReLU(alpha=0.01)(x)
    x = MaxPooling2D(pool_size=(2, 2))(x)

    # Layer 2 - 5
    for i in range(0, 4):
        x = Conv2D(32 * (2 ** i), (3, 3), strides=(1, 1), padding='same', use_bias=False,
                   kernel_regularizer=kr,
                   kernel_initializer=ki)(x)
        x = BatchNormalization()(x)
        x = LeakyReLU(alpha=0.01)(x)
        x = MaxPooling2D(pool_size=(2, 2))(x)
        x = SpatialDropout2D(0.25)(x)

    # Layer 6
    x = Conv2D(512, (3, 3), strides=(1, 1), padding='same', use_bias=False, kernel_regularizer=kr,
               kernel_initializer=ki)(x)
    x = BatchNormalization()(x)
    x = LeakyReLU(alpha=0.01)(x)
    x = MaxPooling2D(pool_size=(2, 2), strides=(1, 1), padding='same')(x)

    # Layer 7 - 8
    for _ in range(0, 2):
        x = Conv2D(1024, (3, 3), strides=(1, 1), padding='same', use_bias=False,
                   kernel_regularizer=kr,
                   kernel_initializer=ki)(x)
        x = BatchNormalization()(x)
        x = LeakyReLU(alpha=0.01)(x)
        x = SpatialDropout2D(0.25)(x)

    # Layer 9
    cls = x
    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, inp, cls

class Mish(Layer):
    '''
    Mish Activation Function.
    .. math::
        mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + e^{x}))
    Shape:
        - Input: Arbitrary. Use the keyword argument `input_shape`
        (tuple of integers, does not include the samples axis)
        when using this layer as the first layer in a model.
        - Output: Same shape as the input.
    Examples:
        >>> X_input = Input(input_shape)
        >>> X = Mish()(X_input)
    '''

    def __init__(self, **kwargs):
        super(Mish, self).__init__(**kwargs)
        self.supports_masking = True

    def call(self, inputs):
        return inputs * K.tanh(K.softplus(inputs))

    def get_config(self):
        config = super(Mish, self).get_config()
        return config

    def compute_output_shape(self, input_shape):
        return input_shape

def enc_miniGoogleNet(params):
    input_shape = [params.NORM_H, params.NORM_W, 3]

    mgn = MiniGoogLeNet()

    model = mgn.build2(params.NORM_W, params.NORM_H, 3, 3)

    model.summary()

    inp = model.input
    x = model.output
    x = AveragePooling2D()(x)
    cls = x
    # x = Dropout(0.3)(x)

    x = add_YOLO_Head(params, x, ks=1, stride=1)

    return x, inp, cls

def MYMODEL(params, net_name='resnet'):
    input_shape = [params.NORM_H, params.NORM_W, 3]
    inputs = Input(shape=input_shape)

    cls = None

    if (net_name == 'resnet'):
        x, inp = enc_resnet(params)

    if (net_name == 'resnet25'):
        x, inp, _ = enc_resnet25(params)

    if (net_name == 'efficient'):
        x, inp = enc_EfficientNetB0(params)

    if (net_name == 'nasmobile'):
        x, inp = enc_nasmobile(params)

    if (net_name == 'mobileV2'):
        x, inp = enc_mobileV2(params)

    if (net_name == 'mobile'):
        x, inp = enc_mobile(params)

    if (net_name == 'vgg'):
        x, inp, _ = enc_vggnet(params)

    if (net_name == 'miniGoogleNet'):
        x, inp, _ = enc_miniGoogleNet(params)

    if (net_name == 'yolopeds'):
        x, inp, _ = enc_yolopeds(params,name=net_name)

    if (net_name == 'dense'):
        x, inp, _ = enc_dense(params)

    if (net_name == 'dronet'):
        x, inp, _ = enc_dronet(params,name=net_name)

    if (net_name == 'tinyyolov2'):
        x, inp, _ = enc_tinyyolov2(params)

    if (cls == None):
        m = Model(inputs=[inp], outputs=[x])

    else:
        cls = conv_block(cls, channels=params.CLASS, kernel_size=1, dropout_rate=0.25)
        cls = GlobalAveragePooling2D()(cls)

        cls = Activation('sigmoid', name='class_branch')(cls)
        m = Model(inputs=[inp], outputs=[x, cls])

    return m


text:
import re
import random
import cPickle as pickle
import numpy as np
import pandas as pd
from numba import jit
from data_helper import DataHelper
from os.path import expanduser
from sampler import MultinomialSampler
from collections import defaultdict

home = expanduser('~')
data_root =  './data'

def get_data(data_name, conf, reverse_samping=False):
    # prepare data
    data_helper = _get_data(data_name)
    train = data_helper.data['train']
    test_seen = data_helper.data['test_seen']
    test = data_helper.data['test']
    C = data_helper.data['C']

    user_count = max(np.max(train[:, 0]), np.max(test[:, 0])) + 1
    word_count = np.max(C) + 1
    item_count = C.shape[0]
    max_content_len = C.shape[1]
    class DataSpec(object):
        def __init__(self, user_count, word_count,
                     item_count, max_content_len):
            self.user_count = user_count
            self.word_count = word_count
            self.item_count = item_count
            self.max_content_len = max_content_len
    data_spec = DataSpec(user_count, word_count,
                         item_count, max_content_len)
    if conf is not None:
        data_spec.W_pretrain, data_spec.C_pretrain = get_pretrained_vectors(conf, data_spec, data_helper)

        neg_dist = conf.neg_dist
        try:
            neg_sampling_power = conf.neg_sampling_power
            # print '[INFO] Setting neg_sampling_power to user defined {}'.format(neg_sampling_power)
        except:
            neg_sampling_power = 0.75
        sampler_dict = {}
        # user negative sampling (if needed)
        if reverse_samping:
            train = train[:, [1, 0, 2]]
            sample_u = get_sampler(train, neg_dist=neg_dist,
                                   neg_sampling_power=neg_sampling_power,
                                   batch_mode=False)
            sample_batch_u = get_sampler(train, neg_dist=neg_dist,
                                         neg_sampling_power=neg_sampling_power,
                                         batch_mode=True)
            train = train[:, [1, 0, 2]]
            sampler_dict['sample_u'] = sample_u
            sampler_dict['sample_batch_u'] = sample_batch_u
        # item negative sampling
        sample = get_sampler(train, neg_dist=neg_dist,
                             neg_sampling_power=neg_sampling_power,
                             batch_mode=False)
        sample_batch = get_sampler(train, neg_dist=neg_dist,
                                   neg_sampling_power=neg_sampling_power,
                                   batch_mode=True)
        sampler_dict['sample'] = sample
        sampler_dict['sample_batch'] = sample_batch
        data_helper.sampler_dict = sampler_dict
    data_helper.data_spec = data_spec

    return data_helper

def _get_data(data_name='citeulike_title_only'):
    data_helper = DataHelper()
    def error():
        assert False, '[ERROR] unseen data_name %s' % data_name
    sub_folder = ''
    fold = re.findall('fold(\d+)', data_name)
    if len(fold) == 1:
        sub_folder = 'fold%d' % int(fold[0])
    if data_name.startswith('citeulike'):
        if data_name.startswith('citeulike_title_only'):
            content_file = data_root + '/citeulike/title_only/%s/data_content.pkl' % sub_folder
            split_file = data_root + '/citeulike/title_only/%s/data_split_cold_item.pkl' % sub_folder
        elif data_name.startswith('citeulike_title_and_abstract'):
            content_file = data_root + '/citeulike/title_and_abstract/%s/data_content.pkl' % sub_folder
            split_file = data_root + '/citeulike/title_and_abstract/%s/data_split_cold_item.pkl' % sub_folder
        else:
            error()
    elif data_name.startswith('news'):
        if  data_name.startswith('news_title_only'):
            content_file = data_root + '/news/title_only/%s/data_content.pkl' % sub_folder
            split_file = data_root + '/news/title_only/%s/data_split_cold_item.pkl' % sub_folder
        elif  data_name.startswith('news_title_and_abstract'):
            content_file = data_root + '/news/title_and_abstract/%s/data_content.pkl' % sub_folder
            split_file = data_root + '/news/title_and_abstract/%s/data_split_cold_item.pkl' % sub_folder
    else:
        error()

    # data_helper.load_data(content_file)
    with open(split_file) as fp:
        split_data = pickle.load(fp)
    data_helper.data = split_data
    return data_helper

def get_pretrain_folder(data_name, aug=True):
    # used in conf
    if data_name.startswith('citeulike_title_only'):
        data_name_short = 'citeulike/title_only/'
    elif data_name.startswith('citeulike_title_and_abstract'):
        data_name_short = 'citeulike/title_and_abstract/'
    elif data_name.startswith('news_title_only'):
        data_name_short = 'news/title_only/'
    elif data_name.startswith('news_title_and_abstract'):
        data_name_short = 'news/title_and_abstract/'
    else:
        assert False, 'data_name %s unkown' % data_name

    if aug:
        aug = 'aug'
    else:
        aug = ''

    pretrain_folder = data_root + '/%s/pretrain/%s/' % (data_name_short, aug)
    return pretrain_folder

def get_pretrained_vectors(conf, data_spec, data_helper):
    # load pretrained word vectors, and sentence vectors
    if conf.pretrain:
        wordvec_filepath = conf.pretrain['wordvec_filepath']
        sentvec_filepath = conf.pretrain['sentvec_filepath']
    else:
        wordvec_filepath = None
        sentvec_filepath = None

    if wordvec_filepath:
        if wordvec_filepath.endswith('pkl'):
            # matrix in pkl format
            with open(wordvec_filepath) as fp:
                W_pretrain = pickle.load(fp)
        else:
            # original text format
            # discard the first line, each line is word emb0 emb1 ...
            # last column is NaN and discarded
            df = pd.read_csv(wordvec_filepath, skiprows=1, header=None, sep=' ')
            df.index = df[0]
            del df[0]
            del df[df.shape[1]]

            W_pretrain = np.zeros((data_spec.word_count, df.shape[1]))
            for word, wid in data_helper.word2id.iteritems():
                try:
                    w = df.loc[word]
                except:
                    continue
                W_pretrain[wid] = w
    else:
        W_pretrain = None

    if sentvec_filepath:
        if sentvec_filepath.endswith('pkl'):
            # matrix in pkl format
            with open(sentvec_filepath) as fp:
                C_pretrain = pickle.load(fp)
        else:
            # original text format
            # each line is id emb0 emb1 ...
            # last column is NaN and discarded
            df = pd.read_csv(sentvec_filepath, skiprows=0, header=None, sep=' ')
            df[0] = df[0].apply(lambda x: x[2:])
            df[0] = df[0].astype('int')
            df.index = df[0]
            del df[0]
            del df[df.shape[1]]
            item_count = data_spec.item_count
            C_pretrain = np.zeros((item_count, df.shape[1]))
            for row in df.iterrows():
                if row[0] < item_count:
                    C_pretrain[row[0]] = row[1]
    else:
        C_pretrain = None

    return W_pretrain, C_pretrain

@jit
def count_jit(ls, c):
    # count list into dict
    for l in ls: c[l] += 1

# prepare sampler for training
def get_sampler(ratings, neg_dist='unigram', neg_sampling_power=0.75,
                column=1, rand_seed=0, batch_mode=True):
    '''
    Return a sampling function

    Assuming the triplets are in shape of (user, item, rating), i.e. dst/item will be sampled as
        negatives
    '''
    neg_dist = neg_dist.split('_')[0]  # support suffix tricks
    assert neg_dist == 'uniform' or neg_dist == 'unigram', [neg_dist]

    dist = [0. for _ in range(np.max(ratings[:, column]) + 1)]
    count_jit(ratings[:, column], dist)
    dist = np.array(dist, dtype=float)
    if neg_dist == 'uniform':
        dist[dist > 0] = 1
        # dist[:] = 1

    s = MultinomialSampler(dist, dist.size, neg_sampling_power, rand_seed)
    if batch_mode:
        return s.sample_batch
    else:
        return s.sample

def group_shuffle_train(train, by='item', chop=0, iidx=None):
    ''' shuffle train randomly but group by items (or users), e.g.
        (2, 1, _), (1, 1, _), (44, 1,  _), (3, 0, _), (1, 0, _), ...
        train: in format of user item response(optional)
        chop: segment into size of chop, then shuffle.
            in expectation, when chop is small, chop will be close to
            average links per group/item/user in a mini-batch
        iidx : idx list of all users/items
    '''
    if by == 'user': col = 0
    else: col = 1
    if iidx is None:
        iidx = np.arange(np.max(train[:, col]) + 1)
    np.random.shuffle(iidx)
    np.random.shuffle(train)
    train = np.hstack((train, iidx[train[:, col]].reshape((train.shape[0], 1))))
    train = train[train[:, -1].argsort()][:, :-1]
    if chop > 0:
        bulk_len = (train.shape[0] // chop) * chop
        train_bulk = train[:bulk_len].reshape((-1, chop, train.shape[-1]))
        np.random.shuffle(train_bulk)
        train_bulk = train_bulk.reshape((-1, train_bulk.shape[-1]))
        train = np.vstack([train_bulk, train[bulk_len:]])
    return train

class GroupSampler(object):
    ''' First sample a group/item, then sample its positive members/users
        Followed by sampling negative members/users (whose number is decided)
    '''
    def __init__(self, train, group_by='item', chop=1,
                 neg_dist='unigram', neg_sign=0):
        ''' chop: int
            group size for each item/user

            neg_dist & neg_sign only used when sample_with_negs() called
        '''
        if neg_dist == 'uniform_no_correction':
            neg_dist = 'uniform'
            no_correction = True
        else:
            no_correction = False
        self.train = train
        self.group_by = group_by
        self.chop = chop
        self.neg_dist = neg_dist
        self.neg_sign = neg_sign

        if group_by == 'item':
            group_column = 1
            member_column = 0
        elif group_by == 'user':
            group_column = 0
            member_column = 1
        else:
            assert False, '[ERROR] Illegal group_by {}'.format(group_by)

        try:
            neg_sampling_power = conf.neg_sampling_power
            print '[INFO] In GroupSampler, setting neg_sampling_power' \
                  'to user defined {}'.format(neg_sampling_power)
        except:
            neg_sampling_power = 0.75
        self.group_sampler = get_sampler(train, neg_dist='unigram', \
            neg_sampling_power=1., column=group_column, batch_mode=True)
        self.member_neg_sampler = get_sampler(train, neg_dist=neg_dist, \
            neg_sampling_power=neg_sampling_power, column=member_column, \
            batch_mode=True)

        group_set = set(train[:, group_column])
        p_n_div_p_d = {}
        if neg_dist == 'unigram' or no_correction:
            for group in group_set:
                p_n_div_p_d[group] = 1.
        elif neg_dist == 'uniform':
            dist = [0. for _ in range(max(group_set) + 1)]
            count_jit(train[:, group_column], dist)
            dist /= np.sum(dist)
            group_set_size = len(group_set)
            for group in group_set:
                p_n_div_p_d[group] = 1. / group_set_size / dist[group]
        else:
            assert False, '[ERROR] Illegal neg_dist {}'.format(neg_dist)
        self.p_n_div_p_d = p_n_div_p_d

        group_members = defaultdict(list)
        for each in train:
            group_members[each[group_column]].append(each[member_column])
        self.group_members = group_members

    def sample(self, batch_size_p, strict_return_shape=True):
        ''' strict_return_shape: bool
            When set True, the returned batch size has to be batch_size_p
            Otherwise it can be smaller
        '''
        group_members = self.group_members
        if strict_return_shape:
            num_groups = int(np.ceil(float(batch_size_p) / self.chop))
        else:
            num_groups = batch_size_p // self.chop
        sampled_groups = self.group_sampler(num_groups)
        sampled_group_members = []
        for group in sampled_groups:
            gms = group_members[group]
            sampled_group_members.append( \
                np.random.choice(gms, self.chop))
        sampled_batch = np.vstack((np.hstack(sampled_group_members),
                                   np.repeat(sampled_groups, self.chop))).T
        sampled_batch = np.hstack((sampled_batch, \
            np.ones((sampled_batch.shape[0], 1), dtype=int)))
        if self.group_by == 'user':
            sampled_batch = sampled_batch[:, [1, 0, 2]]
        if strict_return_shape:
            return sampled_batch[:batch_size_p]
        else:
            return sampled_batch

    def sample_with_negs(self, batch_size_p, k, strict_return_shape=True):
        ''' k: int
            number of negative items per positive-link

            returned batch in following format:
                first ~batch_size_p are positives, with response = 1
                rest ~batch_size_p * k are negatives, with response = neg_sign
        '''

        group_members = self.group_members
        p_n_div_p_d = self.p_n_div_p_d
        batch_size_whole = batch_size_p * (1 + k)
        if strict_return_shape:
            num_groups = int(np.ceil(float(batch_size_p) / self.chop))
        else:
            num_groups = batch_size_p // self.chop
        sampled_groups = self.group_sampler(num_groups)
        sampled_groups_repeat_pos = []
        sampled_group_members_pos = []
        sampled_groups_repeat_neg = []
        sampled_group_members_neg = []

        def sample_members_for_groups(sampled_groups):
            for group in sampled_groups:
                # sample pos members
                sampled_group_members_pos.append( \
                    np.random.choice(group_members[group], self.chop))
                sampled_groups_repeat_pos.extend([group] * self.chop)

                # sample neg members
                random_rounding = lambda x: int(x) + (random.random() < x - int(x))
                num_negatives = random_rounding(k * self.chop * p_n_div_p_d[group])
                if num_negatives > 0:
                    sampled_groups_repeat_neg.extend([group] * num_negatives)
                    sampled_group_members_neg.append( \
                       self.member_neg_sampler(num_negatives))

        sample_members_for_groups(sampled_groups)

        while strict_return_shape:
            try: _here += 1
            except: _here = 1
            if _here > 10: assert False, \
                '[WARNING] the code here should be optimized if this is shown.'
            curt_batch_size = len(sampled_groups_repeat_pos) + \
                len(sampled_groups_repeat_neg)
            size_diff = batch_size_whole - curt_batch_size
            if size_diff <= 0:
                break
            if size_diff <= 100:
                num_groups_to_sample = size_diff
            else:
                num_groups_to_sample = size_diff // (self.chop * (1 + k)) + 100
            sampled_groups = self.group_sampler(num_groups_to_sample)
            sample_members_for_groups(sampled_groups)

        sampled_batch_pos = np.vstack((np.hstack(sampled_group_members_pos),
                                       sampled_groups_repeat_pos)).T
        sampled_batch_pos = np.hstack((sampled_batch_pos, \
            np.ones((sampled_batch_pos.shape[0], 1), dtype=int)))

        sampled_batch_neg = np.vstack((np.hstack(sampled_group_members_neg),
                                       sampled_groups_repeat_neg)).T
        sampled_batch_neg = np.hstack((sampled_batch_neg, self.neg_sign * \
            np.ones((sampled_batch_neg.shape[0], 1), dtype=int)))

        sampled_batch = np.vstack((sampled_batch_pos, sampled_batch_neg))

        if self.group_by == 'user':
            sampled_batch = sampled_batch[:, [1, 0, 2]]
        if strict_return_shape:
            return sampled_batch[:batch_size_whole]
        else:
            return sample_batch

max_stars_repo_name: lastBird/NNCF

max_issues_repo_name: lastBird/NNCF

max_forks_repo_name: lastBird/NNCF

reason: import numpy,from numba


text:
import numpy as np
from itertools import product
import pytest
import tensorflow as tf
from alibi_detect.utils.tensorflow import GaussianRBF, mmd2, mmd2_from_kernel_matrix, permed_lsdds
from alibi_detect.utils.tensorflow import relative_euclidean_distance, squared_pairwise_distance

n_features = [2, 5]
n_instances = [(100, 100), (100, 75)]
tests_pairwise = list(product(n_features, n_instances))
n_tests_pairwise = len(tests_pairwise)

@pytest.fixture
def pairwise_params(request):
    return tests_pairwise[request.param]

@pytest.mark.parametrize('pairwise_params', list(range(n_tests_pairwise)), indirect=True)
def test_pairwise(pairwise_params):
    n_features, n_instances = pairwise_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    np.random.seed(0)
    x = tf.convert_to_tensor(np.random.random(xshape).astype('float32'))
    y = tf.convert_to_tensor(np.random.random(yshape).astype('float32'))

    dist_xx = squared_pairwise_distance(x, x).numpy()
    dist_xy = squared_pairwise_distance(x, y).numpy()

    assert dist_xx.shape == (xshape[0], xshape[0])
    assert dist_xy.shape == n_instances
    np.testing.assert_almost_equal(dist_xx.trace(), 0., decimal=5)

tests_mmd = tests_pairwise
n_tests_mmd = n_tests_pairwise

@pytest.fixture
def mmd_params(request):
    return tests_mmd[request.param]

@pytest.mark.parametrize('mmd_params', list(range(n_tests_mmd)), indirect=True)
def test_mmd(mmd_params):
    n_features, n_instances = mmd_params
    xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features)
    np.random.seed(0)
    x = tf.convert_to_tensor(np.random.random(xshape).astype('float32'))
    y = tf.convert_to_tensor(np.random.random(yshape).astype('float32'))
    mmd_xx = mmd2(x, x, kernel=GaussianRBF(sigma=tf.ones(1)))
    mmd_xy = mmd2(x, y, kernel=GaussianRBF(sigma=tf.ones(1)))
    assert mmd_xy > mmd_xx

n = [10, 100]
m = [10, 100]
permute = [True, False]
zero_diag = [True, False]
tests_mmd_from_kernel_matrix = list(product(n, m, permute, zero_diag))
n_tests_mmd_from_kernel_matrix = len(tests_mmd_from_kernel_matrix)

@pytest.fixture
def mmd_from_kernel_matrix_params(request):
    return tests_mmd_from_kernel_matrix[request.param]

@pytest.mark.parametrize('mmd_from_kernel_matrix_params',
                         list(range(n_tests_mmd_from_kernel_matrix)), indirect=True)
def test_mmd_from_kernel_matrix(mmd_from_kernel_matrix_params):
    n, m, permute, zero_diag = mmd_from_kernel_matrix_params
    n_tot = n + m
    shape = (n_tot, n_tot)
    kernel_mat = np.random.uniform(0, 1, size=shape)
    kernel_mat_2 = kernel_mat.copy()
    kernel_mat_2[-m:, :-m] = 1.
    kernel_mat_2[:-m, -m:] = 1.
    kernel_mat = tf.convert_to_tensor(kernel_mat)
    kernel_mat_2 = tf.convert_to_tensor(kernel_mat_2)
    if not zero_diag:
        kernel_mat -= tf.linalg.diag(tf.linalg.diag_part(kernel_mat))
        kernel_mat_2 -= tf.linalg.diag(tf.linalg.diag_part(kernel_mat_2))
    mmd = mmd2_from_kernel_matrix(kernel_mat, m, permute=permute, zero_diag=zero_diag)
    mmd_2 = mmd2_from_kernel_matrix(kernel_mat_2, m, permute=permute, zero_diag=zero_diag)
    if not permute:
        assert mmd_2.numpy() < mmd.numpy()

def test_relative_euclidean_distance():
    x = tf.convert_to_tensor(np.random.rand(5, 3))
    y = tf.convert_to_tensor(np.random.rand(5, 3))

    assert (relative_euclidean_distance(x, y).numpy() == relative_euclidean_distance(y, x).numpy()).all()
    assert (relative_euclidean_distance(x, x).numpy() == relative_euclidean_distance(y, y).numpy()).all()
    assert (relative_euclidean_distance(x, y).numpy() >= 0.).all()

n = [10]
m = [10]
d = [3]
B = [20]
n_kcs = [5]
tests_permed_lsdds = list(product(n, m, d, B, n_kcs))
n_tests_permed_lsdds = len(tests_permed_lsdds)

@pytest.fixture
def permed_lsdds_params(request):
    return tests_permed_lsdds[request.param]

@pytest.mark.parametrize('permed_lsdds_params',
                         list(range(n_tests_permed_lsdds)), indirect=True)
def test_permed_lsdds(permed_lsdds_params):
    n, m, d, B, n_kcs = permed_lsdds_params

    kcs = tf.random.normal((n_kcs, d))
    x_ref = tf.random.normal((n, d))
    x_cur = 10 + 0.2*tf.random.normal((m, d))

    x_full = tf.concat([x_ref, x_cur], axis=0)
    sigma = tf.constant((1.,))
    k_all_c = GaussianRBF(sigma)(x_full, kcs)
    H = GaussianRBF(np.sqrt(2.)*sigma)(kcs, kcs)

    perms = [tf.random.shuffle(tf.range(n+m)) for _ in range(B)]
    x_perms = [perm[:n] for perm in perms]
    y_perms = [perm[n:] for perm in perms]

    lsdd_perms, H_lam_inv, lsdd_unpermed = permed_lsdds(
        k_all_c, x_perms, y_perms, H, return_unpermed=True
    )

    assert int(tf.reduce_sum(tf.cast(lsdd_perms > lsdd_unpermed, float))) == 0
    assert H_lam_inv.shape == (n_kcs, n_kcs)


text:
# -*- coding:utf-8 -*-

import numpy as np
from convolve import *

WIDTH = 4

class Filter:

    @staticmethod
    def normalize(mask):
        length = len(mask)
        sum = 0
        for i in range(1,length):
            sum += abs(mask[i])
        sum = 2*sum + abs(mask[0])
        for i in range(length):
            mask[i] /= sum
        return mask

    @staticmethod
    def make_gauss(sigma):
        sigma = max(sigma,0.01)
        length = int(np.ceil(sigma * WIDTH) + 1)
        mask = np.zeros(length)
        for i in range(length):
            mask[i] = np.exp(-0.5 * np.square(i/sigma))
        return mask

    @staticmethod
    def smooth(src,sigma):
        mask = Filter.make_gauss(sigma)
        mask = Filter.normalize(mask)
        tmp = np.zeros(src.T.shape)
        dst = np.zeros(src.shape)
        tmp = convolve_even(src,tmp,mask)
        dst = convolve_even(tmp,dst,mask)
        return dst

max_stars_repo_path: Graph-Algorithm/GraphBasedImageSegmentation(C|Python|with or not OpenCV)/GraphBasedImageSegmentation(Python without OpenCV)/filter.py

max_stars_repo_name: ZhongHouyu/CVCode

max_issues_repo_name: ZhongHouyu/CVCode

max_forks_repo_name: ZhongHouyu/CVCode


text:
"""
home-credit-default-competition repository

Functions for preprocessing and extracting features
from previous_application.csv file.
"""

import os
import gc
import numpy as np
import pandas as pd
import utils
import config

def get_previous_applications(path, num_rows=None):
    """Preprocess and extract features from previous_application.csv.

    Arguments:
        path: Path to the folder where files are saved (string).
        num_rows: Number of rows to read; None reads all rows (int, default: None).

    Returns:
        df: DataFrame with processed data.
    """
    prev = pd.read_csv(os.path.join(path, 'previous_application.csv'), nrows=num_rows)
    pay = pd.read_csv(os.path.join(path, 'installments_payments.csv'), nrows=num_rows)

    # One-hot encode most important categorical features
    ohe_columns = [
        'NAME_CONTRACT_STATUS', 'NAME_CONTRACT_TYPE', 'CHANNEL_TYPE',
        'NAME_TYPE_SUITE', 'NAME_YIELD_GROUP', 'PRODUCT_COMBINATION',
        'NAME_PRODUCT_TYPE', 'NAME_CLIENT_TYPE']
    prev, categorical_cols = utils.one_hot_encoder(prev, ohe_columns, nan_as_category=False)

    # Feature engineering: ratios and difference
    prev['APPLICATION_CREDIT_DIFF'] = prev['AMT_APPLICATION'] - prev['AMT_CREDIT']
    prev['APPLICATION_CREDIT_RATIO'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']
    prev['CREDIT_TO_ANNUITY_RATIO'] = prev['AMT_CREDIT']/prev['AMT_ANNUITY']
    prev['DOWN_PAYMENT_TO_CREDIT'] = prev['AMT_DOWN_PAYMENT'] / prev['AMT_CREDIT']
    # Interest ratio on previous application (simplified)
    total_payment = prev['AMT_ANNUITY'] * prev['CNT_PAYMENT']
    prev['SIMPLE_INTERESTS'] = (total_payment/prev['AMT_CREDIT'] - 1)/prev['CNT_PAYMENT']

    # Active loans - approved and not complete yet (last_due 365243)
    approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]
    active_df = approved[approved['DAYS_LAST_DUE'] == 365243]
    # Find how much was already payed in active loans (using installments csv)
    active_pay = pay[pay['SK_ID_PREV'].isin(active_df['SK_ID_PREV'])]
    active_pay_agg = active_pay.groupby('SK_ID_PREV')[['AMT_INSTALMENT', 'AMT_PAYMENT']].sum()
    active_pay_agg.reset_index(inplace= True)
    # Active loans: difference of what was payed and installments
    active_pay_agg['INSTALMENT_PAYMENT_DIFF'] = active_pay_agg['AMT_INSTALMENT'] - active_pay_agg['AMT_PAYMENT']
    # Merge with active_df
    active_df = active_df.merge(active_pay_agg, on= 'SK_ID_PREV', how= 'left')
    active_df['REMAINING_DEBT'] = active_df['AMT_CREDIT'] - active_df['AMT_PAYMENT']
    active_df['REPAYMENT_RATIO'] = active_df['AMT_PAYMENT'] / active_df['AMT_CREDIT']
    # Perform aggregations for active applications
    active_agg_df = utils.group(active_df, 'PREV_ACTIVE_', config.PREVIOUS_ACTIVE_AGG)
    active_agg_df['TOTAL_REPAYMENT_RATIO'] = active_agg_df['PREV_ACTIVE_AMT_PAYMENT_SUM']/\
                                             active_agg_df['PREV_ACTIVE_AMT_CREDIT_SUM']
    del active_pay, active_pay_agg, active_df
    gc.collect()

    # Change 365.243 values to nan (missing)
    prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace=True)
    prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace=True)
    prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace=True)
    prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace=True)
    prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace=True)
    # Days last due difference (scheduled x done)
    prev['DAYS_LAST_DUE_DIFF'] = prev['DAYS_LAST_DUE_1ST_VERSION'] - prev['DAYS_LAST_DUE']
    approved['DAYS_LAST_DUE_DIFF'] = approved['DAYS_LAST_DUE_1ST_VERSION'] - approved['DAYS_LAST_DUE']

    # Categorical features
    categorical_agg = {key: ['mean'] for key in categorical_cols}
    # Perform general aggregations
    agg_prev = utils.group(prev, 'PREV_', {**config.PREVIOUS_AGG, **categorical_agg})
    # Merge active loans dataframe on agg_prev
    agg_prev = agg_prev.merge(active_agg_df, how='left', on='SK_ID_CURR')
    del active_agg_df
    gc.collect()
    # Aggregations for approved and refused loans
    agg_prev = utils.group_and_merge(approved, agg_prev, 'APPROVED_', config.PREVIOUS_APPROVED_AGG)
    refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]
    agg_prev = utils.group_and_merge(refused, agg_prev, 'REFUSED_', config.PREVIOUS_REFUSED_AGG)
    del approved, refused
    gc.collect()
    # Aggregations for Consumer loans and Cash loans
    for loan_type in ['Consumer loans', 'Cash loans']:
        type_df = prev[prev['NAME_CONTRACT_TYPE_{}'.format(loan_type)] == 1]
        prefix = 'PREV_' + loan_type.split(" ")[0] + '_'
        agg_prev = utils.group_and_merge(type_df, agg_prev, prefix, config.PREVIOUS_LOAN_TYPE_AGG)
        del type_df
        gc.collect()

    # Get the SK_ID_PREV for loans with late payments (days past due)
    pay['LATE_PAYMENT'] = pay['DAYS_ENTRY_PAYMENT'] - pay['DAYS_INSTALMENT']
    pay['LATE_PAYMENT'] = pay['LATE_PAYMENT'].apply(lambda x: 1 if x > 0 else 0)
    dpd_id = pay[pay['LATE_PAYMENT'] > 0]['SK_ID_PREV'].unique()
    # Aggregations for loans with late payments
    agg_dpd = utils.group_and_merge(prev[prev['SK_ID_PREV'].isin(dpd_id)], agg_prev,
                                    'PREV_LATE_', config.PREVIOUS_LATE_PAYMENTS_AGG)
    del agg_dpd, dpd_id; gc.collect()
    # Aggregations for loans in the last x months
    for time_frame in [6, 12, 24]:
        time_frame_df = prev[prev['DAYS_DECISION'] >= -30*time_frame]
        prefix = 'PREV_LAST{}M_'.format(time_frame)
        agg_prev = utils.group_and_merge(time_frame_df, agg_prev, prefix, config.PREVIOUS_TIME_AGG)
        del time_frame_df; gc.collect()
    del prev
    gc.collect()
    return agg_prev


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# This is try to code spatial suppression motion task

import numpy as np
from gym import spaces
import neurogym as ngym
import sys

class SpatialSuppressMotion(ngym.TrialEnv):
    '''
    Spatial suppression motion task. This task is useful to study center-surround interaction in monkey MT and human psychophysical performance in motion perception.

    Tha task is derived from (Tadin et al. Nature, 2003). In this task, there is no fixation or decision stage. We only present a stimulus and a subject needs to perform a 4-AFC motion direction judgement. The ground-truth is the probabilities for choosing the four directions at a given time point. The probabilities depend on stimulus contrast and size, and the probabilities are derived from emprically measured human psychophysical performance.

    In this version, the input size is 4 (directions) x 8 (size) = 32 neurons. This setting aims to simulate four pools (8 neurons in each pool) of neurons that are selective for four directions.

    Args:
        <dt>: millisecs per image frame, default: 8.3 (given 120HZ monitor)
        <win_size>: size per image frame
        <timing>: millisecs, stimulus duration, default: 8.3 * 36 frames ~ 300 ms.
            This is the longest duration we need (i.e., probability reach ceilling)

    Note that please input default seq_len = 36 frames when creating dataset object.

    '''
    metadata = {
        'paper_link': 'https://www.nature.com/articles/nature01800',
        'paper_name': '''Perceptual consequences of centre–surround antagonism in visual motion processing ''',
        'tags': ['perceptual', 'plaid', 'motion', 'center-surround']
    }

    def __init__(self, dt=8.3, timing={'stimulus':300}, rewards=None):
        super().__init__(dt=dt)

        from numpy import pi

        # Rewards
        self.rewards = {'abort': -0.1, 'correct': +1., 'fail': 0.}
        if rewards:
            self.rewards.update(rewards)

        # Timing
        self.timing = {
            'stimulus': 300,  # we only need stimulus period for psychophysical task
            }
        if timing:
            self.timing.update(timing)

        self.abort = False

        # define action space four directions
        self.action_space = spaces.Box(0, 1, shape=(4,), dtype=np.float32) # the probabilities for four direction

        # define observation space
        self.observation_space = spaces.Box(
            0, np.inf, shape=(32,), dtype=np.float32) # observation space, 4 directions * 8 sizes
        # larger stimulus could elicit more neurons to fire

        self.directions = [1, 2, 3, 4] # motion direction left/right/up/down
        self.theta = [-pi/2, pi/2, 0, pi] # direction angle of the four directions
        self.directions_anti = [2, 1, 4, 3]
        self.directions_ortho = [[3, 4], [3, 4], [1, 2], [1, 2]]

    def _new_trial(self, diameter=None, contrast=None, direction=None):
        '''
        To define a stimulus, we need diameter, contrast, duration, direction
        <diameter>: 0~11, stimulus size in norm units
        <contrast>: 0~1, stimulus contrast
        <direction>: int(1/2/3/4), left/right/up/down
        '''
        #import ipdb;ipdb.set_trace();import matplotlib.pyplot as plt;

        # if no stimulus information provided, we random sample stimulus parameters
        if direction is None:
            direction = self.rng.choice(self.directions)
        if contrast is None:
            #contrast = self.rng.uniform(0, 1) # stimlus contrast
            contrast = self.rng.choice([0.05, 0.99])  # high/low contrast
        if diameter is None:
            #diameter = self.rng.uniform(0, 11) # stimlus size
            diameter = self.rng.choice([1, 11])  # small/large size

        trial = {
            'diameter': diameter,
            'contrast': contrast,
            'direction': direction,
        }

        # Periods and Timing
        # we only need stimulus period for this psychophysical task
        periods = ['stimulus']
        self.add_period(periods)

        # We need ground_truth
        # the probablities to choose four directions given stimulus parameters
        trial['ground_truth'] = self.getgroundtruth(trial)

        # create the stimulus
        ob = self.view_ob(period='stimulus')
        ob = np.zeros((ob.shape[0], ob.shape[1]))

        stim = (np.cos(np.array(self.theta) - self.theta[direction-1])+1) * contrast / 2
        stim = np.tile(stim, [diameter, 1])

        if diameter != 11:
            tmp = np.zeros((11-diameter, 4))
            stim = np.vstack((stim, tmp)).T.flatten()
        stim = stim.T.flatten()
        stim = np.tile(stim, [ob.shape[0], 1])

        ob = stim.copy()

        # set observation and groundtruth
        self.set_ob(ob, 'stimulus')
        self.set_groundtruth(trial['ground_truth'], 'stimulus')

        return trial

    def _step(self, action):
        '''
        We need output for every single step until to the end, no need to check action every step and calculate reward. Just let this function complete all steps.

        The _step function is useful for making a choice early in a trial or the situation when breaking the fixation.

        '''
        new_trial = False
        # rewards
        reward = 0
        gt = self.gt_now
        # # observations
        # if self.in_period('stimulus'): # start a new trial once step into decision stage
        #          new_trial = True
        return self.ob_now, reward, False, {'new_trial': new_trial, 'gt': gt}

    def getgroundtruth(self, trial):
        '''
        The utility function to obtain ground truth probabilities for four direction

        Input trial is a dict, contains fields <duration>, <contrast>, <diameter>

        We output a (4,) tuple indicate the probabilities to perceive left/right/up/down direction. This label comes from emprically measured human performance
        '''
        from scipy.interpolate import interp1d
        from numpy import zeros

        #duration = [5, 7.296, 10.65, 15.54, 22.67, 33.08, 48.27, 70.44, 102.8]
        frame_ind = [8, 9, 10, 11, 12, 14, 17, 20, 28, 37, 38, 39] # this is derived by envelop.m function
        xx = [1, 2, 3, 4, 5, 6, 7]
        yy = [0.249] * 7

        frame_ind = xx + frame_ind  # to fill in the first a few frames
        frame_ind = [i-1 for i in frame_ind] # frame index start from

        seq_len = self.view_ob(period='stimulus').shape[0]
        xnew = np.arange(seq_len)

        if trial['diameter'] == 1 & trial['contrast'] == 0.05: # small low contrast
            prob_corr = yy + [0.2889, 0.2278, 0.2944, 0.2722, 0.4611, 0.7167, 0.9000, 0.9611, 0.9444, 0.9611, 0.99, 0.99]
            prob_anti = yy + [0.2556, 0.2778, 0.2667, 0.2611, 0.2056, 0.0889, 0.0500, 0.0222, 0.0500, 0.0333, 0.003, 0.003]

        elif trial['diameter'] == 11 & trial['contrast'] == 0.99: # large high contrast
            prob_corr = yy + [0.2056, 0.2167, 0.2833, 0.2389, 0.4000, 0.7444, 0.8833, 0.9389, 0.9333, 0.9833, 0.99, 0.99]
            prob_anti = yy + [0.2667, 0.2444, 0.2722, 0.3611, 0.4333, 0.2222, 0.1111, 0.0556, 0.0611, 0.0111, 0.003, 0.003]

        elif trial['diameter'] == 1 & trial['contrast'] == 0.99: # small high contrast, note here we smooth the curve
            prob_corr = [0.2500, 0.2500, 0.2685, 0.2870, 0.3056, 0.3241, 0.3426] + [0.3889, 0.3611, 0.4111, 0.5556, 0.7833, 0.9056, 0.9444, 0.9889, 0.9944, 0.99, 0.99, 0.99]
            prob_anti = [0.2500, 0.2500, 0.2639, 0.2778, 0.2917, 0.3056, 0.3194] + [0.3389, 0.3333, 0.3278, 0.2833, 0.1889, 0.0833, 0.0444, 0.0111, 0.0056, 0.003, 0.003, 0.003]

        elif trial['diameter'] == 11 & trial['contrast'] == 0.05: # large low contrast:
            prob_corr = yy + [0.2278, 0.2611, 0.3222, 0.3333, 0.7944, 0.9778, 0.9778, 0.9833, 0.9889, 0.9944, 0.9944, 0.9944]
            prob_anti = yy + [0.2333, 0.2333, 0.2111, 0.1611, 0.1222, 0.0111, 0.0222, 0.0167, 0.0111, 0.0056, 0.0056, 0.0056]

        corr_prob = interp1d(frame_ind, prob_corr, kind='slinear', fill_value='extrapolate')(xnew)
        anti_prob = interp1d(frame_ind, prob_anti, kind='slinear', fill_value='extrapolate')(xnew)
        ortho_prob = (1-(corr_prob + anti_prob)) / 2

        direction = trial['direction']-1
        direction_anti = self.directions_anti[direction]-1
        direction_ortho = [i-1 for i in self.directions_ortho[direction]]

        gt = zeros((4, seq_len))
        gt[direction, :] = corr_prob
        gt[direction_anti, :] = anti_prob
        gt[direction_ortho, :] = ortho_prob

        #import ipdb;ipdb.set_trace();import matplotlib.pyplot as plt;
        gt = gt.T
        # gt is a seq_len x 4 numpy array
        return gt

max_stars_repo_name: ruyuanzhang/neurogym

max_issues_repo_name: ruyuanzhang/neurogym

max_forks_repo_name: ruyuanzhang/neurogym

reason: import numpy,from numpy,from scipy


text:
import os
import numpy as np

import gc

import matplotlib as mpl

mpl.use('Agg')
from matplotlib import pyplot as plt

'''
实验三 selection的试验
分为三种selection
1. random selection 无需做，由概率获得 k= wrong/all
2. additional selection 取数据里面跟已经选择的样本们的交集最大
3. total selection 直接取得数据中单一最大的覆盖情况
'''

attack_type = 'gaussian_noise'
threshold = 0
type = 1
m = n = 1
base_dir_path = '../data/mnist'

# additional
if __name__ == '__main__':
    for layer_num in [3, 5, 10]:
        wrong_coverage_data_path = os.path.join(base_dir_path,'mnist_wrong_active_data/coverage/' + attack_type + '/threshold' + str(threshold) + '/adjacent_' + str(m) + '_' + str(n) + '/type' + str(type))
        print(wrong_coverage_data_path)
        # str(layer_num)_hidden_layers_coverage.npy
        right_coverage_data_path = os.path.join(base_dir_path,'mnist_right_active_data/coverage/threshold' + str(threshold) + '/adjacent_' + str(m) + '_' + str(n) + '/type' + str(type))
        print(right_coverage_data_path)
        # str(layer_num)_hidden_layers_coverage.npy

        pass

max_stars_repo_name: ISE-DeepLearning/CoverageDNN

max_issues_repo_name: ISE-DeepLearning/CoverageDNN

max_forks_repo_name: ISE-DeepLearning/CoverageDNN


text:
'''
SYNBIOCHEM (c) University of Manchester 2019

SYNBIOCHEM is licensed under the MIT License.

To view a copy of this license, visit <http://opensource.org/licenses/MIT/>.

@author:  neilswainston
'''
# pylint: disable=invalid-name
import itertools
import os.path
import sys

from scipy.spatial.distance import pdist, squareform

import numpy as np
import pandas as pd

def analyse_comp_profiles(df):
    '''Analyse compound profiles.'''
    # Scale values to between 0 and 1:
    df = df / 100.00

    # Calculate Euclidean distances between all compound profiles:
    metric = 'euclidean'
    distances = pdist(df.values, metric=metric)
    dist_matrix = squareform(distances)

    # Update DataFrame with Euclidean distance between VAR3 and all compound
    # profiles:
    df[metric] = dist_matrix[1]  # 1 is index of VAR3 in data

    return df.sort_values(metric)

def analyse_plasticity(df):
    '''Analyse positional plasticity.'''
    # Scale values between 0 and 1:
    df = df / df.sum()

    # Consolidate all residue frequencies, ignoring codons:
    df.index = df.index.droplevel(level=0)
    df = df.groupby(level=0).sum().reset_index()
    df.index = df[('Residue', '', '')]
    df.index.name = 'Residue'
    df = df.drop(('Residue', '', ''), axis=1)

    # Remove silent mutations:
    # df = _remove_silent_mutations(df)

    # Calculate plasticity, as overlap between distribution and
    # uniform distribution:
    uniform = [1 / len(df)] * len(df)

    name = 'plasticity'
    return df.apply(lambda col: _get_intersection(col, uniform)
                    ).to_frame(name=name)

def _get_intersection(dist1, dist2):
    '''Get intersection between distributions.'''
    minima = np.minimum(dist1, dist2)
    return np.true_divide(np.sum(minima), np.sum(dist2))

def _remove_silent_mutations(df):
    '''Remove silent mitations.'''
    var3 = df.columns.get_level_values('VAR3')

    sim_df = pd.DataFrame(
        np.array(
            [val[0] != val[1] for val in itertools.product(df.index, var3)]
        ).reshape(len(df.index), len(var3)))

    return pd.DataFrame(df.values * sim_df.values,
                        columns=df.columns, index=df.index)

def main(args):
    '''main method.'''
    out_dir = args[0]
    curr_dir = os.path.dirname(os.path.realpath(__file__))
    data_dir = os.path.join(curr_dir, 'data')

    if not os.path.exists(out_dir):
        os.makedirs(out_dir)

    # Calculate distances between native and variant compound profiles:
    prof_in_df = pd.read_excel(os.path.join(data_dir,
                                            'Variant product profiles.xlsx'),
                               sheet_name='Relative_incl ger',
                               index_col='Variant')

    prof_out_df = analyse_comp_profiles(prof_in_df)
    prof_out_df.to_csv(os.path.join(out_dir, 'Variant product profiles.csv'))

    # Analyse position plasticity:
    aa_in_df = pd.read_excel(os.path.join(data_dir,
                                          'Amino acid occurence.xlsx'),
                             sheet_name='Sheet1',
                             header=[0, 1, 2],
                             index_col=[0, 1])

    aa_out_df = analyse_plasticity(aa_in_df)
    aa_out_df.to_csv(os.path.join(out_dir, 'plasticity.csv'))

if __name__ == '__main__':
    main(sys.argv[1:])

max_stars_repo_name: neilswainston/sbc-leferink

max_issues_repo_name: neilswainston/sbc-leferink

max_forks_repo_name: neilswainston/sbc-leferink

reason: import numpy,from scipy