text:
# -*- coding:utf8 -*-
# ==============================================================================
# Copyright 2017 Baidu.com, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Implements data parsers for different tasks on DuReader dataset.
"""

import copy
import itertools
import logging
import json
import numpy as np
import random
import sys
import paddle.v2 as paddle

from utils import find_best_question_match

logger = logging.getLogger("paddle")
logger.setLevel(logging.INFO)

class Dataset(object):
    """
    Base dataset class for various tasks.
    """
    def __init__(self,
                 file_names=None,
                 vocab_file=None,
                 vocab_size=0,
                 shuffle=False,
                 selected=-1,
                 preload=True,
                 append_raw=False,
                 is_infer=False,
                 max_p_len=500):
        self.file_names = file_names
        self.data = []
        self.raw = []
        self.vocab = self.read_vocab(vocab_file, vocab_size) \
                        if vocab_file else {}
        self.shuffle = shuffle
        self.selected = selected
        self.unk_id = 0
        self.preload = preload
        self.max_p_len = max_p_len
        self.append_raw = append_raw
        self.is_infer = is_infer
        self.doc_num = 5
        if preload:
            logger.info('Preloading data...')
            self.load()
            logger.info('Done, data[{}]'.format(len(self.data)))

    def load(self):
        """
        Loads all data records into self.data.
        """
        self.data = []
        for file_name in self.file_names:
            with open(file_name, 'r') as src:
                for line in src:
                    self.data += self.parse(line.strip())
        if self.shuffle:
            logger.info('Shuffling data...')
            random.shuffle(self.data)
        if self.selected > 0:
            self.data = self.data[:self.selected]

    def read_vocab(self, vocab_file, vocab_size):
        """
        Builds vocabulary dictionary.

        Args:
            vocab_file: file name of the vocabulary file, vocabulary file
                        should contain 2 columns separated by tab. The 1st
                        column is token, the 2nd is the count of the token
                        in the dataset.
            vocab_size: An integer indicates size of the vocabulary dictionary,
                        the size includes UNK.
        Returns:
            A dictionary mapping a token to a index. The size of the returned
            dictionary is vocab_size - 1, because UNK is not in the dict,
            the index of UNK is 0, if a token is not in this dict, it will be
            assigned to index 0. Tokens in this dict are indexed from 1.
        """
        vocab = {}
        with open(vocab_file, 'r') as vf:
            ln_cnt = 1
            for line in vf:
                if vocab_size > 0 and ln_cnt > vocab_size - 1:
                    break
                line = unicode(line, encoding='utf8')
                w, c = line.split('\t')
                vocab[w] = ln_cnt
                ln_cnt += 1

        # unk is not in vocab dict. but is counted in the vocab_size
        assert len(vocab) == vocab_size - 1, \
                "{} vs {}".format(len(vocab), vocab_size)
        logger.info('vocab size: {}'.format(len(vocab) + 1))
        return vocab

    def parse(self, line):
        """
        Implements parser for specific task, parses one line and returns a
        record as described by self.schema.
        """
        raise NotImplementedError

    def create_reader(self):
        """
        Creates reader generator.

        Returns:
            A generator, which yields one data record once.
        """
        def _reader_preload():
            if self.shuffle:
                logger.info('shuffling data ...')
                random.shuffle(self.data)
            for line in self.data:
                if not line:
                    logger.info("skip empty line: {}".format(line))
                    continue
                yield line

        def _reader_stream():
            for file_name in self.file_names:
                with open(file_name, 'r') as fn:
                    for line in fn:
                        data = self.parse(line.strip())
                        if not data:
                            continue
                        for d in data:
                            yield d

        if not self.preload:
            return _reader_stream
        return _reader_preload

class DuReaderYesNo(Dataset):
    """
    Implements parser for yesno task.
    """
    def __init__(self, *args, **kwargs):
        self.labels = {'Yes': 0, 'No': 1, 'Depends': 2}
        super(DuReaderYesNo, self).__init__(*args, **kwargs)
        self.schema = ['q_ids', 'a_ids', 'label']
        self.feeding = {name: i for i, name in enumerate(self.schema)}
        if self.is_infer:
            assert self.shuffle == False, 'Shuffling is forbidden for inference'

    def _get_id(self, s):
        s_ids = []
        if not isinstance(s, list):
            s = s.split(' ')
        for t in s:
            s_ids.append(self.vocab.get(t, self.unk_id))
        return s_ids

    def parse_train(self, line):
        """
        Parses one line for training.

        Args:
            line: A legal json string.

        Returns:
            A record as self.schema describes.
        """

        obj = json.loads(line.strip())
        ret = []
        if obj['question_type'] != 'YES_NO':
            return ret
        label_ids = [self.labels[l] for l in obj['yesno_answers']]
        question = [
                self.vocab.get(x, self.unk_id)
                for x in obj['segmented_question']]
        paras = map(self._get_id, obj['segmented_answers'])

        if not question or not paras:
            return ret
        for para, lbl in zip(paras, label_ids):
            ret.append((question, para, lbl))
        return ret

    def parse_infer(self, line):
        """
        Parses one line for inferring.

        Args:
            line: A legal json string.

        Returns:
            A record as self.schema describes.
        """
        obj = json.loads(line.strip())
        ret = []
        paras = map(self._get_id, obj['answers'])
        question = [self.vocab.get(x, self.unk_id) for x in obj['question']]
        fake_label = 0
        for idx, para in enumerate(paras):
            info = copy.deepcopy(obj)
            info['answer_idx'] = idx
            info['yesno_answers_ref'] = info['yesno_answers_ref']
            info['yesno_answers'] = []
            ret.append((question, para, fake_label, info))
        return ret

    def parse(self, line):
        """
        Parses one line for inferring.

        Args:
            line: A legal json string.

        Returns:
            A record as self.schema describes.
        """
        if self.is_infer:
            return self.parse_infer(line)
        return self.parse_train(line)

class DuReaderQA(Dataset):
    """
    Implements parser for QA task.
    """
    def __init__(self, *args, **kwargs):
        super(DuReaderQA, self).__init__(*args, **kwargs)
        doc_names = ['doc' + str(i) for i in range(self.doc_num)]
        start_label_names = ['start_pos' + str(i) for i in range(self.doc_num)]
        end_label_names = ['end_pos' + str(i) for i in range(self.doc_num)]
        doc_len_names = ['len' + str(i) for i in range(self.doc_num)]

        self.schema = ['q_ids'] \
                      + doc_names \
                      + doc_len_names \
                      + start_label_names \
                      + end_label_names

        self.feeding = {name: i for i, name in enumerate(self.schema)}

    def _find_ans_span(self, question_tokens, span, answer_docs, docs):
        assert len(span) == 1, 'Multiple spans: {}'.format(span)
        assert len(answer_docs) == 1, \
                'Multiple answer docs: {}'.format(answer_docs)
        selected_paras = []
        answer_docs = set(answer_docs)
        para_tokens = []
        for i, doc in enumerate(docs):
            if not self.is_infer:
                para_idx = doc['most_related_para']
            else:
                para_idx = find_best_question_match(doc, question_tokens)
            para = doc['segmented_paragraphs'][para_idx]
            if len(para) == 0:
                continue
            ans_span = (-1, -1)
            if i in answer_docs:
                ans_span = span[0]
            if len(para) > self.max_p_len:
                para = para[:self.max_p_len]
            s, e = ans_span
            if s >= len(para):
                logger.info('Skip span out of para length.')
                continue
            e = min(len(para) - 1, e)
            ans_span = (s, e)
            para_ids = [self.vocab.get(x, self.unk_id) for x in para]
            selected_paras.append((para_ids, ans_span))
            para_tokens.append(para)
        return selected_paras, para_tokens

    def _make_sample(self, question_ids, para_infos):
        def _get_label(idx, ref):
            ret = [0.0] * len(ref)
            if idx > 0:
                ret[idx] = 1.0
            return [[x] for x in ret]

        paras, start_labels, end_labels, para_lens = [], [], [], []
        default_para_info = ([0], (-1, -1))
        selected = para_infos[:self.doc_num]
        if len(selected) < self.doc_num:
            selected += [default_para_info] * (self.doc_num - len(selected))
        for para_ids, ans_span in selected:
            s, e = ans_span
            start_label = _get_label(s, para_ids)
            end_label = _get_label(e, para_ids)
            paras.append(para_ids)
            start_labels.append(start_label)
            end_labels.append(end_label)
            para_lens.append([[len(para_ids)]])
        sample = [question_ids] + paras + para_lens + start_labels + end_labels
        return sample

    def _get_infer_info(self, obj, paras):
        info = {}
        info['tokens'] = list(itertools.chain(*paras))
        info['answers'] = []
        info['answers_ref'] = obj.get('segmented_answers', [])
        info['question'] = obj['segmented_question']
        info['question_id'] = obj['question_id']
        info['question_type'] = obj['question_type']
        info['yesno_answers'] = []
        info['yesno_answers_ref'] = obj.get('yesno_answers', [])
        info['entity_answers'] = [[]]
        info['entity_answers_ref'] = obj.get('entity_answers', [[]])
        return info

    def parse(self, line):
        """
        Parses one line.

        Args:
            line: A legal json string.

        Returns:
            A record as self.schema describes.
        """
        ret = []
        obj = json.loads(line)
        if len(obj['answer_docs']) != 1:
            logger.info('skip, wrong answer docs')
            return ret
        if obj['answer_docs'][0] > 5:
            logger.info('skip, answer doc out of range.')
            return ret
        q_ids = [self.vocab.get(x, self.unk_id) for x in obj['segmented_question']]
        if len(q_ids) == 0:
            return ret
        selected_paras, para_tokens = self._find_ans_span(
                obj['segmented_question'],
                obj['answer_spans'],
                obj['answer_docs'],
                obj['documents'])
        if not selected_paras:
            return ret
        sample = self._make_sample(q_ids, selected_paras)
        if self.is_infer:
            sample.append(self._get_infer_info(obj, para_tokens))
        ret.append(sample)
        return ret

if __name__ == '__main__':
    data = sys.argv[1]
    vocab = sys.argv[2]
    dataset = DuReaderYesNo(file_names=data,
            vocab_file=vocab,
            preload=False,
            max_p_len=300,
            is_infer=False,
            append_raw=False,
            vocab_size=218967)

    # test reader
    reader = dataset.create_reader()
    for r in reader():
        print r

max_stars_repo_name: yaoleo/DuReader

max_issues_repo_name: yaoleo/DuReader

max_forks_repo_name: yaoleo/DuReader


text:
# Import the used packages.
import numpy as np
import numpy.random as rd

# Import the util functions
from utils import *

# Define the Game class
class Game:
    """
    The 2048 game class.

    ...

    Attributes
    ----------
    board : numpy array
        Board state of the game (default is np.zeros((4, 4))).
    score : float
        Score associated to the board (default is 0.0).
    lost : boolean
        Boolean to indicate if the game is lost (default is False).
    zeros : numpy array
        An array that contains the indexes of the zeros.
    len_zeros : int
        The number of zeros on the board.

    Methods
    -------
    add_random_tile():
        Randomly add a new tile on a empty tile of the board.
    move(direction, copy):
        Updates the board with the given direction.
    update_zeros():
        Updates the zeros attributes.
    check_lost():
        Checks if the game is over.
    update_score():
        Updates the score
    iterate(direction):
        Makes a move and all the steps of the game
        (add new tile, check for game over, ...).
    print():
        Prints the board
    """

    def __init__(self, board=np.zeros((4, 4)), score=0.0, lost=False):
        """
        Constructs all the necessary attributes for the game object.

        Parameters
        ----------
            board : numpy array, optional
                Board state of the game (default is np.zeros((4,4))).
            score : float, optional
                Score associated to the board (default is 0.0).
        """
        self.board = board
        self.score = score
        self.lost = lost
        self.zeros = np.argwhere(board == 0)
        self.len_zeros = len(self.zeros)
        if (board == np.zeros((4, 4))).all():
            self.add_random_tile()

    def add_random_tile(self):
        """
        Adds a new tile with value 2 or 4 (with a 0.9/0.1 probability distribution)
        in a empty tile of the board.

        Modifies the board array.

        Returns
        -------
        None
        """
        if self.len_zeros == 0:  # Should never be True
            self.lost = True
        else:
            p = rd.randint(self.len_zeros)
            ind = self.zeros[p]
            self.board[ind[0], ind[1]] = rd.choice([2, 4], p=[0.9, 0.1])
            self.zeros = np.delete(self.zeros, p, axis=0)
            self.len_zeros -= 1

    def move(self, direction, copy=False):
        """
        Updates the board with the given direction.

        Parameters
        ----------
        direction : int
            The direction with which to update the board.
            Values must be between 0 and 3 for *left*, *right*, *top* and *bottom*.
        copy : boolean
            Weither or not to make a copy the board then return it.

        Returns
        -------
        tuple
            (If copy is True, returns the modified array else only returns the boolean,
            A boolean if indicating if changes occurred)
        """
        assert direction in list(
            range(4)
        ), f'Direction must be in {list(range(4))}, not "{direction}".'

        modif = []

        if copy:
            m = self.board.copy()
        else:
            m = self.board

        for i in range(4):
            if direction == 0:
                col = m[i, ::-1]
                modif.append(combine_column(col))
                m[i] = col[::-1]
            elif direction == 1:
                col = m[i]
                modif.append(combine_column(col))
                m[i] = col
            elif direction == 2:
                col = m[::-1, i]
                modif.append(combine_column(col))
                m[:, i] = col[::-1]
            elif direction == 3:
                col = m[:, i]
                modif.append(combine_column(col))
                m[:, i] = col

        if copy:
            return m, any(modif)
        else:
            return any(modif)

    def update_zeros(self):
        """
        Updates the zeros attributes of the class.

        Returns
        -------
        None
        """
        self.zeros = np.argwhere(self.board == 0)
        self.len_zeros = len(self.zeros)

    def check_lost(self):
        """
        Checks if the game is over.

        Returns
        -------
        None
        """
        if self.len_zeros != 0:
            return False

        moved = [self.move(d, copy=True)[1] for d in range(4)]

        if any(moved):
            return False
        return True

    def update_score(self):
        pass

    def iterate(self, direction):
        """
        Function to iterate the game with the given direction.

        It doesn't move if the game is already lost.
        It tries to move with the given direction,
        if it doesn't move, returns False.
        Then it finds the zeros and add a new tile.
        Then it updates the score.
        Finally, it checks if the game is lost when there are no zeros left.

        Parameters
        ----------
        direction : int
            The direction with which to update the board.
            Values must be between 0 and 3 for *left*, *right*, *top* and *bottom*.

        Returns
        -------
        bool
            Returns None if the game is lost, else returns if there were changes.
        """
        if self.lost:
            return

        bool = self.move(direction=direction)
        if not bool:
            return False

        self.update_zeros()
        self.add_random_tile()
        self.update_score()

        if self.len_zeros == 0:
            self.lost = self.check_lost()
            if self.lost:
                return

        return True

    def print(self):
        print(self.board)

    def __str__(self):
        """
        A rapid display for the game.

        Prints the array and the number of zeros.

        Returns
        -------
        str
            The number of zeros
        """
        print(self.board)
        if self.lost:
            return f"The game is lost."
        return f"There is {self.len_zeros} zeros."

ext: py

reason: import numpy


text:
"""
Polygon discretization
----------------------
Group which groups all the utilities of polygon based discretization.

TODO
----
Fit from distribution of points tagged with region types.

"""

import numpy as np
from shapely import ops
from shapely.geometry import Point
from utils import match_regions, tesselation

from ..metricdiscretizor import BaseMetricDiscretizor

############################### Polygon based #################################
###############################################################################
class IrregularSpatialDisc(BaseMetricDiscretizor):
    """Grid spatial discretization."""
    n_dim = 2
    multiple = None

    def __init__(self, polygons, regions_id=None):
        "Main information to built the regions."
        self.multiple = None
        self._initialization()
        self.borders = polygons
        if regions_id is None:
            self.regions_id = np.arange(len(polygons))
        else:
            self.regions_id = regions_id
        self.regionlocs = self.map_regionid2regionlocs()
        self.limits = self.compute_limits()

    ################################ Functions ################################
    ###########################################################################
    def map_loc2regionid(self, locs):
        """Discretize locs returning their region_id.

        Parameters
        ----------
        locs: numpy.ndarray
            the locations for which we want to obtain their region given that
            discretization.

        Returns
        -------
        regionid: numpy.ndarray
            the region_id of each locs for this discretization.

        """
        regionid = -1*np.ones(locs.shape[0]).astype(int)
        for i in xrange(len(self.borders)):
            ## See what regions contain these points in their bounding boxes
            limits = np.array(self.borders[i].bounds).reshape((2, 2)).T
            logi_i = np.logical_and(locs[:, 0] >= limits[0, 0],
                                    locs[:, 0] < limits[0, 1])
            logi_i = np.logical_and(logi_i, locs[:, 1] >= limits[1, 0])
            logi_i = np.logical_and(logi_i, locs[:, 1] < limits[1, 1])
            ## Decide region
            for j in np.where(logi_i)[0]:
                if regionid[j] == -1:
                    if self.borders.intersects(Point(locs[j, :])):
                        regionid[j] = self.regions_id[i]
        return regionid

    def map_regionid2regionlocs(self, regions=None):
        """Function which maps the regions ID to their most representative
        location.

        Parameters
        ----------
        locs: np.ndarray, shape (n, 2)
            the locations for which we want to obtain their region given that
            discretization.

        Returns
        -------
        regionlocs: array_like, shape (n, 2)
            the region locations of the each assigned location region.

        """
        if regions is None:
            n_dim = np.array(self.borders[0].centroid).shape[0]
            n = len(self.borders)
            regionlocs = np.zeros((n, n_dim))
            for i in xrange(n):
                regionlocs[i, :] = np.array(self.borders[i].centroid)
        else:
            regionlocs = np.zeros((regions.shape[0],
                                   self.regionslocs.shape[1]))
            for i in xrange(regions.shape):
                idx = np.where(self.regions_id == regions[i])[0][0]
                regionlocs[i, :] = self.regionlocs[idx, :]
        return regionlocs

    def map_locs2regionlocs(self, locs):
        """Map locations to regionlocs.

        Parameters
        ----------
        locs: array_like, shape (n, 2)
            the locations.

        Returns
        -------
        regionlocs: array_like, shape (n, 2)
            the region locations of the each assigned location region.

        """
        regionid = self.map_loc2regionid(locs)
        regionlocs = self.map_regionid2regionlocs(regionid)
        return regionlocs

    def map2aggloc_spec(self, locs):
        agglocs = np.zeros(self.regionlocs.shape).astype(float)
        regions = self.map_loc2regionid(locs)
        # Average between all the locs circles
        u_regions = regions.unique()
        for i in xrange(u_regions.shape[0]):
            logi = (regions == u_regions[i])
            agglocs[i, :] = np.mean(locs[logi, :], axis=0)
        return agglocs

    def _compute_limits(self, region_id=None):
        """Compute the limits of the whole system or of a specific region_id.

        Parameters
        ----------
        region_id: integer or None (default)
            the region id information.

        Returns
        -------
        limits: array_like
            the limits information.

        """
        if region_id is None:
            whole = ops.cascaded_union(self.borders)
            limits = np.array(whole.bounds).reshape((2, 2)).T
        else:
            idxs = np.where(self.regions_id == region_id)[0]
            reg = ops.cascaded_union([self.borders[i] for i in idxs])
            limits = np.array(reg.bounds).reshape((2, 2)).T
        return limits

    ################################ TODO: ###############################
    ##########################################################################
    def compute_contiguity_geom(self, limits):
        "Compute which regions are contiguous and returns a graph."
        # TODO:
        raise Exception("Not implemented function yet.")
        ## Obtain random points around all the r_points
        ## Compute the two nearest points with different region_id
        ## Remove repeated pairs
        return

def fit_polygondiscretizer(regionlocs, regions_id):
    polygons = tesselation(regionlocs)
    i_r = match_regions(polygons, regionlocs)
    regionsid = regions_id[i_r]
    return IrregularSpatialDisc(polygons, regionsid)

"""
Properties to compute
---------------------
* Intelligent pairs
Ovelaps of the bounding boxes
* Disjoint discretization
Intelligent selection of pairs and polygon1.intersection(polygon2).area == 0
or polygon1.overlaps(polygon2)
* Contiguous_geom
Intelligent selection of pairs and polygon1.intersection(polygon2).area == 0

"""

max_stars_repo_name: tgquintela/pySpatialTools

max_issues_repo_name: tgquintela/pySpatialTools

max_forks_repo_name: tgquintela/pySpatialTools


text:
import torch
from math import sqrt
import numpy as np
from copy import deepcopy

class Stationary:
    """Here we follow the structure of GPy to build a sub class of stationary kernel. All the classes (i.e. the class
    of stationary kernels) derived from this class use the scaled distance to compute the Gram matrix."""

    def __init__(self, lengthscale=1., outputscale=1., **kwargs):
        super(Stationary, self).__init__(**kwargs)
        self.lengthscale = lengthscale
        self._gram = None
        self._train = None
        self.outputscale = outputscale

    def forward(self, x1, x2=None, l=None, **params):
        if l is not None:
            return _scaled_distance(l, x1, x2)
        return _scaled_distance(self.lengthscale, x1, x2)

    def fit_transform(self, x1, l=None, rebuild_model=True, save_gram_matrix=True, ):
        if not rebuild_model and self._gram is not None:
            return self._gram
        K = self.forward(x1, l=l)
        if save_gram_matrix:
            self._train = deepcopy(x1)
            self._gram = K.clone()
        return K

    def transform(self, x1, l=None, ):
        if self._gram is None:
            raise ValueError("The kernel has not been fitted. Run fit_transform first")
        return self.forward(self._train, x1, l=l)

    def __call__(self, *args, **kwargs):
        return self.forward(*args, **kwargs)

    def forward_t(self, x2, x1=None, l=None):
        if x1 is None:
            x1 = torch.tensor(self._train)
        x2 = torch.tensor(x2).requires_grad_(True)
        K = self.forward(x1, x2, l)
        return K, x2

class RBFKernel(Stationary):
    def forward(self, x1, x2=None, l=None, **kwargs):
        if l is None:
            dist_sq = _scaled_distance(self.lengthscale, x1, x2, sq_dist=True)
        else:
            dist_sq = _scaled_distance(l, x1, x2, sq_dist=True)
        if isinstance(dist_sq, torch.Tensor):
            return self.outputscale * torch.exp(-0.5 * dist_sq)
        return self.outputscale * np.exp(-0.5 * dist_sq)

class LayeredRBFKernel(RBFKernel):
    """
    Same as the conventional RBF kernel, but adapted in a way as a midway between spherical RBF and ARD RBF. In this
    case, one weight is assigned to each Weisfiler-Lehman iteration only (e.g. one weight for h=0, another for h=1 and
    etc.)
    """

    def forward(self, ard_dims, x1, x2=None, l=None, **kwargs):
        l = l if l is not None else self.lengthscale
        assert l.shape[0] == ard_dims.shape[0], "LayeredRBF expects the lengthscale vector to have the same " \
                                                "dimensionality as the " \
                                                "number of WL iterations, but got lengthscale vector of shape" + str(
            l.shape[0]) + \
                                                "and WL iteration of shape " + str(ard_dims.shape[0])
        if not isinstance(ard_dims, torch.Tensor):
            ard_dims = torch.tensor(ard_dims)
        M = torch.cat([torch.ones(int(ard_dims[i])) * l[i] for i in range(ard_dims.shape[0])])
        return super(LayeredRBFKernel, self).forward(x1, x2, M, **kwargs)

class Matern32Kernel(Stationary):
    def forward(self, x1, x2=None, l=None, **kwargs):
        if l is None:
            dist = _scaled_distance(self.lengthscale, x1, x2, )
        else:
            dist = _scaled_distance(l, x1, x2, )
        if isinstance(dist, torch.Tensor):
            return self.outputscale * (1 + sqrt(3.) * dist) * torch.exp(-sqrt(3.) * dist)
        return self.outputscale * (1 + sqrt(3.) * dist) * np.exp(-sqrt(3.) * dist)

class Matern52Kernel(Stationary):
    def forward(self, x1, x2=None, l=None, **kwargs):
        if l is None:
            dist = _scaled_distance(self.lengthscale, x1, x2, )
        else:
            dist = _scaled_distance(l, x1, x2, )
        sq_dist = dist ** 2
        if isinstance(dist, torch.Tensor):
            return self.outputscale * (1 + sqrt(5.) * dist + 5. / 3. * sq_dist) * torch.exp(-sqrt(5.) * dist)
        return self.outputscale * (1 + sqrt(5.) * dist + 5. / 3. * sq_dist) * np.exp(-sqrt(5.) * dist)

class RationalQuadraticKernel(Stationary):
    def __init__(self, lengthscale, outputscale=1., power=2., **kwargs):
        super(RationalQuadraticKernel, self).__init__(lengthscale, outputscale, **kwargs)
        self.power = power

    def forward(self, x1, x2=None, **kwargs):
        dist_sq = _scaled_distance(self.lengthscale, x1, x2, sq_dist=True)
        return self.outputscale * (1 + dist_sq / 2.) ** (-self.power)

def _unscaled_distance(X, X2=None, sq_dist=False):
    """The unscaled distance between X and X2. if x2 is not supplied, then the squared Euclidean distance is
    computed within X"""
    if isinstance(X, torch.Tensor):
        assert X.ndimension() == 2
        if X2 is not None:
            assert isinstance(X2, torch.Tensor)
            assert X2.ndimension() == 2
        if X2 is None:
            Xsq = torch.sum(X ** 2, 1)
            r2 = -2 * X @ X.t() + Xsq[:, None] + Xsq[None, :]
        else:
            X1sq = torch.sum(X ** 2, 1)
            X2sq = torch.sum(X2 ** 2, 1)
            r2 = -2 * X @ X2.t() + X1sq[:, None] + X2sq[None, :]
        r2 += 1e-8
        if not sq_dist:
            r2 = torch.sqrt(r2)
    else:
        assert X.ndim == 2
        if X2 is not None:
            assert X2.ndim == 2
        if X2 is None:
            Xsq = np.sum(X ** 2, 1)
            r2 = -2 * X @ X.transpose() + Xsq[:, None] + Xsq[None, :]
        else:
            X1sq = np.sum(X ** 2, 1)
            X2sq = np.sum(X2 ** 2, 1)
            r2 = -2 * X @ X2.transpose() + X1sq[:, None] + X2sq[None, :]
        if not sq_dist:
            r2 = np.sqrt(r2)
    return r2

def _scaled_distance(lengthscale, X, X2=None, sq_dist=False):
    """Compute the *scaled* distance between X and x2 (or, if X2 is not supplied, the distance between X and itself)
    by the lengthscale.
    if a scalar (float) or a dim=1 lengthscale vector is supplied, then it is assumed that we use one lengthscale for
    all dimensions. Otherwise we have an ARD kernel and in which case the length of the lengthscale vector must be
    the same as the dimensionality of the problem."""
    if X2 is None: X2 = X
    if isinstance(lengthscale, float) or len(lengthscale) == 1:
        return _unscaled_distance(X, X2) / lengthscale if sq_dist is False else \
            _unscaled_distance(X, X2, sq_dist=True) / (lengthscale ** 2)
    else:
        # ARD kernel - one lengthscale per dimension
        assert len(lengthscale) == X.shape[1], "For a non-scaler theta, it needs to be of the same length as the dim" \
                                               " of the input data, but got input dim of " + str(X.shape[1]) + \
                                               " and lengthscale dimension of " + str(lengthscale.shape[0])
        dist = _unscaled_distance(X / lengthscale, X2 / lengthscale)
        return dist if not sq_dist else dist ** 2

max_stars_repo_name: xingchenwan/nasbowl

max_issues_repo_name: xingchenwan/nasbowl

max_forks_repo_name: xingchenwan/nasbowl


text:
#dependencies (numpy, matplotlib, and keras)
import numpy as np
from keras.layers import Input, Dense, Lambda
from keras.models import Model
from keras import backend as K
from keras import objectives
import cv2
import os
from PIL import Image
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import sys
sys.path.append("modules")
from load_model import load_image
import argparse

parser = argparse.ArgumentParser()
help_ = "Label to refer to the model and all related output files by"
parser.add_argument("-l", "--label", help=help_)
help_ = "Number of epochs to train for"
parser.add_argument("-e", "--epochs", help=help_, type=int)
help_ = "Number of layers in the model"
parser.add_argument("-n", "--layers", help=help_, type=int)
args = parser.parse_args()

#hyperparameters
batch_size = 1
original_dim = 16384
latent_dim = 10
#intermediate_dim = 256
decrease_factor = 4 # Factor by which the size of dense layers decreases each layer
nb_epoch = args.epochs
n_layers = args.layers
epsilon_std = 1.0
keyword = args.label

#encoder
x = Input(batch_shape = (batch_size, original_dim))
dim = original_dim
dimstore = []
for i in range(n_layers):
    dim = dim // decrease_factor
    dimstore.append(dim)
    h = Dense(dim, activation = 'relu')(x if i == 0 else h)
dimstore.append(latent_dim)
#h = Dense(intermediate_dim, activation = 'relu')(x)
z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim, activation = "softmax")(h)

print(z_mean)
print(z_log_var)

def sampling(args):
    z_mean, z_log_var = args
    epsilon = K.random_normal(shape = (batch_size, latent_dim), mean = 0.)
    return z_mean + K.exp(z_log_var/2) * epsilon

#note that "output_shape" isn't necessarily with the Tensorflow backend
z = Lambda (sampling, output_shape = (latent_dim,))([z_mean, z_log_var])

#latent hidden state

print(z)

#decoder
dimstore = list(reversed(dimstore))
print(dimstore, "DIMSTORE")
for i in range(n_layers + 1):
    decoder_h = Dense(dimstore[i], activation = 'relu')(z if i == 0 else decoder_h)
#decoder_h = Dense(intermediate_dim, activation = 'relu')
#decoder_mean = Dense(original_dim, activation = 'sigmoid')
#h_decoded = decoder_h(z)
x_decoded_mean = Dense(original_dim, activation = 'sigmoid')(decoder_h)#decoder_mean(h_decoded)

print(x_decoded_mean)

#TODO: Make loss function stop returning nan
def vae_loss(x, x_decoded_mean):
    xent_loss = original_dim * objectives.binary_crossentropy(x, x_decoded_mean)
    kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
    return xent_loss + kl_loss

vae = Model(x, x_decoded_mean)
print(vae.summary())
vae.compile(optimizer='rmsprop', loss=vae_loss) #loss="binary_crossentropy")

filenames = ["images/" + o for o in os.listdir("images") if o.endswith(".jpg") and (not o.startswith("."))]

#x_train = np.array([np.asarray(cv2.imread(o)) for o in tqdm(filenames)])
#print(x_train[0])
X = np.asarray([load_image(o, resize=(128,128)) for o in tqdm(filenames)])

X = X.astype('float32') / 256.
#X = np.load("data/images_over_256.npy")
X = X.reshape((len(X), np.prod(X.shape[1:])))

x_train, x_test = train_test_split(X, test_size = 0.0) #TODO: Change test size back to 0.5

vae.fit(x_train, x_train, shuffle = True, epochs = nb_epoch, batch_size = batch_size, validation_data = (x_test, x_test), verbose = 1)

encoder = Model(x, (z_mean, z_log_var))

encoder_json = encoder.to_json()
json_file = open("models/encoder.%s.json" % keyword, "w")
json_file.write(encoder_json)
json_file.close()
encoder.save_weights("models/encoder.%s.h5" % keyword)

decoder = Model((z_mean, z_log_var), x_decoded_mean)

decoder_json = decoder.to_json()
json_file = open("models/decoder.%s.json" % keyword, "w")
json_file.write(decoder_json)
json_file.close()
decoder.save_weights("models/decoder.%s.h5" % keyword)

max_stars_repo_name: jonmarty/Roman-Coin-Image-Search

max_issues_repo_name: jonmarty/Roman-Coin-Image-Search

max_forks_repo_name: jonmarty/Roman-Coin-Image-Search


text:
import os
import glob
import numpy as np
import pandas as pd
from PIL import Image
from torch.utils.data import Dataset
import torch

class HeatDataset(Dataset):
    """dataset for heat sensor."""

    def __init__(self, root_dir, transform=None):
        self.root_dir = root_dir
        self.transform = transform
        self.heat_data_labels = {}

        heat_files = os.listdir(os.path.join(root_dir))
        for heat_file in heat_files:
            if not heat_file.split(".")[-1] == "txt":
                continue
            name = heat_file[:-4]
            label_folder = name + "_small_images_label_indexed"
            if not os.path.exists(os.path.join(root_dir, label_folder)):
                continue
            self.heat_data_labels[name] = os.listdir(
                os.path.join(root_dir, label_folder))

    def load_heat_data(self, path, index):
        line = open(path).readlines()[index]
        heat_data = line.split()[-1].split(",")[-64:]
        heat_data = np.array(heat_data, dtype=int)
        heat_data = heat_data.reshape((8, 8))
        heat_data = np.rot90(heat_data)
        return heat_data

    def load_label(self, path):
        mask = np.asarray(Image.open(path))
        mask = np.equal.outer(np.unique(mask), mask).view(
            'i1')
        # Skip first "class" which is nothing
        return mask[1:]

    def get_subsets(self):
        return [*self.heat_data_labels.keys()]

    def __len__(self):
        return sum([len(entries) for entries in self.heat_data_labels.values()])

    def __getitem__(self, idx):
        if idx >= len(self):
            raise IndexError("Index out of range.")
        acc_counter = 0
        for name_id, files in self.heat_data_labels.items():
            if idx < (acc_counter + len(files)):
                data = self.load_heat_data(os.path.join(
                    self.root_dir, name_id + ".txt"), idx-acc_counter)
                file_name = files[idx-acc_counter]
                label = self.load_label(os.path.join(
                    self.root_dir, name_id + "_small_images_label_indexed", file_name))
                break
            acc_counter += len(files)

        if self.transform:
            data = self.transform(data)
            label = self.transform(label)
        else:
            data = torch.as_tensor(data.copy(), dtype=torch.float32)
            label = torch.as_tensor(label, dtype=torch.uint8)

        # Get bounding box
        areas = [np.where(layer == 1) for layer in label]
        boxes = [[np.min(a[1]), np.min(a[0]), np.max(a[1])+1, np.max(a[0])+1] for a in areas]
        boxes = torch.FloatTensor(boxes)

        return (data, {
            "image_id": torch.tensor(idx, dtype=torch.int64),
            "boxes": boxes,
            "labels": torch.tensor([0 for _ in label], dtype=torch.int64),
            "masks": label,
        })

max_stars_repo_name: zimonitrome/PyTorch-Simple-MaskRCNN

max_issues_repo_name: zimonitrome/PyTorch-Simple-MaskRCNN

max_forks_repo_name: zimonitrome/PyTorch-Simple-MaskRCNN


text:
import os, re, sys, logging

import numpy as np
import subprocess as sp
import cPickle as pickle

from pdb import set_trace

sys.path.insert(0, os.path.dirname(os.getcwd()))
from Core.CoreSystem import CoreHash, CoreGotoh

class clsParameter(object):

    def __init__(self):

        if len(sys.argv) > 1:
            self.strForwardFqPath = sys.argv[1]
            self.strReverseFqPath = sys.argv[2]
            self.strRefFa         = sys.argv[3]
            self.strPair          = sys.argv[4]
            self.floOg            = float(sys.argv[5])
            self.floOe            = float(sys.argv[6])
            self.intInsertionWin  = int(sys.argv[7])
            self.intDeletionWin   = int(sys.argv[8])
            self.strPamType       = sys.argv[9].upper()  ## Cpf1, Cas9
            self.strBarcodePamPos = sys.argv[10]  ## PAM - BARCODE type (reverse) or BARCODE - PAM type (forward)
            self.intQualCutoff    = int(sys.argv[11])
            self.strOutputdir     = sys.argv[12]
            self.strLogPath       = sys.argv[13]
            self.strEDNAFULL      = os.path.abspath('../EDNAFULL')

        else:
            sManual = """
            Usage:

            python2.7 ./indel_search_ver1.0.py splitted_input_1.fq splitted_input_2.fq reference.fa

            splitted_input_1.fq : forward
            splitted_input_2.fq : reverse

            Total FASTQ(fq) lines / 4 = remainder 0.
            """
            print(sManual)
            sys.exit()

class clsFastqOpener(object):

    def __init__(self, InstParameter):

        self.strForwardFqPath = InstParameter.strForwardFqPath
        self.strReverseFqPath = InstParameter.strReverseFqPath

    def OpenFastqForward(self):

        listFastqForward = []
        listStore        = []

        with open(self.strForwardFqPath) as Fastq1:

            for i, strRow in enumerate(Fastq1):

                i = i + 1
                strRow = strRow.replace('\n', '').upper()

                if i % 4 == 1 or i % 4 == 2:
                    listStore.append(strRow)
                elif i % 4 == 0:
                    listQual = [ord(i) - 33 for i in strRow]
                    listStore.append(listQual)
                    listFastqForward.append(tuple(listStore))
                    listStore = []

        return listFastqForward

    def OpenFastqReverse(self):

        listFastqReverse = []
        listStore        = []

        dictRev = {'A': 'T', 'C': 'G', 'G': 'C', 'T': 'A', 'N': 'N'}

        #with open('./6_AsD0_2_small_test.fq') as fa_2:
        with open(self.strReverseFqPath) as Fastq2:

            for i, strRow in enumerate(Fastq2):
                i = i + 1
                strRow = strRow.replace('\n', '').upper()

                if i % 4 == 1:
                    listStore.append(strRow)
                elif i % 4 == 2:
                    listStore.append(''.join([dictRev[strNucle] for strNucle in strRow[::-1]]))
                elif i % 4 == 0:
                    listQual = [ord(i) - 33 for i in strRow][::-1]
                    listStore.append(listQual)
                    listFastqReverse.append(tuple(listStore))
                    listStore = []

        return listFastqReverse
        #end1: return
    #end: def

class clsIndelSearchParser(object):

    def __init__(self, InstParameter):

        # index name, constant variable.
        self.intNumOfTotal = 0
        self.intNumOfIns   = 1
        self.intNumOfDel   = 2
        self.intNumofCom   = 3
        self.intTotalFastq = 4
        self.intInsFastq   = 5
        self.intDelFastq   = 6
        self.intComFastq   = 7
        self.intIndelInfo  = 8

        self.strRefFa        = InstParameter.strRefFa
        self.floOg           = InstParameter.floOg
        self.floOe           = InstParameter.floOe
        self.strEDNAFULL     = InstParameter.strEDNAFULL
        self.strPamType      = InstParameter.strPamType
        self.intInsertionWin = InstParameter.intInsertionWin
        self.intDeletionWin  = InstParameter.intDeletionWin
        self.intQualCutoff   = InstParameter.intQualCutoff

    def SearchBarcodeIndelPosition(self, sBarcode_PAM_pos):

        dRef    = {}
        dResult = {}

        with open(self.strRefFa) as Ref:

            sBarcode       = ""
            sTarget_region = ""
            intBarcodeLen  = 0

            for i, sRow in enumerate(Ref):

                if i % 2 == 0: ## >CGCTCTACGTAGACA:CTCTATTACTCGCCCCACCTCCCCCAGCCC
                    sBarcode, sTarget_region, intBarcodeLen = self._SeperateFaHeader(sRow, sBarcode, sTarget_region,
                                                                                intBarcodeLen, sBarcode_PAM_pos)

                elif i % 2 != 0: ## AGCATCGATCAGCTACGATCGATCGATCACTAGCTACGATCGATCA
                    sRef_seq, iIndel_start_pos, iIndel_end_pos = self._SearchIndelPos(sRow, sBarcode_PAM_pos, sTarget_region)

                    try:
                        self._MakeRefAndResultTemplate(sRef_seq, sBarcode, iIndel_start_pos, iIndel_end_pos,
                                                       sTarget_region, dRef, dResult)
                    except ValueError:
                        continue

        assert len(dRef.keys()) == len(dResult.keys())

        return dRef, dResult
        # end1: return

    def _SeperateFaHeader(self, sRow, sBarcode, sTarget_region, intBarcodeLen, sBarcode_PAM_pos):

        #      barcode               target region
        # >CGCTCTACGTAGACA:CTCTATTACTCGCCCCACCTCCCCCAGCCC
        sBarcode_indel_seq = sRow.strip().replace('\n', '').replace('\r', '').split(':')
        sBarcode           = sBarcode_indel_seq[0].replace('>', '')

        if intBarcodeLen > 0:
            assert intBarcodeLen == len(sBarcode), 'All of the barcode lengths must be same.'
        intBarcodeLen = len(sBarcode)

        sTarget_region = sBarcode_indel_seq[1]

        ## Reverse the sentence. If it is done, all methods are same before work.
        if sBarcode_PAM_pos == 'Reverse':
            sBarcode = sBarcode[::-1]
            sTarget_region = sTarget_region[::-1]

        return (sBarcode, sTarget_region, intBarcodeLen)

    def _SearchIndelPos(self, sRow, sBarcode_PAM_pos, sTarget_region):

        sRef_seq = sRow.strip().replace('\n', '').replace('\r', '')

        if sBarcode_PAM_pos == 'Reverse':
            sRef_seq = sRef_seq[::-1]

        Seq_matcher = re.compile(r'(?=(%s))' % sTarget_region)
        # iIndel_start_pos       = sRef_seq.index(sTarget_region)               # There is possible to exist two indel.
        iIndel_start_pos = Seq_matcher.finditer(sRef_seq)

        for i, match in enumerate(iIndel_start_pos):
            iIndel_start_pos = match.start()
        # print iIndel_start_pos
        # print len(sTarget_region)
        # print sRef_seq
        iIndel_end_pos = iIndel_start_pos + len(sTarget_region) - 1

        return (sRef_seq, iIndel_start_pos, iIndel_end_pos)

    def _MakeRefAndResultTemplate(self, sRef_seq, sBarcode, iIndel_start_pos,
                                 iIndel_end_pos, sTarget_region, dRef, dResult):
        iBarcode_start_pos = sRef_seq.index(sBarcode)

        # if iIndel_start_pos <= iBarcode_start_pos:
        #    print(iIndel_start_pos, iBarcode_start_pos)
        #    raise IndexError('indel is before barcode')

        iBarcode_end_pos       = iBarcode_start_pos + len(sBarcode) - 1
        sRef_seq_after_barcode = sRef_seq[iBarcode_end_pos + 1:]

        # modified. to -1
        iIndel_end_next_pos_from_barcode_end   = iIndel_end_pos - iBarcode_end_pos - 1
        iIndel_start_next_pos_from_barcode_end = iIndel_start_pos - iBarcode_end_pos - 1

        #  "barcode"-------------*(N) that distance.
        #          ^  ^            ^
        #   *NNNN*NNNN
        #    ^    ^     indel pos, the sequence matcher selects indel event pos front of it.

        ## Result
        dRef[sBarcode] = (sRef_seq, sTarget_region, sRef_seq_after_barcode, iIndel_start_next_pos_from_barcode_end,
                          iIndel_end_next_pos_from_barcode_end, iIndel_start_pos, iIndel_end_pos)  # total matched reads, insertion, deletion, complex
        dResult[sBarcode] = [0, 0, 0, 0, [], [], [], [], []]

    def SearchIndel(self, lFASTQ=[], dRef = {}, dResult={}, sBarcode_PAM_pos=""):

        # lFASTQ : [(seq, qual),(seq, qual)]
        # lRef   : [(ref_seq, ref_seq_after_barcode, barcode, barcode end pos, indel end pos, indel from barcode),(...)]
        # dResult = [# of total, # of ins, # of del, # of com, [total FASTQ], [ins FASTQ], [del FASTQ], [com FASTQ]]
        iCount = 0
        intBarcodeLen = len(dRef.keys()[0])
        #print('intBarcodeLen', intBarcodeLen)

        InstGotoh = CoreGotoh(strEDNAFULL=self.strEDNAFULL, floOg=self.floOg, floOe=self.floOe)

        for lCol_FASTQ in lFASTQ:
            sName = lCol_FASTQ[0]
            if sBarcode_PAM_pos == 'Reverse':
                sSeq  = lCol_FASTQ[1][::-1]
                lQual = lCol_FASTQ[2][::-1]
            else:
                sSeq  = lCol_FASTQ[1]
                lQual = lCol_FASTQ[2]

            assert isinstance(sName, str) and isinstance(sSeq, str) and isinstance(lQual, list)

            listSeqWindow = CoreHash.MakeHashTable(sSeq, intBarcodeLen)

            iBarcode_matched = 0
            iInsert_count    = 0
            iDelete_count    = 0
            iComplex_count   = 0

            intFirstBarcode  = 0 ## check whether a barcode is one in a sequence.

            for strSeqWindow in listSeqWindow:

                if intFirstBarcode == 1: break ## A second barcode in a sequence is not considerable.

                try:
                    lCol_ref, sBarcode, intFirstBarcode = CoreHash.IndexHashTable(dRef, strSeqWindow, intFirstBarcode)
                except KeyError:
                    continue

                sRef_seq                      = lCol_ref[0]
                sTarget_region                = lCol_ref[1]
                iIndel_seq_len                = len(sTarget_region)
                sRef_seq_after_barcode        = lCol_ref[2]
                iIndel_start_from_barcode_pos = lCol_ref[3]
                iIndel_end_from_barcode_pos   = lCol_ref[4]
                try:
                    if self.strPamType == 'CAS9':
                        iKbp_front_Indel_end = iIndel_end_from_barcode_pos - 6  ## cas9:-6, cpf1:-4
                    elif self.strPamType == 'CAF1':
                        iKbp_front_Indel_end = iIndel_end_from_barcode_pos - 4  ## NN(N)*NNN(N)*NNNN
                except Exception:
                    set_trace()

                """
                                                     *     ^ : iIndel_end_from_barcode_pos
                                  GGCG   TCGCTCATGTACCTCCCGT
                TATAGTCTGTCATGCGATGGCG---TCGCTCATGTACCTCCCGTTACAGCCACAAAGCAGGA
                     *
                GGCGTC GCTCATGTACCTCCCGT
                  6          17
                """

                ## bug fix
                if sBarcode == "": continue

                (sSeq, iBarcode_matched, sQuery_seq_after_barcode, lQuery_qual_after_barcode) = \
                    self._CheckBarcodePosAndRemove(sSeq, sBarcode, iBarcode_matched, lQual)

                ## Alignment Seq to Ref
                npGapIncentive = InstGotoh.GapIncentive(sRef_seq_after_barcode)

                try:
                    lResult = InstGotoh.RunCRISPResso2(sQuery_seq_after_barcode.upper(),
                                                       sRef_seq_after_barcode.upper(),
                                                       npGapIncentive)
                except Exception as e:
                    logging.error(e, exc_info=True)
                    continue

                sQuery_needle_ori = lResult[0]
                sRef_needle_ori   = lResult[1]

                sRef_needle, sQuery_needle            = self._TrimRedundantSideAlignment(sRef_needle_ori, sQuery_needle_ori)
                lInsertion_in_read, lDeletion_in_read = self._MakeIndelPosInfo(sRef_needle, sQuery_needle)

                # print 'sQuery_needle', sQuery_needle
                # print 'lInsertion_in_read: onebase', lInsertion_in_read
                # print 'lDeletion_in_read: onebase', lDeletion_in_read
                # print 'i5bp_front_Indel_end', i5bp_front_Indel_end
                # print 'iIndel_end_from_barcode_pos', iIndel_end_from_barcode_pos

                lTarget_indel_result = []  # ['20M2I', '23M3D' ...]

                iInsert_count = self._TakeInsertionFromAlignment(lInsertion_in_read, iKbp_front_Indel_end, lTarget_indel_result,
                                                                 iIndel_end_from_barcode_pos, iInsert_count)

                iDelete_count = self._TakeDeletionFromAlignment(lDeletion_in_read, iKbp_front_Indel_end, lTarget_indel_result,
                                                                iIndel_end_from_barcode_pos, iDelete_count)

                if iInsert_count == 1 and iDelete_count == 1:
                    iComplex_count = 1
                    iInsert_count = 0
                    iDelete_count = 0

                # """ test set
                # print 'sBarcode', sBarcode
                # print 'sTarget_region', sTarget_region
                # print 'sRef_seq_after_barcode', sRef_seq_after_barcode
                # print 'sSeq_after_barcode', sQuery_seq
                # print 'iIndel_start_from_barcode_pos', iIndel_start_from_barcode_pos
                # print 'iIndel_end_from_barcode_pos', iIndel_end_from_barcode_pos
                # """

                listResultFASTQ = self._MakeAndStoreQuality(sName, sSeq, lQual, dResult, sBarcode)

                """
                iQual_end_pos + 1 is not correct, because the position is like this.
                *NNNN*(N)
                So, '+ 1' is removed.
                Howerver, seqeunce inspects until (N) position. indel is detected front of *(N).
                """
                ################################################################
                #print(lTarget_indel_result)
                #set_trace()
                # len(sQuery_seq_after_barcode) == len(lQuery_qual_after_barcode)
                if np.mean(lQuery_qual_after_barcode[iIndel_start_from_barcode_pos : iIndel_end_from_barcode_pos + 1]) >= self.intQualCutoff: ## Quality cutoff

                    """
                    23M3I
                    23M is included junk_seq after barcode,

                    barcorde  junk   targetseq   others
                    *********ACCCT-------------ACACACACC
                    so should select target region.
                    If junk seq is removed by target region seq index pos.
                    """
                    # filter start,
                    iTarget_start_from_barcode   = sRef_seq_after_barcode.index(sTarget_region)
                    lTrimmed_target_indel_result = self._FixPos(lTarget_indel_result, iTarget_start_from_barcode)

                    # print 'Check'
                    # print sRef_seq_after_barcode
                    # print sQuery_seq_after_barcode
                    # print lTrimmed_target_indel_result
                    # print('Trimmed', lTrimmed_target_indel_result)

                    sRef_seq_after_barcode, sQuery_seq_after_barcode = self._StoreToDictResult(sRef_seq_after_barcode, sQuery_seq_after_barcode, iTarget_start_from_barcode,
                                                                       dResult, sBarcode, lTrimmed_target_indel_result, sTarget_region, sRef_needle_ori,
                                                                       sQuery_needle_ori, iInsert_count, iDelete_count, iComplex_count, listResultFASTQ)
                else:
                    iInsert_count  = 0
                    iDelete_count  = 0
                    iComplex_count = 0

                # total matched reads, insertion, deletion, complex
                dResult[sBarcode][self.intNumOfTotal] += iBarcode_matched
                dResult[sBarcode][self.intNumOfIns] += iInsert_count
                dResult[sBarcode][self.intNumOfDel] += iDelete_count
                dResult[sBarcode][self.intNumofCom] += iComplex_count

                iBarcode_matched = 0
                iInsert_count    = 0
                iDelete_count    = 0
                iComplex_count   = 0

            #End:for
        #END:for
        return dResult

    def _CheckBarcodePosAndRemove(self, sSeq, sBarcode, iBarcode_matched, lQual):

        # Check the barcode pos and remove it.
        sSeq = sSeq.replace('\r', '')
        iBarcode_start_pos_FASTQ = sSeq.index(sBarcode)
        iBarcode_matched += 1
        iBarcode_end_pos_FASTQ = iBarcode_start_pos_FASTQ + len(sBarcode) - 1

        """
            junk seq  target region
        ref: AGGAG    AGAGAGAGAGA
        que: AGGAG    AGAGAGAGAGA
        But, It doesnt know where is the target region because of existed indels.
        So, There is no way not to include it.
        """
        # Use this.
        sQuery_seq_after_barcode = sSeq[iBarcode_end_pos_FASTQ + 1:]
        lQuery_qual_after_barcode = lQual[iBarcode_end_pos_FASTQ:]

        return (sSeq, iBarcode_matched, sQuery_seq_after_barcode, lQuery_qual_after_barcode)

    def _TrimRedundantSideAlignment(self, sRef_needle_ori, sQuery_needle_ori):

        # detach forward ---, backward ---
        # e.g.    ref   ------AAAGGCTACGATCTGCG------
        #         query AAAAAAAAATCGCTCTCGCTCTCCGATCT
        # trimmed ref         AAAGGCTACGATCTGCG
        # trimmed qeury       AAATCGCTCTCGCTCTC
        iReal_ref_needle_start = 0
        iReal_ref_needle_end = len(sRef_needle_ori)
        iRef_needle_len = len(sRef_needle_ori)

        for i, sRef_nucle in enumerate(sRef_needle_ori):
            if sRef_nucle in ['A', 'C', 'G', 'T']:
                iReal_ref_needle_start = i
                break

        for i, sRef_nucle in enumerate(sRef_needle_ori[::-1]):
            if sRef_nucle in ['A', 'C', 'G', 'T']:
                iReal_ref_needle_end = iRef_needle_len - (i + 1)
                # forward 0 1 2  len : 3
                # reverse 2 1 0,  len - (2 + 1) = 0
                break

        sRef_needle = sRef_needle_ori[iReal_ref_needle_start:iReal_ref_needle_end + 1]
        if iReal_ref_needle_start:
            sQuery_needle = sQuery_needle_ori[:iReal_ref_needle_end]
        sQuery_needle = sQuery_needle_ori[:len(sRef_needle)]
        # detaching completion
        return (sRef_needle, sQuery_needle)

    def _MakeIndelPosInfo(self, sRef_needle, sQuery_needle):

        # indel info making.
        iNeedle_match_pos_ref   = 0
        iNeedle_match_pos_query = 0
        iNeedle_insertion       = 0
        iNeedle_deletion        = 0

        lInsertion_in_read = []  # insertion result [[100, 1], [119, 13]]
        lDeletion_in_read  = []  # deletion result  [[97, 1], [102, 3]]

        # print 'sRef_needle', sRef_needle
        # print 'sQuery_needle', sQuery_needle
        for i, (sRef_nucle, sQuery_nucle) in enumerate(zip(sRef_needle, sQuery_needle)):

            if sRef_nucle == '-':
                iNeedle_insertion += 1

            if sQuery_nucle == '-':
                iNeedle_deletion += 1

            if sRef_nucle in ['A', 'C', 'G', 'T']:
                if iNeedle_insertion:
                    lInsertion_in_read.append([iNeedle_match_pos_ref, iNeedle_insertion])
                    iNeedle_insertion = 0
                iNeedle_match_pos_ref += 1

            if sQuery_nucle in ['A', 'C', 'G', 'T']:
                if iNeedle_deletion:
                    lDeletion_in_read.append([iNeedle_match_pos_query, iNeedle_deletion])
                    iNeedle_match_pos_query += iNeedle_deletion
                    iNeedle_deletion = 0
                iNeedle_match_pos_query += 1
                # print 'sRef_needle', sRef_needle

        return (lInsertion_in_read, lDeletion_in_read)

    def _TakeInsertionFromAlignment(self, lInsertion_in_read, iKbp_front_Indel_end, lTarget_indel_result,
                                    iIndel_end_from_barcode_pos, iInsert_count):
        """
        ins case
        ...............................NNNNNNNNNNNNNN....NNNNNNNNNNNNNNNNNNN*NNNNNAGCTT
        """
        for iMatch_pos, iInsertion_pos in lInsertion_in_read:
            if self.strPamType == 'CAS9':
                # if i5bp_front_Indel_end == iMatch_pos -1 or iIndel_end_from_barcode_pos == iMatch_pos -1: # iMatch_pos is one base # original ver
                if iKbp_front_Indel_end - self.intInsertionWin <= iMatch_pos - 1 <= iKbp_front_Indel_end + self.intInsertionWin:  # iMatch_pos is one base
                    iInsert_count = 1
                    lTarget_indel_result.append(str(iMatch_pos) + 'M' + str(iInsertion_pos) + 'I')

            elif self.strPamType == 'CPF1':
                if iKbp_front_Indel_end - self.intInsertionWin <= iMatch_pos - 1 <= iKbp_front_Indel_end + self.intInsertionWin or \
                        iIndel_end_from_barcode_pos - self.intInsertionWin <= iMatch_pos - 1 <= iIndel_end_from_barcode_pos + self.intInsertionWin:  # iMatch_pos is one base
                    iInsert_count = 1
                    lTarget_indel_result.append(str(iMatch_pos) + 'M' + str(iInsertion_pos) + 'I')

        return iInsert_count

    def _TakeDeletionFromAlignment(self, lDeletion_in_read, iKbp_front_Indel_end, lTarget_indel_result,
                                   iIndel_end_from_barcode_pos, iDelete_count):

        """
        del case 1
        ...............................NNNNNNNNNNNNNN....NNNNNNNNNNNNNNNNNNNNN**NNNAGCTT
        del case 2
        ...............................NNNNNNNNNNNNNN....NNNNNNNNNNNNNNNNNNNNN**NNNNNCTT
        """
        for iMatch_pos, iDeletion_pos in lDeletion_in_read:
            """
            Insertion: 30M3I
                   ^
            ACGT---ACGT
            ACGTTTTACGT -> check this seq
            Insertion just check two position

            Deletion: 30M3D
                 ^
            ACGTTTTACGT
            ACGT---ACGT -> check this seq
            But deletion has to includes overlap deletion.
            """
            if self.strPamType == 'CAS9':
                if (iMatch_pos - self.intDeletionWin - 1 <= iKbp_front_Indel_end and iKbp_front_Indel_end < (iMatch_pos + iDeletion_pos + self.intDeletionWin - 1)):
                    iDelete_count = 1
                    lTarget_indel_result.append(str(iMatch_pos) + 'M' + str(iDeletion_pos) + 'D')
            elif self.strPamType == 'CPF1':
                if (iMatch_pos - self.intDeletionWin - 1 <= iKbp_front_Indel_end and iKbp_front_Indel_end < (iMatch_pos + iDeletion_pos + self.intDeletionWin - 1)) or \
                   (iMatch_pos - self.intDeletionWin - 1 <= iIndel_end_from_barcode_pos and iIndel_end_from_barcode_pos < (iMatch_pos + iDeletion_pos + self.intDeletionWin - 1)):
                    iDelete_count = 1
                    lTarget_indel_result.append(str(iMatch_pos) + 'M' + str(iDeletion_pos) + 'D')

        return iDelete_count

    def _MakeAndStoreQuality(self, sName, sSeq, lQual, dResult, sBarcode):
        listResultFASTQ = [sName, sSeq, '+', ''.join(chr(i + 33) for i in lQual)]
        dResult[sBarcode][self.intTotalFastq].append(listResultFASTQ)
        return listResultFASTQ

    def _FixPos(self, lTarget_indel_result, iTarget_start_from_barcode):

        lTrimmed_target_indel_result = []

        for sINDEL in lTarget_indel_result:
            # B - A is not included B position, so +1
            iMatch_target_start = int(sINDEL.split('M')[0]) - iTarget_start_from_barcode
            """ This part determines a deletion range.
                                      ^ current match pos
            AGCTACGATCAGCATCTGACTTACTTC[barcode]

                           ^ fix the match start at here. (target region)
            AGCTACGATCAGCATC TGACTTACTTC[barcode]

            if iMatch_target_start < 0:
                sContinue = 1

            But, this method has some problems.

                           ^ barcode start
            AGCTACGATCAGCAT*********C[barcode]
            Like this pattern doesn't seleted. because, deletion checking is begun the target region start position.
            Thus, I have fixed this problem.
            """

            if iMatch_target_start <= -(iTarget_start_from_barcode):
                # print(iMatch_target_start, iTarget_start_from_barcode)
                continue

            lTrimmed_target_indel_result.append(str(iMatch_target_start) + 'M' + sINDEL.split('M')[1])
        # filter end
        return lTrimmed_target_indel_result

    def _StoreToDictResult(self, sRef_seq_after_barcode, sQuery_seq_after_barcode, iTarget_start_from_barcode,
                           dResult, sBarcode, lTrimmed_target_indel_result, sTarget_region, sRef_needle_ori, sQuery_needle_ori,
                           iInsert_count, iDelete_count, iComplex_count, listResultFASTQ):

        sRef_seq_after_barcode   = sRef_seq_after_barcode[iTarget_start_from_barcode:]
        sQuery_seq_after_barcode = sQuery_seq_after_barcode[iTarget_start_from_barcode:]

        dResult[sBarcode][self.intIndelInfo].append([sRef_seq_after_barcode, sQuery_seq_after_barcode, lTrimmed_target_indel_result,
                                                     sTarget_region, sRef_needle_ori, sQuery_needle_ori])
        if iInsert_count:
            dResult[sBarcode][self.intInsFastq].append(listResultFASTQ)
        elif iDelete_count:
            dResult[sBarcode][self.intDelFastq].append(listResultFASTQ)
        elif iComplex_count:
            dResult[sBarcode][self.intComFastq].append(listResultFASTQ)

        return (sRef_seq_after_barcode, sQuery_seq_after_barcode)

    def CalculateIndelFrequency(self, dResult):
        dResult_INDEL_freq = {}

        for sBarcode, lValue in dResult.items():  # lValue[gINDEL_info] : [[sRef_seq_after_barcode, sQuery_seq_after_barcode, lTarget_indel_result, sTarget_region], ..])
            sRef_seq_loop = ''
            llINDEL_store = []  # ['ACAGACAGA', ['20M2I', '23M3D']]
            dINDEL_freq   = {}

            if lValue[self.intIndelInfo]:
                for sRef_seq_loop, sQuery_seq, lINDEL, sTarget_region, sRef_needle, sQuery_needle in lValue[self.intIndelInfo]: # llINDEL : [['20M2I', '23M3D'], ...]
                    # print 'lINDEL', lINDEL
                    for sINDEL in lINDEL:
                        llINDEL_store.append([sQuery_seq, sINDEL, sRef_needle, sQuery_needle])

                iTotal = len([lINDEL for sQuery_seq, lINDEL, sRef_needle, sQuery_needle in llINDEL_store])

                for sQuery_seq, sINDEL, sRef_needle, sQuery_needle in llINDEL_store:
                    dINDEL_freq[sINDEL] = [[], 0, [], []]

                for sQuery_seq, sINDEL, sRef_needle, sQuery_needle in llINDEL_store:
                    dINDEL_freq[sINDEL][1] += 1
                    dINDEL_freq[sINDEL][0].append(sQuery_seq)
                    dINDEL_freq[sINDEL][2].append(sRef_needle)
                    dINDEL_freq[sINDEL][3].append(sQuery_needle)

                for sINDEL in dINDEL_freq:
                    lQuery        = dINDEL_freq[sINDEL][0]
                    iFreq         = dINDEL_freq[sINDEL][1]
                    lRef_needle   = dINDEL_freq[sINDEL][2]
                    lQuery_needle = dINDEL_freq[sINDEL][3]

                    try:
                        dResult_INDEL_freq[sBarcode].append([sRef_seq_loop, lQuery, sINDEL, float(iFreq) / iTotal,
                                                             sTarget_region, lRef_needle, lQuery_needle])
                    except (KeyError, TypeError, AttributeError) as e:
                        dResult_INDEL_freq[sBarcode] = []
                        dResult_INDEL_freq[sBarcode].append([sRef_seq_loop, lQuery, sINDEL, float(iFreq) / iTotal,
                                                             sTarget_region, lRef_needle, lQuery_needle])
            # end: if lValue[gINDEL_info]
        # end: for sBarcode, lValue
        return dResult_INDEL_freq
        # end1: return
    # end: def
#END:class

class clsOutputMaker(object):

    def __init__(self, InstParameter):

        self.strOutputdir     = InstParameter.strOutputdir
        self.strForwardFqPath = InstParameter.strForwardFqPath

    def MakePickleOutput(self, dictResult, dictResultIndelFreq, strBarcodePamPos=''):

        dictOutput = {'dictResult': dictResult,
                      'dictResultIndelFreq': dictResultIndelFreq,
                      'strBarcodePamPos': strBarcodePamPos}

        with open('{outdir}/Tmp/Pickle/{fq}.pickle'.format(outdir=self.strOutputdir, fq=os.path.basename(self.strForwardFqPath)), 'wb') as Pickle:
            pickle.dump(dictOutput, Pickle)

def Main():

    InstParameter = clsParameter()
    logging.basicConfig(format='%(process)d %(levelname)s %(asctime)s : %(message)s',
                        level=logging.DEBUG,
                        filename=InstParameter.strLogPath,
                        filemode='a')

    logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

    logging.info('Program start : %s' % InstParameter.strForwardFqPath)

    logging.info('File Open')
    InstFileOpen     = clsFastqOpener(InstParameter)
    listFastqForward = InstFileOpen.OpenFastqForward()
    if InstParameter.strPair == 'True':
        listFastqReverse = InstFileOpen.OpenFastqReverse()

    InstIndelSearch = clsIndelSearchParser(InstParameter)

    InstOutput = clsOutputMaker(InstParameter)

    if InstParameter.strPamType == 'CPF1':
        logging.info('Search barcode INDEL pos')
        dRef, dResult = InstIndelSearch.SearchBarcodeIndelPosition(InstParameter.strBarcodePamPos)  # ref check.

        logging.info('Search INDEL forward')
        dResultForward = InstIndelSearch.SearchIndel(listFastqForward, dRef, dResult)

        if InstParameter.strPair == 'True':
            logging.info('Search INDEL reverse')
            dResultReverse = InstIndelSearch.SearchIndel(listFastqReverse, dRef, dResultForward)

            logging.info('Calculate INDEL frequency')
            dictResultIndelFreq = InstIndelSearch.CalculateIndelFrequency(dResultReverse)

            logging.info('Make pickle output forward')
            InstOutput.MakePickleOutput(dResultReverse, dictResultIndelFreq)

        else:
            logging.info('Calculate INDEL frequency')
            dictResultIndelFreq = InstIndelSearch.CalculateIndelFrequency(dResultForward)

            logging.info('Make pickle output forward')
            InstOutput.MakePickleOutput(dResultForward, dictResultIndelFreq)

    elif InstParameter.strPamType == 'CAS9':
        logging.info('Search barcode INDEL pos')
        dRef, dResult   = InstIndelSearch.SearchBarcodeIndelPosition(InstParameter.strBarcodePamPos)
        logging.info('Search INDEL')
        dResult_forward = InstIndelSearch.SearchIndel(listFastqForward, dRef, dResult, InstParameter.strBarcodePamPos)
        logging.info('Calculate INDEL frequency')
        dResult_INDEL_freq = InstIndelSearch.CalculateIndelFrequency(dResult_forward)

        logging.info('Make pickle output forward')
        InstOutput.MakePickleOutput(dResult_forward, dResult_INDEL_freq, InstParameter.strBarcodePamPos)

    logging.info('Program end : %s' % InstParameter.strForwardFqPath)
#END:def

if __name__ == '__main__':
    Main()