<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import utool as ut
import numpy as np
import vtool as vt
import six
import cv2
import itertools
from six.moves import zip, map, range
from functools import partial
#from ibeis_cnn import utils
from ibeis_cnn import draw_results  # NOQA
print, rrr, profile = ut.inject2(__name__, '[ibeis_cnn.ingest_ibeis]')


FIX_HASH = True


def get_aidpairs_partmatch(ibs, acfg_name):
    """

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --exec-get_aidpairs_partmatch

    SeeAlso:
        extract_annotpair_training_chips

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> ibs = ibeis.opendb(defaultdb='PZ_Master1')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_FlankHack')
        >>> acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        ...                             type_=str,
        ...                             default='ctrl:pername=None,excluderef=False,contrib_contains=FlankHack')
        >>> aid_pairs, label_list, flat_metadata = get_aidpairs_partmatch(ibs, acfg_name)
    """
    print('NEW WAY OF FILTERING')
    from ibeis.expt import experiment_helpers
    acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
        ibs, [acfg_name])
    #acfg = acfg_list[0]
    expanded_aids = expanded_aids_list[0]
    qaid_list, daid_list = expanded_aids
    available_aids = np.unique(ut.flatten([qaid_list, daid_list]))
    tup = ibs.partition_annots_into_corresponding_groups(qaid_list, daid_list)
    aids1_list, aids2_list, other_aids1, other_aids2 = tup
    multiton_aids = np.unique(ut.flatten(aids1_list + aids2_list))
    #singletons = ut.flatten(other_aids2 + other_aids1)

    ibs.print_annotconfig_stats(qaid_list, daid_list, bigstr=True)

    # Positive Examples
    print('Sampling positive examples')
    nested_pairs = list(
        map(list, itertools.starmap(ut.iprod, zip(aids1_list, aids2_list))))
    pos_aid_pairs = np.vstack(nested_pairs)
    # Filter Self
    flag_list = pos_aid_pairs.T[0] != pos_aid_pairs.T[1]
    pos_aid_pairs = pos_aid_pairs.compress(flag_list, axis=0)
    # Filter bad viewpoints
    TAU = np.pi * 2
    _np_get_annot_yaws = ut.accepts_numpy(ibs.get_annot_yaws_asfloat.im_func)
    yaw_pairs = _np_get_annot_yaws(ibs, pos_aid_pairs)
    yawdist = vt.ori_distance(yaw_pairs.T[0], yaw_pairs.T[1])
    flag_list = np.logical_or(np.isnan(yawdist), yawdist < TAU / 8.)
    pos_aid_pairs = pos_aid_pairs.compress(flag_list, axis=0)
    #pos_aid_pairs = vt.unique_rows(pos_aid_pairs)  # should be unncessary
    assert len(vt.unique_rows(pos_aid_pairs)) == len(pos_aid_pairs)
    print('pos_aid_pairs.shape = %r' % (pos_aid_pairs.shape,))

    # Hard Negative Examples
    print('Sampling hard negative examples')
    num_hard_neg_per_aid = max(1, len(pos_aid_pairs) // len(multiton_aids))
    cfgdict = {
        'affine_invariance': False,
        'fg_on': not ut.WIN32,
    }
    cm_list, qreq_ = ibs.query_chips(
        qaid_list, daid_list, return_request=True, cfgdict=cfgdict,
        return_cm=True)
    hardneg_aids1 = [[cm.qaid] for cm in (cm_list)]
    hardneg_aids2 = [cm.get_top_gf_aids(ibs, ntop=num_hard_neg_per_aid)
                     for cm in cm_list]
    hardneg_aid_pairs = np.array(ut.flatten(
        itertools.starmap(ut.iprod, zip(hardneg_aids1, hardneg_aids2))))

    # Random Negative Examples
    # TODO: may be able to say not a match from viewpoint?
    print('Sampling random negative examples')
    num_rand_neg_per_aid = max(1, len(pos_aid_pairs) // len(multiton_aids))
    rng = np.random.RandomState(0)
    randneg_aid_pairs = []
    neg_aid_pool = available_aids
    neg_nid_pool = np.array(ibs.get_annot_nids(neg_aid_pool))
    for aid, nid in zip(neg_aid_pool, neg_nid_pool):
        is_valid = np.not_equal(neg_nid_pool, nid)
        p = is_valid / (is_valid.sum())
        chosen = rng.choice(neg_aid_pool, size=num_rand_neg_per_aid,
                            replace=False, p=p)
        chosen_pairs = list(ut.iprod([aid], chosen))
        randneg_aid_pairs.extend(chosen_pairs)
    randneg_aid_pairs = np.array(randneg_aid_pairs)

    # Concatenate both types of negative examples
    #print('Building negative examples')
    #_neg_aid_pairs = np.vstack((hardneg_aid_pairs, randneg_aid_pairs))
    #neg_aid_pairs = vt.unique_rows(_neg_aid_pairs)

    # Unsure Examples
    # TODO: use quality and viewoint labelings to determine this
    # as well as metadata from the annotmatch table
    print('hardneg_aid_pairs.shape = %r' % (hardneg_aid_pairs.shape,))
    print('randneg_aid_pairs.shape = %r' % (randneg_aid_pairs.shape,))
    print('pos_aid_pairs.shape = %r' % (pos_aid_pairs.shape,))

    print('Building labels')
    const = ibs.const
    unflat_pairs = (pos_aid_pairs, hardneg_aid_pairs, randneg_aid_pairs)
    type_labels = (const.TRUTH_MATCH, const.TRUTH_NOT_MATCH,
                   const.TRUTH_NOT_MATCH)
    type_meta_labels = ('pos', 'hardneg', 'randneg')

    def _expand(type_list):
        return [[item] * len(pairs)
                for pairs, item in zip(unflat_pairs, type_list)]

    _aid_pairs = np.vstack(unflat_pairs)
    _labels = np.hstack(_expand(type_labels))

    flat_metadata = {
        'meta_label': ut.flatten(_expand(type_meta_labels))
    }

    print('Filtering Duplicates')
    nonunique_flags = vt.nonunique_row_flags(_aid_pairs)
    print('Filtered %d duplicate pairs' % (nonunique_flags.sum()))
    print('Nonunique stats:')
    for key, val in flat_metadata.items():
        print(ut.dict_hist(ut.list_compress(val, nonunique_flags)))
    unique_flags = ~nonunique_flags
    # Do filtering
    aid_pairs = _aid_pairs.compress(unique_flags, axis=0)
    label_list = _labels.compress(unique_flags, axis=0)
    for key, val in flat_metadata.items():
        flat_metadata[key] = ut.list_compress(val, unique_flags)
    print('Final Stats')
    for key, val in flat_metadata.items():
        print(ut.dict_hist(val))

    groupsizes = map(len, vt.group_indices(aid_pairs.T[0])[1])
    ut.print_dict(ut.dict_hist(groupsizes), 'groupsize freq')
    flat_metadata['aid_pairs'] = aid_pairs
    return aid_pairs, label_list, flat_metadata


def extract_annotpair_training_chips(ibs, aid_pairs, **kwargs):
    """

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --exec-extract_annotpair_training_chips --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> if False:
        >>>     ibs = ibeis.opendb(defaultdb='PZ_Master1')
        >>>     acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        >>>                                 type_=str,
        >>>                                 default='ctrl:pername=None,excluderef=False,contrib_contains=FlankHack')
        >>> else:
        >>>     ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>>     acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        >>>                                  type_=str,
        >>>                                  default='ctrl:pername=None,excluderef=False')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_FlankHack')
        >>> aid_pairs, label_list, flat_metadata = get_aidpairs_partmatch(ibs, acfg_name)
        >>> s = slice(2, len(aid_pairs), len(aid_pairs) // ut.get_argval('--x', type_=int, default=8))
        >>> #aid_pairs = aid_pairs[s]
        >>> #label_list = label_list[s]
        >>> #flat_metadata = dict([(key, val[s]) for key, val in flat_metadata.items()])
        >>> rchip1_list, rchip2_list = extract_annotpair_training_chips(ibs, aid_pairs)
        >>> ut.quit_if_noshow()
        >>> from ibeis_cnn import draw_results  # NOQA
        >>> interact = draw_results.interact_patches(label_list, (rchip1_list, rchip2_list), flat_metadata, chunck_sizes=(2, 2), ibs=ibs)
        >>> ut.show_if_requested()
    """

    # TODO extract chips in a sane manner
    import ibeis.model.hots.vsone_pipeline
    kwargs = kwargs.copy()
    part_chip_width  = kwargs.pop('part_chip_width', 256)
    part_chip_height = kwargs.pop('part_chip_height', 128)
    colorspace = kwargs.pop('colorspace', 'gray')
    assert len(kwargs) == 0, 'unhandled arguments %r' % (kwargs,)

    size = (part_chip_width, part_chip_height)

    #cfgdict = {}
    import ibeis.control.IBEISControl
    import ibeis.model.hots.query_request
    assert isinstance(ibs, ibeis.control.IBEISControl.IBEISController)
    qreq_ = ibs.new_query_request(aid_pairs.T[0][0:1], aid_pairs.T[1][0:1])
    assert isinstance(qreq_, ibeis.model.hots.query_request.QueryRequest)
    qconfig2_ = qreq_.extern_query_config2
    dconfig2_ = qreq_.extern_data_config2

    warped = True

    def compute_alignment(pair_metadata, qreq_=qreq_):
        aid1, aid2 = pair_metadata['aid1'], pair_metadata['aid2']
        print('Computing alignment aidpair=(%r, %r)' % (aid1, aid2))
        matches, match_metadata = ibeis.model.hots.vsone_pipeline.vsone_single(
            aid1, aid2, qreq_, verbose=False)
        match_metadata.clear_stored(['dlen_sqrd2', 'vecs2', 'kpts2', 'kpts1', 'vecs1'])
        return match_metadata

    def make_warped_chips(pair_metadata, size=size):
        match_metadata = pair_metadata['match_metadata']
        print('Warping Chips aidpair=(%r, %r)' % (
            pair_metadata['aid1'], pair_metadata['aid2']))
        rchip1 = match_metadata['rchip1']
        rchip2 = match_metadata['rchip2']
        if warped:
            H1 = match_metadata['H_RAT']
            #print('WARPING')
            # Initial Warping
            wh2 = vt.get_size(rchip2)
            rchip1_ = vt.warpHomog(rchip1, H1, wh2) if H1 is not None else rchip1
            # Cropping to remove parts of the image that (probably) cannot match
            if True:
                isfill = vt.get_pixel_dist(rchip1_, np.array([0, 0, 0])) == 0
                rowslice, colslice = vt.get_crop_slices(isfill)
                rchip1_crop = rchip1_[rowslice, colslice]
                rchip2_crop = rchip2[rowslice, colslice]
            else:
                rchip1_crop = rchip1_
                rchip2_crop = rchip2
            rchip1 = rchip1_crop
            rchip2 = rchip2_crop
            # Make sure match_metadata doesn't take up too much memory
            match_metadata.clear_evaluated()
        # Resize to fit into a neural network
        rchip1_sz = cv2.resize(rchip1, size, interpolation=cv2.INTER_LANCZOS4)
        rchip2_sz = cv2.resize(rchip2, size, interpolation=cv2.INTER_LANCZOS4)
        # hack
        rchip1_sz = vt.convert_image_list_colorspace([rchip1_sz], colorspace)[0]
        rchip2_sz = vt.convert_image_list_colorspace([rchip2_sz], colorspace)[0]
        return (rchip1_sz, rchip2_sz)

    def make_lazy_resize_funcs(pair_metadata):
        tmp_meta = ut.LazyDict(verbose=False)
        tmp_meta['warped_chips'] = partial(make_warped_chips, pair_metadata)
        def lazy_rchip1_sz(tmp_meta=tmp_meta):
            return tmp_meta['warped_chips'][0]
        def lazy_rchip2_sz(tmp_meta=tmp_meta):
            return tmp_meta['warped_chips'][1]
        return lazy_rchip1_sz, lazy_rchip2_sz

    # Compute alignments

    pairmetadata_list = []
    for aid1, aid2 in ut.ProgressIter(aid_pairs, lbl='Align Info', adjust=True):
        pair_metadata = ibs.get_annot_pair_lazy_dict(
            ibs, aid1, aid2, qconfig2_, dconfig2_)
        pair_metadata['match_metadata'] = partial(compute_alignment, pair_metadata)
        pairmetadata_list.append(pair_metadata)

    # Warp the Chips

    rchip1_list = ut.LazyList(verbose=False)
    rchip2_list = ut.LazyList(verbose=False)
    for pair_metadata in ut.ProgressIter(pairmetadata_list, lbl='Building Warped Chips', adjust=True):
        #rchip1_sz, rchip2_sz = make_warped_chips(pair_metadata)
        rchip1_sz, rchip2_sz = make_lazy_resize_funcs(pair_metadata)
        rchip1_list.append(rchip1_sz)
        rchip2_list.append(rchip2_sz)

    return rchip1_list, rchip2_list
    """
    import plottool as pt
    pt.imshow(vt.stack_images(rchip1, rchip2)[0])
    pt.imshow(vt.stack_images(rchip1_sz, rchip2_sz)[0])
    """


def get_aidpair_patchmatch_training_data(ibs, aid1_list, aid2_list,
                                         kpts1_m_list, kpts2_m_list, fm_list,
                                         metadata_lists, patch_size,
                                         colorspace):
    """
    FIXME: errors on get_aidpairs_and_matches(ibs, 1)

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --test-get_aidpair_patchmatch_training_data --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>> tup = get_aidpairs_and_matches(ibs, 6)
        >>> (aid1_list, aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists) = tup
        >>> pmcfg = PatchMetricDataConfig()
        >>> patch_size = pmcfg.patch_size
        >>> colorspace = pmcfg.colorspace
        >>> # execute function
        >>> tup = get_aidpair_patchmatch_training_data(ibs, aid1_list,
        ...     aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists,
        ...     patch_size, colorspace)
        >>> aid1_list_, aid2_list_, warped_patch1_list, warped_patch2_list, flat_metadata = tup
        >>> ut.quit_if_noshow()
        >>> label_list = get_aidpair_training_labels(ibs, aid1_list_, aid2_list_)
        >>> draw_results.interact_patches(label_list, (warped_patch1_list, warped_patch2_list), flat_metadata)
        >>> ut.show_if_requested()
    """
    # Flatten to only apply chip operations once
    print('get_aidpair_patchmatch_training_data num_pairs = %r' % (len(aid1_list)))
    assert len(aid1_list) == len(aid2_list)
    assert len(aid1_list) == len(kpts1_m_list)
    assert len(aid1_list) == len(kpts2_m_list)
    assert len(aid1_list) == len(fm_list)
    print('geting_unflat_chips')
    flat_unique, reconstruct_tup = ut.inverable_unique_two_lists(aid1_list, aid2_list)
    print('grabbing %d unique chips' % (len(flat_unique)))
    chip_list = ibs.get_annot_chips(flat_unique)  # TODO config2_
    # convert to approprate colorspace
    chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
    ut.print_object_size(chip_list, 'chip_list')
    chip1_list, chip2_list = ut.uninvert_unique_two_lists(chip_list, reconstruct_tup)
    print('warping')

    class PatchExtractCache(object):

        def __init__(self, patch_size):
            self.patch_size = patch_size
            self.cache_ = {}

        def idcache_find_misses(self, id_list, cache_):
            # Generalize?
            val_list = [cache_.get(id_, None) for id_ in id_list]
            ismiss_list = [val is None for val in val_list]
            return val_list, ismiss_list

        def idcache_save(self, ismiss_list, miss_vals, id_list, val_list, cache_):
            # Generalize?
            miss_indices = ut.list_where(ismiss_list)
            miss_ids  = ut.filter_items(id_list, ismiss_list)
            # overwrite missed output
            for index, val in zip(miss_indices, miss_vals):
                val_list[index] = val
            # cache save
            for id_, val in zip(miss_ids, miss_vals):
                cache_[id_] = val

        def cacheget_wraped_patches(self, aid, fxs, chip, kpts):
            # +-- Custom ids
            id_list = [(aid, fx) for fx in fxs]
            # L__
            val_list, ismiss_list = self.idcache_find_misses(id_list, self.cache_)
            if any(ismiss_list):
                # +-- Custom evaluate misses
                kpts_miss = kpts.compress(ismiss_list, axis=0)
                miss_vals = vt.get_warped_patches(chip, kpts_miss, patch_size=self.patch_size)[0]
                # L__
                self.idcache_save(ismiss_list, miss_vals, id_list, val_list, self.cache_)
            return val_list

    fx1_list = [fm.T[0] for fm in fm_list]
    fx2_list = [fm.T[1] for fm in fm_list]
    warp_iter1 = ut.ProgressIter(zip(aid1_list, fx1_list, chip1_list,<commit_msg>Made compress the default over filter_items<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
import utool as ut
import numpy as np
import vtool as vt
import six
import cv2
import itertools
from six.moves import zip, map, range
from functools import partial
#from ibeis_cnn import utils
from ibeis_cnn import draw_results  # NOQA
print, rrr, profile = ut.inject2(__name__, '[ibeis_cnn.ingest_ibeis]')


FIX_HASH = True


def get_aidpairs_partmatch(ibs, acfg_name):
    """

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --exec-get_aidpairs_partmatch

    SeeAlso:
        extract_annotpair_training_chips

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> ibs = ibeis.opendb(defaultdb='PZ_Master1')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_FlankHack')
        >>> acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        ...                             type_=str,
        ...                             default='ctrl:pername=None,excluderef=False,contrib_contains=FlankHack')
        >>> aid_pairs, label_list, flat_metadata = get_aidpairs_partmatch(ibs, acfg_name)
    """
    print('NEW WAY OF FILTERING')
    from ibeis.expt import experiment_helpers
    acfg_list, expanded_aids_list = experiment_helpers.get_annotcfg_list(
        ibs, [acfg_name])
    #acfg = acfg_list[0]
    expanded_aids = expanded_aids_list[0]
    qaid_list, daid_list = expanded_aids
    available_aids = np.unique(ut.flatten([qaid_list, daid_list]))
    tup = ibs.partition_annots_into_corresponding_groups(qaid_list, daid_list)
    aids1_list, aids2_list, other_aids1, other_aids2 = tup
    multiton_aids = np.unique(ut.flatten(aids1_list + aids2_list))
    #singletons = ut.flatten(other_aids2 + other_aids1)

    ibs.print_annotconfig_stats(qaid_list, daid_list, bigstr=True)

    # Positive Examples
    print('Sampling positive examples')
    nested_pairs = list(
        map(list, itertools.starmap(ut.iprod, zip(aids1_list, aids2_list))))
    pos_aid_pairs = np.vstack(nested_pairs)
    # Filter Self
    flag_list = pos_aid_pairs.T[0] != pos_aid_pairs.T[1]
    pos_aid_pairs = pos_aid_pairs.compress(flag_list, axis=0)
    # Filter bad viewpoints
    TAU = np.pi * 2
    _np_get_annot_yaws = ut.accepts_numpy(ibs.get_annot_yaws_asfloat.im_func)
    yaw_pairs = _np_get_annot_yaws(ibs, pos_aid_pairs)
    yawdist = vt.ori_distance(yaw_pairs.T[0], yaw_pairs.T[1])
    flag_list = np.logical_or(np.isnan(yawdist), yawdist < TAU / 8.)
    pos_aid_pairs = pos_aid_pairs.compress(flag_list, axis=0)
    #pos_aid_pairs = vt.unique_rows(pos_aid_pairs)  # should be unncessary
    assert len(vt.unique_rows(pos_aid_pairs)) == len(pos_aid_pairs)
    print('pos_aid_pairs.shape = %r' % (pos_aid_pairs.shape,))

    # Hard Negative Examples
    print('Sampling hard negative examples')
    num_hard_neg_per_aid = max(1, len(pos_aid_pairs) // len(multiton_aids))
    cfgdict = {
        'affine_invariance': False,
        'fg_on': not ut.WIN32,
    }
    cm_list, qreq_ = ibs.query_chips(
        qaid_list, daid_list, return_request=True, cfgdict=cfgdict,
        return_cm=True)
    hardneg_aids1 = [[cm.qaid] for cm in (cm_list)]
    hardneg_aids2 = [cm.get_top_gf_aids(ibs, ntop=num_hard_neg_per_aid)
                     for cm in cm_list]
    hardneg_aid_pairs = np.array(ut.flatten(
        itertools.starmap(ut.iprod, zip(hardneg_aids1, hardneg_aids2))))

    # Random Negative Examples
    # TODO: may be able to say not a match from viewpoint?
    print('Sampling random negative examples')
    num_rand_neg_per_aid = max(1, len(pos_aid_pairs) // len(multiton_aids))
    rng = np.random.RandomState(0)
    randneg_aid_pairs = []
    neg_aid_pool = available_aids
    neg_nid_pool = np.array(ibs.get_annot_nids(neg_aid_pool))
    for aid, nid in zip(neg_aid_pool, neg_nid_pool):
        is_valid = np.not_equal(neg_nid_pool, nid)
        p = is_valid / (is_valid.sum())
        chosen = rng.choice(neg_aid_pool, size=num_rand_neg_per_aid,
                            replace=False, p=p)
        chosen_pairs = list(ut.iprod([aid], chosen))
        randneg_aid_pairs.extend(chosen_pairs)
    randneg_aid_pairs = np.array(randneg_aid_pairs)

    # Concatenate both types of negative examples
    #print('Building negative examples')
    #_neg_aid_pairs = np.vstack((hardneg_aid_pairs, randneg_aid_pairs))
    #neg_aid_pairs = vt.unique_rows(_neg_aid_pairs)

    # Unsure Examples
    # TODO: use quality and viewoint labelings to determine this
    # as well as metadata from the annotmatch table
    print('hardneg_aid_pairs.shape = %r' % (hardneg_aid_pairs.shape,))
    print('randneg_aid_pairs.shape = %r' % (randneg_aid_pairs.shape,))
    print('pos_aid_pairs.shape = %r' % (pos_aid_pairs.shape,))

    print('Building labels')
    const = ibs.const
    unflat_pairs = (pos_aid_pairs, hardneg_aid_pairs, randneg_aid_pairs)
    type_labels = (const.TRUTH_MATCH, const.TRUTH_NOT_MATCH,
                   const.TRUTH_NOT_MATCH)
    type_meta_labels = ('pos', 'hardneg', 'randneg')

    def _expand(type_list):
        return [[item] * len(pairs)
                for pairs, item in zip(unflat_pairs, type_list)]

    _aid_pairs = np.vstack(unflat_pairs)
    _labels = np.hstack(_expand(type_labels))

    flat_metadata = {
        'meta_label': ut.flatten(_expand(type_meta_labels))
    }

    print('Filtering Duplicates')
    nonunique_flags = vt.nonunique_row_flags(_aid_pairs)
    print('Filtered %d duplicate pairs' % (nonunique_flags.sum()))
    print('Nonunique stats:')
    for key, val in flat_metadata.items():
        print(ut.dict_hist(ut.list_compress(val, nonunique_flags)))
    unique_flags = ~nonunique_flags
    # Do filtering
    aid_pairs = _aid_pairs.compress(unique_flags, axis=0)
    label_list = _labels.compress(unique_flags, axis=0)
    for key, val in flat_metadata.items():
        flat_metadata[key] = ut.list_compress(val, unique_flags)
    print('Final Stats')
    for key, val in flat_metadata.items():
        print(ut.dict_hist(val))

    groupsizes = map(len, vt.group_indices(aid_pairs.T[0])[1])
    ut.print_dict(ut.dict_hist(groupsizes), 'groupsize freq')
    flat_metadata['aid_pairs'] = aid_pairs
    return aid_pairs, label_list, flat_metadata


def extract_annotpair_training_chips(ibs, aid_pairs, **kwargs):
    """

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --exec-extract_annotpair_training_chips --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> if False:
        >>>     ibs = ibeis.opendb(defaultdb='PZ_Master1')
        >>>     acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        >>>                                 type_=str,
        >>>                                 default='ctrl:pername=None,excluderef=False,contrib_contains=FlankHack')
        >>> else:
        >>>     ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>>     acfg_name = ut.get_argval(('--aidcfg', '--acfg', '-a'),
        >>>                                  type_=str,
        >>>                                  default='ctrl:pername=None,excluderef=False')
        >>> #ibs = ibeis.opendb(defaultdb='PZ_FlankHack')
        >>> aid_pairs, label_list, flat_metadata = get_aidpairs_partmatch(ibs, acfg_name)
        >>> s = slice(2, len(aid_pairs), len(aid_pairs) // ut.get_argval('--x', type_=int, default=8))
        >>> #aid_pairs = aid_pairs[s]
        >>> #label_list = label_list[s]
        >>> #flat_metadata = dict([(key, val[s]) for key, val in flat_metadata.items()])
        >>> rchip1_list, rchip2_list = extract_annotpair_training_chips(ibs, aid_pairs)
        >>> ut.quit_if_noshow()
        >>> from ibeis_cnn import draw_results  # NOQA
        >>> interact = draw_results.interact_patches(label_list, (rchip1_list, rchip2_list), flat_metadata, chunck_sizes=(2, 2), ibs=ibs)
        >>> ut.show_if_requested()
    """

    # TODO extract chips in a sane manner
    import ibeis.model.hots.vsone_pipeline
    kwargs = kwargs.copy()
    part_chip_width  = kwargs.pop('part_chip_width', 256)
    part_chip_height = kwargs.pop('part_chip_height', 128)
    colorspace = kwargs.pop('colorspace', 'gray')
    assert len(kwargs) == 0, 'unhandled arguments %r' % (kwargs,)

    size = (part_chip_width, part_chip_height)

    #cfgdict = {}
    import ibeis.control.IBEISControl
    import ibeis.model.hots.query_request
    assert isinstance(ibs, ibeis.control.IBEISControl.IBEISController)
    qreq_ = ibs.new_query_request(aid_pairs.T[0][0:1], aid_pairs.T[1][0:1])
    assert isinstance(qreq_, ibeis.model.hots.query_request.QueryRequest)
    qconfig2_ = qreq_.extern_query_config2
    dconfig2_ = qreq_.extern_data_config2

    warped = True

    def compute_alignment(pair_metadata, qreq_=qreq_):
        aid1, aid2 = pair_metadata['aid1'], pair_metadata['aid2']
        print('Computing alignment aidpair=(%r, %r)' % (aid1, aid2))
        matches, match_metadata = ibeis.model.hots.vsone_pipeline.vsone_single(
            aid1, aid2, qreq_, verbose=False)
        match_metadata.clear_stored(['dlen_sqrd2', 'vecs2', 'kpts2', 'kpts1', 'vecs1'])
        return match_metadata

    def make_warped_chips(pair_metadata, size=size):
        match_metadata = pair_metadata['match_metadata']
        print('Warping Chips aidpair=(%r, %r)' % (
            pair_metadata['aid1'], pair_metadata['aid2']))
        rchip1 = match_metadata['rchip1']
        rchip2 = match_metadata['rchip2']
        if warped:
            H1 = match_metadata['H_RAT']
            #print('WARPING')
            # Initial Warping
            wh2 = vt.get_size(rchip2)
            rchip1_ = vt.warpHomog(rchip1, H1, wh2) if H1 is not None else rchip1
            # Cropping to remove parts of the image that (probably) cannot match
            if True:
                isfill = vt.get_pixel_dist(rchip1_, np.array([0, 0, 0])) == 0
                rowslice, colslice = vt.get_crop_slices(isfill)
                rchip1_crop = rchip1_[rowslice, colslice]
                rchip2_crop = rchip2[rowslice, colslice]
            else:
                rchip1_crop = rchip1_
                rchip2_crop = rchip2
            rchip1 = rchip1_crop
            rchip2 = rchip2_crop
            # Make sure match_metadata doesn't take up too much memory
            match_metadata.clear_evaluated()
        # Resize to fit into a neural network
        rchip1_sz = cv2.resize(rchip1, size, interpolation=cv2.INTER_LANCZOS4)
        rchip2_sz = cv2.resize(rchip2, size, interpolation=cv2.INTER_LANCZOS4)
        # hack
        rchip1_sz = vt.convert_image_list_colorspace([rchip1_sz], colorspace)[0]
        rchip2_sz = vt.convert_image_list_colorspace([rchip2_sz], colorspace)[0]
        return (rchip1_sz, rchip2_sz)

    def make_lazy_resize_funcs(pair_metadata):
        tmp_meta = ut.LazyDict(verbose=False)
        tmp_meta['warped_chips'] = partial(make_warped_chips, pair_metadata)
        def lazy_rchip1_sz(tmp_meta=tmp_meta):
            return tmp_meta['warped_chips'][0]
        def lazy_rchip2_sz(tmp_meta=tmp_meta):
            return tmp_meta['warped_chips'][1]
        return lazy_rchip1_sz, lazy_rchip2_sz

    # Compute alignments

    pairmetadata_list = []
    for aid1, aid2 in ut.ProgressIter(aid_pairs, lbl='Align Info', adjust=True):
        pair_metadata = ibs.get_annot_pair_lazy_dict(
            ibs, aid1, aid2, qconfig2_, dconfig2_)
        pair_metadata['match_metadata'] = partial(compute_alignment, pair_metadata)
        pairmetadata_list.append(pair_metadata)

    # Warp the Chips

    rchip1_list = ut.LazyList(verbose=False)
    rchip2_list = ut.LazyList(verbose=False)
    for pair_metadata in ut.ProgressIter(pairmetadata_list, lbl='Building Warped Chips', adjust=True):
        #rchip1_sz, rchip2_sz = make_warped_chips(pair_metadata)
        rchip1_sz, rchip2_sz = make_lazy_resize_funcs(pair_metadata)
        rchip1_list.append(rchip1_sz)
        rchip2_list.append(rchip2_sz)

    return rchip1_list, rchip2_list
    """
    import plottool as pt
    pt.imshow(vt.stack_images(rchip1, rchip2)[0])
    pt.imshow(vt.stack_images(rchip1_sz, rchip2_sz)[0])
    """


def get_aidpair_patchmatch_training_data(ibs, aid1_list, aid2_list,
                                         kpts1_m_list, kpts2_m_list, fm_list,
                                         metadata_lists, patch_size,
                                         colorspace):
    """
    FIXME: errors on get_aidpairs_and_matches(ibs, 1)

    CommandLine:
        python -m ibeis_cnn.ingest_ibeis --test-get_aidpair_patchmatch_training_data --show

    Example:
        >>> # DISABLE_DOCTEST
        >>> from ibeis_cnn.ingest_ibeis import *  # NOQA
        >>> import ibeis
        >>> # build test data
        >>> ibs = ibeis.opendb(defaultdb='PZ_MTEST')
        >>> tup = get_aidpairs_and_matches(ibs, 6)
        >>> (aid1_list, aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists) = tup
        >>> pmcfg = PatchMetricDataConfig()
        >>> patch_size = pmcfg.patch_size
        >>> colorspace = pmcfg.colorspace
        >>> # execute function
        >>> tup = get_aidpair_patchmatch_training_data(ibs, aid1_list,
        ...     aid2_list, kpts1_m_list, kpts2_m_list, fm_list, metadata_lists,
        ...     patch_size, colorspace)
        >>> aid1_list_, aid2_list_, warped_patch1_list, warped_patch2_list, flat_metadata = tup
        >>> ut.quit_if_noshow()
        >>> label_list = get_aidpair_training_labels(ibs, aid1_list_, aid2_list_)
        >>> draw_results.interact_patches(label_list, (warped_patch1_list, warped_patch2_list), flat_metadata)
        >>> ut.show_if_requested()
    """
    # Flatten to only apply chip operations once
    print('get_aidpair_patchmatch_training_data num_pairs = %r' % (len(aid1_list)))
    assert len(aid1_list) == len(aid2_list)
    assert len(aid1_list) == len(kpts1_m_list)
    assert len(aid1_list) == len(kpts2_m_list)
    assert len(aid1_list) == len(fm_list)
    print('geting_unflat_chips')
    flat_unique, reconstruct_tup = ut.inverable_unique_two_lists(aid1_list, aid2_list)
    print('grabbing %d unique chips' % (len(flat_unique)))
    chip_list = ibs.get_annot_chips(flat_unique)  # TODO config2_
    # convert to approprate colorspace
    chip_list = vt.convert_image_list_colorspace(chip_list, colorspace)
    ut.print_object_size(chip_list, 'chip_list')
    chip1_list, chip2_list = ut.uninvert_unique_two_lists(chip_list, reconstruct_tup)
    print('warping')

    class PatchExtractCache(object):

        def __init__(self, patch_size):
            self.patch_size = patch_size
            self.cache_ = {}

        def idcache_find_misses(self, id_list, cache_):
            # Generalize?
            val_list = [cache_.get(id_, None) for id_ in id_list]
            ismiss_list = [val is None for val in val_list]
            return val_list, ismiss_list

        def idcache_save(self, ismiss_list, miss_vals, id_list, val_list, cache_):
            # Generalize?
            miss_indices = ut.list_where(ismiss_list)
            miss_ids  = ut.list_compress(id_list, ismiss_list)
            # overwrite missed output
            for index, val in zip(miss_indices, miss_vals):
                val_list[index] = val
            # cache save
            for id_, val in zip(miss_ids, miss_vals):
                cache_[id_] = val

        def cacheget_wraped_patches(self, aid, fxs, chip, kpts):
            # +-- Custom ids
            id_list = [(aid, fx) for fx in fxs]
            # L__
            val_list, ismiss_list = self.idcache_find_misses(id_list, self.cache_)
            if any(ismiss_list):
                # +-- Custom evaluate misses
                kpts_miss = kpts.compress(ismiss_list, axis=0)
                miss_vals = vt.get_warped_patches(chip, kpts_miss, patch_size=self.patch_size)[0]
                # L__
                self.idcache_save(ismiss_list, miss_vals, id_list, val_list, self.cache_)
            return val_list

    fx1_list = [fm.T[0] for fm in fm_list]
    fx2_list = [fm.T[1] for fm in fm_list]
    warp_iter1 = ut.ProgressIter(zip(aid1_list, fx1_list, chip1_list,
{'fraction_non_alphanumeric': 0.10814611034693565, 'fraction_numerical': 0.01228794131132508, 'mean_word_length': 2.964614638875424, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 309, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6241959', 'n_tokens_mistral': 11781, 'n_tokens_neox': 11529, 'n_words': 2016}