<commit_before>"""
Methods for performing binary set operations on IndexSets.

"direct" methods refer to building a new 2d array of locations that is the
result of a binary set operation. They are always lexicographically sorted
and contain only unique rows. These methods return nothing else.

"contrib" methods also build the final locations, but they additionally
provide information that indicates which locations from the parent IndexSets
contributed to the result. This is used for merging the data associated with
the input IndexSets.

"indirect" Like contrib, these provide the final result locations and extra
information. Indirect gives the locations in the final array that come from
a given IndexSet. They are the size of the final array with the number of
True values equal to the number of values in the IndexSet.
"""
import numpy as np
import numba
from concurrent.futures import ThreadPoolExecutor

from ...settings import parse_chunk_args
from ...chunk import gen_cochunks
from ...indexset import IndexSet, concat_sorted_nonoverlapping
from ...flags import SORTED, UNIQUE
from ...data import all_short_circuit_merges

from ._intersection_contrib import (_intersection_contrib_1d,
  _intersection_contrib_2d, _intersection_contrib_3d,
  _intersection_contrib_4d, _intersection_contrib_Nd)

from ._intersection_direct import (_intersection_direct_1d,
  _intersection_direct_2d, _intersection_direct_3d,
  _intersection_direct_4d, _intersection_direct_Nd)


from ...data import merge_data_direct


@numba.njit(nogil=True)
def intersection_(a, b, MERGE=None):
    """ Find the locations common to both a and b, merging data if necessary.
    
    Arguments
    ---------
    a, b : IndexSet instances
    MERGE : array of uint8 | None
        Per-dimension flag indicating how to merge data from locations present
        in both IndexSets. This affects the resulting data even if one of the
        inputs has None for data.
        
        `MERGE` may have a single value, in which case the merge action is
        applied to all data columns, or have a length equal to the number of
        data columns, each data column is merged according to the
        corresponding flag. This argument must be an array.
    
    Returns
    -------
    IndexSet with the coordinates that are in both a and b. Data in the
    overlap will be merged according to MERGE flags.
    
    Notes
    -----
    The following flags are can be used in the MERGE array:
    * 2 DATA_NANSUM
        Add all data in corresponding locations, treating NaNs as zero
    * 3 DATA_SUM
        Add all data in corresponding locations, propagating NaNs.
    * 4 DATA_NANMAX
        Take the maximum contributing datum for each location, ignoring
        NaN values. If all contributors are NaN, the result is NaN.
    * 5 DATA_MAX
        Take the maximum datum at each location, propagating NaNs. If any
        contributing data are NaN, then the result is NaN
    * 6 _BLAHB_DATA_NANMIN
        Take the minimum contributing datum for each location, ignoring
        NaN values. If all contributors are NaN, the result is NaN.
    * 7 _BLAHB_DATA_MIN
        Take the minimum datum at each location, propagating NaNs. If any
        contributing data are NaN, then the result is NaN
    * 8 DATA_NANFIRST
        Find the first non-NaN data from the contributing IndexSets, in
        order of their input. If all data are NaN, then the result is NaN.
    * 10 DATA_NANLAST
        Find the last non-NaN data from the contributing IndexSets, in
        order of their input. If all data are NaN, then the result is NaN.
    
    The default merge mode is _BLAHB_DATA_FIRST will be applied if None
    is given instead of an array.
    """
    if a.is_empty:
        return a
    elif b.is_empty:
        return b
    <commit_msg>Changes to data-flag name<commit_after>"""
Methods for performing binary set operations on IndexSets.

"direct" methods refer to building a new 2d array of locations that is the
result of a binary set operation. They are always lexicographically sorted
and contain only unique rows. These methods return nothing else.

"contrib" methods also build the final locations, but they additionally
provide information that indicates which locations from the parent IndexSets
contributed to the result. This is used for merging the data associated with
the input IndexSets.

"indirect" Like contrib, these provide the final result locations and extra
information. Indirect gives the locations in the final array that come from
a given IndexSet. They are the size of the final array with the number of
True values equal to the number of values in the IndexSet.
"""
import numpy as np
import numba
from concurrent.futures import ThreadPoolExecutor

from ...settings import parse_chunk_args
from ...chunk import gen_cochunks
from ...indexset import IndexSet, concat_sorted_nonoverlapping
from ...flags import SORTED, UNIQUE
from ...data import all_short_circuit_merges

from ._intersection_contrib import (_intersection_contrib_1d,
  _intersection_contrib_2d, _intersection_contrib_3d,
  _intersection_contrib_4d, _intersection_contrib_Nd)

from ._intersection_direct import (_intersection_direct_1d,
  _intersection_direct_2d, _intersection_direct_3d,
  _intersection_direct_4d, _intersection_direct_Nd)


from ...data import merge_data_direct


@numba.njit(nogil=True)
def intersection_(a, b, MERGE=None):
    """ Find the locations common to both a and b, merging data if necessary.
    
    Arguments
    ---------
    a, b : IndexSet instances
    MERGE : array of uint8 | None
        Per-dimension flag indicating how to merge data from locations present
        in both IndexSets. This affects the resulting data even if one of the
        inputs has None for data.
        
        `MERGE` may have a single value, in which case the merge action is
        applied to all data columns, or have a length equal to the number of
        data columns, each data column is merged according to the
        corresponding flag. This argument must be an array.
    
    Returns
    -------
    IndexSet with the coordinates that are in both a and b. Data in the
    overlap will be merged according to MERGE flags.
    
    Notes
    -----
    The following flags are can be used in the MERGE array:
    * 2 DATA_NANSUM
        Add all data in corresponding locations, treating NaNs as zero
    * 3 DATA_SUM
        Add all data in corresponding locations, propagating NaNs.
    * 4 DATA_NANMAX
        Take the maximum contributing datum for each location, ignoring
        NaN values. If all contributors are NaN, the result is NaN.
    * 5 DATA_MAX
        Take the maximum datum at each location, propagating NaNs. If any
        contributing data are NaN, then the result is NaN
    * 6 DATA_NANMIN
        Take the minimum contributing datum for each location, ignoring
        NaN values. If all contributors are NaN, the result is NaN.
    * 7 DATA_MIN
        Take the minimum datum at each location, propagating NaNs. If any
        contributing data are NaN, then the result is NaN
    * 8 DATA_NANFIRST
        Find the first non-NaN data from the contributing IndexSets, in
        order of their input. If all data are NaN, then the result is NaN.
    * 10 DATA_NANLAST
        Find the last non-NaN data from the contributing IndexSets, in
        order of their input. If all data are NaN, then the result is NaN.
    
    The default merge mode is _BLAHB_DATA_FIRST will be applied if None
    is given instead of an array.
    """
    if a.is_empty:
        return a
    elif b.is_empty:
        return b
    
{'fraction_non_alphanumeric': 0.04152475766308619, 'fraction_numerical': 0.004977731202515065, 'mean_word_length': 3.085072231139647, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233125', 'n_tokens_mistral': 2054, 'n_tokens_neox': 1971, 'n_words': 1011}