<commit_before># -*- coding: utf-8 -*-

"""
Datashape normalization. This handles Ellipses and broadcasting.
"""

from itertools import chain
from collections import defaultdict, deque

from blaze import error
from . import transform, tzip
from .coretypes import DataShape, Ellipsis, Fixed, CType

#------------------------------------------------------------------------
# Normalization
#------------------------------------------------------------------------

def normalize(constraints, broadcasting=None):
    """
    Parameters
    ----------

    constraints : [(DataShape, DataShape)]
        List of constraints (datashape type equations)
    broadcasting: [bool]
        indicates for each constraint whether the two DataShapes broadcast

    Returns: (constraints, broadcast_env)
        A two-tuple containing a list of normalized constraints and a
        broadcasting environment listing all type variables which may
        broadcast together.
    """
    broadcasting_env = None
    result = [normalize_simple(a, b) for a, b in constraints]
    return result, broadcasting_env

def normalize_simple(a, b):
    if isinstance(a, (CType, DataShape)) and isinstance(a, (CType, DataShape)):
        if (type(a), type(b)) == (DataShape, DataShape):
            a, b = normalize_ellipses(a, b)
        a, b = normalize_broadcasting(a, b)
    else:
        a, b = tzip(normalize_simple, a, b)

    return a, b

#------------------------------------------------------------------------
# DataShape Normalizers
#------------------------------------------------------------------------

def normalize_ellipses(a, b):
    """Eliminate ellipses in DataShape"""
    S = _normalize_ellipses(a, b)
    return substitute(S, a), substitute(S, b)

def normalize_broadcasting(a, b):
    """Add broadcasting dimensions to DataShapes"""
    return _normalize_broadcasting(a, b)

#------------------------------------------------------------------------
# Ellipses<commit_msg>Handle 0d-array/scalar type unification mismatch in normalization phase<commit_after># -*- coding: utf-8 -*-

"""
Datashape normalization. This handles Ellipses and broadcasting.
"""

from itertools import chain
from collections import defaultdict, deque

from blaze import error
from . import transform, tzip
from .coretypes import DataShape, Ellipsis, Fixed, CType

#------------------------------------------------------------------------
# Normalization
#------------------------------------------------------------------------

def normalize(constraints, broadcasting=None):
    """
    Parameters
    ----------

    constraints : [(DataShape, DataShape)]
        List of constraints (datashape type equations)
    broadcasting: [bool]
        indicates for each constraint whether the two DataShapes broadcast

    Returns: (constraints, broadcast_env)
        A two-tuple containing a list of normalized constraints and a
        broadcasting environment listing all type variables which may
        broadcast together.
    """
    broadcasting_env = None
    result = [normalize_simple(a, b) for a, b in constraints]
    return result, broadcasting_env

def normalize_simple(a, b):
    # Normalize (CType, DataShape) pairs
    if (type(a), type(b)) == (CType, DataShape):
        a = DataShape(a)
    if (type(a), type(b)) == (DataShape, CType):
        b = DataShape(b)

    # Normalize ellipses and broadcasting
    if (type(a), type(b)) == (DataShape, DataShape):
        if (len(a.parameters), len(b.parameters)) != (1, 1):
            a, b = normalize_ellipses(a, b)
        a, b = normalize_broadcasting(a, b)
    else:
        a, b = tzip(normalize_simple, a, b)

    return a, b

#------------------------------------------------------------------------
# DataShape Normalizers
#------------------------------------------------------------------------

def normalize_ellipses(a, b):
    """Eliminate ellipses in DataShape"""
    S = _normalize_ellipses(a, b)
    return substitute(S, a), substitute(S, b)

def normalize_broadcasting(a, b):
    """Add broadcasting dimensions to DataShapes"""
    return _normalize_broadcasting(a, b)

#------------------------------------------------------------------------
# Ellipses
{'fraction_non_alphanumeric': 0.25910064239828695, 'fraction_numerical': 0.0011896264572924102, 'mean_word_length': 1.774917491749175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080052', 'n_tokens_mistral': 1081, 'n_tokens_neox': 1044, 'n_words': 339}