<commit_before>import builtins
import functools
import math
import re
from abc import ABC, abstractmethod
from enum import Enum
from threading import RLock
from types import TracebackType
from typing import (
    Any,
    Callable,
    ContextManager,
    Dict,
    Generic,
    Iterable,
    Iterator,
    List,
    Optional,
    Protocol,
    Sized,
    Tuple,
    Type,
    TypeVar,
    Union,
    cast,
)
from urllib.parse import quote, unquote


class SupportsLessThan(Protocol):
    @abstractmethod
    def __lt__(self, __other: Any) -> bool:
        raise NotImplementedError


_T = TypeVar("_T")
_TSupportsLessThan = TypeVar("_TSupportsLessThan", bound=SupportsLessThan)


class ObjectDisposedException(Exception):
    def __init__(self):
        super().__init__("Cannot access a disposed object")


class IDisposable:
    @abstractmethod
    def Dispose(self) -> None:
        ...

    def __enter__(self):
        """Enter context management."""
        return self

    def __exit__(
        self,
        exctype: Optional[Type[BaseException]],
        excinst: Optional[BaseException],
        exctb: Optional[TracebackType],
    ) -> bool:
        """Exit context management."""

        self.Dispose()
        return False

    @staticmethod
    def create(action: Callable[[], None]):
        """Create disposable from action. Will call action when
        disposed."""
        return AnonymousDisposable(action)


Disposable = TypeVar("Disposable", bound=IDisposable)


class AnonymousDisposable(IDisposable):
    def __init__(self, action: Callable[[], None]):
        self._is_disposed = False
        self._action = action
        self._lock = RLock()

    def Dispose(self) -> None:
        """Performs the task of cleaning up resources."""

        dispose = False
        with self._lock:
            if not self._is_disposed:
                dispose = True
                self._is_disposed = True

        if dispose:
            self._action()

    def __enter__(self):
        if self._is_disposed:
            raise ObjectDisposedException()
        return self


class IEquatable(ABC):
    def GetHashCode(self):
        return hash(self)

    @abstractmethod
    def __eq__(self, other: Any) -> bool:
        return NotImplemented

    @abstractmethod
    def __hash__(self) -> int:
        raise NotImplementedError


class IComparable(IEquatable):
    def CompareTo(self, other: Any) -> int:
        if self < other:
            return -1
        elif self == other:
            return 0
        return 1

    @abstractmethod
    def __lt__(self, other: Any) -> bool:
        raise NotImplementedError


class IComparer(Generic[_T]):
    """Defines a method that a type implements to compare two objects.

    https://docs.microsoft.com/en-us/dotnet/api/system.collections.generic.icomparer-1
    """

    @abstractmethod
    def Compare(self, y: _T) -> int:
        ...


class IEqualityComparer(Generic[_T]):
    def Equals(self, y: _T) -> bool:
        return self == y

    def GetHashCode(self) -> int:
        return hash(self)

    @abstractmethod
    def __eq__(self, other: Any) -> bool:
        return NotImplemented

    @abstractmethod
    def __hash__(self) -> int:
        raise NotImplementedError


class DateKind(Enum):
    Unspecified = 0
    UTC = 1
    Local = 2


def equals(a: Any, b: Any) -> bool:
    return a == b


def is_comparable(x: Any) -> bool:
    return hasattr(x, "CompareTo") and callable(x.CompareTo)


def compare_dicts(x: Dict[str, Any], y: dict[str, Any]) -> int:
    """Compare Python dicts with string keys.

    Python cannot do this natively.
    """
    x_keys = x.keys()
    y_keys = y.keys()

    if len(x_keys) != len(y_keys):
        return -1 if len(x_keys) < len(y_keys) else 1

    x_keys = sorted(x_keys)
    y_keys = sorted(y_keys)

    j = 0
    for i, key in enumerate(x_keys):
        if key != y_keys[i]:
            return -1 if key < y_keys[i] else 1
        else:
            j = compare(x[key], y[key])
            if j != 0:
                return j

    return 0


def compare_arrays(xs: List[Any], ys: List[Any]):<commit_msg>fix for py38 (#2711)<commit_after>import builtins
import functools
import math
import re
from abc import ABC, abstractmethod
from enum import Enum
from threading import RLock
from types import TracebackType
from typing import (
    Any,
    Callable,
    ContextManager,
    Dict,
    Generic,
    Iterable,
    Iterator,
    List,
    Optional,
    Protocol,
    Sized,
    Tuple,
    Type,
    TypeVar,
    Union,
    cast,
)
from urllib.parse import quote, unquote


class SupportsLessThan(Protocol):
    @abstractmethod
    def __lt__(self, __other: Any) -> bool:
        raise NotImplementedError


_T = TypeVar("_T")
_TSupportsLessThan = TypeVar("_TSupportsLessThan", bound=SupportsLessThan)


class ObjectDisposedException(Exception):
    def __init__(self):
        super().__init__("Cannot access a disposed object")


class IDisposable:
    @abstractmethod
    def Dispose(self) -> None:
        ...

    def __enter__(self):
        """Enter context management."""
        return self

    def __exit__(
        self,
        exctype: Optional[Type[BaseException]],
        excinst: Optional[BaseException],
        exctb: Optional[TracebackType],
    ) -> bool:
        """Exit context management."""

        self.Dispose()
        return False

    @staticmethod
    def create(action: Callable[[], None]):
        """Create disposable from action. Will call action when
        disposed."""
        return AnonymousDisposable(action)


Disposable = TypeVar("Disposable", bound=IDisposable)


class AnonymousDisposable(IDisposable):
    def __init__(self, action: Callable[[], None]):
        self._is_disposed = False
        self._action = action
        self._lock = RLock()

    def Dispose(self) -> None:
        """Performs the task of cleaning up resources."""

        dispose = False
        with self._lock:
            if not self._is_disposed:
                dispose = True
                self._is_disposed = True

        if dispose:
            self._action()

    def __enter__(self):
        if self._is_disposed:
            raise ObjectDisposedException()
        return self


class IEquatable(ABC):
    def GetHashCode(self):
        return hash(self)

    @abstractmethod
    def __eq__(self, other: Any) -> bool:
        return NotImplemented

    @abstractmethod
    def __hash__(self) -> int:
        raise NotImplementedError


class IComparable(IEquatable):
    def CompareTo(self, other: Any) -> int:
        if self < other:
            return -1
        elif self == other:
            return 0
        return 1

    @abstractmethod
    def __lt__(self, other: Any) -> bool:
        raise NotImplementedError


class IComparer(Generic[_T]):
    """Defines a method that a type implements to compare two objects.

    https://docs.microsoft.com/en-us/dotnet/api/system.collections.generic.icomparer-1
    """

    @abstractmethod
    def Compare(self, y: _T) -> int:
        ...


class IEqualityComparer(Generic[_T]):
    def Equals(self, y: _T) -> bool:
        return self == y

    def GetHashCode(self) -> int:
        return hash(self)

    @abstractmethod
    def __eq__(self, other: Any) -> bool:
        return NotImplemented

    @abstractmethod
    def __hash__(self) -> int:
        raise NotImplementedError


class DateKind(Enum):
    Unspecified = 0
    UTC = 1
    Local = 2


def equals(a: Any, b: Any) -> bool:
    return a == b


def is_comparable(x: Any) -> bool:
    return hasattr(x, "CompareTo") and callable(x.CompareTo)


def compare_dicts(x: Dict[str, Any], y: Dict[str, Any]) -> int:
    """Compare Python dicts with string keys.

    Python cannot do this natively.
    """
    x_keys = x.keys()
    y_keys = y.keys()

    if len(x_keys) != len(y_keys):
        return -1 if len(x_keys) < len(y_keys) else 1

    x_keys = sorted(x_keys)
    y_keys = sorted(y_keys)

    j = 0
    for i, key in enumerate(x_keys):
        if key != y_keys[i]:
            return -1 if key < y_keys[i] else 1
        else:
            j = compare(x[key], y[key])
            if j != 0:
                return j

    return 0


def compare_arrays(xs: List[Any], ys: List[Any]):
{'fraction_non_alphanumeric': 0.10265422373045692, 'fraction_numerical': 0.004120712640892013, 'mean_word_length': 2.3490259740259742, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6708609', 'n_tokens_mistral': 2623, 'n_tokens_neox': 2467, 'n_words': 720}