<commit_before>import collections
import event_model
import itertools
from intake.catalog.utils import RemoteCatalogError
import numpy
import ophyd.sim
import pytest


def normalize(gen):
    """
    Converted any pages to singles.
    """
    for name, doc in gen:
        if name == 'event_page':
            for event in event_model.unpack_event_page(doc):
                yield 'event', event
        elif name == 'datum_page':
            for datum in event_model.unpack_datum_page(doc):
                yield 'datum', datum
        else:
            yield name, doc


def compare(a, b):
    a = normalize(a)
    b = normalize(b)
    a_indexed = {}
    b_indexed = {}
    for name, doc in a:
        if name == 'resource':
            # Check for an extraneous duplicate key in old documents.
            if 'id' in doc:
                assert doc['id'] == doc['uid']
                doc = doc.copy()
                doc.pop('id')
        if name == 'datum':
            a_indexed[('datum', doc['datum_id'])] = doc
        # v0 yields {'_name": 'RunStop'} is the stop doc is missing; v2 yields None.
        elif name == 'stop' and doc is None or 'uid' not in doc:
            a_indexed[(name, None)] = None
        else:
            a_indexed[(name, doc['uid'])] = doc
    for name, doc in b:
        if name == 'resource':
            # Check for an extraneous duplicate key in old documents.
            if 'id' in doc:
                assert doc['id'] == doc['uid']
                doc = doc.copy()
                doc.pop('id')
        if name == 'datum':
            b_indexed[('datum', doc['datum_id'])] = doc
        # v0 yields {'_name": 'RunStop'} is the stop doc is missing; v2 yields None.
        elif name == 'stop' and doc is None or 'uid' not in doc:
            b_indexed[(name, None)] = None
        else:
            b_indexed[(name, doc['uid'])] = doc
    # Same total number of documents?
    assert len(a_indexed) == len(b_indexed)
    # Same number of each type of document?
    a_counter = collections.Counter(name for _, uid in a_indexed)
    b_counter = collections.Counter(name for _, uid in b_indexed)
    assert a_counter == b_counter
    # Same uids and names?
    assert set(a_indexed) == set(b_indexed)<commit_msg>Remove unhelpful and redundant check.<commit_after>import collections
import event_model
import itertools
from intake.catalog.utils import RemoteCatalogError
import numpy
import ophyd.sim
import pytest


def normalize(gen):
    """
    Converted any pages to singles.
    """
    for name, doc in gen:
        if name == 'event_page':
            for event in event_model.unpack_event_page(doc):
                yield 'event', event
        elif name == 'datum_page':
            for datum in event_model.unpack_datum_page(doc):
                yield 'datum', datum
        else:
            yield name, doc


def compare(a, b):
    a = normalize(a)
    b = normalize(b)
    a_indexed = {}
    b_indexed = {}
    for name, doc in a:
        if name == 'resource':
            # Check for an extraneous duplicate key in old documents.
            if 'id' in doc:
                assert doc['id'] == doc['uid']
                doc = doc.copy()
                doc.pop('id')
        if name == 'datum':
            a_indexed[('datum', doc['datum_id'])] = doc
        # v0 yields {'_name": 'RunStop'} is the stop doc is missing; v2 yields None.
        elif name == 'stop' and doc is None or 'uid' not in doc:
            a_indexed[(name, None)] = None
        else:
            a_indexed[(name, doc['uid'])] = doc
    for name, doc in b:
        if name == 'resource':
            # Check for an extraneous duplicate key in old documents.
            if 'id' in doc:
                assert doc['id'] == doc['uid']
                doc = doc.copy()
                doc.pop('id')
        if name == 'datum':
            b_indexed[('datum', doc['datum_id'])] = doc
        # v0 yields {'_name": 'RunStop'} is the stop doc is missing; v2 yields None.
        elif name == 'stop' and doc is None or 'uid' not in doc:
            b_indexed[(name, None)] = None
        else:
            b_indexed[(name, doc['uid'])] = doc
    # Same number of each type of document?
    a_counter = collections.Counter(name for _, uid in a_indexed)
    b_counter = collections.Counter(name for _, uid in b_indexed)
    assert a_counter == b_counter
    # Same uids and names?
    assert set(a_indexed) == set(b_indexed)
{'fraction_non_alphanumeric': 0.11273627362736274, 'fraction_numerical': 0.0018001800180018, 'mean_word_length': 1.9993252361673415, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '347243', 'n_tokens_mistral': 1350, 'n_tokens_neox': 1324, 'n_words': 451}