<commit_before>"""Combine source and outputs from two notebooks
"""
import re
from .cell_metadata import _IGNORE_CELL_METADATA
from .header import _DEFAULT_NOTEBOOK_METADATA
from .metadata_filter import restore_filtered_metadata
from .formats import long_form_one_format

_BLANK_LINE = re.compile(r'^\s*$')


def black_invariant(text, chars=None):
    """Remove characters that may be changed when reformatting the text with black"""
    if chars is None:
        chars = [' ', '\t', '\n', ',', "'", '"', '(', ')', '\\']

    for char in chars:
        text = text.replace(char, '')
    return text


def same_content(ref, test, endswith=False):
    """Is the content of two cells the same, up to reformating by black"""
    if endswith:
        return black_invariant(ref).endswith(black_invariant(test))
    return black_invariant(ref) == black_invariant(test)


def combine_inputs_with_outputs(nb_source, nb_outputs, fmt=None):
    """Copy outputs and metadata of the second notebook into the first one."""
    # nbformat version number taken from the notebook with outputs
    assert nb_outputs.nbformat == nb_source.nbformat, \
        "The notebook with outputs is in format {}.{}, please upgrade it to {}.x".format(
            nb_outputs.nbformat, nb_outputs.nbformat_minor, nb_source.nbformat)
    nb_source.nbformat_minor = nb_outputs.nbformat_minor

    fmt = long_form_one_format(fmt)
    text_repr = nb_source.metadata.get('jupytext', {}).get('text_representation', {})
    ext = fmt.get('extension') or text_repr.get('extension')
    format_name = fmt.get('format_name') or text_repr.get('format_name')<commit_msg>An empty cell is not the end of another cell<commit_after>"""Combine source and outputs from two notebooks
"""
import re
from .cell_metadata import _IGNORE_CELL_METADATA
from .header import _DEFAULT_NOTEBOOK_METADATA
from .metadata_filter import restore_filtered_metadata
from .formats import long_form_one_format

_BLANK_LINE = re.compile(r'^\s*$')


def black_invariant(text, chars=None):
    """Remove characters that may be changed when reformatting the text with black"""
    if chars is None:
        chars = [' ', '\t', '\n', ',', "'", '"', '(', ')', '\\']

    for char in chars:
        text = text.replace(char, '')
    return text


def same_content(ref, test, endswith=False):
    """Is the content of two cells the same, up to reformating by black"""
    ref = black_invariant(ref)
    test = black_invariant(test)

    if endswith and test:
        return ref.endswith(test)
    return ref == test


def combine_inputs_with_outputs(nb_source, nb_outputs, fmt=None):
    """Copy outputs and metadata of the second notebook into the first one."""
    # nbformat version number taken from the notebook with outputs
    assert nb_outputs.nbformat == nb_source.nbformat, \
        "The notebook with outputs is in format {}.{}, please upgrade it to {}.x".format(
            nb_outputs.nbformat, nb_outputs.nbformat_minor, nb_source.nbformat)
    nb_source.nbformat_minor = nb_outputs.nbformat_minor

    fmt = long_form_one_format(fmt)
    text_repr = nb_source.metadata.get('jupytext', {}).get('text_representation', {})
    ext = fmt.get('extension') or text_repr.get('extension')
    format_name = fmt.get('format_name') or text_repr.get('format_name')
{'fraction_non_alphanumeric': 0.1166109930154874, 'fraction_numerical': 0.0, 'mean_word_length': 4.154929577464789, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1505224', 'n_tokens_mistral': 1055, 'n_tokens_neox': 1007, 'n_words': 292}