<commit_before>#!/usr/bin/env python3

"""
ipyrmd - Convert between IPython/Jupyter notebooks and R Markdown documents

This script provides conversion in both directions between the IPython notebook
format (JSON with seperate markdown and code) and the R markdown format
(markdown text with annotated code blocks).

TODO:

 * Options for how to handle inline R blocks
 * Consider using nbconvert machinery for ipynb -> Rmd conversion
 * Consider whether any chunk options can be emulated with IRdisplay calls
"""

import nbformat
import yaml
import re

# cell.source can be either "source" or ["source", "source"]
# notebook does not insert implicit newlines in the list case
maybe_join = lambda x: x if isinstance(x, str) else "".join(x)

# join two blocks of text, adding a newline if there is not already
# a leading or trailing one
maybe_newline = lambda x, y: x+y if x.endswith("\n") or y.startswith("\n") else x+"\n"+y


def join_with_emptylines(text):
    """
    Join a list of strings with \n\n (one empty line, to force separate
    markdown paragraphs, stripping any extra newlines.
    """
    if len(text) == 1:
        return text[0]
    result = text[0]
    for t in text[1:]:
        result = result.rstrip("\n") + "\n\n" + t.lstrip("\n")
    return result


def prepend_lines(text, prefix):
    """
    Insert a prefix at the beginning of each line, eg #'
    """
    return "\n".join([prefix+t for t in text.split("\n")])

unprepend_line = lambda x, p: x[len(p):] if x.startswith(p) else x


def unprepend_lines(text, prefix):
    return "\n".join([unprepend_line(t, prefix) for t in text.split("\n")])


# ensure NotebookNode objects are represented as plain dicts in YAML
def NN_representer(dumper, data):
    return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data),
                                    flow_style=False)

yaml.add_representer(nbformat.NotebookNode, NN_representer)


def read_ipynb(infile, header=None):<commit_msg>join_with_emptylines didn't handle the empty list<commit_after>#!/usr/bin/env python3

"""
ipyrmd - Convert between IPython/Jupyter notebooks and R Markdown documents

This script provides conversion in both directions between the IPython notebook
format (JSON with seperate markdown and code) and the R markdown format
(markdown text with annotated code blocks).

TODO:

 * Options for how to handle inline R blocks
 * Consider using nbconvert machinery for ipynb -> Rmd conversion
 * Consider whether any chunk options can be emulated with IRdisplay calls
"""

import nbformat
import yaml
import re

# cell.source can be either "source" or ["source", "source"]
# notebook does not insert implicit newlines in the list case
maybe_join = lambda x: x if isinstance(x, str) else "".join(x)

# join two blocks of text, adding a newline if there is not already
# a leading or trailing one
maybe_newline = lambda x, y: x+y if x.endswith("\n") or y.startswith("\n") else x+"\n"+y


def join_with_emptylines(text):
    """
    Join a list of strings with \n\n (one empty line, to force separate
    markdown paragraphs, stripping any extra newlines.
    """
    if len(text) == 0:
        return ""
    if len(text) == 1:
        return text[0]
    result = text[0]
    for t in text[1:]:
        result = result.rstrip("\n") + "\n\n" + t.lstrip("\n")
    return result


def prepend_lines(text, prefix):
    """
    Insert a prefix at the beginning of each line, eg #'
    """
    return "\n".join([prefix+t for t in text.split("\n")])

unprepend_line = lambda x, p: x[len(p):] if x.startswith(p) else x


def unprepend_lines(text, prefix):
    return "\n".join([unprepend_line(t, prefix) for t in text.split("\n")])


# ensure NotebookNode objects are represented as plain dicts in YAML
def NN_representer(dumper, data):
    return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data),
                                    flow_style=False)

yaml.add_representer(nbformat.NotebookNode, NN_representer)


def read_ipynb(infile, header=None):
{'fraction_non_alphanumeric': 0.10681307711504867, 'fraction_numerical': 0.004741702021462441, 'mean_word_length': 3.66046511627907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3256690', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1230, 'n_words': 464}