<commit_before>import re
import sys
import argparse

from IPython.nbformat.v3.rwbase import NotebookReader
from IPython.nbformat.v3.nbjson import JSONWriter

import IPython.nbformat.v3.nbbase as nbbase


class MarkdownReader(NotebookReader):
    """Import markdown to IPython Notebook.

    The markdown is split into blocks: code and not-code. These
    blocks are used as the source for cells in the notebook. Code
    blocks become code cells; not-code blocks become markdown cells.

    Only supports two kinds of notebook cell: code and markdown.
    """
    ## type identifiers
    code = u'code'
    markdown = u'markdown'
    python = u'python'

    ## regular expressions to match a code block, splitting into groups
    ## N.B you can't share group names between these patterns.
    ## this is necessary for format agnostic code block detection.
    ## These two pattern strings are ORed to create a master pattern
    ## and the python re module doesn't allow sharing group names
    ## in a single regular expression.

    # fenced code
    fenced_regex = r"""
    \n*                     # any number of newlines followed by
    ^(?P<fence>`{3,}|~{3,}) # a line starting with a fence of 3 or more ` or ~
    ?\{(?P<language>           # followed by the group 'language',
    [\w+-]*)                # a word of alphanumerics, _, - or +
    [ ]*                    # followed by spaces
    (?P<options>.*)         # followed by any text
    \n                      # followed by a newline<commit_msg>taking another shot at this<commit_after>import re
import sys
import argparse

from IPython.nbformat.v3.rwbase import NotebookReader
from IPython.nbformat.v3.nbjson import JSONWriter

import IPython.nbformat.v3.nbbase as nbbase


class MarkdownReader(NotebookReader):
    """Import markdown to IPython Notebook.

    The markdown is split into blocks: code and not-code. These
    blocks are used as the source for cells in the notebook. Code
    blocks become code cells; not-code blocks become markdown cells.

    Only supports two kinds of notebook cell: code and markdown.
    """
    ## type identifiers
    code = u'code'
    markdown = u'markdown'
    python = u'python'

    ## regular expressions to match a code block, splitting into groups
    ## N.B you can't share group names between these patterns.
    ## this is necessary for format agnostic code block detection.
    ## These two pattern strings are ORed to create a master pattern
    ## and the python re module doesn't allow sharing group names
    ## in a single regular expression.

    # fenced code
    fenced_regex = r"""
    \n*                     # any number of newlines followed by
    ^(?P<fence>`{3,}|~{3,}) # a line starting with a fence of 3 or more ` or ~
    \{*(?P<language>           # followed by the group 'language',
    [\w+-]*)                # a word of alphanumerics, _, - or +
    [ ]*                    # followed by spaces
    (?P<options>.*)         # followed by any text
    \n                      # followed by a newline
{'fraction_non_alphanumeric': 0.08163265306122448, 'fraction_numerical': 0.003949967083607637, 'mean_word_length': 2.5132947976878612, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '727851', 'n_tokens_mistral': 863, 'n_tokens_neox': 826, 'n_words': 355}