<commit_before>#!/usr/bin/env python
from __future__ import print_function  # Must be at the beginning of the file

import argparse
import io
import os
import os.path
import re
import sys
from collections import Counter

import requests
from pyang import plugin, error
from pyang.plugins.name import emit_name
from requests.packages.urllib3 import disable_warnings

from .yangParser import create_context

__author__    = '<EMAIL>, <EMAIL>, <EMAIL>, <EMAIL>'
__copyright__ = "Copyright(c) 2015, 2016, 2017, 2020 Cisco Systems, Inc."
__license__   = "New-style BSD"
__email__     = "<EMAIL>"


if sys.version_info < (2, 7, 9):
    disable_warnings()

try:
    xrange
except:
    xrange = range

def hexdump(src, length=16, sep='.'):
    """
    Hexdump function by sbz and 7h3rAm on Github:
    (https://gist.github.com/7h3rAm/5603718).
    :param src: Source, the string to be shown in hexadecimal format
    :param length: Number of hex characters to print in one row
    :param sep: Unprintable characters representation
    :return:
    """
    filtr = ''.join([(len(repr(chr(x))) == 3) and chr(x) or sep for x in range(256)])
    lines = []
    for c in xrange(0, len(src), length):
        chars = src[c:c+length]
        hexstring = ' '.join(["%02x" % ord(x) for x in chars])
        if len(hexstring) > 24:
            hexstring = "%s %s" % (hexstring[:24], hexstring[24:])
        printable = ''.join(["%s" % ((ord(x) <= 127 and filtr[ord(x)]) or sep) for x in chars])
        lines.append("     %02x:  %-*s  |%s|\n" % (c, length*3, hexstring, printable))
    print(''.join(lines))


def finalize_model(input_model):
    """
    Extracts string from the model data. This function is always the last
    stage in the model post-processing pipeline.
    :param input_model: Model to be processed
    :return: list of strings, ready to be written to a module file
    """
    finalized_output = []
    for mline in input_model:
        finalized_output.append(mline[0])
    return finalized_output


class YangModuleExtractor:
    """
    Extract YANG modules from IETF RFC or draft text string.
    """
    MODULE_STATEMENT = re.compile('''^[ \t]*(sub)?module +(["'])?([-A-Za-z0-9]*(@[0-9-]*)?)(["'])? *\{.*$''')
    PAGE_TAG = re.compile('.*\[Page [0-9]*\].*')
    CODE_ENDS_TAG = re.compile('^[} \t]*<CODE ENDS>.*$')
    CODE_BEGINS_TAG = re.compile('^[ \t\r\n]*<CODE BEGINS>( *file(\W+"(.*)")?)?[ \t\r\n]*$')
    EXAMPLE_TAG = re.compile('^(example-)')

    def __init__(self, src_id, dst_dir, strict=True, strict_examples=True, strict_name=False, add_line_refs=False,
                 debug_level=0, skip_modules=None, parse_only_modules=None):
        """
        Initializes class-global variables.
        :param src_id: text string containing the draft or RFC text from which YANG
                      module(s) are to be extracted
        :param dst_dir: Directory where to put the extracted YANG module(s)
        :param strict: Mode - if 'True', enforce <CODE BEGINS> / <CODE ENDS>;
                       if 'False', just look for 'module <name> {' and '}'
        :param strict_examples: Only output valid examples when in strict mode
        :param strict_name: enforce name from module
        :param debug_level: If > 0 print some debug statements to the console
        :param skip_modules: it will skip modules in this list
        :param parse_only_modules: it will parse only modules in this list
        :return:<commit_msg>Parse multiline filenames in <CODE BEGINS> statements<commit_after>#!/usr/bin/env python
from __future__ import print_function  # Must be at the beginning of the file

import argparse
import io
import os
import os.path
import re
import sys
from collections import Counter

import requests
from pyang import plugin, error
from pyang.plugins.name import emit_name
from requests.packages.urllib3 import disable_warnings

from .yangParser import create_context

__author__    = '<EMAIL>, <EMAIL>, <EMAIL>, <EMAIL>'
__copyright__ = "Copyright(c) 2015, 2016, 2017, 2020 Cisco Systems, Inc."
__license__   = "New-style BSD"
__email__     = "<EMAIL>"


if sys.version_info < (2, 7, 9):
    disable_warnings()

try:
    xrange
except:
    xrange = range

def hexdump(src, length=16, sep='.'):
    """
    Hexdump function by sbz and 7h3rAm on Github:
    (https://gist.github.com/7h3rAm/5603718).
    :param src: Source, the string to be shown in hexadecimal format
    :param length: Number of hex characters to print in one row
    :param sep: Unprintable characters representation
    :return:
    """
    filtr = ''.join([(len(repr(chr(x))) == 3) and chr(x) or sep for x in range(256)])
    lines = []
    for c in xrange(0, len(src), length):
        chars = src[c:c+length]
        hexstring = ' '.join(["%02x" % ord(x) for x in chars])
        if len(hexstring) > 24:
            hexstring = "%s %s" % (hexstring[:24], hexstring[24:])
        printable = ''.join(["%s" % ((ord(x) <= 127 and filtr[ord(x)]) or sep) for x in chars])
        lines.append("     %02x:  %-*s  |%s|\n" % (c, length*3, hexstring, printable))
    print(''.join(lines))


def finalize_model(input_model):
    """
    Extracts string from the model data. This function is always the last
    stage in the model post-processing pipeline.
    :param input_model: Model to be processed
    :return: list of strings, ready to be written to a module file
    """
    finalized_output = []
    for mline in input_model:
        finalized_output.append(mline[0])
    return finalized_output


class YangModuleExtractor:
    """
    Extract YANG modules from IETF RFC or draft text string.
    """
    MODULE_STATEMENT = re.compile('''^[ \t]*(sub)?module +(["'])?([-A-Za-z0-9]*(@[0-9-]*)?)(["'])? *\{.*$''')
    PAGE_TAG = re.compile('.*\[Page [0-9]*\].*')
    CODE_ENDS_TAG = re.compile('^[} \t]*<CODE ENDS>.*$')
    CODE_BEGINS_TAG = re.compile('^[ \t\r\n]*<CODE BEGINS>( *file(?:(\W+"(.*)")|(?:\W+"(.*)))?)?[ \t\r\n]*$')
    EXAMPLE_TAG = re.compile('^(example-)')

    def __init__(self, src_id, dst_dir, strict=True, strict_examples=True, strict_name=False, add_line_refs=False,
                 debug_level=0, skip_modules=None, parse_only_modules=None):
        """
        Initializes class-global variables.
        :param src_id: text string containing the draft or RFC text from which YANG
                      module(s) are to be extracted
        :param dst_dir: Directory where to put the extracted YANG module(s)
        :param strict: Mode - if 'True', enforce <CODE BEGINS> / <CODE ENDS>;
                       if 'False', just look for 'module <name> {' and '}'
        :param strict_examples: Only output valid examples when in strict mode
        :param strict_name: enforce name from module
        :param debug_level: If > 0 print some debug statements to the console
        :param skip_modules: it will skip modules in this list
        :param parse_only_modules: it will parse only modules in this list
        :return:
{'fraction_non_alphanumeric': 0.1347507542019825, 'fraction_numerical': 0.017526217497485994, 'mean_word_length': 3.1317507418397628, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '42529', 'n_tokens_mistral': 2366, 'n_tokens_neox': 2228, 'n_words': 725}