<commit_before>import os
import subprocess
from tempfile import NamedTemporaryFile
from lxml import etree

dirname = os.path.dirname(__file__)

def validate(cnxmlstr, validator='lxml'):
    schemafn = os.path.join(dirname, 'schema/cnxml/rng/0.7/cnxml.rng')
    if validator == 'lxml':
        schemafile = open(schemafn)
        relaxng_doc = etree.parse(schemafile)
        relaxng = etree.RelaxNG(relaxng_doc)
        cnxmldoc= etree.fromstring(cnxmlstr)
        valid = relaxng.validate(cnxmldoc)
        if valid:
            print "Document validates against cnxml 0.7"
        else:
            print relaxng.error_log
        return valid, relaxng.error_log
    elif validator == 'jing':
        tmpfile = NamedTemporaryFile() # doesn't take stdin, so make a file
        tmpfile.write(cnxmlstr)
        tmpfile.flush()
        tmploc = tmpfile.name
        cmdargs = ["jing", schemafn, tmploc]
        process = subprocess.Popen(cmdargs,
            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        stdout, stderr = process.communicate(cnxmlstr)
        valid = stdout is None
        msg = ''
        if valid:
            return valid, msg

        cnxmlstr = cnxmlstr.split('\n')
        for line in stdout.split('\n'):
            if not line:
                continue
            parts = line.split(':')
            line = parts[1]
            column = parts[2]
            
            try:
                line = int(line)
                column = int(column)
                msg += 'On line %s, column %s: %s\n' % (line, column,
                                                        ''.join(parts[3:]))
                msg += '\tcontext: %s\n\n' % cnxmlstr[line-1][column-50:column+50]
            except ValueError:
                # not a line number, so use the whole thing
                line = 0
                column = 0
                
                # known specific exceptions
                if line == 'fatal: exception "java.io.IOException" thrown: Stream closed.':
                    msg += "DOCTYPE declaration not allowed."

        return valid, msg<commit_msg>fix check for valid cnxml<commit_after>import os
import subprocess
from tempfile import NamedTemporaryFile
from lxml import etree

dirname = os.path.dirname(__file__)

def validate(cnxmlstr, validator='lxml'):
    schemafn = os.path.join(dirname, 'schema/cnxml/rng/0.7/cnxml.rng')
    if validator == 'lxml':
        schemafile = open(schemafn)
        relaxng_doc = etree.parse(schemafile)
        relaxng = etree.RelaxNG(relaxng_doc)
        cnxmldoc= etree.fromstring(cnxmlstr)
        valid = relaxng.validate(cnxmldoc)
        if valid:
            print "Document validates against cnxml 0.7"
        else:
            print relaxng.error_log
        return valid, relaxng.error_log
    elif validator == 'jing':
        tmpfile = NamedTemporaryFile() # doesn't take stdin, so make a file
        tmpfile.write(cnxmlstr)
        tmpfile.flush()
        tmploc = tmpfile.name
        cmdargs = ["jing", schemafn, tmploc]
        process = subprocess.Popen(cmdargs,
            stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        stdout, stderr = process.communicate(cnxmlstr)
        valid = not stdout
        msg = ''
        if valid:
            return valid, msg

        cnxmlstr = cnxmlstr.split('\n')
        for line in stdout.split('\n'):
            if not line:
                continue
            parts = line.split(':')
            line = parts[1]
            column = parts[2]
            
            try:
                line = int(line)
                column = int(column)
                msg += 'On line %s, column %s: %s\n' % (line, column,
                                                        ''.join(parts[3:]))
                msg += '\tcontext: %s\n\n' % cnxmlstr[line-1][column-50:column+50]
            except ValueError:
                # not a line number, so use the whole thing
                line = 0
                column = 0
                
                # known specific exceptions
                if line == 'fatal: exception "java.io.IOException" thrown: Stream closed.':
                    msg += "DOCTYPE declaration not allowed."

        return valid, msg
{'fraction_non_alphanumeric': 0.0985781990521327, 'fraction_numerical': 0.006635071090047393, 'mean_word_length': 1.6480552070263488, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 31}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4721194', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1199, 'n_words': 308}