<commit_before>import os
from random import choice
from nose.exc import SkipTest
import rdflib

import logging
log = logging.getLogger(__file__)


skiptests = {
    'example3': "C++ style comment causes parse exception",
}
class Envelope(object):
    def __init__(self, n, f):
        self.name = n
        self.file = f
    def __repr__(self):
        return self.name

def genericbadjsontest(e):
    """Documentation"""
    if e.skip is not None:
        raise SkipTest("%s skipped, %s" % (e.name, e.skip))
    g = rdflib.Graph()
    try:
        g.parse(e.file, format="json-ld")
        assert True is False
    except ValueError:
        assert True is True

def generictest(e):
    """Documentation"""
    if e.skip is not None:
        raise SkipTest("%s skipped, %s" % (e.name, e.skip))
    gjson = rdflib.Graph()
    gn3 = rdflib.Graph()
    gjson.parse(e.file+'.json', format="json-ld")
    gn3.parse(e.file+'.nt', format="nt")
    log.debug(gjson.serialize(format="nt"))
    log.debug(gn3.serialize(format="nt"))
    (s, p, o) = choice(list(gn3.triples((None, None, None))))
    assert gjson.subjects(p, o) is not None

def test_cases():
    from copy import deepcopy
    allfiles = os.listdir(os.getcwd()+'/test/json')
    badfiles = [f for f in allfiles if f.startswith('bad')]
    examples = [f.split('.')[0] for f in allfiles if f.startswith('example') and f.endswith('.json')]
    for tfile in set(badfiles):
        gname = tfile.split('.')[0]
        e = Envelope(gname, os.getcwd()+'/test/json/'+tfile)
        if gname in skiptests:
            e.skip = skiptests[gname]
        else:
            e.skip = None
        # e.skip = True
        gt = deepcopy(genericbadjsontest)
        gt.__doc__ = tfile
        yield gt, e
    for example in examples:
        gname = example.split('.')[0]
        e = Envelope(gname, os.getcwd()+'/test/json/'+example)
        if gname in skiptests:
            e.skip = skiptests[gname]
        else:
            e.skip = None
        gt = deepcopy(generictest)
        gt.__doc__ = tfile
        yield gt, e


import sys
if sys.version_info <= (2,5):
    raise SkipTest('test execution code broken in Python 2.4')

if __name__ == "__main__":
    test_cases()<commit_msg>Circumvent py24 deepcopy issue.<commit_after>import os, sys
from random import choice
from nose.exc import SkipTest
import rdflib
import logging
log = logging.getLogger(__file__)


skiptests = {
    'example3': "C++ style comment causes parse exception",
}
class Envelope(object):
    def __init__(self, n, f):
        self.name = n
        self.file = f
    def __repr__(self):
        return self.name

def genericbadjsontest(e):
    """Documentation"""
    if e.skip is not None:
        raise SkipTest("%s skipped, %s" % (e.name, e.skip))
    g = rdflib.Graph()
    try:
        g.parse(e.file, format="json-ld")
        assert True is False
    except ValueError:
        assert True is True

def generictest(e):
    """Documentation"""
    if e.skip is not None:
        raise SkipTest("%s skipped, %s" % (e.name, e.skip))
    gjson = rdflib.Graph()
    gn3 = rdflib.Graph()
    gjson.parse(e.file+'.json', format="json-ld")
    gn3.parse(e.file+'.nt', format="nt")
    log.debug(gjson.serialize(format="nt"))
    log.debug(gn3.serialize(format="nt"))
    (s, p, o) = choice(list(gn3.triples((None, None, None))))
    assert gjson.subjects(p, o) is not None

def test_cases():
    from copy import deepcopy
    allfiles = os.listdir(os.getcwd()+'/test/json')
    badfiles = [f for f in allfiles if f.startswith('bad')]
    examples = [f.split('.')[0] for f in allfiles if f.startswith('example') and f.endswith('.json')]
    for tfile in set(badfiles):
        gname = tfile.split('.')[0]
        e = Envelope(gname, os.getcwd()+'/test/json/'+tfile)
        if gname in skiptests:
            e.skip = skiptests[gname]
        else:
            e.skip = None
        # e.skip = True
        if sys.version_info[:2] == (2,4):
            import pickle
            gbjt = pickle.dumps(genericbadjsontest)
            gt = pickle.loads(gbjt)
        else:
            gt = deepcopy(genericbadjsontest)
        gt.__doc__ = tfile
        yield gt, e
    for example in examples:
        gname = example.split('.')[0]
        e = Envelope(gname, os.getcwd()+'/test/json/'+example)
        if gname in skiptests:
            e.skip = skiptests[gname]
        else:
            e.skip = None
        if sys.version_info[:2] == (2,4):
            import pickle
            gjt = pickle.dumps(generictest)
            gt = pickle.loads(gjt)
        else:
            gt = deepcopy(generictest)
        gt.__doc__ = tfile
        yield gt, e


if __name__ == "__main__":
    test_cases()
{'fraction_non_alphanumeric': 0.13927813163481953, 'fraction_numerical': 0.0059447983014862, 'mean_word_length': 2.3746418338108883, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7369750', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1611, 'n_words': 406}