<commit_before># -*- coding: utf-8 -*-

from autotest import MD5Info, PathInfo, globjoin
from autotest import run_single

import sys, os
import nose.plugins.skip

class RunTest:
    def __init__(self,f):
        basename = os.path.basename(f)
        self.description = basename 
        mprefix = os.path.join(PathInfo.md5dir, basename)[:-4]
        md5file = mprefix + '.json'
        info=MD5Info()
        if os.path.exists(md5file):
            f = open(md5file, 'rb')
            exec f in info
            f.close()
        if info.good_md5 in [[],['sentinel']]:
            # This is an open issue or something that can't be checked automatically
            self.skip=True
        else:
            self.skip=False
    def __call__(self,f):
        if self.skip:
            raise nose.plugins.skip.SkipTest
        else:
            key, errcode = run_single(f)
            if key in ['incomplete']:
                raise nose.plugins.skip.SkipTest
            assert key == 'good', '%s is not good: %s'%(f,key)

def test():
    testfiles = globjoin(PathInfo.inpdir, '*.txt')
    results = {}
    for fname in testfiles:
        yield RunTest(fname), fname

def setup():
    PathInfo.add_coverage()<commit_msg>Make nosetest skip like autotest does (for CI)<commit_after># -*- coding: utf-8 -*-

from autotest import MD5Info, PathInfo, globjoin
from autotest import run_single

import sys, os
import nose.plugins.skip

class RunTest:
    def __init__(self,f):
        basename = os.path.basename(f)
        self.description = basename 
        mprefix = os.path.join(PathInfo.md5dir, basename)[:-4]
        md5file = mprefix + '.json'
        ignfile = os.path.join(PathInfo.inpdir , basename)+'.ignore'
        info=MD5Info()
        self.skip=False
        if os.path.exists(ignfile):
            self.skip=True
        if os.path.exists(md5file):
            f = open(md5file, 'rb')
            exec f in info
            f.close()
        if info.good_md5 in [[],['sentinel']]:
            # This is an open issue or something that can't be checked automatically
            self.skip=True
            
    def __call__(self,f):
        if self.skip:
            raise nose.plugins.skip.SkipTest
        else:
            key, errcode = run_single(f)
            if key in ['incomplete']:
                raise nose.plugins.skip.SkipTest
            assert key == 'good', '%s is not good: %s'%(f,key)

def test():
    testfiles = globjoin(PathInfo.inpdir, '*.txt')
    results = {}
    for fname in testfiles:
        yield RunTest(fname), fname

def setup():
    PathInfo.add_coverage()
{'fraction_non_alphanumeric': 0.11576923076923076, 'fraction_numerical': 0.006923076923076923, 'mean_word_length': 2.056404230317274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3586364', 'n_tokens_mistral': 826, 'n_tokens_neox': 809, 'n_words': 215}