<commit_before>import unittest
import os
import sys
import shutil
import json
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))

from api import process_docs
import logging

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)


class TestAPI(unittest.TestCase):

    def tearDown(self):
        shutil.rmtree('archive/TEST')

    def test_process_raw(self):
        doc = json.dumps({'Hello':  'world'})
        source = "TEST"
        doc_id = 37
        filetype = "json"

        assert process_docs.process_raw(doc, source, doc_id, filetype)

        found = False
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                found = True
        assert found

    def test_process_legal(self):
        source = "TEST"
        doc_id = 38
        filetype = 'json'
        doc = {
            'title': "TEST PROJECT",
            'contributors': ['Me, Myself', 'And I'],
            'properties': {
                'description': 'science stuff',
                'email': 'email stuff'
            },
            'meta': {},
            'id': doc_id,
            'source': source
        }
        process_docs.process_raw(json.dumps(doc), source, doc_id, filetype)
        timestamp = None
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                timestamp = dirname.split('/')[-1]
        assert timestamp

        assert process_docs.process(doc, timestamp)
        found = False
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/parsed.json'):
                found = True
        assert found

    def test_process_illegal(self):
        source = "TEST"
        doc_id = 37
        filetype = 'json'
        doc = {
            'title': "TEST PROJECT",
            'contributors': ['Me, Myself', 'And I'],
            'properties': {
                'description': 'science stuff',
                'email': 'email stuff'
            },
            'meta': {},
            'id': doc_id,
        }

        process_docs.process_raw(json.dumps(doc), source, doc_id, filetype)
        timestamp = None
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                timestamp = dirname.split('/')[-1]
        assert timestamp

        assert not process_docs.process(doc, timestamp)
<commit_msg>Updated tests to reflect changes<commit_after>import unittest
import os
import sys
import shutil
import json
sys.path.insert(1, os.path.join(
    os.path.dirname(os.path.abspath(__file__)),
    os.pardir,
))

from api import process_docs
import logging

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)


class TestAPI(unittest.TestCase):

    def tearDown(self):
        shutil.rmtree('archive/TEST')

    def test_process_raw(self):
        doc = json.dumps({'Hello':  'world'})
        source = "TEST"
        doc_id = 37
        filetype = "json"

        assert process_docs.process_raw(doc, source, doc_id, filetype, 'test-version')

        found = False
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                found = True
        assert found

    def test_process_legal(self):
        source = "TEST"
        doc_id = 38
        filetype = 'json'
        doc = {
            'title': "TEST PROJECT",
            'contributors': ['Me, Myself', 'And I'],
            'properties': {
                'description': 'science stuff',
                'email': 'email stuff'
            },
            'meta': {},
            'id': doc_id,
            'source': source
        }
        process_docs.process_raw(json.dumps(doc), source, doc_id, filetype, 'test-version')
        timestamp = None
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                timestamp = dirname.split('/')[-1]
        assert timestamp

        assert process_docs.process(doc, timestamp)
        found = False
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/parsed.json'):
                found = True
        assert found

    def test_process_illegal(self):
        source = "TEST"
        doc_id = 37
        filetype = 'json'
        doc = {
            'title': "TEST PROJECT",
            'contributors': ['Me, Myself', 'And I'],
            'properties': {
                'description': 'science stuff',
                'email': 'email stuff'
            },
            'meta': {},
            'id': doc_id,
        }

        process_docs.process_raw(json.dumps(doc), source, doc_id, filetype, 'test-version')
        timestamp = None
        for dirname, dirnames, filenames in os.walk('archive/TEST/{0}'.format(doc_id)):
            if os.path.isfile(dirname + '/raw.json'):
                timestamp = dirname.split('/')[-1]
        assert timestamp

        assert not process_docs.process(doc, timestamp)

{'fraction_non_alphanumeric': 0.1301995896288006, 'fraction_numerical': 0.004849841447491139, 'mean_word_length': 1.9673491975650248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6493573', 'n_tokens_mistral': 1616, 'n_tokens_neox': 1563, 'n_words': 357}