<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
import random
import unittest
import eurostat
import pymongo
import types
import lxml.etree
import urllib

class TestEurostat(unittest.TestCase):
    """Test Eurostat"""

    def setUp(self):
        self.toto=eurostat.Eurostat()
        webpage = urllib.request.urlopen("file:///home/versiontest/27jan/dlstats/src/fetchers/table_of_contents.xml","r")
        table_of_contents = webpage.read()
        self.toto.table_of_contents = lxml.etree.fromstring(table_of_contents)
        webpage.close()
        self.toto.db = self.toto.client.t1
        self.toto.update_categories_db()

    def test_adummy(self):
        datatstat=self.toto.db.command("dbstats")
        for key, value in datatstat.items():
            print( key, value)
            
    def test_allelement(self):
        datatest=list(self.toto.db.categories.find())
        data = []
        with open("mjson.json") as f:
            data =[json.loads(line) for line in f]
        dataname=[]
        dataname=[r.get("name", "Missing: name") for r in data]
        rootid=self.toto.db.categories.find_one({"name": "Database by themes"})["_id"]
        rootchildid=self.toto.db.categories.find_one({"_id": rootid})
        testrootid=[r.get("_id", "Missing: _id") for r in data if r.get("name", "Missing: name")=='Database by themes']
        print('testrootid',testrootid)
        def testtree(idparent,testidparent):
            parentname=self.toto.db.categories.find_one({"_id": idparent})["name"]
            testparentname=[r.get("name", "Missing: name") for r in data if r.get("_id", "Missing: _id")==testidparent[0]][0]
            self.assertTrue(testparentname==parentname)
            try:
                childid=self.toto.db.categories.find_one({"_id": idparent})["children"]
                testchildid=[r.get("children", "Missing: children")[0] for r in data if r.get("_id", "Missing: _id")==testidparent[0]]
                print('testchildid:',testchildid)     
                for children in childid:
                    testtree(children,testchildid)
            except Exception:
                pass
        
        testtree(rootid,testrootid)            

    def test_Nbelement(self):
        mongo_NBm=self.toto.db.categories.find().count()    
        print('Nb of categories after update_categories: ', mongo_NBm)
        self.assertEqual(mongo_NBm, 7736)

    def test_firstelement(self):
        firstelement=self.toto.db.categories.find_one({"name": "Consumers - monthly data"})["name"]
        self.assertTrue(firstelement)

    def tearDown(self):
        self.toto.db = self.toto.client.t1  
        self.toto.db.command("dropDatabase")
        mongo_NBm=self.toto.db.categories.find().count()    
        print('Nb of categories after teardown: ', mongo_NBm)

if __name__ == '__main__':
    unittest.main()<commit_msg>Update eurostat_unit_test.py<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
import random
import unittest
import eurostat
import pymongo
import types
import lxml.etree
import urllib

class TestEurostat(unittest.TestCase):
    """Test Eurostat"""

    def setUp(self):
        self.toto=eurostat.Eurostat()
        webpage = urllib.request.urlopen("file:///home/versiontest/27jan/dlstats/src/fetchers/table_of_contents.xml","r")
        table_of_contents = webpage.read()
        self.toto.table_of_contents = lxml.etree.fromstring(table_of_contents)
        webpage.close()
        self.toto.db = self.toto.client.t1
        self.toto.update_categories_db()

    def test_adummy(self):
        datatstat=self.toto.db.command("dbstats")
        for key, value in datatstat.items():
            print( key, value)
            
    def test_allelement(self):
        datatest=list(self.toto.db.categories.find())
        data = []
        with open("mjson.json") as f:
            data =[json.loads(line) for line in f]
        dataname=[]
        dataname=[r.get("name", "Missing: name") for r in data]
        rootid=self.toto.db.categories.find_one({"name": "Database by themes"})["_id"]
        rootchildid=self.toto.db.categories.find_one({"_id": rootid}) 
        testrootid=[r.get("_id", "Missing: _id") for r in data if r.get("name", "Missing: name")=='Database by themes']
        def testtree(idparent,testidparent):
            parentname=self.toto.db.categories.find_one({"_id": idparent})["name"]
            testparentname=[r.get("name", "Missing: name") for r in data if r.get("_id", "Missing: _id")==testidparent[0]][0]
            print('testidparent',testidparent)  
            self.assertTrue(testparentname==parentname)
            try:
                childid=self.toto.db.categories.find_one({"_id": idparent})["children"]
                testchildid=[r.get("children", "Missing: children") for r in data if r.get("_id", "Missing: _id")==testidparent[0]][0]
                dictchild={}
                for i in range(len(childid)):
                    dictchild[childid[i]] = [testchildid[i]]
                for children, testchildren in dict.items(dictchild):
                    testtree(children,testchildren)
            except Exception as e:
                print(e)
                pass
        
        testtree(rootid,testrootid)     

    def test_Nbelement(self):
        mongo_NBm=self.toto.db.categories.find().count()    
        print('Nb of categories after update_categories: ', mongo_NBm)
        self.assertEqual(mongo_NBm, 7736)

    def test_firstelement(self):
        firstelement=self.toto.db.categories.find_one({"name": "Consumers - monthly data"})["name"]
        self.assertTrue(firstelement)

    def tearDown(self):
        self.toto.db = self.toto.client.t1  
        self.toto.db.command("dropDatabase")
        mongo_NBm=self.toto.db.categories.find().count()    
        print('Nb of categories after teardown: ', mongo_NBm)

if __name__ == '__main__':
    unittest.main()
{'fraction_non_alphanumeric': 0.1380590717299578, 'fraction_numerical': 0.004725738396624472, 'mean_word_length': 2.667079207920792, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1061550', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1893, 'n_words': 336}