<commit_before><commit_msg>added skeleton and first tests of scrapper<commit_after>import unittest
import datetime
from .scrapper import Scrapper

class ScrapperTests(unittest.TestCase):

    def testDirectoryDatePattern(self):
        s = Scrapper('%Y/%m/%d/%Y%m%d_%H%M%S_59.fit.gz')
        testpath = '2014/03/05/20140305_013000_59.fit.gz'
        d = datetime.datetime(2014,3,5,1,30)
        self.failUnless(s.matches(testpath, d))
    def testDirectoryDatePatternFalse(self):
        s = Scrapper('%Y/%m/%d/%Y%m%d_%H%M%S_59.fit.gz')
        testpath = '2013/03/05/20140305_013000_59.fit.gz'
        d = datetime.datetime(2014,3,5,1,30)
        self.failIf(s.matches(testpath, d))
    def testDirectoryObsPattern(self):
        s = Scrapper('%y%m%d/{observatory}_%Y%m%d.fits', observatory = 'SDO')
        testpath = '140305/SDO_20140305.fits'
        d = datetime.datetime(2014,3,5)
        self.failUnless(s.matches(testpath, d))
    def testDirectoryRange(self):
        s = Scrapper('%Y/%m/%d/%Y%m%d_%H.fit.gz')
        directory_list = ['2009/12/30/', '2009/12/31/', '2010/01/01/', 
                          '2010/01/02/', '2010/01/03/']
        startdate = datetime.datetime(2009,12,30)
        enddate = datetime.datetime(2010,1,3)
        self.failUnless(s.range(startdate, enddate) == directory_list)
    def testDirectoryRangeFalse(self):
        s = Scrapper('%Y%m%d/%Y%m%d_%H.fit.gz')
        directory_list = ['20091230/', '20091231/', '20100101/', 
                          '20090102/', '20090103/']
        startdate = datetime.datetime(2009,12,30)
        enddate = datetime.datetime(2010,1,3)
        self.failIf(s.range(startdate, enddate) == directory_list)
# check when there's not directory structure (no /s)         
                        

def main():
    unittest.main()

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.15915263443780553, 'fraction_numerical': 0.10755024443237371, 'mean_word_length': 2.7745901639344264, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3598664', 'n_tokens_mistral': 785, 'n_tokens_neox': 654, 'n_words': 97}