<commit_before>"""
tests the pysat meta object and code
"""
import pysat
import pandas as pds
from nose.tools import assert_raises, raises
import nose.tools
import pysat.instruments.pysat_testing
import numpy as np
import os
import tempfile

import sys
if sys.version_info[0] >= 3:
    if sys.version_info[1] < 4:
        import imp
        re_load = imp.reload
    else:
        import importlib
        re_load = importlib.reload
else:
    re_load = reload

def prep_dir(inst=None):
    import os
    import shutil

    if inst is None:
        inst = pysat.Instrument(platform='pysat', name='testing')
    # create data directories
    try:
        os.makedirs(inst.files.data_path)
        #print ('Made Directory')
    except OSError:
        pass

def remove_files(inst):
    # remove any files
    dir = inst.files.data_path
    for the_file in os.listdir(dir):
        if (the_file[0:13] == 'pysat_testing') & (the_file[-19:] == '.pysat_testing_file'):
            file_path = os.path.join(dir, the_file)
            if os.path.isfile(file_path):
                #print(file_path)
                os.unlink(file_path)
                #elif os.path.isdir(file_path): shutil.rmtree(file_path)


# create year doy file set
#def create_files(inst, start=None, stop=None, freq='1D', use_doy=True,
#                 root_fname=None):
#
#    # create a bunch of files
#    if start is None:
#        start = pysat.datetime(2009, 1, 1)
#    if stop is None:
#        stop = pysat.datetime(2013, 12, 31)
#    dates = pysat.utils.season_date_range(start, stop, freq=freq)
#    
#    if root_fname is None:
#        root_fname = 'pysat_testing_junk_{year:04d}_gold_{day:03d}_stuff.pysat_testing_file'
#        
#    for date in dates:
#        yr, doy = pysat.utils.getyrdoy(date)
#        if use_doy:
#            doy = doy
#        else:
#            doy = date.day        
#            
#        fname = os.path.join(inst.files.data_path, root_fname.format(year=yr, 
#                             day=doy, month=date.month, hour=date.hour, min=date.minute, sec=date.second))
#        f = open(fname, 'w')
#        f.close()




class TestBasics:
    def setup(self):
        """Runs before every method to create a clean testing setup."""
        # store current pysat directory
        self.data_path = pysat.data_dir
        
        # create temporary directory  
        dir_name = tempfile.gettempdir()
        pysat.utils.set_data_dir(dir_name, store=False)

        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, 
                                        clean_level='clean')
        # create testing directory
        prep_dir(self.testInst)

    def teardown(self):
        """Runs after every method to clean up previous testing."""
        remove_files(self.testInst)
        try:
            pysat.utils.set_data_dir(self.data_path, store=False)
        except:
            pass
        del self.testInst
    
    def test_basic_writing_and_reading_netcdf4_default_format(self):
        # create a bunch of files by year and doy<commit_msg>Removed old code<commit_after>"""
tests the pysat utils area
"""
import pysat
import pandas as pds
from nose.tools import assert_raises, raises
import nose.tools
import pysat.instruments.pysat_testing
import numpy as np
import os
import tempfile

import sys
if sys.version_info[0] >= 3:
    if sys.version_info[1] < 4:
        import imp
        re_load = imp.reload
    else:
        import importlib
        re_load = importlib.reload
else:
    re_load = reload

def prep_dir(inst=None):
    import os
    import shutil

    if inst is None:
        inst = pysat.Instrument(platform='pysat', name='testing')
    # create data directories
    try:
        os.makedirs(inst.files.data_path)
        #print ('Made Directory')
    except OSError:
        pass

def remove_files(inst):
    # remove any files
    dir = inst.files.data_path
    for the_file in os.listdir(dir):
        if (the_file[0:13] == 'pysat_testing') & (the_file[-19:] == '.pysat_testing_file'):
            file_path = os.path.join(dir, the_file)
            if os.path.isfile(file_path):
                os.unlink(file_path)

class TestBasics:
    def setup(self):
        """Runs before every method to create a clean testing setup."""
        # store current pysat directory
        self.data_path = pysat.data_dir
        
        # create temporary directory  
        dir_name = tempfile.gettempdir()
        pysat.utils.set_data_dir(dir_name, store=False)

        self.testInst = pysat.Instrument(inst_module=pysat.instruments.pysat_testing, 
                                        clean_level='clean')
        # create testing directory
        prep_dir(self.testInst)

    def teardown(self):
        """Runs after every method to clean up previous testing."""
        remove_files(self.testInst)
        try:
            pysat.utils.set_data_dir(self.data_path, store=False)
        except:
            pass
        del self.testInst
    
    def test_basic_writing_and_reading_netcdf4_default_format(self):
        # create a bunch of files by year and doy
{'fraction_non_alphanumeric': 0.0950800468566966, 'fraction_numerical': 0.007614213197969543, 'mean_word_length': 2.313712807244502, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2195731', 'n_tokens_mistral': 1690, 'n_tokens_neox': 1634, 'n_words': 394}