<commit_before># Copyright 2017 <NAME>, GPLv3

import unittest
from pkg_resources import load_entry_point
import contextlib
import glob
import io
import os
import shutil
import sys
import time
import tempfile
import tqdm

import matplotlib
matplotlib.use('Agg')

VERBOSE = False

def _replace_in_file(fname_src, fname_dest, str_src, str_dest):
    with open(fname_src) as f:
        text = f.read()
    text = text.replace(str_src, str_dest)
    with open(fname_dest, 'w') as f:
        f.write(text)


class TestCase(unittest.TestCase):

    def setUp(self):
        args = sys.argv[1:]
        if '-v' in args:
            global VERBOSE
            VERBOSE = True
        self.permanent_tempdir = '-p' in args
        if self.permanent_tempdir:
            tempdir = os.path.join(tempfile.gettempdir(), 'yam_test')
            if os.path.exists(tempdir) and '-d' in args:
                shutil.rmtree(tempdir)
            if not os.path.exists(tempdir):
                os.mkdir(tempdir)
        else:
            tempdir = tempfile.mkdtemp(prefix='yam_test_')
        self.cwd = os.getcwd()
        os.chdir(tempdir)
        self.tempdir = tempdir
        # for coverage put .coveragerc config file into tempdir
        # and append correct data_file parameter to config file
        covfn = os.path.join(self.cwd, '.coverage')
        if not os.path.exists('.coveragerc') and os.path.exists(covfn + 'rc'):
            _replace_in_file(covfn + 'rc', '.coveragerc', '[run]',
                             '[run]\ndata_file = ' + covfn)

    def redirect_output(self, args, raises_systemexit=False):
        with io.StringIO() as f:
            with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f):
                try:
                    self.script(args.split())
                except SystemExit:
                    if not raises_systemexit:
                        raise
            return f.getvalue()

    def out(self, cmd, text=None):
        t2 = self.redirect_output(cmd)
        if text is not None:
            if VERBOSE:
                print(t2)
            self.assertIn(text, t2)
        self.pbar.update(1)
        return t2

    def err(self, cmd, text=None, verbose=False):
        t2 = self.redirect_output(cmd, True)
        if text is not None:
            if VERBOSE:
                print(t2, file=sys.stderr)
            self.assertIn(text, t2)
        self.pbar.update(1)
        return t2

    def run_(self, cmd):
        self.script(cmd.split())
        self.pbar.update(1)
        self.pbar.refresh()

    def checkplot(self, bname):
        fname = os.path.join(self.plotdir, bname)
        self.assertTrue(os.path.exists(fname), msg='%s missing' % fname)

    def test_cli(self):

        self.plotdir = os.path.join(self.tempdir, 'plots')
        self.script = load_entry_point('yam', 'console_scripts', 'yam')
        total = 72 - 2 * self.permanent_tempdir
        self.pbar = tqdm.tqdm(total=total, desc='CLI tests passed')

        # create tutorial
        self.err('-h', 'print information about')
        self.err('--version', '.')
        self.err('bla', 'invalid choice')
        if not self.permanent_tempdir:
            self.err('info', "No such file or directory: 'conf.json'")
        self.out('create', '')
        if not self.permanent_tempdir:
            self.out('info', 'Not found')
        self.out('create --tutorial')
        self.run_('create --tutorial')
        _replace_in_file('conf.json', 'conf.json', '#"verbose"', '"verbose"')

        # check basics
        _replace_in_file('conf.json', 'conf2.json', '"io"', '"io",')
        self.err('-c conf2.json info', 'parsing the conf')
        self.out('info', '3 stations')
        self.out('info stations', 'CX.PATCX..BHZ')
        self.out('info data', 'example_data/CX.PATCX')
        self.out('print stations', 'CX.PATCX..BHZ')
        pr = self.out('print data CX.PATCX..BHZ 2010-02-03', '2010-02-03')
        self.out('print data CX.PATCX..BHZ 2010-034', '2010-02-03')
        self.out('print prepdata CX.PATCX..BHZ 2010-02-03 1', '864001 samples')
        self.err('print data', 'seedid')
        self.err('print prepdata CX.PATCX..BHZ 2010-02-03', 'corrid')
        _replace_in_file('conf.json', 'conf2.json', '"clip_factor"',
                         '"clip_set_zero": true, "clip_factor"')
        self.out('-c conf2.json print prepdata CX.PATCX..BHZ 2010-02-03 1a')

        # use self.out instead of self.run_ for plots to not show warnings
        try:
            self.out('plot stations')
        except ImportError:
            self.pbar.update(1)<commit_msg>hopefully fix travis by shutting down logging after each call of yam<commit_after># Copyright 2017 <NAME>, GPLv3

import unittest
from pkg_resources import load_entry_point
import contextlib
import glob
import io
import logging
import importlib
import os
import shutil
import sys
import time
import tempfile
import tqdm

import matplotlib
matplotlib.use('Agg')

VERBOSE = False

def _replace_in_file(fname_src, fname_dest, str_src, str_dest):
    with open(fname_src) as f:
        text = f.read()
    text = text.replace(str_src, str_dest)
    with open(fname_dest, 'w') as f:
        f.write(text)


class TestCase(unittest.TestCase):

    def setUp(self):
        args = sys.argv[1:]
        if '-v' in args:
            global VERBOSE
            VERBOSE = True
        self.permanent_tempdir = '-p' in args
        if self.permanent_tempdir:
            tempdir = os.path.join(tempfile.gettempdir(), 'yam_test')
            if os.path.exists(tempdir) and '-d' in args:
                shutil.rmtree(tempdir)
            if not os.path.exists(tempdir):
                os.mkdir(tempdir)
        else:
            tempdir = tempfile.mkdtemp(prefix='yam_test_')
        self.cwd = os.getcwd()
        os.chdir(tempdir)
        self.tempdir = tempdir
        # for coverage put .coveragerc config file into tempdir
        # and append correct data_file parameter to config file
        covfn = os.path.join(self.cwd, '.coverage')
        if not os.path.exists('.coveragerc') and os.path.exists(covfn + 'rc'):
            _replace_in_file(covfn + 'rc', '.coveragerc', '[run]',
                             '[run]\ndata_file = ' + covfn)

    def redirect_output(self, args, raises_systemexit=False):
        with io.StringIO() as f:
            with contextlib.redirect_stdout(f), contextlib.redirect_stderr(f):
                try:
                    self.script(args.split())
                except SystemExit:
                    if not raises_systemexit:
                        raise
                finally:
                    logging.shutdown()
                    importlib.reload(logging)
            return f.getvalue()

    def out(self, cmd, text=None):
        t2 = self.redirect_output(cmd)
        if text is not None:
            if VERBOSE:
                print(t2)
            self.assertIn(text, t2)
        self.pbar.update(1)
        return t2

    def err(self, cmd, text=None, verbose=False):
        t2 = self.redirect_output(cmd, True)
        if text is not None:
            if VERBOSE:
                print(t2, file=sys.stderr)
            self.assertIn(text, t2)
        self.pbar.update(1)
        return t2

    def run_(self, cmd):
        try:
            self.script(cmd.split())
        finally:
            logging.shutdown()
            importlib.reload(logging)
        self.pbar.update(1)
        self.pbar.refresh()

    def checkplot(self, bname):
        fname = os.path.join(self.plotdir, bname)
        self.assertTrue(os.path.exists(fname), msg='%s missing' % fname)

    def test_cli(self):

        self.plotdir = os.path.join(self.tempdir, 'plots')
        self.script = load_entry_point('yam', 'console_scripts', 'yam')
        total = 72 - 2 * self.permanent_tempdir
        self.pbar = tqdm.tqdm(total=total, desc='CLI tests passed')

        # create tutorial
        self.err('-h', 'print information about')
        self.err('--version', '.')
        self.err('bla', 'invalid choice')
        if not self.permanent_tempdir:
            self.err('info', "No such file or directory: 'conf.json'")
        self.out('create', '')
        if not self.permanent_tempdir:
            self.out('info', 'Not found')
        self.out('create --tutorial')
        self.run_('create --tutorial')
        if VERBOSE:
            _replace_in_file('conf.json', 'conf.json',
                             '#"verbose"', '"verbose"')

        # check basics
        _replace_in_file('conf.json', 'conf2.json', '"io"', '"io",')
        self.err('-c conf2.json info', 'parsing the conf')
        self.out('info', '3 stations')
        self.out('info stations', 'CX.PATCX..BHZ')
        self.out('info data', 'example_data/CX.PATCX')
        self.out('print stations', 'CX.PATCX..BHZ')
        pr = self.out('print data CX.PATCX..BHZ 2010-02-03', '2010-02-03')
        self.out('print data CX.PATCX..BHZ 2010-034', '2010-02-03')
        self.out('print prepdata CX.PATCX..BHZ 2010-02-03 1', '864001 samples')
        self.err('print data', 'seedid')
        self.err('print prepdata CX.PATCX..BHZ 2010-02-03', 'corrid')
        _replace_in_file('conf.json', 'conf2.json', '"clip_factor"',
                         '"clip_set_zero": true, "clip_factor"')
        self.out('-c conf2.json print prepdata CX.PATCX..BHZ 2010-02-03 1a')

        # use self.out instead of self.run_ for plots to not show warnings
        try:
            self.out('plot stations')
        except ImportError:
            self.pbar.update(1)
{'fraction_non_alphanumeric': 0.12768184196755625, 'fraction_numerical': 0.018628990057561485, 'mean_word_length': 2.0687219010918434, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5412299', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3021, 'n_words': 699}