<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Unit tests for the Python WCON parser

"""
import os
import sys
from six import StringIO  # StringIO.StringIO in 2.x, io.StringIO in 3.x
import json
import jsonschema
import unittest
import filecmp
import glob
import collections

import pandas as pd
idx = pd.IndexSlice
import numpy as np
import time

sys.path.append('..')
from wcon import WCONWorms, MeasurementUnit
from wcon.measurement_unit import MeasurementUnitAtom

if __name__ == '__main__':
    print("BIG TEST: Test load and save and load and save")
    files_to_test = [sys.argv[1]]

    for JSON_path in files_to_test:
        for pretty in [True, False]:
            print("LOADING FOR TEST: " + JSON_path +
                  " (PRETTY = " + str(pretty) + ")")

            start_time = time.monotonic()
            w1 = WCONWorms.load_from_file(JSON_path,
                                          validate_against_schema=False)
            print("Time to load w1: " + str(time.monotonic() - start_time))

            # Save these worm tracks to a file, then load that file
            test_path = 'test.wcon'
            start_time = time.monotonic()
            w1.save_to_file(test_path, pretty_print=pretty)
            print("Time to save w1: " + str(time.monotonic() - start_time))

            start_time = time.monotonic()
            w2 = WCONWorms.load_from_file(test_path,
                                          validate_against_schema=False)
            print("Time to load w2: " + str(time.monotonic() - start_time))
<commit_msg>Make diagnostic_test Python 2 compatible<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Unit tests for the Python WCON parser

"""
import os
import sys
from six import StringIO  # StringIO.StringIO in 2.x, io.StringIO in 3.x
import json
import jsonschema
import unittest
import filecmp
import glob
import collections

import pandas as pd
idx = pd.IndexSlice
import numpy as np
import time

sys.path.append('..')
from wcon import WCONWorms, MeasurementUnit
from wcon.measurement_unit import MeasurementUnitAtom


def timing_function():
    """
    There's a better timing function available in Python 3.3+
    Otherwise use the old one.

    """
    if sys.version_info[0] >= 3 and sys.version_info[1] >= 3:
        return time.monotonic()
    else:
        return time.time()

if __name__ == '__main__':
    # def test_big_file():
    print("BIG TEST: Test load and save and load and save")
    files_to_test = [sys.argv[1]]

    for JSON_path in files_to_test:
        for pretty in [True]:  # , False]:
            print("LOADING FOR TEST: " + JSON_path +
                  " (PRETTY = " + str(pretty) + ")")

            start_time = timing_function()
            w1 = WCONWorms.load_from_file(JSON_path,
                                          validate_against_schema=False)
            print("Time to load w1: " + str(timing_function() - start_time))

            continue

            # Save these worm tracks to a file, then load that file
            test_path = 'test.wcon'
            start_time = timing_function()
            w1.save_to_file(test_path, pretty_print=pretty)
            print("Time to save w1: " + str(timing_function() - start_time))

            start_time = timing_function()
            w2 = WCONWorms.load_from_file(test_path,
                                          validate_against_schema=False)
            print("Time to load w2: " + str(timing_function() - start_time))

{'fraction_non_alphanumeric': 0.09537407195888065, 'fraction_numerical': 0.007709880068532267, 'mean_word_length': 2.3266856600189936, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7641563', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1030, 'n_words': 302}