<commit_before>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# <NAME> <<EMAIL>>
# Wed 11 Dec 15:14:08 2013 CET
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland

"""Tests the IO functionality of bob.measure."""

import bob.measure
import tempfile, os, shutil

import bob.io.base.test_utils

def test_load_scores():
  # This function tests the IO functionality of loading score files in different ways

  scores = []
  load_functions = {'4col' : bob.measure.load.four_column, '5col' : bob.measure.load.five_column}
  cols = {'4col' : 4, '5col' : 5}

  for variant in ('4col', '5col'):
    # read score file in normal way
    normal_score_file = bob.io.base.test_utils.datafile('dev-%s.txt' % variant, 'bob.measure')
    normal_scores = list(load_functions[variant](normal_score_file))

    assert len(normal_scores) == 910
    assert all(len(s) == cols[variant] for s in normal_scores)

    # read the compressed score file
    compressed_score_file = bob.io.base.test_utils.datafile('dev-%s.tar.gz' % variant, 'bob.measure')
    compressed_scores = list(load_functions[variant](compressed_score_file))

    assert len(compressed_scores) == len(normal_scores)
    assert all(len(c) == cols[variant] for c in compressed_scores)
    assert all(c[i] == s[i] for c,s in zip(compressed_scores, normal_scores) for i in range(cols[variant]))


def _check_binary_identical(name1, name2):
  # see: http://www.peterbe.com/plog/using-md5-to-check-equality-between-files
  import sys
  if sys.version_info[0] == 2:
    from md5 import new as md5
  else:
    from hashlib import md5
  # tests if two files are binary identical
  with open(name1,'rb') as f1, open(name2,'rb') as f2:
    assert md5(f1.read()).digest() == md5(f2.read()).digest()


def test_convert_openbr():
  # This function tests that the conversion to the OpenBR file works as expected
  temp_dir = tempfile.mkdtemp(prefix='bob_test')

  # define output files
  openbr_extensions = ('.mtx', '.mask')
  matrix_file, mask_file = [os.path.join(temp_dir, "scores%s") % ext for ext in openbr_extensions]

  try:
    for variant in ('4col', '5col'):
      # get score file
      score_file = bob.io.base.test_utils.datafile('scores-cmc-%s.txt' % variant, 'bob.measure')

      # first round, do not define keyword arguments -- let the file get the gallery and probe ids automatically
      kwargs = {}
      for i in range(2):
        # get the files by automatically obtaining the identities
        bob.measure.openbr.write_matrix(score_file, matrix_file, mask_file, score_file_format = "%sumn" % variant, **kwargs)
<commit_msg>Fixed next py26 issue<commit_after>#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
# <NAME> <<EMAIL>>
# Wed 11 Dec 15:14:08 2013 CET
#
# Copyright (C) 2011-2013 Idiap Research Institute, Martigny, Switzerland

"""Tests the IO functionality of bob.measure."""

import bob.measure
import tempfile, os, shutil

import bob.io.base.test_utils

def test_load_scores():
  # This function tests the IO functionality of loading score files in different ways

  scores = []
  load_functions = {'4col' : bob.measure.load.four_column, '5col' : bob.measure.load.five_column}
  cols = {'4col' : 4, '5col' : 5}

  for variant in ('4col', '5col'):
    # read score file in normal way
    normal_score_file = bob.io.base.test_utils.datafile('dev-%s.txt' % variant, 'bob.measure')
    normal_scores = list(load_functions[variant](normal_score_file))

    assert len(normal_scores) == 910
    assert all(len(s) == cols[variant] for s in normal_scores)

    # read the compressed score file
    compressed_score_file = bob.io.base.test_utils.datafile('dev-%s.tar.gz' % variant, 'bob.measure')
    compressed_scores = list(load_functions[variant](compressed_score_file))

    assert len(compressed_scores) == len(normal_scores)
    assert all(len(c) == cols[variant] for c in compressed_scores)
    assert all(c[i] == s[i] for c,s in zip(compressed_scores, normal_scores) for i in range(cols[variant]))


def _check_binary_identical(name1, name2):
  # see: http://www.peterbe.com/plog/using-md5-to-check-equality-between-files
  import sys
  if sys.version_info[0] == 2:
    from md5 import new as md5
  else:
    from hashlib import md5
  # tests if two files are binary identical
  with open(name1,'rb') as f1:
    with open(name2,'rb') as f2:
      assert md5(f1.read()).digest() == md5(f2.read()).digest()


def test_convert_openbr():
  # This function tests that the conversion to the OpenBR file works as expected
  temp_dir = tempfile.mkdtemp(prefix='bob_test')

  # define output files
  openbr_extensions = ('.mtx', '.mask')
  matrix_file, mask_file = [os.path.join(temp_dir, "scores%s") % ext for ext in openbr_extensions]

  try:
    for variant in ('4col', '5col'):
      # get score file
      score_file = bob.io.base.test_utils.datafile('scores-cmc-%s.txt' % variant, 'bob.measure')

      # first round, do not define keyword arguments -- let the file get the gallery and probe ids automatically
      kwargs = {}
      for i in range(2):
        # get the files by automatically obtaining the identities
        bob.measure.openbr.write_matrix(score_file, matrix_file, mask_file, score_file_format = "%sumn" % variant, **kwargs)

{'fraction_non_alphanumeric': 0.12397482357428953, 'fraction_numerical': 0.019835971771886323, 'mean_word_length': 4.066666666666666, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '316934', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1713, 'n_words': 486}