<commit_before>import os
import time
import datetime
import pytz
import pytest

from scipy.misc import imread, imsave
from scipy.ndimage.interpolation import zoom

import numpy as np

import localizer.config
from pipeline import Pipeline
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.stages import Localizer, PipelineStage, ImageReader, \
    LocalizerPreprocessor, TagSimilarityEncoder, Decoder, DecoderPreprocessor, \
    ResultCrownVisualizer, LocalizerVisualizer, SaliencyVisualizer

from pipeline.objects import Filename, Image, Timestamp, CameraIndex, IDs, \
    PipelineResult, Candidates, Regions, Descriptors, LocalizerInputImage, \
    SaliencyImage, PaddedCandidates, PaddedImage, Orientations
from bb_binary import Repository, DataSource, FrameContainer


def test_empty_pipeline():
    pipeline = Pipeline([], [])
    assert(len(pipeline.pipeline) == 0)


def test_stages_are_instantiated(pipeline_config):
    pipeline = Pipeline([Image], [LocalizerInputImage], **pipeline_config)
    assert(all([issubclass(type(stage), PipelineStage) for stage in pipeline.pipeline]))


def _assert_types(actual, expected):
    for actual_type, expected_type in zip(actual, expected):
        assert(type(actual_type) == expected_type)


def test_simple_pipeline(pipeline_config):
    pipeline = Pipeline([Filename], [SaliencyImage, Descriptors], **pipeline_config)

    expected_stages = [ImageReader,
                       LocalizerPreprocessor,
                       Localizer,
                       TagSimilarityEncoder]
    _assert_types(pipeline.pipeline, expected_stages)


def test_imagereader(bees_image, pipeline_config):
    pipeline = Pipeline([Filename], [Image, Timestamp, CameraIndex], **pipeline_config)

    expected_stages = [ImageReader]
    _assert_types(pipeline.pipeline, expected_stages)

    outputs = pipeline([bees_image])
    assert(len(outputs) == 3)

    assert Image in outputs
    assert Timestamp in outputs
    assert CameraIndex in outputs

    im = outputs[Image]
    ts = outputs[Timestamp]
    idx = outputs[CameraIndex]

    tz = pytz.timezone('Europe/Berlin')
    dt = datetime.datetime.fromtimestamp(ts, tz=pytz.utc)
    dt = dt.astimezone(tz)
    assert(im.shape == (3000, 4000))
    assert(dt.year == 2015)
    assert(dt.month == 8)
    assert(dt.day == 21)
    assert(dt.hour == 16)
    assert(dt.minute == 15)
    assert(dt.second == 30)
    assert(dt.microsecond == 884267)
    assert(idx == 2)


def test_localizer(pipeline_config):
    pipeline = Pipeline([Filename], [Regions, Candidates], **pipeline_config)

    expected_stages = [ImageReader,
                       LocalizerPreprocessor,
                       Localizer]
    _assert_types(pipeline.pipeline, expected_stages)

    fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'

    outputs = pipeline([fname])

    assert len(outputs) == 2
    assert Regions in outputs
    assert Candidates in outputs

    regions = outputs[Regions]
    assert(len(regions) > 0)

    candidates = outputs[Candidates]
    assert(len(regions) == len(candidates))

    for candidate in candidates:
        assert(candidate[0] >= 0 and candidate[0] < 3000)
        assert(candidate[1] >= 0 and candidate[1] < 4000)


def test_padding(pipeline_config):
    pipeline = Pipeline([Filename], [PaddedCandidates, Candidates,
                                     PaddedImage, Image],
                        **pipeline_config)

    fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'
    outputs = pipeline([fname])

    assert len(outputs) == 4
    assert PaddedImage in outputs
    assert PaddedCandidates in outputs

    assert(localizer.config.data_imsize[0] == localizer.config.data_imsize[1])
    assert(localizer.config.data_imsize[0] % 2 == 0)
    offset = localizer.config.data_imsize[0] // 2
    for padded, original in zip(outputs[PaddedCandidates], outputs[Candidates]):
        assert(all([(pc - offset) == oc for pc, oc in zip(padded, original)]))


@pytest.mark.slow
def test_decoder(pipeline_config):<commit_msg>Fix localizer tests<commit_after>import os
import time
import datetime
import pytz
import pytest

from scipy.misc import imread, imsave
from scipy.ndimage.interpolation import zoom

import numpy as np

from pipeline import Pipeline
from pipeline.pipeline import GeneratorProcessor, get_auto_config
from pipeline.io import BBBinaryRepoSink, video_generator
from pipeline.stages import Localizer, PipelineStage, ImageReader, \
    LocalizerPreprocessor, TagSimilarityEncoder, Decoder, DecoderPreprocessor, \
    ResultCrownVisualizer, LocalizerVisualizer, SaliencyVisualizer

from pipeline.objects import Filename, Image, Timestamp, CameraIndex, IDs, \
    PipelineResult, Candidates, Regions, Descriptors, LocalizerInputImage, \
    SaliencyImage, PaddedCandidates, PaddedImage, Orientations, LocalizerShapes
from bb_binary import Repository, DataSource, FrameContainer


def test_empty_pipeline():
    pipeline = Pipeline([], [])
    assert(len(pipeline.pipeline) == 0)


def test_stages_are_instantiated(pipeline_config):
    pipeline = Pipeline([Image], [LocalizerInputImage], **pipeline_config)
    assert(all([issubclass(type(stage), PipelineStage) for stage in pipeline.pipeline]))


def _assert_types(actual, expected):
    for actual_type, expected_type in zip(actual, expected):
        assert(type(actual_type) == expected_type)


def test_simple_pipeline(pipeline_config):
    pipeline = Pipeline([Filename], [SaliencyImage, Descriptors], **pipeline_config)

    expected_stages = [ImageReader,
                       LocalizerPreprocessor,
                       Localizer,
                       TagSimilarityEncoder]
    _assert_types(pipeline.pipeline, expected_stages)


def test_imagereader(bees_image, pipeline_config):
    pipeline = Pipeline([Filename], [Image, Timestamp, CameraIndex], **pipeline_config)

    expected_stages = [ImageReader]
    _assert_types(pipeline.pipeline, expected_stages)

    outputs = pipeline([bees_image])
    assert(len(outputs) == 3)

    assert Image in outputs
    assert Timestamp in outputs
    assert CameraIndex in outputs

    im = outputs[Image]
    ts = outputs[Timestamp]
    idx = outputs[CameraIndex]

    tz = pytz.timezone('Europe/Berlin')
    dt = datetime.datetime.fromtimestamp(ts, tz=pytz.utc)
    dt = dt.astimezone(tz)
    assert(im.shape == (3000, 4000))
    assert(dt.year == 2015)
    assert(dt.month == 8)
    assert(dt.day == 21)
    assert(dt.hour == 16)
    assert(dt.minute == 15)
    assert(dt.second == 30)
    assert(dt.microsecond == 884267)
    assert(idx == 2)


def test_localizer(pipeline_config):
    pipeline = Pipeline([Filename], [Regions, Candidates], **pipeline_config)

    expected_stages = [ImageReader,
                       LocalizerPreprocessor,
                       Localizer]
    _assert_types(pipeline.pipeline, expected_stages)

    fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'

    outputs = pipeline([fname])

    assert len(outputs) == 2
    assert Regions in outputs
    assert Candidates in outputs

    regions = outputs[Regions]
    assert(len(regions) > 0)

    candidates = outputs[Candidates]
    assert(len(regions) == len(candidates))

    for candidate in candidates:
        assert(candidate[0] >= 0 and candidate[0] < 3000)
        assert(candidate[1] >= 0 and candidate[1] < 4000)


def test_padding(pipeline_config):
    pipeline = Pipeline([Filename], [PaddedCandidates, Candidates,
                                     PaddedImage, Image, LocalizerShapes],
                        **pipeline_config)

    fname = os.path.dirname(__file__) + '/data/Cam_2_20150821161530_884267.jpeg'
    outputs = pipeline([fname])

    assert len(outputs) == 5
    assert PaddedImage in outputs
    assert PaddedCandidates in outputs

    offset = outputs[LocalizerShapes]['roi_size'] // 2
    for padded, original in zip(outputs[PaddedCandidates], outputs[Candidates]):
        assert(all([(pc - offset) == oc for pc, oc in zip(padded, original)]))


@pytest.mark.slow
def test_decoder(pipeline_config):
{'fraction_non_alphanumeric': 0.09857264852994999, 'fraction_numerical': 0.02269122849823106, 'mean_word_length': 3.794152046783626, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3180587', 'n_tokens_mistral': 2614, 'n_tokens_neox': 2423, 'n_words': 558}