<commit_before>import Images
using Color, Base.Test, FixedPointNumbers

# Comparison of each element in arrays with a scalar
approx_equal(ar, v) = all(abs(ar.-v) .< sqrt(eps(v)))
approx_equal(ar::Images.AbstractImage, v) = approx_equal(Images.data(ar), v)

# arithmetic
img = convert(Images.Image, zeros(3,3))
img2 = (img .+ 3)/2
@test all(img2 .== 1.5)
img3 = 2img2
@test all(img3 .== 3)
img3 = copy(img2)
img3[img2 .< 4] = -1
@test all(img3 .== -1)
img = convert(Images.Image, rand(3,4))
A = rand(3,4)
img2 = img .* A
@test all(Images.data(img2) == Images.data(img).*A)
img2 = convert(Images.Image, A)
img2 = img2 .- 0.5
img3 = 2img .* img2
img2 = img ./ A
img2 = (2img).^2
# Same operations with ColorValue images
img = Images.colorim(zeros(Float32,3,4,5))
img2 = (img .+ RGB{Float32}(1,1,1))/2
@test all(img2 .== RGB{Float32}(1,1,1)/2)
img3 = 2img2
@test all(img3 .== RGB{Float32}(1,1,1))
A = fill(2, 4, 5)
@test all(A.*img2 .== fill(RGB{Float32}(1,1,1), 4, 5))
img2 = img2 .- RGB{Float32}(1,1,1)/2
A = rand(Uint8,3,4)
img = reinterpret(Images.Gray{Ufixed8}, Images.grayim(A))
imgm = mean(img)
imgn = img/imgm
@test_approx_eq reinterpret(Float32, Images.data(imgn)) convert(Array{Float32}, A/mean(A))

# Reductions
let
    A = rand(5,5,3)
    img = Images.colorim(A, "RGB")
    s12 = sum(img, (1,2))
    @test Images.colorspace(s12) == "RGB"
    s3 = sum(img, (3,))
    @test Images.colorspace(s3) == "Unknown"
    A = [NaN, 1, 2, 3]
    @test_approx_eq Images.meanfinite(A, 1) [2]
    A = [NaN 1 2 3;
         NaN 6 5 4]
    @test_approx_eq Images.meanfinite(A, 1) [NaN 3.5 3.5 3.5]
    @test_approx_eq Images.meanfinite(A, 2) [2, 5]'
    @test_approx_eq Images.meanfinite(A, (1,2)) [3.5]
    @test Images.minfinite(A) == 1
    @test Images.maxfinite(A) == 6
    @test Images.maxabsfinite(A) == 6
    A = rand(Float32,3,5,5)
    img = Images.colorim(A, "RGB")
    dc = Images.data(Images.meanfinite(img, 1))-reinterpret(RGB{Float32}, mean(A, 2), (1,5))
    @test maximum(map(abs, dc)) < 1e-6

    a = convert(Array{Uint8}, [1, 1, 1])
    b = convert(Array{Uint8}, [134, 252, 4])
    @test sad(a, b) == 387
    @test ssd(a, b) == 80699
    af = reinterpret(Ufixed8, a)
    bf = reinterpret(Ufixed8, b)
    @test_approx_eq sad(af, bf) 387f0/255
    @test_approx_eq ssd(af, bf) 80699f0/255^2
    ac = reinterpret(RGB{Ufixed8}, a)
    bc = reinterpret(RGB{Ufixed8}, b)
    @test_approx_eq sad(ac, bc) 387f0/255
    @test_approx_eq ssd(ac, bc) 80699f0/255^2
    ag = reinterpret(RGB{Ufixed8}, a)
    bg = reinterpret(RGB{Ufixed8}, b)
    @test_approx_eq sad(ag, bg) 387f0/255
    @test_approx_eq ssd(ag, bg) 80699f0/255^2<commit_msg>Add missing module qualifiers to tests<commit_after>import Images
using Color, Base.Test, FixedPointNumbers

# Comparison of each element in arrays with a scalar
approx_equal(ar, v) = all(abs(ar.-v) .< sqrt(eps(v)))
approx_equal(ar::Images.AbstractImage, v) = approx_equal(Images.data(ar), v)

# arithmetic
img = convert(Images.Image, zeros(3,3))
img2 = (img .+ 3)/2
@test all(img2 .== 1.5)
img3 = 2img2
@test all(img3 .== 3)
img3 = copy(img2)
img3[img2 .< 4] = -1
@test all(img3 .== -1)
img = convert(Images.Image, rand(3,4))
A = rand(3,4)
img2 = img .* A
@test all(Images.data(img2) == Images.data(img).*A)
img2 = convert(Images.Image, A)
img2 = img2 .- 0.5
img3 = 2img .* img2
img2 = img ./ A
img2 = (2img).^2
# Same operations with ColorValue images
img = Images.colorim(zeros(Float32,3,4,5))
img2 = (img .+ RGB{Float32}(1,1,1))/2
@test all(img2 .== RGB{Float32}(1,1,1)/2)
img3 = 2img2
@test all(img3 .== RGB{Float32}(1,1,1))
A = fill(2, 4, 5)
@test all(A.*img2 .== fill(RGB{Float32}(1,1,1), 4, 5))
img2 = img2 .- RGB{Float32}(1,1,1)/2
A = rand(Uint8,3,4)
img = reinterpret(Images.Gray{Ufixed8}, Images.grayim(A))
imgm = mean(img)
imgn = img/imgm
@test_approx_eq reinterpret(Float32, Images.data(imgn)) convert(Array{Float32}, A/mean(A))

# Reductions
let
    A = rand(5,5,3)
    img = Images.colorim(A, "RGB")
    s12 = sum(img, (1,2))
    @test Images.colorspace(s12) == "RGB"
    s3 = sum(img, (3,))
    @test Images.colorspace(s3) == "Unknown"
    A = [NaN, 1, 2, 3]
    @test_approx_eq Images.meanfinite(A, 1) [2]
    A = [NaN 1 2 3;
         NaN 6 5 4]
    @test_approx_eq Images.meanfinite(A, 1) [NaN 3.5 3.5 3.5]
    @test_approx_eq Images.meanfinite(A, 2) [2, 5]'
    @test_approx_eq Images.meanfinite(A, (1,2)) [3.5]
    @test Images.minfinite(A) == 1
    @test Images.maxfinite(A) == 6
    @test Images.maxabsfinite(A) == 6
    A = rand(Float32,3,5,5)
    img = Images.colorim(A, "RGB")
    dc = Images.data(Images.meanfinite(img, 1))-reinterpret(RGB{Float32}, mean(A, 2), (1,5))
    @test maximum(map(abs, dc)) < 1e-6

    a = convert(Array{Uint8}, [1, 1, 1])
    b = convert(Array{Uint8}, [134, 252, 4])
    @test Images.sad(a, b) == 387
    @test Images.ssd(a, b) == 80699
    af = reinterpret(Ufixed8, a)
    bf = reinterpret(Ufixed8, b)
    @test_approx_eq Images.sad(af, bf) 387f0/255
    @test_approx_eq Images.ssd(af, bf) 80699f0/255^2
    ac = reinterpret(RGB{Ufixed8}, a)
    bc = reinterpret(RGB{Ufixed8}, b)
    @test_approx_eq Images.sad(ac, bc) 387f0/255
    @test_approx_eq Images.ssd(ac, bc) 80699f0/255^2
    ag = reinterpret(RGB{Ufixed8}, a)
    bg = reinterpret(RGB{Ufixed8}, b)
    @test_approx_eq Images.sad(ag, bg) 387f0/255
    @test_approx_eq Images.ssd(ag, bg) 80699f0/255^2
{'fraction_non_alphanumeric': 0.18552459322984852, 'fraction_numerical': 0.08378530016831869, 'mean_word_length': 3.861818181818182, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7514181', 'n_tokens_mistral': 2531, 'n_tokens_neox': 2318, 'n_words': 508}