<commit_before>#Trimmed mean; amount of trimming defaults to 20% or 0.2
function tmean{S <: Real}(x::Vector{S}; tr::Real=0.2)
    if tr < 0 || tr > 0.5
        error("tr cannot be smaller than 0 or larger than 0.5")
    elseif tr == 0
        return mean(x)
    elseif tr == .5
        return median(x, checknan=false)
    else 
        n   = length(x)
        lo  = floor(n*tr)+1
        hi  = n+1-lo
        return mean(sort!(copy(x))[lo:hi])
    end
end
#tmean{S <: Real}(x::DataVector{S}; tr::Real=0.2)=tmean(removeNA(x), tr=tr)

function tmean!{S <: Real}(x::Vector{S}; tr::Real=0.2)
    if tr < 0 || tr > 0.5
        error("tr cannot be smaller than 0 or larger than 0.5")
    elseif tr == 0
        return mean(x)
    elseif tr == .5
        return median!(x, checknan=false)
    else 
        n   = length(x)
        lo  = floor(n*tr)+1
        hi  = n+1-lo
        return mean(sort!(x)[lo:hi])
    end
end


#Winsorize data
function winval{S <: Real}(x::Vector{S}; tr::Real=0.2)
    xcopy   = copy(x)
    n       = length(x)
    xcopy   = sort!(xcopy)
    ibot    = floor(tr*n)+1
    itop    = n-ibot+1
    xbot, xtop = xcopy[ibot], xcopy[itop]
    return  [x[i]<=xbot?xbot:(x[i]>=xtop?xtop:x[i]) for i=1:n]
end
winval{S <: Real}(x::DataVector{S}; tr::Real=0.2)=winval(removeNA(x), tr=tr)

#Winsorized mean
winmean{S <: Real}(x::Vector{S}; tr=0.2)=mean(winval(x, tr=tr))
#winmean{S <: Real}(x::DataVector{S}; tr::Real=0.2)=mean(winval(removeNA(x), tr=tr))

#Winsorized variance
winvar{S <: Real}(x::Vector{S}; tr=0.2)=var(winval(x, tr=tr))
#winvar{S <: Real}(x::DataVector{S}; tr::Real=0.2)=var(winval(remove(x), tr=tr))

#winsorized standard deviation
winstd{S <: Real}(x::Vector{S}; tr=0.2)=std(winval(x, tr=tr))
#winstd{S <: Real}(x::DataVector{S}; tr::Real=0.2)=std(winval(remove(x), tr=tr))


#Estimate the standard error of the gamma trimmed mean
trimse{S <: Real}(x::Vector{S}; tr::Real=0.2)=
    sqrt(winvar(x,tr=tr))/((1-2tr)*sqrt(length(x)))
#trimse{S <: Real}(x::DataVector{S}; tr::Real=0.2)=trimse(removeNA(x), tr=tr)

#Compute a 1-alpha confidence interval for the trimmed mean
function trimci{S <: Real}(x::Vector{S}; tr::Real=0.2, alpha::Real=0.05, nullvalue::Real=0, method=true)
    se  = trimse(x, tr=tr)
    n   = length(x)
    df::Int64   = n-2*floor(tr*n)-1
    estimate    = tmean(x, tr=tr)
    confint     = [estimate-Rmath.qt(1-alpha/2, df)*se, 
                   estimate+Rmath.qt(1-alpha/2, df)*se]
    statistic   = (estimate-nullvalue)/se
    pval        = 2*(1-Rmath.pt(abs(statistic),df))
    METHOD      = method ? "1-alpha confidence interval for the trimmed mean\n": nothing
    output           = testOutput()
    output.method    = METHOD
    output.df        = df
    output.estimate  = estimate
    output.ci        = confint
    output.statistic = statistic
    output.p         = pval
    return output
end
#trimci{S <: Real}(x::DataVector{S}; tr=0.2, alpha=0.05, nullvalue=0, method=true)=
    trimci(removeNA(x), tr=tr, alpha=alpha, nullvalue=nullvalue, method=method)


#Stein's method
function stein1{S <: Real}(x::Vector{S}, delta::Real; alpha::Real=0.05, pow::Real=0.8, oneside::Bool=false, n=nothing, variance=nothing)
    delta    = abs(delta)
    n        = n==nothing?length(x):n 
    variance = variance==nothing?var(x):variance 
    df       = n-1
    alpha    = !oneside?alpha/2:alpha
    d        = (delta/(Rmath.qt(pow, df)-Rmath.qt(alpha, df)))*(delta/(Rmath.qt(pow, df)-Rmath.qt(alpha, df)))
    N        = max([n, floor(variance/d)+1])
    return integer(N)
end
#stein1{S <: Real}(x::DataVector{S}, delta::Real; alpha::Real=0.05, pow::Real=0.8, oneside::Bool=false, n=nothing, variance=nothing)=
 #   stein1(removeNA(x), delta, alpha=alpha, pow=pow, oneside=oneside, n=n, variance=variance)

    <commit_msg>Update functions.jl<commit_after>#Trimmed mean; amount of trimming defaults to 20% or 0.2
function tmean{S <: Real}(x::Vector{S}; tr::Real=0.2)
    if tr < 0 || tr > 0.5
        error("tr cannot be smaller than 0 or larger than 0.5")
    elseif tr == 0
        return mean(x)
    elseif tr == .5
        return median(x, checknan=false)
    else 
        n   = length(x)
        lo  = floor(n*tr)+1
        hi  = n+1-lo
        return mean(sort!(copy(x))[lo:hi])
    end
end
#tmean{S <: Real}(x::DataVector{S}; tr::Real=0.2)=tmean(removeNA(x), tr=tr)

function tmean!{S <: Real}(x::Vector{S}; tr::Real=0.2)
    if tr < 0 || tr > 0.5
        error("tr cannot be smaller than 0 or larger than 0.5")
    elseif tr == 0
        return mean(x)
    elseif tr == .5
        return median!(x, checknan=false)
    else 
        n   = length(x)
        lo  = floor(n*tr)+1
        hi  = n+1-lo
        return mean(sort!(x)[lo:hi])
    end
end


#Winsorize data
function winval{S <: Real}(x::Vector{S}; tr::Real=0.2)
    xcopy   = copy(x)
    n       = length(x)
    xcopy   = sort!(xcopy)
    ibot    = floor(tr*n)+1
    itop    = n-ibot+1
    xbot, xtop = xcopy[ibot], xcopy[itop]
    return  [x[i]<=xbot?xbot:(x[i]>=xtop?xtop:x[i]) for i=1:n]
end
winval{S <: Real}(x::DataVector{S}; tr::Real=0.2)=winval(removeNA(x), tr=tr)

#Winsorized mean
winmean{S <: Real}(x::Vector{S}; tr=0.2)=mean(winval(x, tr=tr))
#winmean{S <: Real}(x::DataVector{S}; tr::Real=0.2)=mean(winval(removeNA(x), tr=tr))

#Winsorized variance
winvar{S <: Real}(x::Vector{S}; tr=0.2)=var(winval(x, tr=tr))
#winvar{S <: Real}(x::DataVector{S}; tr::Real=0.2)=var(winval(remove(x), tr=tr))

#winsorized standard deviation
winstd{S <: Real}(x::Vector{S}; tr=0.2)=std(winval(x, tr=tr))
#winstd{S <: Real}(x::DataVector{S}; tr::Real=0.2)=std(winval(remove(x), tr=tr))


#Estimate the standard error of the gamma trimmed mean
trimse{S <: Real}(x::Vector{S}; tr::Real=0.2)=
    sqrt(winvar(x,tr=tr))/((1-2tr)*sqrt(length(x)))
#trimse{S <: Real}(x::DataVector{S}; tr::Real=0.2)=trimse(removeNA(x), tr=tr)

#Compute a 1-alpha confidence interval for the trimmed mean
function trimci{S <: Real}(x::Vector{S}; tr::Real=0.2, alpha::Real=0.05, nullvalue::Real=0, method=true)
    se  = trimse(x, tr=tr)
    n   = length(x)
    df::Int64   = n-2*floor(tr*n)-1
    estimate    = tmean(x, tr=tr)
    confint     = [estimate-Rmath.qt(1-alpha/2, df)*se, 
                   estimate+Rmath.qt(1-alpha/2, df)*se]
    statistic   = (estimate-nullvalue)/se
    pval        = 2*(1-Rmath.pt(abs(statistic),df))
    METHOD      = method ? "1-alpha confidence interval for the trimmed mean\n": nothing
    output           = testOutput()
    output.method    = METHOD
    output.df        = df
    output.estimate  = estimate
    output.ci        = confint
    output.statistic = statistic
    output.p         = pval
    return output
end
#trimci{S <: Real}(x::DataVector{S}; tr=0.2, alpha=0.05, nullvalue=0, method=true)=
 #   trimci(removeNA(x), tr=tr, alpha=alpha, nullvalue=nullvalue, method=method)


#Stein's method
function stein1{S <: Real}(x::Vector{S}, delta::Real; alpha::Real=0.05, pow::Real=0.8, oneside::Bool=false, n=nothing, variance=nothing)
    delta    = abs(delta)
    n        = n==nothing?length(x):n 
    variance = variance==nothing?var(x):variance 
    df       = n-1
    alpha    = !oneside?alpha/2:alpha
    d        = (delta/(Rmath.qt(pow, df)-Rmath.qt(alpha, df)))*(delta/(Rmath.qt(pow, df)-Rmath.qt(alpha, df)))
    N        = max([n, floor(variance/d)+1])
    return integer(N)
end
#stein1{S <: Real}(x::DataVector{S}, delta::Real; alpha::Real=0.05, pow::Real=0.8, oneside::Bool=false, n=nothing, variance=nothing)=
 #   stein1(removeNA(x), delta, alpha=alpha, pow=pow, oneside=oneside, n=n, variance=variance)

    
{'fraction_non_alphanumeric': 0.18869921926690486, 'fraction_numerical': 0.025142252216488026, 'mean_word_length': 3.336201950659782, 'pattern_counts': {'":': 2, '<': 43, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6565047', 'n_tokens_mistral': 3084, 'n_tokens_neox': 2990, 'n_words': 593}