<commit_before>
@doc doc"""
# HDFSException
Thrown by HDFS APIs.
""" ->
type HDFSException <: Exception
    message::AbstractString
end


@doc doc"""
HDFSProtocol: Hadoop RPC client for namenode protocol.
""" ->
typealias HDFSProtocol HadoopRpcProtocol{ClientNamenodeProtocolBlockingStub}

for fn in (:getListing, :getFileInfo, :getBlockLocations, :getServerDefaults, :getFsStats, :setReplication, :getContentSummary, :mkdirs, :rename2, :rename, :delete, :create, :complete, :addBlock, :renewLease, :setTimes)
    @eval begin
        (hadoop.hdfs.$fn)(p::HDFSProtocol, inp) = (hadoop.hdfs.$fn)(p.stub, p.controller, inp)
    end
end

@doc doc"""
# HDFSClient
A client to the namenode in a HDFS cluster and holds a connection to it.
It also stores the folder context for using relative paths in APIs that use the client.
""" ->
type HDFSClient
    nn_conn::HDFSProtocol
    wd::AbstractString
    server_defaults::Nullable{FsServerDefaultsProto}

    function HDFSClient(host::AbstractString, port::Integer, ugi::UserGroupInformation=UserGroupInformation())
        nn_conn = HDFSProtocol(host, port, ugi)
        new(nn_conn, "/", Nullable{FsServerDefaultsProto}())
    end
end

function show(io::IO, client::HDFSClient)
    show(client.nn_conn)
    println(io, "    pwd: $(client.wd)")
    nothing
end

set_debug(client::HDFSClient, debug::Bool) = set_debug(client.nn_conn, debug)

abspath(client::HDFSClient, path::AbstractString) = abspath(client.wd, path)
function cd(client::HDFSClient, path::AbstractString)
    wd = abspath(client, path)
    !isdir(client, wd) && throw(HDFSException("Directory not found: $wd"))
    client.wd = wd
end
pwd(client::HDFSClient) = client.wd



@doc doc"""
# HDFSFile
Holds a HDFSClient and path pointing to a file on HDFS
""" ->
type HDFSFile
    client::HDFSClient
    path::AbstractString
end

function HDFSFile(uristr::AbstractString)
    uri = URI(uristr)
    (uri.schema == "hdfs") || throw(HDFSException("Not a HDFS URI: $uristr"))
    client = HDFSClient(uri.host, uri.port, UserGroupInformation(uri.userinfo))
    HDFSFile(client, uri.path)
end

function show(io::IO, file::HDFSFile)
    println(io, "HDFSFile: ", string(convert(URI, file)))
    nothing
end

function convert(::Type{URI}, file::HDFSFile)
    client = file.client
    ch = client.nn_conn.channel
    user = username(ch.ugi)
    user_spec = isempty(user) ? user : "$(user)@"
    URI("hdfs://$(user_spec)$(ch.host):$(ch.port)$(abspath(client, file.path))")
end

@doc doc"""
# HDFSFileInfo
Stat structure for file/foledr in HDFS
""" ->
type HDFSFileInfo
    kind::Int32
    name::AbstractString
    size::UInt64
    replications::UInt32<commit_msg>fix for URIParser changes (schema to scheme)<commit_after>
@doc doc"""
# HDFSException
Thrown by HDFS APIs.
""" ->
type HDFSException <: Exception
    message::AbstractString
end


@doc doc"""
HDFSProtocol: Hadoop RPC client for namenode protocol.
""" ->
typealias HDFSProtocol HadoopRpcProtocol{ClientNamenodeProtocolBlockingStub}

for fn in (:getListing, :getFileInfo, :getBlockLocations, :getServerDefaults, :getFsStats, :setReplication, :getContentSummary, :mkdirs, :rename2, :rename, :delete, :create, :complete, :addBlock, :renewLease, :setTimes)
    @eval begin
        (hadoop.hdfs.$fn)(p::HDFSProtocol, inp) = (hadoop.hdfs.$fn)(p.stub, p.controller, inp)
    end
end

@doc doc"""
# HDFSClient
A client to the namenode in a HDFS cluster and holds a connection to it.
It also stores the folder context for using relative paths in APIs that use the client.
""" ->
type HDFSClient
    nn_conn::HDFSProtocol
    wd::AbstractString
    server_defaults::Nullable{FsServerDefaultsProto}

    function HDFSClient(host::AbstractString, port::Integer, ugi::UserGroupInformation=UserGroupInformation())
        nn_conn = HDFSProtocol(host, port, ugi)
        new(nn_conn, "/", Nullable{FsServerDefaultsProto}())
    end
end

function show(io::IO, client::HDFSClient)
    show(client.nn_conn)
    println(io, "    pwd: $(client.wd)")
    nothing
end

set_debug(client::HDFSClient, debug::Bool) = set_debug(client.nn_conn, debug)

abspath(client::HDFSClient, path::AbstractString) = abspath(client.wd, path)
function cd(client::HDFSClient, path::AbstractString)
    wd = abspath(client, path)
    !isdir(client, wd) && throw(HDFSException("Directory not found: $wd"))
    client.wd = wd
end
pwd(client::HDFSClient) = client.wd



@doc doc"""
# HDFSFile
Holds a HDFSClient and path pointing to a file on HDFS
""" ->
type HDFSFile
    client::HDFSClient
    path::AbstractString
end

function HDFSFile(uristr::AbstractString)
    uri = URI(uristr)
    (uri.scheme == "hdfs") || throw(HDFSException("Not a HDFS URI: $uristr"))
    client = HDFSClient(uri.host, uri.port, UserGroupInformation(uri.userinfo))
    HDFSFile(client, uri.path)
end

function show(io::IO, file::HDFSFile)
    println(io, "HDFSFile: ", string(convert(URI, file)))
    nothing
end

function convert(::Type{URI}, file::HDFSFile)
    client = file.client
    ch = client.nn_conn.channel
    user = username(ch.ugi)
    user_spec = isempty(user) ? user : "$(user)@"
    URI("hdfs://$(user_spec)$(ch.host):$(ch.port)$(abspath(client, file.path))")
end

@doc doc"""
# HDFSFileInfo
Stat structure for file/foledr in HDFS
""" ->
type HDFSFileInfo
    kind::Int32
    name::AbstractString
    size::UInt64
    replications::UInt32
{'fraction_non_alphanumeric': 0.1343311506080449, 'fraction_numerical': 0.0026192703461178674, 'mean_word_length': 4.855421686746988, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3584355', 'n_tokens_mistral': 1898, 'n_tokens_neox': 1789, 'n_words': 435}