<commit_before>%%
%% %CopyrightBegin%
%% 
%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
%% 
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%% 
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%% 
%% %CopyrightEnd%

-module(filelib).

%% File utilities.

-export([wildcard/1, wildcard/2, is_dir/1, is_file/1, is_regular/1, 
	 compile_wildcard/1]).
-export([fold_files/5, last_modified/1, file_size/1, ensure_dir/1]).

-export([wildcard/3, is_dir/2, is_file/2, is_regular/2]).
-export([fold_files/6, last_modified/2, file_size/2]).

-include_lib("kernel/include/file.hrl").

-define(HANDLE_ERROR(Expr),
	try
	    Expr
	catch
	    error:{badpattern,_}=UnUsUalVaRiAbLeNaMe ->
		%% Get the stack backtrace correct.
		erlang:error(UnUsUalVaRiAbLeNaMe)
	end).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec wildcard(name()) -> [file:filename()].
wildcard(Pattern) when is_list(Pattern) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, file)).

-spec wildcard(name(), name() | atom()) -> [file:filename()].
wildcard(Pattern, Cwd) when is_list(Pattern), is_list(Cwd) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, file));
wildcard(Pattern, Mod) when is_list(Pattern), is_atom(Mod) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Mod)).

-spec wildcard(name(), name(), atom()) -> [file:filename()].
wildcard(Pattern, Cwd, Mod)
  when is_list(Pattern), is_list(Cwd), is_atom(Mod) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, Mod)).

-spec is_dir(name()) -> boolean().
is_dir(Dir) ->
    do_is_dir(Dir, file).

-spec is_dir(name(), atom()) -> boolean().
is_dir(Dir, Mod) when is_atom(Mod) ->
    do_is_dir(Dir, Mod).

-spec is_file(name()) -> boolean().
is_file(File) ->
    do_is_file(File, file).

-spec is_file(name(), atom()) -> boolean().
is_file(File, Mod) when is_atom(Mod) ->
    do_is_file(File, Mod).

-spec is_regular(name()) -> boolean().
is_regular(File) ->
    do_is_regular(File, file).
    
-spec is_regular(name(), atom()) -> boolean().
is_regular(File, Mod) when is_atom(Mod) ->
    do_is_regular(File, Mod).
    
-spec fold_files(name(), string(), boolean(), fun((_,_) -> _), _) -> _.
fold_files(Dir, RegExp, Recursive, Fun, Acc) ->
    do_fold_files(Dir, RegExp, Recursive, Fun, Acc, file).

-spec fold_files(name(), string(), boolean(), fun((_,_) -> _), _, atom()) -> _.
fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod) when is_atom(Mod) ->
    do_fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod).

-spec last_modified(name()) -> date_time() | 0.
last_modified(File) ->
    do_last_modified(File, file).

-spec last_modified(name(), atom()) -> date_time() | 0.
last_modified(File, Mod) when is_atom(Mod) ->
    do_last_modified(File, Mod).

-spec file_size(name()) -> non_neg_integer().
file_size(File) ->
    do_file_size(File, file).

-spec file_size(name(), atom()) -> non_neg_integer().
file_size(File, Mod) when is_atom(Mod) ->
    do_file_size(File, Mod).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

do_wildcard(Pattern, Mod) when is_list(Pattern) ->
    do_wildcard_comp(do_compile_wildcard(Pattern), Mod).

do_wildcard_comp({compiled_wildcard,{exists,File}}, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok,_} -> [File];
	_ -> []
    end;
do_wildcard_comp({compiled_wildcard,[Base|Rest]}, Mod) ->
    do_wildcard_1([Base], Rest, Mod).

do_wildcard(Pattern, Cwd, Mod) when is_list(Pattern), is_list(Cwd) ->
    do_wildcard_comp(do_compile_wildcard(Pattern), Cwd, Mod).

do_wildcard_comp({compiled_wildcard,{exists,File}}, Cwd, Mod) ->
    case eval_read_file_info(filename:absname(File, Cwd), Mod) of
	{ok,_} -> [File];
	_ -> []
    end;
do_wildcard_comp({compiled_wildcard,[current|Rest]}, Cwd0, Mod) ->
    Cwd = filename:join([Cwd0]),		%Slash away redundant slashes.
    PrefixLen = length(Cwd)+1,
    [lists:nthtail(PrefixLen, N) || N <- do_wildcard_1([Cwd], Rest, Mod)];
do_wildcard_comp({compiled_wildcard,[Base|Rest]}, _Cwd, Mod) ->
    do_wildcard_1([Base], Rest, Mod).

do_is_dir(Dir, Mod) ->
    case eval_read_file_info(Dir, Mod) of
	{ok, #file_info{type=directory}} ->
	    true;
	_ ->
	    false
    end.

do_is_file(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, #file_info{type=regular}} ->
	    true;
	{ok, #file_info{type=directory}} ->
	    true;
        _ ->
            false
    end.

do_is_regular(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, #file_info{type=regular}} ->
	    true;
        _ ->
            false
    end.

%% fold_files(Dir, RegExp, Recursive, Fun, AccIn).

%% folds the function Fun(F, Acc) -> Acc1 over
%%   all files <F> in <Dir> that match the regular expression <RegExp>
%%   If <Recursive> is true all sub-directories to <Dir> are processed

do_fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod) ->
    {ok, Re1} = re:compile(RegExp),
    do_fold_files1(Dir, Re1, Recursive, Fun, Acc, Mod).

do_fold_files1(Dir, RegExp, Recursive, Fun, Acc, Mod) ->
    case eval_list_dir(Dir, Mod) of
	{ok, Files} -> do_fold_files2(Files, Dir, RegExp, Recursive, Fun, Acc, Mod);
	{error, _}  -> Acc
    end.

do_fold_files2([], _Dir, _RegExp, _Recursive, _Fun, Acc, _Mod) -> 
    Acc;
do_fold_files2([File|T], Dir, RegExp, Recursive, Fun, Acc0, Mod) ->
    FullName = filename:join(Dir, File),
    case do_is_regular(FullName, Mod) of
	true  ->
	    case re:run(File, RegExp, [{capture,none}]) of
		match  -> 
		    Acc = Fun(FullName, Acc0),
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc, Mod);
		nomatch ->
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod)
	    end;
	false ->
	    case Recursive andalso do_is_dir(FullName, Mod) of
		true ->
		    Acc1 = do_fold_files1(FullName, RegExp, Recursive,
					  Fun, Acc0, Mod),
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc1, Mod);
		false ->
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod)
	    end
    end.

do_last_modified(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, Info} ->
	    Info#file_info.mtime;
	_ ->
	    0
    end.

do_file_size(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, Info} ->
	    Info#file_info.size;
	_ ->
	    0
    end.

%%----------------------------------------------------------------------
%% +type ensure_dir(X) -> ok | {error, Reason}.
%% +type X = filename() | dirname()
%% ensures that the directory name required to create D exists

-spec ensure_dir(name()) -> 'ok' | {'error', posix()}.
ensure_dir("/") ->
    ok;
ensure_dir(F) ->
    Dir = filename:dirname(F),
    case do_is_dir(Dir, file) of
	true ->
	    ok;
	false ->
	    ensure_dir(Dir),
	    file:make_dir(Dir)
    end.



%%%
%%% Pattern matching using a compiled wildcard.
%%%

do_wildcard_1(Files, Pattern, Mod) ->
    do_wildcard_2(Files, Pattern, [], Mod).

do_wildcard_2([File|Rest], Pattern, Result, Mod) ->
    do_wildcard_2(Rest, Pattern, do_wildcard_3(File, Pattern, Result, Mod), Mod);
do_wildcard_2([], _, Result, _Mod) -><commit_msg>Don't return a false {error,eexist} in filelib:ensure_dir/1<commit_after>%%
%% %CopyrightBegin%
%% 
%% Copyright Ericsson AB 1997-2009. All Rights Reserved.
%% 
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%% 
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%% 
%% %CopyrightEnd%

-module(filelib).

%% File utilities.

-export([wildcard/1, wildcard/2, is_dir/1, is_file/1, is_regular/1, 
	 compile_wildcard/1]).
-export([fold_files/5, last_modified/1, file_size/1, ensure_dir/1]).

-export([wildcard/3, is_dir/2, is_file/2, is_regular/2]).
-export([fold_files/6, last_modified/2, file_size/2]).

-include_lib("kernel/include/file.hrl").

-define(HANDLE_ERROR(Expr),
	try
	    Expr
	catch
	    error:{badpattern,_}=UnUsUalVaRiAbLeNaMe ->
		%% Get the stack backtrace correct.
		erlang:error(UnUsUalVaRiAbLeNaMe)
	end).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

-spec wildcard(name()) -> [file:filename()].
wildcard(Pattern) when is_list(Pattern) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, file)).

-spec wildcard(name(), name() | atom()) -> [file:filename()].
wildcard(Pattern, Cwd) when is_list(Pattern), is_list(Cwd) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, file));
wildcard(Pattern, Mod) when is_list(Pattern), is_atom(Mod) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Mod)).

-spec wildcard(name(), name(), atom()) -> [file:filename()].
wildcard(Pattern, Cwd, Mod)
  when is_list(Pattern), is_list(Cwd), is_atom(Mod) ->
    ?HANDLE_ERROR(do_wildcard(Pattern, Cwd, Mod)).

-spec is_dir(name()) -> boolean().
is_dir(Dir) ->
    do_is_dir(Dir, file).

-spec is_dir(name(), atom()) -> boolean().
is_dir(Dir, Mod) when is_atom(Mod) ->
    do_is_dir(Dir, Mod).

-spec is_file(name()) -> boolean().
is_file(File) ->
    do_is_file(File, file).

-spec is_file(name(), atom()) -> boolean().
is_file(File, Mod) when is_atom(Mod) ->
    do_is_file(File, Mod).

-spec is_regular(name()) -> boolean().
is_regular(File) ->
    do_is_regular(File, file).
    
-spec is_regular(name(), atom()) -> boolean().
is_regular(File, Mod) when is_atom(Mod) ->
    do_is_regular(File, Mod).
    
-spec fold_files(name(), string(), boolean(), fun((_,_) -> _), _) -> _.
fold_files(Dir, RegExp, Recursive, Fun, Acc) ->
    do_fold_files(Dir, RegExp, Recursive, Fun, Acc, file).

-spec fold_files(name(), string(), boolean(), fun((_,_) -> _), _, atom()) -> _.
fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod) when is_atom(Mod) ->
    do_fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod).

-spec last_modified(name()) -> date_time() | 0.
last_modified(File) ->
    do_last_modified(File, file).

-spec last_modified(name(), atom()) -> date_time() | 0.
last_modified(File, Mod) when is_atom(Mod) ->
    do_last_modified(File, Mod).

-spec file_size(name()) -> non_neg_integer().
file_size(File) ->
    do_file_size(File, file).

-spec file_size(name(), atom()) -> non_neg_integer().
file_size(File, Mod) when is_atom(Mod) ->
    do_file_size(File, Mod).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

do_wildcard(Pattern, Mod) when is_list(Pattern) ->
    do_wildcard_comp(do_compile_wildcard(Pattern), Mod).

do_wildcard_comp({compiled_wildcard,{exists,File}}, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok,_} -> [File];
	_ -> []
    end;
do_wildcard_comp({compiled_wildcard,[Base|Rest]}, Mod) ->
    do_wildcard_1([Base], Rest, Mod).

do_wildcard(Pattern, Cwd, Mod) when is_list(Pattern), is_list(Cwd) ->
    do_wildcard_comp(do_compile_wildcard(Pattern), Cwd, Mod).

do_wildcard_comp({compiled_wildcard,{exists,File}}, Cwd, Mod) ->
    case eval_read_file_info(filename:absname(File, Cwd), Mod) of
	{ok,_} -> [File];
	_ -> []
    end;
do_wildcard_comp({compiled_wildcard,[current|Rest]}, Cwd0, Mod) ->
    Cwd = filename:join([Cwd0]),		%Slash away redundant slashes.
    PrefixLen = length(Cwd)+1,
    [lists:nthtail(PrefixLen, N) || N <- do_wildcard_1([Cwd], Rest, Mod)];
do_wildcard_comp({compiled_wildcard,[Base|Rest]}, _Cwd, Mod) ->
    do_wildcard_1([Base], Rest, Mod).

do_is_dir(Dir, Mod) ->
    case eval_read_file_info(Dir, Mod) of
	{ok, #file_info{type=directory}} ->
	    true;
	_ ->
	    false
    end.

do_is_file(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, #file_info{type=regular}} ->
	    true;
	{ok, #file_info{type=directory}} ->
	    true;
        _ ->
            false
    end.

do_is_regular(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, #file_info{type=regular}} ->
	    true;
        _ ->
            false
    end.

%% fold_files(Dir, RegExp, Recursive, Fun, AccIn).

%% folds the function Fun(F, Acc) -> Acc1 over
%%   all files <F> in <Dir> that match the regular expression <RegExp>
%%   If <Recursive> is true all sub-directories to <Dir> are processed

do_fold_files(Dir, RegExp, Recursive, Fun, Acc, Mod) ->
    {ok, Re1} = re:compile(RegExp),
    do_fold_files1(Dir, Re1, Recursive, Fun, Acc, Mod).

do_fold_files1(Dir, RegExp, Recursive, Fun, Acc, Mod) ->
    case eval_list_dir(Dir, Mod) of
	{ok, Files} -> do_fold_files2(Files, Dir, RegExp, Recursive, Fun, Acc, Mod);
	{error, _}  -> Acc
    end.

do_fold_files2([], _Dir, _RegExp, _Recursive, _Fun, Acc, _Mod) -> 
    Acc;
do_fold_files2([File|T], Dir, RegExp, Recursive, Fun, Acc0, Mod) ->
    FullName = filename:join(Dir, File),
    case do_is_regular(FullName, Mod) of
	true  ->
	    case re:run(File, RegExp, [{capture,none}]) of
		match  -> 
		    Acc = Fun(FullName, Acc0),
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc, Mod);
		nomatch ->
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod)
	    end;
	false ->
	    case Recursive andalso do_is_dir(FullName, Mod) of
		true ->
		    Acc1 = do_fold_files1(FullName, RegExp, Recursive,
					  Fun, Acc0, Mod),
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc1, Mod);
		false ->
		    do_fold_files2(T, Dir, RegExp, Recursive, Fun, Acc0, Mod)
	    end
    end.

do_last_modified(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, Info} ->
	    Info#file_info.mtime;
	_ ->
	    0
    end.

do_file_size(File, Mod) ->
    case eval_read_file_info(File, Mod) of
	{ok, Info} ->
	    Info#file_info.size;
	_ ->
	    0
    end.

%%----------------------------------------------------------------------
%% +type ensure_dir(X) -> ok | {error, Reason}.
%% +type X = filename() | dirname()
%% ensures that the directory name required to create D exists

-spec ensure_dir(name()) -> 'ok' | {'error', posix()}.
ensure_dir("/") ->
    ok;
ensure_dir(F) ->
    Dir = filename:dirname(F),
    case do_is_dir(Dir, file) of
	true ->
	    ok;
	false ->
	    ensure_dir(Dir),
	    case file:make_dir(Dir) of
		{error,eexist}=EExist ->
		    case do_is_dir(Dir, file) of
			true ->
			    ok;
			false ->
			    EExist
		    end;
		Err ->
		    Err
	    end
    end.



%%%
%%% Pattern matching using a compiled wildcard.
%%%

do_wildcard_1(Files, Pattern, Mod) ->
    do_wildcard_2(Files, Pattern, [], Mod).

do_wildcard_2([File|Rest], Pattern, Result, Mod) ->
    do_wildcard_2(Rest, Pattern, do_wildcard_3(File, Pattern, Result, Mod), Mod);
do_wildcard_2([], _, Result, _Mod) ->
{'fraction_non_alphanumeric': 0.18735409857933702, 'fraction_numerical': 0.008470619622490495, 'mean_word_length': 3.78125, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 183, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7190187', 'n_tokens_mistral': 5727, 'n_tokens_neox': 5458, 'n_words': 1275}