<commit_before>%% Author: eric.yutao
%% Created: 2013-12-2
%% Description: TODO: Add description to uitl
-module(util).

%%
%% Include files
%%

%%
%% Exported Functions
%%
-export([]).
-compile(export_all).

%%
%% API Functions
%%
string_to_term(String) ->
	case erl_scan:string(String++".") of
		{ok, Tokens, _} ->
			case erl_parse:parse_term(Tokens) of
				{ok, Term} ->
					{ok, Term};
				Reason ->
					io:format("error string_to_term ~p~n", [{?MODULE, Reason}]),
					parse_error
			end;
		Reason ->
			io:format("error string_to_term ~p~n", [{?MODULE, Reason}]),
			scan_error
	end.

term_to_string(Term) ->
	lists:flatten(io_lib:format("~w", [Term])).


-spec uniq(List::list()) -> List1::list().
uniq([X, X|Xs]) -> uniq([X|Xs]);
uniq([X|Xs]) -> [X|uniq(Xs)];
uniq([]) -> [].


%%
%% Local Functions
%%
<commit_msg>	modified:   src/util_tools/util.erl<commit_after>%% Author: eric.yutao
%% Created: 2013-12-2
%% Description: TODO: Add description to uitl
-module(util).

%%
%% Include files
%%

%%
%% Exported Functions
%%
-export([]).
-compile(export_all).

%%
%% API Functions
%%
string_to_term(String) ->
	case erl_scan:string(String++".") of
		{ok, Tokens, _} ->
			case erl_parse:parse_term(Tokens) of
				{ok, Term} ->
					{ok, Term};
				Reason ->
					io:format("error string_to_term ~p~n", [{?MODULE, Reason}]),
					parse_error
			end;
		Reason ->
			io:format("error string_to_term ~p~n", [{?MODULE, Reason}]),
			scan_error
	end.

term_to_string(Term) ->
	lists:flatten(io_lib:format("~w", [Term])).

%% Ye olde uniq function
-spec uniq(List::list()) -> List1::list().
uniq([X, X|Xs]) -> uniq([X|Xs]);
uniq([X|Xs]) -> [X|uniq(Xs)];
uniq([]) -> [].


%%
%% Local Functions
%%

{'fraction_non_alphanumeric': 0.22366097704532079, 'fraction_numerical': 0.009417304296645085, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1038511', 'n_tokens_mistral': 765, 'n_tokens_neox': 680, 'n_words': 132}