<commit_before>%%
%%
%%

-module(eiconv).
-author("<NAME> <<EMAIL>>").

-export([open/2, conv/2, chunk/2, reset/1, close/1]).

% easy api one shot convert api
-export([convert/2, convert/3]).

-on_load(init/0).

% @doc Load the nif
%
init() ->
    NifName = "eiconv_nif",
    NifFileName = case code:priv_dir(esqlite) of
        {error, bad_name} -> filename:join("priv", NifName);
        Dir -> filename:join(Dir, NifName)
    end,
    ok = erlang:load_nif(NifFileName, 0).

% @doc Open a new encoder which can be used to convert text from FromCode into ToCode.
%
open(_ToCode, _FromCode) ->
    exit(nif_library_not_loaded).

% @doc Convert Input into the requested encoding.
%
conv(_Cd, _Input) ->
    exit(nif_library_not_loaded).

% @doc Convert a chunk, returns {ok, Converted, Rest}
%
chunk(_Cd, _Input) ->
    exit(nif_library_not_loaded).

% @doc Reset the cd structure, returns{ok, Rest}
%
reset(_Cd) ->
    exit(nif_library_not_loaded).<commit_msg>Fixed the loading of the nif<commit_after>%%
%%
%%

-module(eiconv).
-author("<NAME> <<EMAIL>>").

-export([open/2, conv/2, chunk/2, reset/1, close/1]).

% easy api one shot convert api
-export([convert/2, convert/3]).

-on_load(init/0).

% @doc Load the nif
%
init() ->
    NifName = "eiconv_nif",
    NifFileName = case code:priv_dir(eiconv) of
        {error, bad_name} -> filename:join(["priv", NifName]);
        Dir -> filename:join([Dir, NifName])
    end,

    ok = erlang:load_nif(NifFileName, 0).

% @doc Open a new encoder which can be used to convert text from FromCode into ToCode.
%
open(_ToCode, _FromCode) ->
    exit(nif_library_not_loaded).

% @doc Convert Input into the requested encoding.
%
conv(_Cd, _Input) ->
    exit(nif_library_not_loaded).

% @doc Convert a chunk, returns {ok, Converted, Rest}
%
chunk(_Cd, _Input) ->
    exit(nif_library_not_loaded).

% @doc Reset the cd structure, returns{ok, Rest}
%
reset(_Cd) ->
    exit(nif_library_not_loaded).
{'fraction_non_alphanumeric': 0.1557503867973182, 'fraction_numerical': 0.009283135636926251, 'mean_word_length': 3.7665847665847667, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2918451', 'n_tokens_mistral': 733, 'n_tokens_neox': 721, 'n_words': 182}