<commit_before>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module contains eunit tests of tokens module.
%%% @end
%%%-------------------------------------------------------------------
-module(tokens_tests).
-author("<NAME>").

-ifdef(TEST).

-include_lib("eunit/include/eunit.hrl").

-include("onedata.hrl").
-include("aai/aai.hrl").
-include("graph_sync/graph_sync.hrl").
-include("errors.hrl").

-define(OZ_DOMAIN, <<"onezone.example.com">>).<commit_msg>VFS-5731 Add missing header file in tokens_tests<commit_after>%%%-------------------------------------------------------------------
%%% @author <NAME>
%%% @copyright (C) 2019 ACK CYFRONET AGH
%%% This software is released under the MIT license
%%% cited in 'LICENSE.txt'.
%%% @end
%%%-------------------------------------------------------------------
%%% @doc
%%% This module contains eunit tests of tokens module.
%%% @end
%%%-------------------------------------------------------------------
-module(tokens_tests).
-author("<NAME>").

-ifdef(TEST).

-include_lib("eunit/include/eunit.hrl").

-include("aai/aai.hrl").
-include("errors.hrl").
-include("graph_sync/graph_sync.hrl").
-include("http/headers.hrl").
-include("onedata.hrl").

-define(OZ_DOMAIN, <<"onezone.example.com">>).
{'fraction_non_alphanumeric': 0.42876076872100727, 'fraction_numerical': 0.007952286282306162, 'mean_word_length': 1.5724020442930153, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6688914', 'n_tokens_mistral': 442, 'n_tokens_neox': 421, 'n_words': 88}