<commit_before>#!perl

use lib 'lib', '../lib';
use Data::Dump;
use ProfilesEasyJSON::MegaUCSF;
use Test::More;
use Test::NoWarnings;
use strict;
use warnings;

plan tests => 13;

# TODO:
# - test a valid FNO

*preprocess_ucsf_identifier
    = \&ProfilesEasyJSON::MegaUCSF::preprocess_ucsf_identifier;

is_deeply( [ preprocess_ucsf_identifier( 'PrettyURL', 'anirvan.chatterjee' ) ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid PrettyURL' );

is_deeply( [ preprocess_ucsf_identifier( 'Person', '5396511' ) ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid Person' );

is_deeply( [ preprocess_ucsf_identifier( 'EmployeeID', '028272045' ) ],
           [ 'UserName', '<EMAIL>' ],
           'valid EmployeeID' );

is_deeply( [  preprocess_ucsf_identifier(
                           'URL', 'https://profiles.ucsf.edu/anirvan.chatterjee'
              )
           ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid URL with PrettyURL'
);

is_deeply( [ preprocess_ucsf_identifier(
                  'URL',
                  'https://profiles.ucsf.edu/ProfileDetails.aspx?Person=5396511'
             )
           ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid URL with Person ID'
);

is_deeply( [  preprocess_ucsf_identifier(
                              'URL', 'https://profiles.ucsf.edu/profile/1234567'
              )
           ],
           [ 'ProfilesNodeID', '1234567' ],
           'valid URL with node ID'
);

{<commit_msg>added an FNO test<commit_after>#!perl

use lib 'lib', '../lib';
use Data::Dump;
use ProfilesEasyJSON::MegaUCSF;
use Test::More;
use Test::NoWarnings;
use strict;
use warnings;

plan tests => 14;

*preprocess_ucsf_identifier
    = \&ProfilesEasyJSON::MegaUCSF::preprocess_ucsf_identifier;

is_deeply( [ preprocess_ucsf_identifier( 'PrettyURL', 'anirvan.chatterjee' ) ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid PrettyURL' );

is_deeply( [ preprocess_ucsf_identifier( 'Person', '5396511' ) ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid Person' );

is_deeply( [  preprocess_ucsf_identifier( 'FNO', '<EMAIL>'
              )
           ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid FNO'
);

is_deeply( [ preprocess_ucsf_identifier( 'EmployeeID', '028272045' ) ],
           [ 'UserName', '<EMAIL>' ],
           'valid EmployeeID' );

is_deeply( [  preprocess_ucsf_identifier(
                           'URL', 'https://profiles.ucsf.edu/anirvan.chatterjee'
              )
           ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid URL with PrettyURL'
);

is_deeply( [ preprocess_ucsf_identifier(
                  'URL',
                  'https://profiles.ucsf.edu/ProfileDetails.aspx?Person=5396511'
             )
           ],
           [ 'PrettyURL', 'anirvan.chatterjee' ],
           'valid URL with Person ID'
);

is_deeply( [  preprocess_ucsf_identifier(
                              'URL', 'https://profiles.ucsf.edu/profile/1234567'
              )
           ],
           [ 'ProfilesNodeID', '1234567' ],
           'valid URL with node ID'
);

{
{'fraction_non_alphanumeric': 0.14096916299559473, 'fraction_numerical': 0.024543738200125866, 'mean_word_length': 2.198189134808853, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4498417', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1042, 'n_words': 172}