<commit_before>#!/usr/bin/perl
use strict;
use warnings;

use MaxMind::DB::Writer::Tree;
use Net::Works::Network;
use Text::CSV;

# http://www.quickmeme.com/img/d6/d6a1143f571184db25f94613edd43b40af6d3a629221aba00d9efdcfef5efd84.jpg
#   --klady

# MMDB format docs: https://maxmind.github.io/MaxMind-DB/
# MaxMind::DB:Writer::Tree API docs: https://metacpan.org/pod/MaxMind::DB::Writer::Tree

my %types = (
    proxy_type => 'utf8_string',
    proxy_level => 'utf8_string',
);

# record_size: there's no actual documentation as to how
# to choose a value in any of the documentation online.
# If I understand it correctly (I probably don't), it's
# the arity of each node. I chose 24 because the example
# in the docs used it, and I can't justify changing it.
# In retrospect, "chose" is a bit strong. -klady
my $tree = MaxMind::DB::Writer::Tree->new(
    ip_version            => 4,
    record_size           => 24,
    database_type         => 'Neustar-IP-Gold',
    languages             => ['en'],
    description           => { en => 'Neustar IP Intelligence Gold Edition' },
    map_key_type_callback => sub { $types{ $_[0] } },
);


# I have no clue why it needs to be in binary mode,
# but it just fails to parse everything coming out of
# Python's csv module, so binary it is! -klady
my $csv = Text::CSV->new({ binary => 1 });<commit_msg>Generate mmdb refactor for Tor and Neustar<commit_after>#!/usr/bin/perl
use strict;
use warnings;

use MaxMind::DB::Writer::Tree;
use Net::Works::Network;
use Text::CSV;

# http://www.quickmeme.com/img/d6/d6a1143f571184db25f94613edd43b40af6d3a629221aba00d9efdcfef5efd84.jpg
#   --klady

# MMDB format docs: https://maxmind.github.io/MaxMind-DB/
# MaxMind::DB:Writer::Tree API docs: https://metacpan.org/pod/MaxMind::DB::Writer::Tree

my $num_args = $#ARGV + 1;
if ($num_args != 1) {
    print "Usage: generate_mmdb.pl <tor or neustar>\n";
    exit;
}

my $db_type = $ARGV[0];
my $db_name = "";
my $db_description = "";
if ($db_type eq "tor") {
    $db_name = "Tor-Project-List";
    $db_description = "Tor project exit address listing";
} elsif ($db_type eq "neustar") {
    $db_name = "Neustar-IP-Gold";
    $db_description = "Neustar IP Intelligence Gold Edition";
} else {
    print "Usage: generate_mmdb.pl <tor or neustar>\n";
    exit;
}

my %types = (
    proxy_type => 'utf8_string',
    proxy_level => 'utf8_string',
);

# record_size: there's no actual documentation as to how
# to choose a value in any of the documentation online.
# If I understand it correctly (I probably don't), it's
# the arity of each node. I chose 24 because the example
# in the docs used it, and I can't justify changing it.
# In retrospect, "chose" is a bit strong. -klady
my $tree = MaxMind::DB::Writer::Tree->new(
    ip_version            => 4,
    record_size           => 24,
    database_type         => $db_name,
    languages             => ['en'],
    description           => { en => $db_description },
    map_key_type_callback => sub { $types{ $_[0] } },
);


# I have no clue why it needs to be in binary mode,
# but it just fails to parse everything coming out of
# Python's csv module, so binary it is! -klady
my $csv = Text::CSV->new({ binary => 1 });
{'fraction_non_alphanumeric': 0.1377264209868832, 'fraction_numerical': 0.02966895690193629, 'mean_word_length': 3.448611111111111, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707666', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1107, 'n_words': 337}