<commit_before>
try mkdir 'install';
try mkdir 'install/languages';
try mkdir 'install/languages/perl6';
try mkdir 'install/languages/perl6/site';
try mkdir 'install/languages/perl6/site/panda';

my $state-file    = 'install/languages/perl6/site/panda/state';
my $projects-file = 'install/languages/perl6/site/panda/projects.json';

fetch-projects-json($projects-file);

my $projects = from-json $projects-file.IO.slurp;

# In case we ship a project that is just a fork of a project listed in the ecosystem, add
# the mapping here.
my %ex =
#    'git://github.com/FROGGS/perl6-digest-md5' => 'git://github.com/cosimo/perl6-digest-md5',
Nil;

# Walk the submodules and put its project information in panda's state file.
my $fh = $state-file.IO.open(:w);
for '.gitmodules'.IO.lines.grep(/^\turl/).map({ /$<url>=[\S+]$/; ~$<url> }) -> $url {
    my $p          = $projects.first({defined .<source-url> && $_.<source-url> ~~ /^ "{%ex{$url} // $url}" '.git'? $/});
    $p<repo-type>  = 'git';
    $p<source-url> = $url;
    $fh.say: $p<name> ~ ' installed ' ~ to-json($p).subst(/\n+/, '', :g);
}
$fh.close;

say $state-file;
say $projects-file;

sub fetch-projects-json($to) {
    try unlink $to;
    my $s;
    if %*ENV<http_proxy> {
        my ($host, $port) = %*ENV<http_proxy>.split('/').[2].split(':');
        $s = IO::Socket::INET.new(host=>$host, port=>$port.Int);
        $s.send("GET http://ecosystem-api.p6c.org/projects.json HTTP/1.1\nHost: ecosystem-api.p6c.org\nAccept: */*\nConnection: Close\n\n");
    }
    else {
        $s = IO::Socket::INET.new(:host<ecosystem-api.p6c.org>, :port(80));
        $s.send("GET /projects.json HTTP/1.0\n\n");
    }
    my ($buf, $g) = '';
    $buf ~= $g while $g = $s.get;

    if %*ENV<http_proxy> {
        $buf.=subst(:g,/'git://'/,'http://');
    }
    
    given open($to, :w) {
        .say: $buf.split(/\r?\n\r?\n/, 2)[1];
        .close;
    }<commit_msg>Unify get request for proxy/non-proxy<commit_after>
try mkdir 'install';
try mkdir 'install/languages';
try mkdir 'install/languages/perl6';
try mkdir 'install/languages/perl6/site';
try mkdir 'install/languages/perl6/site/panda';

my $state-file    = 'install/languages/perl6/site/panda/state';
my $projects-file = 'install/languages/perl6/site/panda/projects.json';

fetch-projects-json($projects-file);

my $projects = from-json $projects-file.IO.slurp;

# In case we ship a project that is just a fork of a project listed in the ecosystem, add
# the mapping here.
my %ex =
#    'git://github.com/FROGGS/perl6-digest-md5' => 'git://github.com/cosimo/perl6-digest-md5',
Nil;

# Walk the submodules and put its project information in panda's state file.
my $fh = $state-file.IO.open(:w);
for '.gitmodules'.IO.lines.grep(/^\turl/).map({ /$<url>=[\S+]$/; ~$<url> }) -> $url {
    my $p          = $projects.first({defined .<source-url> && $_.<source-url> ~~ /^ "{%ex{$url} // $url}" '.git'? $/});
    $p<repo-type>  = 'git';
    $p<source-url> = $url;
    $fh.say: $p<name> ~ ' installed ' ~ to-json($p).subst(/\n+/, '', :g);
}
$fh.close;

say $state-file;
say $projects-file;

sub fetch-projects-json($to) {
    try unlink $to;
    my $s;
    if %*ENV<http_proxy> {
        my ($host, $port) = %*ENV<http_proxy>.split('/').[2].split(':');
        $s = IO::Socket::INET.new(host=>$host, port=>$port.Int);
    }
    else {
        $s = IO::Socket::INET.new(:host<ecosystem-api.p6c.org>, :port(80));
    }
   $s.send("GET http://ecosystem-api.p6c.org/projects.json HTTP/1.1\nHost: ecosystem-api.p6c.org\nAccept: */*\nConnection: Close\n\n");
    my ($buf, $g) = '';
    $buf ~= $g while $g = $s.get;

    if %*ENV<http_proxy> {
        $buf.=subst(:g,/'git://'/,'http://');
    }
    
    given open($to, :w) {
        .say: $buf.split(/\r?\n\r?\n/, 2)[1];
        .close;
    }
{'fraction_non_alphanumeric': 0.22865450700502246, 'fraction_numerical': 0.010573618821041502, 'mean_word_length': 3.0169851380042463, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5903360', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1508, 'n_words': 287}