<commit_before>#!/usr/bin/env perl
use strict;
use warnings;
use 5.010;
use open ':encoding(utf8)';

use Mojo::UserAgent;

die 'no images specified' unless @ARGV;

my $ua = Mojo::UserAgent->new->max_redirects(10);
$ua->transactor->name('Docker');

my $maxRetries = 3;
sub ua_req {
	my $method = shift;
	my $tries = $maxRetries;
	my $tx;
	do {
		--$tries;
		$tx = $ua->$method(@_);
		return $tx if $tx->success;
	} while ($tries > 0);
	return $tx;
}

sub split_image_name {
	my $image = shift;
	if ($image =~ m{
		^
		(?: ([^/:]+) / )? # optional namespace
		([^/:]+)          # image name
		(?: : ([^/:]+) )? # optional tag
		$
	}x) {
		my ($namespace, $name, $tag) = (
			$1 // 'library', # namespace
			$2,              # image name
			$3 // 'latest',  # tag
		);
		return ("$namespace/$name", $tag);
	}
	die "unrecognized image name format in: $image";
}

sub get_token {
	my $repo = shift;
	state %tokens;
	return $tokens{$repo} if $tokens{$repo};
	my $realmTx = $ua->get("https://registry-1.docker.io/v2/$repo/tags/list");
	my $auth = $realmTx->res->headers->www_authenticate;
	die "unexpected WWW-Authenticate header: $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x;
	my $realm = $1;
	my $url = Mojo::URL->new;
	while ($realm =~ m{
		# key="val",
		([^=]+)
		=
		"([^"]+)"
		,?
	}xg) {
		my ($key, $val) = ($1, $2);
		if ($key eq 'realm') {
			$url->base(Mojo::URL->new($val));
		} else {
			$url->query->append($key => $val);
		}
	}
	$url = $url->to_abs;
	my $tokenTx = ua_req(get => $url);
	die "failed to fetch token for $repo" unless $tokenTx->success;
	return $tokens{$repo} = $tokenTx->res->json->{token};
}

sub get_manifest {
	my ($repo, $tag) = @_;
	my $image = "$repo:$tag";
	state %manifests;
	return $manifests{$image} if $manifests{$image};

	my $token = get_token($repo);
	my $authorizationHeader = { Authorization => "Bearer $token" };

	my $manifestTx = ua_req(get => "https://registry-1.docker.io/v2/$repo/manifests/$tag" => $authorizationHeader);
	die "failed to get manifest for $image" unless $manifestTx->success;
	return (
		$manifestTx->res->headers->header('Docker-Content-Digest'),
		$manifests{$image} = $manifestTx->res->json,
	);
}

sub get_blob_headers {
	my ($repo, $blob) = @_;
	my $key = $repo . '@' . $blob;
	state %headers;
	return $headers{$key} if $headers{$key};

	my $token = get_token($repo);
	my $authorizationHeader = { Authorization => "Bearer $token" };

	my $headersTx = ua_req(head => "https://registry-1.docker.io/v2/$repo/blobs/$blob" => $authorizationHeader);
	die "failed to get headers for $key" unless $headersTx->success;
	return $headers{$key} = $headersTx->res->headers;
}

sub get_layer_data {
	my ($repo, $id, $blob, $v1) = @_;
	$id //= $v1->{id};
	state %layers;
	return $layers{$id} if $layers{$id};
	die "missing v1/blob data for layer $id" unless $blob and $v1;
	my $data = {
		map({ $_ => $v1->{$_} } qw(id created parent docker_version)),
		container_command => $v1->{container_config}{Cmd},
		virtual_size => $v1->{Size},
		blob => $blob,
	};<commit_msg>Fix minor bug with generate-tag-details.pl related to "Size" fields that were null instead of 0<commit_after>#!/usr/bin/env perl
use strict;
use warnings;
use 5.010;
use open ':encoding(utf8)';

use Mojo::UserAgent;

die 'no images specified' unless @ARGV;

my $ua = Mojo::UserAgent->new->max_redirects(10);
$ua->transactor->name('Docker');

my $maxRetries = 3;
sub ua_req {
	my $method = shift;
	my $tries = $maxRetries;
	my $tx;
	do {
		--$tries;
		$tx = $ua->$method(@_);
		return $tx if $tx->success;
	} while ($tries > 0);
	return $tx;
}

sub split_image_name {
	my $image = shift;
	if ($image =~ m{
		^
		(?: ([^/:]+) / )? # optional namespace
		([^/:]+)          # image name
		(?: : ([^/:]+) )? # optional tag
		$
	}x) {
		my ($namespace, $name, $tag) = (
			$1 // 'library', # namespace
			$2,              # image name
			$3 // 'latest',  # tag
		);
		return ("$namespace/$name", $tag);
	}
	die "unrecognized image name format in: $image";
}

sub get_token {
	my $repo = shift;
	state %tokens;
	return $tokens{$repo} if $tokens{$repo};
	my $realmTx = $ua->get("https://registry-1.docker.io/v2/$repo/tags/list");
	my $auth = $realmTx->res->headers->www_authenticate;
	die "unexpected WWW-Authenticate header: $auth" unless $auth =~ m{ ^ Bearer \s+ (\S.*) $ }x;
	my $realm = $1;
	my $url = Mojo::URL->new;
	while ($realm =~ m{
		# key="val",
		([^=]+)
		=
		"([^"]+)"
		,?
	}xg) {
		my ($key, $val) = ($1, $2);
		if ($key eq 'realm') {
			$url->base(Mojo::URL->new($val));
		} else {
			$url->query->append($key => $val);
		}
	}
	$url = $url->to_abs;
	my $tokenTx = ua_req(get => $url);
	die "failed to fetch token for $repo" unless $tokenTx->success;
	return $tokens{$repo} = $tokenTx->res->json->{token};
}

sub get_manifest {
	my ($repo, $tag) = @_;
	my $image = "$repo:$tag";
	state %manifests;
	return $manifests{$image} if $manifests{$image};

	my $token = get_token($repo);
	my $authorizationHeader = { Authorization => "Bearer $token" };

	my $manifestTx = ua_req(get => "https://registry-1.docker.io/v2/$repo/manifests/$tag" => $authorizationHeader);
	die "failed to get manifest for $image" unless $manifestTx->success;
	return (
		$manifestTx->res->headers->header('Docker-Content-Digest'),
		$manifests{$image} = $manifestTx->res->json,
	);
}

sub get_blob_headers {
	my ($repo, $blob) = @_;
	my $key = $repo . '@' . $blob;
	state %headers;
	return $headers{$key} if $headers{$key};

	my $token = get_token($repo);
	my $authorizationHeader = { Authorization => "Bearer $token" };

	my $headersTx = ua_req(head => "https://registry-1.docker.io/v2/$repo/blobs/$blob" => $authorizationHeader);
	die "failed to get headers for $key" unless $headersTx->success;
	return $headers{$key} = $headersTx->res->headers;
}

sub get_layer_data {
	my ($repo, $id, $blob, $v1) = @_;
	$id //= $v1->{id};
	state %layers;
	return $layers{$id} if $layers{$id};
	die "missing v1/blob data for layer $id" unless $blob and $v1;
	my $data = {
		map({ $_ => $v1->{$_} } qw(id created parent docker_version)),
		container_command => $v1->{container_config}{Cmd},
		virtual_size => $v1->{Size} // 0,
		blob => $blob,
	};
{'fraction_non_alphanumeric': 0.21485583971330835, 'fraction_numerical': 0.009447792800130314, 'mean_word_length': 3.7893915756630263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 95, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6069748', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2397, 'n_words': 571}