<commit_before>#!/usr/bin/env perl
use strict;
use warnings;
use 5.010;
use open ':encoding(utf8)';

use File::Basename qw(basename fileparse);
use File::Temp;
use Getopt::Long;
use Mojo::File;
use Mojo::UserAgent;
use Mojo::Util qw(b64_encode decode encode trim);

use Term::UI;
use Term::ReadLine;

my $hubLengthLimit = 25_000;
my $githubBase = 'https://github.com/docker-library/docs/tree/master'; # TODO point this at the correct "dist-xxx" branch based on "namespace"

my $username;
my $password;
my $batchmode;
my $namespace;
my $logos;
GetOptions(
	'u|username=s' => \$username,
	'p|password=s' => \$password,
	'batchmode!' => \$batchmode,
	'namespace=s' => \$namespace,
	'logos!' => \$logos,
) or die 'bad args';

die 'no repos specified' unless @ARGV;

my $ua = Mojo::UserAgent->new->max_redirects(10);
$ua->transactor->name('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36');

my $term = Term::ReadLine->new('docker-library-docs-push');
unless (defined $username) {
	$username = $term->get_reply(prompt => 'Hub Username');
}
unless (defined $password) {
	$password = $term->get_reply(prompt => 'Hub Password'); # TODO hide the input? O:)
}

my $login = $ua->post('https://hub.docker.com/v2/users/login/' => {} => json => { username => $username, password => $password });
die 'login failed' unless $login->res->is_success;

my $token = $login->res->json->{token};

my $csrf;
for my $cookie (@{ $login->res->cookies }) {
	if ($cookie->name eq 'csrftoken') {
		$csrf = $cookie->value;
		last;
	}
}
die 'missing CSRF token' unless defined $csrf;

my $attemptLogin = $ua->post('https://hub.docker.com/attempt-login/' => {} => json => { jwt => $token });
die 'attempt-login failed' unless $attemptLogin->res->is_success;

my $authorizationHeader = {
	Authorization => "JWT $token",
	'X-CSRFToken' => $csrf,
};

my $userData = $ua->get('https://hub.docker.com/v2/user/' => $authorizationHeader);
die 'user failed' unless $userData->res->is_success;
$userData = $userData->res->json;

sub prompt_for_edit {
	my $currentText = shift;
	my $proposedFile = shift;
	my $lengthLimit = shift // 0;
	
	my $proposedText = Mojo::File->new($proposedFile)->slurp // '**  FILE MISSING!  **';
	$proposedText = trim(decode('UTF-8', $proposedText));
	
	# remove our warning about generated files (Hub doesn't support HTML comments in Markdown)
	$proposedText =~ s% ^ <!-- .*? --> \s* %%sx;
	
	my $alwaysShortTags = ($proposedFile eq 'neo4j/README.md');
	
	if ($alwaysShortTags || ($lengthLimit > 0 && length($proposedText) > $lengthLimit)) {
		# TODO https://github.com/docker/hub-beta-feedback/issues/238
		my $fullUrl = "$githubBase/$proposedFile";
		my $shortTags = "-\tSee [\"Supported tags and respective \`Dockerfile\` links\" at $fullUrl]($fullUrl#supported-tags-and-respective-dockerfile-links)\n\n";
		my $tagsNote = "**Note:** the description for this image is longer than the Hub length limit of $lengthLimit, so the \"Supported tags\" list has been trimmed to compensate.  See [docker/hub-beta-feedback#238](https://github.com/docker/hub-beta-feedback/issues/238) for more information.\n\n" . $shortTags;
		my $genericNote = "**Note:** the description for this image is longer than the Hub length limit of $lengthLimit, so has been trimmed.  The full description can be found at [$fullUrl]($fullUrl).  See [docker/hub-beta-feedback#238](https://github.com/docker/hub-beta-feedback/issues/238) for more information.\n\n";
		
		$tagsNote = $shortTags if $alwaysShortTags;
		
		my $trimmedText = $proposedText;
		
		# if our text is too long for the Hub length limit, let's first try removing the "Supported tags" list and add $tagsNote and see if that's enough to let us put the full image documentation
		$trimmedText =~ s%^(# Supported tags and respective `Dockerfile` links\n\n).*?\n(?=# |\[)%$1$tagsNote%ms;
		# (we scrape until the next "h1" or a line starting with a link which is likely a build status badge for an architecture-namespace)
		
		$proposedText = $trimmedText if $alwaysShortTags;
		
		if (length($trimmedText) > $lengthLimit) {<commit_msg>Preserve Docker-provided "Sponsored Links" while updating descriptions<commit_after>#!/usr/bin/env perl
use strict;
use warnings;
use 5.010;
use open ':encoding(utf8)';

use File::Basename qw(basename fileparse);
use File::Temp;
use Getopt::Long;
use Mojo::File;
use Mojo::UserAgent;
use Mojo::Util qw(b64_encode decode encode trim);

use Term::UI;
use Term::ReadLine;

my $hubLengthLimit = 25_000;
my $githubBase = 'https://github.com/docker-library/docs/tree/master'; # TODO point this at the correct "dist-xxx" branch based on "namespace"

my $username;
my $password;
my $batchmode;
my $namespace;
my $logos;
GetOptions(
	'u|username=s' => \$username,
	'p|password=s' => \$password,
	'batchmode!' => \$batchmode,
	'namespace=s' => \$namespace,
	'logos!' => \$logos,
) or die 'bad args';

die 'no repos specified' unless @ARGV;

my $ua = Mojo::UserAgent->new->max_redirects(10);
$ua->transactor->name('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36');

my $term = Term::ReadLine->new('docker-library-docs-push');
unless (defined $username) {
	$username = $term->get_reply(prompt => 'Hub Username');
}
unless (defined $password) {
	$password = $term->get_reply(prompt => 'Hub Password'); # TODO hide the input? O:)
}

my $login = $ua->post('https://hub.docker.com/v2/users/login/' => {} => json => { username => $username, password => $password });
die 'login failed' unless $login->res->is_success;

my $token = $login->res->json->{token};

my $csrf;
for my $cookie (@{ $login->res->cookies }) {
	if ($cookie->name eq 'csrftoken') {
		$csrf = $cookie->value;
		last;
	}
}
die 'missing CSRF token' unless defined $csrf;

my $attemptLogin = $ua->post('https://hub.docker.com/attempt-login/' => {} => json => { jwt => $token });
die 'attempt-login failed' unless $attemptLogin->res->is_success;

my $authorizationHeader = {
	Authorization => "JWT $token",
	'X-CSRFToken' => $csrf,
};

my $userData = $ua->get('https://hub.docker.com/v2/user/' => $authorizationHeader);
die 'user failed' unless $userData->res->is_success;
$userData = $userData->res->json;

my $supportedTagsRegex = qr%^(# Supported tags and respective `Dockerfile` links\n\n)(.*?\n)(?=# |\[)%ms;

sub prompt_for_edit {
	my $currentText = shift;
	my $proposedFile = shift;
	my $lengthLimit = shift // 0;
	
	my $proposedText = Mojo::File->new($proposedFile)->slurp // '**  FILE MISSING!  **';
	$proposedText = trim(decode('UTF-8', $proposedText));
	
	# remove our warning about generated files (Hub doesn't support HTML comments in Markdown)
	$proposedText =~ s% ^ <!-- .*? --> \s* %%sx;
	
	# extract/re-inject sponsored links
	my $sponsoredLinks = '';
	if ($currentText =~ m{ ( ^ --- \n+ \Q*Useful Resources*\E \n .*? \n --- \n ) }smx) {
		$sponsoredLinks = $1 . "\n";
		say $sponsoredLinks;
		$proposedText =~ s%$supportedTagsRegex%$1$2$sponsoredLinks%;
	}
	
	my $alwaysShortTags = ($proposedFile eq 'neo4j/README.md');
	
	if ($alwaysShortTags || ($lengthLimit > 0 && length($proposedText) > $lengthLimit)) {
		# TODO https://github.com/docker/hub-beta-feedback/issues/238
		my $fullUrl = "$githubBase/$proposedFile";
		my $shortTags = "-\tSee [\"Supported tags and respective \`Dockerfile\` links\" at $fullUrl]($fullUrl#supported-tags-and-respective-dockerfile-links)\n\n";
		my $tagsNote = "**Note:** the description for this image is longer than the Hub length limit of $lengthLimit, so the \"Supported tags\" list has been trimmed to compensate.  See [docker/hub-beta-feedback#238](https://github.com/docker/hub-beta-feedback/issues/238) for more information.\n\n" . $shortTags;
		my $genericNote = "**Note:** the description for this image is longer than the Hub length limit of $lengthLimit, so has been trimmed.  The full description can be found at [$fullUrl]($fullUrl).  See [docker/hub-beta-feedback#238](https://github.com/docker/hub-beta-feedback/issues/238) for more information.\n\n";
		
		$tagsNote = $shortTags if $alwaysShortTags;
		
		my $trimmedText = $proposedText;
		
		# if our text is too long for the Hub length limit, let's first try removing the "Supported tags" list and add $tagsNote and see if that's enough to let us put the full image documentation
		$trimmedText =~ s%$supportedTagsRegex%$1$tagsNote$sponsoredLinks%ms;
		# (we scrape until the next "h1" or a line starting with a link which is likely a build status badge for an architecture-namespace)
		
		$proposedText = $trimmedText if $alwaysShortTags;
		
		if (length($trimmedText) > $lengthLimit) {
{'fraction_non_alphanumeric': 0.1691006026889198, 'fraction_numerical': 0.015414928140936486, 'mean_word_length': 4.599610642439974, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 103, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1472323', 'n_tokens_mistral': 3146, 'n_tokens_neox': 2888, 'n_words': 817}