<commit_before><commit_msg>GetWebText - Download a remote text thru http(s) [210203]<commit_after>// IdExtenso wants to run in INDD.
// ---
#target 'indesign'

// Path to IdExtenso entry point.
// ---
#include '../$$.jsxinc'

// Web module.
// ---
#include '../etc/$$.Web.jsxlib'

// Load the framework in TRACE mode
// ---
$$.load(-1);

// =============================================================================
// GetWebText [190323] [210203]
// Download a remote text resource.
// ---
// Demonstrates:
// - `$$.Web(url,1)`, with `wantText` option.
// =============================================================================

try
{
	// URL of the resource.
	// ---
	var url = prompt(
		__("Enter the URL of the resource:"),
		"http://indiscripts.com/pages/help",
		"GetWebText"
		);
	if( !url || !(url=url.trim()) )
	{
		exit(0);
	}
	
	// Check the URL.
	// ---
	url = $$.Web.parseURI(url);
	if( !url.protocol )
	{
		$$.error(__("Invalid protocol in %1.", url.source) );
	}

	// GET the resource.
	// ---
	var res = $$.Web(url.source,/*wantText*/1);
	
	if( res.error )
	{
		$$.error(res.error);
	}
	
	// Show what it looks like.
	// ---
	var sel = app.properties.selection;
	var txt = res.data;

	if( sel && (sel=sel[0]) && ('texts' in sel) )
	{
		var t = txt.replace(/^\s*\<!DOCTYPE [^\>]+\>/,'').trim();

		try{ txt=XML(t).toString(); }
		catch(_){ $$.trace( __("ExtendScript cannot interprete data as XML: %1.",_) ); }

		sel.texts[0].contents = txt;
	}
	else
	{
		const MAX_LEN = 10000;
		if( txt.length > MAX_LEN )
			$$.trace( __("Data (sample): %1", txt.slice(0,MAX_LEN).toSource()) );
		else
			$$.trace( __("Data: %1", txt.toSource()) );
	}
}

catch(e)
{
	// Just in case something goes wrong.
	// ---
	$$.receiveError(e);
}
// =============================================================================


// Please, unload the framework to cleanup memory.
// Also, this auto-opens the log file (if active.)
// ---
$$.unload();
{'fraction_non_alphanumeric': 0.3259866735007688, 'fraction_numerical': 0.01742696053305997, 'mean_word_length': 2.983673469387755, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5176808', 'n_tokens_mistral': 738, 'n_tokens_neox': 673, 'n_words': 164}