<commit_before><?php
define('BY1',     1);
define('BY2',     2);
define('BY21',    3);
define('BY25',    4);
define('BY3',     5);
define('BYSA1',   6);
define('BYSA2',   7);
define('BYSA21',  8);
define('BYSA25',  9);
define('BYSA3',  10);
define('LAL',    11);

// Arrays containing regular expressions for each license type
// (so we can support multiple URL formats in the future if needed)
$by1 = array('http://creativecommons.org/licenses/by/1.0/?.*');
$by2 = array('http://creativecommons.org/licenses/by/2.0/?.*');
$by21 = array('http://creativecommons.org/licenses/by/2.1/?.*');
$by25 = array('http://creativecommons.org/licenses/by/2.5/?.*');
$by3 = array('http://creativecommons.org/licenses/by/3.0/?.*');
$bysa1 = array('http://creativecommons.org/licenses/by-sa/1.0/?.*');
$bysa2 = array('http://creativecommons.org/licenses/by-sa/2.0/?.*');
$bysa21 = array('http://creativecommons.org/licenses/by-sa/2.1/?.*');
$bysa25 = array('http://creativecommons.org/licenses/by-sa/2.5/?.*');
$bysa3 = array('http://creativecommons.org/licenses/by-sa/3.0/?.*');
$lal = array('http://artlibre.org/licence.php/lal.html');

// map licenses to ids by array position
$licenses = array(array(), $by1, $by2, $by21, $by25, $by3, $bysa1, $bysa2, $bysa21, $bysa25, $bysa3, $lal);

function simplify_license($license) {
	global $licenses;

	foreach ($licenses as $key => $l) {
		foreach ($l as $urlschema) {<commit_msg>Add public domain to valid list of licenses<commit_after><?php
define('BY1',     1);
define('BY2',     2);
define('BY21',    3);
define('BY25',    4);
define('BY3',     5);
define('BYSA1',   6);
define('BYSA2',   7);
define('BYSA21',  8);
define('BYSA25',  9);
define('BYSA3',  10);
define('LAL',    11);
define('PD',     12);

// Arrays containing regular expressions for each license type
// (so we can support multiple URL formats in the future if needed)
$by1 = array('http://creativecommons.org/licenses/by/1.0/?.*');
$by2 = array('http://creativecommons.org/licenses/by/2.0/?.*');
$by21 = array('http://creativecommons.org/licenses/by/2.1/?.*');
$by25 = array('http://creativecommons.org/licenses/by/2.5/?.*');
$by3 = array('http://creativecommons.org/licenses/by/3.0/?.*');
$bysa1 = array('http://creativecommons.org/licenses/by-sa/1.0/?.*');
$bysa2 = array('http://creativecommons.org/licenses/by-sa/2.0/?.*');
$bysa21 = array('http://creativecommons.org/licenses/by-sa/2.1/?.*');
$bysa25 = array('http://creativecommons.org/licenses/by-sa/2.5/?.*');
$bysa3 = array('http://creativecommons.org/licenses/by-sa/3.0/?.*');
$lal = array('http://artlibre.org/licence.php/lal.html');
$pd = array('http://creativecommons.org/licenses/publicdomain/?.*');

// map licenses to ids by array position
$licenses = array(array(), $by1, $by2, $by21, $by25, $by3, $bysa1, $bysa2, $bysa21, $bysa25, $bysa3, $lal, $pd);

function simplify_license($license) {
	global $licenses;

	foreach ($licenses as $key => $l) {
		foreach ($l as $urlschema) {
{'fraction_non_alphanumeric': 0.23727087576374745, 'fraction_numerical': 0.051595383570943655, 'mean_word_length': 4.4073394495412845, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3713393', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1117, 'n_words': 208}