<commit_before><!DOCTYPE html>
<div>test</div>
<script>
'use strict';

function text_size(text, px) {
  var node = document.createElement('span');
  node.innerHTML = text;
  node.style = 'position: absolute; visibility: hidden; height: auto; width: auto; white-space: nowrap; font-family: monospace;';
  node.style.fontSize = px.toString() + 'px';
  document.body.appendChild(node);
  var result = [node.offsetWidth, node.offsetHeight];
  document.body.removeChild(node);
  return result;
}

function text_width(text, px) {
  var cvs = document.createElement('canvas');
  var ctx = cvs.getContext('2d');
  ctx.font = px.toString() + 'px monospace';
  return ctx.measureText(text).width;
}

var nil = null;
function pair(hd, tl) { return [hd, tl]; }
function head(pr) { return pr[0]; }
function tail(pr) { return pr[1]; }
<commit_msg>Define HTML and URI codecs<commit_after><!DOCTYPE html>
<div>test</div>
<script>
'use strict';

function text_size(text, px) {
  var node = document.createElement('span');
  node.innerHTML = text;
  node.style = 'position: absolute; visibility: hidden; height: auto; width: auto; white-space: nowrap; font-family: monospace;';
  node.style.fontSize = px.toString() + 'px';
  document.body.appendChild(node);
  var result = [node.offsetWidth, node.offsetHeight];
  document.body.removeChild(node);
  return result;
}

function text_width(text, px) {
  var cvs = document.createElement('canvas');
  var ctx = cvs.getContext('2d');
  ctx.font = px.toString() + 'px monospace';
  return ctx.measureText(text).width;
}

// `html_encode` should be sufficient as long as we control the construction
// of HTML/CSS/JS.  User content must pass through our construction APIs.
// Since we maintain the original content, there's no need for `html_decode`.
var html_encode_replacement = {
  '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#x27;',
  '/': '&#x2F', '`': '&#x60'
};
function html_encode_replace(str) { return html_encode_replacement[str]; }
function html_encode(str) {
  return str.replace(/[&<>"'\/`]/g, html_encode_replace);
}
// `encodeURI` is not safe for user input, as it preserves special characters
// used across a full URI.  Instead, build URIs manually using these.
function uri_path_encode(str) { return encodeURIComponent(str); }
function uri_path_decode(up) { return decodeURIComponent(str); }
// Replace space with + rather than %20 in application/x-www-form-urlencoded
// content, such as anything to the right of ? in a URL
function uri_query_encode(str) {
  return encodeURIComponent(str).replace(/%20/g, '+');
}
function uri_query_decode(uq) {
  return decodeURIComponent(uq.replace(/\+/g, '%20'));
}

var nil = null;
function pair(hd, tl) { return [hd, tl]; }
function head(pr) { return pr[0]; }
function tail(pr) { return pr[1]; }

{'fraction_non_alphanumeric': 0.1625580564487317, 'fraction_numerical': 0.006073597713469096, 'mean_word_length': 4.894736842105263, 'pattern_counts': {'":': 1, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1418901', 'n_tokens_mistral': 939, 'n_tokens_neox': 935, 'n_words': 253}