<commit_before>#!/usr/bin/env phantomjs

// This script looks at the HTML files in the offline folder
// and isolates the markdown section in each page.

var page = require('webpage').create();
var fs = require('fs');

page.onConsoleMessage = function (msg) { console.log(msg); };

function fixPage() {
  // executed inside page

  var file = document.querySelector('.file');
  file.style.marginTop = "0";
  document.body.innerHTML = file.outerHTML;
  document.querySelector('.file-header').innerHTML = "";

  document.body.style.width = "900px";

  var links = document.querySelectorAll('a');
  var i, link;
  for (i=0, len=links.length; i<len; i++) {
    link = links[i];<commit_msg>correct title of offline pages in clean script<commit_after>#!/usr/bin/env phantomjs

// This script looks at the HTML files in the offline folder
// and isolates the markdown section in each page.

var page = require('webpage').create();
var fs = require('fs');

page.onConsoleMessage = function (msg) { console.log(msg); };

// code executed inside page
function fixPage() {

  function trim(s) {
    return s.replace(/^\s+|\s+$/g, '');
  }

  // set title to first header text
  // (replace any redundant mentions of ClojureScript)
  var header = document.querySelector('h1') || document.querySelector('h2');
  document.title = trim(header.innerText.replace('ClojureScript',''));

  // replace everything with .file box (markdown view)
  var file = document.querySelector('.file');
  file.style.marginTop = "0";
  document.body.innerHTML = file.outerHTML;

  // remove the file header
  document.querySelector('.file-header').innerHTML = "";

  // set width to tame formatting
  document.body.style.width = "900px";

  // make anchor links work
  var links = document.querySelectorAll('a');
  var i, link;
  for (i=0, len=links.length; i<len; i++) {
    link = links[i];
{'fraction_non_alphanumeric': 0.15016146393972013, 'fraction_numerical': 0.006458557588805167, 'mean_word_length': 3.905013192612137, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6461620', 'n_tokens_mistral': 584, 'n_tokens_neox': 557, 'n_words': 167}