<commit_before>/**---------------------------------------------------------------------------------------------------------------------
 * tgi-spec/lib/tgi-spec.source.js
 **/
var Spec = function () {
  this.scripts = []; // array of scripts
  this.nodes = []; // array of Spec.Node's
  this.testsCreated = 0;
  this.testsPending = 0;
  this.testsFailed = 0;
};
Spec.prototype.test = function (testSource, testName, testScript) {
  this.scripts.push({testSource: testSource, testName: testName, testScript: testScript});
};
Spec.prototype.runTests = function (callback) {
  this.testFunction = callback;
  // Load the scripts
  for (var i = 0; i < this.scripts.length; i++) {
    var script = this.scripts[i];
    this.testFunction({log: 'loading test script: ' + script.testName});
    script.testScript(callback);
  }
  this.completionCheck();
};
Spec.prototype.completionCheck = function (force) {
  var spec = this;
  if (!spec.testsPending || force) {
    spec.testFunction({
      done: true,
      testsCreated: spec.testsCreated,
      testsPending: spec.testsPending,
      testsFailed: spec.testsFailed
    });
  } else {
    // Give time for async to finish
    spec.watchdog = spec.watchdog || setTimeout(function () {
      spec.completionCheck(true); // force completion
    }, 500);
  }
};
Spec.prototype.githubMarkdown = function () {
  var spec = this;
  var text = '';
  var i, j, k;
  var arrayOfLines;
  var prettyCode;
  var leadingSpaces = 0;
  var line,line2;
  for (i = 0; i < spec.nodes.length; i++) {
    var node = spec.nodes[i];
    if (i)
      text += '\n';
    switch (node.type) {
      case 'h':
        text += '#### ' + node.text;
        break;
      case 'e':
        arrayOfLines = node.test.testFunction.toString().match(/[^\r\n]+/g);
        prettyCode = '';
        for (j = 1; j < arrayOfLines.length - 1; j++) {
          line = arrayOfLines[j];
          if (j == 1) {
            leadingSpaces=0;
            for (k = 0; k < line.length && line[k] == ' '; k++) {
              leadingSpaces++;
            }
          }
          for (k = 0; k < line.length && k < leadingSpaces && line[k] == ' '; k++) {
            // eat spaces
          }
          line2 = '';
          for (; k < line.length; k++) {
            line2 += line[k];
            // eat spaces
          }
          if (prettyCode.length)
            prettyCode += line2 + '\n';
          prettyCode += line2;
        }
        text += '<strong><i>EXAMPLE </i> ' + node.text + ':</strong>';
        text += '\n```javascript' +
        '\n' + prettyCode +
        '\n```';
        //if (true) {
          text += '\n<blockquote>' +
          JSON.stringify(node.test) +
          '</blockquote>';
        //}
        break;
      default:
      case 'p':
        text += '<p>' + node.text + '</p>';
        break;
    }
  }
  return text;
};
/**
 * Spec.Node object for each piece of spec types as follows:
 * h - heading
 * p - paragraph
 * e - example (test)
 **/
Spec.Node = function (args) {
  args = args || {};
  this.type = args.type || null;
  if (!this.type || ((this.type != 'h') && (this.type != 'p') && (this.type != 'e'))) {
    throw new Error('Spec.Node type must be h, p or e');
  }
};
/**
 * Spec.Test for each example
 */
Spec.Test = function (spec, expectedValue, testFunction) {
  var test = this;
  var testExpectedValue;
  var testReturnValue;
  spec.testsCreated++;
  spec.testsPending++;<commit_msg>close to fin<commit_after>/**---------------------------------------------------------------------------------------------------------------------
 * tgi-spec/lib/tgi-spec.source.js
 **/
var Spec = function () {
  this.scripts = []; // array of scripts
  this.nodes = []; // array of Spec.Node's
  this.testsCreated = 0;
  this.testsPending = 0;
  this.testsFailed = 0;
};
Spec.prototype.test = function (testSource, testName, testScript) {
  this.scripts.push({testSource: testSource, testName: testName, testScript: testScript});
};
Spec.prototype.runTests = function (callback) {
  this.testFunction = callback;
  // Load the scripts
  for (var i = 0; i < this.scripts.length; i++) {
    var script = this.scripts[i];
    this.testFunction({log: 'loading test script: ' + script.testName});
    script.testScript(callback);
  }
  this.completionCheck();
};
Spec.prototype.completionCheck = function (force) {
  var spec = this;
  if (!spec.testsPending || force) {
    spec.testFunction({
      done: true,
      testsCreated: spec.testsCreated,
      testsPending: spec.testsPending,
      testsFailed: spec.testsFailed
    });
  } else {
    // Give time for async to finish
    spec.watchdog = spec.watchdog || setTimeout(function () {
      spec.completionCheck(true); // force completion
    }, 500);
  }
};
Spec.prototype.githubMarkdown = function () {
  var spec = this;
  var text = '';
  var i, j, k;
  var arrayOfLines;
  var prettyCode;
  var leadingSpaces = 0;
  var line, line2;
  for (i = 0; i < spec.nodes.length; i++) {
    var node = spec.nodes[i];
    if (i)
      text += '\n';
    switch (node.type) {
      case 'h':
        text += '#### ' + node.text;
        break;
      case 'e':
        text += getCodeChunk();
        break;
      default:
      case 'p':
        text += '<p>' + node.text + '</p>';
        break;
    }
  }
  return text;

  function getCodeChunk() {
    var text = '';

    arrayOfLines = node.test.testFunction.toString().match(/[^\r\n]+/g);
    prettyCode = '';
    for (j = 1; j < arrayOfLines.length - 1; j++) {
      line = arrayOfLines[j];
      if (j == 1) {
        leadingSpaces = 0;
        for (k = 0; k < line.length && line[k] == ' '; k++) {
          leadingSpaces++;
        }
      }
      for (k = 0; k < line.length && k < leadingSpaces && line[k] == ' '; k++) {
        // eat spaces
      }
      line2 = '';
      for (; k < line.length; k++) {
        line2 += line[k];
        // eat spaces
      }
      if (prettyCode.length)
        prettyCode += line2 + '\n';
      prettyCode += line2;
    }
    text += '<strong><i>EXAMPLE </i> ' + node.text + ':</strong>';
    text += '\n```javascript' +
    '\n' + prettyCode +
    '\n```';
    //if (true) {
    text += '\n<blockquote>' +
    JSON.stringify(node.test) +
    '</blockquote>';
    //}

    return text;
  }


};
/**
 * Spec.Node object for each piece of spec types as follows:
 * h - heading
 * p - paragraph
 * e - example (test)
 **/
Spec.Node = function (args) {
  args = args || {};
  this.type = args.type || null;
  if (!this.type || ((this.type != 'h') && (this.type != 'p') && (this.type != 'e'))) {
    throw new Error('Spec.Node type must be h, p or e');
  }
};
/**
 * Spec.Test for each example
 */
Spec.Test = function (spec, expectedValue, testFunction) {
  var test = this;
  var testExpectedValue;
  var testReturnValue;
  spec.testsCreated++;
  spec.testsPending++;
{'fraction_non_alphanumeric': 0.20213232072440485, 'fraction_numerical': 0.0058419745874105445, 'mean_word_length': 1.9903930131004366, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5358422', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2167, 'n_words': 547}