<commit_before>'use strict';

const Promise   = require('bluebird');
const rp        = require('request-promise');
const fs        = Promise.promisifyAll(require('fs'));
const recursive = Promise.promisify(require('recursive-readdir'));

const SPEC_MD       = 'https://raw.githubusercontent.com/ietf-wg-acme/acme/master/draft-ietf-acme-acme.md';
const SPEC_HTML     = 'https://github.com/ietf-wg-acme/acme/blob/master/draft-ietf-acme-acme.md';
const NORMATIVE_RE  = /MUST|SHOULD|MAY|REQUIRED/;
const CODE_BASE     = 'https://github.com/bifurcation/rocket-skates/tree/master/';
const CONCORD_RE    = /{{[a-z0-9-]+}}/;
const COVERAGE_FILE = 'SPEC_COVERAGE.md';
const TITLE         = '# Coverage of the ACME Specification\n';
const INTRO = `
This file is a summary of how well
[rocket-skates](https://github.com/bifurcation/rocket-skates) covers the
behaviors specified in the [ACME
specification](https://ietf-wg-acme.github.io/acme).  For each section of the
spec, we indicate where the behaviors in that section are implemented.  Sections
with no normative requirements are omitted.

`;

// Returns [[tag, index, text]]
function getSpecTags(url) {
  return rp.get(url)
    .then(spec => {
      let headers = [];
      let sections = {};
      let currHeader;
      let currText = '';
      spec.split('\n')
        .map(line => {
          if (line.match(/^#/)) {
            if (currHeader) {
              headers.push(currHeader);
              sections[currHeader] = currText;
            }

            currHeader = line;
            currText = '';
          } else {
            currText += line + '\n';
          }
        })


      let tagInfo = [];
      let lastLevel = 0;
      let counters = [0, 0, 0, 0, 0];
      for (let heading of headers) {
        let level = heading.replace(/[^#].*$/, '').length;
        if (level < lastLevel) {
          for (let i = level; i < counters.length; i += 1) {
            counters[i] = 0;
          }
        }
        lastLevel = level;

        let text = heading.substr(level)
                            .replace(/^[# ]*/, '')
                            .replace(/[# ]*$/, '');
        let tag = text.replace(/[^a-zA-Z0-9 -]/g, '')
                      .toLowerCase().replace(/ /g, '-');
        counters[level-1] += 1;

        // Only collect coverage data for sections that have RFC2119 keywords.
        // This means we have to manually exclude the terminology section, which
        // lists the keywords, but is not normative.
        if (sections[heading].match(NORMATIVE_RE) && (tag !== 'terminology')) {
          let index = counters.filter(x => x > 0).join('.');
          tagInfo.push({
            tag: tag,
            index: index,
            text: text
          });
        }
      }

      return tagInfo;
    });
}

// Returns [tag, fileName, lineNo]
function findTagsInJS(dir) {
  let fileNames;
  return recursive(__dirname + dir)
    .then(files => {
      fileNames = files.map(x => x.replace(/^.*rocket-skates\/lib/, 'lib'));
      let reads = files.map(file => fs.readFileAsync(file, 'utf8'));
      return Promise.all(reads);
    })
    .then(reads => {
      let concords = [];
      reads.map((content, i) => {
        content.split('\n')
          .map((line, lineNo) => {
            let match = line.match(CONCORD_RE);
            if (match) {
              let tag = match[0].replace(/[{}]/g, '');
              let file = fileNames[i].replace(/^.*\//, '');
              concords.push({
                tag: tag,
                file: file,
                path: fileNames[i],
                line: lineNo
              });
            }
          });
      });

      return concords;
    });
}

function joinTagTables(spec, client, server) {
  let data = [];
  for (let heading of spec) {
    data.push({
      tag:    heading.tag,
      spec:   heading,
      client: client.filter(code => code.tag === heading.tag),
      server: server.filter(code => code.tag === heading.tag)
    });
  }
  return data;
}
<commit_msg>Make spec coverage links more accurate<commit_after>'use strict';

const Promise   = require('bluebird');
const rp        = require('request-promise');
const fs        = Promise.promisifyAll(require('fs'));
const recursive = Promise.promisify(require('recursive-readdir'));

const SPEC_MD       = 'https://raw.githubusercontent.com/ietf-wg-acme/acme/master/draft-ietf-acme-acme.md';
const SPEC_HTML     = 'https://github.com/ietf-wg-acme/acme/blob/master/draft-ietf-acme-acme.md';
const NORMATIVE_RE  = /MUST|SHOULD|MAY|REQUIRED/;
const CODE_BASE     = 'https://github.com/bifurcation/rocket-skates/tree/master/';
const CONCORD_RE    = /{{[a-z0-9-]+}}/;
const COVERAGE_FILE = 'SPEC_COVERAGE.md';
const TITLE         = '# Coverage of the ACME Specification\n';
const INTRO = `
This file is a summary of how well
[rocket-skates](https://github.com/bifurcation/rocket-skates) covers the
behaviors specified in the [ACME
specification](https://ietf-wg-acme.github.io/acme).  For each section of the
spec, we indicate where the behaviors in that section are implemented.  Sections
with no normative requirements are omitted.

`;

// Returns [[tag, index, text]]
function getSpecTags(url) {
  return rp.get(url)
    .then(spec => {
      let headers = [];
      let sections = {};
      let currHeader;
      let currText = '';
      spec.split('\n')
        .map(line => {
          if (line.match(/^#/)) {
            if (currHeader) {
              headers.push(currHeader);
              sections[currHeader] = currText;
            }

            currHeader = line;
            currText = '';
          } else {
            currText += line + '\n';
          }
        })


      let tagInfo = [];
      let lastLevel = 0;
      let counters = [0, 0, 0, 0, 0];
      for (let heading of headers) {
        let level = heading.replace(/[^#].*$/, '').length;
        if (level < lastLevel) {
          for (let i = level; i < counters.length; i += 1) {
            counters[i] = 0;
          }
        }
        lastLevel = level;

        let text = heading.substr(level)
                            .replace(/^[# ]*/, '')
                            .replace(/[# ]*$/, '');
        let tag = text.replace(/[^a-zA-Z0-9 -]/g, '')
                      .toLowerCase().replace(/ /g, '-');
        counters[level-1] += 1;

        // Only collect coverage data for sections that have RFC2119 keywords.
        // This means we have to manually exclude the terminology section, which
        // lists the keywords, but is not normative.
        if (sections[heading].match(NORMATIVE_RE) && (tag !== 'terminology')) {
          let index = counters.filter(x => x > 0).join('.');
          tagInfo.push({
            tag: tag,
            index: index,
            text: text
          });
        }
      }

      return tagInfo;
    });
}

// Returns [tag, fileName, lineNo]
function findTagsInJS(dir) {
  let fileNames;
  return recursive(__dirname + dir)
    .then(files => {
      fileNames = files.map(x => x.replace(/^.*rocket-skates\/lib/, 'lib'));
      let reads = files.map(file => fs.readFileAsync(file, 'utf8'));
      return Promise.all(reads);
    })
    .then(reads => {
      let concords = [];
      reads.map((content, i) => {
        content.split('\n')
          .map((line, lineNo) => {
            let match = line.match(CONCORD_RE);
            if (match) {
              let tag = match[0].replace(/[{}]/g, '');
              let file = fileNames[i].replace(/^.*\//, '');
              concords.push({
                tag: tag,
                file: file,
                path: fileNames[i],
                line: lineNo + 1 // lines are indexed from 1
              });
            }
          });
      });

      return concords;
    });
}

function joinTagTables(spec, client, server) {
  let data = [];
  for (let heading of spec) {
    data.push({
      tag:    heading.tag,
      spec:   heading,
      client: client.filter(code => code.tag === heading.tag),
      server: server.filter(code => code.tag === heading.tag)
    });
  }
  return data;
}

{'fraction_non_alphanumeric': 0.15486342850080337, 'fraction_numerical': 0.005438141144481523, 'mean_word_length': 1.8858773181169757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1152432', 'n_tokens_mistral': 2589, 'n_tokens_neox': 2501, 'n_words': 587}