<commit_before>
/*

  Metadata.js

  Metadata is stored as comments near the beginning of an HTML file.
  e.g. DATA=recipes:/api/recipes{slug,url}?recipe.key=<slug>,/users

*/

var Utils     = require("./utils"),
    Universql = require("universql"),
    Metadata  = {};


module.exports = Metadata;


/*

  function extract

  Extracts pertinent metadata for querying.

*/

Metadata.extract = function(str) {

  var names = ["QUERY", "DATA", "COOKIES", "SESSION"],
      re    = new RegExp("(\\b(?:" + names.join("|") + ")\\=[^\\n\\r]+)", "g"),
      meta  = {};


  Utils.getRegExpMatches(str, re, 1).map(Metadata.parse).forEach(function(metadata) {
    meta[metadata["name"]] = meta[metadata["name"]] || [];
    meta[metadata["name"]].push(metadata.value);
  });

  return meta;

}



/*<commit_msg>Metadata#extract: removes DATA (folding into QUERY)<commit_after>
/*

  Metadata.js

  Metadata is stored as comments near the beginning of an HTML file.
  e.g. DATA=recipes:/api/recipes{slug,url}?recipe.key=<slug>,/users

*/

var Utils     = require("./utils"),
    Universql = require("universql"),
    Metadata  = {};


module.exports = Metadata;


/*

  function extract

  Extracts pertinent metadata for querying.

*/

Metadata.extract = function(str) {

  var names = ["QUERY", "COOKIES", "SESSION"],
      re    = new RegExp("(\\b(?:" + names.join("|") + ")\\=[^\\n\\r]+)", "g"),
      meta  = {};


  Utils.getRegExpMatches(str, re, 1).map(Metadata.parse).forEach(function(metadata) {
    meta[metadata["name"]] = meta[metadata["name"]] || [];
    meta[metadata["name"]].push(metadata.value);
  });

  return meta;

}



/*
{'fraction_non_alphanumeric': 0.20012239902080783, 'fraction_numerical': 0.0012239902080783353, 'mean_word_length': 3.36, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259485', 'n_tokens_mistral': 592, 'n_tokens_neox': 556, 'n_words': 112}