<commit_before>// JSON-Schema documentation generator
// ===================================
// The core Generator class to take a set of schemas, templates, and options
// and build one or more HTML files.
'use strict';

var _ = require('lodash'),
	fs = require('fs'),
	Promise = require('bluebird'),
	getFiles = require('./lib/get-files'),
	SchemaResolver = require('./lib/resolver'),
	path = require('path'),
	// Default endpoint options. All of these can be overridden
	// via your own configuration
	endpointOptionDefaults = {
		// Attributes to include with each `link` endpoint (these will be in
		// addition so some generated properties of this package)
		attributes : ['title', 'description', 'method'],
		// The attributes to build for each parameter listing of an object
		// definition and `link` schema (e.g., required/optional inputs)
		attribtueParameters : ['name', 'type', 'description', 'example'],
		// Additional cURL headers to include with each example cURL request
		// Useful for authentication, etc
		curlHeaders : {}
	},
	// constructor
	Generator = function( options ){
		// Base URL used with example cURLs
		this.apiURL = options.apiURL;
		// Globs of file paths we should fetch
		this.schemas = options.schemas;
		// File paths we shouldn't parse.
		this.dontParse = options.dontParse || [];
		// Schemas that shouldn't have docs. Useful
		// for base objects that others extend from,
		// but have no use for documentation
		this.noDocs = options.noDocs || [];
		// Globs of template files
		this.templates = options.templates || [];
		// Pages to build, referenced by template name
		this.pages = options.pages;
		// Each endpoint section will get these parameters
		// for the template
		this.endpointOptions = _.defaults({}, options.endpointOptions, endpointOptionDefaults);
		// Additional template parameters for each page.
		// Here might be a good place to configure things like an API version, or any global
		// variables you want accessible inside of your templates
		this.templateOptions = options.templateOptions;
		// Default curl command format.
		this.curl = options.curl || '$ curl -X';
	},
	proto = {};

module.exports = Generator;
Generator.prototype = proto;

_.extend(proto, {

	// Resolve schemas and template globs
	//
	// @return Promise
	resolvePaths : function () {
		return Promise.props({
			schemas: this.parseSchemas(),
			templates: this.parseTemplates()
		});
	},

	// Get schemas contents from the filesystem and resolve references
	//
	// @return Promise
	parseSchemas : function () {
		return new Promise(function (resolve, reject) {
			// Get only the files we want to parse
			var files = this.schemas.filter(function(path){
					return !_.include(this.dontParse, path);
				}.bind(this));

			getFiles.asJSON(files).bind(this).then(function(files){
				// Pass back schema map..
				resolve(_.reduce(files, function(acc, schema){
					acc[schema.id] = schema;
					return acc;
				}, {}, this));
			}, reject);
		}.bind(this));
	},

	// Recursively resolve all references schemas for an array of schemas
	//
	// @param schemas object - Valid schema objects, keyed by schema ID
	// @return array - resolve schema objects
	resolveSchemas : function (schemas) {
		var resolver = this.resolver = new SchemaResolver({schemas: schemas});
		return resolver.resolve();
	},

	// Get the contents of each template file, and key the results by file name.
	// (useful for nesting templates / including (named) partials)
	//
	// @return Promise
	parseTemplates : function () {
		return new Promise(function (resolve, reject) {
			getFiles.raw(this.templates).bind(this).then(function (templates) {
				// Map file name to contents
				resolve(_.reduce(templates, function (acc, contents, p) {
					acc[path.basename(p, path.extname(p))] = contents;
					return acc;
				}, {}, this));
			}, reject);
		}.bind(this));
	},

	// Run the contents of each template through the provided compiler
	//
	// @param Compiler function - a compiler function like Handlebars.compile
	// @param templates object - object of templates keyed by template ID (file basename)
	// @return object - template contents, keyed by file basename
	compileTemplates : function (Compiler, templates) {
		return _.reduce(templates, function (acc, t, id) {
			acc[id] = Compiler(t);
			return acc;
		}, {}, this);
	},

	// Build the objects needed for the template and
	// create the configured HTML files
	//
	// @param templates object - Map of templates, keyed by file name
	// @return object - HTML contents, keyed by page basename
	makeHTML : function (templates, schemas) {
		var sections = this.buildSchemaDocObjects(schemas);

		return _.reduce(this.pages, function(acc, includeSchemas, page){
			var template = templates[page] || templates.index; // Temporary?
			acc[page] = template(_.extend({}, this.templateOptions, {
				sections: this.getSectionsForPage(sections, includeSchemas)
			}));
			return acc;
		}, {}, this);
	},

	// Each page can have an array of schema IDs that should
	// be included with it. This method takes all available sections/schemas
	// and returns only the ones that should be included
	//
	// @param sections array - The prepared sections of documentation. This is a one-to-one correlation with all of the schemas that will have documentation
	// @param include array - Schema IDs whose sections should be returned
	// @return array - array of sections
	getSectionsForPage : function (sections, include) {
		// Special case for all schemas on one page
		if (include === '*') {return sections;}

		return _.filter(sections, function (section) {
			return _.contains(include, section._id);
		});
	},

	// Expects to resolve a relative URI from the given schema ID
	// @todo Allow this to resolve any pointer reference, right now it assumes relative
	//
	// @param href string - `href` from a `link` object
	// @param id string - root schema ID
	// @param replaceWithData boolean - Replace references with example content or not
	// @return string - resolved URI
	resolveURI : function(href, id, replaceWithData) {
		// This will pull out all {/schema/pointers}
		var pat = /((?:{(?:#?(\/[\w\/]+))})+)+/g,
			matches = href.match(pat);

		_.each(matches, function (match) {
			var stripped, definition, replacement;
				// Remove the brackets so we can find the definition
			stripped = match.replace(/[{}]/g, '');
			definition = this.resolver.get(id+stripped);<commit_msg>Returning the sections/schemas for each page in the order they were defined<commit_after>// JSON-Schema documentation generator
// ===================================
// The core Generator class to take a set of schemas, templates, and options
// and build one or more HTML files.
'use strict';

var _ = require('lodash'),
	fs = require('fs'),
	Promise = require('bluebird'),
	getFiles = require('./lib/get-files'),
	SchemaResolver = require('./lib/resolver'),
	path = require('path'),
	// Default endpoint options. All of these can be overridden
	// via your own configuration
	endpointOptionDefaults = {
		// Attributes to include with each `link` endpoint (these will be in
		// addition so some generated properties of this package)
		attributes : ['title', 'description', 'method'],
		// The attributes to build for each parameter listing of an object
		// definition and `link` schema (e.g., required/optional inputs)
		attribtueParameters : ['name', 'type', 'description', 'example'],
		// Additional cURL headers to include with each example cURL request
		// Useful for authentication, etc
		curlHeaders : {}
	},
	// constructor
	Generator = function( options ){
		// Base URL used with example cURLs
		this.apiURL = options.apiURL;
		// Globs of file paths we should fetch
		this.schemas = options.schemas;
		// File paths we shouldn't parse.
		this.dontParse = options.dontParse || [];
		// Schemas that shouldn't have docs. Useful
		// for base objects that others extend from,
		// but have no use for documentation
		this.noDocs = options.noDocs || [];
		// Globs of template files
		this.templates = options.templates || [];
		// Pages to build, referenced by template name
		this.pages = options.pages;
		// Each endpoint section will get these parameters
		// for the template
		this.endpointOptions = _.defaults({}, options.endpointOptions, endpointOptionDefaults);
		// Additional template parameters for each page.
		// Here might be a good place to configure things like an API version, or any global
		// variables you want accessible inside of your templates
		this.templateOptions = options.templateOptions;
		// Default curl command format.
		this.curl = options.curl || '$ curl -X';
	},
	proto = {};

module.exports = Generator;
Generator.prototype = proto;

_.extend(proto, {

	// Resolve schemas and template globs
	//
	// @return Promise
	resolvePaths : function () {
		return Promise.props({
			schemas: this.parseSchemas(),
			templates: this.parseTemplates()
		});
	},

	// Get schemas contents from the filesystem and resolve references
	//
	// @return Promise
	parseSchemas : function () {
		return new Promise(function (resolve, reject) {
			// Get only the files we want to parse
			var files = this.schemas.filter(function(path){
					return !_.include(this.dontParse, path);
				}.bind(this));

			getFiles.asJSON(files).bind(this).then(function(files){
				// Pass back schema map..
				resolve(_.reduce(files, function(acc, schema){
					acc[schema.id] = schema;
					return acc;
				}, {}, this));
			}, reject);
		}.bind(this));
	},

	// Recursively resolve all references schemas for an array of schemas
	//
	// @param schemas object - Valid schema objects, keyed by schema ID
	// @return array - resolve schema objects
	resolveSchemas : function (schemas) {
		var resolver = this.resolver = new SchemaResolver({schemas: schemas});
		return resolver.resolve();
	},

	// Get the contents of each template file, and key the results by file name.
	// (useful for nesting templates / including (named) partials)
	//
	// @return Promise
	parseTemplates : function () {
		return new Promise(function (resolve, reject) {
			getFiles.raw(this.templates).bind(this).then(function (templates) {
				// Map file name to contents
				resolve(_.reduce(templates, function (acc, contents, p) {
					acc[path.basename(p, path.extname(p))] = contents;
					return acc;
				}, {}, this));
			}, reject);
		}.bind(this));
	},

	// Run the contents of each template through the provided compiler
	//
	// @param Compiler function - a compiler function like Handlebars.compile
	// @param templates object - object of templates keyed by template ID (file basename)
	// @return object - template contents, keyed by file basename
	compileTemplates : function (Compiler, templates) {
		return _.reduce(templates, function (acc, t, id) {
			acc[id] = Compiler(t);
			return acc;
		}, {}, this);
	},

	// Build the objects needed for the template and
	// create the configured HTML files
	//
	// @param templates object - Map of templates, keyed by file name
	// @return object - HTML contents, keyed by page basename
	makeHTML : function (templates, schemas) {
		var sections = this.buildSchemaDocObjects(schemas);

		return _.reduce(this.pages, function(acc, includeSchemas, page){
			var template = templates[page] || templates.index; // Temporary?
			acc[page] = template(_.extend({}, this.templateOptions, {
				sections: this.getSectionsForPage(sections, includeSchemas)
			}));
			return acc;
		}, {}, this);
	},

	// Each page can have an array of schema IDs that should
	// be included with it. This method takes all available sections/schemas
	// and returns only the ones that should be included
	// This will return the `sections` in the order defined by `include`
	//
	// @param sections array - The prepared sections of documentation. This is a one-to-one correlation with all of the schemas that will have documentation
	// @param include array - Schema IDs whose sections should be returned
	// @return array - array of sections
	getSectionsForPage : function (sections, include) {
		var includeSections = [];
		// Special case for all schemas on one page.
		// NOTE: No guarantee of order here.
		if (include === '*') {return sections;}

		// Specifically doing this in order so the schemas that should be included
		// are return in their configured order.
		includeSections = _.filter(sections, function (section) {
			return _.contains(include, section._id);
		});

		return _.reduce(include, function (acc, schemaID) {
			acc.push(_.detect(includeSections, function (section) { return section._id === schemaID; }));
			return acc;
		}, []);
	},

	// Expects to resolve a relative URI from the given schema ID
	// @todo Allow this to resolve any pointer reference, right now it assumes relative
	//
	// @param href string - `href` from a `link` object
	// @param id string - root schema ID
	// @param replaceWithData boolean - Replace references with example content or not
	// @return string - resolved URI
	resolveURI : function(href, id, replaceWithData) {
		// This will pull out all {/schema/pointers}
		var pat = /((?:{(?:#?(\/[\w\/]+))})+)+/g,
			matches = href.match(pat);

		_.each(matches, function (match) {
			var stripped, definition, replacement;
				// Remove the brackets so we can find the definition
			stripped = match.replace(/[{}]/g, '');
			definition = this.resolver.get(id+stripped);
{'fraction_non_alphanumeric': 0.1234115712363582, 'fraction_numerical': 0.0, 'mean_word_length': 3.5214599526867185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5224881', 'n_tokens_mistral': 4036, 'n_tokens_neox': 3665, 'n_words': 1406}