<commit_before>'use strict';

var rfr = require('rfr');
var _ = require('lodash');
var Component = rfr('src/component');
var matter = require('gray-matter');

class Parser {

	/**
	 * Parses components from source content.
	 *
	 * @param {String} source
	 * @param {String} [language]
	 * @return {Array.<Component>}
	 */
	parse(content, language) {
		var components = _(getDocBlocks(content, language))
			.map(parseDocBlock)
			.filter()
			.thru(Component.merge)
			.value();

		return components;
	}
}

function getDocBlocks(content, language) {
	return isMarkdown(language)
		? getMarkdownDocBlocks(content)
		: getSourceCodeDocBlocks(content);
}

function isMarkdown(language) {
	return _.includes(['markdown', 'mdown', 'md'], language);
}

function getMarkdownDocBlocks(content) {
	return [content];
}

function getSourceCodeDocBlocks(content) {
	var docBlocks = content.match(/\/\*([\s\S]+?)\*\//g);

	// Removes extraneous asterisks from the start & end of comment blocks
	docBlocks = _.map(docBlocks, (docBlock) => /\/\*[\s\*]*([\s\S]+?)?[ \t\*]*\*\//g.exec(docBlock)[1]);

	return docBlocks;
}

// @todo Refactor below

function parseDocBlock(docBlock) {<commit_msg>Guard against malformed docblocks<commit_after>'use strict';

var rfr = require('rfr');
var _ = require('lodash');
var Component = rfr('src/component');
var matter = require('gray-matter');

class Parser {

	/**
	 * Parses components from source content.
	 *
	 * @param {String} source
	 * @param {String} [language]
	 * @return {Array.<Component>}
	 */
	parse(content, language) {
		var components = _(getDocBlocks(content, language))
			.map(parseDocBlock)
			.filter()
			.thru(Component.merge)
			.value();

		return components;
	}
}

function getDocBlocks(content, language) {
	return isMarkdown(language)
		? getMarkdownDocBlocks(content)
		: getSourceCodeDocBlocks(content);
}

function isMarkdown(language) {
	return _.includes(['markdown', 'mdown', 'md'], language);
}

function getMarkdownDocBlocks(content) {
	return [content];
}

function getSourceCodeDocBlocks(content) {
	var docBlocks = content.match(/\/\*([\s\S]+?)\*\//g);

	// Removes extraneous asterisks from the start & end of comment blocks
	docBlocks = _.map(docBlocks, (docBlock) => /\/\*[\s\*]*([\s\S]+?)?[ \t\*]*\*\//g.exec(docBlock)[1]);

	return docBlocks;
}

// @todo Refactor below

function parseDocBlock(docBlock) {

	if (!_.isString(docBlock)) {
		// Malformed docblock
		return null;
	}

{'fraction_non_alphanumeric': 0.1744281045751634, 'fraction_numerical': 0.0008169934640522876, 'mean_word_length': 4.394273127753304, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6907904', 'n_tokens_mistral': 871, 'n_tokens_neox': 809, 'n_words': 179}