<commit_before><?php

/*!
 * Data Class
 *
 * Copyright (c) 2014 <NAME>, http://dmolsen.com
 * Licensed under the MIT license
 *
 * Acts as the overall data store for Pattern Lab. Takes in data found in JSON and YAML files.
 *
 */

namespace PatternLab;

use \PatternLab\Config;
use \PatternLab\Console;
use \PatternLab\Dispatcher;
use \PatternLab\Timer;
use \Symfony\Component\Yaml\Exception\ParseException;
use \Symfony\Component\Yaml\Yaml;

class Data {
	
	protected static $store        = array();
	protected static $reservedKeys = array("listItems","cacheBuster","link","patternSpecific","patternLabHead","patternLabFoot");
	
	/**
	* Gather data from any JSON and YAML files in source/_data
	*
	* Reserved attributes:
	*    - Data::$store["listItems"] : listItems from listitems.json, duplicated into separate arrays for Data::$store["listItems"]["one"], Data::$store["listItems"]["two"]... etc.
	*    - Data::$store["link"] : the links to each pattern
	*    - Data::$store["cacheBuster"] : the cache buster value to be appended to URLs
	*    - Data::$store["patternSpecific"] : holds attributes from the pattern-specific data files
	*
	* @return {Array}        populates Data::$store
	*/
	public static function gather($options = array()) {
		
		// set-up the dispatcher
		$dispatcherInstance = Dispatcher::getInstance();
		
		// dispatch that the data gather has started
		$dispatcherInstance->dispatch("data.gatherStart");
		
		// default vars
		$found         = false;
		$dataJSON      = array();
		$dataYAML      = array();
		$listItemsJSON = array();
		$listItemsYAML = array();
		$sourceDir     = Config::getOption("sourceDir");
		
		// iterate over all of the other files in the source directory
		if (!is_dir($sourceDir."/_data/")) {
			Console::writeLine("<path>_data/</path><warning> doesn't exist so you won't have dynamic data...</warning>");
			exit;
		}
		$directoryIterator = new \RecursiveDirectoryIterator($sourceDir."/_data/");
		$objects           = new \RecursiveIteratorIterator($directoryIterator, \RecursiveIteratorIterator::SELF_FIRST);
		
		// make sure dots are skipped
		$objects->setFlags(\FilesystemIterator::SKIP_DOTS);
		
		foreach ($objects as $name => $object) {
			
			$ext           = $object->getExtension();<commit_msg>making the _data dir if it doesn't exist<commit_after><?php

/*!
 * Data Class
 *
 * Copyright (c) 2014 <NAME>, http://dmolsen.com
 * Licensed under the MIT license
 *
 * Acts as the overall data store for Pattern Lab. Takes in data found in JSON and YAML files.
 *
 */

namespace PatternLab;

use \PatternLab\Config;
use \PatternLab\Console;
use \PatternLab\Dispatcher;
use \PatternLab\Timer;
use \Symfony\Component\Yaml\Exception\ParseException;
use \Symfony\Component\Yaml\Yaml;

class Data {
	
	protected static $store        = array();
	protected static $reservedKeys = array("listItems","cacheBuster","link","patternSpecific","patternLabHead","patternLabFoot");
	
	/**
	* Gather data from any JSON and YAML files in source/_data
	*
	* Reserved attributes:
	*    - Data::$store["listItems"] : listItems from listitems.json, duplicated into separate arrays for Data::$store["listItems"]["one"], Data::$store["listItems"]["two"]... etc.
	*    - Data::$store["link"] : the links to each pattern
	*    - Data::$store["cacheBuster"] : the cache buster value to be appended to URLs
	*    - Data::$store["patternSpecific"] : holds attributes from the pattern-specific data files
	*
	* @return {Array}        populates Data::$store
	*/
	public static function gather($options = array()) {
		
		// set-up the dispatcher
		$dispatcherInstance = Dispatcher::getInstance();
		
		// dispatch that the data gather has started
		$dispatcherInstance->dispatch("data.gatherStart");
		
		// default vars
		$found         = false;
		$dataJSON      = array();
		$dataYAML      = array();
		$listItemsJSON = array();
		$listItemsYAML = array();
		$sourceDir     = Config::getOption("sourceDir");
		
		// iterate over all of the other files in the source directory
		if (!is_dir($sourceDir."/_data/")) {
			Console::writeLine("<path>_data/</path><warning> doesn't exist so you won't have dynamic data...</warning>");
			exit;
			mkdir($sourceDir."/_data/");
		}
		$directoryIterator = new \RecursiveDirectoryIterator($sourceDir."/_data/");
		$objects           = new \RecursiveIteratorIterator($directoryIterator, \RecursiveIteratorIterator::SELF_FIRST);
		
		// make sure dots are skipped
		$objects->setFlags(\FilesystemIterator::SKIP_DOTS);
		
		foreach ($objects as $name => $object) {
			
			$ext           = $object->getExtension();
{'fraction_non_alphanumeric': 0.1413921012437268, 'fraction_numerical': 0.0017455814968361336, 'mean_word_length': 3.8201892744479493, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479759', 'n_tokens_mistral': 1461, 'n_tokens_neox': 1326, 'n_words': 371}