<commit_before><?php

/**
 * Copyright 2015-2016 Unicity International
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

declare(strict_types = 1);

namespace Unicity\VLD\Parser {

	use \Unicity\Common;
	use \Unicity\Config;
	use \Unicity\Core;
	use \Unicity\IO;
	use \Unicity\Locale;
	use \Unicity\ORM;
	use \Unicity\VLD;

	class Feedback extends Core\Object {

		/**
		 * @var Common\Mutable\HashSet
		 */
		protected $recommendations;

		/**
		 * @var string
		 */
		protected $root;

		/**
		 * @var Common\Mutable\HashSet
		 */
		protected $violations;

		public function __construct(string $root) {
			$this->root = $root;
			$this->recommendations = new Common\Mutable\HashSet();
			$this->violations = new Common\Mutable\HashSet();
		}

		public function addRecommendation(VLD\Parser\RuleType $type, array $fields, string $message, array $values = []) : void {
			ksort($values);
			ksort($fields);
			$this->recommendations->putValue([
				'fields' => static::mapRecommendations($this->root, $fields),
				'message' => strtr(static::localize($message), $values),
				'type' => (string) $type,
			]);
		}

		public function addRecommendations(VLD\Parser\Feedback $feedback) : void {
			$this->recommendations->putValues($feedback->recommendations);
		}

		public function addViolation(VLD\Parser\RuleType $type, array $fields, string $message, array $values = []) : void {
			ksort($values);
			sort($fields);
			$this->violations->putValue([
				'fields' => static::mapViolations($this->root, $fields),
				'message' => strtr(static::localize($message), $values),
				'type' => (string) $type,
			]);
		}

		public function addViolations(VLD\Parser\Feedback $feedback) : void {
			$this->violations->putValues($feedback->violations);
		}

		public function getNumberOfRecommendations() : int {
			return $this->recommendations->count();
		}

		public function getNumberOfViolations() : int {
			return $this->violations->count();
		}

		public function toMap() : Common\IMap {
			$feedback = new ORM\JSON\Model\HashMap('\\Unicity\\VLD\\Parser\\Model\\Feedback');
			$feedback->recommendations = $this->recommendations;
			$feedback->violations = $this->violations;
			return $feedback;
		}

		protected static $localization = null;

		protected static function localize(string $message) {
			if (static::$localization === null) {
				static::$localization = static::localize_();
			}
			if (static::$localization->hasKey($message)) {
				return static::$localization->getValue($message);
			}
			return $message;
		}

		protected static function localize_() {
			$languages = Locale\Info::getLanguages();
			$file = dirname(__FILE__) . DIRECTORY_SEPARATOR . 'Messages';
			foreach ($languages as $language => $q) {
				$ext = '_' . str_replace('-', '_', $language) . '.properties';
				$uri = $file . $ext;
				if (file_exists($uri)) {
					return Config\Properties\Reader::load(new IO\File($uri))->read();
				}
			}
			$uri = $file . '.properties';
			return Config\Properties\Reader::load(new IO\File($uri))->read();
		}

		protected static function mapRecommendations(string $root, array $fields) {
			$buffer = array();
			$i = 0;
			foreach ($fields as $k => $v) {
				$buffer[$i]['field'] = static::formatKey($root, (string) $k);
				$buffer[$i]['to'] = $v;
				$i++;
			}
			return $buffer;
		}

		protected static function mapViolations(string $root, array $fields) {
			$buffer = array();
			foreach ($fields as $i => $v) {
				$buffer[$i]['field'] = static::formatKey($root, (string) $v);
			}
			return $buffer;
		}

		/**
		 * This method returns the JSONPath for the given key.
		 *
		 * @access protected
		 * @static
		 * @param string $path                                      the current path
		 * @param string $key                                       the key to be affixed
		 * @return string                                           the new path
		 */
		protected static function formatKey(?string $path, string $key) {
			$buffer = array('$');
			$pattern = (!is_null($path)) ? explode('.', $path) : array();
			foreach ($pattern as $segment) {
				if (!in_array($segment, ['', '$', '@'])) {
					$buffer[] = $segment;
				}
			}
			$pattern = explode('.', $key);
			foreach ($pattern as $segment) {
				if (!in_array($segment, ['', '@'])) {
					$buffer[] = $segment;
				}
			}
			return implode('.', $buffer);
		}

	}

}<commit_msg>Improving JSONPath format for indexes.<commit_after><?php

/**
 * Copyright 2015-2016 Unicity International
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

declare(strict_types = 1);

namespace Unicity\VLD\Parser {

	use \Unicity\Common;
	use \Unicity\Config;
	use \Unicity\Core;
	use \Unicity\IO;
	use \Unicity\Locale;
	use \Unicity\ORM;
	use \Unicity\VLD;

	class Feedback extends Core\Object {

		/**
		 * @var Common\Mutable\HashSet
		 */
		protected $recommendations;

		/**
		 * @var string
		 */
		protected $root;

		/**
		 * @var Common\Mutable\HashSet
		 */
		protected $violations;

		public function __construct(string $root) {
			$this->root = $root;
			$this->recommendations = new Common\Mutable\HashSet();
			$this->violations = new Common\Mutable\HashSet();
		}

		public function addRecommendation(VLD\Parser\RuleType $type, array $fields, string $message, array $values = []) : void {
			ksort($values);
			ksort($fields);
			$this->recommendations->putValue([
				'fields' => static::mapRecommendations($this->root, $fields),
				'message' => strtr(static::localize($message), $values),
				'type' => (string) $type,
			]);
		}

		public function addRecommendations(VLD\Parser\Feedback $feedback) : void {
			$this->recommendations->putValues($feedback->recommendations);
		}

		public function addViolation(VLD\Parser\RuleType $type, array $fields, string $message, array $values = []) : void {
			ksort($values);
			sort($fields);
			$this->violations->putValue([
				'fields' => static::mapViolations($this->root, $fields),
				'message' => strtr(static::localize($message), $values),
				'type' => (string) $type,
			]);
		}

		public function addViolations(VLD\Parser\Feedback $feedback) : void {
			$this->violations->putValues($feedback->violations);
		}

		public function getNumberOfRecommendations() : int {
			return $this->recommendations->count();
		}

		public function getNumberOfViolations() : int {
			return $this->violations->count();
		}

		public function toMap() : Common\IMap {
			$feedback = new ORM\JSON\Model\HashMap('\\Unicity\\VLD\\Parser\\Model\\Feedback');
			$feedback->recommendations = $this->recommendations;
			$feedback->violations = $this->violations;
			return $feedback;
		}

		protected static $localization = null;

		protected static function localize(string $message) {
			if (static::$localization === null) {
				static::$localization = static::localize_();
			}
			if (static::$localization->hasKey($message)) {
				return static::$localization->getValue($message);
			}
			return $message;
		}

		protected static function localize_() {
			$languages = Locale\Info::getLanguages();
			$file = dirname(__FILE__) . DIRECTORY_SEPARATOR . 'Messages';
			foreach ($languages as $language => $q) {
				$ext = '_' . str_replace('-', '_', $language) . '.properties';
				$uri = $file . $ext;
				if (file_exists($uri)) {
					return Config\Properties\Reader::load(new IO\File($uri))->read();
				}
			}
			$uri = $file . '.properties';
			return Config\Properties\Reader::load(new IO\File($uri))->read();
		}

		protected static function mapRecommendations(string $root, array $fields) {
			$buffer = array();
			$i = 0;
			foreach ($fields as $k => $v) {
				$buffer[$i]['field'] = static::formatKey($root, (string) $k);
				$buffer[$i]['to'] = $v;
				$i++;
			}
			return $buffer;
		}

		protected static function mapViolations(string $root, array $fields) {
			$buffer = array();
			foreach ($fields as $i => $v) {
				$buffer[$i]['field'] = static::formatKey($root, (string) $v);
			}
			return $buffer;
		}

		/**
		 * This method returns the JSONPath for the given key.
		 *
		 * @access protected
		 * @static
		 * @param string $path                                      the current path
		 * @param string $key                                       the key to be affixed
		 * @return string                                           the new path
		 */
		protected static function formatKey(?string $path, string $key) {
			$buffer = array('$');
			$pattern = (!is_null($path)) ? explode('.', $path) : array();
			foreach ($pattern as $segment) {
				if (!in_array($segment, ['', '$', '@'])) {
					$buffer[] = is_numeric($segment) ? "[{$segment}]" : $segment;
				}
			}
			$pattern = explode('.', $key);
			foreach ($pattern as $segment) {
				if (!in_array($segment, ['', '$', '@'])) {
					$buffer[] = is_numeric($segment) ? "[{$segment}]" : $segment;
				}
			}
			return preg_replace('/\.\[(0|[1-9][0-9]*)\]/', '[$1]', implode('.', $buffer));
		}

	}

}
{'fraction_non_alphanumeric': 0.16282115869017633, 'fraction_numerical': 0.0034256926952141056, 'mean_word_length': 3.2582582582582584, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1871558', 'n_tokens_mistral': 3616, 'n_tokens_neox': 3032, 'n_words': 764}