<commit_before>#!/usr/bin/env python

#
# file: generateChipYaml.py
#
# author: Copyright (C) 2018 <NAME> http://www.distortec.com http://www.freddiechopin.info
#
# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
# distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import argparse
import ast
import common
from common import Reference
import csv
import datetime
import os
import re
import ruamel.yaml

def addLabels(dictionary, labels):
	"""Add `'$labels'` keys to appropriate nodes in dictionary and return this dictionary.

	* `dictionary` is a dictionary representing an assembled row of CSV file
	* `labels` is a list of labels which should be added to the dictionary
	"""
	for keys, label in labels:
		try:
			node = dictionary
			for key in keys:
				node = node[key]
			node['$labels'] = [label]
		except KeyError:
			pass

	return dictionary

def parseFirstRow(firstRow):
	"""Parse first row of the CSV file into list of paths and list of labels and return tuple with the list of paths.

	List of paths has one entry for each column in the CSV file. Each entry is a list of keys for the element in that
	column.

	List of labels has one entry for any path which should have the label added. Each entry is a tuple of path (list of
	keys) and a label for the node on that path.

	* `firstRow` is a list of strings representing the first row of the CSV file
	"""
	previousKeys = firstRow[0].split('/')
	paths = []
	labels = []
	for header in firstRow:
		keys = header.split('/')

		i = 0
		while keys[i] == '..':
			i += 1
		keys[0:i] = previousKeys[0:-i]

		for i, key in enumerate(keys):
			if isinstance(key, str) == True:
				key, separator, label = key.partition('$')
				key = parseString(key)
				keys[i] = key
				if separator == '$':
					labels.append((keys[:i + 1], label or key))

		paths.append(keys)
		previousKeys = keys

	return paths, labels

def parseRow(row, paths):
	"""Assemble single row from CSV file into a dictionary and return this dictionary.

	* `row` is a list of strings representing a single row of CSV file
	* `paths` is a list of paths parsed from the first row of CSV file
	"""
	dictionary = {}
	for column, element in enumerate(row):
		if element != '':
			node = dictionary
			for key in paths[column][:-1]:
				node = node.setdefault(key, {})
			node[paths[column][-1]] = parseString(element)

	return dictionary

def parseString(string):
	"""Try to parse `string` into proper type and return it.

	* `string` is the string which will be parsed
	"""
	try:
		return ast.literal_eval(string)
	except ValueError:
		match = re.match("Reference\(label = '(\w+)'\)\Z", string)
		if match:
			return Reference(label = match[1])
		else:
			return string

########################################################################################################################
# main
########################################################################################################################

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('csvFile', help = 'input CSV file')
	parser.add_argument('-o','--outputPath', help = 'output path')
	arguments = parser.parse_args()

	if not arguments.outputPath:
		arguments.outputPath = os.path.dirname(arguments.csvFile) + '/chipYaml'

	year = datetime.date.today().year

	with open(arguments.csvFile) as csvFile:
		csvReader = csv.reader(csvFile)
		firstRow = next(csvReader)
		paths, labels = parseFirstRow(firstRow)
		yaml = ruamel.yaml.YAML()
		yaml.register_class(Reference)<commit_msg>Slightly change the way the header is parsed in generateChipYaml.py<commit_after>#!/usr/bin/env python

#
# file: generateChipYaml.py
#
# author: Copyright (C) 2018 <NAME> http://www.distortec.com http://www.freddiechopin.info
#
# This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not
# distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#

import argparse
import ast
import common
from common import Reference
import csv
import datetime
import os
import re
import ruamel.yaml

def addLabels(dictionary, labels):
	"""Add `'$labels'` keys to appropriate nodes in dictionary and return this dictionary.

	* `dictionary` is a dictionary representing an assembled row of CSV file
	* `labels` is a list of labels which should be added to the dictionary
	"""
	for keys, label in labels:
		try:
			node = dictionary
			for key in keys:
				node = node[key]
			node['$labels'] = [label]
		except KeyError:
			pass

	return dictionary

def parseFirstRow(firstRow):
	"""Parse first row of the CSV file into list of paths and list of labels and return tuple with the list of paths.

	List of paths has one entry for each column in the CSV file. Each entry is a list of keys for the element in that
	column.

	List of labels has one entry for any path which should have the label added. Each entry is a tuple of path (list of
	keys) and a label for the node on that path.

	* `firstRow` is a list of strings representing the first row of the CSV file
	"""
	previousKeys = firstRow[0].split('/')
	paths = []
	labels = []
	for header in firstRow:
		keys = header.split('/')

		i = 0
		while keys[i] == '..':
			i += 1
		prefix = previousKeys[0:-i]
		keys[0:i] = prefix
		prefixLength = len(prefix)

		for i, key in enumerate(keys[prefixLength:], start = prefixLength):
			key = parseString(key)
			keys[i] = key
			if isinstance(key, str) == True:
				key, separator, label = key.partition('$')
				keys[i] = key
				if separator == '$':
					labels.append((keys[:i + 1], label or key))

		paths.append(keys)
		previousKeys = keys

	return paths, labels

def parseRow(row, paths):
	"""Assemble single row from CSV file into a dictionary and return this dictionary.

	* `row` is a list of strings representing a single row of CSV file
	* `paths` is a list of paths parsed from the first row of CSV file
	"""
	dictionary = {}
	for column, element in enumerate(row):
		if element != '':
			node = dictionary
			for key in paths[column][:-1]:
				node = node.setdefault(key, {})
			node[paths[column][-1]] = parseString(element)

	return dictionary

def parseString(string):
	"""Try to parse `string` into proper type and return it.

	* `string` is the string which will be parsed
	"""
	try:
		return ast.literal_eval(string)
	except (SyntaxError, ValueError):
		match = re.match("Reference\(label = '(\w+)'\)\Z", string)
		if match:
			return Reference(label = match[1])
		else:
			return string

########################################################################################################################
# main
########################################################################################################################

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('csvFile', help = 'input CSV file')
	parser.add_argument('-o','--outputPath', help = 'output path')
	arguments = parser.parse_args()

	if not arguments.outputPath:
		arguments.outputPath = os.path.dirname(arguments.csvFile) + '/chipYaml'

	year = datetime.date.today().year

	with open(arguments.csvFile) as csvFile:
		csvReader = csv.reader(csvFile)
		firstRow = next(csvReader)
		paths, labels = parseFirstRow(firstRow)
		yaml = ruamel.yaml.YAML()
		yaml.register_class(Reference)
{'fraction_non_alphanumeric': 0.1684011352885525, 'fraction_numerical': 0.004595215569671577, 'mean_word_length': 4.103448275862069, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6172184', 'n_tokens_mistral': 2376, 'n_tokens_neox': 2127, 'n_words': 839}