<commit_before>import sys
import pprint
import json
import toml
pp = pprint.PrettyPrinter();
import pymongo
from pymongo import MongoClient

# NOTE: installations required for the javascript-based config system
#import PyV8
#ctx = PyV8.JSContext()
#ctx.enter()

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created on Wed Aug 22 19:55:11 PDT 2018

@author: jpitts

Requirements:
    - toml, pymongo need to be installed
    - mongodb needs to be running
    - brigade-matchmaker web needs to be running
        (populates the users table in the brigade-matchmaker collection)

Installation:
		pip3 install pymomgo
		pip3 install toml

"""

# load config file
#file = open("../../etc/development.js","w")
#print(file.read())

# database configuration
collection_name = 'brigade-matchmaker'
client = MongoClient('localhost', 27017)
db = client[collection_name]


# Tree Structure for UI Rendering
# SEE: https://github.com/designforsf/brigade-matchmaker/blob/master/docs/taxonomy.md#tree-structure-for-ui-rendering
tax_dict = {
	"skills": {
		"itemsBySection": {}
	},
	"learnSkills": {
		"itemsBySection": {}
	},
	"interests": {
		"itemsBySection": {}
	}
}

# load the taxonomy data from the database
taxonomies = []
taxonomy_count = 0
for tax in db.projecttaxonomies.find({}):
	#print(tax['parent'], '/', tax['name'])
	taxonomies.append(tax)

# process level 0 - taxonomy root and sections
for tax_index, tax in enumerate(taxonomies):
	
	# remove from tax array
	taxonomies.pop(tax_index)

	for root in tax_dict.keys():
		if tax['parent'] == root:
			
			# legacy clean up
			tax.pop('_id', None)
			tax.pop('className', None)

			tax_dict[root]['itemsBySection'][tax['name']] = tax

			# array for items under this section
			tax_dict[root]['itemsBySection'][tax['name']]['items'] = []

# process level 1 - sections and items

for root in tax_dict.keys():
	print('ROOT: ', root)

	for section in tax_dict[root]['itemsBySection'].keys():
		#print('SECTION: ', section)

		for tax_index, tax in enumerate(taxonomies):
			if tax['parent'] == section:

				# legacy clean up
				tax.pop('_id', None)
				tax.pop('className', None)

				tax_dict[root]['itemsBySection'][section]['items'].append(tax)
				#print('ITEM: ', tax['name'])


# save the toml output
tax_toml = toml.dumps(tax_dict)

print(tax_toml)
<commit_msg>Taxonomy: Updated the importer to use "Tree Structure for Human Editing" for output.<commit_after>import sys
import pprint
import json
import toml
pp = pprint.PrettyPrinter();
import pymongo
from pymongo import MongoClient

# NOTE: installations required for the javascript-based config system
#import PyV8
#ctx = PyV8.JSContext()
#ctx.enter()

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Created on Wed Aug 22 19:55:11 PDT 2018

@author: jpitts

Requirements:
    - toml, pymongo need to be installed
    - mongodb needs to be running
    - brigade-matchmaker web needs to be running
        (populates the users table in the brigade-matchmaker collection)

Installation:
		pip3 install pymomgo
		pip3 install toml

"""

# load config file
#file = open("../../etc/development.js","w")
#print(file.read())

# database configuration
collection_name = 'brigade-matchmaker'
client = MongoClient('localhost', 27017)
db = client[collection_name]

# taxonomy domains
# 	Tree Structure for UI Rendering
# 	(dump this out instead of tax_output for reference)
#
# SEE: https://github.com/designforsf/brigade-matchmaker/blob/master/docs/taxonomy.md#tree-structure-for-ui-rendering

tax_domains = {
	"skills": {
		"itemsBySection": {}
	},
	"interests": {
		"itemsBySection": {}
	}
}

# taxonomy output
# 	Tree Structure for Human Editing
# 	(output)
#
# SEE: https://github.com/designforsf/brigade-matchmaker/blob/master/docs/taxonomy.md#tree-structure-for-ui-rendering

tax_output = {}


# load the taxonomy data from the database
taxonomies = []
taxonomy_count = 0
for tax in db.projecttaxonomies.find({}):
	#print(tax['parent'], '/', tax['name'])
	taxonomies.append(tax)

# process level 0 - taxonomy root and sections
for tax_index, tax in enumerate(taxonomies):
	
	# remove from tax array
	#taxonomies.pop(tax_index)

	for root in tax_domains.keys():
		if tax['parent'] == root:

			tax_domains[root]['itemsBySection'][tax['name']] = tax

			# array for items under this section
			tax_domains[root]['itemsBySection'][tax['name']]['items'] = []


# process level 1 - sections and items

for root in tax_domains.keys():
	#print('ROOT: ', root)

	# process level 1 - sections and items

	for section in tax_domains[root]['itemsBySection'].keys():
		#print('SECTION: ', section)


		outputTax = tax_domains[root]['itemsBySection'][section].copy()

		# legacy clean up
		outputTax.pop('_id', None)
		outputTax.pop('className', None)
		outputTax.pop('name', None)
		outputTax.pop('parent', None)
		outputTax.pop('items', None)

		path = root + '/' + section;
		tax_output[path] = outputTax;

		for tax_index, tax in enumerate(taxonomies):
			if tax['parent'] == section:

				outputTax = tax.copy()

				# legacy clean up
				outputTax.pop('_id', None)
				outputTax.pop('className', None)
				outputTax.pop('name', None)
				outputTax.pop('parent', None)

				tax_domains[root]['itemsBySection'][section]['items'].append(tax)
				#print('ITEM: ', tax['name'])
				
				path = root + '/' + section + '/' + tax['name'];
				tax_output[path] = outputTax;


# output the Tree Structure for UI Rendering
#print(toml.dumps(tax_domains))

# output the Tree Structure for Human Editing
print(toml.dumps(tax_output))

{'fraction_non_alphanumeric': 0.14055716353111433, 'fraction_numerical': 0.00958755426917511, 'mean_word_length': 4.311239193083574, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7381603', 'n_tokens_mistral': 2024, 'n_tokens_neox': 1813, 'n_words': 481}