<commit_before># Copyright 2012-2013 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import yaml
import os
from os.path import splitext, split, exists, join
import re
import copy
import openscad,freecad, html

_re_angled = re.compile("([^<]*)<([^>]*)")

current_version = 0.2

#this is not super-precise, but allows to do some rough checks
_specification = {
	"root" : (["collection","classes"],[]),
	"collection" : (["author","license","blt-version"],["name","description"]),
	"class" : (["naming","source","id"],["drawing","description","standard","status","replaces","parameters","url","notes"]),
	"naming" : (["template"],["substitute"]),
	"parameters" : ([],["literal","free","tables","types"]),
	"table" : (["index","columns","data"],[])
}

class VersionError(Exception):
	def __init__(self,version):
		self.version = version
	def __str__(self):
		return "Unknown version. File: %f Current: %f" % \
			(self.version,current_version)

class UnknownFieldError(Exception):
	def __init__(self,fieldname):
		self.field = fieldname
	def __str__(self):
		return "Unknown Field: %s" % self.field

class MissingFieldError(Exception):
	def __init__(self,fieldname):
		self.field = fieldname
	def __str__(self):
		return "Missing Field: %s" % self.field

class MalformedRepositoryError(Exception):
	def __init__(self,msg):
		self.msg = msg
	def __str(self):
		return self.msg

class MalformedCollectionError(Exception):
	def __init__(self,msg):
		self.msg = msg
	def __str(self):
		return self.msg

def check_dict(array,spec):
	man = spec[0][:]
	opt = spec[1][:]
	for key in array.keys():
		if key in man:
			man.remove(key)
		elif key in opt:
			opt.remove(key)
		else:
			raise UnknownFieldError(key)
	if len(man) > 0:
		raise MissingFieldError(man)


class BOLTSRepository:
	#order is important
	standard_bodies = ["DINENISO","DINEN","DINISO","DIN","EN","ISO"]
	def __init__(self,path):
		self.path = path
		self.collections = []

		#load collection data
		for filename in os.listdir(join(path,"data")):
			if splitext(filename)[1] == ".blt":
				self.collections.append(BOLTSCollection(join(path,"data",filename)))

		self.standardized = {body:[] for body in self.standard_bodies}

		#find standard parts and their respective standard bodies
		for coll in self.collections:
			for cl in coll.classes:
				#order is important
				for body in self.standard_bodies:
					if cl.name.startswith(body):
						self.standardized[body].append(cl)
						cl.body = body
						break

		#fill in obsolescence data
		for coll in self.collections:
			for cl in coll.classes:
				if cl.replaces is None:
					continue
				#order is important
				for body in self.standard_bodies:
					if cl.replaces.startswith(body):
						idx = [c.name for c in self.standardized[body]].index(cl.replaces)
						self.standardized[body][idx].replacedby = cl.name
						break


		if not exists(join(path,"drawings")):
			raise MalformedRepositoryError("drawings folder is missing")

		#load backend data
		self.openscad = None
		if exists(join(path,"openscad")):
			self.openscad = openscad.OpenSCADData(path)
		self.freecad = None
		if exists(join(path,"freecad")):
			self.freecad = freecad.FreeCADData(path)
		self.html = None
		if exists(join(path,"html")):
			self.html = html.HTMLData(path)

class BOLTSCollection:
	def __init__(self,bltname):
		self.id = splitext(split(bltname)[1])[0]
		if self.id in ["common","output","data"]:
			raise MalformedCollectionError(
					"Forbidden collection id: %s" % self.id)
		coll = list(yaml.load_all(open(bltname)))
		if len(coll) == 0:
			raise MalformedCollectionError(
					"No YAML document found in file %s" % bltname)
		if len(coll) > 1:
			raise MalformedCollectionError(
					"More than one YAML document found in file %s" % bltname)
		coll = coll[0]

		version = coll["collection"]["blt-version"]
		if version != current_version:
			raise VersionError(version)

		#Check Conformity<commit_msg>add downloads backend to commandline and repo<commit_after># Copyright 2012-2013 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import yaml
import os
from os.path import splitext, split, exists, join
import re
import copy
import openscad,freecad, html, downloads

_re_angled = re.compile("([^<]*)<([^>]*)")

current_version = 0.2

#this is not super-precise, but allows to do some rough checks
_specification = {
	"root" : (["collection","classes"],[]),
	"collection" : (["author","license","blt-version"],["name","description"]),
	"class" : (["naming","source","id"],["drawing","description","standard","status","replaces","parameters","url","notes"]),
	"naming" : (["template"],["substitute"]),
	"parameters" : ([],["literal","free","tables","types"]),
	"table" : (["index","columns","data"],[])
}

class VersionError(Exception):
	def __init__(self,version):
		self.version = version
	def __str__(self):
		return "Unknown version. File: %f Current: %f" % \
			(self.version,current_version)

class UnknownFieldError(Exception):
	def __init__(self,fieldname):
		self.field = fieldname
	def __str__(self):
		return "Unknown Field: %s" % self.field

class MissingFieldError(Exception):
	def __init__(self,fieldname):
		self.field = fieldname
	def __str__(self):
		return "Missing Field: %s" % self.field

class MalformedRepositoryError(Exception):
	def __init__(self,msg):
		self.msg = msg
	def __str(self):
		return self.msg

class MalformedCollectionError(Exception):
	def __init__(self,msg):
		self.msg = msg
	def __str(self):
		return self.msg

def check_dict(array,spec):
	man = spec[0][:]
	opt = spec[1][:]
	for key in array.keys():
		if key in man:
			man.remove(key)
		elif key in opt:
			opt.remove(key)
		else:
			raise UnknownFieldError(key)
	if len(man) > 0:
		raise MissingFieldError(man)


class BOLTSRepository:
	#order is important
	standard_bodies = ["DINENISO","DINEN","DINISO","DIN","EN","ISO"]
	def __init__(self,path):
		self.path = path
		self.collections = []

		#load collection data
		for filename in os.listdir(join(path,"data")):
			if splitext(filename)[1] == ".blt":
				self.collections.append(BOLTSCollection(join(path,"data",filename)))

		self.standardized = {body:[] for body in self.standard_bodies}

		#find standard parts and their respective standard bodies
		for coll in self.collections:
			for cl in coll.classes:
				#order is important
				for body in self.standard_bodies:
					if cl.name.startswith(body):
						self.standardized[body].append(cl)
						cl.body = body
						break

		#fill in obsolescence data
		for coll in self.collections:
			for cl in coll.classes:
				if cl.replaces is None:
					continue
				#order is important
				for body in self.standard_bodies:
					if cl.replaces.startswith(body):
						idx = [c.name for c in self.standardized[body]].index(cl.replaces)
						self.standardized[body][idx].replacedby = cl.name
						break


		if not exists(join(path,"drawings")):
			raise MalformedRepositoryError("drawings folder is missing")

		#load backend data
		self.openscad = None
		if exists(join(path,"openscad")):
			self.openscad = openscad.OpenSCADData(path)
		self.freecad = None
		if exists(join(path,"freecad")):
			self.freecad = freecad.FreeCADData(path)
		self.html = None
		if exists(join(path,"html")):
			self.html = html.HTMLData(path)
		self.downloads = None
		if exists(join(path,"downloads")):
			self.downloads = downloads.DownloadsData(path)

class BOLTSCollection:
	def __init__(self,bltname):
		self.id = splitext(split(bltname)[1])[0]
		if self.id in ["common","output","data"]:
			raise MalformedCollectionError(
					"Forbidden collection id: %s" % self.id)
		coll = list(yaml.load_all(open(bltname)))
		if len(coll) == 0:
			raise MalformedCollectionError(
					"No YAML document found in file %s" % bltname)
		if len(coll) > 1:
			raise MalformedCollectionError(
					"More than one YAML document found in file %s" % bltname)
		coll = coll[0]

		version = coll["collection"]["blt-version"]
		if version != current_version:
			raise VersionError(version)

		#Check Conformity
{'fraction_non_alphanumeric': 0.1342572542226072, 'fraction_numerical': 0.0043308791684712, 'mean_word_length': 4.531137724550899, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3458253', 'n_tokens_mistral': 3335, 'n_tokens_neox': 2941, 'n_words': 858}