<commit_before>from gnowsys_ndf.ndf.models  import *
import os
from django.core.management.base import BaseCommand, CommandError
from django.core import serializers
from bson.json_util import dumps,loads,object_hook
import json
class Command(BaseCommand):
    help = "setup the initial database"
    def handle(self,*args,**options):
		file_names = ['metatype.json','AttributeType.json','RelationType.json','GSystemType.json','factorydata.json']
		for i in file_names:	
			PROJECT_ROOT = os.path.abspath(os.path.dirname(os.pardir))
			refcatpath = os.path.join(PROJECT_ROOT + '/GRef.cat./' + i)
			path_val = os.path.exists(refcatpath)
			if path_val:
				with open(refcatpath, 'r') as json_file:
					json_data = json_file.read()
				json_documents_list = json.loads(json_data)
				
			# Process data in proper format
			
			
			# process data based on the structure type			
			parsed_json_document = []
			for j,json_data in enumerate(json_documents_list):
				converted_data = node_collection.from_json(json.dumps(json_data))
				#check if node already exist
				existing_node = node_collection.one({"_id":ObjectId(converted_data["_id"])})	
				if existing_node is None:
					if i ==  'GSystemType.json':
						node = node_collection.collection.GSystemType()
					elif i == 'factorydata.json':
						if json_data['_type'] == 'Group':
							node = node_collection.collection.Group()
						else:					
							node = node_collection.collection.GSystem()
					elif i == 'metatype.json':
						node = node_collection.collection.MetaType()
					elif i == 'RelationType.json': 
						node = node_collection.collection.RelationType()
					elif i == 'AttributeType.json':
						node = node_collection.collection.AttributeType()
					if converted_data['name']:
					
							for key, values in converted_data.items():
								if values and type(values) == list:
									oid_ObjectId_list = []
									oid_list_str = values.__str__()
									if key == 'attribute_type_set' or key == 'relation_type_set' or key == 'meta_type_set':
										value = []	
										if values is not None:
											value = process_list(values)
											node[key] = value
									elif '$oid' in oid_list_str:
										for oid_dict in values:
											oid_ObjectId = ObjectId(oid_dict['$oid'])
											oid_ObjectId_list.append(oid_ObjectId)

										node[key] = oid_ObjectId_list
									 
							
								else:
									if key != "_type":
										node[key] = values
					
						
							node.save()
					
				else:<commit_msg>code modified to add modified by while restoring data<commit_after>from gnowsys_ndf.ndf.models  import *
import os
from django.core.management.base import BaseCommand, CommandError
from django.core import serializers
from bson.json_util import dumps,loads,object_hook
import json
class Command(BaseCommand):
    help = "setup the initial database"
    def handle(self,*args,**options):
		file_names = ['metatype.json','AttributeType.json','RelationType.json','GSystemType.json','factorydata.json']
		for i in file_names:	
			PROJECT_ROOT = os.path.abspath(os.path.dirname(os.pardir))
			refcatpath = os.path.join(PROJECT_ROOT + '/GRef.cat./' + i)
			path_val = os.path.exists(refcatpath)
			if path_val:
				with open(refcatpath, 'r') as json_file:
					json_data = json_file.read()
				json_documents_list = json.loads(json_data)
				
			# Process data in proper format
			
			
			# process data based on the structure type			
			parsed_json_document = []
			for j,json_data in enumerate(json_documents_list):
				converted_data = node_collection.from_json(json.dumps(json_data))
				#check if node already exist
				existing_node = node_collection.one({"_id":ObjectId(converted_data["_id"])})	
				if existing_node is None:
					if i ==  'GSystemType.json':
						node = node_collection.collection.GSystemType()
					elif i == 'factorydata.json':
						if json_data['_type'] == 'Group':
							node = node_collection.collection.Group()
						else:					
							node = node_collection.collection.GSystem()
					elif i == 'metatype.json':
						node = node_collection.collection.MetaType()
					elif i == 'RelationType.json': 
						node = node_collection.collection.RelationType()
					elif i == 'AttributeType.json':
						node = node_collection.collection.AttributeType()
					if converted_data['name']:
							if converted_data['modified_by'] is None:
								converted_data['modified_by'] = 1	
							
							for key, values in converted_data.items():
								
								if values and type(values) == list:
									oid_ObjectId_list = []
									oid_list_str = values.__str__()
									if key == 'attribute_type_set' or key == 'relation_type_set' or key == 'meta_type_set':
										value = []	
										if values is not None:
											value = process_list(values)
											node[key] = value
									elif '$oid' in oid_list_str:
										for oid_dict in values:
											oid_ObjectId = ObjectId(oid_dict['$oid'])
											oid_ObjectId_list.append(oid_ObjectId)

										node[key] = oid_ObjectId_list
									 
							
								else:
									if key != "_type":
										node[key] = values
					
						
							node.save()
					
				else:
{'fraction_non_alphanumeric': 0.10795124782356355, 'fraction_numerical': 0.0001934610176049526, 'mean_word_length': 2.832468495181616, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7589059', 'n_tokens_mistral': 2235, 'n_tokens_neox': 1585, 'n_words': 353}