<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from viz.models import PollingStation, Election, Party, RegionalElectoralDistrict, State, District, Municipality, List
import json
import datetime

class Command(BaseCommand):

	help = 'Import basic data into database'

	def handle(self, *args, **options):

		config = {
			'party_location': 'austria',
			'log_detail' : 'low'
		}

		# import elections
		elections = json.loads(self.open_file('data/setup/elections.json'))
		self.import_elections(elections, config)

		# import regional electoral districts
		reds = json.loads(self.open_file('data/setup/regional-electoral-districts_20170101.json'))
		self.import_reds(reds, config)

		# import parties
		parties = json.loads(self.open_file('data/setup/parties.json'))
		self.import_parties(parties, config)
		
		# import lists
		lists = json.loads(self.open_file('data/setup/lists.json'))
		self.import_lists(lists, config)

		# import states and districts
		states_districts = json.loads(self.open_file('data/setup/states-to-districts_20170101.json'))
		self.import_states_districts(states_districts, config)

		# import municipalities
		municipalities = json.loads(self.open_file('data/setup/municipalities_20170101_2.json'))
		muns2reds = json.loads(self.open_file('data/setup/municipality2red_20170101.json'))
		self.import_municipalities(municipalities, muns2reds, config)

	def open_file(self, filename,):
		"""
		Open file.
		"""

		try:
			with open(filename) as data_file:
				return data_file.read()
		except IOError:
			print('Error: can\'t find file or read data')

	def open_jsonfile(self, filename):

		try:
			data = json.loads(self.open_file(filename))
			return data
		except JSONDecodeError:
			print('Error: File is not valid JSON.')

	def import_elections(self, elections, config):
		"""
		Import elections data into database.
		"""

		num_entries_created = 0
		num_entries_updated = 0

		for key, value in elections.items():
			ts = datetime.datetime.strptime(value['election_day'], "%Y-%m-%d")
			ts = timezone.make_aware(ts, timezone.get_current_timezone())

			e = Election.objects.update_or_create(
				short_name = value['short_name'],
				short_name_text = value['short_name_text'],
				full_name = value['full_name'],
				election_type = value['election_type'],
				wikidata_id = value['wikidata_id'],
				administrative_level = value['administrative_level'],
				election_day = ts<commit_msg>Added UTF8 Import for Textfiles (JSON and Plain-Textfiles)<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone
from viz.models import PollingStation, Election, Party, RegionalElectoralDistrict, State, District, Municipality, List
import json
import datetime

class Command(BaseCommand):

	help = 'Import basic data into database'

	def handle(self, *args, **options):

		config = {
			'party_location': 'austria',
			'log_detail' : 'low'
		}

		# import elections
		elections = self.open_jsonfile('data/setup/elections.json')
		self.import_elections(elections, config)

		# import regional electoral districts
		reds = self.open_jsonfile('data/setup/regional-electoral-districts_20170101.json')
		self.import_reds(reds, config)

		# import parties
		parties = self.open_jsonfile('data/setup/parties.json')
		self.import_parties(parties, config)
		
		# import lists
		lists = self.open_jsonfile('data/setup/lists.json')
		self.import_lists(lists, config)

		# import states and districts
		states_districts = self.open_jsonfile('data/setup/states-to-districts_20170101.json')
		self.import_states_districts(states_districts, config)

		# import municipalities
		municipalities = self.open_jsonfile('data/setup/municipalities_20170101_2.json')
		muns2reds = self.open_jsonfile('data/setup/municipality2red_20170101.json')
		self.import_municipalities(municipalities, muns2reds, config)

	def open_file(self, filename,):
		"""
		Open file.
		"""

		try:
			with open(filename, encoding='utf-8') as data_file:
				return data_file.read()
		except IOError:
			print('Error: can\'t find file or read data')

	def open_jsonfile(self, filename):

		try:
			data = json.loads(open(filename, encoding='utf-8-sig').read())
			return data
		except JSONDecodeError:
			print('Error: File is not valid JSON.')

	def import_elections(self, elections, config):
		"""
		Import elections data into database.
		"""

		num_entries_created = 0
		num_entries_updated = 0

		for key, value in elections.items():
			ts = datetime.datetime.strptime(value['election_day'], "%Y-%m-%d")
			ts = timezone.make_aware(ts, timezone.get_current_timezone())

			e = Election.objects.update_or_create(
				short_name = value['short_name'],
				short_name_text = value['short_name_text'],
				full_name = value['full_name'],
				election_type = value['election_type'],
				wikidata_id = value['wikidata_id'],
				administrative_level = value['administrative_level'],
				election_day = ts
{'fraction_non_alphanumeric': 0.1166, 'fraction_numerical': 0.0158, 'mean_word_length': 4.960667461263409, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7380524', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1587, 'n_words': 330}