<commit_before>from django.contrib.gis.db import models
from datetime import datetime
from django.utils.timezone import utc
from django.conf import settings
from django.db.models import Q#, Count
from django.contrib.auth import get_user_model

import json, urllib, pycurl, decimal  


BATCH_SIZE = 5000
class Dataset(models.Model):
	name = models.CharField(max_length=200)
	url = models.URLField(blank=True,max_length=300)
	cached = models.DateTimeField(null=True,blank=True)
	cache_max_age = models.IntegerField('age when cache should be replaced in days',default=1)
	#field names
	remote_id_field = models.CharField('column name of key field on the remote server',blank=True, max_length=50, default='id')
	name_field = models.CharField(max_length=50,default='name')
	lat_field = models.CharField(max_length=50,default='latitude')
	lon_field = models.CharField(max_length=50,default='longitude')
	street_field = models.CharField(max_length=50,default='street')
	city_field = models.CharField(max_length=50,default='city')
	state_field = models.CharField(max_length=50,default='state')
	zipcode_field = models.CharField(max_length=50,default='zip')
	county_field = models.CharField(max_length=50,default='county')
	field1_en = models.CharField(blank=True,max_length=150)
	field1_name = models.CharField(blank=True,max_length=50)
	field2_en = models.CharField(blank=True,max_length=150)
	field2_name = models.CharField(blank=True,max_length=50)
	field3_en = models.CharField(blank=True,max_length=150)
	field3_name = models.CharField(blank=True,max_length=50)
	needs_geocoding = models.BooleanField(default = False)

	def __unicode__(self):  # Python 3: def __str__(self):
		return self.name

	#recurses through structure of fields and the json
	# , = level
	# + = concatenation of 2 or more fields.
	def reach_field(self, json_item, location):
		result = ''
		if len(location) > 1:
			for field in location[0]:
				if field in json_item:
					result += self.reach_field(json_item[field], location[1:]) + ' '
		elif len(location) == 1:
			for field in location[0]:
				if field in json_item:
					result += json_item[field].strip() + ' '<commit_msg>Add field classes for Dataset<commit_after>from django.contrib.gis.db import models
from datetime import datetime
from django.utils.timezone import utc
from django.conf import settings
from django.db.models import Q#, Count
from django.contrib.auth import get_user_model

import json, urllib, pycurl, decimal  
 
class Location(models.Model):
	street_field = models.CharField(max_length=50,default='street')
	city_field = models.CharField(max_length=50,default='city')
	state_field = models.CharField(max_length=50,default='state')
	zipcode_field = models.CharField(max_length=50,default='zip')
	county_field = models.CharField(max_length=50,default='county')

class GeoCoordinate(models.Model):
	lat_field = models.CharField(max_length=50,default='latitude')
	lon_field = models.CharField(max_length=50,default='longitude')
		
BATCH_SIZE = 5000
class Dataset(models.Model):
	name = models.CharField(max_length=200)
	url = models.URLField(blank=True,max_length=300)
	cached = models.DateTimeField(null=True,blank=True)
	cache_max_age = models.IntegerField('age when cache should be replaced in days',default=1)
	#field names
	remote_id_field = models.CharField('column name of key field on the remote server',blank=True, max_length=50, default='id')
	name_field = models.CharField(max_length=50,default='name')	
	field1_en = models.CharField(blank=True,max_length=150)
	field1_name = models.CharField(blank=True,max_length=50)
	field2_en = models.CharField(blank=True,max_length=150)
	field2_name = models.CharField(blank=True,max_length=50)
	field3_en = models.CharField(blank=True,max_length=150)
	field3_name = models.CharField(blank=True,max_length=50)
	needs_geocoding = models.BooleanField(default = False)

	location = models.ForeignKey(Location)
	geo_coordinate = models.ForeignKey(GeoCoordinate)

	def __unicode__(self):  # Python 3: def __str__(self):
		return self.name

	#recurses through structure of fields and the json
	# , = level
	# + = concatenation of 2 or more fields.
	def reach_field(self, json_item, location):
		result = ''
		if len(location) > 1:
			for field in location[0]:
				if field in json_item:
					result += self.reach_field(json_item[field], location[1:]) + ' '
		elif len(location) == 1:
			for field in location[0]:
				if field in json_item:
					result += json_item[field].strip() + ' '
{'fraction_non_alphanumeric': 0.11533303531515422, 'fraction_numerical': 0.02548055431381314, 'mean_word_length': 6.372322899505766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3172882', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1472, 'n_words': 310}