<commit_before>import logging

import urllib
import httplib2
import simplejson
import yaml
import json
import random
import time

from datetime import datetime
from datetime import timedelta

import config
from webapp2_extras.appengine.auth.models import User
from google.appengine.ext import ndb

from lib.utils import generate_token
from utter_apiobjects import schemes
from utter_apiobjects.model_mixin import ModelSchemaMixin


# user model - extends webapp2 User model
class User(User):
	uid = ndb.StringProperty()
	username = ndb.StringProperty()
	email = ndb.StringProperty()
	name = ndb.StringProperty()
	timezone = ndb.StringProperty()
	country = ndb.StringProperty()
	company = ndb.StringProperty()
	blogger = ndb.BooleanProperty(default=False)
	activated = ndb.BooleanProperty(default=False)
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	last_login = ndb.DateTimeProperty()
	tfsecret = ndb.StringProperty()
	tfenabled = ndb.BooleanProperty(default=False)

	@classmethod
	def get_by_email(cls, email):
		return cls.query(cls.email == email).get()

	@classmethod
	def get_by_uid(cls, uid):
		return cls.query(cls.uid == uid).get()

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.created).fetch()


# appliance group model
class Group(ndb.Model):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	owner = ndb.KeyProperty(kind=User)

	@classmethod
	def get_by_name(cls, name):
		return cls.query(cls.name == name).get()

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.created).fetch()

	@classmethod
	def get_by_owner(cls, owner):
		return cls.query(cls.owner == owner).order(-cls.created).fetch()


# class for group membership
class GroupMembers(ndb.Model):
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	group = ndb.KeyProperty(kind=Group)    
	email = ndb.StringProperty()
	member = ndb.KeyProperty(kind=User)
	invitor = ndb.KeyProperty(kind=User)
	token = ndb.StringProperty()
	active = ndb.BooleanProperty(default=False)

	@classmethod
	def invite(cls, email, group, invitor):
		# do we have this combo already?
		entry = cls.query().filter(cls.email == email, cls.group == group).get()
		
		if not entry:
			# generate new token and create new entry 
			token = "%s" % generate_token(size=16, caselimit=True)
			entry = GroupMembers(
				group = group,
				email = email,
				invitor = invitor,
				token = token,
				active = False
			)
			entry.put()

		return entry

	@classmethod
	def get_group_user_count(cls, group):
		return cls.query().filter(cls.group == group, cls.active == True).count()

	@classmethod
	def get_by_token(cls, token):
		return cls.query().filter(cls.token == token).fetch()

	@classmethod
	def get_by_userid_groupid(cls, user, group):
		return cls.query().filter(cls.member == user, cls.group == group).get()

	@classmethod
	def get_new_owner(cls, user, group):
		return cls.query().filter(cls.group == group, cls.member != user).get()

	@classmethod
	def get_user_groups(cls, user):
		groups = []
		entries = cls.query(cls.member == user).fetch()
		for entry in entries:
			group = Group.get_by_id(entry.group.id())
			groups.append(group)
		return groups

	@classmethod
	def get_group_users(cls, group):
		users = []
		entries = cls.query().filter(cls.group == group, cls.active == True).fetch()
		for entry in entries:
			stuff = entry.member
			user = User.get_by_id(entry.member.id())
			users.append(user)
		return users

	@classmethod
	def is_member(cls, user, group):
		entry = cls.query().filter(cls.member == user, cls.group == group).get()
		if entry:
			return True
		else:
			return False


# appliance model
class Appliance(ndb.Model):
	activated = ndb.BooleanProperty(default=True)
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	token = ndb.StringProperty()
	owner = ndb.KeyProperty(kind=User)
	group = ndb.KeyProperty(kind=Group)
	dynamicimages = ndb.BooleanProperty(default=True)
	location = ndb.GeoPtProperty()
	ipv4enabled = ndb.BooleanProperty(default=False)
	ipv6enabled = ndb.BooleanProperty(default=False)
	ipv4_address = ndb.StringProperty()

	@classmethod
	def get_by_token(cls, token):
		return cls.query(cls.token == token).get()

	@classmethod
	def get_by_user(cls, user):
		return cls.query().filter(cls.owner == user).order(-cls.created).fetch()

	@classmethod
	def get_by_group(cls, group):
		return cls.query().filter(cls.group == group).fetch()

	@classmethod
	def get_by_user_group(cls, user, group):
		return cls.query().filter(cls.owner == user, cls.group == group).fetch()

	@classmethod
	def get_appliance_count_by_user_group(cls, user, group):
		return cls.query().filter(cls.owner == user, cls.group == group).count()

	@classmethod
	def get_geopoints(cls):
		# fetch public appliances
		appliances = cls.query().filter(cls.group == None).fetch()
		
		# geopoint array
		geopoints = []

		# loop through the appliances
		for appliance in appliances:
			geopoints.append({
				"latitude": appliance.location.lat, 
				"longitude": appliance.location.lon, 
				"name": appliance.name,
				"id": appliance.key.id()
			})

		return geopoints

	@classmethod
	def authenticate(cls, apitoken):
		appliance = cls.get_by_token(apitoken)
		if not (appliance and appliance.activated):
			return False
		return True


# image model
class Image(ndb.Model):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	url = ndb.StringProperty()
	size = ndb.IntegerProperty()
	diskformat = ndb.StringProperty()
	containerformat = ndb.StringProperty()
	active = ndb.BooleanProperty(default=False)
	dynamic = ndb.BooleanProperty(default=False)

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(cls.created).fetch()

	@classmethod
	def get_by_name(cls, name):
		image_query = cls.query().filter(cls.name == name)
		image = image_query.get()
		return image


# flavor model
class Flavor(ndb.Model, ModelSchemaMixin):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	vpus = ndb.IntegerProperty()
	memory = ndb.IntegerProperty()
	disk = ndb.IntegerProperty()
	network_up = ndb.IntegerProperty()
	network_down = ndb.IntegerProperty()
	rate = ndb.IntegerProperty() # current market rate
	hot = ndb.IntegerProperty() # number hot
	launches = ndb.IntegerProperty() # number of launches
	active = ndb.BooleanProperty(default=False)
	# specifies where the flavor originated from
	# 0 - originated from pool
	# 1 - originated from an appliance
	# 2 - merge generated in pool
	locality = ndb.IntegerProperty(default=0)
	appliances = ndb.KeyProperty(kind=Appliance, repeated=True)

	# criteria based on which we decide if another flavor is same or not
	comparison_criteria = [
		'vpus',
		'memory',
		'disk',
		'network_up',
		'network_down']

	object_schema = schemes['FlavorSchema']
	object_list_schema = schemes['FlavorListSchema']

	@property
	def flags(self):
		if self.active:
			return 1
		return 8

	# see if another flavor that's equal already exists
	@classmethod
	def find_match(cls, *args, **kwargs):
		qry = cls.query()
		for crit in cls.comparison_criteria:
			qry = qry.filter(getattr(cls, crit) == kwargs[crit])
		return qry.get()

	# used to retreive a flavor by merging it's specs. if the searched specs
	# don't exist yet, it creates a new auto-generated merge-flavor.
	@classmethod
	def get_by_merge(cls, *args, **kwargs):
		criteria = dict((x, kwargs[x]) for x in cls.comparison_criteria)

		# search for flavor that matches the criteria from cls.comparison_criteria
		flavor = cls.find_match(**criteria)
		if not flavor:
			flavor = Flavor(
				# create a name that includes all specs of flavor
				name='Merge flavor ' + ' '.join([
					str(criteria[key]) for key in sorted(criteria.keys())
				]),
				#[x[key] for key in sorted(x.keys())]
				description='This is a flavor that was autogenerated by the flavor-merge',
				locality=2, # merge generated in pool
				hot=2, # because we feel like 2 is a good number
				launches=0,
				active=True,
				rate=kwargs['ask'] # initial rate at first launch
			)
			for key in criteria.keys():
				setattr(flavor, key, criteria[key])
			flavor.put()
		return flavor

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.rate).fetch()

	@classmethod
	def get_all_active(cls):
		return cls.query().filter(cls.active == True).order(-cls.rate).fetch()

	@classmethod
	def get_by_name(cls, name):
		flavor_query = cls.query().filter(cls.name == name)
		flavor = flavor_query.get()
		return flavor


# address model
class Address(ndb.Model):
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	address = ndb.StringProperty()<commit_msg>change rate to manual number<commit_after>import logging

import urllib
import httplib2
import simplejson
import yaml
import json
import random
import time

from datetime import datetime
from datetime import timedelta

import config
from webapp2_extras.appengine.auth.models import User
from google.appengine.ext import ndb

from lib.utils import generate_token
from utter_apiobjects import schemes
from utter_apiobjects.model_mixin import ModelSchemaMixin


# user model - extends webapp2 User model
class User(User):
	uid = ndb.StringProperty()
	username = ndb.StringProperty()
	email = ndb.StringProperty()
	name = ndb.StringProperty()
	timezone = ndb.StringProperty()
	country = ndb.StringProperty()
	company = ndb.StringProperty()
	blogger = ndb.BooleanProperty(default=False)
	activated = ndb.BooleanProperty(default=False)
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	last_login = ndb.DateTimeProperty()
	tfsecret = ndb.StringProperty()
	tfenabled = ndb.BooleanProperty(default=False)

	@classmethod
	def get_by_email(cls, email):
		return cls.query(cls.email == email).get()

	@classmethod
	def get_by_uid(cls, uid):
		return cls.query(cls.uid == uid).get()

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.created).fetch()


# appliance group model
class Group(ndb.Model):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	owner = ndb.KeyProperty(kind=User)

	@classmethod
	def get_by_name(cls, name):
		return cls.query(cls.name == name).get()

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.created).fetch()

	@classmethod
	def get_by_owner(cls, owner):
		return cls.query(cls.owner == owner).order(-cls.created).fetch()


# class for group membership
class GroupMembers(ndb.Model):
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	group = ndb.KeyProperty(kind=Group)    
	email = ndb.StringProperty()
	member = ndb.KeyProperty(kind=User)
	invitor = ndb.KeyProperty(kind=User)
	token = ndb.StringProperty()
	active = ndb.BooleanProperty(default=False)

	@classmethod
	def invite(cls, email, group, invitor):
		# do we have this combo already?
		entry = cls.query().filter(cls.email == email, cls.group == group).get()
		
		if not entry:
			# generate new token and create new entry 
			token = "%s" % generate_token(size=16, caselimit=True)
			entry = GroupMembers(
				group = group,
				email = email,
				invitor = invitor,
				token = token,
				active = False
			)
			entry.put()

		return entry

	@classmethod
	def get_group_user_count(cls, group):
		return cls.query().filter(cls.group == group, cls.active == True).count()

	@classmethod
	def get_by_token(cls, token):
		return cls.query().filter(cls.token == token).fetch()

	@classmethod
	def get_by_userid_groupid(cls, user, group):
		return cls.query().filter(cls.member == user, cls.group == group).get()

	@classmethod
	def get_new_owner(cls, user, group):
		return cls.query().filter(cls.group == group, cls.member != user).get()

	@classmethod
	def get_user_groups(cls, user):
		groups = []
		entries = cls.query(cls.member == user).fetch()
		for entry in entries:
			group = Group.get_by_id(entry.group.id())
			groups.append(group)
		return groups

	@classmethod
	def get_group_users(cls, group):
		users = []
		entries = cls.query().filter(cls.group == group, cls.active == True).fetch()
		for entry in entries:
			stuff = entry.member
			user = User.get_by_id(entry.member.id())
			users.append(user)
		return users

	@classmethod
	def is_member(cls, user, group):
		entry = cls.query().filter(cls.member == user, cls.group == group).get()
		if entry:
			return True
		else:
			return False


# appliance model
class Appliance(ndb.Model):
	activated = ndb.BooleanProperty(default=True)
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	token = ndb.StringProperty()
	owner = ndb.KeyProperty(kind=User)
	group = ndb.KeyProperty(kind=Group)
	dynamicimages = ndb.BooleanProperty(default=True)
	location = ndb.GeoPtProperty()
	ipv4enabled = ndb.BooleanProperty(default=False)
	ipv6enabled = ndb.BooleanProperty(default=False)
	ipv4_address = ndb.StringProperty()

	@classmethod
	def get_by_token(cls, token):
		return cls.query(cls.token == token).get()

	@classmethod
	def get_by_user(cls, user):
		return cls.query().filter(cls.owner == user).order(-cls.created).fetch()

	@classmethod
	def get_by_group(cls, group):
		return cls.query().filter(cls.group == group).fetch()

	@classmethod
	def get_by_user_group(cls, user, group):
		return cls.query().filter(cls.owner == user, cls.group == group).fetch()

	@classmethod
	def get_appliance_count_by_user_group(cls, user, group):
		return cls.query().filter(cls.owner == user, cls.group == group).count()

	@classmethod
	def get_geopoints(cls):
		# fetch public appliances
		appliances = cls.query().filter(cls.group == None).fetch()
		
		# geopoint array
		geopoints = []

		# loop through the appliances
		for appliance in appliances:
			geopoints.append({
				"latitude": appliance.location.lat, 
				"longitude": appliance.location.lon, 
				"name": appliance.name,
				"id": appliance.key.id()
			})

		return geopoints

	@classmethod
	def authenticate(cls, apitoken):
		appliance = cls.get_by_token(apitoken)
		if not (appliance and appliance.activated):
			return False
		return True


# image model
class Image(ndb.Model):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	url = ndb.StringProperty()
	size = ndb.IntegerProperty()
	diskformat = ndb.StringProperty()
	containerformat = ndb.StringProperty()
	active = ndb.BooleanProperty(default=False)
	dynamic = ndb.BooleanProperty(default=False)

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(cls.created).fetch()

	@classmethod
	def get_by_name(cls, name):
		image_query = cls.query().filter(cls.name == name)
		image = image_query.get()
		return image


# flavor model
class Flavor(ndb.Model, ModelSchemaMixin):
	name = ndb.StringProperty()
	description = ndb.StringProperty()
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	vpus = ndb.IntegerProperty()
	memory = ndb.IntegerProperty()
	disk = ndb.IntegerProperty()
	network_up = ndb.IntegerProperty()
	network_down = ndb.IntegerProperty()
	rate = ndb.IntegerProperty() # current market rate
	hot = ndb.IntegerProperty() # number hot
	launches = ndb.IntegerProperty() # number of launches
	active = ndb.BooleanProperty(default=False)
	# specifies where the flavor originated from
	# 0 - originated from pool
	# 1 - originated from an appliance
	# 2 - merge generated in pool
	locality = ndb.IntegerProperty(default=0)
	appliances = ndb.KeyProperty(kind=Appliance, repeated=True)

	# criteria based on which we decide if another flavor is same or not
	comparison_criteria = [
		'vpus',
		'memory',
		'disk',
		'network_up',
		'network_down']

	object_schema = schemes['FlavorSchema']
	object_list_schema = schemes['FlavorListSchema']

	@property
	def flags(self):
		if self.active:
			return 1
		return 8

	# see if another flavor that's equal already exists
	@classmethod
	def find_match(cls, *args, **kwargs):
		qry = cls.query()
		for crit in cls.comparison_criteria:
			qry = qry.filter(getattr(cls, crit) == kwargs[crit])
		return qry.get()

	# used to retreive a flavor by merging it's specs. if the searched specs
	# don't exist yet, it creates a new auto-generated merge-flavor.
	@classmethod
	def get_by_merge(cls, *args, **kwargs):
		criteria = dict((x, kwargs[x]) for x in cls.comparison_criteria)

		# search for flavor that matches the criteria from cls.comparison_criteria
		flavor = cls.find_match(**criteria)
		if not flavor:
			flavor = Flavor(
				# create a name that includes all specs of flavor
				name='Merge flavor ' + ' '.join([
					str(criteria[key]) for key in sorted(criteria.keys())
				]),
				#[x[key] for key in sorted(x.keys())]
				description='This is a flavor that was autogenerated by the flavor-merge',
				locality=2, # merge generated in pool
				hot=2, # because we feel like 2 is a good number
				launches=0,
				active=True,
				rate=0 # it's a manual number for now
			)
			for key in criteria.keys():
				setattr(flavor, key, criteria[key])
			flavor.put()
		return flavor

	@classmethod
	def get_all(cls):
		return cls.query().filter().order(-cls.rate).fetch()

	@classmethod
	def get_all_active(cls):
		return cls.query().filter(cls.active == True).order(-cls.rate).fetch()

	@classmethod
	def get_by_name(cls, name):
		flavor_query = cls.query().filter(cls.name == name)
		flavor = flavor_query.get()
		return flavor


# address model
class Address(ndb.Model):
	created = ndb.DateTimeProperty(auto_now_add=True)
	updated = ndb.DateTimeProperty(auto_now=True)
	address = ndb.StringProperty()
{'fraction_non_alphanumeric': 0.12184988793527579, 'fraction_numerical': 0.0020226316077187998, 'mean_word_length': 4.66728624535316, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1836340', 'n_tokens_mistral': 6525, 'n_tokens_neox': 6081, 'n_words': 1442}