<commit_before>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

// Package datastore retrieves stores data for the ciao controller.
// This package caches most data in memory, and uses a sql
// database as persistent storage.
package datastore

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"net"
	"sort"
	"sync"
	"time"

	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp"
	"github.com/01org/ciao/ssntp/uuid"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

// custom errors
var (
	ErrNoTenant            = errors.New("Tenant not found")
	ErrNoBlockData         = errors.New("Block Device not found")
	ErrNoStorageAttachment = errors.New("No Volume Attached")
)

// Config contains configuration information for the datastore.
type Config struct {
	DBBackend         persistentStore
	PersistentURI     string
	TransientURI      string
	InitWorkloadsPath string
}

type userEventType string

const (
	userInfo  userEventType = "info"
	userError userEventType = "error"
)

type tenant struct {
	types.Tenant
	network   map[int]map[int]bool
	subnets   []int
	instances map[string]*types.Instance
	devices   map[string]types.BlockData
	workloads []types.Workload
}

type node struct {
	types.Node
	instances map[string]*types.Instance
}

type attachment struct {
	instanceID string
	volumeID   string
}

type persistentStore interface {
	init(config Config) error
	disconnect()

	// interfaces related to logging
	logEvent(tenantID string, eventType string, message string) error
	clearLog() error
	getEventLog() (logEntries []*types.LogEntry, err error)

	// interfaces related to workloads
	updateWorkload(wl types.Workload) error
	deleteWorkload(ID string) error

	// interfaces related to tenants
	addTenant(id string, MAC string) (err error)
	getTenant(id string) (t *tenant, err error)
	getTenants() ([]*tenant, error)
	updateTenant(t *tenant) (err error)
	releaseTenantIP(tenantID string, subnetInt int, rest int) (err error)
	claimTenantIP(tenantID string, subnetInt int, rest int) (err error)

	// interfaces related to instances
	getInstances() (instances []*types.Instance, err error)
	addInstance(instance *types.Instance) (err error)
	deleteInstance(instanceID string) (err error)

	// interfaces related to statistics
	addNodeStat(stat payloads.Stat) (err error)
	getNodeSummary() (Summary []*types.NodeSummary, err error)
	addInstanceStats(stats []payloads.InstanceStat, nodeID string) (err error)
	addFrameStat(stat payloads.FrameTrace) (err error)
	getBatchFrameSummary() (stats []types.BatchFrameSummary, err error)
	getBatchFrameStatistics(label string) (stats []types.BatchFrameStat, err error)

	// storage interfaces
	getWorkloadStorage(ID string) ([]types.StorageResource, error)
	getAllBlockData() (map[string]types.BlockData, error)
	addBlockData(data types.BlockData) error
	updateBlockData(data types.BlockData) error
	deleteBlockData(string) error
	getTenantDevices(tenantID string) (map[string]types.BlockData, error)
	addStorageAttachment(a types.StorageAttachment) error
	getAllStorageAttachments() (map[string]types.StorageAttachment, error)
	deleteStorageAttachment(ID string) error

	// external IP interfaces
	addPool(pool types.Pool) error
	updatePool(pool types.Pool) error
	getAllPools() map[string]types.Pool
	deletePool(ID string) error

	addMappedIP(m types.MappedIP) error
	deleteMappedIP(ID string) error
	getMappedIPs() map[string]types.MappedIP

	// quotas
	updateQuotas(tenantID string, qds []types.QuotaDetails) error
	getQuotas(tenantID string) ([]types.QuotaDetails, error)
}

// Datastore provides context for the datastore package.
type Datastore struct {
	db persistentStore

	cnciAddedChans map[string]chan bool
	cnciAddedLock  *sync.Mutex

	nodeLastStat     map[string]types.CiaoNode
	nodeLastStatLock *sync.RWMutex

	instanceLastStat     map[string]types.CiaoServerStats
	instanceLastStatLock *sync.RWMutex

	tenants     map[string]*tenant
	tenantsLock *sync.RWMutex

	cnciWorkload types.Workload

	nodes     map[string]*node
	nodesLock *sync.RWMutex

	instances     map[string]*types.Instance
	instancesLock *sync.RWMutex

	tenantUsage     map[string][]types.CiaoUsage
	tenantUsageLock *sync.RWMutex

	blockDevices map[string]types.BlockData
	bdLock       *sync.RWMutex

	attachments     map[string]types.StorageAttachment
	instanceVolumes map[attachment]string
	attachLock      *sync.RWMutex
	// maybe add a map[instanceid][]types.StorageAttachment
	// to make retrieval of volumes faster.

	pools           map[string]types.Pool
	externalSubnets map[string]bool
	externalIPs     map[string]bool
	mappedIPs       map[string]types.MappedIP
	poolsLock       *sync.RWMutex
}

func (ds *Datastore) initExternalIPs() {
	ds.poolsLock = &sync.RWMutex{}
	ds.externalSubnets = make(map[string]bool)
	ds.externalIPs = make(map[string]bool)

	ds.pools = ds.db.getAllPools()

	for _, pool := range ds.pools {
		for _, subnet := range pool.Subnets {
			ds.externalSubnets[subnet.CIDR] = true
		}

		for _, IP := range pool.IPs {
			ds.externalIPs[IP.Address] = true
		}
	}

	ds.mappedIPs = ds.db.getMappedIPs()
}

// Init initializes the private data for the Datastore object.
// The sql tables are populated with initial data from csv
// files if this is the first time the database has been
// created.  The datastore caches are also filled.
func (ds *Datastore) Init(config Config) error {
	ps := config.DBBackend

	if ps == nil {
		ps = &sqliteDB{}
	}

	err := ps.init(config)
	if err != nil {
		return errors.Wrap(err, "error initialising persistent store")
	}

	ds.db = ps

	ds.cnciAddedChans = make(map[string]chan bool)
	ds.cnciAddedLock = &sync.Mutex{}

	ds.nodeLastStat = make(map[string]types.CiaoNode)
	ds.nodeLastStatLock = &sync.RWMutex{}

	ds.instanceLastStat = make(map[string]types.CiaoServerStats)
	ds.instanceLastStatLock = &sync.RWMutex{}

	// warning, do not use the tenant cache to get
	// networking information right now.  that is not
	// updated, just the resources
	ds.tenants = make(map[string]*tenant)
	ds.tenantsLock = &sync.RWMutex{}

	// cache all our instances prior to getting tenants
	ds.instancesLock = &sync.RWMutex{}
	ds.instances = make(map[string]*types.Instance)

	instances, err := ds.db.getInstances()
	if err != nil {
		return errors.Wrap(err, "error getting instances from database")
	}

	for i := range instances {
		ds.instances[instances[i].ID] = instances[i]
	}

	// cache our current tenants into a map that we can
	// quickly index
	tenants, err := ds.db.getTenants()
	if err != nil {
		return errors.Wrap(err, "error getting tenants from database")
	}
	for i := range tenants {
		ds.tenants[tenants[i].ID] = tenants[i]
	}

	ds.nodesLock = &sync.RWMutex{}
	ds.nodes = make(map[string]*node)

	for key, i := range ds.instances {
		_, ok := ds.nodes[i.NodeID]
		if !ok {
			newNode := types.Node{
				ID: i.NodeID,
			}
			n := &node{
				Node:      newNode,
				instances: make(map[string]*types.Instance),
			}
			ds.nodes[i.NodeID] = n
		}
		ds.nodes[i.NodeID].instances[key] = i

		// ds.tenants.instances should point to the same
		// instances that we have in ds.instances, otherwise they
		// will not get updated when we get new stats.

		tenant := ds.tenants[i.TenantID]
		if tenant != nil {
			tenant.instances[i.ID] = i
		}
	}

	ds.tenantUsage = make(map[string][]types.CiaoUsage)
	ds.tenantUsageLock = &sync.RWMutex{}

	ds.blockDevices, err = ds.db.getAllBlockData()
	if err != nil {
		return errors.Wrap(err, "error getting block devices from database")
	}

	ds.bdLock = &sync.RWMutex{}

	ds.attachments, err = ds.db.getAllStorageAttachments()
	if err != nil {
		return errors.Wrap(err, "error getting storage attachments from database")
	}

	ds.instanceVolumes = make(map[attachment]string)

	for key, value := range ds.attachments {
		link := attachment{
			instanceID: value.InstanceID,
			volumeID:   value.BlockID,
		}

		ds.instanceVolumes[link] = key

		instance := ds.instances[value.InstanceID]
		if instance != nil {
			instance.Attachments = append(instance.Attachments, value)
		}
	}

	ds.attachLock = &sync.RWMutex{}

	ds.initExternalIPs()

	return nil
}

// Exit will disconnect the backing database.
func (ds *Datastore) Exit() {
	ds.db.disconnect()
}

// AddTenantChan allows a caller to pass in a channel for CNCI Launch status.
// When a CNCI has been added to the datastore and a channel exists,
// success will be indicated on the channel.  If a CNCI failure occurred
// and a channel exists, failure will be indicated on the channel.
func (ds *Datastore) AddTenantChan(c chan bool, tenantID string) {
	ds.cnciAddedLock.Lock()
	ds.cnciAddedChans[tenantID] = c
	ds.cnciAddedLock.Unlock()
}

func newHardwareAddr() (net.HardwareAddr, error) {
	buf := make([]byte, 6)
	_, err := rand.Read(buf)
	if err != nil {
		return nil, errors.Wrap(err, "error reading random data")
	}

	// vnic creation seems to require not just the
	// bit 1 to be set, but the entire byte to be
	// set to 2.  Also, ensure that we get no
	// overlap with tenant mac addresses by not allowing
	// byte 1 to ever be zero.
	buf[0] = 2
	if buf[1] == 0 {
		buf[1] = 3
	}

	hw := net.HardwareAddr(buf)

	return hw, nil
}

// AddTenant stores information about a tenant into the datastore.
// it creates a MAC address for the tenant network and makes sure
// that this new tenant is cached.
func (ds *Datastore) AddTenant(id string) (*types.Tenant, error) {
	hw, err := newHardwareAddr()
	if err != nil {
		return nil, errors.Wrap(err, "error creating MAC address")
	}

	err = ds.db.addTenant(id, hw.String())
	if err != nil {
		return nil, errors.Wrapf(err, "error adding tenant (%v) to database", id)
	}

	t, err := ds.getTenant(id)
	if err != nil || t == nil {
		return nil, err
	}

	ds.tenantsLock.Lock()
	ds.tenants[id] = t
	ds.tenantsLock.Unlock()

	return &t.Tenant, nil
}

func (ds *Datastore) getTenant(id string) (*tenant, error) {
	// check cache first
	ds.tenantsLock.RLock()
	t := ds.tenants[id]
	ds.tenantsLock.RUnlock()

	if t != nil {
		return t, nil
	}

	t, err := ds.db.getTenant(id)
	return t, errors.Wrapf(err, "error getting tenant (%v) from database", id)
}

// GetTenant returns details about a tenant referenced by the uuid
func (ds *Datastore) GetTenant(id string) (*types.Tenant, error) {
	t, err := ds.getTenant(id)
	if err != nil || t == nil {
		return nil, err
	}

	return &t.Tenant, nil
}

// AddWorkload is used to add a new workload to the datastore.
// Both cache and persistent store are updated.
func (ds *Datastore) AddWorkload(w types.Workload) error {
	ds.tenantsLock.Lock()
	defer ds.tenantsLock.Unlock()

	tenant, ok := ds.tenants[w.TenantID]
	if !ok {
		return ErrNoTenant
	}

	err := ds.db.updateWorkload(w)
	if err != nil {
		return errors.Wrapf(err, "error updating workload (%v) in database", w.ID)
	}

	// cache it.
	ds.tenants[w.TenantID].workloads = append(tenant.workloads, w)

	return nil
}

// DeleteWorkload will delete an unused workload from the datastore.
// workload ID out of the datastore.
func (ds *Datastore) DeleteWorkload(tenantID string, workloadID string) error {
	// make sure that this workload is not in use.
	// always get from cache
	ds.instancesLock.RLock()
	defer ds.instancesLock.RUnlock()

	if len(ds.instances) > 0 {
		for _, val := range ds.instances {
			if val.WorkloadID == workloadID {
				// we can't go on.
				return types.ErrWorkloadInUse
			}
		}
	}

	// workload is not being used, find it so we can delete it.
	ds.tenantsLock.Lock()
	defer ds.tenantsLock.Unlock()

	t, ok := ds.tenants[tenantID]
	if !ok {
		return types.ErrTenantNotFound
	}

	for i, wl := range t.workloads {
		if wl.ID == workloadID {
			// delete from persistent datastore.
			err := ds.db.deleteWorkload(workloadID)
			if err != nil {
				return errors.Wrapf(err, "error deleting workload %v from database", wl.ID)
			}

			// delete from cache.
			ds.tenants[tenantID].workloads = append(ds.tenants[tenantID].workloads[:i], ds.tenants[tenantID].workloads[i+1:]...)
			return nil
		}
	}

	return types.ErrWorkloadNotFound
}

// GetWorkload returns details about a specific workload referenced by id
func (ds *Datastore) GetWorkload(tenantID string, ID string) (types.Workload, error) {
	if ID == ds.cnciWorkload.ID {
		return ds.cnciWorkload, nil
	}

	ds.tenantsLock.RLock()
	defer ds.tenantsLock.RUnlock()

	// get any public workloads. These are part of our
	// dummy tenant "public".
	public, ok := ds.tenants["public"]
	if ok {
		for _, wl := range public.workloads {
			if wl.ID == ID {
				return wl, nil
			}
		}
	}

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		return types.Workload{}, ErrNoTenant
	}

	for _, wl := range tenant.workloads {
		if wl.ID == ID {
			return wl, nil
		}
	}

	return types.Workload{}, types.ErrWorkloadNotFound
}

// GetWorkloads retrieves the list of workloads for a particular tenant.
// if there are any public workloads, they will be included in the returned list.
func (ds *Datastore) GetWorkloads(tenantID string) ([]types.Workload, error) {
	var workloads []types.Workload

	// check the cache first
	ds.tenantsLock.RLock()
	defer ds.tenantsLock.RUnlock()

	// get any public workloads. These are part of our
	// dummy tenant "public".
	public, ok := ds.tenants["public"]
	if ok {
		for _, wl := range public.workloads {
			workloads = append(workloads, wl)
		}
	}

	// if there isn't a tenant here, it isn't necessarily an
	// error.
	tenant, ok := ds.tenants[tenantID]
	if !ok {
		return workloads, nil
	}

	workloads = append(workloads, tenant.workloads...)

	return workloads, nil
}

// AddCNCIIP will associate a new IP address with an existing CNCI
// via the mac address
func (ds *Datastore) AddCNCIIP(cnciMAC string, ip string) error {
	var ok bool
	var tenantID string
	var tenant *tenant

	ds.tenantsLock.Lock()

	for tenantID, tenant = range ds.tenants {
		if tenant.CNCIMAC == cnciMAC {
			ok = true
			break
		}
	}

	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIIP = ip

	ds.tenantsLock.Unlock()

	err := ds.db.updateTenant(tenant)

	ds.cnciAddedLock.Lock()

	c, ok := ds.cnciAddedChans[tenantID]
	if ok {
		delete(ds.cnciAddedChans, tenantID)
	}

	ds.cnciAddedLock.Unlock()

	if c != nil {
		c <- true
	}

	return errors.Wrap(err, "error updating tenant in database")
}

// AddTenantCNCI will associate a new CNCI instance with a specific tenant.
// The instanceID of the new CNCI instance and the MAC address of the new instance
// are stored in the sql database and updated in the cache.
func (ds *Datastore) AddTenantCNCI(tenantID string, instanceID string, mac string) error {
	// update tenants cache
	ds.tenantsLock.Lock()

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIID = instanceID
	tenant.CNCIMAC = mac

	ds.tenantsLock.Unlock()

	return ds.db.updateTenant(tenant)
}

func (ds *Datastore) removeTenantCNCI(tenantID string) error {
	// update tenants cache
	ds.tenantsLock.Lock()

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIID = ""
	tenant.CNCIIP = ""

	ds.tenantsLock.Unlock()

	return errors.Wrap(ds.db.updateTenant(tenant), "error updating tenant in database")
}

// GetAllTenants returns all the tenants from the datastore.
func (ds *Datastore) GetAllTenants() ([]*types.Tenant, error) {
	var tenants []*types.Tenant

	for _, t := range ds.tenants {
		tenants = append(tenants, &t.Tenant)
	}

	return tenants, nil
}

// ReleaseTenantIP will return an IP address previously allocated to the pool.
// Once a tenant IP address is released, it can be reassigned to another
// instance.
func (ds *Datastore) ReleaseTenantIP(tenantID string, ip string) error {
	ipAddr := net.ParseIP(ip)
	if ipAddr == nil {
		return errors.New("Invalid IPv4 Address")
	}

	ipBytes := ipAddr.To4()
	if ipBytes == nil {
		return errors.New("Unable to convert ip to bytes")
	}

	subnetInt := binary.BigEndian.Uint16(ipBytes[1:3])

	// clear from cache
	ds.tenantsLock.Lock()

	if ds.tenants[tenantID] != nil {
		ds.tenants[tenantID].network[int(subnetInt)][int(ipBytes[3])] = false
	}

	ds.tenantsLock.Unlock()

	return ds.db.releaseTenantIP(tenantID, int(subnetInt), int(ipBytes[3]))
}

// AllocateTenantIP will find a free IP address within a tenant network.
// For now we make each tenant have unique subnets even though it
// isn't actually needed because of a docker issue.
func (ds *Datastore) AllocateTenantIP(tenantID string) (net.IP, error) {
	var subnetInt uint16
	subnetInt = 0

	ds.tenantsLock.Lock()

	network := ds.tenants[tenantID].network
	subnets := ds.tenants[tenantID].subnets

	// find any subnet assigned to this tenant with available addresses
	sort.Ints(subnets)

	for _, k := range subnets {
		if len(network[k]) < 253 {
			subnetInt = uint16(k)
		}
	}

	var subnetBytes = []byte{16, 0}

	if subnetInt == 0 {
		i := binary.BigEndian.Uint16(subnetBytes)

		for {
			// check for new subnet.
			_, ok := network[int(i)]
			if !ok {
				sub := make(map[int]bool)
				network[int(i)] = sub

				break
			}

			if subnetBytes[1] == 255 {
				if subnetBytes[0] == 31 {
					// out of possible subnets
					glog.Warning("Out of Subnets")
					ds.tenantsLock.Unlock()
					return nil, errors.New("Out of subnets")
				}
				subnetBytes[0]++
				subnetBytes[1] = 0
			} else {
				subnetBytes[1]++
			}

			i = binary.BigEndian.Uint16(subnetBytes)
		}

		subnetInt = i

		ds.tenants[tenantID].subnets = append(subnets, int(subnetInt))
	} else {
		binary.BigEndian.PutUint16(subnetBytes, subnetInt)
	}

	hosts := network[int(subnetInt)]

	rest := 2

	for {
		if hosts[rest] == false {
			hosts[rest] = true
			break
		}

		if rest == 255 {
			// this should never happen
			glog.Warning("ran out of host numbers")

			ds.tenantsLock.Unlock()

			return nil, errors.New("rand out of host numbers")
		}

		rest++
	}

	ds.tenantsLock.Unlock()

	err := ds.db.claimTenantIP(tenantID, int(subnetInt), rest)
	if err != nil {
		return nil, errors.Wrap(err, "Error claiming tenant IP in database")
	}

	// convert to IP type.
	next := net.IPv4(172, subnetBytes[0], subnetBytes[1], byte(rest))

	return next, nil
}

// GetAllInstances retrieves all instances out of the datastore.
func (ds *Datastore) GetAllInstances() ([]*types.Instance, error) {
	var instances []*types.Instance

	// always get from cache
	ds.instancesLock.RLock()

	if len(ds.instances) > 0 {
		for _, val := range ds.instances {
			instances = append(instances, val)
		}
	}

	ds.instancesLock.RUnlock()

	return instances, nil
}

// GetInstance retrieves an instance out of the datastore.
func (ds *Datastore) GetInstance(id string) (*types.Instance, error) {
	// always get from cache
	ds.instancesLock.RLock()

	value, ok := ds.instances[id]

	ds.instancesLock.RUnlock()

	if !ok {
		return nil, types.ErrInstanceNotFound
	}

	return value, nil
}

// GetTenantInstance retrieves a tenant instance out of the datastore.
func (ds *Datastore) GetTenantInstance(tenantID string, instanceID string) (*types.Instance, error) {
	// always get from cache
	ds.instancesLock.RLock()

	value, ok := ds.instances[instanceID]

	ds.instancesLock.RUnlock()

	if !ok || value.TenantID != tenantID {
		return nil, types.ErrInstanceNotFound
	}

	return value, nil
}

// GetAllInstancesFromTenant will retrieve all instances belonging to a specific tenant
func (ds *Datastore) GetAllInstancesFromTenant(tenantID string) ([]*types.Instance, error) {
	var instances []*types.Instance

	ds.tenantsLock.RLock()

	t, ok := ds.tenants[tenantID]
	if ok {
		for _, val := range t.instances {
			instances = append(instances, val)
		}

		ds.tenantsLock.RUnlock()

		return instances, nil
	}

	ds.tenantsLock.RUnlock()

	return nil, nil
}

// GetAllInstancesByNode will retrieve all the instances running on a specific compute Node.
func (ds *Datastore) GetAllInstancesByNode(nodeID string) ([]*types.Instance, error) {
	var instances []*types.Instance

	ds.nodesLock.RLock()

	n, ok := ds.nodes[nodeID]
	if ok {
		for _, val := range n.instances {
			instances = append(instances, val)
		}
	}

	ds.nodesLock.RUnlock()

	return instances, nil
}

// AddInstance will store a new instance in the datastore.
// The instance will be updated both in the cache and in the database
func (ds *Datastore) AddInstance(instance *types.Instance) error {
	err := ds.db.addInstance(instance)

	if err != nil {
		return errors.Wrap(err, "Error adding instance to database")
	}

	// add to cache
	ds.instancesLock.Lock()

	ds.instances[instance.ID] = instance

	instanceStat := types.CiaoServerStats{
		ID:        instance.ID,
		TenantID:  instance.TenantID,
		NodeID:    instance.NodeID,
		Timestamp: time.Now(),
		Status:    instance.State,
	}

	ds.instanceLastStatLock.Lock()
	ds.instanceLastStat[instance.ID] = instanceStat
	ds.instanceLastStatLock.Unlock()

	ds.instancesLock.Unlock()

	ds.tenantsLock.Lock()
	tenant := ds.tenants[instance.TenantID]
	if tenant != nil {
		tenant.instances[instance.ID] = instance
	}
	ds.tenantsLock.Unlock()

	return nil
}

// RestartFailure logs a RestartFailure in the datastore
func (ds *Datastore) RestartFailure(instanceID string, reason payloads.RestartFailureReason) error {
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Restart Failure %s: %s", instanceID, reason.String())
	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// StopFailure logs a StopFailure in the datastore
func (ds *Datastore) StopFailure(instanceID string, reason payloads.StopFailureReason) error {
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Stop Failure %s: %s", instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// StartFailure will clean up after a failure to start an instance.
// If an instance was a CNCI, this function will remove the CNCI instance
// for this tenant. If the instance was a normal tenant instance, the
// IP address will be released and the instance will be deleted from the
// datastore.
//
// Only instances whose status is pending are removed when a StartFailure event
// is received.  StartFailure errors may also be generated when restarting an
// exited instance and we want to make sure that a failure to restart such
// an instance does not result in it being deleted.
func (ds *Datastore) StartFailure(instanceID string, reason payloads.StartFailureReason, migration bool) error {
	var tenantID string
	var cnci bool

	ds.tenantsLock.RLock()

	for key, t := range ds.tenants {
		if t.CNCIID == instanceID {
			cnci = true
			tenantID = key
			break
		}
	}

	ds.tenantsLock.RUnlock()

	if cnci == true {
		glog.Warning("CNCI ", instanceID, " Failed to start")

		err := ds.removeTenantCNCI(tenantID)

		msg := fmt.Sprintf("CNCI Start Failure %s: %s", instanceID, reason.String())
		ds.db.logEvent(tenantID, string(userError), msg)

		ds.cnciAddedLock.Lock()

		c, ok := ds.cnciAddedChans[tenantID]
		if ok {
			delete(ds.cnciAddedChans, tenantID)
		}

		ds.cnciAddedLock.Unlock()

		if c != nil {
			c <- false
		}

		return errors.Wrap(err, "error removing CNCI for tenant")
	}

	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	if reason.IsFatal() && !migration {
		ds.deleteInstance(instanceID)
	}

	msg := fmt.Sprintf("Start Failure %s: %s", instanceID, reason.String())
	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// AttachVolumeFailure will clean up after a failure to attach a volume.
// The volume state will be changed back to available, and an error message
// will be logged.
func (ds *Datastore) AttachVolumeFailure(instanceID string, volumeID string, reason payloads.AttachVolumeFailureReason) error {
	// update the block data to reflect correct state
	data, err := ds.GetBlockDevice(volumeID)
	if err != nil {
		return errors.Wrapf(err, "error getting block device for volume (%v)", volumeID)
	}

	oldState := data.State
	data.State = types.Available
	err = ds.UpdateBlockDevice(data)
	if err != nil {
		data.State = oldState
		return errors.Wrapf(err, "error updating block device for volume (%v)", volumeID)
	}

	// get owner of this instance
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Attach Volume Failure %s to %s: %s", volumeID, instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// DetachVolumeFailure will clean up after a failure to detach a volume.
// The volume state will be changed back to available, and an error message
// will be logged.
func (ds *Datastore) DetachVolumeFailure(instanceID string, volumeID string, reason payloads.DetachVolumeFailureReason) error {
	// update the block data to reflect correct state
	data, err := ds.GetBlockDevice(volumeID)
	if err != nil {
		return errors.Wrapf(err, "error getting block device for volume (%v)", volumeID)
	}

	// because controller wouldn't allow a detach if state
	// wasn't initially InUse, we can blindly set this back
	// to InUse.
	oldState := data.State
	data.State = types.InUse
	err = ds.UpdateBlockDevice(data)
	if err != nil {
		data.State = oldState
		return errors.Wrapf(err, "error updating block device for volume (%v)", volumeID)
	}

	// get owner of this instance
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Detach Volume Failure %s from %s: %s", volumeID, instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)
	return nil
}

func (ds *Datastore) deleteInstance(instanceID string) (string, error) {
	ds.instanceLastStatLock.Lock()
	delete(ds.instanceLastStat, instanceID)
	ds.instanceLastStatLock.Unlock()

	ds.instancesLock.Lock()
	i := ds.instances[instanceID]
	delete(ds.instances, instanceID)
	ds.instancesLock.Unlock()

	ds.tenantsLock.Lock()
	tenant := ds.tenants[i.TenantID]
	if tenant != nil {
		delete(tenant.instances, instanceID)
	}
	ds.tenantsLock.Unlock()

	// we may not have received any node stats for this instance
	if i.NodeID != "" {
		ds.nodesLock.Lock()
		delete(ds.nodes[i.NodeID].instances, instanceID)
		ds.nodesLock.Unlock()
	}

	var err error
	if tmpErr := ds.db.deleteInstance(i.ID); tmpErr != nil {
		glog.Warningf("error deleting instance (%v): %v", i.ID, err)
		err = errors.Wrapf(tmpErr, "error deleting instance from database (%v)", i.ID)
	}

	if tmpErr := ds.ReleaseTenantIP(i.TenantID, i.IPAddress); tmpErr != nil {
		glog.Warningf("error releasing IP for instance (%v): %v", i.ID, err)
		if err == nil {
			err = errors.Wrapf(tmpErr, "error releasing IP for instance (%v)", i.ID)
		}
	}

	ds.updateStorageAttachments(instanceID, nil)<commit_msg>datastore: Remove instance from database before updating cache<commit_after>/*
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

// Package datastore retrieves stores data for the ciao controller.
// This package caches most data in memory, and uses a sql
// database as persistent storage.
package datastore

import (
	"crypto/rand"
	"encoding/binary"
	"fmt"
	"net"
	"sort"
	"sync"
	"time"

	"github.com/01org/ciao/ciao-controller/types"
	"github.com/01org/ciao/payloads"
	"github.com/01org/ciao/ssntp"
	"github.com/01org/ciao/ssntp/uuid"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

// custom errors
var (
	ErrNoTenant            = errors.New("Tenant not found")
	ErrNoBlockData         = errors.New("Block Device not found")
	ErrNoStorageAttachment = errors.New("No Volume Attached")
)

// Config contains configuration information for the datastore.
type Config struct {
	DBBackend         persistentStore
	PersistentURI     string
	TransientURI      string
	InitWorkloadsPath string
}

type userEventType string

const (
	userInfo  userEventType = "info"
	userError userEventType = "error"
)

type tenant struct {
	types.Tenant
	network   map[int]map[int]bool
	subnets   []int
	instances map[string]*types.Instance
	devices   map[string]types.BlockData
	workloads []types.Workload
}

type node struct {
	types.Node
	instances map[string]*types.Instance
}

type attachment struct {
	instanceID string
	volumeID   string
}

type persistentStore interface {
	init(config Config) error
	disconnect()

	// interfaces related to logging
	logEvent(tenantID string, eventType string, message string) error
	clearLog() error
	getEventLog() (logEntries []*types.LogEntry, err error)

	// interfaces related to workloads
	updateWorkload(wl types.Workload) error
	deleteWorkload(ID string) error

	// interfaces related to tenants
	addTenant(id string, MAC string) (err error)
	getTenant(id string) (t *tenant, err error)
	getTenants() ([]*tenant, error)
	updateTenant(t *tenant) (err error)
	releaseTenantIP(tenantID string, subnetInt int, rest int) (err error)
	claimTenantIP(tenantID string, subnetInt int, rest int) (err error)

	// interfaces related to instances
	getInstances() (instances []*types.Instance, err error)
	addInstance(instance *types.Instance) (err error)
	deleteInstance(instanceID string) (err error)

	// interfaces related to statistics
	addNodeStat(stat payloads.Stat) (err error)
	getNodeSummary() (Summary []*types.NodeSummary, err error)
	addInstanceStats(stats []payloads.InstanceStat, nodeID string) (err error)
	addFrameStat(stat payloads.FrameTrace) (err error)
	getBatchFrameSummary() (stats []types.BatchFrameSummary, err error)
	getBatchFrameStatistics(label string) (stats []types.BatchFrameStat, err error)

	// storage interfaces
	getWorkloadStorage(ID string) ([]types.StorageResource, error)
	getAllBlockData() (map[string]types.BlockData, error)
	addBlockData(data types.BlockData) error
	updateBlockData(data types.BlockData) error
	deleteBlockData(string) error
	getTenantDevices(tenantID string) (map[string]types.BlockData, error)
	addStorageAttachment(a types.StorageAttachment) error
	getAllStorageAttachments() (map[string]types.StorageAttachment, error)
	deleteStorageAttachment(ID string) error

	// external IP interfaces
	addPool(pool types.Pool) error
	updatePool(pool types.Pool) error
	getAllPools() map[string]types.Pool
	deletePool(ID string) error

	addMappedIP(m types.MappedIP) error
	deleteMappedIP(ID string) error
	getMappedIPs() map[string]types.MappedIP

	// quotas
	updateQuotas(tenantID string, qds []types.QuotaDetails) error
	getQuotas(tenantID string) ([]types.QuotaDetails, error)
}

// Datastore provides context for the datastore package.
type Datastore struct {
	db persistentStore

	cnciAddedChans map[string]chan bool
	cnciAddedLock  *sync.Mutex

	nodeLastStat     map[string]types.CiaoNode
	nodeLastStatLock *sync.RWMutex

	instanceLastStat     map[string]types.CiaoServerStats
	instanceLastStatLock *sync.RWMutex

	tenants     map[string]*tenant
	tenantsLock *sync.RWMutex

	cnciWorkload types.Workload

	nodes     map[string]*node
	nodesLock *sync.RWMutex

	instances     map[string]*types.Instance
	instancesLock *sync.RWMutex

	tenantUsage     map[string][]types.CiaoUsage
	tenantUsageLock *sync.RWMutex

	blockDevices map[string]types.BlockData
	bdLock       *sync.RWMutex

	attachments     map[string]types.StorageAttachment
	instanceVolumes map[attachment]string
	attachLock      *sync.RWMutex
	// maybe add a map[instanceid][]types.StorageAttachment
	// to make retrieval of volumes faster.

	pools           map[string]types.Pool
	externalSubnets map[string]bool
	externalIPs     map[string]bool
	mappedIPs       map[string]types.MappedIP
	poolsLock       *sync.RWMutex
}

func (ds *Datastore) initExternalIPs() {
	ds.poolsLock = &sync.RWMutex{}
	ds.externalSubnets = make(map[string]bool)
	ds.externalIPs = make(map[string]bool)

	ds.pools = ds.db.getAllPools()

	for _, pool := range ds.pools {
		for _, subnet := range pool.Subnets {
			ds.externalSubnets[subnet.CIDR] = true
		}

		for _, IP := range pool.IPs {
			ds.externalIPs[IP.Address] = true
		}
	}

	ds.mappedIPs = ds.db.getMappedIPs()
}

// Init initializes the private data for the Datastore object.
// The sql tables are populated with initial data from csv
// files if this is the first time the database has been
// created.  The datastore caches are also filled.
func (ds *Datastore) Init(config Config) error {
	ps := config.DBBackend

	if ps == nil {
		ps = &sqliteDB{}
	}

	err := ps.init(config)
	if err != nil {
		return errors.Wrap(err, "error initialising persistent store")
	}

	ds.db = ps

	ds.cnciAddedChans = make(map[string]chan bool)
	ds.cnciAddedLock = &sync.Mutex{}

	ds.nodeLastStat = make(map[string]types.CiaoNode)
	ds.nodeLastStatLock = &sync.RWMutex{}

	ds.instanceLastStat = make(map[string]types.CiaoServerStats)
	ds.instanceLastStatLock = &sync.RWMutex{}

	// warning, do not use the tenant cache to get
	// networking information right now.  that is not
	// updated, just the resources
	ds.tenants = make(map[string]*tenant)
	ds.tenantsLock = &sync.RWMutex{}

	// cache all our instances prior to getting tenants
	ds.instancesLock = &sync.RWMutex{}
	ds.instances = make(map[string]*types.Instance)

	instances, err := ds.db.getInstances()
	if err != nil {
		return errors.Wrap(err, "error getting instances from database")
	}

	for i := range instances {
		ds.instances[instances[i].ID] = instances[i]
	}

	// cache our current tenants into a map that we can
	// quickly index
	tenants, err := ds.db.getTenants()
	if err != nil {
		return errors.Wrap(err, "error getting tenants from database")
	}
	for i := range tenants {
		ds.tenants[tenants[i].ID] = tenants[i]
	}

	ds.nodesLock = &sync.RWMutex{}
	ds.nodes = make(map[string]*node)

	for key, i := range ds.instances {
		_, ok := ds.nodes[i.NodeID]
		if !ok {
			newNode := types.Node{
				ID: i.NodeID,
			}
			n := &node{
				Node:      newNode,
				instances: make(map[string]*types.Instance),
			}
			ds.nodes[i.NodeID] = n
		}
		ds.nodes[i.NodeID].instances[key] = i

		// ds.tenants.instances should point to the same
		// instances that we have in ds.instances, otherwise they
		// will not get updated when we get new stats.

		tenant := ds.tenants[i.TenantID]
		if tenant != nil {
			tenant.instances[i.ID] = i
		}
	}

	ds.tenantUsage = make(map[string][]types.CiaoUsage)
	ds.tenantUsageLock = &sync.RWMutex{}

	ds.blockDevices, err = ds.db.getAllBlockData()
	if err != nil {
		return errors.Wrap(err, "error getting block devices from database")
	}

	ds.bdLock = &sync.RWMutex{}

	ds.attachments, err = ds.db.getAllStorageAttachments()
	if err != nil {
		return errors.Wrap(err, "error getting storage attachments from database")
	}

	ds.instanceVolumes = make(map[attachment]string)

	for key, value := range ds.attachments {
		link := attachment{
			instanceID: value.InstanceID,
			volumeID:   value.BlockID,
		}

		ds.instanceVolumes[link] = key

		instance := ds.instances[value.InstanceID]
		if instance != nil {
			instance.Attachments = append(instance.Attachments, value)
		}
	}

	ds.attachLock = &sync.RWMutex{}

	ds.initExternalIPs()

	return nil
}

// Exit will disconnect the backing database.
func (ds *Datastore) Exit() {
	ds.db.disconnect()
}

// AddTenantChan allows a caller to pass in a channel for CNCI Launch status.
// When a CNCI has been added to the datastore and a channel exists,
// success will be indicated on the channel.  If a CNCI failure occurred
// and a channel exists, failure will be indicated on the channel.
func (ds *Datastore) AddTenantChan(c chan bool, tenantID string) {
	ds.cnciAddedLock.Lock()
	ds.cnciAddedChans[tenantID] = c
	ds.cnciAddedLock.Unlock()
}

func newHardwareAddr() (net.HardwareAddr, error) {
	buf := make([]byte, 6)
	_, err := rand.Read(buf)
	if err != nil {
		return nil, errors.Wrap(err, "error reading random data")
	}

	// vnic creation seems to require not just the
	// bit 1 to be set, but the entire byte to be
	// set to 2.  Also, ensure that we get no
	// overlap with tenant mac addresses by not allowing
	// byte 1 to ever be zero.
	buf[0] = 2
	if buf[1] == 0 {
		buf[1] = 3
	}

	hw := net.HardwareAddr(buf)

	return hw, nil
}

// AddTenant stores information about a tenant into the datastore.
// it creates a MAC address for the tenant network and makes sure
// that this new tenant is cached.
func (ds *Datastore) AddTenant(id string) (*types.Tenant, error) {
	hw, err := newHardwareAddr()
	if err != nil {
		return nil, errors.Wrap(err, "error creating MAC address")
	}

	err = ds.db.addTenant(id, hw.String())
	if err != nil {
		return nil, errors.Wrapf(err, "error adding tenant (%v) to database", id)
	}

	t, err := ds.getTenant(id)
	if err != nil || t == nil {
		return nil, err
	}

	ds.tenantsLock.Lock()
	ds.tenants[id] = t
	ds.tenantsLock.Unlock()

	return &t.Tenant, nil
}

func (ds *Datastore) getTenant(id string) (*tenant, error) {
	// check cache first
	ds.tenantsLock.RLock()
	t := ds.tenants[id]
	ds.tenantsLock.RUnlock()

	if t != nil {
		return t, nil
	}

	t, err := ds.db.getTenant(id)
	return t, errors.Wrapf(err, "error getting tenant (%v) from database", id)
}

// GetTenant returns details about a tenant referenced by the uuid
func (ds *Datastore) GetTenant(id string) (*types.Tenant, error) {
	t, err := ds.getTenant(id)
	if err != nil || t == nil {
		return nil, err
	}

	return &t.Tenant, nil
}

// AddWorkload is used to add a new workload to the datastore.
// Both cache and persistent store are updated.
func (ds *Datastore) AddWorkload(w types.Workload) error {
	ds.tenantsLock.Lock()
	defer ds.tenantsLock.Unlock()

	tenant, ok := ds.tenants[w.TenantID]
	if !ok {
		return ErrNoTenant
	}

	err := ds.db.updateWorkload(w)
	if err != nil {
		return errors.Wrapf(err, "error updating workload (%v) in database", w.ID)
	}

	// cache it.
	ds.tenants[w.TenantID].workloads = append(tenant.workloads, w)

	return nil
}

// DeleteWorkload will delete an unused workload from the datastore.
// workload ID out of the datastore.
func (ds *Datastore) DeleteWorkload(tenantID string, workloadID string) error {
	// make sure that this workload is not in use.
	// always get from cache
	ds.instancesLock.RLock()
	defer ds.instancesLock.RUnlock()

	if len(ds.instances) > 0 {
		for _, val := range ds.instances {
			if val.WorkloadID == workloadID {
				// we can't go on.
				return types.ErrWorkloadInUse
			}
		}
	}

	// workload is not being used, find it so we can delete it.
	ds.tenantsLock.Lock()
	defer ds.tenantsLock.Unlock()

	t, ok := ds.tenants[tenantID]
	if !ok {
		return types.ErrTenantNotFound
	}

	for i, wl := range t.workloads {
		if wl.ID == workloadID {
			// delete from persistent datastore.
			err := ds.db.deleteWorkload(workloadID)
			if err != nil {
				return errors.Wrapf(err, "error deleting workload %v from database", wl.ID)
			}

			// delete from cache.
			ds.tenants[tenantID].workloads = append(ds.tenants[tenantID].workloads[:i], ds.tenants[tenantID].workloads[i+1:]...)
			return nil
		}
	}

	return types.ErrWorkloadNotFound
}

// GetWorkload returns details about a specific workload referenced by id
func (ds *Datastore) GetWorkload(tenantID string, ID string) (types.Workload, error) {
	if ID == ds.cnciWorkload.ID {
		return ds.cnciWorkload, nil
	}

	ds.tenantsLock.RLock()
	defer ds.tenantsLock.RUnlock()

	// get any public workloads. These are part of our
	// dummy tenant "public".
	public, ok := ds.tenants["public"]
	if ok {
		for _, wl := range public.workloads {
			if wl.ID == ID {
				return wl, nil
			}
		}
	}

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		return types.Workload{}, ErrNoTenant
	}

	for _, wl := range tenant.workloads {
		if wl.ID == ID {
			return wl, nil
		}
	}

	return types.Workload{}, types.ErrWorkloadNotFound
}

// GetWorkloads retrieves the list of workloads for a particular tenant.
// if there are any public workloads, they will be included in the returned list.
func (ds *Datastore) GetWorkloads(tenantID string) ([]types.Workload, error) {
	var workloads []types.Workload

	// check the cache first
	ds.tenantsLock.RLock()
	defer ds.tenantsLock.RUnlock()

	// get any public workloads. These are part of our
	// dummy tenant "public".
	public, ok := ds.tenants["public"]
	if ok {
		for _, wl := range public.workloads {
			workloads = append(workloads, wl)
		}
	}

	// if there isn't a tenant here, it isn't necessarily an
	// error.
	tenant, ok := ds.tenants[tenantID]
	if !ok {
		return workloads, nil
	}

	workloads = append(workloads, tenant.workloads...)

	return workloads, nil
}

// AddCNCIIP will associate a new IP address with an existing CNCI
// via the mac address
func (ds *Datastore) AddCNCIIP(cnciMAC string, ip string) error {
	var ok bool
	var tenantID string
	var tenant *tenant

	ds.tenantsLock.Lock()

	for tenantID, tenant = range ds.tenants {
		if tenant.CNCIMAC == cnciMAC {
			ok = true
			break
		}
	}

	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIIP = ip

	ds.tenantsLock.Unlock()

	err := ds.db.updateTenant(tenant)

	ds.cnciAddedLock.Lock()

	c, ok := ds.cnciAddedChans[tenantID]
	if ok {
		delete(ds.cnciAddedChans, tenantID)
	}

	ds.cnciAddedLock.Unlock()

	if c != nil {
		c <- true
	}

	return errors.Wrap(err, "error updating tenant in database")
}

// AddTenantCNCI will associate a new CNCI instance with a specific tenant.
// The instanceID of the new CNCI instance and the MAC address of the new instance
// are stored in the sql database and updated in the cache.
func (ds *Datastore) AddTenantCNCI(tenantID string, instanceID string, mac string) error {
	// update tenants cache
	ds.tenantsLock.Lock()

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIID = instanceID
	tenant.CNCIMAC = mac

	ds.tenantsLock.Unlock()

	return ds.db.updateTenant(tenant)
}

func (ds *Datastore) removeTenantCNCI(tenantID string) error {
	// update tenants cache
	ds.tenantsLock.Lock()

	tenant, ok := ds.tenants[tenantID]
	if !ok {
		ds.tenantsLock.Unlock()
		return ErrNoTenant
	}

	tenant.CNCIID = ""
	tenant.CNCIIP = ""

	ds.tenantsLock.Unlock()

	return errors.Wrap(ds.db.updateTenant(tenant), "error updating tenant in database")
}

// GetAllTenants returns all the tenants from the datastore.
func (ds *Datastore) GetAllTenants() ([]*types.Tenant, error) {
	var tenants []*types.Tenant

	for _, t := range ds.tenants {
		tenants = append(tenants, &t.Tenant)
	}

	return tenants, nil
}

// ReleaseTenantIP will return an IP address previously allocated to the pool.
// Once a tenant IP address is released, it can be reassigned to another
// instance.
func (ds *Datastore) ReleaseTenantIP(tenantID string, ip string) error {
	ipAddr := net.ParseIP(ip)
	if ipAddr == nil {
		return errors.New("Invalid IPv4 Address")
	}

	ipBytes := ipAddr.To4()
	if ipBytes == nil {
		return errors.New("Unable to convert ip to bytes")
	}

	subnetInt := binary.BigEndian.Uint16(ipBytes[1:3])

	// clear from cache
	ds.tenantsLock.Lock()

	if ds.tenants[tenantID] != nil {
		ds.tenants[tenantID].network[int(subnetInt)][int(ipBytes[3])] = false
	}

	ds.tenantsLock.Unlock()

	return ds.db.releaseTenantIP(tenantID, int(subnetInt), int(ipBytes[3]))
}

// AllocateTenantIP will find a free IP address within a tenant network.
// For now we make each tenant have unique subnets even though it
// isn't actually needed because of a docker issue.
func (ds *Datastore) AllocateTenantIP(tenantID string) (net.IP, error) {
	var subnetInt uint16
	subnetInt = 0

	ds.tenantsLock.Lock()

	network := ds.tenants[tenantID].network
	subnets := ds.tenants[tenantID].subnets

	// find any subnet assigned to this tenant with available addresses
	sort.Ints(subnets)

	for _, k := range subnets {
		if len(network[k]) < 253 {
			subnetInt = uint16(k)
		}
	}

	var subnetBytes = []byte{16, 0}

	if subnetInt == 0 {
		i := binary.BigEndian.Uint16(subnetBytes)

		for {
			// check for new subnet.
			_, ok := network[int(i)]
			if !ok {
				sub := make(map[int]bool)
				network[int(i)] = sub

				break
			}

			if subnetBytes[1] == 255 {
				if subnetBytes[0] == 31 {
					// out of possible subnets
					glog.Warning("Out of Subnets")
					ds.tenantsLock.Unlock()
					return nil, errors.New("Out of subnets")
				}
				subnetBytes[0]++
				subnetBytes[1] = 0
			} else {
				subnetBytes[1]++
			}

			i = binary.BigEndian.Uint16(subnetBytes)
		}

		subnetInt = i

		ds.tenants[tenantID].subnets = append(subnets, int(subnetInt))
	} else {
		binary.BigEndian.PutUint16(subnetBytes, subnetInt)
	}

	hosts := network[int(subnetInt)]

	rest := 2

	for {
		if hosts[rest] == false {
			hosts[rest] = true
			break
		}

		if rest == 255 {
			// this should never happen
			glog.Warning("ran out of host numbers")

			ds.tenantsLock.Unlock()

			return nil, errors.New("rand out of host numbers")
		}

		rest++
	}

	ds.tenantsLock.Unlock()

	err := ds.db.claimTenantIP(tenantID, int(subnetInt), rest)
	if err != nil {
		return nil, errors.Wrap(err, "Error claiming tenant IP in database")
	}

	// convert to IP type.
	next := net.IPv4(172, subnetBytes[0], subnetBytes[1], byte(rest))

	return next, nil
}

// GetAllInstances retrieves all instances out of the datastore.
func (ds *Datastore) GetAllInstances() ([]*types.Instance, error) {
	var instances []*types.Instance

	// always get from cache
	ds.instancesLock.RLock()

	if len(ds.instances) > 0 {
		for _, val := range ds.instances {
			instances = append(instances, val)
		}
	}

	ds.instancesLock.RUnlock()

	return instances, nil
}

// GetInstance retrieves an instance out of the datastore.
func (ds *Datastore) GetInstance(id string) (*types.Instance, error) {
	// always get from cache
	ds.instancesLock.RLock()

	value, ok := ds.instances[id]

	ds.instancesLock.RUnlock()

	if !ok {
		return nil, types.ErrInstanceNotFound
	}

	return value, nil
}

// GetTenantInstance retrieves a tenant instance out of the datastore.
func (ds *Datastore) GetTenantInstance(tenantID string, instanceID string) (*types.Instance, error) {
	// always get from cache
	ds.instancesLock.RLock()

	value, ok := ds.instances[instanceID]

	ds.instancesLock.RUnlock()

	if !ok || value.TenantID != tenantID {
		return nil, types.ErrInstanceNotFound
	}

	return value, nil
}

// GetAllInstancesFromTenant will retrieve all instances belonging to a specific tenant
func (ds *Datastore) GetAllInstancesFromTenant(tenantID string) ([]*types.Instance, error) {
	var instances []*types.Instance

	ds.tenantsLock.RLock()

	t, ok := ds.tenants[tenantID]
	if ok {
		for _, val := range t.instances {
			instances = append(instances, val)
		}

		ds.tenantsLock.RUnlock()

		return instances, nil
	}

	ds.tenantsLock.RUnlock()

	return nil, nil
}

// GetAllInstancesByNode will retrieve all the instances running on a specific compute Node.
func (ds *Datastore) GetAllInstancesByNode(nodeID string) ([]*types.Instance, error) {
	var instances []*types.Instance

	ds.nodesLock.RLock()

	n, ok := ds.nodes[nodeID]
	if ok {
		for _, val := range n.instances {
			instances = append(instances, val)
		}
	}

	ds.nodesLock.RUnlock()

	return instances, nil
}

// AddInstance will store a new instance in the datastore.
// The instance will be updated both in the cache and in the database
func (ds *Datastore) AddInstance(instance *types.Instance) error {
	err := ds.db.addInstance(instance)

	if err != nil {
		return errors.Wrap(err, "Error adding instance to database")
	}

	// add to cache
	ds.instancesLock.Lock()

	ds.instances[instance.ID] = instance

	instanceStat := types.CiaoServerStats{
		ID:        instance.ID,
		TenantID:  instance.TenantID,
		NodeID:    instance.NodeID,
		Timestamp: time.Now(),
		Status:    instance.State,
	}

	ds.instanceLastStatLock.Lock()
	ds.instanceLastStat[instance.ID] = instanceStat
	ds.instanceLastStatLock.Unlock()

	ds.instancesLock.Unlock()

	ds.tenantsLock.Lock()
	tenant := ds.tenants[instance.TenantID]
	if tenant != nil {
		tenant.instances[instance.ID] = instance
	}
	ds.tenantsLock.Unlock()

	return nil
}

// RestartFailure logs a RestartFailure in the datastore
func (ds *Datastore) RestartFailure(instanceID string, reason payloads.RestartFailureReason) error {
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Restart Failure %s: %s", instanceID, reason.String())
	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// StopFailure logs a StopFailure in the datastore
func (ds *Datastore) StopFailure(instanceID string, reason payloads.StopFailureReason) error {
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Stop Failure %s: %s", instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// StartFailure will clean up after a failure to start an instance.
// If an instance was a CNCI, this function will remove the CNCI instance
// for this tenant. If the instance was a normal tenant instance, the
// IP address will be released and the instance will be deleted from the
// datastore.
//
// Only instances whose status is pending are removed when a StartFailure event
// is received.  StartFailure errors may also be generated when restarting an
// exited instance and we want to make sure that a failure to restart such
// an instance does not result in it being deleted.
func (ds *Datastore) StartFailure(instanceID string, reason payloads.StartFailureReason, migration bool) error {
	var tenantID string
	var cnci bool

	ds.tenantsLock.RLock()

	for key, t := range ds.tenants {
		if t.CNCIID == instanceID {
			cnci = true
			tenantID = key
			break
		}
	}

	ds.tenantsLock.RUnlock()

	if cnci == true {
		glog.Warning("CNCI ", instanceID, " Failed to start")

		err := ds.removeTenantCNCI(tenantID)

		msg := fmt.Sprintf("CNCI Start Failure %s: %s", instanceID, reason.String())
		ds.db.logEvent(tenantID, string(userError), msg)

		ds.cnciAddedLock.Lock()

		c, ok := ds.cnciAddedChans[tenantID]
		if ok {
			delete(ds.cnciAddedChans, tenantID)
		}

		ds.cnciAddedLock.Unlock()

		if c != nil {
			c <- false
		}

		return errors.Wrap(err, "error removing CNCI for tenant")
	}

	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	if reason.IsFatal() && !migration {
		ds.deleteInstance(instanceID)
	}

	msg := fmt.Sprintf("Start Failure %s: %s", instanceID, reason.String())
	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// AttachVolumeFailure will clean up after a failure to attach a volume.
// The volume state will be changed back to available, and an error message
// will be logged.
func (ds *Datastore) AttachVolumeFailure(instanceID string, volumeID string, reason payloads.AttachVolumeFailureReason) error {
	// update the block data to reflect correct state
	data, err := ds.GetBlockDevice(volumeID)
	if err != nil {
		return errors.Wrapf(err, "error getting block device for volume (%v)", volumeID)
	}

	oldState := data.State
	data.State = types.Available
	err = ds.UpdateBlockDevice(data)
	if err != nil {
		data.State = oldState
		return errors.Wrapf(err, "error updating block device for volume (%v)", volumeID)
	}

	// get owner of this instance
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Attach Volume Failure %s to %s: %s", volumeID, instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)

	return nil
}

// DetachVolumeFailure will clean up after a failure to detach a volume.
// The volume state will be changed back to available, and an error message
// will be logged.
func (ds *Datastore) DetachVolumeFailure(instanceID string, volumeID string, reason payloads.DetachVolumeFailureReason) error {
	// update the block data to reflect correct state
	data, err := ds.GetBlockDevice(volumeID)
	if err != nil {
		return errors.Wrapf(err, "error getting block device for volume (%v)", volumeID)
	}

	// because controller wouldn't allow a detach if state
	// wasn't initially InUse, we can blindly set this back
	// to InUse.
	oldState := data.State
	data.State = types.InUse
	err = ds.UpdateBlockDevice(data)
	if err != nil {
		data.State = oldState
		return errors.Wrapf(err, "error updating block device for volume (%v)", volumeID)
	}

	// get owner of this instance
	i, err := ds.GetInstance(instanceID)
	if err != nil {
		return errors.Wrapf(err, "error getting instance (%v)", instanceID)
	}

	msg := fmt.Sprintf("Detach Volume Failure %s from %s: %s", volumeID, instanceID, reason.String())

	ds.db.logEvent(i.TenantID, string(userError), msg)
	return nil
}

func (ds *Datastore) deleteInstance(instanceID string) (string, error) {
	if err := ds.db.deleteInstance(instanceID); err != nil {
		glog.Warningf("error deleting instance (%v): %v", instanceID, err)
		return "", errors.Wrapf(err, "error deleting instance from database (%v)", instanceID)
	}

	ds.instanceLastStatLock.Lock()
	delete(ds.instanceLastStat, instanceID)
	ds.instanceLastStatLock.Unlock()

	ds.instancesLock.Lock()
	i := ds.instances[instanceID]
	delete(ds.instances, instanceID)
	ds.instancesLock.Unlock()

	ds.tenantsLock.Lock()
	tenant := ds.tenants[i.TenantID]
	if tenant != nil {
		delete(tenant.instances, instanceID)
	}
	ds.tenantsLock.Unlock()

	// we may not have received any node stats for this instance
	if i.NodeID != "" {
		ds.nodesLock.Lock()
		delete(ds.nodes[i.NodeID].instances, instanceID)
		ds.nodesLock.Unlock()
	}

	var err error
	if tmpErr := ds.ReleaseTenantIP(i.TenantID, i.IPAddress); tmpErr != nil {
		glog.Warningf("error releasing IP for instance (%v): %v", i.ID, err)
		if err == nil {
			err = errors.Wrapf(tmpErr, "error releasing IP for instance (%v)", i.ID)
		}
	}

	ds.updateStorageAttachments(instanceID, nil)
{'fraction_non_alphanumeric': 0.11561995025146612, 'fraction_numerical': 0.002759772681881729, 'mean_word_length': 3.8702803077195154, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6880595', 'n_tokens_mistral': 19557, 'n_tokens_neox': 18565, 'n_words': 5458}