from django.db import models
from core import Process, Stream, MaterialStatus, File
from libs.file import netcdf as nc
import calendar
from libs import matrix
import pytz
from datetime import datetime


class Compact(Process):
	class Meta(object):
			app_label = 'plumbing'
	extension = models.TextField()
	resultant_stream = models.ForeignKey(Stream, null=True, default=None)

	def do(self, stream):
		filename = "%spkg.%s.nc" % (self.resultant_stream.root_path,stream.tags.make_filename())
		f = self.do_file(filename,stream)
		fs = MaterialStatus(material=f,stream=self.resultant_stream)
		fs.save()
		return self.resultant_stream

	def getdatetimenow(self):
		return datetime.utcnow().replace(tzinfo=pytz.UTC)

	def do_file(self, filename, stream):
		# create compact file and initialize basic settings
		root, is_new = nc.open(filename)
		if is_new:
			sample = nc.open(stream.files.all()[0].file.completepath())[0]
			shape = sample.variables['data'].shape
			nc.getdim(root,'northing', shape[1])
			nc.getdim(root,'easting', shape[2])
			nc.getdim(root,'timing')
			v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
			v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
			v_lon[:] = nc.getvar(sample, 'lon')[:]
			v_lat[:] = nc.getvar(sample, 'lat')[:]
			nc.close(sample)
			nc.sync(root)
		self.do_var(root, 'data', stream)
		# save the content inside the compact file
		if not root is None: nc.close(root)
		f = File(localname=filename)
		f.save()
		return f

	def do_var(self, root, var_name, stream):
		file_statuses = sorted(self.stream.unprocessed(), key=lambda fs: fs.material.filename())
		shape = nc.getvar(root,'lat').shape
		for fs in file_statuses:
			# join the distributed content
			v_ch   = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
			v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',))
			try:
				rootimg = nc.open(fs.file.completepath())[0]
				data = (nc.getvar(rootimg, 'data'))[:]
				# Force all the channels to the same shape
				if not (data.shape[1:3] == shape):
					print data.shape[1:3], shape
					data = matrix.adapt(data, shape)
				if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
					index = v_ch.shape[0]
					v_ch[index,:] = data
					v_ch_t[index] = calendar.timegm(fs.file.datetime().utctimetuple())
				nc.close(rootimg)
				nc.sync(root)
			except RuntimeError:
				print fs.file.completepath()

from django.db import models
from core import Process, Stream, MaterialStatus, File
from libs.file import netcdf as nc
import calendar
from libs import matrix
import pytz
from datetime import datetime


class Compact(Process):
	class Meta(object):
			app_label = 'plumbing'
	extension = models.TextField()
	resultant_stream = models.ForeignKey(Stream, null=True, default=None)

	def do(self, stream):
		filename = "%spkg.%s.nc" % (self.resultant_stream.root_path,stream.tags.make_filename())
		f = self.do_file(filename,stream)
		ms = MaterialStatus(material=f,stream=self.resultant_stream)
		ms.save()
		return self.resultant_stream

	def getdatetimenow(self):
		return datetime.utcnow().replace(tzinfo=pytz.UTC)

	def do_file(self, filename, stream):
		# create compact file and initialize basic settings
		root, is_new = nc.open(filename)
		if is_new:
			sample = nc.open(stream.files.all()[0].file.completepath())[0]
			shape = sample.variables['data'].shape
			nc.getdim(root,'northing', shape[1])
			nc.getdim(root,'easting', shape[2])
			nc.getdim(root,'timing')
			v_lat = nc.getvar(root,'lat', 'f4', ('northing','easting',), 4)
			v_lon = nc.getvar(root,'lon', 'f4', ('northing','easting',), 4)
			v_lon[:] = nc.getvar(sample, 'lon')[:]
			v_lat[:] = nc.getvar(sample, 'lat')[:]
			nc.close(sample)
			nc.sync(root)
		self.do_var(root, 'data', stream)
		# save the content inside the compact file
		if not root is None: nc.close(root)
		f = File(localname=filename)
		f.save()
		return f

	def do_var(self, root, var_name, stream):
		material_statuses = sorted(stream.unprocessed(), key=lambda ms: ms.material.filename())
		shape = nc.getvar(root,'lat').shape
		for ms in material_statuses:
			# join the distributed content
			v_ch   = nc.getvar(root,var_name, 'f4', ('timing','northing','easting',), 4)
			v_ch_t = nc.getvar(root,var_name + '_time', 'f4', ('timing',))
			try:
				rootimg = nc.open(ms.material.completepath())[0]
				data = (nc.getvar(rootimg, 'data'))[:]
				# Force all the channels to the same shape
				if not (data.shape[1:3] == shape):
					print data.shape[1:3], shape
					data = matrix.adapt(data, shape)
				if v_ch.shape[1] == data.shape[1] and v_ch.shape[2] == data.shape[2]:
					index = v_ch.shape[0]
					v_ch[index,:] = data
					v_ch_t[index] = calendar.timegm(ms.material.datetime().utctimetuple())
				nc.close(rootimg)
				nc.sync(root)
			except RuntimeError:
				print ms.material.completepath()

Work on issues #11 and #13.