# Path: query/models.py
# class DayStatistic(models.Model):
#     """DayStatistic is used to generate timeline graphs. Data for the
#     DayStatistic table is gathered with the 'gatherstatistics' management
#     command.
#     """
#     date = models.DateField()
#     distribution = models.ForeignKey(Distribution, blank=True, null=True)
#     article_type = models.ForeignKey(ArticleType, blank=True, null=True)
#     count = models.IntegerField()
#     checked = models.DateTimeField(auto_now=True)
# 
#     class Meta:
#         unique_together = (('date', 'distribution', 'article_type'),)
# 
#     def __unicode__(self):
#         return '{}: {}'.format(str(self.date), self.count)
# 
# class ArticleType(models.Model):
#     """Model to store the names of article types that are available in the
#     elasticsearch index.
# 
#     The article types are part of the metadata of a query.
#     """
#     id = models.CharField(max_length=15, primary_key=True)
#     name = models.CharField(max_length=35)
# 
#     def __unicode__(self):
#         return self.id
# 
# class Distribution(models.Model):
#     """Model that stores the names of distributions that are available in the
#     elasticsearch index.
# 
#     The distributions are part of the metadata of a query.
#     """
#     id = models.CharField(max_length=15, primary_key=True)
#     name = models.CharField(max_length=35)
# 
#     def __unicode__(self):
#         return self.id
# 
# Path: services/es.py
# def day_statistics(idx, typ, date_range, agg_name, distribution=None, article_type=None):
#     """Gather day statistics for all dates in the date range
# 
#     This function is used by the gatherstatistics management command.
#     """
#     q = create_day_statistics_query(date_range, agg_name, distribution, article_type)
# 
#     results = _es().search(index=idx, doc_type=typ, body=q, size=0)
# 
#     if 'took' in results:
#         return results
#     return None


from datetime import datetime
from django.core.management.base import BaseCommand
from django.db import DatabaseError
from django.conf import settings
from query.models import DayStatistic, ArticleType, Distribution
from services.es import day_statistics
from texcavator.utils import daterange2dates
import sys
        agg_name = 'daystatistic'

        for year in range(year_lower, year_upper+1):
            date_range = {
                'lower': '{y}-01-01'.format(y=year),
                'upper': '{y}-12-31'.format(y=year)
            }

            print year

            for article_type in ArticleType.objects.all():
                for distribution in Distribution.objects.all():
                    results = day_statistics(settings.ES_INDEX,
                                             settings.ES_DOCTYPE,
                                             date_range,
                                             agg_name,
                                             distribution.id,
                                             article_type.id)
                    self.save_to_database(agg_name, results, distribution, article_type)

    @staticmethod
    def save_to_database(agg_name, results, distribution=None, article_type=None):
        if results:
            # save results to database
            agg_data = results['aggregations'][agg_name]['buckets']

            for date in agg_data:
                try:
                    d = datetime.strptime(date['key_as_string'],
                                          '%Y-%m-%dT00:00:00.000Z').date()

# Path: query/models.py
# class DayStatistic(models.Model):
#     """DayStatistic is used to generate timeline graphs. Data for the
#     DayStatistic table is gathered with the 'gatherstatistics' management
#     command.
#     """
#     date = models.DateField()
#     distribution = models.ForeignKey(Distribution, blank=True, null=True)
#     article_type = models.ForeignKey(ArticleType, blank=True, null=True)
#     count = models.IntegerField()
#     checked = models.DateTimeField(auto_now=True)
# 
#     class Meta:
#         unique_together = (('date', 'distribution', 'article_type'),)
# 
#     def __unicode__(self):
#         return '{}: {}'.format(str(self.date), self.count)
# 
# class ArticleType(models.Model):
#     """Model to store the names of article types that are available in the
#     elasticsearch index.
# 
#     The article types are part of the metadata of a query.
#     """
#     id = models.CharField(max_length=15, primary_key=True)
#     name = models.CharField(max_length=35)
# 
#     def __unicode__(self):
#         return self.id
# 
# class Distribution(models.Model):
#     """Model that stores the names of distributions that are available in the
#     elasticsearch index.
# 
#     The distributions are part of the metadata of a query.
#     """
#     id = models.CharField(max_length=15, primary_key=True)
#     name = models.CharField(max_length=35)
# 
#     def __unicode__(self):
#         return self.id
# 
# Path: services/es.py
# def day_statistics(idx, typ, date_range, agg_name, distribution=None, article_type=None):
#     """Gather day statistics for all dates in the date range
# 
#     This function is used by the gatherstatistics management command.
#     """
#     q = create_day_statistics_query(date_range, agg_name, distribution, article_type)
# 
#     results = _es().search(index=idx, doc_type=typ, body=q, size=0)
# 
#     if 'took' in results:
#         return results
#     return None


# Path: query/management/commands/gatherstatistics.py
from datetime import datetime
from django.core.management.base import BaseCommand
from django.db import DatabaseError
from django.conf import settings
from query.models import DayStatistic, ArticleType, Distribution
from services.es import day_statistics
from texcavator.utils import daterange2dates
import sys

        agg_name = 'daystatistic'

        for year in range(year_lower, year_upper+1):
            date_range = {
                'lower': '{y}-01-01'.format(y=year),
                'upper': '{y}-12-31'.format(y=year)
            }

            print year

            for article_type in ArticleType.objects.all():
                for distribution in Distribution.objects.all():
                    results = day_statistics(settings.ES_INDEX,
                                             settings.ES_DOCTYPE,
                                             date_range,
                                             agg_name,
                                             distribution.id,
                                             article_type.id)
                    self.save_to_database(agg_name, results, distribution, article_type)

    @staticmethod
    def save_to_database(agg_name, results, distribution=None, article_type=None):
        if results:
            # save results to database
            agg_data = results['aggregations'][agg_name]['buckets']

            for date in agg_data:
                try:
                    d = datetime.strptime(date['key_as_string'],
                                          '%Y-%m-%dT00:00:00.000Z').date()

                    DayStatistic.objects.create(date=str(d),