# Path: archive_chan/models.py
# class Update(models.Model):
#     CURRENT = 0
#     FAILED = 1
#     COMPLETED = 2
# 
#     STATUS_CHOICES = (
#         (CURRENT, 'Started'),
#         (FAILED, 'Failed'),
#         (COMPLETED, 'Completed'),
#     )
# 
#     board = models.ForeignKey('Board')
#     status = models.SmallIntegerField(choices=STATUS_CHOICES, default=CURRENT)
#     start = models.DateTimeField()
#     end = models.DateTimeField(null=True)
# 
#     used_threads = models.IntegerField()
# 
#     total_time = models.FloatField(default=0)
#     wait_time = models.FloatField(default=0)
#     download_time = models.FloatField(default=0)
# 
#     processed_threads = models.IntegerField(default=0)
#     added_posts = models.IntegerField(default=0)
#     removed_posts = models.IntegerField(default=0)
# 
#     downloaded_images = models.IntegerField(default=0)
#     downloaded_thumbnails = models.IntegerField(default=0)
#     downloaded_threads = models.IntegerField(default=0)
# 
#     class Meta:
#         ordering = ['-start']
# 
# class Board(models.Model):
#     name = models.CharField(max_length=255, primary_key = True)
#     active = models.BooleanField(
#         default=True,
#         help_text='Should this board be updated with new posts?'
#     )
#     store_threads_for = models.IntegerField(
#         default=48,
#         help_text='[hours] After that much time passes from the last reply in a NOT SAVED thread it will be deleted. Set to 0 to preserve threads forever.'
#     )
#     replies_threshold = models.IntegerField(
#         default=20,
#         help_text='Store threads after they reach that many replies.'
#     )
# 
#     class Meta:
#         ordering = ['name']
# 
#     def __str__(self):
#         return format("/%s/" % self.name)
# 
#     def get_absolute_url(self):
#         return reverse('archive_chan:board', args=[self.name])
# 
# Path: archive_chan/settings.py
# class AppSettings:
#     app_settings = {
#         'API_WAIT': 1, # [seconds] Delay between two API calls (catalog/list of posts). This should follow the API rules.
#         'FILE_WAIT': 0, # [seconds] Delay between two file downloads (images/thumbnails). This should follow the API rules (no limit at this point).
#         'CONNECTION_TIMEOUT': 10, # [seconds] Code downloading the data will stop waiting for a response after that time.
#         'RECENT_POSTS_AGE': 48, # [hours] Used for selecting statistics when the board stores posts forever without deleting them. Read more in views.ajax_board_stats
#         'SCRAPER_THREADS_NUMBER': 4, # Number of additional program threads running at the same time. In other words that many 4chan threads will be updated at the same time.
#         'VIEW_CACHE_AGE': 60 * 5, # [seconds] max age of the dynamic pages eg. board
#         'VIEW_CACHE_AGE_STATIC': 60 * 60 * 24, # [seconds] max age of the static pages eg. stats
#         'MEDIA_URL': settings.MEDIA_URL, # You can override the URL from which the downloaded photos are served.
#     }
# 
#     @classmethod
#     def get(self, name):
#         return getattr(settings, 'ARCHIVE_CHAN_' + name, self.app_settings[name])


from datetime import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.utils.timezone import utc
from archive_chan.models import Update, Board
from archive_chan.settings import AppSettings
import sys, re



class Command(BaseCommand):
    args = '<file_path file_path ...>'
    help = 'Imports data from scraper log files.'
    option_list = BaseCommand.option_list + (
        make_option(
            '-t',
            '--used-threads',
            action="store",
            dest='used_threads',
            help='Specify the number of threads to set in the database. Log does not contain that data.',
        ),
    )

    def handle(self, *args, **options):
        boards = {}

        if options['used_threads'] is None:
            raise ValueError('Specify the number of used threads.')

        options['used_threads'] = int(options['used_threads'])

        sucessful = 0
        failed = 0
        added = 0


# Path: archive_chan/models.py
# class Update(models.Model):
#     CURRENT = 0
#     FAILED = 1
#     COMPLETED = 2
# 
#     STATUS_CHOICES = (
#         (CURRENT, 'Started'),
#         (FAILED, 'Failed'),
#         (COMPLETED, 'Completed'),
#     )
# 
#     board = models.ForeignKey('Board')
#     status = models.SmallIntegerField(choices=STATUS_CHOICES, default=CURRENT)
#     start = models.DateTimeField()
#     end = models.DateTimeField(null=True)
# 
#     used_threads = models.IntegerField()
# 
#     total_time = models.FloatField(default=0)
#     wait_time = models.FloatField(default=0)
#     download_time = models.FloatField(default=0)
# 
#     processed_threads = models.IntegerField(default=0)
#     added_posts = models.IntegerField(default=0)
#     removed_posts = models.IntegerField(default=0)
# 
#     downloaded_images = models.IntegerField(default=0)
#     downloaded_thumbnails = models.IntegerField(default=0)
#     downloaded_threads = models.IntegerField(default=0)
# 
#     class Meta:
#         ordering = ['-start']
# 
# class Board(models.Model):
#     name = models.CharField(max_length=255, primary_key = True)
#     active = models.BooleanField(
#         default=True,
#         help_text='Should this board be updated with new posts?'
#     )
#     store_threads_for = models.IntegerField(
#         default=48,
#         help_text='[hours] After that much time passes from the last reply in a NOT SAVED thread it will be deleted. Set to 0 to preserve threads forever.'
#     )
#     replies_threshold = models.IntegerField(
#         default=20,
#         help_text='Store threads after they reach that many replies.'
#     )
# 
#     class Meta:
#         ordering = ['name']
# 
#     def __str__(self):
#         return format("/%s/" % self.name)
# 
#     def get_absolute_url(self):
#         return reverse('archive_chan:board', args=[self.name])
# 
# Path: archive_chan/settings.py
# class AppSettings:
#     app_settings = {
#         'API_WAIT': 1, # [seconds] Delay between two API calls (catalog/list of posts). This should follow the API rules.
#         'FILE_WAIT': 0, # [seconds] Delay between two file downloads (images/thumbnails). This should follow the API rules (no limit at this point).
#         'CONNECTION_TIMEOUT': 10, # [seconds] Code downloading the data will stop waiting for a response after that time.
#         'RECENT_POSTS_AGE': 48, # [hours] Used for selecting statistics when the board stores posts forever without deleting them. Read more in views.ajax_board_stats
#         'SCRAPER_THREADS_NUMBER': 4, # Number of additional program threads running at the same time. In other words that many 4chan threads will be updated at the same time.
#         'VIEW_CACHE_AGE': 60 * 5, # [seconds] max age of the dynamic pages eg. board
#         'VIEW_CACHE_AGE_STATIC': 60 * 60 * 24, # [seconds] max age of the static pages eg. stats
#         'MEDIA_URL': settings.MEDIA_URL, # You can override the URL from which the downloaded photos are served.
#     }
# 
#     @classmethod
#     def get(self, name):
#         return getattr(settings, 'ARCHIVE_CHAN_' + name, self.app_settings[name])


# Path: archive_chan/management/commands/archive_chan_import_log.py
from datetime import datetime
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from django.utils.timezone import utc
from archive_chan.models import Update, Board
from archive_chan.settings import AppSettings
import sys, re




class Command(BaseCommand):
    args = '<file_path file_path ...>'
    help = 'Imports data from scraper log files.'
    option_list = BaseCommand.option_list + (
        make_option(
            '-t',
            '--used-threads',
            action="store",
            dest='used_threads',
            help='Specify the number of threads to set in the database. Log does not contain that data.',
        ),
    )

    def handle(self, *args, **options):
        boards = {}

        if options['used_threads'] is None:
            raise ValueError('Specify the number of used threads.')

        options['used_threads'] = int(options['used_threads'])

        sucessful = 0
        failed = 0
        added = 0


        for file_path in args: