# Path: privacyscore/backend/models.py
# class RawScanResult(models.Model):
#     """Raw scan result of a test."""
#     scan = models.ForeignKey(
#         Scan, on_delete=models.CASCADE, related_name='raw_results')
# 
#     scan_host = models.CharField(max_length=80)
#     test = models.CharField(max_length=80)
#     identifier = models.CharField(max_length=80)
# 
#     mime_type = models.CharField(max_length=80)
#     file_name = models.CharField(max_length=80, null=True, blank=True)
#     data = models.BinaryField(null=True, blank=True)
# 
#     def get_data_as_string(self):
#         return bytes(self.retrieve()).decode()
# 
#     def __str__(self) -> str:
#         return '{}: {}'.format(str(self.scan), self.test)
# 
#     @property
#     def in_db(self) -> bool:
#         return self.file_name is None
# 
#     @staticmethod
#     def store_raw_data(data: bytes, mime_type: str, scan_host: str, test: str,
#                        identifier: str, scan_pk: int):
#         """Store data in db or filesystem."""
#         if len(data) > settings.RAW_DATA_DB_MAX_SIZE:
#             # store in filesystem
# 
#             # TODO: ensure uniqueness
#             file_name = str(uuid4())
#             path = os.path.join(settings.RAW_DATA_DIR, file_name)
#             if mime_type in settings.RAW_DATA_UNCOMPRESSED_TYPES:
#                 # do not compress
#                 with open(path, 'wb') as f:
#                     f.write(data)
#             else:
#                 file_name += '.gz'
#                 path += '.gz'
#                 # compress
#                 with gzip.open(path, 'wb') as f:
#                     f.write(data)
# 
#             RawScanResult.objects.create(
#                 scan_id=scan_pk,
#                 scan_host=scan_host,
#                 test=test,
#                 identifier=identifier,
#                 mime_type=mime_type,
#                 file_name=file_name)
#         else:
#             RawScanResult.objects.create(
#                 scan_id=scan_pk,
#                 scan_host=scan_host,
#                 test=test,
#                 identifier=identifier,
#                 mime_type=mime_type,
#                 data=data)
# 
#     def retrieve(self) -> bytes:
#         """Retrieve the raw data."""
#         if self.in_db:
#             if isinstance(self.data, memoryview):
#                 return self.data.tobytes()
#             return self.data
#         path = os.path.join(settings.RAW_DATA_DIR, self.file_name)
#         if path.endswith('.gz'):
#             with gzip.open(path, 'rb') as f:
#                 return f.read()
#         with open(path, 'rb') as f:
#             return f.read()


import os
import sys
from django.conf import settings
from django.core.management import BaseCommand
from django.utils import timezone
from privacyscore.backend.models import RawScanResult
class Command(BaseCommand):
    help = 'Cleans up old raw data and removes raw data from filesystem not known to db.'

    def handle(self, *args, **options):
        """Handle command."""
        # remove raw data db entries older than configured max allowed
        outdated = RawScanResult.objects.filter(
            scan__start__lte=timezone.now() - settings.RAW_DATA_DELETE_AFTER)
        deleted = outdated.delete()[0]
        self.stdout.write('Deleted {} database entries'.format(deleted))

        # find files from file system unknown to db
        known_files = [v['file_name'] for v in RawScanResult.objects.filter(
            file_name__isnull=False).values('file_name')]

        deleted = 0
        for file in os.listdir(settings.RAW_DATA_DIR):
            if file not in known_files:
                os.remove(os.path.join(
                    settings.RAW_DATA_DIR, file))
                deleted += 1
        print('Deleted {} files from file system'.format(deleted))

        # find files from db unknown to filesystem
        known_files = os.listdir(settings.RAW_DATA_DIR)
        to_delete = []
        for elem in RawScanResult.objects.filter(
                file_name__isnull=False).values('id', 'file_name'):
            if elem['file_name'] not in known_files:
                to_delete.append(elem['id'])

# Path: privacyscore/backend/models.py
# class RawScanResult(models.Model):
#     """Raw scan result of a test."""
#     scan = models.ForeignKey(
#         Scan, on_delete=models.CASCADE, related_name='raw_results')
# 
#     scan_host = models.CharField(max_length=80)
#     test = models.CharField(max_length=80)
#     identifier = models.CharField(max_length=80)
# 
#     mime_type = models.CharField(max_length=80)
#     file_name = models.CharField(max_length=80, null=True, blank=True)
#     data = models.BinaryField(null=True, blank=True)
# 
#     def get_data_as_string(self):
#         return bytes(self.retrieve()).decode()
# 
#     def __str__(self) -> str:
#         return '{}: {}'.format(str(self.scan), self.test)
# 
#     @property
#     def in_db(self) -> bool:
#         return self.file_name is None
# 
#     @staticmethod
#     def store_raw_data(data: bytes, mime_type: str, scan_host: str, test: str,
#                        identifier: str, scan_pk: int):
#         """Store data in db or filesystem."""
#         if len(data) > settings.RAW_DATA_DB_MAX_SIZE:
#             # store in filesystem
# 
#             # TODO: ensure uniqueness
#             file_name = str(uuid4())
#             path = os.path.join(settings.RAW_DATA_DIR, file_name)
#             if mime_type in settings.RAW_DATA_UNCOMPRESSED_TYPES:
#                 # do not compress
#                 with open(path, 'wb') as f:
#                     f.write(data)
#             else:
#                 file_name += '.gz'
#                 path += '.gz'
#                 # compress
#                 with gzip.open(path, 'wb') as f:
#                     f.write(data)
# 
#             RawScanResult.objects.create(
#                 scan_id=scan_pk,
#                 scan_host=scan_host,
#                 test=test,
#                 identifier=identifier,
#                 mime_type=mime_type,
#                 file_name=file_name)
#         else:
#             RawScanResult.objects.create(
#                 scan_id=scan_pk,
#                 scan_host=scan_host,
#                 test=test,
#                 identifier=identifier,
#                 mime_type=mime_type,
#                 data=data)
# 
#     def retrieve(self) -> bytes:
#         """Retrieve the raw data."""
#         if self.in_db:
#             if isinstance(self.data, memoryview):
#                 return self.data.tobytes()
#             return self.data
#         path = os.path.join(settings.RAW_DATA_DIR, self.file_name)
#         if path.endswith('.gz'):
#             with gzip.open(path, 'rb') as f:
#                 return f.read()
#         with open(path, 'rb') as f:
#             return f.read()


# Path: privacyscore/backend/management/commands/rawdatagc.py
import os
import sys
from django.conf import settings
from django.core.management import BaseCommand
from django.utils import timezone
from privacyscore.backend.models import RawScanResult

class Command(BaseCommand):
    help = 'Cleans up old raw data and removes raw data from filesystem not known to db.'

    def handle(self, *args, **options):
        """Handle command."""
        # remove raw data db entries older than configured max allowed
        outdated = RawScanResult.objects.filter(
            scan__start__lte=timezone.now() - settings.RAW_DATA_DELETE_AFTER)
        deleted = outdated.delete()[0]
        self.stdout.write('Deleted {} database entries'.format(deleted))

        # find files from file system unknown to db
        known_files = [v['file_name'] for v in RawScanResult.objects.filter(
            file_name__isnull=False).values('file_name')]

        deleted = 0
        for file in os.listdir(settings.RAW_DATA_DIR):
            if file not in known_files:
                os.remove(os.path.join(
                    settings.RAW_DATA_DIR, file))
                deleted += 1
        print('Deleted {} files from file system'.format(deleted))

        # find files from db unknown to filesystem
        known_files = os.listdir(settings.RAW_DATA_DIR)
        to_delete = []
        for elem in RawScanResult.objects.filter(
                file_name__isnull=False).values('id', 'file_name'):
            if elem['file_name'] not in known_files:
                to_delete.append(elem['id'])

        deleted = RawScanResult.objects.filter(id__in=to_delete).delete()[0]