<commit_before>import os
import base64
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.image import Image

class SyncImages(OpenStackSyncStep):
    provides=[Image]
    requested_interval=0
    observes=Image

    def fetch_pending(self, deleted):
        # Images come from the back end
        # You can't delete them
        if (deleted):
            return []

        # get list of images on disk
        images_path = Config().observer_images_directory 
        available_images = {}
        for f in os.listdir(images_path):
            if os.path.isfile(os.path.join(images_path ,f)):
                available_images[f] = os.path.join(images_path ,f)

        images = Image.objects.all()
        image_names = [image.name for image in images]

        for image_name in available_images:
            #remove file extension
            clean_name = ".".join(image_name.split('.')[:-1])
            if clean_name not in image_names:
                image = Image(name=clean_name,
                              disk_format='raw',
                              container_format='bare', 
                              path = available_images[image_name])
                image.save()
       
        
        return Image.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) 

    def sync_record(self, image):
        image.save()<commit_msg>Check the existence of the images_path<commit_after>import os
import base64
from django.db.models import F, Q
from xos.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.image import Image

class SyncImages(OpenStackSyncStep):
    provides=[Image]
    requested_interval=0
    observes=Image

    def fetch_pending(self, deleted):
        # Images come from the back end
        # You can't delete them
        if (deleted):
            return []

        # get list of images on disk
        images_path = Config().observer_images_directory

        available_images = {}
        if os.path.exists(images_path):
            for f in os.listdir(images_path):
                filename = os.path.join(images_path, f)
                if os.path.isfile(filename):
                    available_images[f] = filename

        images = Image.objects.all()
        image_names = [image.name for image in images]

        for image_name in available_images:
            #remove file extension
            clean_name = ".".join(image_name.split('.')[:-1])
            if clean_name not in image_names:
                image = Image(name=clean_name,
                              disk_format='raw',
                              container_format='bare', 
                              path = available_images[image_name])
                image.save()

        return Image.objects.filter(Q(enacted__lt=F('updated')) | Q(enacted=None)) 

    def sync_record(self, image):
        image.save()
{'fraction_non_alphanumeric': 0.08784920902053181, 'fraction_numerical': 0.0026926960619320095, 'mean_word_length': 2.026476578411405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7386051', 'n_tokens_mistral': 819, 'n_tokens_neox': 785, 'n_words': 201}