<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2016 Parisson SARL
# Copyright (c) 2014-2016 <NAME> <<EMAIL>>
# Copyright (c) 2014-2016 <NAME> <<EMAIL>>

# This file is part of TimeSide.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Authors:
# <NAME> <<EMAIL>>
# <NAME> <<EMAIL>>


from __future__ import unicode_literals
import os
import uuid
import mimetypes
import ast
import time
import datetime
import gc
from shutil import copyfile
from builtins import str

import timeside.core
from timeside.plugins.decoder.utils import sha1sum_file, sha1sum_url
from timeside.core.tools.parameters import DEFAULT_SCHEMA
from django.db import models
from django.utils.functional import lazy
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import python_2_unicode_compatible
from django.contrib.auth.models import User
from django.db.models.signals import post_save, pre_save
from django.conf import settings
from django.dispatch import receiver
from rest_framework.authtoken.models import Token
from django.core.files import File
from celery.utils.log import get_task_logger

import jsonfield
import json
import youtube_dl
from requests import get

#XMLtoJSON
from xml.etree.ElementTree import fromstring
from xmljson import abdera as ab

worker_logger = get_task_logger(__name__)
app = 'timeside'

processors = timeside.core.processor.processors(timeside.core.api.IProcessor)
#providers = timeside.core.provider.providers(timeside.core.api.IProvider)


_processor_types = {'Analyzers': timeside.core.api.IAnalyzer,
                    'Encoders': timeside.core.api.IEncoder,
                    'Graphers': timeside.core.api.IGrapher}


def get_processor_pids():
    return [(name, [(processor.id(), processor.id())
                    for processor
                    in timeside.core.processor.processors(proc_type)])
            for name, proc_type in _processor_types.items()]

public_extra_types = {
    '.webm': 'video/webm',
}

encoders = timeside.core.processor.processors(timeside.core.api.IEncoder)
for encoder in encoders:
    public_extra_types['.' + encoder.file_extension()] = encoder.mime_type()

private_extra_types = {
    '.eaf': 'text/xml',  # ELAN Annotation Format
    '.trs': 'text/xml',  # Trancriber Annotation Format
    '.svl': 'text/xml',  # Sonic Visualiser layer file
    '.TextGrid': 'text/praat-textgrid',  # Praat TextGrid annotation file
    '.h5': 'application/x-hdf5',    # HDF5
    '.hdf5': 'application/x-hdf5',  # HDF5
}

for ext, mime_type in public_extra_types.items():
    if not mimetypes.guess_extension(mime_type) == ext:
        mimetypes.add_type(mime_type, ext)

for ext, mime_type in private_extra_types.items():
    mimetypes.add_type(mime_type, ext)


# Status
_FAILED, _DRAFT, _PENDING, _RUNNING, _DONE = 0, 1, 2, 3, 4
STATUS = ((_FAILED, _('failed')), (_DRAFT, _('draft')),
          (_PENDING, _('pending')), (_RUNNING, _('running')),
          (_DONE, _('done')))


RESULTS_ROOT = os.path.join(settings.MEDIA_ROOT, 'results')
if not os.path.exists(RESULTS_ROOT):
    os.makedirs(RESULTS_ROOT)

class UUIDEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, UUID):
            # if the obj is uuid, we simply return the value of uuid
            return obj.hex
        return json.JSONEncoder.default(self, obj)


def get_mime_type(path):
    return mimetypes.guess_type(path)[0]

#make every user having an automatically generated Token catching the User's post_save signal
@receiver(post_save, sender=settings.AUTH_USER_MODEL)
def create_auth_token(sender, instance=None, created=False, **kwargs):
    if created:
        Token.objects.create(user=instance)

# --- Abstract classes -----
class Dated(models.Model):

    date_added = models.DateTimeField(_('date added'), auto_now_add=True, null=True)
    date_modified = models.DateTimeField(_('date modified'), auto_now=True, null=True)

    class Meta:
        abstract = True


class UUID(models.Model):

    uuid = models.UUIDField(_('uuid'), default=uuid.uuid4, primary_key=True, blank=False, max_length=255, editable=False)
    #uuid = models.CharField(_('uuid'), unique=True, blank=True, max_length=255, editable=False)
    class Meta:
        abstract = True

    @classmethod
    def get_first_or_create(cls, **kwargs):
        """
        duplicates-compatible get or create method
        """
        objs = cls.objects.filter(**kwargs)
        if objs:
            obj = objs[0]
            created = False
        else:
            obj = cls(**kwargs)
            obj.save()
            created = True
        return obj, created


@python_2_unicode_compatible
class Titled(models.Model):

    title = models.CharField(_('title'), blank=True, max_length=512)
    description = models.TextField(_('description'), blank=True)

    def __str__(self):
        return self.title

    class Meta:
        abstract = True


@python_2_unicode_compatible
class Named(models.Model):

    name = models.CharField(_('name'), blank=True, max_length=512)
    description = models.TextField(_('description'), blank=True)

    def __str__(self):
        return str(self.name)

    class Meta:
        abstract = True


class Shareable(models.Model):

    author = models.ForeignKey(User, related_name="%(class)s", verbose_name=_('author'), blank=True, null=True, on_delete=models.SET_NULL)
    is_public = models.BooleanField(default=True)

    class Meta:
        abstract = True


@python_2_unicode_compatible
class Provider(Named, UUID):

    pid = models.CharField(_('pid'), blank=True, max_length=128)
    source_access =  models.BooleanField(default=False)

    def __str__(self):
        return str(self.pid)

    def get_provider(self):
        return timeside.core.get_provider(self.pid)

    def get_source_from_url(self, url, download=False):
        DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT,'items','download','')
        return self.get_provider()().get_source_from_url(url, DOWNLOAD_ROOT, download)

    def get_source_from_id(self, external_id, download=False):
        DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT,'items','download','')
        return self.get_provider()().get_source_from_id(external_id, DOWNLOAD_ROOT, download)

    def get_id_from_url(self, url):
        return self.get_provider()().get_id_from_url(url)

class Selection(Titled, UUID, Dated, Shareable):

    items = models.ManyToManyField('Item', related_name="selections", verbose_name=_('items'), blank=True)
    selections = models.ManyToManyField('Selection', related_name="other_selections", verbose_name=_('other selections'), blank=True)

    class Meta:
        verbose_name = _('selection')

    def get_all_items(self):
        qs_items = self.items.all()
        for selection in self.selections.all():
            qs_items |= selection.get_all_items()
        return qs_items


@python_2_unicode_compatible
class Item(Titled, UUID, Dated, Shareable):

    element_type = 'timeside_item'

    source_file = models.FileField(_('file'), upload_to='items/%Y/%m/%d', blank=True, max_length=1024)
    source_url = models.URLField(_('URL'), blank=True, max_length=1024)
    audio_duration = models.FloatField(_('duration'), blank=True, null=True)
    sha1 = models.CharField(_('sha1'), blank=True, max_length=512)
    mime_type = models.CharField(_('mime type'), blank=True, max_length=256)
    hdf5 = models.FileField(_('HDF5 result file'), upload_to='results/%Y/%m/%d', blank=True, max_length=1024)
    lock = models.BooleanField(default=False)
    external_uri = models.CharField(_('external_uri'), blank=True, max_length=1024)
    external_id = models.CharField(_('external_id'), blank=True, max_length=256)
    provider = models.ForeignKey('Provider', verbose_name=_('provider'), blank=True, null=True, on_delete=models.SET_NULL)

    class Meta:
        ordering = ['title']
        verbose_name = _('item')

    def __str__(self):
        return '_'.join([str(self.title), str(self.uuid)[:4]])

    def results(self):
        return [result for result in self.results.all()]

    def lock_setter(self, lock):
        self.lock = lock
        self.save()

    def get_source(self, download=False):
        if not (self.source_url or self.source_file) and self.provider:
            if download:
                if self.external_uri:
                    self.source_file = self.provider.get_source_from_url(self.external_uri,download).replace(settings.MEDIA_ROOT, '')
                elif self.external_id:
                    self.source_file = self.provider.get_source_from_id(self.external_id,download).replace(settings.MEDIA_ROOT, '')
            else:
                if self.external_uri:
                    self.source_url = self.provider.get_source_from_url(self.external_uri,download)
                elif self.external_id:
                    self.source_url = self.provider.get_source_from_id(self.external_id,download)
            super(Item, self).save()

    def get_external_id(self):
        if not (self.source_url or self.external_id) and self.external_uri:
            self.external_id = self.provider.get_id_from_url(self.external_uri)
            super(Item, self).save()

    def get_uri(self):
        """Return the Item source"""
        if self.source_file and os.path.exists(self.source_file.path):
            return self.source_file.path
        elif self.source_url:
            return self.source_url
        return None

    def get_results_path(self):
        """
        Return Item result path
        """
        result_path = os.path.join(RESULTS_ROOT, str(self.uuid))
        if not os.path.exists(result_path):
            os.makedirs(result_path)
        return result_path

    def get_audio_duration(self, force=False):
        """
        Return item audio duration
        """
        if (force or not self.audio_duration) and self.source_file:
            decoder = timeside.core.get_processor('file_decoder')(
                uri=self.get_uri())
            self.audio_duration = decoder.uri_total_duration
            super(Item, self).save()

    def get_hash(self, force=False):
        "Set SHA1 hash from file binary content"
        if force or (self.sha1 is None):
            if self.source_file:
                sha1 = sha1sum_file(self.source_file.path)
            elif self.source_url:
                sha1 = sha1sum_url(self.source_url)
            else:
                return
            self.sha1 = sha1
            super(Item, self).save()

    def get_mimetype(self, force=False):
        if force or (self.mime_type is None):
            if self.source_url:
                path = self.source_url
            elif self.source_file:
                path = self.source_file.path
            if os.path.exists(path):
                self.mime_type = get_mime_type(path)
            super(Item, self).save()

    def get_single_selection(self):
        # TODO : have singleton selection has a foreign key of Item ???
        sel_title = "Singleton selection for item %d" % self.uuid
        sel_description = ("Singleton selection for item %d\n"
                           "Automatically generated by the TimeSide "
                           "application.") % self.uuid
        selection, created = Selection.objects.get_or_create(
            title=sel_title,
            description=sel_description)
        if created:
            selection.save()
            selection.items.add(self)
        return selection

    def run(self, experience):
        result_path = self.get_results_path()
        # get audio source
        uri = self.get_uri()

        # decode audio source
        decoder = timeside.plugins.decoder.file.FileDecoder(uri=uri,
                                                            sha1=self.sha1)
        presets = {}
        pipe = decoder
        parent_analyzers = []

        # search for parent analyzer presets not to add as duplicates in the pipe
        for preset in experience.presets.all():
            proc = preset.processor.get_processor()
            if proc.type in ['analyzer', 'grapher']:
                proc = proc(**json.loads(preset.parameters))
                if 'analyzer' in proc.parents:
                    parent_analyzers.append(proc.parents['analyzer'])

        for preset in experience.presets.all():
            # get core audio metaProcessor corresponding to preset.processor.pid
            proc = preset.processor.get_processor()
            if proc.type == 'encoder':
                result, c = Result.objects.get_or_create(preset=preset,
                                                         item=self)
                media_file = '.'.join([str(result.uuid),
                                       proc.file_extension()])
                result.file = os.path.join(result_path, media_file).replace(settings.MEDIA_ROOT, '')
                result.save()
                # instantiate a core processor of an encoder
                proc = proc(result.file.path, overwrite=True,
                            streaming=False)
                logger_worker.info(f'Run {str(proc)} on {str(self)}')
            elif proc.type in ['analyzer', 'grapher']:
                # instantiate a core processor of an analyzer or a grapher
                proc = proc(**json.loads(preset.parameters))
                logger_worker.info(f'Run {str(proc)} on {str(self)} with {str(preset.parameters)}')

            if proc not in parent_analyzers:
                presets[preset] = proc
                pipe |= proc

        # item.lock_setter(True)

        if not self.hdf5:
            hdf5_file = str(experience.uuid) + '.hdf5'
            self.hdf5 = os.path.join(result_path, hdf5_file).replace(settings.MEDIA_ROOT, '')
            self.save()

        pipe.run()

        def set_results_from_processor(proc, preset=None):
            if preset:
                parameters = preset.parameters
            elif preset is None:
                processor, c = Processor.get_first_or_create(pid=proc.id())
                parameters = json.dumps(processor.get_parameters_default())
                preset, c = Preset.get_first_or_create(processor=processor,
                                                    parameters=parameters)
            result, c = Result.objects.get_or_create(preset=preset,
                                                    item=self)


            if not hasattr(proc, 'external'):
                hdf5_file = str(result.uuid) + '.hdf5'
                result.hdf5 = os.path.join(result_path, hdf5_file).replace(settings.MEDIA_ROOT, '')
                # while result.lock:<commit_msg>[server] fix log worker name bug #129<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2014-2016 <NAME>
# Copyright (c) 2014-2016 <NAME> <<EMAIL>>
# Copyright (c) 2014-2016 <NAME> <<EMAIL>>

# This file is part of TimeSide.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# Authors:
# <NAME> <<EMAIL>>
# <NAME> <<EMAIL>>


from __future__ import unicode_literals
import os
import uuid
import mimetypes
import ast
import time
import datetime
import gc
from shutil import copyfile
from builtins import str

import timeside.core
from timeside.plugins.decoder.utils import sha1sum_file, sha1sum_url
from timeside.core.tools.parameters import DEFAULT_SCHEMA
from django.db import models
from django.utils.functional import lazy
from django.utils.text import slugify
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import python_2_unicode_compatible
from django.contrib.auth.models import User
from django.db.models.signals import post_save, pre_save
from django.conf import settings
from django.dispatch import receiver
from rest_framework.authtoken.models import Token
from django.core.files import File
from celery.utils.log import get_task_logger

import jsonfield
import json
import youtube_dl
from requests import get

#XMLtoJSON
from xml.etree.ElementTree import fromstring
from xmljson import abdera as ab

worker_logger = get_task_logger(__name__)
app = 'timeside'

processors = timeside.core.processor.processors(timeside.core.api.IProcessor)
#providers = timeside.core.provider.providers(timeside.core.api.IProvider)


_processor_types = {'Analyzers': timeside.core.api.IAnalyzer,
                    'Encoders': timeside.core.api.IEncoder,
                    'Graphers': timeside.core.api.IGrapher}


def get_processor_pids():
    return [(name, [(processor.id(), processor.id())
                    for processor
                    in timeside.core.processor.processors(proc_type)])
            for name, proc_type in _processor_types.items()]

public_extra_types = {
    '.webm': 'video/webm',
}

encoders = timeside.core.processor.processors(timeside.core.api.IEncoder)
for encoder in encoders:
    public_extra_types['.' + encoder.file_extension()] = encoder.mime_type()

private_extra_types = {
    '.eaf': 'text/xml',  # ELAN Annotation Format
    '.trs': 'text/xml',  # Trancriber Annotation Format
    '.svl': 'text/xml',  # Sonic Visualiser layer file
    '.TextGrid': 'text/praat-textgrid',  # Praat TextGrid annotation file
    '.h5': 'application/x-hdf5',    # HDF5
    '.hdf5': 'application/x-hdf5',  # HDF5
}

for ext, mime_type in public_extra_types.items():
    if not mimetypes.guess_extension(mime_type) == ext:
        mimetypes.add_type(mime_type, ext)

for ext, mime_type in private_extra_types.items():
    mimetypes.add_type(mime_type, ext)


# Status
_FAILED, _DRAFT, _PENDING, _RUNNING, _DONE = 0, 1, 2, 3, 4
STATUS = ((_FAILED, _('failed')), (_DRAFT, _('draft')),
          (_PENDING, _('pending')), (_RUNNING, _('running')),
          (_DONE, _('done')))


RESULTS_ROOT = os.path.join(settings.MEDIA_ROOT, 'results')
if not os.path.exists(RESULTS_ROOT):
    os.makedirs(RESULTS_ROOT)

class UUIDEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, UUID):
            # if the obj is uuid, we simply return the value of uuid
            return obj.hex
        return json.JSONEncoder.default(self, obj)


def get_mime_type(path):
    return mimetypes.guess_type(path)[0]

#make every user having an automatically generated Token catching the User's post_save signal
@receiver(post_save, sender=settings.AUTH_USER_MODEL)
def create_auth_token(sender, instance=None, created=False, **kwargs):
    if created:
        Token.objects.create(user=instance)

# --- Abstract classes -----
class Dated(models.Model):

    date_added = models.DateTimeField(_('date added'), auto_now_add=True, null=True)
    date_modified = models.DateTimeField(_('date modified'), auto_now=True, null=True)

    class Meta:
        abstract = True


class UUID(models.Model):

    uuid = models.UUIDField(_('uuid'), default=uuid.uuid4, primary_key=True, blank=False, max_length=255, editable=False)
    #uuid = models.CharField(_('uuid'), unique=True, blank=True, max_length=255, editable=False)
    class Meta:
        abstract = True

    @classmethod
    def get_first_or_create(cls, **kwargs):
        """
        duplicates-compatible get or create method
        """
        objs = cls.objects.filter(**kwargs)
        if objs:
            obj = objs[0]
            created = False
        else:
            obj = cls(**kwargs)
            obj.save()
            created = True
        return obj, created


@python_2_unicode_compatible
class Titled(models.Model):

    title = models.CharField(_('title'), blank=True, max_length=512)
    description = models.TextField(_('description'), blank=True)

    def __str__(self):
        return self.title

    class Meta:
        abstract = True


@python_2_unicode_compatible
class Named(models.Model):

    name = models.CharField(_('name'), blank=True, max_length=512)
    description = models.TextField(_('description'), blank=True)

    def __str__(self):
        return str(self.name)

    class Meta:
        abstract = True


class Shareable(models.Model):

    author = models.ForeignKey(User, related_name="%(class)s", verbose_name=_('author'), blank=True, null=True, on_delete=models.SET_NULL)
    is_public = models.BooleanField(default=True)

    class Meta:
        abstract = True


@python_2_unicode_compatible
class Provider(Named, UUID):

    pid = models.CharField(_('pid'), blank=True, max_length=128)
    source_access =  models.BooleanField(default=False)

    def __str__(self):
        return str(self.pid)

    def get_provider(self):
        return timeside.core.get_provider(self.pid)

    def get_source_from_url(self, url, download=False):
        DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT,'items','download','')
        return self.get_provider()().get_source_from_url(url, DOWNLOAD_ROOT, download)

    def get_source_from_id(self, external_id, download=False):
        DOWNLOAD_ROOT = os.path.join(settings.MEDIA_ROOT,'items','download','')
        return self.get_provider()().get_source_from_id(external_id, DOWNLOAD_ROOT, download)

    def get_id_from_url(self, url):
        return self.get_provider()().get_id_from_url(url)

class Selection(Titled, UUID, Dated, Shareable):

    items = models.ManyToManyField('Item', related_name="selections", verbose_name=_('items'), blank=True)
    selections = models.ManyToManyField('Selection', related_name="other_selections", verbose_name=_('other selections'), blank=True)

    class Meta:
        verbose_name = _('selection')

    def get_all_items(self):
        qs_items = self.items.all()
        for selection in self.selections.all():
            qs_items |= selection.get_all_items()
        return qs_items


@python_2_unicode_compatible
class Item(Titled, UUID, Dated, Shareable):

    element_type = 'timeside_item'

    source_file = models.FileField(_('file'), upload_to='items/%Y/%m/%d', blank=True, max_length=1024)
    source_url = models.URLField(_('URL'), blank=True, max_length=1024)
    audio_duration = models.FloatField(_('duration'), blank=True, null=True)
    sha1 = models.CharField(_('sha1'), blank=True, max_length=512)
    mime_type = models.CharField(_('mime type'), blank=True, max_length=256)
    hdf5 = models.FileField(_('HDF5 result file'), upload_to='results/%Y/%m/%d', blank=True, max_length=1024)
    lock = models.BooleanField(default=False)
    external_uri = models.CharField(_('external_uri'), blank=True, max_length=1024)
    external_id = models.CharField(_('external_id'), blank=True, max_length=256)
    provider = models.ForeignKey('Provider', verbose_name=_('provider'), blank=True, null=True, on_delete=models.SET_NULL)

    class Meta:
        ordering = ['title']
        verbose_name = _('item')

    def __str__(self):
        return '_'.join([str(self.title), str(self.uuid)[:4]])

    def results(self):
        return [result for result in self.results.all()]

    def lock_setter(self, lock):
        self.lock = lock
        self.save()

    def get_source(self, download=False):
        if not (self.source_url or self.source_file) and self.provider:
            if download:
                if self.external_uri:
                    self.source_file = self.provider.get_source_from_url(self.external_uri,download).replace(settings.MEDIA_ROOT, '')
                elif self.external_id:
                    self.source_file = self.provider.get_source_from_id(self.external_id,download).replace(settings.MEDIA_ROOT, '')
            else:
                if self.external_uri:
                    self.source_url = self.provider.get_source_from_url(self.external_uri,download)
                elif self.external_id:
                    self.source_url = self.provider.get_source_from_id(self.external_id,download)
            super(Item, self).save()

    def get_external_id(self):
        if not (self.source_url or self.external_id) and self.external_uri:
            self.external_id = self.provider.get_id_from_url(self.external_uri)
            super(Item, self).save()

    def get_uri(self):
        """Return the Item source"""
        if self.source_file and os.path.exists(self.source_file.path):
            return self.source_file.path
        elif self.source_url:
            return self.source_url
        return None

    def get_results_path(self):
        """
        Return Item result path
        """
        result_path = os.path.join(RESULTS_ROOT, str(self.uuid))
        if not os.path.exists(result_path):
            os.makedirs(result_path)
        return result_path

    def get_audio_duration(self, force=False):
        """
        Return item audio duration
        """
        if (force or not self.audio_duration) and self.source_file:
            decoder = timeside.core.get_processor('file_decoder')(
                uri=self.get_uri())
            self.audio_duration = decoder.uri_total_duration
            super(Item, self).save()

    def get_hash(self, force=False):
        "Set SHA1 hash from file binary content"
        if force or (self.sha1 is None):
            if self.source_file:
                sha1 = sha1sum_file(self.source_file.path)
            elif self.source_url:
                sha1 = sha1sum_url(self.source_url)
            else:
                return
            self.sha1 = sha1
            super(Item, self).save()

    def get_mimetype(self, force=False):
        if force or (self.mime_type is None):
            if self.source_url:
                path = self.source_url
            elif self.source_file:
                path = self.source_file.path
            if os.path.exists(path):
                self.mime_type = get_mime_type(path)
            super(Item, self).save()

    def get_single_selection(self):
        # TODO : have singleton selection has a foreign key of Item ???
        sel_title = "Singleton selection for item %d" % self.uuid
        sel_description = ("Singleton selection for item %d\n"
                           "Automatically generated by the TimeSide "
                           "application.") % self.uuid
        selection, created = Selection.objects.get_or_create(
            title=sel_title,
            description=sel_description)
        if created:
            selection.save()
            selection.items.add(self)
        return selection

    def run(self, experience):
        result_path = self.get_results_path()
        # get audio source
        uri = self.get_uri()

        # decode audio source
        decoder = timeside.plugins.decoder.file.FileDecoder(uri=uri,
                                                            sha1=self.sha1)
        presets = {}
        pipe = decoder
        parent_analyzers = []

        # search for parent analyzer presets not to add as duplicates in the pipe
        for preset in experience.presets.all():
            proc = preset.processor.get_processor()
            if proc.type in ['analyzer', 'grapher']:
                proc = proc(**json.loads(preset.parameters))
                if 'analyzer' in proc.parents:
                    parent_analyzers.append(proc.parents['analyzer'])

        for preset in experience.presets.all():
            # get core audio metaProcessor corresponding to preset.processor.pid
            proc = preset.processor.get_processor()
            if proc.type == 'encoder':
                result, c = Result.objects.get_or_create(preset=preset,
                                                         item=self)
                media_file = '.'.join([str(result.uuid),
                                       proc.file_extension()])
                result.file = os.path.join(result_path, media_file).replace(settings.MEDIA_ROOT, '')
                result.save()
                # instantiate a core processor of an encoder
                proc = proc(result.file.path, overwrite=True,
                            streaming=False)
                worker_logger.info(f'Run {str(proc)} on {str(self)}')
            elif proc.type in ['analyzer', 'grapher']:
                # instantiate a core processor of an analyzer or a grapher
                proc = proc(**json.loads(preset.parameters))
                worker_logger.info(f'Run {str(proc)} on {str(self)} with {str(preset.parameters)}')

            if proc not in parent_analyzers:
                presets[preset] = proc
                pipe |= proc

        # item.lock_setter(True)

        if not self.hdf5:
            hdf5_file = str(experience.uuid) + '.hdf5'
            self.hdf5 = os.path.join(result_path, hdf5_file).replace(settings.MEDIA_ROOT, '')
            self.save()

        pipe.run()

        def set_results_from_processor(proc, preset=None):
            if preset:
                parameters = preset.parameters
            elif preset is None:
                processor, c = Processor.get_first_or_create(pid=proc.id())
                parameters = json.dumps(processor.get_parameters_default())
                preset, c = Preset.get_first_or_create(processor=processor,
                                                    parameters=parameters)
            result, c = Result.objects.get_or_create(preset=preset,
                                                    item=self)


            if not hasattr(proc, 'external'):
                hdf5_file = str(result.uuid) + '.hdf5'
                result.hdf5 = os.path.join(result_path, hdf5_file).replace(settings.MEDIA_ROOT, '')
                # while result.lock:
{'fraction_non_alphanumeric': 0.10708790309379604, 'fraction_numerical': 0.007366181044360779, 'mean_word_length': 2.6661065770523282, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1430283', 'n_tokens_mistral': 9224, 'n_tokens_neox': 8819, 'n_words': 2090}