<commit_before># -*- coding: utf-8 -*-
# Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/
import json
import musicbrainzngs

from dashboard.log import logger
from dashboard import release_importer
import carnatic.models

import compmusic
from compmusic import mb

def remove_deleted_items():
    """ Search musicbrainz for all items in the database and if they
    have been deleted then remove them """

    logger.info("Scanning works...")
    for w in carnatic.models.Work.objects.all():
        try:
            mb.get_work_by_id(w.mbid)
        except mb.ResponseError:
            logger.info("work %s (%s) missing; deleting" % (w, w.mbid))
            w.delete()

    logger.info("Scanning recordings...")
    for r in carnatic.models.Recording.objects.all():
        try:
            mb.get_recording_by_id(r.mbid)
        except mb.ResponseError:
            logger.info("recording %s (%s) missing; deleting" % (r, r.mbid))
            r.delete()

    logger.info("Scanning concerts...")
    for c in carnatic.models.Concert.objects.all():
        try:
            mb.get_release_by_id(c.mbid)
        except mb.ResponseError:
            logger.info("release %s (%s) missing; deleting" % (c, c.mbid))
            c.delete()

    logger.info("Scanning artists...")
    for a in carnatic.models.Artist.objects.all():
        try:
            mb.get_artist_by_id(a.mbid)
        except mb.ResponseError:
            # We import dummy artists to be gurus, leave them here
            if not a.dummy:
                logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
                a.delete()

    logger.info("Scanning composers...")
    for a in carnatic.models.Composer.objects.all():
        try:
            mb.get_artist_by_id(a.mbid)
        except mb.ResponseError:
            logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
            a.delete()

class CarnaticReleaseImporter(release_importer.ReleaseImporter):
    _ArtistClass = carnatic.models.Artist
    _ArtistAliasClass = carnatic.models.ArtistAlias
    _ComposerClass = carnatic.models.Composer
    _ComposerAliasClass = carnatic.models.ComposerAlias
    _ReleaseClass = carnatic.models.Concert
    _RecordingClass = carnatic.models.Recording
    _InstrumentClass = carnatic.models.Instrument
    _WorkClass = carnatic.models.Work

    def _link_release_recording(self, concert, recording, trackorder, mnum, tnum):
        if not concert.recordings.filter(pk=recording.pk).exists():
            carnatic.models.ConcertRecording.objects.create(
                concert=concert, recording=recording, track=trackorder, disc=mnum, disctrack=tnum)

    def _add_work_attributes(self, work, mbwork, created):
        """ Read raaga and taala attributes from the webservice query
        and add them to the object """
        if created or self.overwrite:
            raaga_attr = self._get_raaga_mb(mbwork)
            taala_attr = self._get_taala_mb(mbwork)
            raaga = self._get_raaga(raaga_attr)
            taala = self._get_taala(taala_attr)
            if raaga:
                work.raaga = raaga
            if taala:
                work.taala = taala
            work.save()

    def _join_recording_and_works(self, recording, works):
        # A carnatic recording only has many works.
        for i, w in list(enumerate(works)):
            carnatic.models.RecordingWork.objects.create(work=w, recording=recording, sequence=i)

    def _apply_tags(self, recording, works, tags):
        if self.overwrite:
            recording.forms.clear()
            recording.raagas.clear()
            recording.taalas.clear()

        # Check that there is not more than one form
        forms = recording.forms.all()
        if len(forms) > 1:
            raise release_importer.ImportFailedException('TODO: Support more than one form per recording')
        elif len(forms) < 1:
            form = self._get_form_tag(tags)
            if form:
                # TODO: If there is more than one form, set sequence properly
                carnatic.models.RecordingForm.objects.create(recording=recording, form=form, sequence=1)
            else:
                raise release_importer.ImportFailedException("The recording doesn't have a form")
        else:
            form = forms[0]

        if form.attrfromrecording:
            # Create the relation with Raaga and Taala

            raaga_tag = self._get_raaga_tags(tags)
            taala_tag = self._get_taala_tags(tags)

            if raaga_tag:
                r = self._get_raaga(raaga_tag)
                carnatic.models.RecordingRaaga.objects.create(recording=recording, raaga=r, sequence=1)

            if taala_tag:
                t = self._get_taala(taala_tag)
                carnatic.models.RecordingTaala.objects.create(recording=recording, taala=t, sequence=1)


    def _get_form_tag(self, tags):
        for t in tags:
            name = t["name"].lower()
            if compmusic.tags.has_carnatic_form(name):
                form_tag = compmusic.tags.parse_carnatic_form(name)
                # forms are returned in (num, tag) - for now we just<commit_msg>import even if we have no form<commit_after># -*- coding: utf-8 -*-
# Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
#
# This file is part of Dunya
#
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/
import json
import musicbrainzngs

from dashboard.log import logger
from dashboard import release_importer
import carnatic.models

import compmusic
from compmusic import mb

def remove_deleted_items():
    """ Search musicbrainz for all items in the database and if they
    have been deleted then remove them """

    logger.info("Scanning works...")
    for w in carnatic.models.Work.objects.all():
        try:
            mb.get_work_by_id(w.mbid)
        except mb.ResponseError:
            logger.info("work %s (%s) missing; deleting" % (w, w.mbid))
            w.delete()

    logger.info("Scanning recordings...")
    for r in carnatic.models.Recording.objects.all():
        try:
            mb.get_recording_by_id(r.mbid)
        except mb.ResponseError:
            logger.info("recording %s (%s) missing; deleting" % (r, r.mbid))
            r.delete()

    logger.info("Scanning concerts...")
    for c in carnatic.models.Concert.objects.all():
        try:
            mb.get_release_by_id(c.mbid)
        except mb.ResponseError:
            logger.info("release %s (%s) missing; deleting" % (c, c.mbid))
            c.delete()

    logger.info("Scanning artists...")
    for a in carnatic.models.Artist.objects.all():
        try:
            mb.get_artist_by_id(a.mbid)
        except mb.ResponseError:
            # We import dummy artists to be gurus, leave them here
            if not a.dummy:
                logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
                a.delete()

    logger.info("Scanning composers...")
    for a in carnatic.models.Composer.objects.all():
        try:
            mb.get_artist_by_id(a.mbid)
        except mb.ResponseError:
            logger.info("artist %s (%s) missing; deleting" % (a, a.mbid))
            a.delete()

class CarnaticReleaseImporter(release_importer.ReleaseImporter):
    _ArtistClass = carnatic.models.Artist
    _ArtistAliasClass = carnatic.models.ArtistAlias
    _ComposerClass = carnatic.models.Composer
    _ComposerAliasClass = carnatic.models.ComposerAlias
    _ReleaseClass = carnatic.models.Concert
    _RecordingClass = carnatic.models.Recording
    _InstrumentClass = carnatic.models.Instrument
    _WorkClass = carnatic.models.Work

    def _link_release_recording(self, concert, recording, trackorder, mnum, tnum):
        if not concert.recordings.filter(pk=recording.pk).exists():
            carnatic.models.ConcertRecording.objects.create(
                concert=concert, recording=recording, track=trackorder, disc=mnum, disctrack=tnum)

    def _add_work_attributes(self, work, mbwork, created):
        """ Read raaga and taala attributes from the webservice query
        and add them to the object """
        if created or self.overwrite:
            raaga_attr = self._get_raaga_mb(mbwork)
            taala_attr = self._get_taala_mb(mbwork)
            raaga = self._get_raaga(raaga_attr)
            taala = self._get_taala(taala_attr)
            if raaga:
                work.raaga = raaga
            if taala:
                work.taala = taala
            work.save()

    def _join_recording_and_works(self, recording, works):
        # A carnatic recording only has many works.
        for i, w in list(enumerate(works)):
            carnatic.models.RecordingWork.objects.create(work=w, recording=recording, sequence=i)

    def _apply_tags(self, recording, works, tags):
        if self.overwrite:
            recording.forms.clear()
            recording.raagas.clear()
            recording.taalas.clear()

        # Check that there is not more than one form
        noform = False
        forms = recording.forms.all()
        if len(forms) > 1:
            raise release_importer.ImportFailedException('TODO: Support more than one form per recording')
        elif len(forms) < 1:
            form = self._get_form_tag(tags)
            if form:
                # TODO: If there is more than one form, set sequence properly
                carnatic.models.RecordingForm.objects.create(recording=recording, form=form, sequence=1)
            else:
                # If we have no form, we import anyway and put the tags on the recording
                noform = True
        else:
            form = forms[0]

        if noform or form.attrfromrecording:
            # Create the relation with Raaga and Taala

            raaga_tag = self._get_raaga_tags(tags)
            taala_tag = self._get_taala_tags(tags)

            if raaga_tag:
                r = self._get_raaga(raaga_tag)
                carnatic.models.RecordingRaaga.objects.create(recording=recording, raaga=r, sequence=1)

            if taala_tag:
                t = self._get_taala(taala_tag)
                carnatic.models.RecordingTaala.objects.create(recording=recording, taala=t, sequence=1)
        elif not form.attrfromrecording:
            # In this case, we read attributes from the work. If the work has no attributes
            # we should add these tags to the recording anyway.
            pass


    def _get_form_tag(self, tags):
        for t in tags:
            name = t["name"].lower()
            if compmusic.tags.has_carnatic_form(name):
                form_tag = compmusic.tags.parse_carnatic_form(name)
                # forms are returned in (num, tag) - for now we just
{'fraction_non_alphanumeric': 0.08931112413158115, 'fraction_numerical': 0.0026784966937306435, 'mean_word_length': 2.4001138303927148, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2243672', 'n_tokens_mistral': 3566, 'n_tokens_neox': 3425, 'n_words': 983}