<commit_before>import json
import urllib2

import compmusic.dunya.conn
import compmusic.dunya.docserver
import compmusic.extractors
import pydub

from symbtrsynthesis.adaptivesynthesizer import AdaptiveSynthesizer
from symbtrsynthesis.musicxmlreader import MusicXMLReader


class ScoreSynthesis(compmusic.extractors.ExtractorModule):
    _version = "0.1"
    _sourcetype = "symbtrxml"
    _slug = "synthesis"

    def set_settings(self, recid='aeu'):
        self._output = {recid: {'mp3': {"extension": "mp3",
                                        "mimetype": "audio/mp3"},
                                'onsets': {"extension": "json",
                                           "mimetype": "application/json"}}
                        }

    def set_parameters(self, measures, bpm, stablenotes, tnc_sym):
        self.measures = measures
        self.bpm = bpm
        self.stablenotes = stablenotes
        self.tnc_sym = tnc_sym

    def run(self, musicbrainzid, fname):
        try:
            # synthesize
            audio_karplus, onsets_karplus = AdaptiveSynthesizer.synth_from_tuning(
                measures=self.measures, bpm=self.bpm,
                stable_notes=self.stablenotes, tonic_sym=self.tnc_sym,
                synth_type='karplus', verbose=True)

            # mp3 conversion
            audio_obj = pydub.AudioSegment(data=audio_karplus)
            audio_mp3 = audio_obj.export()

            return {musicbrainzid: {'mp3': audio_mp3,
                                    'onsets': onsets_karplus}
                    }

        except compmusic.dunya.conn.HTTPError:
            print(workid, 'is not exist')


token = ''
compmusic.dunya.conn.set_token(token)

# fetch symbtrs
symbtrs = compmusic.dunya.get_collection('makam-symbtr')['documents']

response = urllib2.urlopen('https://raw.githubusercontent.com/MTG/otmm_tuning_intonation_dataset/master/dataset.json')
dataset = json.loads(response.read())
MAKAMS = dataset.keys()


synthesizer = ScoreSynthesis()

for xx, symbtr in enumerate(symbtrs):
    workid = symbtr['external_identifier']

    try:
        # get metadata
        metadata = json.loads(compmusic.dunya.docserver.file_for_document(
            workid, 'scoreanalysis', 'metadata'))
        makam = metadata['makam']['attribute_key']

        # fetch score
        musicxml = compmusic.dunya.docserver.file_for_document(
            recordingid=workid, thetype='score', subtype='xmlscore')

        # read musicxml
        (measures, makam, usul, form, time_sigs, keysig, work_title,
         composer, lyricist, bpm, tnc_sym) = MusicXMLReader.read(musicxml)

        # check if it is in selected makams
        if makam in MAKAMS:
            recids = dataset[makam]
            for recid in recids:
                stablenotes = json.loads(compmusic.dunya.file_for_document(
                    recid, 'audioanalysis', 'note_models'))

                synthesizer.set_settings(recid)
                synthesizer.set_parameters(measures=measures, bpm=bpm,
                                           stablenotes=stablenotes,
                                           tnc_sym=tnc_sym)
                print(xx, makam, recid)
                synthesizer.run(recid, None)
        print('aeu', makam)

        synthesizer.set_settings()
        synthesizer.set_parameters(measures=measures, bpm=bpm,
                                   stablenotes=None, tnc_sym=tnc_sym)
        synthesizer.run('aeu', None)

    except compmusic.dunya.conn.HTTPError:
        print(workid, 'is not exist')<commit_msg>Move extractor code as script<commit_after>import json
import urllib2

import compmusic.dunya.conn
import compmusic.dunya.docserver
import compmusic.extractors
import pydub

from symbtrsynthesis.adaptivesynthesizer import AdaptiveSynthesizer
from symbtrsynthesis.musicxmlreader import MusicXMLReader


class ScoreSynthesis(compmusic.extractors.ExtractorModule):
    _version = "0.1"
    _sourcetype = "symbtrxml"
    _slug = "synthesis"

    def set_settings(self, recid='aeu'):
        self._output = {recid: {'mp3': {"extension": "mp3",
                                        "mimetype": "audio/mp3"},
                                'onsets': {"extension": "json",
                                           "mimetype": "application/json"}}
                        }

    def set_parameters(self, measures, bpm, stablenotes, tnc_sym):
        self.measures = measures
        self.bpm = bpm
        self.stablenotes = stablenotes
        self.tnc_sym = tnc_sym

    def run(self, musicbrainzid, fname):
        try:
            # synthesize
            audio_karplus, onsets_karplus = AdaptiveSynthesizer.synth_from_tuning(
                measures=self.measures, bpm=self.bpm,
                stable_notes=self.stablenotes, tonic_sym=self.tnc_sym,
                synth_type='karplus', verbose=True)

            # mp3 conversion
            audio_obj = pydub.AudioSegment(data=audio_karplus)
            audio_mp3 = audio_obj.export()

            return {musicbrainzid: {'mp3': audio_mp3,
                                    'onsets': onsets_karplus}
                    }

        except compmusic.dunya.conn.HTTPError:
            print(workid, 'is not exist')

if __name__ == "__main__":
    token = ''
    compmusic.dunya.conn.set_token(token)

    # fetch symbtrs
    symbtrs = compmusic.dunya.get_collection('makam-symbtr')['documents']

    response = urllib2.urlopen('https://raw.githubusercontent.com/MTG/otmm_tuning_intonation_dataset/master/dataset.json')
    dataset = json.loads(response.read())
    MAKAMS = dataset.keys()


    synthesizer = ScoreSynthesis()

    for xx, symbtr in enumerate(symbtrs):
        workid = symbtr['external_identifier']

        try:
            # get metadata
            metadata = json.loads(compmusic.dunya.docserver.file_for_document(
                workid, 'scoreanalysis', 'metadata'))
            makam = metadata['makam']['attribute_key']

            # fetch score
            musicxml = compmusic.dunya.docserver.file_for_document(
                recordingid=workid, thetype='score', subtype='xmlscore')

            # read musicxml
            (measures, makam, usul, form, time_sigs, keysig, work_title,
             composer, lyricist, bpm, tnc_sym) = MusicXMLReader.read(musicxml)

            # check if it is in selected makams
            if makam in MAKAMS:
                recids = dataset[makam]
                for recid in recids:
                    stablenotes = json.loads(compmusic.dunya.file_for_document(
                        recid, 'audioanalysis', 'note_models'))

                    synthesizer.set_settings(recid)
                    synthesizer.set_parameters(measures=measures, bpm=bpm,
                                               stablenotes=stablenotes,
                                               tnc_sym=tnc_sym)
                    print(xx, makam, recid)
                    synthesizer.run(recid, None)
            print('aeu', makam)

            synthesizer.set_settings()
            synthesizer.set_parameters(measures=measures, bpm=bpm,
                                       stablenotes=None, tnc_sym=tnc_sym)
            synthesizer.run('aeu', None)

        except compmusic.dunya.conn.HTTPError:
            print(workid, 'is not exist')
{'fraction_non_alphanumeric': 0.09438356164383561, 'fraction_numerical': 0.0030136986301369864, 'mean_word_length': 2.029460580912863, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1449835', 'n_tokens_mistral': 2299, 'n_tokens_neox': 2165, 'n_words': 384}