<commit_before># Copyright 2020 The Magenta Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Beam pipeline for MAESTRO dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import hashlib
import os

import apache_beam as beam
from apache_beam.metrics import Metrics

from magenta.models.onsets_frames_transcription import audio_label_data_utils
from magenta.models.onsets_frames_transcription import create_dataset_lib
from magenta.models.onsets_frames_transcription import data
from magenta.music import audio_io
from magenta.music.protobuf import music_pb2

import tensorflow.compat.v1 as tf

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('output_directory', None, 'Path to output_directory')
tf.app.flags.DEFINE_integer('min_length', 5, 'minimum length for a segment')
tf.app.flags.DEFINE_integer('max_length', 20, 'maximum length for a segment')
tf.app.flags.DEFINE_integer('sample_rate', 16000,
                            'sample_rate of the output files')
tf.app.flags.DEFINE_boolean('preprocess_examples', False,
                            'Whether to preprocess examples.')
tf.app.flags.DEFINE_integer(
    'preprocess_train_example_multiplier', 1,
    'How many times to run data preprocessing on each training example. '
    'Useful if preprocessing involves a stochastic process that is useful to '
    'sample multiple times.')
tf.app.flags.DEFINE_string('config', 'onsets_frames',
                           'Name of the config to use.')
tf.app.flags.DEFINE_string('dataset_config', 'maestro',
                           'Name of the dataset config to use.')
tf.app.flags.DEFINE_string(
    'hparams', '',
    'A comma-separated list of `name=value` hyperparameter values.')
tf.app.flags.DEFINE_string(
    'pipeline_options', '--runner=DirectRunner',
    'Command line flags to use in constructing the Beam pipeline options.')
tf.app.flags.DEFINE_boolean(
    'load_audio_with_librosa', False,
    'Whether to use librosa for sampling audio')


def split_wav(input_example, min_length, max_length, sample_rate,
              debug_output_directory, split_example, load_audio_with_librosa):
  """Splits wav and midi files for the dataset."""
  tf.logging.info('Splitting %s',
                  input_example.features.feature['id'].bytes_list.value[0])

  wav_data = input_example.features.feature['audio'].bytes_list.value[0]

  ns = music_pb2.NoteSequence.FromString(
      input_example.features.feature['sequence'].bytes_list.value[0])

  Metrics.counter('split_wav', 'read_midi_wav_to_split').inc()

  if not split_example:
    split_examples = audio_label_data_utils.process_record(
        wav_data,
        ns,
        ns.id,
        min_length=0,
        max_length=-1,
        sample_rate=sample_rate,
        load_audio_with_librosa=load_audio_with_librosa)

    for example in split_examples:
      Metrics.counter('split_wav', 'full_example').inc()
      yield example
  else:
    try:
      split_examples = audio_label_data_utils.process_record(
          wav_data,
          ns,
          ns.id,
          min_length=min_length,
          max_length=max_length,
          sample_rate=sample_rate,
          load_audio_with_librosa=load_audio_with_librosa)

      for example in split_examples:
        Metrics.counter('split_wav', 'split_example').inc()
        yield example
    except AssertionError:
      output_file = 'badexample-' + hashlib.md5(ns.id).hexdigest() + '.proto'
      output_path = os.path.join(debug_output_directory, output_file)
      tf.logging.error('Exception processing %s. Writing file to %s', ns.id,
                       output_path)<commit_msg>allow empty sequences when not splitting. useful for creating inference-only datasets.<commit_after># Copyright 2020 The Magenta Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Beam pipeline for MAESTRO dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import hashlib
import os

import apache_beam as beam
from apache_beam.metrics import Metrics

from magenta.models.onsets_frames_transcription import audio_label_data_utils
from magenta.models.onsets_frames_transcription import create_dataset_lib
from magenta.models.onsets_frames_transcription import data
from magenta.music import audio_io
from magenta.music.protobuf import music_pb2

import tensorflow.compat.v1 as tf

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('output_directory', None, 'Path to output_directory')
tf.app.flags.DEFINE_integer('min_length', 5, 'minimum length for a segment')
tf.app.flags.DEFINE_integer('max_length', 20, 'maximum length for a segment')
tf.app.flags.DEFINE_integer('sample_rate', 16000,
                            'sample_rate of the output files')
tf.app.flags.DEFINE_boolean('preprocess_examples', False,
                            'Whether to preprocess examples.')
tf.app.flags.DEFINE_integer(
    'preprocess_train_example_multiplier', 1,
    'How many times to run data preprocessing on each training example. '
    'Useful if preprocessing involves a stochastic process that is useful to '
    'sample multiple times.')
tf.app.flags.DEFINE_string('config', 'onsets_frames',
                           'Name of the config to use.')
tf.app.flags.DEFINE_string('dataset_config', 'maestro',
                           'Name of the dataset config to use.')
tf.app.flags.DEFINE_string(
    'hparams', '',
    'A comma-separated list of `name=value` hyperparameter values.')
tf.app.flags.DEFINE_string(
    'pipeline_options', '--runner=DirectRunner',
    'Command line flags to use in constructing the Beam pipeline options.')
tf.app.flags.DEFINE_boolean(
    'load_audio_with_librosa', False,
    'Whether to use librosa for sampling audio')


def split_wav(input_example, min_length, max_length, sample_rate,
              debug_output_directory, split_example, load_audio_with_librosa):
  """Splits wav and midi files for the dataset."""
  tf.logging.info('Splitting %s',
                  input_example.features.feature['id'].bytes_list.value[0])

  wav_data = input_example.features.feature['audio'].bytes_list.value[0]

  ns = music_pb2.NoteSequence.FromString(
      input_example.features.feature['sequence'].bytes_list.value[0])

  Metrics.counter('split_wav', 'read_midi_wav_to_split').inc()

  if not split_example:
    split_examples = audio_label_data_utils.process_record(
        wav_data,
        ns,
        ns.id,
        min_length=0,
        max_length=-1,
        sample_rate=sample_rate,
        allow_empty_notesequence=True,
        load_audio_with_librosa=load_audio_with_librosa)

    for example in split_examples:
      Metrics.counter('split_wav', 'full_example').inc()
      yield example
  else:
    try:
      split_examples = audio_label_data_utils.process_record(
          wav_data,
          ns,
          ns.id,
          min_length=min_length,
          max_length=max_length,
          sample_rate=sample_rate,
          load_audio_with_librosa=load_audio_with_librosa)

      for example in split_examples:
        Metrics.counter('split_wav', 'split_example').inc()
        yield example
    except AssertionError:
      output_file = 'badexample-' + hashlib.md5(ns.id).hexdigest() + '.proto'
      output_path = os.path.join(debug_output_directory, output_file)
      tf.logging.error('Exception processing %s. Writing file to %s', ns.id,
                       output_path)
{'fraction_non_alphanumeric': 0.08884443398446694, 'fraction_numerical': 0.006119086843963286, 'mean_word_length': 3.6065040650406504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6615352', 'n_tokens_mistral': 2569, 'n_tokens_neox': 2456, 'n_words': 674}