<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Defines Module base class and implementations.

Modules are data processing building blocks for creating datasets.
"""


import random

# internal imports
import numpy as np
import tensorflow as tf

from magenta.lib import melodies_lib
from magenta.lib import sequences_lib
from magenta.pipelines import pipeline
from magenta.pipelines import statistics
from magenta.protobuf import music_pb2


class Quantizer(pipeline.Pipeline):
  """A Module that quantizes NoteSequence data."""

  def __init__(self, steps_per_beat=4):
    super(Quantizer, self).__init__(
        input_type=music_pb2.NoteSequence,
        output_type=sequences_lib.QuantizedSequence)
    self.steps_per_beat = steps_per_beat

  def transform(self, note_sequence):
    quantized_sequence = sequences_lib.QuantizedSequence()
    try:
      quantized_sequence.from_note_sequence(note_sequence, self.steps_per_beat)
      return [quantized_sequence]
    except sequences_lib.MultipleTimeSignatureException:
      tf.logging.debug('Multiple time signatures found in NoteSequence')
      return []


class MonophonicMelodyExtractor(pipeline.Pipeline):
  """Extracts monophonic melodies from a QuantizedSequence."""

  def __init__(self, min_bars=7, min_unique_pitches=5, gap_bars=1.0,
               ignore_polyphonic_notes=False):
    super(MonophonicMelodyExtractor, self).__init__(
        input_type=sequences_lib.QuantizedSequence,
        output_type=melodies_lib.MonophonicMelody)
    self.min_bars = min_bars
    self.min_unique_pitches = min_unique_pitches
    self.gap_bars = gap_bars
    self.ignore_polyphonic_notes = False

  def transform(self, quantized_sequence):
    try:
      melodies, stats = melodies_lib.extract_melodies(
          quantized_sequence,
          min_bars=self.min_bars,
          min_unique_pitches=self.min_unique_pitches,
          gap_bars=self.gap_bars,
          ignore_polyphonic_notes=self.ignore_polyphonic_notes)
    except melodies_lib.NonIntegerStepsPerBarException as detail:
      tf.logging.warning('Skipped sequence: %s', detail)
      melodies = []
      stats = [statistics.Counter('non_integer_steps_per_bar', 1)]
    self._set_stats(stats)<commit_msg>Quantizer counts sequences with multiple time signatures (#181)<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Defines Module base class and implementations.

Modules are data processing building blocks for creating datasets.
"""


import random

# internal imports
import numpy as np
import tensorflow as tf

from magenta.lib import melodies_lib
from magenta.lib import sequences_lib
from magenta.pipelines import pipeline
from magenta.pipelines import statistics
from magenta.protobuf import music_pb2


class Quantizer(pipeline.Pipeline):
  """A Module that quantizes NoteSequence data."""

  def __init__(self, steps_per_beat=4):
    super(Quantizer, self).__init__(
        input_type=music_pb2.NoteSequence,
        output_type=sequences_lib.QuantizedSequence)
    self.steps_per_beat = steps_per_beat

  def transform(self, note_sequence):
    quantized_sequence = sequences_lib.QuantizedSequence()
    try:
      quantized_sequence.from_note_sequence(note_sequence, self.steps_per_beat)
      return [quantized_sequence]
    except sequences_lib.MultipleTimeSignatureException:
      tf.logging.debug('Multiple time signatures found in NoteSequence')
      self._set_stats([statistics.Counter(
          'sequences_discarded_because_multiple_time_signatures', 1)])
      return []


class MonophonicMelodyExtractor(pipeline.Pipeline):
  """Extracts monophonic melodies from a QuantizedSequence."""

  def __init__(self, min_bars=7, min_unique_pitches=5, gap_bars=1.0,
               ignore_polyphonic_notes=False):
    super(MonophonicMelodyExtractor, self).__init__(
        input_type=sequences_lib.QuantizedSequence,
        output_type=melodies_lib.MonophonicMelody)
    self.min_bars = min_bars
    self.min_unique_pitches = min_unique_pitches
    self.gap_bars = gap_bars
    self.ignore_polyphonic_notes = False

  def transform(self, quantized_sequence):
    try:
      melodies, stats = melodies_lib.extract_melodies(
          quantized_sequence,
          min_bars=self.min_bars,
          min_unique_pitches=self.min_unique_pitches,
          gap_bars=self.gap_bars,
          ignore_polyphonic_notes=self.ignore_polyphonic_notes)
    except melodies_lib.NonIntegerStepsPerBarException as detail:
      tf.logging.warning('Skipped sequence: %s', detail)
      melodies = []
      stats = [statistics.Counter('non_integer_steps_per_bar', 1)]
    self._set_stats(stats)
{'fraction_non_alphanumeric': 0.0724991286162426, 'fraction_numerical': 0.006273963053328686, 'mean_word_length': 4.348555452003728, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1197234', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1616, 'n_words': 449}