<commit_before># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
# 
# This file is part of Dunya
# 
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
# 
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/

from __future__ import division
from os.path import basename
import numpy as np
import pickle
import pypeaks
from glob import glob
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter


class Raaga:
    def __init__(self, name, path_to_raaga_profiles):
        self.name = name
        self.average_hist = None
        self.average_profile = None
        self.path_to_raaga_profiles = path_to_raaga_profiles

    @staticmethod
    def kldiv(x, y):
        """
        Calculates the KL divergence D(A||B) between the
        normalized distributions A and B.

        Usage: d = kldiv(A,B)
        """
        eps = np.finfo(float).eps*2*len(x)
        x = np.array(x)+eps
        y = np.array(y)+eps
        #eps = 0.01
        #if max(sum(A), sum(B)) > 1+eps:
        #   print max(sum(A), sum(B))
        #   print "Probabilities do not sum to 1."
        #   return np.NaN
        if x.size != y.size:
            print "Arguments are of different length."
            return np.NaN
        return (np.dot(x, np.log2(x)-np.log2(y))+np.dot(y, np.log2(y)-np.log2(x)))/2.0

    def load_average_hist(self):
        self.average_hist = pickle.load(file(self.path_to_raaga_profiles.rstrip("/") + "/" + self.name + ".pickle"))

    def compute_average_hist(self, paths_to_pitch_files, tonics, octave_folded=True, bins=1200):
        """
        Computes average histogram for a list of given pitch files.
        The pitch values are expected to be in hertz.
        """
        all_cents = []
        for i in xrange(len(paths_to_pitch_files)):
            filepath = paths_to_pitch_files[i]
            data = np.loadtxt(filepath)

            if data is None:
                raise Exception("Pitch data not received.")

            #Histogram
            valid_pitch = data[:, 1]
            tonic = tonics[i]
            valid_pitch = [1200*np.log2(i/tonic) for i in valid_pitch if i > 0]
            if octave_folded:
                valid_pitch = map(lambda x: int(x % 1200), valid_pitch)

            all_cents.extend(valid_pitch)
        if not bins:
            bins = max(all_cents)-min(all_cents)

        n, bin_edges = np.histogram(all_cents, bins, normed=True)
        bin_centers = 0.5*(bin_edges[1:]+bin_edges[:-1])
        n = n.reshape(len(n), 1)
        bin_centers = bin_centers.reshape(len(bin_centers), 1)
        self.average_hist = np.append(n, bin_centers, axis=1)

    @staticmethod
    def find_nearest_index(arr, value):<commit_msg>separate file reading from mean calculation using data<commit_after># Copyright 2013,2014 Music Technology Group - Universitat Pompeu Fabra
# 
# This file is part of Dunya
# 
# Dunya is free software: you can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free Software
# Foundation (FSF), either version 3 of the License, or (at your option) any later
# version.
# 
# This program is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along with
# this program.  If not, see http://www.gnu.org/licenses/

from __future__ import division
from os.path import basename
import numpy as np
import pickle
import pypeaks
from glob import glob
import matplotlib.pyplot as plt
from scipy.ndimage.filters import gaussian_filter


class Raaga:
    def __init__(self, name, path_to_raaga_profiles):
        self.name = name
        self.average_hist = None
        self.average_profile = None
        self.path_to_raaga_profiles = path_to_raaga_profiles

    @staticmethod
    def kldiv(x, y):
        """
        Calculates the KL divergence D(A||B) between the
        normalized distributions A and B.

        Usage: d = kldiv(A,B)
        """
        eps = np.finfo(float).eps*2*len(x)
        x = np.array(x)+eps
        y = np.array(y)+eps
        #eps = 0.01
        #if max(sum(A), sum(B)) > 1+eps:
        #   print max(sum(A), sum(B))
        #   print "Probabilities do not sum to 1."
        #   return np.NaN
        if x.size != y.size:
            print "Arguments are of different length."
            return np.NaN
        return (np.dot(x, np.log2(x)-np.log2(y))+np.dot(y, np.log2(y)-np.log2(x)))/2.0

    def load_average_hist(self):
        self.average_hist = pickle.load(file(self.path_to_raaga_profiles.rstrip("/") + "/" + self.name + ".pickle"))

    def compute_average_hist(self, paths_to_pitch_files, tonics, octave_folded=True, bins=1200):
        pitches = []
        for i in xrange(len(paths_to_pitch_files)):
            filepath = paths_to_pitch_files[i]
            data = np.loadtxt(filepath)

            if data is None:
                raise Exception("Pitch data not received.")
            pitches.append(data)

        compute_average_hist_data(pitches, tonics, octave_folded, bins)


    def compute_average_hist_data(self, pitches, tonics, octave_folded=True, bins=1200):
        """
        Computes average histogram for a list of given pitches.
        Pitch data should be a numpy array. `tonics` are the tonics for each
        pitch array.
        The pitch values are expected to be in hertz.
        """
        all_cents = []
        for data, tonic in zip(pitches, tonics):
            #Histogram
            valid_pitch = data[:, 1]
            valid_pitch = [1200*np.log2(i/tonic) for i in valid_pitch if i > 0]
            if octave_folded:
                valid_pitch = map(lambda x: int(x % 1200), valid_pitch)

            all_cents.extend(valid_pitch)
        if not bins:
            bins = max(all_cents)-min(all_cents)

        n, bin_edges = np.histogram(all_cents, bins, normed=True)
        bin_centers = 0.5*(bin_edges[1:]+bin_edges[:-1])
        n = n.reshape(len(n), 1)
        bin_centers = bin_centers.reshape(len(bin_centers), 1)
        self.average_hist = np.append(n, bin_centers, axis=1)

    @staticmethod
    def find_nearest_index(arr, value):
{'fraction_non_alphanumeric': 0.09523809523809523, 'fraction_numerical': 0.01310615989515072, 'mean_word_length': 2.5770833333333334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5924511', 'n_tokens_mistral': 2272, 'n_tokens_neox': 2126, 'n_words': 664}