<commit_before>#! /usr/local/bin/python
# -*- coding: utf-8 -*-
#
# gv.py
#   First ver.: 2017-06-09
#
#   Copyright 2017
#       <NAME> <<EMAIL>>
#
#   Distributed under terms of the MIT license.
#

"""
Calculate log F0 mean and variance of several feature sequence

"""

import os
import numpy as np

from sprocket.util.hdf5 import open_h5files, close_h5files


class F0statistics (object):

    def __init__(self, conf):
        # read pair-dependent yml file
        self.conf = conf

        # open h5list files
        self.h5s = open_h5files(conf, mode='tr')
        self.num_files = len(self.h5s)

    def estimate(self):
        otflags = [0, 1]
        for otflag in otflags:
            if otflag == 1:
                spkr = 'org'
            else:
                spkr = 'tar'

            for i in range(self.num_files):
                nonzeroidx = np.nonzero(self.h5s[i][otflag].read('f0'))
                f0 = self.h5s[i][otflag].read('f0')
                if i == 0:
                    f0s = np.log(f0[nonzeroidx])
                else:
                    f0s = np.r_[f0s, np.log(f0[nonzeroidx])]

            f0stats = np.array([np.mean(f0s), np.std(f0s)])

            f0statspath = self.conf.pairdir + '/stats/' + spkr + '.f0stats'
            if not os.path.exists(os.path.dirname(f0statspath)):
                os.makedirs(os.path.dirname(f0statspath))
            fp = open(f0statspath, 'w')
            fp.write(f0stats)
            fp.close()

            print('F0 statistics estimation for ' + spkr + ' has been done.')

        # close h5class
        close_h5files(self.h5s)

        return
<commit_msg>Inplementation of F0 transformation has been done<commit_after>#! /usr/local/bin/python
# -*- coding: utf-8 -*-
#
# gv.py
#   First ver.: 2017-06-09
#
#   Copyright 2017
#       <NAME> <<EMAIL>>
#
#   Distributed under terms of the MIT license.
#

"""
Calculate log F0 mean and variance of several feature sequence

"""

import os
import numpy as np

from sprocket.util.hdf5 import open_h5files, close_h5files


class F0statistics (object):

    def __init__(self, conf):
        # read pair-dependent yml file
        self.conf = conf

        # open h5list files
        self.h5s = open_h5files(conf, mode='tr')
        self.num_files = len(self.h5s)

    def estimate(self):
        otflags = [0, 1]
        for otflag in otflags:
            if otflag == 0:
                spkr = 'org'
            else:
                spkr = 'tar'

            for i in range(self.num_files):
                nonzeroidx = np.nonzero(self.h5s[i][otflag].read('f0'))
                f0 = self.h5s[i][otflag].read('f0')
                if i == 0:
                    f0s = np.log(f0[nonzeroidx])
                else:
                    f0s = np.r_[f0s, np.log(f0[nonzeroidx])]

            f0stats = np.array([np.mean(f0s), np.std(f0s)])

            f0statspath = self.conf.pairdir + '/stats/' + spkr + '.f0stats'
            if not os.path.exists(os.path.dirname(f0statspath)):
                os.makedirs(os.path.dirname(f0statspath))
            fp = open(f0statspath, 'w')
            fp.write(f0stats)
            fp.close()

            print('F0 statistics estimation for ' + spkr + ' has been done.')

        # close h5class
        close_h5files(self.h5s)

        return

    def read_statistics(self, orgstatsf, tarstatsf):
        # read f0 statistics of source and target from binary
        orgf0stats = np.fromfile(orgstatsf, dtype='d')
        tarf0stats = np.fromfile(tarstatsf, dtype='d')

        self.omean = orgf0stats[0]
        self.ostd = orgf0stats[1]
        self.tmean = tarf0stats[0]
        self.tstd = tarf0stats[1]

        return

    def transform_f0(self, if0):
        assert self.omean, self.ostd is not None
        assert self.tmean, self.tstdis is not None

        # get length and dimension
        T = len(if0)

        # perform f0 transformation
        of0 = np.zeros(T)
        for t in range(T):
            if if0[t] > 0:
                of0[t] = (if0[t] - self.omean) * \
                    self.tstd / self.ostd + self.tmean

        return of0

{'fraction_non_alphanumeric': 0.11981566820276497, 'fraction_numerical': 0.02910502061605627, 'mean_word_length': 1.7511674449633088, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100515', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1390, 'n_words': 321}