<commit_before># -*- encoding: utf-8 -*-

require 'tempfile'
require 'open3'
require 'logger'

module GoogleSpeech

  class Utility

    SOX_ERROR_RE       = /error:/

    class <<self

      def audio_file_duration(path)
        check_local_file(path)

        soxi_duration, err = run_command("soxi -V0 -D #{path}", :nice=>false, :echo_return=>false)
        duration = soxi_duration.chomp.to_f
        duration
      end

      def trim_to_flac(wav_path, duration, flac_path, start, length)
        check_local_file(wav_path)

        command = "sox -t wav '#{wav_path}' -t flac '#{flac_path}' trim #{start.to_i} #{length.to_i} rate 16k"
        out, err = run_command(command)
        response = out + err
        response.split("\n").each{ |l| raise("trim_to_flac: error cmd: '#{command}'\nout: '#{response}'") if l =~ SOX_ERROR_RE }
      end

      # Pass the command to run, and various options
      # :timeout - seconds to wait for command to complete, defaults to 2 hours
      # :echo_return - gets the return value via appended '; echo $?', true by default
      # :nice - call with nice -19 by default, set to false to stop, or integer to set different level
      def run_command(command, options={})
        timeout = options[:timeout] || 7200
        
        # default to adding a nice 19 if nothing specified
        nice = if options.key?(:nice)
          !options[:nice] ? '' : "nice -n #{options[:nice].to_i} "
        else
          'nice -n 19 '
        end
        
        echo_return = (options.key?(:echo_return) && !options[:echo_return]) ? '' : '; echo $?'
        
        cmd = "#{nice}#{command}#{echo_return}"
        
        # logger.debug "run_command:  #{cmd}"<commit_msg>Improved flac encoding command - reduction to mono, companding, normalization<commit_after># -*- encoding: utf-8 -*-

require 'tempfile'
require 'open3'
require 'logger'

module GoogleSpeech

  class Utility

    SOX_ERROR_RE       = /error:/

    class <<self

      def audio_file_duration(path)
        check_local_file(path)

        soxi_duration, err = run_command("soxi -V0 -D #{path}", :nice=>false, :echo_return=>false)
        duration = soxi_duration.chomp.to_f
        duration
      end

      def trim_to_flac(wav_path, duration, flac_path, start, length)
        check_local_file(wav_path)

        #command = "sox -t wav '#{wav_path}' -r 16000 -c 1 -t flac '#{flac_path}' trim #{start.to_i} #{length.to_i} compand .5,2 -80,-80,-75,-50,-30,-15,0,0 norm -0.1"

        command = "sox -t wav '#{wav_path}' -t flac '#{flac_path}' trim #{start.to_i} #{length.to_i} rate 16k"
        
        out, err = run_command(command)
        response = out + err
        response.split("\n").each{ |l| raise("trim_to_flac: error cmd: '#{command}'\nout: '#{response}'") if l =~ SOX_ERROR_RE }
      end

      # Pass the command to run, and various options
      # :timeout - seconds to wait for command to complete, defaults to 2 hours
      # :echo_return - gets the return value via appended '; echo $?', true by default
      # :nice - call with nice -19 by default, set to false to stop, or integer to set different level
      def run_command(command, options={})
        timeout = options[:timeout] || 7200
        
        # default to adding a nice 19 if nothing specified
        nice = if options.key?(:nice)
          !options[:nice] ? '' : "nice -n #{options[:nice].to_i} "
        else
          'nice -n 19 '
        end
        
        echo_return = (options.key?(:echo_return) && !options[:echo_return]) ? '' : '; echo $?'
        
        cmd = "#{nice}#{command}#{echo_return}"
        
        # logger.debug "run_command:  #{cmd}"
{'fraction_non_alphanumeric': 0.13380666302566904, 'fraction_numerical': 0.015292190060076462, 'mean_word_length': 2.485252140818268, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4864820', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1166, 'n_words': 335}