<commit_before>import sys
from .utils import (
    db_to_float,
    ratio_to_db,
    register_pydub_effect,
    make_chunks,
    audioop,
)
from .exceptions import TooManyMissingFrames
from tempfile import NamedTemporaryFile
import subprocess
import os

if sys.version_info >= (3, 0):
    xrange = range

@register_pydub_effect
def noise_reduction(seg, sample_start=0, sample_end=1000, sensitivity=0.2):
    """
    noise_reduction will remove consistent background noise from a recording
        given a sample region which is expected to be silent (but includes the
        background noise)

    sample_start: the time (in ms) at which we should start the sample period
    sample_end: the time (in ms) at which we end the sample period
    sensitivity: (from 0 to 1) how aggresive to be with the noise reduction
    """

    sample_section = seg[sample_start:sample_end]
    sample_section_file = NamedTemporaryFile(mode='wb', delete=False)
    sample_section.export(sample_section_file.name, format="wav")
    
    noise_profile_file = NamedTemporaryFile(mode='wb', delete=False)
    noise_profile_command = ['sox', sample_section_file.name, '-n', 'noiseprof', noise_profile_file.name]
    subprocess.call(noise_profile_command, stderr=open(os.devnull))

    noisy_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    seg.export(noisy_file.name, format="wav")
    denoised_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    noise_reduction_command = ['sox', noisy_file.name, denoised_file.name, 'noisered', noise_profile_file.name, '0.2']
    subprocess.call(noise_reduction_command, stderr=open(os.devnull))

    os.remove(sample_section_file.name)
    os.remove(noise_profile_file.name)
    os.remove(noisy_file.name)

    denoised_segment = seg.from_file(denoised_file.name)
    
    os.remove(denoised_file.name)

    return denoised_segment


@register_pydub_effect
def normalize(seg, headroom=0.1):
    """<commit_msg>added a first pass at sox_compand, a fast dynamic range compressor<commit_after>import sys
from .utils import (
    db_to_float,
    ratio_to_db,
    register_pydub_effect,
    make_chunks,
    audioop,
)
from .exceptions import TooManyMissingFrames
from tempfile import NamedTemporaryFile
import subprocess
import os

if sys.version_info >= (3, 0):
    xrange = range

@register_pydub_effect
def noise_reduction(seg, sample_start=0, sample_end=1000, sensitivity=0.2):
    """
    noise_reduction will remove consistent background noise from a recording
        given a sample region which is expected to be silent (but includes the
        background noise)

    sample_start: the time (in ms) at which we should start the sample period
    sample_end: the time (in ms) at which we end the sample period
    sensitivity: (from 0 to 1) how aggresive to be with the noise reduction
    """

    sample_section = seg[sample_start:sample_end]
    sample_section_file = NamedTemporaryFile(mode='wb', delete=False)
    sample_section.export(sample_section_file.name, format="wav")
    
    noise_profile_file = NamedTemporaryFile(mode='wb', delete=False)
    noise_profile_command = ['sox', sample_section_file.name, '-n', 'noiseprof', noise_profile_file.name]
    subprocess.call(noise_profile_command, stderr=open(os.devnull))

    noisy_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    seg.export(noisy_file.name, format="wav")
    denoised_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    noise_reduction_command = ['sox', noisy_file.name, denoised_file.name, 'noisered', noise_profile_file.name, '0.2']
    subprocess.call(noise_reduction_command, stderr=open(os.devnull))

    os.remove(sample_section_file.name)
    os.remove(noise_profile_file.name)
    os.remove(noisy_file.name)

    denoised_segment = seg.from_file(denoised_file.name)
    
    os.remove(denoised_file.name)

    return denoised_segment

@register_pydub_effect
def sox_compand(seg):
    """
    right now a very simple wrapper around sox's compand

    no params for the time being...
    """
    uncompressed_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    seg.export(uncompressed_file.name, format="wav")

    compressed_file = NamedTemporaryFile(mode='wb', suffix=".wav", delete=False)
    compression_command = ['sox', uncompressed_file.name, compressed_file.name, "compand", "0.3,1", "6:-70,-60,-20", "-5", "-90", "0.2"];
    subprocess.call(compression_command, stderr=open(os.devnull))

    os.remove(uncompressed_file.name)

    compressed_segment = seg.from_file(compressed_file.name)

    os.remove(compressed_file.name)

    return compressed_segment

@register_pydub_effect
def normalize(seg, headroom=0.1):
    """
{'fraction_non_alphanumeric': 0.10382744766335378, 'fraction_numerical': 0.009515753859166844, 'mean_word_length': 4.637663885578069, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716814', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1532, 'n_words': 346}