<commit_before>#!/usr/bin/env python3
"""
Utilities for FFT computation and visualization
"""
import scipy.fftpack
import numpy as np
import numpy.fft
import functools

__fft_windows = {
	"blackman": np.blackman,
	"bartlett": np.bartlett,
	"hamming": np.hamming,
	"hanning": np.hanning,
	"kaiser": lambda sz: np.kaiser(sz, 2.0),
}

def computeFFT(y, samplerate, window="blackman"):
    "Compute the real FFT of a dataset and return (x, y) which can directly be visualized using matplotlib etc"
    windowArr = __fft_windows[window]
    n = len(y)
    w =  scipy.fftpack.fft(y)
    w  = 2.0 * np.abs(w[:n / 2]) / samplerate # Perform amplitude normalization
    x = np.linspace(0.0, samplerate/2, n/2) 
    return (x, w)

def cutFFTDCArtifats(fx, fy):
	"""
	If an FFT contains DC artifacts, i.e. a large value in the first FFT samples,
	this function can be used to remove this area from the FFT value set.
	This function cuts every value up to (but not including the) first local minimum.
	It returns a tuple (x, y)
	"""
	lastVal = fy[0]
	idx = 0
	# Loop until first local minimum
	for y in fy:
		if y > lastVal:
			return (fx[idx:], fy[idx:])
		idx += 1
		lastVal = y
	# No mimum found. We can't remove DC offset, so return something non-empty (= consistent)
	return (fx, fy)<commit_msg>Added function to select frequencies by threshold<commit_after>#!/usr/bin/env python3
"""
Utilities for FFT computation and visualization
"""
import scipy.fftpack
import numpy as np
import numpy.fft
import functools

__fft_windows = {
	"blackman": np.blackman,
	"bartlett": np.bartlett,
	"hamming": np.hamming,
	"hanning": np.hanning,
	"kaiser": lambda sz: np.kaiser(sz, 2.0),
}

def computeFFT(y, samplerate, window="blackman"):
    "Compute the real FFT of a dataset and return (x, y) which can directly be visualized using matplotlib etc"
    windowArr = __fft_windows[window]
    n = len(y)
    w =  scipy.fftpack.fft(y)
    w  = 2.0 * np.abs(w[:n / 2]) / samplerate # Perform amplitude normalization
    x = np.linspace(0.0, samplerate/2, n/2) 
    return (x, w)

def cutFFTDCArtifats(fx, fy):
	"""
	If an FFT contains DC artifacts, i.e. a large value in the first FFT samples,
	this function can be used to remove this area from the FFT value set.
	This function cuts every value up to (but not including the) first local minimum.
	It returns a tuple (x, y)
	"""
	lastVal = fy[0]
	idx = 0
	# Loop until first local minimum
	for y in fy:
		if y > lastVal:
			return (fx[idx:], fy[idx:])
		idx += 1
		lastVal = y
	# No mimum found. We can't remove DC offset, so return something non-empty (= consistent)
	return (fx, fy)

def selectFrequenciesByThreshold(fx, fy, thresh):
	"""
	Select frequencies where a specific absolute threshold applies
	Returns an array of frequencies
	"""
	return np.asarray([fx[idx] for idx in range(len(fy)) if fy[idx] > thresh])

{'fraction_non_alphanumeric': 0.11345275728837373, 'fraction_numerical': 0.0091324200913242, 'mean_word_length': 3.762541806020067, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '802071', 'n_tokens_mistral': 1054, 'n_tokens_neox': 1018, 'n_words': 365}