<jupyter_start><jupyter_text>Data format with dinesionality reduction<jupyter_code>drone = np.load("data/drone_rf_LH.npy", allow_pickle=True)
background = np.load("data/background_rf_LH.npy", allow_pickle=True)
drone.min()
# background.min()<jupyter_output><empty_output><jupyter_text>Viewing RF data<jupyter_code>import matplotlib.pyplot as plt
import random
num = random.randint(0, len(background)-1)
num = 2
channel = 0
plt.plot(background[num][channel], label="background")
# plt.plot(drone[num][channel],label="drone")
# plt.legend(loc='upper right')
plt.xlabel("Time (in 100ms)")
plt.ylabel("Power (dB)")<jupyter_output><empty_output><jupyter_text>Transforming RF data<jupyter_code>num = random.randint(0, len(background)-1)
M = 2**11
from scipy import signal, fftpack
fft = fftpack.fft
bg_transformed = fft(background[num][0], n=int(10**7/M))
# bg_transformed = signal.resample(background[num][0], int(10**7/M))
plt.plot(np.absolute(bg_transformed), label="low")
# bg_transformed = fft(background[num][1], n=int(10**7/M))
# bg_transformed = signal.resample(background[num][1], int(10**7/M))
# plt.plot(bg_transformed, label="high")
# num = random.randint(0, len(background)-1)
M = 2048
from scipy import signal
num =29
# bg_transformed = abs(np.fft.fft(background[num][0], int(10**7/M)))
bg_transformed = signal.resample(background[num][0], int(10**7/M))
plt.plot(np.absolute(bg_transformed), label="low")
# bg_transformed = abs(np.fft.fft(background[num][1], int(10**7/M)))
# bg_transformed = signal.resample(background[num][1], int(10**7/M))
# plt.plot(bg_transformed, label="high")
plt.xlabel("Time (in 100ms)")
plt.ylabel("Power (dB)")
bg_transformed.size
from scipy import signal
num =random.randint(0, len(drone)-1)
from scipy import signal, fftpack
fft = fftpack.fft
dr_transformed = fft(drone[num][0], n=int(10**7/M))
# dr_transformed = signal.resample(drone[num][0], int(10**7/M))
plt.plot(np.absolute(dr_transformed), label="low")
# dr_transformed = abs(np.fft.fft(drone[num][1], int(10**7/M)))
# dr_transformed = signal.resample(drone[num][0], int(10**7/M))
# plt.plot(dr_transformed, label="high")
plt.xlabel("Time (in 100ms)")
plt.ylabel("Power (dB)")
plt.plot(drone[num][0])
M =1
# dr_transformed = abs(np.fft.fft(drone[num][0], int(10**7/M)))
dr_transformed = signal.resample(drone[num][0], int(10**7/M))
plt.plot(dr_transformed, label="low")
# dr_transformed = abs(np.fft.fft(drone[num][1], int(10**7/M)))
dr_transformed = signal.resample(drone[num][0], int(10**7/M))
plt.plot(dr_transformed, label="high")<jupyter_output><empty_output><jupyter_text>Computing the transform for full dataset and saving<jupyter_code>background_resampled = []
drone_resampled = []
from scipy import signal
from tqdm import tqdm
M = 2**2
num_samples = int(10**7/M)
print("Resample size:",num_samples)
print("Processing background RF..")
for sample in tqdm(background):
    resampled_L = signal.resample(sample[0], num_samples)
    resampled_H = signal.resample(sample[1], num_samples)
    background_resampled.append(np.array([resampled_L, resampled_H]))
print("Processing drone RF..")
for sample in tqdm(drone):
    resampled_L = signal.resample(sample[0], num_samples)
    resampled_H = signal.resample(sample[1], num_samples)
    drone_resampled.append(np.array([resampled_L, resampled_H]))
from scipy import signal
from tqdm import tqdm
M = 2**2
num_samples = int(10**7/M)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

print("Processing background RF..")
for sample in tqdm(background):
    resampled_L = signal.resample(sample[0], num_samples)
    resampled_H = signal.resample(sample[1], num_samples)
    resampled_LH = np.array([resampled_L, resampled_H])
    resampled_LH = scaler.fit_transform(resampled_LH)
    background_resampled.append(resampled_LH)
print("Processing drone RF..")
for sample in tqdm(drone):
    resampled_L = signal.resample(sample[0], num_samples)
    resampled_H = signal.resample(sample[1], num_samples)
    resampled_LH = np.array([resampled_L, resampled_H])
    resampled_LH = scaler.fit_transform(resampled_LH)
    drone_resampled.append(resampled_LH)
np.save("data/droneRF_onefourth" ,np.array(drone_resampled))
np.save("data/backgroundRF_onefourth" ,np.array(background_resampled))
from scipy import fftpack
from tqdm import tqdm
fft = fftpack.fft
background_resampled = []
drone_resampled = []
M = 2**11
num_samples = int(10**7/M)
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()

print("Processing background RF..")
for sample in tqdm(background):
    resampled_L = np.absolute(fft(sample[0], num_samples))
    resampled_H = np.absolute(fft(sample[1], num_samples))
    resampled_LH = np.array([resampled_L, resampled_H])
    resampled_LH = scaler.fit_transform(resampled_LH)
    background_resampled.append(resampled_LH)
print("Processing drone RF..")
for sample in tqdm(drone):
    resampled_L = np.absolute(fft(sample[0], num_samples))
    resampled_H = np.absolute(fft(sample[1], num_samples))
    resampled_LH = np.array([resampled_L, resampled_H])
    resampled_LH = scaler.fit_transform(resampled_LH)
    drone_resampled.append(resampled_LH)
np.save("data/droneRF_fft" ,np.array(drone_resampled))
np.save("data/backgroundRF_fft" ,np.array(background_resampled))<jupyter_output><empty_output><jupyter_text>Testing and viewing resampled dataset<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(np.array(background_resampled+drone_resampled).flatten().reshape(-1,1))
np.array(background).shape
plt.plot(background_resampled[1][0])
plt.plot(drone_resampled[2][0])
from scipy import signal
import matplotlib.pyplot as plt
sample_size = 10**7/2**2
print("Sample size:",sample_size)
plt.plot(signal.resample(drone[29][0], int(sample_size)))<jupyter_output>Sample size: 2500000.0
{'fraction_non_alphanumeric': 0.1533573759230637, 'fraction_numerical': 0.02472952086553323, 'mean_word_length': 7.379856115107914, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '531515', 'license': 'MIT', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2214, 'n_words': 357, 'path': 'Data Parser.ipynb', 'repo_name': 'nihadern/cam-rf-drone-detection', 'size': '5823'}