<jupyter_start><jupyter_text>Method4 DCT based DOST + Huffman encoding  Import Libraries<jupyter_code>import mne
import numpy as np
from scipy.fft import fft,fftshift
import matplotlib.pyplot as plt
from scipy.signal import butter, lfilter
from scipy.signal import freqz
from scipy import signal
from scipy.fftpack import fft, dct, idct
from itertools import islice<jupyter_output><empty_output><jupyter_text>Preprocessing Data loading<jupyter_code>file = "/Users/lubaiying/Desktop/big_ideas_lab/DBDP_Compression_Toolbox/DCT_DOST_Huffman/ECG.EDF"
data = mne.io.read_raw_edf(file)
raw_data = data.get_data()
# you can get the metadata included in the file and a list of all channels:
info = data.info
channels = data.ch_names



np.savetxt('raw.txt',raw_data[0])  #Save it as a txt file
sampling_freq = 1/1000  #This is the sampling frequency
N = raw_data[0].size  #The number of point

ECG_array = raw_data[0] #Copy the data into an ndarray, which does the same thing with the former step
ECG_array.size

xf = np.linspace(-N*sampling_freq/2, N*sampling_freq/2,  N)

ECG_array = ECG_array[150000:-300000]  # Remove the signal in first 3minutes and last 5minutes

index = np.linspace(0, round((N-1)*sampling_freq,4), N)<jupyter_output>Extracting EDF parameters from /Users/lubaiying/Desktop/big_ideas_lab/DBDP_Compression_Toolbox/DCT_DOST_Huffman/ECG.EDF...
EDF file detected
Setting channel info structure...
Creating raw.info structure...<jupyter_text>Butterworth Filter to denoise<jupyter_code>def butter_bandpass(lowcut, highcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    high = highcut / nyq
    b, a = butter(order, [low, high], btype='band')
    return b, a


def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

# Sample rate and desired cutoff frequencies (in Hz).
fs = 1000.0
lowcut = 0.5
highcut = 50.0

# Plot the frequency response for a few different orders.
plt.figure(1)
plt.clf()
for order in [1, 2, 3, 4]:
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    w, h = freqz(b, a, worN=2000)
    plt.plot((fs * 0.5 / np.pi) * w, abs(h), label="order = %d" % order)

plt.plot([0, 0.5 * fs], [np.sqrt(0.5), np.sqrt(0.5)],
         '--', label='sqrt(0.5)')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Gain')
plt.grid(True)
plt.legend(loc='best')
y = butter_bandpass_filter(ECG_array, lowcut, highcut, fs, order=2)<jupyter_output><empty_output><jupyter_text>Resampling and Quantization<jupyter_code>resampled_signal = signal.resample(y, 564000)
np.savetxt('resampled_signal_ecg.txt',resampled_signal)
rounded_signal = np.around(resampled_signal, 5)<jupyter_output><empty_output><jupyter_text>Transformation<jupyter_code>aN = dct(rounded_signal, type = 2, norm = 'ortho')
def return_N(target):
    if target > 1:
        for i in range(1, int(target)):
            if (2 ** i >= target):
                return i-1
    else:
        return 1
split_list = [1]
for i in range(0,return_N(aN.size)):
    split_list.append(2 ** i)
temp = iter(aN) 
res = [list(islice(temp, 0, ele)) for ele in split_list] 
cN_idct = [list(idct(res[0], type = 2, norm = 'ortho' )), list(idct(res[1], type = 2, norm = 'ortho' ))]
for k in range(2,len(res)):
    cN_idct.append(list(idct(res[k], type = 2, norm = 'ortho' )))
all_numbers = []
for i in cN_idct:
    for j in i:
        all_numbers.append(j)
all_numbers = np.asarray(all_numbers)
new_cN_idct = all_numbers*10**4
int_cN = np.round(new_cN_idct,4)
np.savetxt('resampled_signal_ecg.txt',resampled_signal,fmt='%.4f')
np.savetxt('int_cN.txt',int_cN, fmt='%.4f')<jupyter_output><empty_output><jupyter_text>Huffman Coding  INSTRUCTION ON HOW TO COMPRESS THE DATA BY HUFFMAN CODING(I used the package "tcmpr 0.2" and "pyhuff 1.1". These two packages provided the same compression result. So here, we just use "tcmpr 0.2")1. Open your termial or git bash, enter "pip install tcmpr" to install the "tcmpr 0.2" package2. Enter the directory which include the file you want to compress OR copy the path of the file you want to compress3. Enter "tcmpr filename.txt" / "tcmpr filepath" to compress the file4. Find the compressed file in the same directory of the original file<jupyter_code># Do Huffman encoding based on the instruction above 
# or run this trunk if this scratch locates in the same directory with the signal you want to encode
os.system('tcmpr int_cN.txt')<jupyter_output><empty_output><jupyter_text>Reconstruction<jupyter_code>os.system('tcmpr -d int_cN.txt.huffman')
decoded_data = np.loadtxt(fname = "int_cN.txt")
recover_signal = decoded_data / 10**4
recover_signal = list(recover_signal)
split_list = [1]
for i in range(0,return_N(len(recover_signal))+1):
    split_list.append(2 ** i)
temp_recovered = iter(recover_signal) 
res_recovered = [list(islice(temp_recovered, 0, ele)) for ele in split_list] 
recover_dct = [list(dct(res_recovered[0], type = 2, norm = 'ortho' )), list(dct(res_recovered[1], type = 2, norm = 'ortho' ))]
for k in range(2,len(res_recovered)):
    recover_dct.append(list(dct(res_recovered[k], type = 2, norm = 'ortho' )))
all_recover = []
for i in recover_dct:
    for j in i:
        all_recover.append(j)
aN_recover = idct(all_recover, type = 2, norm = 'ortho')
final_recovered = aN_recover[:524288]
resampled_signal_shorter = signal.resample(y, len(final_recovered))
#resampled_signal_shorter = rounded_signal[:len(final_recovered)] 
from sklearn.metrics import mean_squared_error
from math import sqrt

def PRD_calculation(original_signal, compressed_signal):
    PRD = sqrt(sum((original_signal-compressed_signal)**2)/(sum(original_signal**2)))
    return PRD

PRD = PRD_calculation(resampled_signal_shorter, final_recovered)
print("The PRD is {}%".format(round(PRD*100,3)))
#plt.plot(resampled_signal_shorter[74000:75000], label = "origianl")
resampled_signal_shorter = signal.resample(y, len(final_recovered))
plt.plot(final_recovered[74000:75000], label = "recovered", color = 'orange')
plt.plot(resampled_signal_shorter[74000:75000], label = "origianl")
plt.legend()
plt.title('ECG')
plt.grid()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12506082725060827, 'fraction_numerical': 0.02611516626115166, 'mean_word_length': 5.266260162601626, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '206996', 'license': 'Apache-2.0', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2317, 'n_words': 574, 'path': 'DOST_Huffman/DCT_DOST_Huffman_ECG.ipynb', 'repo_name': 'BaiyingLu/Data-Compression-Toolbox', 'size': '6165'}