<jupyter_start><jupyter_text>Hamming weight and Pearson CorrelationThis notebook demonstrates how to guess a floating number using Hamming Weight model.We generate a random secret floating number. Then we use hamming weight model to guess the number.<jupyter_code>import pathlib
TEST_RESULT_DIR = './results'

pathlib.Path(TEST_RESULT_DIR).mkdir(parents=True, exist_ok=True) 
import sys
sys.path.append("../")

from pybatina.weight_recovery import *
from pybatina.utils import *
import numpy as np

mantissa_nbits = 7
guess_range=(-5.0, 5.0)
# the number of guess values in the guess range
secret_number = np.random.uniform(guess_range[0], guess_range[1]) 
secret_number = 0.25384521484375
secret_number = int_to_float(float_to_int(secret_number))

print('secret_number =', secret_number)<jupyter_output>secret_number = 0.25384521484375<jupyter_text>Receover the weight number in step-by-step  Correlation of mantissa<jupyter_code>guess_numbers = build_guess_values(component='mantissa', mantissa_nbits=mantissa_nbits, guess_range=guess_range)
known_inputs = build_input_values(mantissa_nbits=mantissa_nbits, component='mantissa')
secret_hw = np.vectorize(hamming_weight)(known_inputs * secret_number)
%%time
mantisa_corr = compute_corr_numbers(secret_hw, known_inputs, guess_numbers)
ax = mantisa_corr.sort_index().plot(figsize=(15,6), label='correlation')
ax.vlines(mantisa_corr.idxmax(), mantisa_corr.min(), mantisa_corr.max(), color='green', label='max correlation')
ax.vlines(secret_number, mantisa_corr.min(), mantisa_corr.max(), color='red', label='secret value')
ax.legend()
ax.grid()
mantisa_corr.sort_values(ascending=False).iloc[:10]<jupyter_output><empty_output><jupyter_text>Correlation exponent<jupyter_code>known_inputs = build_input_values(component='exponent')
secret_hw = np.vectorize(hamming_weight)(known_inputs * secret_number)
guess_numbers = build_guess_values(component='exponent', numbers=mantisa_corr.sort_values(ascending=False).index[:10], guess_range=guess_range)
pd.Series(guess_numbers).sort_values().reset_index(drop=True).plot(figsize=(12,6), marker='.')
%%time
mantisa_exp_corr = compute_corr_numbers(secret_hw, known_inputs, guess_numbers)
ax = mantisa_exp_corr.sort_index().plot(figsize=(15,6), label='correlation')
ax.vlines(mantisa_exp_corr.idxmax(), mantisa_exp_corr.min(), mantisa_exp_corr.max(), color='green', label='max correlation')
ax.vlines(secret_number, mantisa_exp_corr.min(), mantisa_exp_corr.max(), color='red', label='secret value')
ax.legend()
ax.grid()
mantisa_exp_corr.sort_values(ascending=False).iloc[:10]<jupyter_output><empty_output><jupyter_text>Correlation sign<jupyter_code>guess_numbers = build_guess_values(component='sign', numbers=mantisa_exp_corr.sort_values(ascending=False).index[:10], guess_range=guess_range)
known_inputs = build_input_values(component='sign')
secret_hw = np.vectorize(hamming_weight)(known_inputs * secret_number)
pd.Series(guess_numbers).sort_values().reset_index(drop=True).plot(figsize=(12,6), marker='^')
%%time
full_number_corr = compute_corr_numbers(secret_hw, known_inputs, guess_numbers)
ax = full_number_corr.sort_index().plot(figsize=(15,6), label='correlation')
ax.vlines(full_number_corr.idxmax(), full_number_corr.min(), full_number_corr.max(), color='green', label='max correlation')
ax.vlines(secret_number, full_number_corr.min(), full_number_corr.max(), color='red', label='secret value')
ax.legend()
ax.grid()
print('Secret value = %f' % secret_number)
full_number_corr.sort_values(ascending=False).iloc[:10]<jupyter_output>Secret value = 0.253845<jupyter_text>Recover a secret number with helper function<jupyter_code>%%time
from weight_recovery_helper import *
full_number_corr = batina_recover_weight(secret_number=secret_number, 
                                         guess_range=guess_range,
                                         mantissa_nbits=mantissa_nbits,
                                         noise=None)
ax = full_number_corr.sort_index().plot(figsize=(15,6), label='correlation')
ax.vlines(full_number_corr.idxmax(), full_number_corr.min(), full_number_corr.max(), color='green', label='max correlation')
ax.vlines(secret_number, full_number_corr.min(), full_number_corr.max(), color='red', label='secret value')
ax.legend()
ax.grid()
print('Secret value = %f' % secret_number)
full_number_corr.sort_values(ascending=False).iloc[:10]<jupyter_output>Secret value = 0.253845<jupyter_text>Recover multiple secret numbers<jupyter_code>SECRET_VALUE='secret value'
GUESSED_VALUE = 'guessed value'
CORRELATION='correlation'
df_index = [SECRET_VALUE, GUESSED_VALUE, CORRELATION]
def add_noise_to_signal(signal, snr, freq):
    """
    add noise to the signal
    :param signal: signal
    :param snr: signal to noise ratio 
    :param freq: frequency of noise 
    :return: signal with noise
    """
    sig_max = signal.max()
    sig_min = signal.min()
    return pd.Series(signal).apply(lambda x: x  +
                                   np.average(np.random.normal(0.0, (sig_max-sig_min)/snr, freq)))
precision = 1e-3
n_secret_numbers = 2000
secret_numbers = np.concatenate((
    np.random.uniform(guess_range[0], guess_range[1], int(n_secret_numbers*2/4) ),
    np.random.uniform(guess_range[0]*3e-1, guess_range[1]*3e-1, int(n_secret_numbers*1/4)),
    np.random.uniform(guess_range[0]*1e-1, guess_range[1]*1e-1, int(n_secret_numbers*1/4))
))
np.random.shuffle(secret_numbers)
signal_to_noise_ratio = 0.5
print('secret_numbers.shape = %s' % (str(secret_numbers.shape)))<jupyter_output>secret_numbers.shape = (2000,)<jupyter_text>Batina without noise  load existing DB<jupyter_code># read the existing data
from os import path
try:
    fname = path.join(TEST_RESULT_DIR, 'batina_results-%d.csv' % mantissa_nbits)
    last_df = pd.read_csv(fname, index_col=0)
    print('read results in %s' % fname)
    print('number of weight values', len(last_df[SECRET_VALUE].unique()))
except FileNotFoundError:
    last_df = pd.DataFrame()
    pass

batina_results = last_df<jupyter_output><empty_output><jupyter_text>main process<jupyter_code>%%time 
for value in secret_numbers:
    try:
        if value in batina_results[SECRET_VALUE].unique():
            continue
    except KeyError:
        pass
    
    s = batina_recover_weight(secret_number=value,
                              guess_range=guess_range,
                              mantissa_nbits=mantissa_nbits)
    s.name = CORRELATION
    df = pd.DataFrame(s).reset_index().rename(columns={'index': GUESSED_VALUE})
    df[SECRET_VALUE] = value
    batina_results = pd.concat([batina_results, df], axis=0, ignore_index=True)<jupyter_output>/usr/local/lib/python3.8/dist-packages/numpy/lib/function_base.py:2167: RuntimeWarning: overflow encountered in ? (vectorized)
  outputs = ufunc(*inputs)
/usr/local/lib/python3.8/dist-packages/numpy/lib/function_base.py:2167: RuntimeWarning: invalid value encountered in ? (vectorized)
  outputs = ufunc(*inputs)
../pybatina/weight_recovery.py:37: RuntimeWarning: invalid value encountered in greater_equal
  guess_numbers = y[(lo_range <= y) & (y <= hi_range)]
../pybatina/weight_recovery.py:37: RuntimeWarning: invalid value encountered in less_equal
  guess_numbers = y[(lo_range <= y) & (y <= hi_range)]
../pybatina/weight_recovery.py:85: RuntimeWarning: invalid value encountered in multiply
  data=np.vectorize(hamming_weight)(known_inputs.reshape(-1, 1) * guess_numbers))<jupyter_text>merge and save DB<jupyter_code>print('number of weight values', len(batina_results[SECRET_VALUE].unique()))
batina_results.to_csv(fname)
print('save results in %s' % fname)<jupyter_output>number of weight values 108
save results in ./results/batina_results-7.csv<jupyter_text>Batina with noise  load existing DB<jupyter_code># read the existing data
from os import path
try:
    fname = path.join(TEST_RESULT_DIR, 'noise_batina_results-%d.csv' % mantissa_nbits)
    last_df = pd.read_csv(fname, index_col=0)
    print('read results in %s' % fname)
    print('number of weight values', len(last_df[SECRET_VALUE].unique()))
except FileNotFoundError:
    last_df = pd.DataFrame()
    pass
noise_batina_results = last_df<jupyter_output><empty_output><jupyter_text>main process<jupyter_code>%%time
for value in batina_results.index:
    try:
        if value in noise_batina_results[SECRET_VALUE].unique():
            continue
    except KeyError:
        pass
    s = batina_recover_weight(secret_number=value, 
                              guess_range=guess_range,
                              mantissa_nbits=mantissa_nbits,
                              noise=(add_noise_to_signal, signal_to_noise_ratio, 200))
    s.name = CORRELATION
    df = pd.DataFrame(s).reset_index().rename(columns={'index': GUESSED_VALUE})
    df[SECRET_VALUE] = value
    noise_batina_results = pd.concat([noise_batina_results, df], axis=0, ignore_index=True)<jupyter_output>/home/tvk/programming/others/NN-Recovery/tests/weight_recovery_helper.py:30: RuntimeWarning: invalid value encountered in multiply
  exponent_weight_hw = np.vectorize(hamming_weight)(exponent_known_inputs * secret_number)
/home/tvk/programming/others/NN-Recovery/tests/weight_recovery_helper.py:35: RuntimeWarning: invalid value encountered in multiply
  sign_weight_hw = np.vectorize(hamming_weight)(sign_known_inputs * secret_number)<jupyter_text>merge and save DB<jupyter_code>print('number of weight values', len(noise_batina_results[SECRET_VALUE].unique()))
noise_batina_results.to_csv(fname)
print('save results in %s' % fname)<jupyter_output>number of weight values 30
save results in ./results/noise_batina_results-7.csv<jupyter_text>Statistics<jupyter_code>ERROR = 'error'
def statistics(df):
    dft = df[[SECRET_VALUE, GUESSED_VALUE, CORRELATION]].astype(dtype=np.float64)
    dft[ERROR] = np.abs((dft[GUESSED_VALUE] - dft[SECRET_VALUE])/dft[SECRET_VALUE])    
    best_corr = dft.loc[dft.groupby(SECRET_VALUE)[CORRELATION].idxmax()]
    best_err = dft.loc[dft.groupby(SECRET_VALUE)[ERROR].idxmin()]
    return best_corr.T, best_err.T
batina_best_corr, batina_best_err = statistics(batina_results)
noise_batina_best_corr, noise_batina_best_err = statistics(noise_batina_results)
study_score = batina_best_corr
(study_score.T[ERROR] < 1e-2).value_counts()
error_sorted = study_score.T.sort_values(ERROR, ascending=False)
error_sorted[:10]
error_sorted.iloc[0][SECRET_VALUE]
from matplotlib import pyplot as plt

fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(15, 18))
df = batina_best_corr.T
ax[0].plot(guess_range, guess_range, linewidth=1, label='ideal', color='red')
df[[GUESSED_VALUE, SECRET_VALUE]].set_index(SECRET_VALUE)[GUESSED_VALUE].plot(ax=ax[0], marker='.', label='without noise')
ax[0].set_ylim(np.min(guess_range), np.max(guess_range))

ax[0].set_ylabel('Guessed values')
ax[0].set_title('Guessed values (Batina, best corr)')

df = batina_best_corr.T
df[[ERROR, SECRET_VALUE]].set_index(SECRET_VALUE)[ERROR].plot(ax=ax[1], marker='.', label='without noise')
ax[1].set_ylabel('Error')
ax[1].set_title('Guess Error (Batina, best corr)')

df = batina_best_corr.T
df[[CORRELATION, SECRET_VALUE]].set_index(SECRET_VALUE)[CORRELATION].plot(ax=ax[2], marker='.', label='without noise')
ax[2].set_ylabel('Correlation')
ax[2].set_title('Correlation (Batina, best corr)')

for a in ax:
    a.legend()
    a.grid(True)
    a.set_xlabel('secret values')
    a.set_xlim(np.min(guess_range), np.max(guess_range))

from datetime import datetime
from os import path

fname = path.join(TEST_RESULT_DIR, 
                  'hw-%s.jpg' % (datetime.now().strftime("%Y%m%d_%H%M%S%f")))
plt.savefig(fname)
plt.show()
from matplotlib import pyplot as plt

fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(12, 15))
df = batina_best_corr.T
df[[GUESSED_VALUE, SECRET_VALUE]].set_index(SECRET_VALUE)[GUESSED_VALUE].plot(ax=ax[0], marker='.', label='without noise')
df = noise_batina_best_corr.T
ax[0].plot(guess_range, guess_range, linewidth=1, label='ideal', color='red')
df[[GUESSED_VALUE, SECRET_VALUE]].set_index(SECRET_VALUE)[GUESSED_VALUE].plot(ax=ax[0], marker='.', label='with noise')
ax[0].set_ylabel('Guessed values')
ax[0].set_title('Guessed values (Batina, best corr, SNR=%0.2f)' % signal_to_noise_ratio)

df = batina_best_corr.T
df[[ERROR, SECRET_VALUE]].set_index(SECRET_VALUE)[ERROR].plot(ax=ax[1], marker='.', label='without noise')
df = noise_batina_best_corr.T
df[[ERROR, SECRET_VALUE]].set_index(SECRET_VALUE)[ERROR].plot(ax=ax[1], marker='.', label='with noise')
ax[1].set_ylabel('Error')
ax[1].set_title('Guess Error (Batina, best corr, SNR=%0.2f)' % signal_to_noise_ratio)

df = batina_best_corr.T
df[[CORRELATION, SECRET_VALUE]].set_index(SECRET_VALUE)[CORRELATION].plot(ax=ax[2], marker='.', label='without noise')
df = noise_batina_best_corr.T
df[[CORRELATION, SECRET_VALUE]].set_index(SECRET_VALUE)[CORRELATION].plot(ax=ax[2], marker='.', label='with noise')
ax[2].set_ylabel('Correlation')
ax[2].set_title('Correlation (Batina, best corr, SNR=%0.2f)' % signal_to_noise_ratio)

for a in ax:
    a.legend()
    a.grid(True)
    a.set_xlabel('secret values')

from datetime import datetime
from os import path

fname = path.join(TEST_RESULT_DIR, 
                  'nhw-%s.jpg' % (datetime.now().strftime("%Y%m%d_%H%M%S%f")))
plt.savefig(fname)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13426763110307413, 'fraction_numerical': 0.015370705244122965, 'mean_word_length': 6.094067343666489, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '588637', 'license': 'Apache-2.0', 'n_tokens_mistral': 4922, 'n_tokens_neox': 4679, 'n_words': 767, 'path': 'tests/batina_hwp.ipynb', 'repo_name': 'mu-triv/NN-Recovery', 'size': '13272'}