<jupyter_start><jupyter_text><jupyter_code>import numpy as np
from scipy.special import comb
import matplotlib.pyplot as plt
### Parameters
p = 0.01  # Probability of getting a defect
n = 1000  # products per batch in production line

alpha = 0.01  # likelihood of finding these or more defects
def prob_of_these_defects(t):
    # a function that calculated the probability of seeing exactly t defect in a production line
    coef_a = comb(n, t)
    coef_b = np.power(p, t)
    coef_c = np.power(1.0 - p, n - t)
    return coef_a * coef_b * coef_c


def prob_of_seeing_more_defects(k):
    # a function that calculated the prob of k or more defects
    t_vector = [prob_of_these_defects(t) for t in range(k, n + 1)]
    t_sum = np.sum(t_vector)
    return t_sum

### Run calculations
k_values = list()
k_results = list()
for k in range(2, int(n / 2)):
  likelihood_of_k_defects = prob_of_seeing_more_defects(k)
  k_results.append(likelihood_of_k_defects)
  k_values.append(k)

  if likelihood_of_k_defects < alpha * 0.001:
      break  # To save computational effort, when the likelihood drops significantly below alpha, we break
### Find optimal k
idx_of_closest = (np.abs(np.asarray(k_results) - alpha)).argmin()
optimal_k = k_values[idx_of_closest]
optimal_value = k_results[idx_of_closest]
# plot
plt.plot(k_values, k_results, 'black', label='k-to-alpha', linewidth=3.5)
plt.axhline(y=alpha, color='r', label=f'alpha ({alpha:>.3f})')
plt.axvline(x=optimal_k, color='g', label=f'Optimal k ({optimal_k})')

plt.title(f'optimal K for desired alpha (n={n},p={p:>.3f})')
plt.ylabel('alpha')
plt.xlabel('k')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12176009644364075, 'fraction_numerical': 0.013863773357444244, 'mean_word_length': 4.7839721254355405, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '343067', 'license': 'Apache-2.0', 'n_tokens_mistral': 655, 'n_tokens_neox': 619, 'n_words': 163, 'path': 'ex_a_22.ipynb', 'repo_name': 'GuysBarash/MLBook', 'size': '1659'}