<commit_before>from pylab import *

def truth():
  x = linspace(0., 1., 2001)

  # The two lagrange multipliers
  L1 = linspace(0., 20., 101)
  L2 = linspace(0., 20., 101)
  [L1, L2] = meshgrid(L1, L2)
  L2 = L2[::-1, :]

  logZ = L1.copy()
  info = L1.copy()

  for i in xrange(0, 101):
    for j in xrange(0, 101):
      p = exp(-L1[i, j]*(x - 0.5)**2 - L2[i, j]*sin(2.*pi*x/0.5)**2)
      Z = trapz(p, x=x)
      H = trapz(p/Z*log(p/Z), x=x)

      logZ[i, j] = 200*log(Z)
      info[i, j] = H
    print(i+1)

  subplot(1, 2, 1)
  imshow(logZ)

  subplot(1, 2, 2)
  imshow(info)
  show()


if __name__ == '__main__':
  truth()
<commit_msg>Do both old and new thing<commit_after>from pylab import *

x = linspace(0., 1., 2001)

def truth(L1, L2, do_plot=False):
  p = exp(-L1*(x - 0.5)**2 - L2*sin(2.*pi*x/0.5)**2)
  Z = trapz(p, x=x)
  H = trapz(p/Z*log(p/Z), x=x)

  logZ = 200*log(Z)
  H *= 200

  if do_plot:
    plot(x, p)
    title([logZ, H])
    show()

  return [logZ, H]
  

def grid():
  # The two lagrange multipliers
  L1 = linspace(0., 20., 101)
  L2 = linspace(0., 20., 101)
  [L1, L2] = meshgrid(L1, L2)
  L2 = L2[::-1, :]

  logZ = L1.copy()
  info = L1.copy()

  for i in xrange(0, 101):
    for j in xrange(0, 101):
      [logZ[i, j], info[i, j]] = truth(L1[i, j], L2[i, j])
    print(i+1)

  subplot(1, 2, 1)
  imshow(logZ)

  subplot(1, 2, 2)
  imshow(info)
  show()


if __name__ == '__main__':
  truth(10., 1., do_plot=True)
  grid()


{'fraction_non_alphanumeric': 0.2131596984235778, 'fraction_numerical': 0.08361891706648389, 'mean_word_length': 2.5523114355231145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '708393', 'n_tokens_mistral': 797, 'n_tokens_neox': 721, 'n_words': 172}