<commit_before>import hashlib


b = 256
q = 2 ** 255 - 19
l = 2 ** 252 + 27742317777372353535851937790883648493


def H(m):
    return hashlib.sha512(m).digest()


def pow2(x, p):
    """== pow(x, 2**p, q)"""
    while p > 0:
        x = x * x % q
        p -= 1
    return x

def powcyclo(x, p, k):
    """== pow(x, 1 + 2**p + ... + 2**((k-1)*p), q)"""
    c = x
    while k > 1:
        c = x * pow2(c, p) % q
        k -= 1
    return c

def inv(x):
    """$= x^{-1} \mod q$, for x != 0"""
    # q-2 == 32*(2**250 - 1) + 11
    i = pow(x, 32*31, q)         # == x**(32 * (2**5 - 1))
    i = powcyclo(i, 5, 5)        # == x**(32 * (2**25 - 1))
    i = powcyclo(i, 25, 5)       # == x**(32 * (2**125 - 1))
    i = powcyclo(i, 125, 2)      # == x**(32 * (2**250 - 1))
    return i * pow(x, 11, q) % q # == x**(q-2)


d = -121665 * inv(121666)
I = pow(2, (q - 1) / 4, q)


def xrecover(y):
    xx = (y * y - 1) * inv(d * y * y + 1)
    x = pow(xx, (q + 3) / 8, q)

    if (x * x - xx) % q != 0:
        x = (x * I) % q

    if x % 2 != 0:
        x = q-x

    return x


By = 4 * inv(5)
Bx = xrecover(By)
B = (Bx % q, By % q)

<commit_msg>Invert slightly faster still -- thanks, djb<commit_after>import hashlib


b = 256
q = 2 ** 255 - 19
l = 2 ** 252 + 27742317777372353535851937790883648493


def H(m):
    return hashlib.sha512(m).digest()


def pow2(x, p):
    """== pow(x, 2**p, q)"""
    while p > 0:
        x = x * x % q
        p -= 1
    return x

def inv(z):
    """$= z^{-1} \mod q$, for z != 0"""
    # Adapted from curve25519_athlon.c in djb's Curve25519.
    z2 = z * z % q                                # 2
    z9 = pow2(z2, 2) * z % q                      # 9
    z11 = z9 * z2 % q                             # 11
    z2_5_0 = (z11*z11)%q * z9 % q                 # 31 == 2^5 - 2^0
    z2_10_0 = pow2(z2_5_0, 5) * z2_5_0 % q        # 2^10 - 2^0
    z2_20_0 = pow2(z2_10_0, 10) * z2_10_0 % q     # ...
    z2_40_0 = pow2(z2_20_0, 20) * z2_20_0 % q
    z2_50_0 = pow2(z2_40_0, 10) * z2_10_0 % q
    z2_100_0 = pow2(z2_50_0, 50) * z2_50_0 % q
    z2_200_0 = pow2(z2_100_0, 100) * z2_100_0 % q
    z2_250_0 = pow2(z2_200_0, 50) * z2_50_0 % q   # 2^250 - 2^0
    return pow2(z2_250_0, 5) * z11 % q # 2^255 - 2^5 + 11 = q - 2


d = -121665 * inv(121666)
I = pow(2, (q - 1) / 4, q)


def xrecover(y):
    xx = (y * y - 1) * inv(d * y * y + 1)
    x = pow(xx, (q + 3) / 8, q)

    if (x * x - xx) % q != 0:
        x = (x * I) % q

    if x % 2 != 0:
        x = q-x

    return x


By = 4 * inv(5)
Bx = xrecover(By)
B = (Bx % q, By % q)


{'fraction_non_alphanumeric': 0.20125539427226363, 'fraction_numerical': 0.1573165947430365, 'mean_word_length': 1.5731584258324924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '40967', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1215, 'n_words': 333}