<commit_before>"""  The Number Coil  """
#  List a prime; List mod6 of prime; Compare with another set (offset of mod6)
#  to see spin change, determine cell of each prime.
#
#

"""  This code plots the positons in the central triangle of the prime hexagon.

See accompanying notes and images for what these mean.

"""

import sys, os, glob
import logging
import re
import itertools

from collections import namedtuple

import numpy as np
import primesieve

_HAS_PRIMESIEVE_NUMPY = False
try:
    import primesieve.numpy
    _HAS_PRIMESIEVE_NUMPY = True
except:
    pass

_HAS_CYTHON_PRIMEHEX = False
try:
    import primehexagon
    _HAS_CYTHON_PRIMEHEX = True
except:
    pass

# python 2/3 compatibility stuff
try:
    l = long(1)
except:
    long = int

try:
    # python2 version of zip iterator is in itertools
    zipper = itertools.izip
except:
    # new python3 version is no longer in itertools. seriously?
    zipper = zip

logger = None
log_formatter = None

def setup_loggers():

    global logger, log_formatter
    if logger is not None:
        return
    
    # create logger
    logger = logging.getLogger('prime_hexagon')
    logger.setLevel(logging.INFO)

    # create formatter
    log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

setup_loggers()

def list_generate_primes_array(a,b):
    logger.info("starting generating primes from {} to {}".format(a,b))
    logger.info("\tusing primesieve list prime generation")
    a = primesieve.primes(a,b)
    logger.info("\tprimes generated")
    a = np.array(a,dtype=np.uint64)
    logger.info("\toutput array created")
    logger.info("done generating primes")
    return a

def numpy_generate_primes_array(a,b):
    logger.info("starting generating primes from {} to {}".format(a,b))
    logger.info("\tusing numpy prime generator")
    import primesieve.numpy
    a = primesieve.numpy.primes(a,b)
    a = a.astype(np.uint64)
    logger.info("done generating primes")
    return a

def get_primes_generator():
    logger.info("selecting primes generation function")
    func = None
    if _HAS_PRIMESIEVE_NUMPY:
        logger.info("\tusing numpy prime generation...")
        func = numpy_generate_primes_array
    else:
        logger.info("\tfalling back to list primesieve prime generation")
        func = list_generate_primes_array

    logger.info("prime generator selected")
    return func


def _compute_spins(primes, last_prime, last_spin):
    """Returns array of SPINS given an array of PRIMES (assumed to be 1d)

    SPINS result array is same shape as PRIMES, with values stored in SPINS[1:]; 
    SPINS[0] = 0, use that zero element to thread chunked computations together.
    """
    
    logger.info("compute_spins: generating mod6Values")
    m6val  = primes % 6
    m6_offset_sum     = np.empty_like(primes, dtype=np.int32)
    m6_offset_sum[0]  = m6val[0] + (last_prime % 6)            #  seed value from current val + prev m6val
    m6_offset_sum[1:] = m6val[1:] + m6val[:-1]                 #  cur m6val + prev m6val
    logger.info("compute_spins: done mod6Values")
    logger.info("compute_spins: m6_off_sum={}".format(m6_offset_sum))

    logger.info("compute_spins: starting to compute spins")

    z = np.zeros_like(primes, dtype=np.int32)
    z[ m6_offset_sum ==  6] =  1
    z[ m6_offset_sum == 10] = -1
    z[ m6_offset_sum ==  2] = -1
    logger.info("compute_spins: z before last_spin={}".format(z))
    z[0] *= last_spin
    logger.info("compute_spins: z={}".format(z))
    spin = np.cumprod(z)

    logger.info("compute_spins: spin={}".format(spin))
    logger.info("compute_spins: done computing spins")
    return spin

def _compute_positions(spin, seed_pos, seed_spin):
    """Given an array of SPINS and two SEED_POSITION and SEED_SPIN values, compute the positions along the prime hex 

    """

    logger.info("compute_positions: starting aux calculations")
    delta = np.zeros_like(spin)
    delta[0]  = spin[0] - seed_spin      # first delta is cur_spin - prev_spin from seed_spin
    delta[1:] = spin[1:] - spin[:-1]     # delta is cur_spin - prev_spin 
    logger.info("compute_positions: delta={}".format(delta))

    #increments = np.copy(spin)           # copy the spin array,
    increments = np.abs(spin)           # copy the spin array,
    increments[ delta != 0 ] = 0          # set any non-zero delta to zero in the increment array
    logger.info("compute_positions: increments={}".format(increments))

    logger.info("compute_positions:\tdone with aux calculations")
    
    logger.info("compute_positions: starting primary calculation")

    # start at seed, cumulative add
    positions = np.copy(increments)
    #increments[0] = seed_pos
    outpositions = (seed_pos + np.cumsum(increments)) % 6
    logger.info("compute_positions: outpositions={}".format(outpositions))
    logger.info("compute_positions:\tdone with primary calculation")
    return outpositions


def _compute_rotations(positions, pos_seed, rot_seed ):

    logger.info("compute_rotations: starting aux calculations")
    #logger.info("positions array {}".format(positions))
    #logger.debug("rot_seed = {}".format(rot_seed))
    #logger.debug("pos_seed = {}".format(pos_seed))

    delta = np.zeros_like(positions)
    delta[1:] = positions[1:] - positions[:-1]
    delta[0] = positions[0] - pos_seed
    #logger.debug("delta array {}".format(delta))

    z = np.zeros_like(delta)       # zero array like delta
    z[ delta == -5 ] =  1          # where delta = -5, set increment to 1
    z[ delta ==  5 ] = -1          # where delta is 5, set increment to -1

    logger.info("compute_rotations: done with aux calculations")

    logger.info("compute_rotations: starting primary calculations")
    z[0] += rot_seed
    r = np.cumsum( z )
    
    logger.info("compute_rotations: done with primary calculations")

    return r

class PrimeSpinFuncs:
    def __init__(self, **opts):
        self.compute_spins     = opts.get('compute_spins',_compute_spins)
        self.compute_positions = opts.get('compute_positions',_compute_positions)
        self.compute_rotations = opts.get('compute_rotations',_compute_rotations)
        self.generate_primes   = opts.get('generate_primes')
        
_engines = {}
_engines['numpy'] = PrimeSpinFuncs(compute_spins = _compute_spins,
                                   compute_positions = _compute_positions,
                                   compute_rotations = _compute_rotations)

if _HAS_CYTHON_PRIMEHEX:
    _engines['cython'] = PrimeSpinFuncs(compute_spins = primehexagon._compute_spins,
                                        compute_positions = primehexagon._compute_positions,
                                        compute_rotations = primehexagon._compute_rotations)
    
## TODO: figure if we can generalize/refactor these array savers methods
def save_text_arrays( filename, data, save_opts):
    logger.info("start saving text results to file {0}".format(filename))

    skip_interval = save_opts.get('skip_interval',1)
    if skip_interval == 0:
        s = slice(-1,None,None)
    else:
        s = slice(None, None, skip_interval)

    logger.info("start zipping and slicing data together")
    outdata    = zipper(data.prime[s], data.pos[s], data.spin[s], data.rot[s])
    logger.info("\tdone zipping and slicing data")

    with open(filename, "w") as fp:
        w = fp.write
        # write each line as comma separated values
        sep = ','
        newline = '\n'
        for d in outdata:
            line = sep.join(map(str,d))
            w(line)
            w(newline)
    logger.info("done saving text file")
    
def save_binary_arrays( filename, data, save_opts):

    skip_interval = save_opts.get('skip_interval', 1)
    if skip_interval == 0:
        s = slice(-1, None,None)           # only take last value
    else:
        s = slice(None, None, skip_interval)

    do_compress = save_opts.get('compress', 1)
        
    (saver,msg) = (np.savez,"uncompressed") if do_compress else (np.savez_compressed, "compressed")
        
    logger.info("start save binary arrays {} to file {} with slice {}".format(msg, filename, s))
    saver(filename, prime=data.prime[s], spin=data.spin[s], pos=data.pos[s], rot=data.rot[s])
    logger.info("\tdone save binary arrays")


def compute_hex_positions(start, end, boundary_vals, save_opts):
<commit_msg>Fixed bug not saving compressed npz files<commit_after>"""  The Number Coil  """
#  List a prime; List mod6 of prime; Compare with another set (offset of mod6)
#  to see spin change, determine cell of each prime.
#
#

"""  This code plots the positons in the central triangle of the prime hexagon.

See accompanying notes and images for what these mean.

"""

import sys, os, glob
import logging
import re
import itertools

from collections import namedtuple

import numpy as np
import primesieve

_HAS_PRIMESIEVE_NUMPY = False
try:
    import primesieve.numpy
    _HAS_PRIMESIEVE_NUMPY = True
except:
    pass

_HAS_CYTHON_PRIMEHEX = False
try:
    import primehexagon
    _HAS_CYTHON_PRIMEHEX = True
except:
    pass

# python 2/3 compatibility stuff
try:
    l = long(1)
except:
    long = int

try:
    # python2 version of zip iterator is in itertools
    zipper = itertools.izip
except:
    # new python3 version is no longer in itertools. seriously?
    zipper = zip

logger = None
log_formatter = None

def setup_loggers():

    global logger, log_formatter
    if logger is not None:
        return
    
    # create logger
    logger = logging.getLogger('prime_hexagon')
    logger.setLevel(logging.INFO)

    # create formatter
    log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

setup_loggers()

def list_generate_primes_array(a,b):
    logger.info("starting generating primes from {} to {}".format(a,b))
    logger.info("\tusing primesieve list prime generation")
    a = primesieve.primes(a,b)
    logger.info("\tprimes generated")
    a = np.array(a,dtype=np.uint64)
    logger.info("\toutput array created")
    logger.info("done generating primes")
    return a

def numpy_generate_primes_array(a,b):
    logger.info("starting generating primes from {} to {}".format(a,b))
    logger.info("\tusing numpy prime generator")
    import primesieve.numpy
    a = primesieve.numpy.primes(a,b)
    a = a.astype(np.uint64)
    logger.info("done generating primes")
    return a

def get_primes_generator():
    logger.info("selecting primes generation function")
    func = None
    if _HAS_PRIMESIEVE_NUMPY:
        logger.info("\tusing numpy prime generation...")
        func = numpy_generate_primes_array
    else:
        logger.info("\tfalling back to list primesieve prime generation")
        func = list_generate_primes_array

    logger.info("prime generator selected")
    return func


def _compute_spins(primes, last_prime, last_spin):
    """Returns array of SPINS given an array of PRIMES (assumed to be 1d)

    SPINS result array is same shape as PRIMES, with values stored in SPINS[1:]; 
    SPINS[0] = 0, use that zero element to thread chunked computations together.
    """
    
    logger.info("compute_spins: generating mod6Values")
    m6val  = primes % 6
    m6_offset_sum     = np.empty_like(primes, dtype=np.int32)
    m6_offset_sum[0]  = m6val[0] + (last_prime % 6)            #  seed value from current val + prev m6val
    m6_offset_sum[1:] = m6val[1:] + m6val[:-1]                 #  cur m6val + prev m6val
    logger.info("compute_spins: done mod6Values")
    logger.info("compute_spins: m6_off_sum={}".format(m6_offset_sum))

    logger.info("compute_spins: starting to compute spins")

    z = np.zeros_like(primes, dtype=np.int32)
    z[ m6_offset_sum ==  6] =  1
    z[ m6_offset_sum == 10] = -1
    z[ m6_offset_sum ==  2] = -1
    logger.info("compute_spins: z before last_spin={}".format(z))
    z[0] *= last_spin
    logger.info("compute_spins: z={}".format(z))
    spin = np.cumprod(z)

    logger.info("compute_spins: spin={}".format(spin))
    logger.info("compute_spins: done computing spins")
    return spin

def _compute_positions(spin, seed_pos, seed_spin):
    """Given an array of SPINS and two SEED_POSITION and SEED_SPIN values, compute the positions along the prime hex 

    """

    logger.info("compute_positions: starting aux calculations")
    delta = np.zeros_like(spin)
    delta[0]  = spin[0] - seed_spin      # first delta is cur_spin - prev_spin from seed_spin
    delta[1:] = spin[1:] - spin[:-1]     # delta is cur_spin - prev_spin 
    logger.info("compute_positions: delta={}".format(delta))

    #increments = np.copy(spin)           # copy the spin array,
    increments = np.abs(spin)           # copy the spin array,
    increments[ delta != 0 ] = 0          # set any non-zero delta to zero in the increment array
    logger.info("compute_positions: increments={}".format(increments))

    logger.info("compute_positions:\tdone with aux calculations")
    
    logger.info("compute_positions: starting primary calculation")

    # start at seed, cumulative add
    positions = np.copy(increments)
    #increments[0] = seed_pos
    outpositions = (seed_pos + np.cumsum(increments)) % 6
    logger.info("compute_positions: outpositions={}".format(outpositions))
    logger.info("compute_positions:\tdone with primary calculation")
    return outpositions


def _compute_rotations(positions, pos_seed, rot_seed ):

    logger.info("compute_rotations: starting aux calculations")
    #logger.info("positions array {}".format(positions))
    #logger.debug("rot_seed = {}".format(rot_seed))
    #logger.debug("pos_seed = {}".format(pos_seed))

    delta = np.zeros_like(positions)
    delta[1:] = positions[1:] - positions[:-1]
    delta[0] = positions[0] - pos_seed
    #logger.debug("delta array {}".format(delta))

    z = np.zeros_like(delta)       # zero array like delta
    z[ delta == -5 ] =  1          # where delta = -5, set increment to 1
    z[ delta ==  5 ] = -1          # where delta is 5, set increment to -1

    logger.info("compute_rotations: done with aux calculations")

    logger.info("compute_rotations: starting primary calculations")
    z[0] += rot_seed
    r = np.cumsum( z )
    
    logger.info("compute_rotations: done with primary calculations")

    return r

class PrimeSpinFuncs:
    def __init__(self, **opts):
        self.compute_spins     = opts.get('compute_spins',_compute_spins)
        self.compute_positions = opts.get('compute_positions',_compute_positions)
        self.compute_rotations = opts.get('compute_rotations',_compute_rotations)
        self.generate_primes   = opts.get('generate_primes')
        
_engines = {}
_engines['numpy'] = PrimeSpinFuncs(compute_spins = _compute_spins,
                                   compute_positions = _compute_positions,
                                   compute_rotations = _compute_rotations)

if _HAS_CYTHON_PRIMEHEX:
    _engines['cython'] = PrimeSpinFuncs(compute_spins = primehexagon._compute_spins,
                                        compute_positions = primehexagon._compute_positions,
                                        compute_rotations = primehexagon._compute_rotations)
    
## TODO: figure if we can generalize/refactor these array savers methods
def save_text_arrays( filename, data, save_opts):
    logger.info("start saving text results to file {0}".format(filename))

    skip_interval = save_opts.get('skip_interval',1)
    if skip_interval == 0:
        s = slice(-1,None,None)
    else:
        s = slice(None, None, skip_interval)

    logger.info("start zipping and slicing data together")
    outdata    = zipper(data.prime[s], data.pos[s], data.spin[s], data.rot[s])
    logger.info("\tdone zipping and slicing data")

    with open(filename, "w") as fp:
        w = fp.write
        # write each line as comma separated values
        sep = ','
        newline = '\n'
        for d in outdata:
            line = sep.join(map(str,d))
            w(line)
            w(newline)
    logger.info("done saving text file")
    
def save_binary_arrays( filename, data, save_opts):

    skip_interval = save_opts.get('skip_interval', 1)
    if skip_interval == 0:
        s = slice(-1, None,None)           # only take last value
    else:
        s = slice(None, None, skip_interval)

    do_compress = save_opts.get('compress', 1)
        
    if do_compress:
        (saver, msg) = (np.savez_compressed, "compressed")
    else:
        (saver, msg) = (np.savez,"uncompressed")
        
    logger.info("start save binary arrays {} to file {} with slice {}".format(msg, filename, s))
    saver(filename, prime=data.prime[s], spin=data.spin[s], pos=data.pos[s], rot=data.rot[s])
    logger.info("\tdone save binary arrays")


def compute_hex_positions(start, end, boundary_vals, save_opts):

{'fraction_non_alphanumeric': 0.11039657020364416, 'fraction_numerical': 0.00976539240204835, 'mean_word_length': 3.127549766527402, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7076882', 'n_tokens_mistral': 5476, 'n_tokens_neox': 5238, 'n_words': 1383}