<commit_before>"""Read Minigo game examples from a Bigtable.
"""

import collections
import numpy as np
import os
import struct
import time
from google.cloud import bigtable
from google.cloud.bigtable import row_filters
import tensorflow as tf
import utils


# TODO(president.jackson): document these
_project_name = os.environ['PROJECT']
_instance_name = os.environ['CBT_INSTANCE']
_table_name = os.environ['CBT_TABLE']

_bt_table = bigtable.Client(
    _project_name).instance(
        _instance_name).table(
            _table_name)

_tf_table = tf.contrib.cloud.BigtableClient(
    _project_name,
    _instance_name).table(
        _table_name)


def get_latest_game_number():
    """Return the number of the next game to be stored.

    The state of the table is stored in `metadata:game_counter`
    in the `table_state` row only.
    """
    table_state = _bt_table.read_row(
        b'table_state',
        filter_=row_filters.ColumnRangeFilter(
            'metadata', b'game_counter', b'game_counter'))
    value = table_state.cell_value('metadata', b'game_counter')
    return struct.unpack('>q', value)[0]


def get_game_range_row_names(game_begin, game_end):
    """Get the row range containing the given games.

    Sample row name:
      g_0000000001_m001

    Args:
      game_begin:  an integer of the beginning game number.
      game_end:  an integer of the ending game number, exclusive.

    Returns:
      The two string row numbers to pass to Bigtable as the row range.
    """
    row_fmt = 'g_{:0>10}_'
    return row_fmt.format(game_begin), row_fmt.format(game_end)


def make_single_array(ds, batch_size=8*1024):
    batches = []
    with tf.Session() as sess:
        ds = ds.batch(batch_size)
        iterator = ds.make_initializable_iterator()
        sess.run(iterator.initializer)
        get_next = iterator.get_next()
        try:<commit_msg>Fetch and sort bleakest moves from game range<commit_after>"""Read Minigo game examples from a Bigtable.
"""

import collections
import numpy as np
import os
import re
import struct
import time
from google.cloud import bigtable
from google.cloud.bigtable import row_filters
import tensorflow as tf
import utils


# TODO(president.jackson): document these
_project_name = os.environ['PROJECT']
_instance_name = os.environ['CBT_INSTANCE']
_table_name = os.environ['CBT_TABLE']

_bt_table = bigtable.Client(
    _project_name).instance(
        _instance_name).table(
            _table_name)

_tf_table = tf.contrib.cloud.BigtableClient(
    _project_name,
    _instance_name).table(
        _table_name)


def get_latest_game_number():
    """Return the number of the next game to be stored.

    The state of the table is stored in `metadata:game_counter`
    in the `table_state` row only.
    """
    table_state = _bt_table.read_row(
        b'table_state',
        filter_=row_filters.ColumnRangeFilter(
            'metadata', b'game_counter', b'game_counter'))
    value = table_state.cell_value('metadata', b'game_counter')
    return struct.unpack('>q', value)[0]


def get_game_range_row_names(game_begin, game_end):
    """Get the row range containing the given games.

    Sample row name:
      g_0000000001_m001

    Args:
      game_begin:  an integer of the beginning game number.
      game_end:  an integer of the ending game number, exclusive.

    Returns:
      The two string row numbers to pass to Bigtable as the row range.
    """
    row_fmt = 'g_{:0>10}_'
    return row_fmt.format(game_begin), row_fmt.format(game_end)


def get_bleakest_moves(start_game, end_game):
    """Given a range of games, return the bleakest moves.

    Returns a list of (game, move, q) sorted by q.
    """
    row_fmt = 'g_{:0>10}_'
    bleak = b'bleakest_q'
    rows = _bt_table.read_rows(
        row_fmt.format(start_game),
        row_fmt.format(end_game),
        filter_=row_filters.ColumnRangeFilter(
            'metadata', bleak, bleak))
    gm_pat = re.compile(r'g_(\d+)_m_(\d+)')
    bleakest = [gm_pat.match(str(r.row_key, 'utf-8')).groups() +
                (r.cell_value('metadata', bleak),)
                for r in rows]
    # Convert to numbers in a second pass
    bleakest = [(int(g), int(m), float(q)) for g, m, q in bleakest]
    bleakest.sort(key=operator.itemgetter(2))
    return bleakest


def make_single_array(ds, batch_size=8*1024):
    batches = []
    with tf.Session() as sess:
        ds = ds.batch(batch_size)
        iterator = ds.make_initializable_iterator()
        sess.run(iterator.initializer)
        get_next = iterator.get_next()
        try:
{'fraction_non_alphanumeric': 0.09982486865148861, 'fraction_numerical': 0.010726795096322241, 'mean_word_length': 3.3063147973609803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2309357', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1459, 'n_words': 391}