<commit_before>import sys
import importlib
from mpi4py import MPI
import os, random, math

import ga_utils
import pbt

from keras import backend as K

class ModelWorker:
    def __init__(self, rank):
        self.rank = rank

    def ready(self, pbt_client, epoch):
        # read every n epochs, n.b. first epoch is 0
        e = epoch + 1
        ready = e % 2 == 0
        if ready:
            pbt_client.log("{}: ready at epoch {}".format(self.rank, epoch))
        return ready

    def pack_data(self, pbt_client, model, metrics):
        """ Packs relevant hyperparameters and selected score metric into a dict to be
            passed to the datastore.

            :param metrics: the metrics in keras callback log
        """
        lr = float(K.get_value(model.optimizer.lr))
        data = {'lr': lr, 'score': metrics['val_loss']}
        data.update(metrics)
        pbt_client.log("{}: putting data".format(self.rank))
        return data

    def update(self, pbt_client, model, data):
        # data: {'acc': 0.87916666666666665, 'loss': 0.38366817765765721, 'rank': 1,
        # 'score': 0.36156702836354576, 'lr': 0.0010000000474974513, 'val_acc': 0.87870370237915607,
        # 'val_loss': 0.36156702836354576}
        old_lr = data['lr']
        lr = old_lr
        draw = random.random()
        if draw <= 0.33:
            lr = old_lr * 0.8
        elif draw <= 0.66:
            lr = old_lr * 1.2
        # else leave as is
        K.set_value(model.optimizer.lr, lr)
        pbt_client.log("{}: updating from rank {}, lr from {} to {}".format(self.rank, data['rank'], old_lr, lr))

def truncation_select(data, score):
    """
     :param data: list of dict containg each ranks' model data as well as
     rank itself.
     :return a dict that contains all the selected rank's model data, or an
     empty dict if no selection
    """
    # e.g. data: [{'acc': 0.87916666666666665, 'loss': 0.38366817765765721, 'rank': 1,
    # 'score': 0.36156702836354576, 'lr': 0.0010000000474974513, 'val_acc': 0.87870370237915607,
    # 'val_loss': 0.36156702836354576}, ...]
    items = sorted(data, key=lambda item: item['score'])
    size = len(items)
    quintile = int(round(size / 5.0))
    if score >= items[-quintile]['score']:
        # in bottom 20%
        idx = random.randint(0, quintile - 1)
        return items[idx]<commit_msg>Updated logging<commit_after>import sys
import importlib
from mpi4py import MPI
import os, random, math

import ga_utils
import pbt

from keras import backend as K

class ModelWorker:
    def __init__(self, rank):
        self.rank = rank

    def ready(self, pbt_client, epoch):
        # read every n epochs, n.b. first epoch is 0
        e = epoch + 1
        return e % 4 == 0
        # if ready:
        #     pbt_client.log("{}: ready at epoch {}".format(self.rank, epoch))
        # return ready

    def pack_data(self, pbt_client, model, metrics):
        """ Packs relevant hyperparameters and selected score metric into a dict to be
            passed to the datastore.

            :param metrics: the metrics in keras callback log
        """
        lr = float(K.get_value(model.optimizer.lr))
        data = {'lr': lr, 'score': metrics['val_loss']}
        data.update(metrics)
        #pbt_client.log("{}: putting data".format(self.rank))
        return data

    def update(self, pbt_client, model, data):
        # data: {'acc': 0.87916666666666665, 'loss': 0.38366817765765721, 'rank': 1,
        # 'score': 0.36156702836354576, 'lr': 0.0010000000474974513, 'val_acc': 0.87870370237915607,
        # 'val_loss': 0.36156702836354576}
        old_lr = data['lr']
        lr = old_lr
        draw = random.random()
        if draw <= 0.33:
            lr = old_lr * 0.8
        elif draw <= 0.66:
            lr = old_lr * 1.2
        # else leave as is
        K.set_value(model.optimizer.lr, lr)
        pbt_client.log("{},{},{},{},{}".format(self.rank, data['rank'], old_lr, lr, old_lr == lr))
        #pbt_client.log("{}: updating from rank {}, lr from {} to {}".format(self.rank, data['rank'], old_lr, lr))

def truncation_select(data, score):
    """
     :param data: list of dict containg each ranks' model data as well as
     rank itself.
     :return a dict that contains all the selected rank's model data, or an
     empty dict if no selection
    """
    # e.g. data: [{'acc': 0.87916666666666665, 'loss': 0.38366817765765721, 'rank': 1,
    # 'score': 0.36156702836354576, 'lr': 0.0010000000474974513, 'val_acc': 0.87870370237915607,
    # 'val_loss': 0.36156702836354576}, ...]
    items = sorted(data, key=lambda item: item['score'])
    size = len(items)
    quintile = int(round(size / 5.0))
    if score >= items[-quintile]['score']:
        # in bottom 20%
        idx = random.randint(0, quintile - 1)
        return items[idx]
{'fraction_non_alphanumeric': 0.13251762754043966, 'fraction_numerical': 0.10078805474906678, 'mean_word_length': 2.5255847953216373, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '905059', 'n_tokens_mistral': 1937, 'n_tokens_neox': 1607, 'n_words': 466}