def run(self):
    dev = cuda.Device(self.device)
    dev.use()
    self.setup()
    while True:
        (job, data) = self.pipe.recv()
        if (job == 'finalize'):
            dev.synchronize()
            break
        if (job == 'update'):
            self.model.cleargrads()
            batch = self.converter(self.iterator.next(), self.device)
            with self.reporter.scope({}):
                loss = _calc_loss(self.model, batch)
            self.model.cleargrads()
            loss.backward()
            del loss
            gg = gather_grads(self.model)
            nccl_data_type = _get_nccl_data_type(gg.dtype)
            null_stream = cuda.Stream.null
            self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size, nccl_data_type, nccl.NCCL_SUM, 0, null_stream.ptr)
            del gg
            self.model.cleargrads()
            gp = gather_params(self.model)
            nccl_data_type = _get_nccl_data_type(gp.dtype)
            self.comm.bcast(gp.data.ptr, gp.size, nccl_data_type, 0, null_stream.ptr)
            scatter_params(self.model, gp)
            del gp
def run(self):
    dev = cuda.Device(self.device)
    dev.use()
    self.setup()
    while True:
        (job, data) = self.pipe.recv()
        if (job == 'finalize'):
            dev.synchronize()
            break
        if (job == 'update'):
            self.model.cleargrads()
            batch = self.converter(self.iterator.next(), self.device)
            with self.reporter.scope({}):
                loss = _calc_loss(self.model, batch)
            self.model.cleargrads()
            loss.backward()
            del loss
            gg = gather_grads(self.model)
            nccl_data_type = _get_nccl_data_type(gg.dtype)
            null_stream = cuda.Stream.null
            self.comm.reduce(gg.data.ptr, gg.data.ptr, gg.size, nccl_data_type, nccl.NCCL_SUM, 0, null_stream.ptr)
            del gg
            self.model.cleargrads()
            gp = gather_params(self.model)
            nccl_data_type = _get_nccl_data_type(gp.dtype)
            self.comm.bcast(gp.data.ptr, gp.size, nccl_data_type, 0, null_stream.ptr)
            scatter_params(self.model, gp)
            del gp