<commit_before>#include "dynet/tensor.h"
#include "dynet/globals.h"
#include "dynet/except.h"

#include <random>
#include <vector>
#include <cstring>

#ifdef __CUDACC__
#include "dynet/gpu-ops.h"
#include "dynet/cuda.h"
#endif

using namespace std;

namespace dynet {

// ---- CPU only operations

#ifndef __CUDACC__

ostream& operator<<(ostream& os, const Tensor& t) {
  if (t.device->type == DeviceType::CPU) {
    os << (*t);
#if HAVE_CUDA
  } else if (t.device->type == DeviceType::GPU) {
    vector<real> vt = as_vector(t);
    Eigen::Map<Eigen::MatrixXf> m(&vt[0], t.d.rows(), t.d.cols());
    os << m;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return os;
}

real as_scalar(const Tensor& t) {
  if (t.d.size() != 1)
    throw std::runtime_error("Input tensor has more than one element, cannot convert to scalar.");
  real res = 0.;
  if (t.device->type == DeviceType::CPU) {
    return t.v[0];
#if HAVE_CUDA
  } else if (t.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)t.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res, t.v, sizeof(float), cudaMemcpyDeviceToHost));
    return res;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

vector<real> as_vector(const Tensor& v) {
  vector<real> res(v.d.size());
  if (v.device->type == DeviceType::CPU) {
    memcpy(&res[0], v.v, sizeof(real) * res.size());
  } else if (v.device->type == DeviceType::GPU) {
#if HAVE_CUDA
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res[0], v.v, sizeof(real) * res.size(), cudaMemcpyDeviceToHost));
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

vector<Eigen::DenseIndex> as_vector(const IndexTensor& v) {
  vector<Eigen::DenseIndex> res(v.d.size());
  if (v.device->type == DeviceType::CPU) {
    memcpy(&res[0], v.v, sizeof(Eigen::DenseIndex) * res.size());
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res[0], v.v, sizeof(Eigen::DenseIndex) * res.size(), cudaMemcpyDeviceToHost));
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

float TensorTools::access_element(const Tensor& v, int index) {
  float ret = 0.;
  if (v.device->type == DeviceType::CPU) {
    return v.v[index];
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpy(&ret, &v.v[index], sizeof(real), cudaMemcpyDeviceToHost);
    return ret;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return ret;
}

float TensorTools::access_element(const Tensor& v, const Dim& index) {
  if (v.device->type == DeviceType::CPU) {
    return (*v)(index[0], index[1]);
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    DYNET_NO_CUDA_IMPL_ERROR("TensorTools::access_element(Tensor,Dim)")
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return 0;
}

void TensorTools::set_element(const Tensor& v, int index, float value) {
  if (v.device->type == DeviceType::CPU) {
    v.v[index] = value;
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpyAsync(&v.v[index], &value, sizeof(real), cudaMemcpyHostToDevice);
#endif
  } else { throw std::runtime_error("Bad device type"); }
}

void TensorTools::copy_element(const Tensor& l, int lindex, Tensor& r, int rindex) {
  if (l.device->type == DeviceType::CPU) {
    r.v[rindex] = l.v[lindex];
  } else {
    if (l.device != r.device) {
      throw std::invalid_argument("TensorTools::CopyElement doesn't support inter-device copy yet");
#if HAVE_CUDA
    } else if (l.device->type == DeviceType::GPU) {
      cudaMemcpyAsync(&r.v[rindex], &l.v[lindex], sizeof(real), cudaMemcpyDeviceToDevice);
#endif
    } else { throw std::runtime_error("Bad device type"); }
  }
}

void TensorTools::set_elements(const Tensor& v, const vector<float>& vec) {
  if (v.device->type == DeviceType::CPU) {
    memcpy(v.v, &vec[0], sizeof(real) * vec.size());
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpyAsync(v.v, &vec[0], sizeof(real) * vec.size(), cudaMemcpyHostToDevice);
#endif
  } else { throw std::runtime_error("Bad device type"); }
}

void TensorTools::copy_elements(Tensor& v, const Tensor& v_src) {
  if (v.device->type == DeviceType::CPU) {
    if (v_src.device->type == DeviceType::CPU) {
      memcpy(v.v, v_src.v, sizeof(real) * v.d.size());
#if HAVE_CUDA
    } else if (v_src.device->type == DeviceType::GPU) {
      CUDA_CHECK(cudaSetDevice(((Device_GPU*)v_src.device)->cuda_device_id));
      cudaMemcpyAsync(v.v, v_src.v, sizeof(real) * v.d.size(), cudaMemcpyDeviceToHost);
#endif
    } else { throw std::runtime_error("Bad device type"); }<commit_msg>Mutli-gpu support for Tensor::copy_element. (#999)<commit_after>#include "dynet/tensor.h"
#include "dynet/globals.h"
#include "dynet/except.h"

#include <random>
#include <vector>
#include <cstring>

#ifdef __CUDACC__
#include "dynet/gpu-ops.h"
#include "dynet/cuda.h"
#endif

using namespace std;

namespace dynet {

// ---- CPU only operations

#ifndef __CUDACC__

ostream& operator<<(ostream& os, const Tensor& t) {
  if (t.device->type == DeviceType::CPU) {
    os << (*t);
#if HAVE_CUDA
  } else if (t.device->type == DeviceType::GPU) {
    vector<real> vt = as_vector(t);
    Eigen::Map<Eigen::MatrixXf> m(&vt[0], t.d.rows(), t.d.cols());
    os << m;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return os;
}

real as_scalar(const Tensor& t) {
  if (t.d.size() != 1)
    throw std::runtime_error("Input tensor has more than one element, cannot convert to scalar.");
  real res = 0.;
  if (t.device->type == DeviceType::CPU) {
    return t.v[0];
#if HAVE_CUDA
  } else if (t.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)t.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res, t.v, sizeof(float), cudaMemcpyDeviceToHost));
    return res;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

vector<real> as_vector(const Tensor& v) {
  vector<real> res(v.d.size());
  if (v.device->type == DeviceType::CPU) {
    memcpy(&res[0], v.v, sizeof(real) * res.size());
  } else if (v.device->type == DeviceType::GPU) {
#if HAVE_CUDA
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res[0], v.v, sizeof(real) * res.size(), cudaMemcpyDeviceToHost));
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

vector<Eigen::DenseIndex> as_vector(const IndexTensor& v) {
  vector<Eigen::DenseIndex> res(v.d.size());
  if (v.device->type == DeviceType::CPU) {
    memcpy(&res[0], v.v, sizeof(Eigen::DenseIndex) * res.size());
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    CUDA_CHECK(cudaMemcpy(&res[0], v.v, sizeof(Eigen::DenseIndex) * res.size(), cudaMemcpyDeviceToHost));
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return res;
}

float TensorTools::access_element(const Tensor& v, int index) {
  float ret = 0.;
  if (v.device->type == DeviceType::CPU) {
    return v.v[index];
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpy(&ret, &v.v[index], sizeof(real), cudaMemcpyDeviceToHost);
    return ret;
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return ret;
}

float TensorTools::access_element(const Tensor& v, const Dim& index) {
  if (v.device->type == DeviceType::CPU) {
    return (*v)(index[0], index[1]);
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    DYNET_NO_CUDA_IMPL_ERROR("TensorTools::access_element(Tensor,Dim)")
#endif
  } else { throw std::runtime_error("Bad device type"); }
  return 0;
}

void TensorTools::set_element(const Tensor& v, int index, float value) {
  if (v.device->type == DeviceType::CPU) {
    v.v[index] = value;
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpyAsync(&v.v[index], &value, sizeof(real), cudaMemcpyHostToDevice);
#endif
  } else { throw std::runtime_error("Bad device type"); }
}

void TensorTools::copy_element(const Tensor& l, int lindex, Tensor& r, int rindex) {
  if (l.device->type == DeviceType::CPU) {
    if (r.device->type == DeviceType::CPU) {
      r.v[rindex] = l.v[lindex];
#if HAVE_CUDA
    } else if (r.device->type == DeviceType::GPU) {
      CUDA_CHECK(cudaSetDevice(((Device_GPU*)r.device)->cuda_device_id));
      cudaMemcpyAsync(&r.v[rindex], &l.v[lindex], sizeof(real),
                      cudaMemcpyHostToDevice);
#endif
    } else { throw std::runtime_error("Bad device type"); }
#if HAVE_CUDA
  } else if (l.device->type == DeviceType::GPU) {
    if (r.device->type == DeviceType::CPU) {
      CUDA_CHECK(cudaSetDevice(((Device_GPU*)l.device)->cuda_device_id));
      cudaMemcpyAsync(&r.v[rindex], &l.v[lindex], sizeof(real),
                      cudaMemcpyDeviceToHost);
    } else if (r.device->type == DeviceType::GPU) {
      if (l.device == r.device) {
        cudaMemcpyAsync(&r.v[rindex], &l.v[lindex], sizeof(real), cudaMemcpyDeviceToDevice);
      } else {
        cudaMemcpyPeerAsync(&r.v[rindex], ((Device_GPU*)r.device)->cuda_device_id,
                            &l.v[lindex], ((Device_GPU*)l.device)->cuda_device_id,
                            sizeof(real), dynet::default_stream);
      }
    } else { throw std::runtime_error("Bad device type"); }
#endif
  } else { throw std::runtime_error("Bad device type"); }
}

void TensorTools::set_elements(const Tensor& v, const vector<float>& vec) {
  if (v.device->type == DeviceType::CPU) {
    memcpy(v.v, &vec[0], sizeof(real) * vec.size());
#if HAVE_CUDA
  } else if (v.device->type == DeviceType::GPU) {
    CUDA_CHECK(cudaSetDevice(((Device_GPU*)v.device)->cuda_device_id));
    cudaMemcpyAsync(v.v, &vec[0], sizeof(real) * vec.size(), cudaMemcpyHostToDevice);
#endif
  } else { throw std::runtime_error("Bad device type"); }
}

void TensorTools::copy_elements(Tensor& v, const Tensor& v_src) {
  if (v.device->type == DeviceType::CPU) {
    if (v_src.device->type == DeviceType::CPU) {
      memcpy(v.v, v_src.v, sizeof(real) * v.d.size());
#if HAVE_CUDA
    } else if (v_src.device->type == DeviceType::GPU) {
      CUDA_CHECK(cudaSetDevice(((Device_GPU*)v_src.device)->cuda_device_id));
      cudaMemcpyAsync(v.v, v_src.v, sizeof(real) * v.d.size(), cudaMemcpyDeviceToHost);
#endif
    } else { throw std::runtime_error("Bad device type"); }
{'fraction_non_alphanumeric': 0.17890789353333944, 'fraction_numerical': 0.0028354523003750113, 'mean_word_length': 4.461538461538462, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '713737', 'n_tokens_mistral': 4039, 'n_tokens_neox': 3997, 'n_words': 798}