<commit_before>/**
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */


#include <faiss/gpu/impl/IVFFlat.cuh>
#include <faiss/gpu/GpuResources.h>
#include <faiss/gpu/impl/FlatIndex.cuh>
#include <faiss/gpu/impl/IVFAppend.cuh>
#include <faiss/gpu/impl/IVFFlatScan.cuh>
#include <faiss/gpu/impl/RemapIndices.h>
#include <faiss/gpu/utils/ConversionOperators.cuh>
#include <faiss/gpu/utils/CopyUtils.cuh>
#include <faiss/gpu/utils/DeviceDefs.cuh>
#include <faiss/gpu/utils/DeviceUtils.h>
#include <faiss/gpu/utils/Float16.cuh>
#include <faiss/gpu/utils/HostTensor.cuh>
#include <faiss/gpu/utils/Transpose.cuh>
#include <limits>
#include <thrust/host_vector.h>
#include <unordered_map>

namespace faiss { namespace gpu {

IVFFlat::IVFFlat(GpuResources* res,
                 FlatIndex* quantizer,
                 faiss::MetricType metric,
                 float metricArg,
                 bool useResidual,
                 faiss::ScalarQuantizer* scalarQ,
                 IndicesOptions indicesOptions,
                 MemorySpace space) :
    IVFBase(res,
            metric,
            metricArg,
            quantizer,
            indicesOptions,
            space),
    useResidual_(useResidual),
    scalarQ_(scalarQ ? new GpuScalarQuantizer(res, *scalarQ) : nullptr) {
}

IVFFlat::~IVFFlat() {
}

size_t
IVFFlat::getGpuVectorsEncodingSize_(int numVecs) const {
  return (size_t) numVecs *
    // scalar size per dimension
    (scalarQ_ ? scalarQ_->code_size : sizeof(float)) *
    // number of dimensions
    getDim();
}

size_t
IVFFlat::getCpuVectorsEncodingSize_(int numVecs) const {
  return (size_t) numVecs *
    // scalar size per dimension
    (scalarQ_ ? scalarQ_->code_size : sizeof(float)) *
    // number of dimensions
    getDim();
}

std::vector<unsigned char>
IVFFlat::translateCodesToGpu_(std::vector<unsigned char> codes,
                              size_t numVecs) const {
  // nothing to do
  return codes;
}

std::vector<unsigned char>
IVFFlat::translateCodesFromGpu_(std::vector<unsigned char> codes,
                                size_t numVecs) const {
  // nothing to do<commit_msg>GPU IVFSQ code_size fix<commit_after>/**
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */


#include <faiss/gpu/impl/IVFFlat.cuh>
#include <faiss/gpu/GpuResources.h>
#include <faiss/gpu/impl/FlatIndex.cuh>
#include <faiss/gpu/impl/IVFAppend.cuh>
#include <faiss/gpu/impl/IVFFlatScan.cuh>
#include <faiss/gpu/impl/RemapIndices.h>
#include <faiss/gpu/utils/ConversionOperators.cuh>
#include <faiss/gpu/utils/CopyUtils.cuh>
#include <faiss/gpu/utils/DeviceDefs.cuh>
#include <faiss/gpu/utils/DeviceUtils.h>
#include <faiss/gpu/utils/Float16.cuh>
#include <faiss/gpu/utils/HostTensor.cuh>
#include <faiss/gpu/utils/Transpose.cuh>
#include <limits>
#include <thrust/host_vector.h>
#include <unordered_map>

namespace faiss { namespace gpu {

IVFFlat::IVFFlat(GpuResources* res,
                 FlatIndex* quantizer,
                 faiss::MetricType metric,
                 float metricArg,
                 bool useResidual,
                 faiss::ScalarQuantizer* scalarQ,
                 IndicesOptions indicesOptions,
                 MemorySpace space) :
    IVFBase(res,
            metric,
            metricArg,
            quantizer,
            indicesOptions,
            space),
    useResidual_(useResidual),
    scalarQ_(scalarQ ? new GpuScalarQuantizer(res, *scalarQ) : nullptr) {
}

IVFFlat::~IVFFlat() {
}

size_t
IVFFlat::getGpuVectorsEncodingSize_(int numVecs) const {
  return (size_t) numVecs *
    // size per encoded vector
    (scalarQ_ ? scalarQ_->code_size : sizeof(float) * getDim());
}

size_t
IVFFlat::getCpuVectorsEncodingSize_(int numVecs) const {
  return (size_t) numVecs *
    // size per encoded vector
    (scalarQ_ ? scalarQ_->code_size : sizeof(float) * getDim());
}

std::vector<unsigned char>
IVFFlat::translateCodesToGpu_(std::vector<unsigned char> codes,
                              size_t numVecs) const {
  // nothing to do
  return codes;
}

std::vector<unsigned char>
IVFFlat::translateCodesFromGpu_(std::vector<unsigned char> codes,
                                size_t numVecs) const {
  // nothing to do
{'fraction_non_alphanumeric': 0.11256190904997748, 'fraction_numerical': 0.0009004952723998199, 'mean_word_length': 2.9811827956989245, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455094', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1463, 'n_words': 306}