<commit_before>/*!
 * Copyright (c) 2017 by Contributors
 * \file main.cpp
 * \brief model-converter main
 * \author HPI-DeepLearning
*/
#include <stdio.h>
#include <libgen.h>
#include <fstream>

#include <mxnet/ndarray.h>

#include "../src/xnor_cpu.h"

#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

using mxnet::op::xnor_cpu::BITS_PER_BINARY_WORD;
using mxnet::op::xnor_cpu::BINARY_WORD;


void convert_to_binary(mxnet::NDArray& array) {
  assert(mshadow::mshadow_sizeof(array.dtype()) == sizeof(BINARY_WORD));
  assert(array.shape().ndim() == 4); // adjust for FC, flatten
  assert(array.shape()[1] % BITS_PER_BINARY_WORD == 0);
  nnvm::TShape binarized_shape(1);
  size_t size = array.shape().Size();
  binarized_shape[0] = size / BITS_PER_BINARY_WORD;
  mxnet::NDArray temp(binarized_shape, mxnet::Context::CPU(), false, array.dtype());
  mxnet::op::xnor_cpu::get_binary_row((float*) array.data().dptr_, (BINARY_WORD*) temp.data().dptr_, size);
  array = temp;
}

int convert_params_file(const std::string& input_file, const std::string& output_file) {
  std::vector<mxnet::NDArray> data;
  std::vector<std::string> keys;

  std::cout << "loading " << input_file << "..." << std::endl;
  { // loading params file into data and keys
    std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(input_file.c_str(), "r"));
    mxnet::NDArray::Load(fi.get(), &data, &keys);
  }

  const std::string filter_string("qconvolution");
  auto containsSubString = [filter_string](std::string s) {
    return s.find(filter_string) != std::string::npos;}; //@todo: add FC

  auto iter = std::find_if(keys.begin(),
                           keys.end(),
                           containsSubString);

  //Use a while loop, checking whether iter is at the end of myVector
  //Do a find_if starting at the item after iter, std::next(iter)
  while (iter != keys.end())
  {
    std::cout << "|- converting weights " << *iter << "..." << std::endl;
    convert_to_binary(data[iter - keys.begin()]);
    iter = std::find_if(std::next(iter),
                        keys.end(),
                        containsSubString);
  }


  { // saving params back to *_converted
    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(output_file.c_str(), "w"));
    mxnet::NDArray::Save(fo.get(), data, keys);
  }
  std::cout << "wrote converted params to " << output_file << std::endl;
  return 0;
}

int convert_json_file(const std::string& input_fname, const std::string& output_fname) {
  std::cout << "loading " << input_fname << "..." << std::endl;
  std::string json;
  {
    std::ifstream stream(input_fname);
    if (!stream.is_open()) {<commit_msg>binarization also for fully connected layers<commit_after>/*!
 * Copyright (c) 2017 by Contributors
 * \file main.cpp
 * \brief model-converter main
 * \author HPI-DeepLearning
*/
#include <stdio.h>
#include <libgen.h>
#include <fstream>

#include <mxnet/ndarray.h>

#include "../src/xnor_cpu.h"

#include "rapidjson/document.h"
#include "rapidjson/writer.h"
#include "rapidjson/stringbuffer.h"

using mxnet::op::xnor_cpu::BITS_PER_BINARY_WORD;
using mxnet::op::xnor_cpu::BINARY_WORD;


void convert_to_binary(mxnet::NDArray& array) {
  assert(mshadow::mshadow_sizeof(array.dtype()) == sizeof(BINARY_WORD));
  // dim. checks, watch out, second dim is checked to / 32, dont oversee if this really holds for fc
  assert(array.shape().ndim() >= 2); // second dimension is input depth from prev. layer, needed for next line
  assert(array.shape()[1] % BITS_PER_BINARY_WORD == 0); // depth from input has to be divisible by 32 (or 64) since we
  nnvm::TShape binarized_shape(1);
  size_t size = array.shape().Size();
  binarized_shape[0] = size / BITS_PER_BINARY_WORD;
  mxnet::NDArray temp(binarized_shape, mxnet::Context::CPU(), false, array.dtype());
  mxnet::op::xnor_cpu::get_binary_row((float*) array.data().dptr_, (BINARY_WORD*) temp.data().dptr_, size);
  array = temp;
}

int convert_params_file(const std::string& input_file, const std::string& output_file) {
  std::vector<mxnet::NDArray> data;
  std::vector<std::string> keys;

  std::cout << "loading " << input_file << "..." << std::endl;
  { // loading params file into data and keys
    std::unique_ptr<dmlc::Stream> fi(dmlc::Stream::Create(input_file.c_str(), "r"));
    mxnet::NDArray::Load(fi.get(), &data, &keys);
  }

  //const std::string filter_strings();
  const std::vector<std::string> filter_strings {"qconvolution", "qfullyconnected"};


  auto containsFilterString = [filter_strings](std::string line_in_params) {
    auto containsSubString = [line_in_params](std::string filter_string) {
      return line_in_params.find(filter_string) != std::string::npos;};
    return std::find_if(filter_strings.begin(),
                        filter_strings.end(),
                        containsSubString) != filter_strings.end();};

  auto iter = std::find_if(keys.begin(),
                           keys.end(),
                           containsFilterString);

  //Use a while loop, checking whether iter is at the end of myVector
  //Do a find_if starting at the item after iter, std::next(iter)
  while (iter != keys.end())
  {
    std::cout << "|- converting weights " << *iter << "..." << std::endl;
    CHECK((*iter).find("weight") != std::string::npos) << "onyl weight binarization supported currently";

    convert_to_binary(data[iter - keys.begin()]);

    iter = std::find_if(std::next(iter),
                        keys.end(),
                        containsFilterString);
  }


  { // saving params back to *_converted
    std::unique_ptr<dmlc::Stream> fo(dmlc::Stream::Create(output_file.c_str(), "w"));
    mxnet::NDArray::Save(fo.get(), data, keys);
  }
  std::cout << "wrote converted params to " << output_file << std::endl;
  return 0;
}

int convert_json_file(const std::string& input_fname, const std::string& output_fname) {
  std::cout << "loading " << input_fname << "..." << std::endl;
  std::string json;
  {
    std::ifstream stream(input_fname);
    if (!stream.is_open()) {
{'fraction_non_alphanumeric': 0.16820732915153516, 'fraction_numerical': 0.004291845493562232, 'mean_word_length': 3.9827302631578947, 'pattern_counts': {'":': 0, '<': 82, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3496196', 'n_tokens_mistral': 2079, 'n_tokens_neox': 2031, 'n_words': 436}