<commit_before>#include "parse_json.hh"
#include "lightweight_nn_streamers.hh"

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <cassert>
#include <string>

#include <iostream>

namespace {
  using namespace boost::property_tree;
  using namespace lwt;
  lwt::Activation get_activation(const std::string&);
  lwt::Architecture get_architecture(const std::string&);
  void add_dense_info(LayerConfig& lc, const ptree& pt);
}


namespace lwt {

  JSONConfig parse_json(std::istream& json)
  {
    boost::property_tree::ptree pt;
    boost::property_tree::read_json(json, pt);

    JSONConfig cfg;
    for (const auto& v: pt.get_child("inputs")) {
      std::string name = v.second.get<std::string>("name");
      auto offset = v.second.get<double>("offset");
      auto scale = v.second.get<double>("scale");
      Input input{name, offset, scale};
      cfg.inputs.push_back(input);
    }
    for (const auto& v: pt.get_child("layers")) {
      LayerConfig layer;
      layer.activation = get_activation(
        v.second.get<std::string>("activation"));
      layer.architecture = get_architecture(
        v.second.get<std::string>("architecture"));
      // todo: make this weights stuff happen in another function
      for (const auto& wt: v.second.get_child("weights")) {
        layer.weights.push_back(wt.second.get_value<double>());
      }
      for (const auto& bs: v.second.get_child("bias")) {
        layer.bias.push_back(bs.second.get_value<double>());
      }
      // todo: add maxout_tensor filler
      cfg.layers.push_back(layer);
    }
    for (const auto& v: pt.get_child("outputs"))
    {
      assert(v.first.empty()); // array elements have no names
      cfg.outputs.push_back(v.second.data());
    }
    const std::string dname = "defaults";
    if (pt.count(dname)) {
      for (const auto& def: pt.get_child(dname)) {
        cfg.defaults.emplace(def.first, def.second.get_value<double>());
      }
    }
    return cfg;
  }

}

namespace {
  lwt::Activation get_activation(const std::string& str) {
    using namespace lwt;
    if (str == "linear") return Activation::LINEAR;
    if (str == "sigmoid") return Activation::SIGMOID;
    if (str == "rectified") return Activation::RECTIFIED;
    if (str == "softmax") return Activation::SOFTMAX;
    throw std::logic_error("activation function " + str + " not recognized");
  }
  lwt::Architecture get_architecture(const std::string& str) {
    using namespace lwt;
    if (str == "dense") return Architecture::DENSE;
    if (str == "maxout") return Architecture::MAXOUT;
    throw std::logic_error("architecture " + str + " not recognized");
  }
  void add_dense_info(LayerConfig& lc, const ptree& pt) {

  }
}<commit_msg>Maxout seems to be working<commit_after>#include "parse_json.hh"
#include "lightweight_nn_streamers.hh"

#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <cassert>
#include <string>

#include <iostream>

namespace {
  using namespace boost::property_tree;
  using namespace lwt;
  lwt::Activation get_activation(const std::string&);
  lwt::Architecture get_architecture(const std::string&);
  void add_dense_info(LayerConfig& lc, const ptree::value_type& pt);
  void add_maxout_info(LayerConfig& lc, const ptree::value_type& pt);
}


namespace lwt {

  JSONConfig parse_json(std::istream& json)
  {
    boost::property_tree::ptree pt;
    boost::property_tree::read_json(json, pt);

    JSONConfig cfg;
    for (const auto& v: pt.get_child("inputs")) {
      std::string name = v.second.get<std::string>("name");
      auto offset = v.second.get<double>("offset");
      auto scale = v.second.get<double>("scale");
      Input input{name, offset, scale};
      cfg.inputs.push_back(input);
    }
    for (const auto& v: pt.get_child("layers")) {
      LayerConfig layer;
      layer.activation = get_activation(
        v.second.get<std::string>("activation"));
      layer.architecture = get_architecture(
        v.second.get<std::string>("architecture"));

      if (layer.architecture == Architecture::DENSE) {
        add_dense_info(layer, v);
      } else if (layer.architecture == Architecture::MAXOUT) {
        add_maxout_info(layer, v);
      }
      cfg.layers.push_back(layer);
    }
    for (const auto& v: pt.get_child("outputs"))
    {
      assert(v.first.empty()); // array elements have no names
      cfg.outputs.push_back(v.second.data());
    }
    const std::string dname = "defaults";
    if (pt.count(dname)) {
      for (const auto& def: pt.get_child(dname)) {
        cfg.defaults.emplace(def.first, def.second.get_value<double>());
      }
    }
    return cfg;
  }

}

namespace {
  lwt::Activation get_activation(const std::string& str) {
    using namespace lwt;
    if (str == "linear") return Activation::LINEAR;
    if (str == "sigmoid") return Activation::SIGMOID;
    if (str == "rectified") return Activation::RECTIFIED;
    if (str == "softmax") return Activation::SOFTMAX;
    throw std::logic_error("activation function " + str + " not recognized");
  }

  lwt::Architecture get_architecture(const std::string& str) {
    using namespace lwt;
    if (str == "dense") return Architecture::DENSE;
    if (str == "maxout") return Architecture::MAXOUT;
    throw std::logic_error("architecture " + str + " not recognized");
  }

  void add_dense_info(LayerConfig& layer, const ptree::value_type& v) {
    for (const auto& wt: v.second.get_child("weights")) {
      layer.weights.push_back(wt.second.get_value<double>());
    }
    for (const auto& bs: v.second.get_child("bias")) {
      layer.bias.push_back(bs.second.get_value<double>());
    }
  }

  void add_maxout_info(LayerConfig& layer, const ptree::value_type& v) {
    for (const auto& mat: v.second.get_child("weights")) {
      layer.maxout_tensor.push_back(std::vector<double>());
      for (const auto& wt: mat.second) {
        layer.maxout_tensor.back().push_back(wt.second.get_value<double>());
      }
    }
  }

}
{'fraction_non_alphanumeric': 0.1495342648037259, 'fraction_numerical': 0.0, 'mean_word_length': 3.723487824037706, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5153741', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1961, 'n_words': 408}