<commit_before>// -*- coding: utf-8 -*-
// Copyright (C) 2012-2017 MUJIN Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/** \file mujinjson.h
    \brief Wrapper for rapidjson.
 */
#ifndef MUJIN_CONTROLLERCLIENT_JSON_H
#define MUJIN_CONTROLLERCLIENT_JSON_H

#include <array>
#include <boost/shared_ptr.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <stdint.h>
#include <string>
#include <stdexcept>
#include <vector>
#include <map>
#include <iostream>

#include <rapidjson/pointer.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/ostreamwrapper.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/writer.h>
#include <rapidjson/error/en.h>
#include <rapidjson/prettywriter.h>

namespace mujinclient {
namespace mujinjson_external {

enum MujinJSONErrorCode
{
    MJE_Failed=0,
};

inline const char* GetErrorCodeString(MujinJSONErrorCode error)
{
    switch(error) {
    case MJE_Failed: return "Failed";
    }
    // throw an exception?
    return "";
}

/// \brief Exception that MujinJSON internal methods throw; the error codes are held in \ref MujinJSONErrorCode.
class MujinJSONException : public std::exception
{
public:
    MujinJSONException() : std::exception(), _s(""), _error(MJE_Failed) {
    }
    MujinJSONException(const std::string& s, MujinJSONErrorCode error=MJE_Failed ) : std::exception() {
        _error = error;
        _s = "mujin (";
        _s += "): ";
        _s += s;
        _ssub = s;
    }
    virtual ~MujinJSONException() throw() {
    }

    /// \brief outputs everything
    char const* what() const throw() {
        return _s.c_str();
    }

    /// \brief outputs everything
    const std::string& message() const {
        return _s;
    }

    /// \briefs just the sub-message
    const std::string& GetSubMessage() const {
        return _ssub;
    }

    MujinJSONErrorCode GetCode() const {
        return _error;
    }

private:
    std::string _s, _ssub;
    MujinJSONErrorCode _error;
};

template<class T> inline std::string GetJsonString(const T& t);

/// \brief gets a string of the Value type for debugging purposes
inline std::string GetJsonTypeName(const rapidjson::Value& v) {
    int type = v.GetType();
    switch (type) {
    case 0:
        return "Null";
    case 1:
        return "False";
    case 2:
        return "True";
    case 3:
        return "Object";
    case 4:
        return "Array";
    case 5:
        return "String";
    case 6:
        return "Number";
    default:
        return "Unknown";
    }
}

inline std::string DumpJson(const rapidjson::Value& value, const unsigned int indent=0) {
    rapidjson::StringBuffer stringbuffer;
    if (indent == 0) {
        rapidjson::Writer<rapidjson::StringBuffer> writer(stringbuffer);
        value.Accept(writer);
    } else {
        rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringbuffer);
        writer.SetIndent(' ', indent);
        value.Accept(writer);
    }
    return std::string(stringbuffer.GetString(), stringbuffer.GetSize());
}

inline void DumpJson(const rapidjson::Value& value, std::ostream& os, const unsigned int indent=0) {
    rapidjson::OStreamWrapper osw(os);
    if (indent == 0) {
        rapidjson::Writer<rapidjson::OStreamWrapper> writer(osw);
        value.Accept(writer);
    } else {
        rapidjson::PrettyWriter<rapidjson::OStreamWrapper> writer(osw);
        writer.SetIndent(' ', indent);
        value.Accept(writer);
    }
}

inline void ParseJson(rapidjson::Document& d, const std::string& str) {
    // repeatedly calling Parse on the same rapidjson::Document will not release previsouly allocated memory, memory will accumulate until the object is destroyed
    // we use a new temporary Document to parse, and swap content with the original one, so that memory in original Document will be released when this function ends
    // see: https://github.com/Tencent/rapidjson/issues/1333
    rapidjson::Document tempDoc;
    tempDoc.Parse<rapidjson::kParseFullPrecisionFlag>(str.c_str()); // parse float in full precision mode
    if (tempDoc.HasParseError()) {
        std::string substr;
        if (str.length()> 200) {
            substr = str.substr(0, 200);
        } else {
            substr = str;
        }
        throw MujinJSONException(boost::str(boost::format("Json string is invalid (offset %u) %s str=%s")%((unsigned)d.GetErrorOffset())%GetParseError_En(d.GetParseError())%substr), MJE_Failed);
    }
    d.Swap(tempDoc);
}

inline void ParseJson(rapidjson::Document& d, std::istream& is) {
    rapidjson::IStreamWrapper isw(is);
    // see note in: void ParseJson(rapidjson::Document& d, const std::string& str)
    rapidjson::Document(tempDoc);
    tempDoc.ParseStream<rapidjson::kParseFullPrecisionFlag>(isw); // parse float in full precision mode
    if (tempDoc.HasParseError()) {
        throw MujinJSONException(boost::str(boost::format("Json stream is invalid (offset %u) %s")%((unsigned)d.GetErrorOffset())%GetParseError_En(d.GetParseError())), MJE_Failed);
    }
    d.Swap(tempDoc);
}

class JsonSerializable {
public:
    virtual void LoadFromJson(const rapidjson::Value& v) = 0;
    virtual void SaveToJson(rapidjson::Value& v, rapidjson::Document::AllocatorType& alloc) const = 0;
    virtual void SaveToJson(rapidjson::Document& d) const {
        SaveToJson(d, d.GetAllocator());
    }
};

template<class T, class S> inline T LexicalCast(const S& v, const std::string& typeName) {
    try {
        T t = boost::lexical_cast<T>(v);
        return t;
    }
    catch (const boost::bad_lexical_cast& ex) {
        std::stringstream ss;
        ss << v;
        throw MujinJSONException("Cannot convert \"" + ss.str() + "\" to type " + typeName, MJE_Failed);
    }
}

//store a json value to local data structures
//for compatibility with ptree, type conversion is made. will remove them in the future
inline void LoadJsonValue(const rapidjson::Value& v, JsonSerializable& t) {
    t.LoadFromJson(v);
}

inline void LoadJsonValue(const rapidjson::Value& v, std::string& t) {
    if (v.IsString()) {
        t = v.GetString();
    } else if (v.IsInt64()) {
        //TODO: add warnings on all usages of lexical_cast
        t = boost::lexical_cast<std::string>(v.GetInt64());
    } else {
        throw MujinJSONException("Cannot convert json value " + GetJsonString(v) + " to String", MJE_Failed);
    }

}

inline void LoadJsonValue(const rapidjson::Value& v, int& t) {
    if (v.IsInt()) {
        t = v.GetInt();
    } else if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<int>(v.GetString(), "Int");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, int8_t& t) {
    if (v.IsInt()) {
        t = v.GetInt();
    }
    else if (v.IsUint()) {
        t = v.GetUint();
    }
    else if (v.IsString()) {
        t = LexicalCast<int>(v.GetString(), "Int8");
    }
    else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int8", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, uint8_t& t) {
    if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned int>(v.GetString(), "UInt8");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt8", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, unsigned int& t) {
    if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned int>(v.GetString(), "UInt");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, unsigned long long& t) {
    if (v.IsUint64()) {
        t = v.GetUint64();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned long long>(v.GetString(), "ULongLong");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to ULongLong", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, uint64_t& t) {
    if (v.IsUint64()) {
        t = v.GetUint64();
    } else if (v.IsString()) {
        t = LexicalCast<uint64_t>(v.GetString(), "UInt64");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt64", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, int64_t& t) {
    if (v.IsInt64()) {
        t = v.GetInt64();
    } else if (v.IsString()) {
        t = LexicalCast<int64_t>(v.GetString(), "Int64");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int64", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, double& t) {
    if (v.IsNumber()) {
        t = v.GetDouble();
    } else if (v.IsString()) {
        t = LexicalCast<double>(v.GetString(), "Double");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Double", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, float& t) {
    if (v.IsNumber()) {
        t = v.GetDouble();
    } else if (v.IsString()) {
        t = LexicalCast<double>(v.GetString(), "Float");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Float", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, bool& t) {
    if (v.IsInt()) t = v.GetInt();
    else if (v.IsBool()) t = v.GetBool();
    else if (v.IsString())  {
        t = LexicalCast<bool>(v.GetString(), "Bool");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Bool", MJE_Failed);
    }
}

template<class T> inline void LoadJsonValue(const rapidjson::Value& v, std::vector<T>& t);

template<class T, size_t N> inline void LoadJsonValue(const rapidjson::Value& v, std::array<T, N>& t);

template<class T> inline void LoadJsonValue(const rapidjson::Value& v, boost::shared_ptr<T>& ptr) {
    T t;
    LoadJsonValue(v, t);
    ptr = boost::shared_ptr<T>(new T(t));
}<commit_msg>mujinjson: LoadJsonValue: T must be default constructible, also use make_shared.<commit_after>// -*- coding: utf-8 -*-
// Copyright (C) 2012-2017 MUJIN Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/** \file mujinjson.h
    \brief Wrapper for rapidjson.
 */
#ifndef MUJIN_CONTROLLERCLIENT_JSON_H
#define MUJIN_CONTROLLERCLIENT_JSON_H

#include <array>
#include <boost/shared_ptr.hpp>
#include <boost/smart_ptr/make_shared.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
#include <stdint.h>
#include <string>
#include <stdexcept>
#include <vector>
#include <map>
#include <iostream>

#include <rapidjson/pointer.h>
#include <rapidjson/stringbuffer.h>
#include <rapidjson/ostreamwrapper.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/writer.h>
#include <rapidjson/error/en.h>
#include <rapidjson/prettywriter.h>

namespace mujinclient {
namespace mujinjson_external {

enum MujinJSONErrorCode
{
    MJE_Failed=0,
};

inline const char* GetErrorCodeString(MujinJSONErrorCode error)
{
    switch(error) {
    case MJE_Failed: return "Failed";
    }
    // throw an exception?
    return "";
}

/// \brief Exception that MujinJSON internal methods throw; the error codes are held in \ref MujinJSONErrorCode.
class MujinJSONException : public std::exception
{
public:
    MujinJSONException() : std::exception(), _s(""), _error(MJE_Failed) {
    }
    MujinJSONException(const std::string& s, MujinJSONErrorCode error=MJE_Failed ) : std::exception() {
        _error = error;
        _s = "mujin (";
        _s += "): ";
        _s += s;
        _ssub = s;
    }
    virtual ~MujinJSONException() throw() {
    }

    /// \brief outputs everything
    char const* what() const throw() {
        return _s.c_str();
    }

    /// \brief outputs everything
    const std::string& message() const {
        return _s;
    }

    /// \briefs just the sub-message
    const std::string& GetSubMessage() const {
        return _ssub;
    }

    MujinJSONErrorCode GetCode() const {
        return _error;
    }

private:
    std::string _s, _ssub;
    MujinJSONErrorCode _error;
};

template<class T> inline std::string GetJsonString(const T& t);

/// \brief gets a string of the Value type for debugging purposes
inline std::string GetJsonTypeName(const rapidjson::Value& v) {
    int type = v.GetType();
    switch (type) {
    case 0:
        return "Null";
    case 1:
        return "False";
    case 2:
        return "True";
    case 3:
        return "Object";
    case 4:
        return "Array";
    case 5:
        return "String";
    case 6:
        return "Number";
    default:
        return "Unknown";
    }
}

inline std::string DumpJson(const rapidjson::Value& value, const unsigned int indent=0) {
    rapidjson::StringBuffer stringbuffer;
    if (indent == 0) {
        rapidjson::Writer<rapidjson::StringBuffer> writer(stringbuffer);
        value.Accept(writer);
    } else {
        rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(stringbuffer);
        writer.SetIndent(' ', indent);
        value.Accept(writer);
    }
    return std::string(stringbuffer.GetString(), stringbuffer.GetSize());
}

inline void DumpJson(const rapidjson::Value& value, std::ostream& os, const unsigned int indent=0) {
    rapidjson::OStreamWrapper osw(os);
    if (indent == 0) {
        rapidjson::Writer<rapidjson::OStreamWrapper> writer(osw);
        value.Accept(writer);
    } else {
        rapidjson::PrettyWriter<rapidjson::OStreamWrapper> writer(osw);
        writer.SetIndent(' ', indent);
        value.Accept(writer);
    }
}

inline void ParseJson(rapidjson::Document& d, const std::string& str) {
    // repeatedly calling Parse on the same rapidjson::Document will not release previsouly allocated memory, memory will accumulate until the object is destroyed
    // we use a new temporary Document to parse, and swap content with the original one, so that memory in original Document will be released when this function ends
    // see: https://github.com/Tencent/rapidjson/issues/1333
    rapidjson::Document tempDoc;
    tempDoc.Parse<rapidjson::kParseFullPrecisionFlag>(str.c_str()); // parse float in full precision mode
    if (tempDoc.HasParseError()) {
        std::string substr;
        if (str.length()> 200) {
            substr = str.substr(0, 200);
        } else {
            substr = str;
        }
        throw MujinJSONException(boost::str(boost::format("Json string is invalid (offset %u) %s str=%s")%((unsigned)d.GetErrorOffset())%GetParseError_En(d.GetParseError())%substr), MJE_Failed);
    }
    d.Swap(tempDoc);
}

inline void ParseJson(rapidjson::Document& d, std::istream& is) {
    rapidjson::IStreamWrapper isw(is);
    // see note in: void ParseJson(rapidjson::Document& d, const std::string& str)
    rapidjson::Document(tempDoc);
    tempDoc.ParseStream<rapidjson::kParseFullPrecisionFlag>(isw); // parse float in full precision mode
    if (tempDoc.HasParseError()) {
        throw MujinJSONException(boost::str(boost::format("Json stream is invalid (offset %u) %s")%((unsigned)d.GetErrorOffset())%GetParseError_En(d.GetParseError())), MJE_Failed);
    }
    d.Swap(tempDoc);
}

class JsonSerializable {
public:
    virtual void LoadFromJson(const rapidjson::Value& v) = 0;
    virtual void SaveToJson(rapidjson::Value& v, rapidjson::Document::AllocatorType& alloc) const = 0;
    virtual void SaveToJson(rapidjson::Document& d) const {
        SaveToJson(d, d.GetAllocator());
    }
};

template<class T, class S> inline T LexicalCast(const S& v, const std::string& typeName) {
    try {
        T t = boost::lexical_cast<T>(v);
        return t;
    }
    catch (const boost::bad_lexical_cast& ex) {
        std::stringstream ss;
        ss << v;
        throw MujinJSONException("Cannot convert \"" + ss.str() + "\" to type " + typeName, MJE_Failed);
    }
}

//store a json value to local data structures
//for compatibility with ptree, type conversion is made. will remove them in the future
inline void LoadJsonValue(const rapidjson::Value& v, JsonSerializable& t) {
    t.LoadFromJson(v);
}

inline void LoadJsonValue(const rapidjson::Value& v, std::string& t) {
    if (v.IsString()) {
        t = v.GetString();
    } else if (v.IsInt64()) {
        //TODO: add warnings on all usages of lexical_cast
        t = boost::lexical_cast<std::string>(v.GetInt64());
    } else {
        throw MujinJSONException("Cannot convert json value " + GetJsonString(v) + " to String", MJE_Failed);
    }

}

inline void LoadJsonValue(const rapidjson::Value& v, int& t) {
    if (v.IsInt()) {
        t = v.GetInt();
    } else if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<int>(v.GetString(), "Int");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, int8_t& t) {
    if (v.IsInt()) {
        t = v.GetInt();
    }
    else if (v.IsUint()) {
        t = v.GetUint();
    }
    else if (v.IsString()) {
        t = LexicalCast<int>(v.GetString(), "Int8");
    }
    else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int8", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, uint8_t& t) {
    if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned int>(v.GetString(), "UInt8");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt8", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, unsigned int& t) {
    if (v.IsUint()) {
        t = v.GetUint();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned int>(v.GetString(), "UInt");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, unsigned long long& t) {
    if (v.IsUint64()) {
        t = v.GetUint64();
    } else if (v.IsString()) {
        t = LexicalCast<unsigned long long>(v.GetString(), "ULongLong");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to ULongLong", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, uint64_t& t) {
    if (v.IsUint64()) {
        t = v.GetUint64();
    } else if (v.IsString()) {
        t = LexicalCast<uint64_t>(v.GetString(), "UInt64");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to UInt64", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, int64_t& t) {
    if (v.IsInt64()) {
        t = v.GetInt64();
    } else if (v.IsString()) {
        t = LexicalCast<int64_t>(v.GetString(), "Int64");
    } else if (v.IsBool()) {
        t = v.GetBool() ? 1 : 0;
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Int64", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, double& t) {
    if (v.IsNumber()) {
        t = v.GetDouble();
    } else if (v.IsString()) {
        t = LexicalCast<double>(v.GetString(), "Double");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Double", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, float& t) {
    if (v.IsNumber()) {
        t = v.GetDouble();
    } else if (v.IsString()) {
        t = LexicalCast<double>(v.GetString(), "Float");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Float", MJE_Failed);
    }
}

inline void LoadJsonValue(const rapidjson::Value& v, bool& t) {
    if (v.IsInt()) t = v.GetInt();
    else if (v.IsBool()) t = v.GetBool();
    else if (v.IsString())  {
        t = LexicalCast<bool>(v.GetString(), "Bool");
    } else {
        throw MujinJSONException("Cannot convert json type " + GetJsonString(v) + " to Bool", MJE_Failed);
    }
}

template<class T> inline void LoadJsonValue(const rapidjson::Value& v, std::vector<T>& t);

template<class T, size_t N> inline void LoadJsonValue(const rapidjson::Value& v, std::array<T, N>& t);

template<class T> inline void LoadJsonValue(const rapidjson::Value& v, boost::shared_ptr<T>& ptr) {
    static_assert(std::is_default_constructible<T>::value, "Shared pointer of type must be default-constructible.");
    ptr = boost::make_shared<T>();
    LoadJsonValue(v, *ptr);
}
{'fraction_non_alphanumeric': 0.1411775106082037, 'fraction_numerical': 0.007956152758132957, 'mean_word_length': 3.11438443353337, 'pattern_counts': {'":': 0, '<': 97, '<?xml version=': 0, '>': 95, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1998941', 'n_tokens_mistral': 7507, 'n_tokens_neox': 7384, 'n_words': 1958}