<commit_before>/******************************************************************************
 * Copyright 2018 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include <Python.h>
#include <string>

#include "python/wrapper/py_record.h"

#define PYOBJECT_NULL_STRING PyString_FromStringAndSize("", 0)

template <typename T>
T PyObjectToPtr(PyObject *pyobj, const std::string &type_ptr) {
  T obj_ptr = (T)PyCapsule_GetPointer(pyobj, type_ptr.c_str());
  if (obj_ptr == nullptr) {
    AINFO << "PyObjectToPtr failed,type->" << type_ptr << "pyobj: " << pyobj;
  }
  return obj_ptr;
}

PyObject *cyber_new_PyRecordReader(PyObject *self, PyObject *args) {
  char *filepath = nullptr;
  Py_ssize_t len = 0;
  if (!PyArg_ParseTuple(args, const_cast<char *>("s#:new_PyRecordReader"),
                        &filepath, &len)) {
    return Py_None;
  }

  apollo::cybertron::record::PyRecordReader *reader =
      new apollo::cybertron::record::PyRecordReader(std::string(filepath, len));
  PyObject *pyobj_rec_reader =
      PyCapsule_New(reader, "apollo_cybertron_record_pyrecordfilereader", NULL);
  return pyobj_rec_reader;
}

PyObject *cyber_delete_PyRecordReader(PyObject *self, PyObject *args) {
  PyObject *pyobj_rec_reader = nullptr;
  if (!PyArg_ParseTuple(args, const_cast<char *>("O:delete_PyRecordReader"),
                        &pyobj_rec_reader)) {
    return Py_None;
  }

  auto reader =
      (apollo::cybertron::record::PyRecordReader *)PyCapsule_GetPointer(
          pyobj_rec_reader, "apollo_cybertron_record_pyrecordfilereader");
  if (nullptr == reader) {
    AINFO << "delete_PyRecordReader:reader ptr is null!";
    return Py_None;
  }
  delete reader;
  return Py_None;
}

PyObject *cyber_PyRecordReader_ReadMessage(PyObject *self, PyObject *args) {
  PyObject *pyobj_reader = nullptr;
  uint64_t begin_time = 0;
  uint64_t end_time = UINT64_MAX;
  if (!PyArg_ParseTuple(args,
                        const_cast<char *>("OKK:PyRecordReader_ReadMessage"),
                        &pyobj_reader, &begin_time, &end_time)) {
    return nullptr;
  }

  auto reader =
      (apollo::cybertron::record::PyRecordReader *)PyCapsule_GetPointer(
          pyobj_reader, "apollo_cybertron_record_pyrecordfilereader");
  if (nullptr == reader) {
    AERROR << "PyRecordReader_ReadMessage ptr is null!";
    return nullptr;
  }

  apollo::cybertron::record::BagMessage result;
  result = reader->ReadMessage(begin_time, end_time);
  PyObject *pyobj_bag_message = pyobj_bag_message = PyDict_New();
  PyDict_SetItemString(pyobj_bag_message, "channel_name",
                       Py_BuildValue("s", result.channel_name.c_str()));
  PyDict_SetItemString(pyobj_bag_message, "data",
                       Py_BuildValue("s", result.data.c_str()));
  PyDict_SetItemString(pyobj_bag_message, "data_type",
                       Py_BuildValue("s", result.data_type.c_str()));
  PyDict_SetItem(pyobj_bag_message, Py_BuildValue("s", "timestamp"),
                 Py_BuildValue("K", result.timestamp));
  PyDict_SetItem(pyobj_bag_message, Py_BuildValue("s", "end"),
                 result.end ? Py_True : Py_False);

  return pyobj_bag_message;
}

PyObject *cyber_PyRecordReader_GetMessageNumber(PyObject *self,
                                                PyObject *args) {
  PyObject *pyobj_reader = nullptr;
  char *channel_name = nullptr;
  Py_ssize_t len = 0;<commit_msg>framework: fix message being truncated at null character issue<commit_after>/******************************************************************************
 * Copyright 2018 The Apollo Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *****************************************************************************/

#include <Python.h>
#include <string>

#include "python/wrapper/py_record.h"

#define PYOBJECT_NULL_STRING PyString_FromStringAndSize("", 0)

template <typename T>
T PyObjectToPtr(PyObject *pyobj, const std::string &type_ptr) {
  T obj_ptr = (T)PyCapsule_GetPointer(pyobj, type_ptr.c_str());
  if (obj_ptr == nullptr) {
    AINFO << "PyObjectToPtr failed,type->" << type_ptr << "pyobj: " << pyobj;
  }
  return obj_ptr;
}

PyObject *cyber_new_PyRecordReader(PyObject *self, PyObject *args) {
  char *filepath = nullptr;
  Py_ssize_t len = 0;
  if (!PyArg_ParseTuple(args, const_cast<char *>("s#:new_PyRecordReader"),
                        &filepath, &len)) {
    return Py_None;
  }

  apollo::cybertron::record::PyRecordReader *reader =
      new apollo::cybertron::record::PyRecordReader(std::string(filepath, len));
  PyObject *pyobj_rec_reader =
      PyCapsule_New(reader, "apollo_cybertron_record_pyrecordfilereader", NULL);
  return pyobj_rec_reader;
}

PyObject *cyber_delete_PyRecordReader(PyObject *self, PyObject *args) {
  PyObject *pyobj_rec_reader = nullptr;
  if (!PyArg_ParseTuple(args, const_cast<char *>("O:delete_PyRecordReader"),
                        &pyobj_rec_reader)) {
    return Py_None;
  }

  auto reader =
      (apollo::cybertron::record::PyRecordReader *)PyCapsule_GetPointer(
          pyobj_rec_reader, "apollo_cybertron_record_pyrecordfilereader");
  if (nullptr == reader) {
    AINFO << "delete_PyRecordReader:reader ptr is null!";
    return Py_None;
  }
  delete reader;
  return Py_None;
}

PyObject *cyber_PyRecordReader_ReadMessage(PyObject *self, PyObject *args) {
  PyObject *pyobj_reader = nullptr;
  uint64_t begin_time = 0;
  uint64_t end_time = UINT64_MAX;
  if (!PyArg_ParseTuple(args,
                        const_cast<char *>("OKK:PyRecordReader_ReadMessage"),
                        &pyobj_reader, &begin_time, &end_time)) {
    return nullptr;
  }

  auto reader =
      (apollo::cybertron::record::PyRecordReader *)PyCapsule_GetPointer(
          pyobj_reader, "apollo_cybertron_record_pyrecordfilereader");
  if (nullptr == reader) {
    AERROR << "PyRecordReader_ReadMessage ptr is null!";
    return nullptr;
  }

  apollo::cybertron::record::BagMessage result;
  result = reader->ReadMessage(begin_time, end_time);
  PyObject *pyobj_bag_message = pyobj_bag_message = PyDict_New();
  PyDict_SetItemString(pyobj_bag_message, "channel_name",
                       Py_BuildValue("s", result.channel_name.c_str()));
  PyDict_SetItemString(pyobj_bag_message, "data",
                       Py_BuildValue("s#", result.data.c_str(), result.data.length()));
  PyDict_SetItemString(pyobj_bag_message, "data_type",
                       Py_BuildValue("s", result.data_type.c_str()));
  PyDict_SetItem(pyobj_bag_message, Py_BuildValue("s", "timestamp"),
                 Py_BuildValue("K", result.timestamp));
  PyDict_SetItem(pyobj_bag_message, Py_BuildValue("s", "end"),
                 result.end ? Py_True : Py_False);

  return pyobj_bag_message;
}

PyObject *cyber_PyRecordReader_GetMessageNumber(PyObject *self,
                                                PyObject *args) {
  PyObject *pyobj_reader = nullptr;
  char *channel_name = nullptr;
  Py_ssize_t len = 0;
{'fraction_non_alphanumeric': 0.1433696190004948, 'fraction_numerical': 0.004453240969816922, 'mean_word_length': 4.0405236907730675, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5620357', 'n_tokens_mistral': 2532, 'n_tokens_neox': 2479, 'n_words': 556}