<commit_before>/*
 * Medical Image Registration ToolKit (MIRTK)
 *
 * Copyright 2013-2015 Imperial College London
 * Copyright 2013-2015 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "mirtk/PointSetUtils.h"

#include "mirtk/Vtk.h"
#include "mirtk/Utils.h"
#include "mirtk/Math.h"
#include "mirtk/Memory.h"
#include "mirtk/Stream.h"
#include "mirtk/Parallel.h"
#include "mirtk/EdgeTable.h"
#include "mirtk/DataStatistics.h"
#include "mirtk/PointSet.h"
#include "mirtk/Voxel.h"
#include "mirtk/BaseImage.h"
#include "mirtk/Vector.h"
#include "mirtk/Vector3D.h"
#include "mirtk/Matrix3x3.h"
#include "mirtk/Vector3.h"
#include "mirtk/Algorithm.h"
#include "mirtk/UnorderedSet.h"
#include "mirtk/List.h"

#include "vtkPolyData.h"
#include "vtkUnstructuredGrid.h"
#include "vtkStructuredGrid.h"
#include "vtkDataArray.h"
#include "vtkUnsignedShortArray.h"
#include "vtkFloatArray.h"
#include "vtkPointData.h"
#include "vtkCellData.h"
#include "vtkDataSetAttributes.h"
#include "vtkCell.h"
#include "vtkTriangle.h"
#include "vtkTetra.h"
#include "vtkGenericCell.h"
#include "vtkImageData.h"
#include "vtkImageStencilData.h"
#include "vtkConnectivityFilter.h"
#include "vtkPolyDataConnectivityFilter.h"

#include "vtkImageStencilIterator.h"
#include "vtkPolyDataToImageStencil.h"
#include "vtkCleanPolyData.h"
#include "vtkTriangleFilter.h"
#include "vtkDataSetSurfaceFilter.h"
#include "vtkHull.h"
#include "vtkMaskPoints.h"
#include "vtkDelaunay3D.h"
#include "vtkMassProperties.h"


namespace mirtk {


// =============================================================================
// VTK / MIRTK type conversion
// =============================================================================

// -----------------------------------------------------------------------------
void AddPoints(PointSet &oset, vtkPointSet *iset)
{
  double p[3];
  oset.Reserve(oset.Size() + static_cast<int>(iset->GetNumberOfPoints()));
  for (vtkIdType ptId = 0; ptId < iset->GetNumberOfPoints(); ++ptId) {
    iset->GetPoint(ptId, p);
    oset.Add(Point(p));
  }
}

// =============================================================================
// Point set domain
// =============================================================================

// -----------------------------------------------------------------------------
int Dimension(vtkDataSet *dataset)
{
  double bounds[6];
  dataset->GetBounds(bounds);
  double sx = bounds[1] - bounds[0];
  double sy = bounds[3] - bounds[2];
  double sz = bounds[5] - bounds[4];
  return ((sx > .0 ? 1 : 0) + (sy > .0 ? 1 : 0) + (sz > .0 ? 1 : 0));
}

// -----------------------------------------------------------------------------
ImageAttributes PointSetDomain(vtkPointSet *data, double dx, double dy, double dz)
{
  ImageAttributes attr;
  attr._dx = dx;
  attr._dy = dy >= .0 ? dy : dx;
  attr._dz = dz >= .0 ? dz : dx;
  attr._dt = .0;
  // Compute eigenvectors of covariance matrix
  double c[3], p[3];
  data->GetCenter(c);
  Matrix3x3 covar(.0);
  for (vtkIdType i = 0; i < data->GetNumberOfPoints(); ++i) {
    data->GetPoint(i, p);
    for (int d = 0; d < 3; ++d) p[d] -= c[d];
    for (int r = 0; r < 3; ++r) {
      for (int c = 0; c < 3; ++c) {
        covar[r][c] += p[r] * p[c];
      }
    }
  }
  double  eigen[3];
  Vector3 axis [3];
  covar.EigenSolveSymmetric(eigen, axis);
  Vector3::GenerateOrthonormalBasis(axis[0], axis[1], axis[2]);
  // Set output origin and orientation
  attr._xorigin = c[0];
  attr._yorigin = c[1];
  attr._zorigin = c[2];
  for (int d = 0; d < 3; ++d) {
    attr._xaxis[d] = axis[0][d];
    attr._yaxis[d] = axis[1][d];
    attr._zaxis[d] = axis[2][d];
  }
  // Determine bounds of reoriented data set
  double x, y, z;
  double xmin = numeric_limits<double>::max(), ymin = xmin, zmin = xmin;
  double xmax = -xmin, ymax = -ymin, zmax = -zmin;
  OrderedSet<double> xs, ys, zs;
  for (vtkIdType i = 0; i < data->GetNumberOfPoints(); ++i) {
    data->GetPoint(i, p);
    x = attr._xaxis[0] * p[0] + attr._xaxis[1] * p[1] + attr._xaxis[2] * p[2];
    y = attr._yaxis[0] * p[0] + attr._yaxis[1] * p[1] + attr._yaxis[2] * p[2];
    z = attr._zaxis[0] * p[0] + attr._zaxis[1] * p[1] + attr._zaxis[2] * p[2];
    if (x < xmin) xmin = x;
    if (x > xmax) xmax = x;
    if (y < ymin) ymin = y;
    if (y > ymax) ymax = y;
    if (z < zmin) zmin = z;
    if (z > zmax) zmax = z;
    if (attr._dx <= .0) xs.insert(x);
    if (attr._dy <= .0) ys.insert(y);
    if (attr._dz <= .0) zs.insert(z);
  }
  // Set output resolution and size
  const double extent[3]  = {xmax - xmin, ymax - ymin, zmax - zmin};
  const double avg_extent = (extent[0] + extent[1] + extent[2]) / 3.0;
  if (attr._dx <= .0) attr._dx = ((extent[0] / avg_extent > 1e-3) ? AverageInterval(xs) : .0);
  if (attr._dy <= .0) attr._dy = ((extent[1] / avg_extent > 1e-3) ? AverageInterval(ys) : .0);
  if (attr._dz <= .0) attr._dz = ((extent[2] / avg_extent > 1e-3) ? AverageInterval(zs) : .0);
  attr._x  = (attr._dx > .0 ? iround(extent[0] / attr._dx) : 0) + 1;
  attr._y  = (attr._dy > .0 ? iround(extent[1] / attr._dy) : 0) + 1;
  attr._z  = (attr._dz > .0 ? iround(extent[2] / attr._dz) : 0) + 1;
  attr._dx = (attr._x  >  1 ? extent[0] / (attr._x - 1) : .0);
  attr._dy = (attr._y  >  1 ? extent[1] / (attr._y - 1) : .0);
  attr._dz = (attr._z  >  1 ? extent[2] / (attr._z - 1) : .0);
  return attr;
}

// -----------------------------------------------------------------------------
ImageAttributes PointSetDomain(vtkPointSet *data, const Vector3D<double> &ds)
{
  return PointSetDomain(data, ds._x, ds._y, ds._z);
}

// -----------------------------------------------------------------------------
ImageAttributes PolyDataDomain(vtkPolyData *data, double dx, double dy, double dz)
{
  return PointSetDomain(data, dx, dy, dz);
}

// -----------------------------------------------------------------------------
ImageAttributes PolyDataDomain(vtkPolyData *data, const Vector3D<double> &ds)
{
  return PointSetDomain(data, ds);
}

// =============================================================================
// Point/cell data
// =============================================================================

// -----------------------------------------------------------------------------
int PolyDataAttributeType(const char *type)
{
  string ltype(type);
  transform(ltype.begin(), ltype.end(), ltype.begin(), ::tolower);
  if (ltype == "scalars")     return vtkDataSetAttributes::SCALARS;
  if (ltype == "vectors")     return vtkDataSetAttributes::VECTORS;
  if (ltype == "normals")     return vtkDataSetAttributes::NORMALS;
  if (ltype == "tcoords")     return vtkDataSetAttributes::TCOORDS;
  if (ltype == "tensors")     return vtkDataSetAttributes::TENSORS;
  if (ltype == "globalids")   return vtkDataSetAttributes::GLOBALIDS;
  if (ltype == "pedigreeids") return vtkDataSetAttributes::PEDIGREEIDS;
  if (ltype == "edgeflag")    return vtkDataSetAttributes::EDGEFLAG;
  return -1;
}

// -----------------------------------------------------------------------------
vtkDataArray *GetArrayByCaseInsensitiveName(vtkDataSetAttributes *data, const char *name, int *loc)
{
  string lname = ToLower(name), lower_name;
  for (int i = 0; i < data->GetNumberOfArrays(); ++i) {
    const char *array_name = data->GetArrayName(i);
    if (array_name) {
      lower_name = ToLower(array_name);
      if (lower_name == lname) {
        if (loc) *loc = i;
        return data->GetArray(i);
      }
    }
  }
  if (loc) *loc = -1;
  return NULL;
}

// -----------------------------------------------------------------------------
int DeepCopyArrayUsingCaseInsensitiveName(vtkDataSetAttributes *dst, vtkDataSetAttributes *src, const char *name, bool deep)
{
  int loc = -1;
  vtkDataArray *src_array = GetArrayByCaseInsensitiveName(src, name);
  if (src_array) {
    vtkDataArray *dst_array = GetArrayByCaseInsensitiveName(dst, name, &loc);
    if (dst_array) {
      dst_array->DeepCopy(src_array);
    } else {
      vtkSmartPointer<vtkDataArray> copy;
      copy = vtkSmartPointer<vtkDataArray>::NewInstance(src_array);
      copy->DeepCopy(src_array);
      loc = dst->AddArray(copy);
    }
  }
  return loc;
}

// =============================================================================
// Cells
// =============================================================================

// -----------------------------------------------------------------------------
// Attention: vtkGenericCell is not derived from the respective cell types
double ComputeArea(vtkCell *cell)
{
  if (cell->GetCellDimension() < 2) return .0;
  vtkPoints *points = cell->GetPoints();
  switch (cell->GetCellType()) {
    case VTK_TRIANGLE: {
      double p1[3], p2[3], p3[3];
      points->GetPoint(0, p1);
      points->GetPoint(1, p2);
      points->GetPoint(2, p3);
      return vtkTriangle::TriangleArea(p1, p2, p3);
    }
    default: return numeric_limits<double>::quiet_NaN();
  }
}

// -----------------------------------------------------------------------------
// Attention: vtkGenericCell is not derived from the respective cell types
double ComputeVolume(vtkCell *cell)
{
  if (cell->GetCellDimension() < 3) return .0;
  vtkPoints *points = cell->GetPoints();
  switch (cell->GetCellType()) {
    case VTK_TETRA: {
      double p1[3], p2[3], p3[3], p4[3];
      points->GetPoint(0, p1);
      points->GetPoint(1, p2);
      points->GetPoint(2, p3);
      points->GetPoint(3, p4);
      return vtkTetra::ComputeVolume(p1, p2, p3, p4);
    }
    default: return numeric_limits<double>::quiet_NaN();
  }
}

// =============================================================================
// Basic point set manipulation
// =============================================================================

// -----------------------------------------------------------------------------
vtkSmartPointer<vtkPolyData> DataSetSurface(vtkSmartPointer<vtkDataSet> dataset, bool passPtIds, bool passCellIds)
{
  vtkSmartPointer<vtkDataSetSurfaceFilter> surface;
  surface = vtkSmartPointer<vtkDataSetSurfaceFilter>::New();
  SetVTKInput(surface, dataset);
  surface->SetPassThroughPointIds(passPtIds);
  surface->SetPassThroughCellIds(passCellIds);
  surface->Update();
  return surface->GetOutput();
}

// -----------------------------------------------------------------------------
void Center(vtkSmartPointer<vtkPointSet> pointset)
{
  double c[3], p[3];<commit_msg>enh: Make shallow copy only when point set is already a surface mesh [PointSet]<commit_after>/*
 * Medical Image Registration ToolKit (MIRTK)
 *
 * Copyright 2013-2015 Imperial College London
 * Copyright 2013-2015 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "mirtk/PointSetUtils.h"

#include "mirtk/Vtk.h"
#include "mirtk/Utils.h"
#include "mirtk/Math.h"
#include "mirtk/Memory.h"
#include "mirtk/Stream.h"
#include "mirtk/Parallel.h"
#include "mirtk/EdgeTable.h"
#include "mirtk/DataStatistics.h"
#include "mirtk/PointSet.h"
#include "mirtk/Voxel.h"
#include "mirtk/BaseImage.h"
#include "mirtk/Vector.h"
#include "mirtk/Vector3D.h"
#include "mirtk/Matrix3x3.h"
#include "mirtk/Vector3.h"
#include "mirtk/Algorithm.h"
#include "mirtk/UnorderedSet.h"
#include "mirtk/List.h"

#include "vtkPolyData.h"
#include "vtkUnstructuredGrid.h"
#include "vtkStructuredGrid.h"
#include "vtkDataArray.h"
#include "vtkUnsignedShortArray.h"
#include "vtkFloatArray.h"
#include "vtkPointData.h"
#include "vtkCellData.h"
#include "vtkDataSetAttributes.h"
#include "vtkCell.h"
#include "vtkTriangle.h"
#include "vtkTetra.h"
#include "vtkGenericCell.h"
#include "vtkImageData.h"
#include "vtkImageStencilData.h"
#include "vtkConnectivityFilter.h"
#include "vtkPolyDataConnectivityFilter.h"

#include "vtkImageStencilIterator.h"
#include "vtkPolyDataToImageStencil.h"
#include "vtkCleanPolyData.h"
#include "vtkTriangleFilter.h"
#include "vtkDataSetSurfaceFilter.h"
#include "vtkHull.h"
#include "vtkMaskPoints.h"
#include "vtkDelaunay3D.h"
#include "vtkMassProperties.h"


namespace mirtk {


// =============================================================================
// VTK / MIRTK type conversion
// =============================================================================

// -----------------------------------------------------------------------------
void AddPoints(PointSet &oset, vtkPointSet *iset)
{
  double p[3];
  oset.Reserve(oset.Size() + static_cast<int>(iset->GetNumberOfPoints()));
  for (vtkIdType ptId = 0; ptId < iset->GetNumberOfPoints(); ++ptId) {
    iset->GetPoint(ptId, p);
    oset.Add(Point(p));
  }
}

// =============================================================================
// Point set domain
// =============================================================================

// -----------------------------------------------------------------------------
int Dimension(vtkDataSet *dataset)
{
  double bounds[6];
  dataset->GetBounds(bounds);
  double sx = bounds[1] - bounds[0];
  double sy = bounds[3] - bounds[2];
  double sz = bounds[5] - bounds[4];
  return ((sx > .0 ? 1 : 0) + (sy > .0 ? 1 : 0) + (sz > .0 ? 1 : 0));
}

// -----------------------------------------------------------------------------
ImageAttributes PointSetDomain(vtkPointSet *data, double dx, double dy, double dz)
{
  ImageAttributes attr;
  attr._dx = dx;
  attr._dy = dy >= .0 ? dy : dx;
  attr._dz = dz >= .0 ? dz : dx;
  attr._dt = .0;
  // Compute eigenvectors of covariance matrix
  double c[3], p[3];
  data->GetCenter(c);
  Matrix3x3 covar(.0);
  for (vtkIdType i = 0; i < data->GetNumberOfPoints(); ++i) {
    data->GetPoint(i, p);
    for (int d = 0; d < 3; ++d) p[d] -= c[d];
    for (int r = 0; r < 3; ++r) {
      for (int c = 0; c < 3; ++c) {
        covar[r][c] += p[r] * p[c];
      }
    }
  }
  double  eigen[3];
  Vector3 axis [3];
  covar.EigenSolveSymmetric(eigen, axis);
  Vector3::GenerateOrthonormalBasis(axis[0], axis[1], axis[2]);
  // Set output origin and orientation
  attr._xorigin = c[0];
  attr._yorigin = c[1];
  attr._zorigin = c[2];
  for (int d = 0; d < 3; ++d) {
    attr._xaxis[d] = axis[0][d];
    attr._yaxis[d] = axis[1][d];
    attr._zaxis[d] = axis[2][d];
  }
  // Determine bounds of reoriented data set
  double x, y, z;
  double xmin = numeric_limits<double>::max(), ymin = xmin, zmin = xmin;
  double xmax = -xmin, ymax = -ymin, zmax = -zmin;
  OrderedSet<double> xs, ys, zs;
  for (vtkIdType i = 0; i < data->GetNumberOfPoints(); ++i) {
    data->GetPoint(i, p);
    x = attr._xaxis[0] * p[0] + attr._xaxis[1] * p[1] + attr._xaxis[2] * p[2];
    y = attr._yaxis[0] * p[0] + attr._yaxis[1] * p[1] + attr._yaxis[2] * p[2];
    z = attr._zaxis[0] * p[0] + attr._zaxis[1] * p[1] + attr._zaxis[2] * p[2];
    if (x < xmin) xmin = x;
    if (x > xmax) xmax = x;
    if (y < ymin) ymin = y;
    if (y > ymax) ymax = y;
    if (z < zmin) zmin = z;
    if (z > zmax) zmax = z;
    if (attr._dx <= .0) xs.insert(x);
    if (attr._dy <= .0) ys.insert(y);
    if (attr._dz <= .0) zs.insert(z);
  }
  // Set output resolution and size
  const double extent[3]  = {xmax - xmin, ymax - ymin, zmax - zmin};
  const double avg_extent = (extent[0] + extent[1] + extent[2]) / 3.0;
  if (attr._dx <= .0) attr._dx = ((extent[0] / avg_extent > 1e-3) ? AverageInterval(xs) : .0);
  if (attr._dy <= .0) attr._dy = ((extent[1] / avg_extent > 1e-3) ? AverageInterval(ys) : .0);
  if (attr._dz <= .0) attr._dz = ((extent[2] / avg_extent > 1e-3) ? AverageInterval(zs) : .0);
  attr._x  = (attr._dx > .0 ? iround(extent[0] / attr._dx) : 0) + 1;
  attr._y  = (attr._dy > .0 ? iround(extent[1] / attr._dy) : 0) + 1;
  attr._z  = (attr._dz > .0 ? iround(extent[2] / attr._dz) : 0) + 1;
  attr._dx = (attr._x  >  1 ? extent[0] / (attr._x - 1) : .0);
  attr._dy = (attr._y  >  1 ? extent[1] / (attr._y - 1) : .0);
  attr._dz = (attr._z  >  1 ? extent[2] / (attr._z - 1) : .0);
  return attr;
}

// -----------------------------------------------------------------------------
ImageAttributes PointSetDomain(vtkPointSet *data, const Vector3D<double> &ds)
{
  return PointSetDomain(data, ds._x, ds._y, ds._z);
}

// -----------------------------------------------------------------------------
ImageAttributes PolyDataDomain(vtkPolyData *data, double dx, double dy, double dz)
{
  return PointSetDomain(data, dx, dy, dz);
}

// -----------------------------------------------------------------------------
ImageAttributes PolyDataDomain(vtkPolyData *data, const Vector3D<double> &ds)
{
  return PointSetDomain(data, ds);
}

// =============================================================================
// Point/cell data
// =============================================================================

// -----------------------------------------------------------------------------
int PolyDataAttributeType(const char *type)
{
  string ltype(type);
  transform(ltype.begin(), ltype.end(), ltype.begin(), ::tolower);
  if (ltype == "scalars")     return vtkDataSetAttributes::SCALARS;
  if (ltype == "vectors")     return vtkDataSetAttributes::VECTORS;
  if (ltype == "normals")     return vtkDataSetAttributes::NORMALS;
  if (ltype == "tcoords")     return vtkDataSetAttributes::TCOORDS;
  if (ltype == "tensors")     return vtkDataSetAttributes::TENSORS;
  if (ltype == "globalids")   return vtkDataSetAttributes::GLOBALIDS;
  if (ltype == "pedigreeids") return vtkDataSetAttributes::PEDIGREEIDS;
  if (ltype == "edgeflag")    return vtkDataSetAttributes::EDGEFLAG;
  return -1;
}

// -----------------------------------------------------------------------------
vtkDataArray *GetArrayByCaseInsensitiveName(vtkDataSetAttributes *data, const char *name, int *loc)
{
  string lname = ToLower(name), lower_name;
  for (int i = 0; i < data->GetNumberOfArrays(); ++i) {
    const char *array_name = data->GetArrayName(i);
    if (array_name) {
      lower_name = ToLower(array_name);
      if (lower_name == lname) {
        if (loc) *loc = i;
        return data->GetArray(i);
      }
    }
  }
  if (loc) *loc = -1;
  return NULL;
}

// -----------------------------------------------------------------------------
int DeepCopyArrayUsingCaseInsensitiveName(vtkDataSetAttributes *dst, vtkDataSetAttributes *src, const char *name, bool deep)
{
  int loc = -1;
  vtkDataArray *src_array = GetArrayByCaseInsensitiveName(src, name);
  if (src_array) {
    vtkDataArray *dst_array = GetArrayByCaseInsensitiveName(dst, name, &loc);
    if (dst_array) {
      dst_array->DeepCopy(src_array);
    } else {
      vtkSmartPointer<vtkDataArray> copy;
      copy = vtkSmartPointer<vtkDataArray>::NewInstance(src_array);
      copy->DeepCopy(src_array);
      loc = dst->AddArray(copy);
    }
  }
  return loc;
}

// =============================================================================
// Cells
// =============================================================================

// -----------------------------------------------------------------------------
// Attention: vtkGenericCell is not derived from the respective cell types
double ComputeArea(vtkCell *cell)
{
  if (cell->GetCellDimension() < 2) return .0;
  vtkPoints *points = cell->GetPoints();
  switch (cell->GetCellType()) {
    case VTK_TRIANGLE: {
      double p1[3], p2[3], p3[3];
      points->GetPoint(0, p1);
      points->GetPoint(1, p2);
      points->GetPoint(2, p3);
      return vtkTriangle::TriangleArea(p1, p2, p3);
    }
    default: return numeric_limits<double>::quiet_NaN();
  }
}

// -----------------------------------------------------------------------------
// Attention: vtkGenericCell is not derived from the respective cell types
double ComputeVolume(vtkCell *cell)
{
  if (cell->GetCellDimension() < 3) return .0;
  vtkPoints *points = cell->GetPoints();
  switch (cell->GetCellType()) {
    case VTK_TETRA: {
      double p1[3], p2[3], p3[3], p4[3];
      points->GetPoint(0, p1);
      points->GetPoint(1, p2);
      points->GetPoint(2, p3);
      points->GetPoint(3, p4);
      return vtkTetra::ComputeVolume(p1, p2, p3, p4);
    }
    default: return numeric_limits<double>::quiet_NaN();
  }
}

// =============================================================================
// Basic point set manipulation
// =============================================================================

// -----------------------------------------------------------------------------
vtkSmartPointer<vtkPolyData> DataSetSurface(vtkSmartPointer<vtkDataSet> dataset, bool passPtIds, bool passCellIds)
{
  if (IsSurfaceMesh(dataset)) {
    vtkSmartPointer<vtkPolyData> surface = vtkSmartPointer<vtkPolyData>::New();
    surface->ShallowCopy(dataset);
    return surface;
  } else {
    vtkSmartPointer<vtkDataSetSurfaceFilter> surface;
    surface = vtkSmartPointer<vtkDataSetSurfaceFilter>::New();
    SetVTKInput(surface, dataset);
    surface->SetPassThroughPointIds(passPtIds);
    surface->SetPassThroughCellIds(passCellIds);
    surface->Update();
    return surface->GetOutput();
  }
}

// -----------------------------------------------------------------------------
void Center(vtkSmartPointer<vtkPointSet> pointset)
{
  double c[3], p[3];
{'fraction_non_alphanumeric': 0.295972850678733, 'fraction_numerical': 0.016923076923076923, 'mean_word_length': 2.668215767634855, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 134, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5883795', 'n_tokens_mistral': 7415, 'n_tokens_neox': 7202, 'n_words': 1665}