<commit_before>///////////////////////////////////////////////////////////////////////////////
// FILE:       CSUW1.cpp
// PROJECT:    MicroManage
// SUBSYSTEM:  DeviceAdapters
//-----------------------------------------------------------------------------
// DESCRIPTION:
// Yokogawa CSUW1 controller adapter
//
// COPYRIGHT:     University of California, San Francisco, 2006
//                All rights reserved
//
// LICENSE:       This library is free software; you can redistribute it and/or
//                modify it under the terms of the GNU Lesser General Public
//                License as published by the Free Software Foundation.
//                
//                You should have received a copy of the GNU Lesser General Public
//                License along with the source distribution; if not, write to
//                the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
//                Boston, MA  02111-1307  USA
//
//                This file is distributed in the hope that it will be useful,
//                but WITHOUT ANY WARRANTY; without even the implied warranty
//                of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
//                IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//                CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
//                INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES.  

// Based on CSUX1 adapter by <NAME>
//
// Based on NikonTE2000 controller adapter by <NAME>
//

#ifdef WIN32
#include <windows.h>
#define snprintf _snprintf 
#endif

#include "CSUW1.h"
#include "CSUW1Hub.h"
#include <cstdio>
#include <string>
#include <math.h>
#include "../../MMDevice/ModuleInterface.h"
#include <sstream>
#include <iostream>

const char* g_CSUW1Hub = "CSUW1-Hub";
const char* g_CSUW1FilterWheel = "CSUW1-Filter Wheel";
const char* g_CSUW1Dichroic = "CSUW1-Dichroic Mirror";
const char* g_CSUW1Shutter = "CSUW1-Shutter";
const char* g_CSUW1DriveSpeed= "CSUW1-Drive Speed";
const char* g_CSUW1BrightField = "CSUW1-Bright Field";
const char* g_CSUW1Disk = "CSUW1-Disk";
const char* g_CSUW1Port = "CSUW1-Port";
const char* g_CSUW1Aperture = "CSUW1-Aperture";
const char* g_CSUW1Frap = "CSUW1-FRAP";
const char* g_CSUW1Magnifier = "CSUW1-Magnifier";
const char* g_CSUW1NIRShutter = "CSUW1-NIR Shutter";

using namespace std;

CSUW1Hub g_hub;

///////////////////////////////////////////////////////////////////////////////
// Exported MMDevice API
///////////////////////////////////////////////////////////////////////////////
MODULE_API void InitializeModuleData()
{
   RegisterDevice(g_CSUW1Hub, MM::GenericDevice, "CSUW1 Hub (Needed for CSUW1. 115200bps)");
   RegisterDevice(g_CSUW1FilterWheel, MM::StateDevice, "Filter Wheel");
   RegisterDevice(g_CSUW1Dichroic, MM::StateDevice, "Dichroic Mirror");
   RegisterDevice(g_CSUW1Shutter, MM::ShutterDevice, "Shutter");
   RegisterDevice(g_CSUW1DriveSpeed, MM::GenericDevice, "Drive Speed");
   RegisterDevice(g_CSUW1BrightField, MM::StateDevice, "Bright Field");
   RegisterDevice(g_CSUW1Disk, MM::StateDevice, "Disk");
   RegisterDevice(g_CSUW1Port, MM::StateDevice, "Port");
   RegisterDevice(g_CSUW1Aperture, MM::StateDevice, "Aperture");
   RegisterDevice(g_CSUW1Frap, MM::StateDevice, "FRAP");
   RegisterDevice(g_CSUW1Magnifier, MM::StateDevice, "Magnifier");
   RegisterDevice(g_CSUW1NIRShutter, MM::ShutterDevice, "NIR Shutter");
}

MODULE_API MM::Device* CreateDevice(const char* deviceName)
{
   if (deviceName == 0)
      return 0;

   if (strcmp(deviceName, g_CSUW1Hub) == 0)
   {
      return new Hub();
   }
   else if (strcmp(deviceName, g_CSUW1FilterWheel) == 0 )
   {
      return new FilterWheel();
   }
   else if (strcmp(deviceName, g_CSUW1Dichroic) == 0 )
   {
      return new Dichroic();
   }
   else if (strcmp(deviceName, g_CSUW1Shutter) == 0 )
   {
      return new Shutter();
   }
   else if (strcmp(deviceName, g_CSUW1DriveSpeed) == 0 )
   {
      return new DriveSpeed();
   }
   else if (strcmp(deviceName, g_CSUW1BrightField) == 0 )
   {
      return new BrightField();
   }
   else if (strcmp(deviceName, g_CSUW1Disk) == 0 )
   {
      return new Disk();
   }
   else if (strcmp(deviceName, g_CSUW1Port) == 0 )
   {
      return new Port();
   }
   else if (strcmp(deviceName, g_CSUW1Aperture) == 0 )
   {
      return new Aperture();
   }
   else if (strcmp(deviceName, g_CSUW1Frap) == 0 )
   {
      return new Frap();
   }
   else if (strcmp(deviceName, g_CSUW1Magnifier) == 0 )
   {
      return new Magnifier();
   }
   else if (strcmp(deviceName, g_CSUW1NIRShutter) == 0 )
   {
      return new NIRShutter();
   }
   return 0;
}

MODULE_API void DeleteDevice(MM::Device* pDevice)
{
   delete pDevice;
}


///////////////////////////////////////////////////////////////////////////////
// CSUW1 Hub
///////////////////////////////////////////////////////////////////////////////

Hub::Hub() :
   initialized_(false),
   port_("Undefined")
{
   InitializeDefaultErrorMessages();

   // custom error messages
   SetErrorText(ERR_COMMAND_CANNOT_EXECUTE, "Command cannot be executed");

   // create pre-initialization properties
   // ------------------------------------

   // Port
   CPropertyAction* pAct = new CPropertyAction (this, &Hub::OnPort);
   CreateProperty(MM::g_Keyword_Port, "Undefined", MM::String, false, pAct, true);

}

Hub::~Hub()
{
   Shutdown();
}

void Hub::GetName(char* name) const
{
   //assert(name_.length() < CDeviceUtils::GetMaxStringLength());   
   CDeviceUtils::CopyLimitedString(name, g_CSUW1Hub);
}

bool Hub::Busy()
{
   return false;
}

int Hub::Initialize()
{
   // set property list
   // -----------------
   
   // Name
   int ret = CreateProperty(MM::g_Keyword_Name, g_CSUW1Hub, MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // Description
   ret = CreateProperty(MM::g_Keyword_Description, "Yokogawa CSUW1 hub", MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   initialized_ = true;
   
   return DEVICE_OK;
}

int Hub::Shutdown()
{
   if (initialized_)
   {
      initialized_ = false;
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// Action handlers
///////////////////////////////////////////////////////////////////////////////
/*
 * Sets the Serial Port to be used.
 * Should be called before initialization
 */
int Hub::OnPort(MM::PropertyBase* pProp, MM::ActionType eAct)
{
   if (eAct == MM::BeforeGet)
   {
      pProp->Set(port_.c_str());
   }
   else if (eAct == MM::AfterSet)
   {
      if (initialized_)
      {
         // revert
         pProp->Set(port_.c_str());
         //return ERR_PORT_CHANGE_FORBIDDEN;
      }

      pProp->Get(port_);
      g_hub.SetPort(port_.c_str());
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// CSUW1 Filter Wheel
///////////////////////////////////////////////////////////////////////////////
FilterWheel::FilterWheel () :
   posMoved_ (0),
   wheelNr_ (1),
   initialized_ (false),
   numPos_ (10),
   pos_ (1),
   name_ (g_CSUW1FilterWheel)
{
   InitializeDefaultErrorMessages();

   lastMoveTime_ = GetCurrentMMTime();

   // Wheel Number
   CPropertyAction* pAct = new CPropertyAction (this, &FilterWheel::OnWheelNr);
   CreateProperty("WheelNumber", "1", MM::Integer, false, pAct, true);
   AddAllowedValue("WheelNumber", "1");
   AddAllowedValue("WheelNumber", "2");
   // Todo: Add custom messages
}

FilterWheel::~FilterWheel ()
{
   Shutdown();
}

void FilterWheel::GetName(char* name) const
{
   assert(name_.length() < CDeviceUtils::GetMaxStringLength());
   CDeviceUtils::CopyLimitedString(name, name_.c_str());
}

int FilterWheel::Initialize()
{
   // Name
   int ret = CreateProperty(MM::g_Keyword_Name, name_.c_str(), MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // Description
   ret = CreateProperty(MM::g_Keyword_Description, g_CSUW1FilterWheel, MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // State
   CPropertyAction* pAct = new CPropertyAction (this, &FilterWheel::OnState);
   ret = CreateProperty(MM::g_Keyword_State, "0", MM::Integer, false, pAct); 
   if (ret != DEVICE_OK) 
      return ret; 

   // create default positions and labels
   pAct = new CPropertyAction (this, &CStateBase::OnLabel);                  
   ret = CreateProperty(MM::g_Keyword_Label, "Undefined", MM::String, false, pAct);
   if (ret != DEVICE_OK)                                                     
      return ret;                                                            
   SetPositionLabel(0, "Filter-1");
   SetPositionLabel(1, "Filter-2");
   SetPositionLabel(2, "Filter-3");
   SetPositionLabel(3, "Filter-4");
   SetPositionLabel(4, "Filter-5");
   SetPositionLabel(5, "Filter-6");
   SetPositionLabel(6, "Filter-7");
   SetPositionLabel(7, "Filter-8");
   SetPositionLabel(8, "Filter-9");
   SetPositionLabel(9, "Filter-10");

   // Speed
   pAct = new CPropertyAction (this, &FilterWheel::OnSpeed);
   ret = CreateProperty("Speed", "2", MM::Integer, false, pAct); 
   if (ret != DEVICE_OK) 
      return ret; 
   AddAllowedValue("Speed", "0");
   AddAllowedValue("Speed", "1");
   AddAllowedValue("Speed", "2");
   AddAllowedValue("Speed", "3");

   // Although we compute an appropriate delay, we allow the user to set an
   // extra delay to wait for vibrations to sibside.
   EnableDelay(true);

   ret = UpdateStatus();
   if (ret != DEVICE_OK) 
      return ret; 

   initialized_ = true;

   return DEVICE_OK;
}

bool FilterWheel::Busy()
{
   MM::MMTime elapsed = GetCurrentMMTime() - lastMoveTime_;
   long msPerPosition;
   switch (speed_) {
   case 3:
      msPerPosition = 40;
      break;
   case 2:
      msPerPosition = 66;
      break;
   case 1:
      msPerPosition = 100;
      break;
   case 0:
   default:
      msPerPosition = 400;
      break;
   }
   long waitTimeMs = posMoved_ * msPerPosition + GetDelayMs();
   return elapsed.getMsec() < waitTimeMs;
}

int FilterWheel::Shutdown()
{
   if (initialized_)
   {
      initialized_ = false;
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// Action handlers                                                           
///////////////////////////////////////////////////////////////////////////////

int FilterWheel::OnState(MM::PropertyBase* pProp, MM::ActionType eAct)
{
   if (eAct == MM::BeforeGet)
   {
      long pos;
      int ret = g_hub.GetFilterWheelPosition(*this, *GetCoreCallback(), wheelNr_, pos);
      if (ret != DEVICE_OK)
         return ret;
      pProp->Set(pos);
      pos_ = pos;
   }
   else if (eAct == MM::AfterSet)
   {<commit_msg>CSUW1: fix compiler warning<commit_after>///////////////////////////////////////////////////////////////////////////////
// FILE:       CSUW1.cpp
// PROJECT:    MicroManage
// SUBSYSTEM:  DeviceAdapters
//-----------------------------------------------------------------------------
// DESCRIPTION:
// Yokogawa CSUW1 controller adapter
//
// COPYRIGHT:     University of California, San Francisco, 2006
//                All rights reserved
//
// LICENSE:       This library is free software; you can redistribute it and/or
//                modify it under the terms of the GNU Lesser General Public
//                License as published by the Free Software Foundation.
//                
//                You should have received a copy of the GNU Lesser General Public
//                License along with the source distribution; if not, write to
//                the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
//                Boston, MA  02111-1307  USA
//
//                This file is distributed in the hope that it will be useful,
//                but WITHOUT ANY WARRANTY; without even the implied warranty
//                of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
//                IN NO EVENT SHALL THE COPYRIGHT OWNER OR
//                CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
//                INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES.  

// Based on CSUX1 adapter by <NAME>
//
// Based on NikonTE2000 controller adapter by <NAME>
//

#ifdef WIN32
#include <windows.h>
#define snprintf _snprintf 
#endif

#include "CSUW1.h"
#include "CSUW1Hub.h"
#include <cstdio>
#include <string>
#include <math.h>
#include "../../MMDevice/ModuleInterface.h"
#include <sstream>
#include <iostream>

const char* g_CSUW1Hub = "CSUW1-Hub";
const char* g_CSUW1FilterWheel = "CSUW1-Filter Wheel";
const char* g_CSUW1Dichroic = "CSUW1-Dichroic Mirror";
const char* g_CSUW1Shutter = "CSUW1-Shutter";
const char* g_CSUW1DriveSpeed= "CSUW1-Drive Speed";
const char* g_CSUW1BrightField = "CSUW1-Bright Field";
const char* g_CSUW1Disk = "CSUW1-Disk";
const char* g_CSUW1Port = "CSUW1-Port";
const char* g_CSUW1Aperture = "CSUW1-Aperture";
const char* g_CSUW1Frap = "CSUW1-FRAP";
const char* g_CSUW1Magnifier = "CSUW1-Magnifier";
const char* g_CSUW1NIRShutter = "CSUW1-NIR Shutter";

using namespace std;

CSUW1Hub g_hub;

///////////////////////////////////////////////////////////////////////////////
// Exported MMDevice API
///////////////////////////////////////////////////////////////////////////////
MODULE_API void InitializeModuleData()
{
   RegisterDevice(g_CSUW1Hub, MM::GenericDevice, "CSUW1 Hub (Needed for CSUW1. 115200bps)");
   RegisterDevice(g_CSUW1FilterWheel, MM::StateDevice, "Filter Wheel");
   RegisterDevice(g_CSUW1Dichroic, MM::StateDevice, "Dichroic Mirror");
   RegisterDevice(g_CSUW1Shutter, MM::ShutterDevice, "Shutter");
   RegisterDevice(g_CSUW1DriveSpeed, MM::GenericDevice, "Drive Speed");
   RegisterDevice(g_CSUW1BrightField, MM::StateDevice, "Bright Field");
   RegisterDevice(g_CSUW1Disk, MM::StateDevice, "Disk");
   RegisterDevice(g_CSUW1Port, MM::StateDevice, "Port");
   RegisterDevice(g_CSUW1Aperture, MM::StateDevice, "Aperture");
   RegisterDevice(g_CSUW1Frap, MM::StateDevice, "FRAP");
   RegisterDevice(g_CSUW1Magnifier, MM::StateDevice, "Magnifier");
   RegisterDevice(g_CSUW1NIRShutter, MM::ShutterDevice, "NIR Shutter");
}

MODULE_API MM::Device* CreateDevice(const char* deviceName)
{
   if (deviceName == 0)
      return 0;

   if (strcmp(deviceName, g_CSUW1Hub) == 0)
   {
      return new Hub();
   }
   else if (strcmp(deviceName, g_CSUW1FilterWheel) == 0 )
   {
      return new FilterWheel();
   }
   else if (strcmp(deviceName, g_CSUW1Dichroic) == 0 )
   {
      return new Dichroic();
   }
   else if (strcmp(deviceName, g_CSUW1Shutter) == 0 )
   {
      return new Shutter();
   }
   else if (strcmp(deviceName, g_CSUW1DriveSpeed) == 0 )
   {
      return new DriveSpeed();
   }
   else if (strcmp(deviceName, g_CSUW1BrightField) == 0 )
   {
      return new BrightField();
   }
   else if (strcmp(deviceName, g_CSUW1Disk) == 0 )
   {
      return new Disk();
   }
   else if (strcmp(deviceName, g_CSUW1Port) == 0 )
   {
      return new Port();
   }
   else if (strcmp(deviceName, g_CSUW1Aperture) == 0 )
   {
      return new Aperture();
   }
   else if (strcmp(deviceName, g_CSUW1Frap) == 0 )
   {
      return new Frap();
   }
   else if (strcmp(deviceName, g_CSUW1Magnifier) == 0 )
   {
      return new Magnifier();
   }
   else if (strcmp(deviceName, g_CSUW1NIRShutter) == 0 )
   {
      return new NIRShutter();
   }
   return 0;
}

MODULE_API void DeleteDevice(MM::Device* pDevice)
{
   delete pDevice;
}


///////////////////////////////////////////////////////////////////////////////
// CSUW1 Hub
///////////////////////////////////////////////////////////////////////////////

Hub::Hub() :
   initialized_(false),
   port_("Undefined")
{
   InitializeDefaultErrorMessages();

   // custom error messages
   SetErrorText(ERR_COMMAND_CANNOT_EXECUTE, "Command cannot be executed");

   // create pre-initialization properties
   // ------------------------------------

   // Port
   CPropertyAction* pAct = new CPropertyAction (this, &Hub::OnPort);
   CreateProperty(MM::g_Keyword_Port, "Undefined", MM::String, false, pAct, true);

}

Hub::~Hub()
{
   Shutdown();
}

void Hub::GetName(char* name) const
{
   //assert(name_.length() < CDeviceUtils::GetMaxStringLength());   
   CDeviceUtils::CopyLimitedString(name, g_CSUW1Hub);
}

bool Hub::Busy()
{
   return false;
}

int Hub::Initialize()
{
   // set property list
   // -----------------
   
   // Name
   int ret = CreateProperty(MM::g_Keyword_Name, g_CSUW1Hub, MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // Description
   ret = CreateProperty(MM::g_Keyword_Description, "Yokogawa CSUW1 hub", MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   initialized_ = true;
   
   return DEVICE_OK;
}

int Hub::Shutdown()
{
   if (initialized_)
   {
      initialized_ = false;
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// Action handlers
///////////////////////////////////////////////////////////////////////////////
/*
 * Sets the Serial Port to be used.
 * Should be called before initialization
 */
int Hub::OnPort(MM::PropertyBase* pProp, MM::ActionType eAct)
{
   if (eAct == MM::BeforeGet)
   {
      pProp->Set(port_.c_str());
   }
   else if (eAct == MM::AfterSet)
   {
      if (initialized_)
      {
         // revert
         pProp->Set(port_.c_str());
         //return ERR_PORT_CHANGE_FORBIDDEN;
      }

      pProp->Get(port_);
      g_hub.SetPort(port_.c_str());
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// CSUW1 Filter Wheel
///////////////////////////////////////////////////////////////////////////////
FilterWheel::FilterWheel () :
   posMoved_ (0),
   wheelNr_ (1),
   initialized_ (false),
   numPos_ (10),
   pos_ (1),
   name_ (g_CSUW1FilterWheel)
{
   InitializeDefaultErrorMessages();

   lastMoveTime_ = GetCurrentMMTime();

   // Wheel Number
   CPropertyAction* pAct = new CPropertyAction (this, &FilterWheel::OnWheelNr);
   CreateProperty("WheelNumber", "1", MM::Integer, false, pAct, true);
   AddAllowedValue("WheelNumber", "1");
   AddAllowedValue("WheelNumber", "2");
   // Todo: Add custom messages
}

FilterWheel::~FilterWheel ()
{
   Shutdown();
}

void FilterWheel::GetName(char* name) const
{
   assert(name_.length() < CDeviceUtils::GetMaxStringLength());
   CDeviceUtils::CopyLimitedString(name, name_.c_str());
}

int FilterWheel::Initialize()
{
   // Name
   int ret = CreateProperty(MM::g_Keyword_Name, name_.c_str(), MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // Description
   ret = CreateProperty(MM::g_Keyword_Description, g_CSUW1FilterWheel, MM::String, true);
   if (DEVICE_OK != ret)
      return ret;

   // State
   CPropertyAction* pAct = new CPropertyAction (this, &FilterWheel::OnState);
   ret = CreateProperty(MM::g_Keyword_State, "0", MM::Integer, false, pAct); 
   if (ret != DEVICE_OK) 
      return ret; 

   // create default positions and labels
   pAct = new CPropertyAction (this, &CStateBase::OnLabel);                  
   ret = CreateProperty(MM::g_Keyword_Label, "Undefined", MM::String, false, pAct);
   if (ret != DEVICE_OK)                                                     
      return ret;                                                            
   SetPositionLabel(0, "Filter-1");
   SetPositionLabel(1, "Filter-2");
   SetPositionLabel(2, "Filter-3");
   SetPositionLabel(3, "Filter-4");
   SetPositionLabel(4, "Filter-5");
   SetPositionLabel(5, "Filter-6");
   SetPositionLabel(6, "Filter-7");
   SetPositionLabel(7, "Filter-8");
   SetPositionLabel(8, "Filter-9");
   SetPositionLabel(9, "Filter-10");

   // Speed
   pAct = new CPropertyAction (this, &FilterWheel::OnSpeed);
   ret = CreateProperty("Speed", "2", MM::Integer, false, pAct); 
   if (ret != DEVICE_OK) 
      return ret; 
   AddAllowedValue("Speed", "0");
   AddAllowedValue("Speed", "1");
   AddAllowedValue("Speed", "2");
   AddAllowedValue("Speed", "3");

   // Although we compute an appropriate delay, we allow the user to set an
   // extra delay to wait for vibrations to sibside.
   EnableDelay(true);

   ret = UpdateStatus();
   if (ret != DEVICE_OK) 
      return ret; 

   initialized_ = true;

   return DEVICE_OK;
}

bool FilterWheel::Busy()
{
   MM::MMTime elapsed = GetCurrentMMTime() - lastMoveTime_;
   long msPerPosition;
   switch (speed_) {
   case 3:
      msPerPosition = 40;
      break;
   case 2:
      msPerPosition = 66;
      break;
   case 1:
      msPerPosition = 100;
      break;
   case 0:
   default:
      msPerPosition = 400;
      break;
   }
   long waitTimeMs = (long) (posMoved_ * msPerPosition + GetDelayMs());
   return elapsed.getMsec() < waitTimeMs;
}

int FilterWheel::Shutdown()
{
   if (initialized_)
   {
      initialized_ = false;
   }
   return DEVICE_OK;
}

///////////////////////////////////////////////////////////////////////////////
// Action handlers                                                           
///////////////////////////////////////////////////////////////////////////////

int FilterWheel::OnState(MM::PropertyBase* pProp, MM::ActionType eAct)
{
   if (eAct == MM::BeforeGet)
   {
      long pos;
      int ret = g_hub.GetFilterWheelPosition(*this, *GetCoreCallback(), wheelNr_, pos);
      if (ret != DEVICE_OK)
         return ret;
      pProp->Set(pos);
      pos_ = pos;
   }
   else if (eAct == MM::AfterSet)
   {
{'fraction_non_alphanumeric': 0.20715252812923104, 'fraction_numerical': 0.014706568934123908, 'mean_word_length': 2.1003039513677813, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5912397', 'n_tokens_mistral': 6882, 'n_tokens_neox': 6542, 'n_words': 1588}