<commit_before>#include "pfPose.h"

using namespace cv;
using namespace std;
using namespace message_filters;

PFTracker::PFTracker()
{
	image_transport::ImageTransport it(nh);
	
	pub = it.advertise("/poseImage",10);
	edge_pub = it.advertise("/handImage",10);
	hand_pub = nh.advertise<handBlobTracker::HFPose2DArray>("/correctedFaceHandPose", 10);
		
	image_sub.subscribe(nh, "/rgb/image_color", 1);
	pose_sub.subscribe(nh, "/faceHandPose", 1); 
		
	sync = new message_filters::TimeSynchronizer<sensor_msgs::Image, handBlobTracker::HFPose2DArray>(image_sub,pose_sub,10);
	sync->registerCallback(boost::bind(&PFTracker::callback, this, _1, _2));
	
	// Load Kinect GMM priors
	std::stringstream ss1;
	std::string left_arm_training;
	ros::param::param<std::string>("left_arm_training", left_arm_training, "/training/data13D_PCA.yml");
	ss1 << ros::package::getPath("mkfbodytracker") << left_arm_training;
	std::stringstream ss2;
	std::string right_arm_training;
	ros::param::param<std::string>("right_arm_training", right_arm_training, "/training/data23D_PCA.yml");
	ss2 << ros::package::getPath("mkfbodytracker") << right_arm_training;
	ROS_INFO("Getting data from %s",ss1.str().c_str());
	ROS_INFO("Getting data from %s",ss2.str().c_str());
	cv::FileStorage fs1(ss1.str(), FileStorage::READ);
	cv::FileStorage fs2(ss2.str(), FileStorage::READ);
	
	cv::Mat means1, weights1, means2, weights2, covs1, covs2, g1, g2;
    fs1["means"] >> means1;
    fs2["means"] >> means2;
  	fs1["covs"] >> covs1;
	fs2["covs"] >> covs2;
	fs1["weights"] >> weights1;
    fs2["weights"] >> weights2;
    fs1["pca_proj"] >> h1_pca;
    h1_pca.convertTo(h1_pca, CV_64F);
    fs2["pca_proj"] >> h2_pca;
    h2_pca.convertTo(h2_pca, CV_64F);
    fs1["pca_mean"] >> m1_pca;
    m1_pca.convertTo(m1_pca, CV_64F);
    fs2["pca_mean"] >> m2_pca;
    m2_pca.convertTo(m2_pca, CV_64F);
    fs2["gamma"] >> g1;
    fs2["gamma"] >> g2;
    fs1.release();
    fs2.release();
    
    d = m1_pca.cols;
	pf1 = new ParticleFilter(d,100); // left arm pf
	pf2 = new ParticleFilter(d,100); // right arm pf

	for (int i = 0; i < means1.rows; i++)
	{
		pf2->gmm.loadGaussian(means1.row(i),covs1(Range(covs1.cols*i,covs1.cols*(i+1)),Range(0,covs1.cols)), h1_pca, m1_pca, weights1.at<double>(0,i), g1.at<double>(0,i));
		pf1->gmm.loadGaussian(means2.row(i),covs2(Range(covs2.cols*i,covs2.cols*(i+1)),Range(0,covs2.cols)), h2_pca, m2_pca, weights2.at<double>(0,i), g2.at<double>(0,i));
	}
	////Load regularising gaussians - default constant velocity KF
	//pf2->gmm.loadGaussian(cv::Mat::zeros(1,means1.cols,CV_64F),0.5*cv::Mat::eye(means1.cols,means1.cols,CV_64F), 1);
	//pf1->gmm.loadGaussian(cv::Mat::zeros(1,means1.cols,CV_64F),0.5*cv::Mat::eye(means1.cols,means1.cols,CV_64F), 1);
	
	swap = false;
	edge_heuristic = 1e-1;
}

PFTracker::~PFTracker()
{
	delete sync;
	delete pf1;
	delete pf2;
}

// Create Rotation matrix from roll, pitch and yaw
cv::Mat PFTracker::rpy(double roll, double pitch, double yaw)
{
	cv::Mat R1 = (Mat_<double>(3, 3) << 1, 0, 0, 0, cos(roll), -sin(roll), 0, sin(roll), cos(roll));
	cv::Mat R2 = (Mat_<double>(3, 3) << cos(pitch), 0, sin(pitch), 0, 1, 0, -sin(pitch), 0, cos(pitch));
	cv::Mat R3 = (Mat_<double>(3, 3) << cos(yaw), -sin(yaw), 0, sin(yaw), cos(yaw), 0, 0, 0, 1);

	return  R3*R2*R1;
}

cv::Mat PFTracker::get3Dpose(cv::Mat estimate)
{
	cv::Mat est;
	estimate.copyTo(est);
	// Webcam
	//~ cv::Mat K  = (Mat_<double>(3, 3) << 660.326889, 0.0, 318.705890, 0.0, 660.857176, 240.784699, 0.0, 0.0, 1.0);
	
	// Kinect
	cv::Mat K  = (Mat_<double>(3, 3) << 525.0, 0.0, 319.5, 0.0, 525.0, 239.5, 0.0, 0.0, 1.0);
	
	// h       e     s       h       n        r  p  y   tx ty tz 
	// 0 1 2  3 4 5  6 7 8 9 10 11  12 13 14  15 16 17  18 19 20
	cv::Mat R = rpy(est.at<double>(0,16),est.at<double>(0,17),est.at<double>(0,15));
	cv::Mat t = (Mat_<double>(3, 1) << est.at<double>(0,18), est.at<double>(0,19), est.at<double>(0,20));
	
	cv::Mat P,T;
	hconcat(R, t, T);
	P = K*T;
	
	cv::Mat im_points(5,3,CV_64FC1);
	cv::Mat temp;
	for (int k = 0; k < 5; k++)
	{
		est.at<double>(0,3*k) = est.at<double>(0,3*k)*est.at<double>(0,3*k+2);
		est.at<double>(0,3*k+1) = est.at<double>(0,3*k+1)*est.at<double>(0,3*k+2);
		temp = (est.rowRange(Range(3*k,3*k+3))).t();
		temp.copyTo(im_points(Range(k,k+1),Range(0,3)));
	}
	
	cv::Mat P_I;
	invert(P(Range(0,3),Range(0,3)),P_I,DECOMP_LU);
	
	cv::Mat pos3D = P_I*(im_points-repeat(P.col(3).t(),5,1)).t();
	return pos3D;
}

bool PFTracker::edgePoseCorrection(cv::Mat image4, handBlobTracker::HFPose2DArray pfPose, cv::Mat image3)
{
	bool value = true;
	
	// Edge detection
	cv::Mat dst;
	Canny(image4, dst, 90, 200, 3);

	// Find lines in edge image
	vector<Vec4i> lines;
	HoughLinesP(dst, lines, 5, 5*CV_PI/180, 10, 5, 10 );
	
	// Declare edge probabilities
	double e1=1e-4,e2=1e-4,e3=1e-4,e4=1e-4;
	
	// Draw stick limbs
	line(image3, Point(pfPose.measurements[1].x, pfPose.measurements[1].y), Point(pfPose.measurements[5].x, pfPose.measurements[5].y), Scalar(0,255,255), 3, CV_AA); //right_arm_l
	line(image3, Point(pfPose.measurements[0].x, pfPose.measurements[0].y), Point(pfPose.measurements[4].x, pfPose.measurements[4].y), Scalar(255,0,255), 3, CV_AA); //left_arm_l
	line(image3, Point(pfPose.measurements[7].x, pfPose.measurements[7].y), Point(pfPose.measurements[5].x, pfPose.measurements[5].y), Scalar(0,255,0), 3, CV_AA); // right_arm_u
	line(image3, Point(pfPose.measurements[6].x, pfPose.measurements[6].y), Point(pfPose.measurements[4].x, pfPose.measurements[4].y), Scalar(255,255,0), 3, CV_AA); //left_arm_u
	
	double right_arm_l_x = (pfPose.measurements[1].x + pfPose.measurements[5].x)/2.0;
	double right_arm_l_y = (pfPose.measurements[1].y + pfPose.measurements[5].y)/2.0;
	double right_arm_l_th  = atan2(pfPose.measurements[1].y-pfPose.measurements[5].y,pfPose.measurements[1].x-pfPose.measurements[5].x);
	
	double left_arm_l_x = (pfPose.measurements[0].x + pfPose.measurements[4].x)/2.0;
	double left_arm_l_y = (pfPose.measurements[0].y + pfPose.measurements[4].y)/2.0;
	double left_arm_l_th  = atan2(pfPose.measurements[0].y-pfPose.measurements[4].y,pfPose.measurements[0].x-pfPose.measurements[4].x);
			
	double right_arm_u_x = (pfPose.measurements[7].x + pfPose.measurements[5].x)/2.0;
	double right_arm_u_y = (pfPose.measurements[7].y + pfPose.measurements[5].y)/2.0;
	double right_arm_u_th  = atan2(pfPose.measurements[7].y-pfPose.measurements[5].y,pfPose.measurements[7].x-pfPose.measurements[5].x);
			
	double left_arm_u_x = (pfPose.measurements[6].x + pfPose.measurements[4].x)/2.0;
	double left_arm_u_y = (pfPose.measurements[6].y + pfPose.measurements[4].y)/2.0;
	double left_arm_u_th  = atan2(pfPose.measurements[6].y-pfPose.measurements[4].y,pfPose.measurements[6].x-pfPose.measurements[4].x);
	
	for (int i = 0; i < (int)lines.size(); i++)
	{
		Vec4i l = lines[i];
		line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,0,255), 1, CV_AA);
	
		double th  = atan2(l[3]-l[1],l[2]-l[0]);
		double x = (l[0]+l[2])/2.0;
		double y = (l[1]+l[3])/2.0;
		
		double d1 = pow(right_arm_l_x-x,2)+pow(right_arm_l_y-y,2);
		double d2 = pow(left_arm_l_x-x,2)+pow(left_arm_l_y-y,2);
		double d3 = pow(left_arm_u_x-x,2)+pow(left_arm_u_y-y,2);
		double d4 = pow(right_arm_u_x-x,2)+pow(right_arm_u_y-y,2);
		
		double temp = exp(-0.5*(1.0/125.0*d1 + 1.0/0.01*pow(atan(sin(right_arm_l_th - th)/cos(right_arm_l_th - th)),2)));
		if (temp > e1)
		{
			e1 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,255,255), 1, CV_AA);
		}		
			
		temp = exp(-0.5*(1.0/125.0*d2 + 1.0/0.01*pow(atan(sin(left_arm_l_th - th)/cos(left_arm_l_th - th)),2)));
		if (temp > e2)
		{
			e2 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255,0,255), 1, CV_AA);
		}
		
		temp = exp(-0.5*(1.0/125.0*d3 + 1.0/0.01*pow(atan(sin(left_arm_u_th - th)/cos(left_arm_u_th - th)),2)));
		if (temp > e3)
		{
			e3 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255,255,0), 1, CV_AA);
		}
		
		temp = exp(-0.5*(1.0/125.0*d4 + 1.0/0.01*pow(atan(sin(right_arm_u_th - th)/cos(right_arm_u_th - th)),2)));
		if (temp > e4)
		{
			e4 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,255,0), 1, CV_AA);
		}
	}
	
	edge_heuristic = 0.95*edge_heuristic+0.05*(e1+e2+e3+e4)/4.0; //smoothing filter to reduce clutter
	ROS_DEBUG("Forearm evidence: %f, r_upper: %f r_lower: %f l_upper: %f l_lower: %f.",edge_heuristic,e4,e1,e3,e2);	
	if ((edge_heuristic < 0.09))
	{
		ROS_WARN("Reset (line check fail): %f",edge_heuristic);
		edge_heuristic = 2e-1;
		value = false;
	}
	
	return value;
}

cv::Mat PFTracker::associateHands(const handBlobTracker::HFPose2DArrayConstPtr& msg)<commit_msg>Set default training as PCA RB 13 d 28 k<commit_after>#include "pfPose.h"

using namespace cv;
using namespace std;
using namespace message_filters;

PFTracker::PFTracker()
{
	image_transport::ImageTransport it(nh);
	
	pub = it.advertise("/poseImage",10);
	edge_pub = it.advertise("/handImage",10);
	hand_pub = nh.advertise<handBlobTracker::HFPose2DArray>("/correctedFaceHandPose", 10);
		
	image_sub.subscribe(nh, "/rgb/image_color", 1);
	pose_sub.subscribe(nh, "/faceHandPose", 1); 
		
	sync = new message_filters::TimeSynchronizer<sensor_msgs::Image, handBlobTracker::HFPose2DArray>(image_sub,pose_sub,10);
	sync->registerCallback(boost::bind(&PFTracker::callback, this, _1, _2));
	
	// Load Kinect GMM priors
	std::stringstream ss1;
	std::string left_arm_training;
	ros::param::param<std::string>("left_arm_training", left_arm_training, "/training/data13D_PCA_100000_29_13.yml");
	ss1 << ros::package::getPath("mkfbodytracker") << left_arm_training;
	std::stringstream ss2;
	std::string right_arm_training;
	ros::param::param<std::string>("right_arm_training", right_arm_training, "/training/data23D_PCA_100000_29_13.yml");
	ss2 << ros::package::getPath("mkfbodytracker") << right_arm_training;
	ROS_INFO("Getting data from %s",ss1.str().c_str());
	ROS_INFO("Getting data from %s",ss2.str().c_str());
	cv::FileStorage fs1(ss1.str(), FileStorage::READ);
	cv::FileStorage fs2(ss2.str(), FileStorage::READ);
	
	cv::Mat means1, weights1, means2, weights2, covs1, covs2, g1, g2;
    fs1["means"] >> means1;
    fs2["means"] >> means2;
  	fs1["covs"] >> covs1;
	fs2["covs"] >> covs2;
	fs1["weights"] >> weights1;
    fs2["weights"] >> weights2;
    fs1["pca_proj"] >> h1_pca;
    h1_pca.convertTo(h1_pca, CV_64F);
    fs2["pca_proj"] >> h2_pca;
    h2_pca.convertTo(h2_pca, CV_64F);
    fs1["pca_mean"] >> m1_pca;
    m1_pca.convertTo(m1_pca, CV_64F);
    fs2["pca_mean"] >> m2_pca;
    m2_pca.convertTo(m2_pca, CV_64F);
    fs2["gamma"] >> g1;
    fs2["gamma"] >> g2;
    fs1.release();
    fs2.release();
    
    d = m1_pca.cols;
	pf1 = new ParticleFilter(d,100); // left arm pf
	pf2 = new ParticleFilter(d,100); // right arm pf

	for (int i = 0; i < means1.rows; i++)
	{
		pf2->gmm.loadGaussian(means1.row(i),covs1(Range(covs1.cols*i,covs1.cols*(i+1)),Range(0,covs1.cols)), h1_pca, m1_pca, weights1.at<double>(0,i), g1.at<double>(0,i));
		pf1->gmm.loadGaussian(means2.row(i),covs2(Range(covs2.cols*i,covs2.cols*(i+1)),Range(0,covs2.cols)), h2_pca, m2_pca, weights2.at<double>(0,i), g2.at<double>(0,i));
	}
	////Load regularising gaussians - default constant velocity KF
	//pf2->gmm.loadGaussian(cv::Mat::zeros(1,means1.cols,CV_64F),0.5*cv::Mat::eye(means1.cols,means1.cols,CV_64F), 1);
	//pf1->gmm.loadGaussian(cv::Mat::zeros(1,means1.cols,CV_64F),0.5*cv::Mat::eye(means1.cols,means1.cols,CV_64F), 1);
	
	swap = false;
	edge_heuristic = 1e-1;
}

PFTracker::~PFTracker()
{
	delete sync;
	delete pf1;
	delete pf2;
}

// Create Rotation matrix from roll, pitch and yaw
cv::Mat PFTracker::rpy(double roll, double pitch, double yaw)
{
	cv::Mat R1 = (Mat_<double>(3, 3) << 1, 0, 0, 0, cos(roll), -sin(roll), 0, sin(roll), cos(roll));
	cv::Mat R2 = (Mat_<double>(3, 3) << cos(pitch), 0, sin(pitch), 0, 1, 0, -sin(pitch), 0, cos(pitch));
	cv::Mat R3 = (Mat_<double>(3, 3) << cos(yaw), -sin(yaw), 0, sin(yaw), cos(yaw), 0, 0, 0, 1);

	return  R3*R2*R1;
}

cv::Mat PFTracker::get3Dpose(cv::Mat estimate)
{
	cv::Mat est;
	estimate.copyTo(est);
	// Webcam
	//~ cv::Mat K  = (Mat_<double>(3, 3) << 660.326889, 0.0, 318.705890, 0.0, 660.857176, 240.784699, 0.0, 0.0, 1.0);
	
	// Kinect
	cv::Mat K  = (Mat_<double>(3, 3) << 525.0, 0.0, 319.5, 0.0, 525.0, 239.5, 0.0, 0.0, 1.0);
	
	// h       e     s       h       n        r  p  y   tx ty tz 
	// 0 1 2  3 4 5  6 7 8 9 10 11  12 13 14  15 16 17  18 19 20
	cv::Mat R = rpy(est.at<double>(0,16),est.at<double>(0,17),est.at<double>(0,15));
	cv::Mat t = (Mat_<double>(3, 1) << est.at<double>(0,18), est.at<double>(0,19), est.at<double>(0,20));
	
	cv::Mat P,T;
	hconcat(R, t, T);
	P = K*T;
	
	cv::Mat im_points(5,3,CV_64FC1);
	cv::Mat temp;
	for (int k = 0; k < 5; k++)
	{
		est.at<double>(0,3*k) = est.at<double>(0,3*k)*est.at<double>(0,3*k+2);
		est.at<double>(0,3*k+1) = est.at<double>(0,3*k+1)*est.at<double>(0,3*k+2);
		temp = (est.rowRange(Range(3*k,3*k+3))).t();
		temp.copyTo(im_points(Range(k,k+1),Range(0,3)));
	}
	
	cv::Mat P_I;
	invert(P(Range(0,3),Range(0,3)),P_I,DECOMP_LU);
	
	cv::Mat pos3D = P_I*(im_points-repeat(P.col(3).t(),5,1)).t();
	return pos3D;
}

bool PFTracker::edgePoseCorrection(cv::Mat image4, handBlobTracker::HFPose2DArray pfPose, cv::Mat image3)
{
	bool value = true;
	
	// Edge detection
	cv::Mat dst;
	Canny(image4, dst, 90, 200, 3);

	// Find lines in edge image
	vector<Vec4i> lines;
	HoughLinesP(dst, lines, 5, 5*CV_PI/180, 10, 5, 10 );
	
	// Declare edge probabilities
	double e1=1e-4,e2=1e-4,e3=1e-4,e4=1e-4;
	
	// Draw stick limbs
	line(image3, Point(pfPose.measurements[1].x, pfPose.measurements[1].y), Point(pfPose.measurements[5].x, pfPose.measurements[5].y), Scalar(0,255,255), 3, CV_AA); //right_arm_l
	line(image3, Point(pfPose.measurements[0].x, pfPose.measurements[0].y), Point(pfPose.measurements[4].x, pfPose.measurements[4].y), Scalar(255,0,255), 3, CV_AA); //left_arm_l
	line(image3, Point(pfPose.measurements[7].x, pfPose.measurements[7].y), Point(pfPose.measurements[5].x, pfPose.measurements[5].y), Scalar(0,255,0), 3, CV_AA); // right_arm_u
	line(image3, Point(pfPose.measurements[6].x, pfPose.measurements[6].y), Point(pfPose.measurements[4].x, pfPose.measurements[4].y), Scalar(255,255,0), 3, CV_AA); //left_arm_u
	
	double right_arm_l_x = (pfPose.measurements[1].x + pfPose.measurements[5].x)/2.0;
	double right_arm_l_y = (pfPose.measurements[1].y + pfPose.measurements[5].y)/2.0;
	double right_arm_l_th  = atan2(pfPose.measurements[1].y-pfPose.measurements[5].y,pfPose.measurements[1].x-pfPose.measurements[5].x);
	
	double left_arm_l_x = (pfPose.measurements[0].x + pfPose.measurements[4].x)/2.0;
	double left_arm_l_y = (pfPose.measurements[0].y + pfPose.measurements[4].y)/2.0;
	double left_arm_l_th  = atan2(pfPose.measurements[0].y-pfPose.measurements[4].y,pfPose.measurements[0].x-pfPose.measurements[4].x);
			
	double right_arm_u_x = (pfPose.measurements[7].x + pfPose.measurements[5].x)/2.0;
	double right_arm_u_y = (pfPose.measurements[7].y + pfPose.measurements[5].y)/2.0;
	double right_arm_u_th  = atan2(pfPose.measurements[7].y-pfPose.measurements[5].y,pfPose.measurements[7].x-pfPose.measurements[5].x);
			
	double left_arm_u_x = (pfPose.measurements[6].x + pfPose.measurements[4].x)/2.0;
	double left_arm_u_y = (pfPose.measurements[6].y + pfPose.measurements[4].y)/2.0;
	double left_arm_u_th  = atan2(pfPose.measurements[6].y-pfPose.measurements[4].y,pfPose.measurements[6].x-pfPose.measurements[4].x);
	
	for (int i = 0; i < (int)lines.size(); i++)
	{
		Vec4i l = lines[i];
		line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,0,255), 1, CV_AA);
	
		double th  = atan2(l[3]-l[1],l[2]-l[0]);
		double x = (l[0]+l[2])/2.0;
		double y = (l[1]+l[3])/2.0;
		
		double d1 = pow(right_arm_l_x-x,2)+pow(right_arm_l_y-y,2);
		double d2 = pow(left_arm_l_x-x,2)+pow(left_arm_l_y-y,2);
		double d3 = pow(left_arm_u_x-x,2)+pow(left_arm_u_y-y,2);
		double d4 = pow(right_arm_u_x-x,2)+pow(right_arm_u_y-y,2);
		
		double temp = exp(-0.5*(1.0/125.0*d1 + 1.0/0.01*pow(atan(sin(right_arm_l_th - th)/cos(right_arm_l_th - th)),2)));
		if (temp > e1)
		{
			e1 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,255,255), 1, CV_AA);
		}		
			
		temp = exp(-0.5*(1.0/125.0*d2 + 1.0/0.01*pow(atan(sin(left_arm_l_th - th)/cos(left_arm_l_th - th)),2)));
		if (temp > e2)
		{
			e2 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255,0,255), 1, CV_AA);
		}
		
		temp = exp(-0.5*(1.0/125.0*d3 + 1.0/0.01*pow(atan(sin(left_arm_u_th - th)/cos(left_arm_u_th - th)),2)));
		if (temp > e3)
		{
			e3 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255,255,0), 1, CV_AA);
		}
		
		temp = exp(-0.5*(1.0/125.0*d4 + 1.0/0.01*pow(atan(sin(right_arm_u_th - th)/cos(right_arm_u_th - th)),2)));
		if (temp > e4)
		{
			e4 = temp;
			line(image3, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(0,255,0), 1, CV_AA);
		}
	}
	
	edge_heuristic = 0.95*edge_heuristic+0.05*(e1+e2+e3+e4)/4.0; //smoothing filter to reduce clutter
	ROS_DEBUG("Forearm evidence: %f, r_upper: %f r_lower: %f l_upper: %f l_lower: %f.",edge_heuristic,e4,e1,e3,e2);	
	if ((edge_heuristic < 0.07))
	{
		ROS_WARN("Reset (line check fail): %f",edge_heuristic);
		edge_heuristic = 2e-1;
		value = false;
	}
	
	return value;
}

cv::Mat PFTracker::associateHands(const handBlobTracker::HFPose2DArrayConstPtr& msg)
{'fraction_non_alphanumeric': 0.20929027801761355, 'fraction_numerical': 0.0764404535773902, 'mean_word_length': 4.84983164983165, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5781684', 'n_tokens_mistral': 8786, 'n_tokens_neox': 8018, 'n_words': 1214}