<commit_before>/* ---------------------------------------------------------------------------
** This software is in the public domain, furnished "as is", without technical
** support, and with no warranty, express or implied, as to its usefulness for
** any purpose.
**
** main.cpp
** 
** V4L2 RTSP streamer                                                                 
**                                                                                    
** H264 capture using V4L2                                                            
** RTSP using live555                                                                 
**                                                                                    
** -------------------------------------------------------------------------*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <signal.h>
#include <sys/ioctl.h>

#include <sstream>

// libv4l2
#include <linux/videodev2.h>

// live555
#include <BasicUsageEnvironment.hh>
#include <GroupsockHelper.hh>

// project
#include "logger.h"

#include "V4l2Device.h"
#include "V4l2Capture.h"
#include "V4l2Output.h"

#include "H264_V4l2DeviceSource.h"
#include "ServerMediaSubsession.h"
#include "HTTPServer.h"

// -----------------------------------------
//    signal handler
// -----------------------------------------
char quit = 0;
void sighandler(int n)
{ 
	printf("SIGINT\n");
	quit =1;
}


// -----------------------------------------
//    create RTSP server
// -----------------------------------------
RTSPServer* createRTSPServer(UsageEnvironment& env, unsigned short rtspPort, unsigned short rtspOverHTTPPort, int timeout, unsigned int hlsSegment)
{
	UserAuthenticationDatabase* authDB = NULL;	
	RTSPServer* rtspServer = HTTPServer::createNew(env, rtspPort, authDB, timeout, hlsSegment);
	if (rtspServer != NULL)
	{
		// set http tunneling
		if (rtspOverHTTPPort)
		{
			rtspServer->setUpTunnelingOverHTTP(rtspOverHTTPPort);
		}
	}
	return rtspServer;
}


// -----------------------------------------
//    create FramedSource server
// -----------------------------------------
FramedSource* createFramedSource(UsageEnvironment* env, const V4L2DeviceParameters& param, V4l2Capture* videoCapture, int outfd, int queueSize, bool useThread, bool repeatConfig, bool muxTS)
{
	FramedSource* source = NULL;
	int format = videoCapture->getFormat();
	if (format == V4L2_PIX_FMT_H264)
	{
		source = H264_V4L2DeviceSource::createNew(*env, param, videoCapture, outfd, queueSize, useThread, repeatConfig, muxTS);
		if (muxTS)
		{
			MPEG2TransportStreamFromESSource* muxer = MPEG2TransportStreamFromESSource::createNew(*env);
			muxer->addNewVideoSource(source, 5);
			source = muxer;
		}
	}
	else
	{
		source = V4L2DeviceSource::createNew(*env, param, videoCapture, outfd, queueSize, useThread);
	}
	return source;
}
	
// -----------------------------------------
//    add an RTSP session
// -----------------------------------------
void addSession(RTSPServer* rtspServer, const std::string & sessionName, ServerMediaSubsession *subSession)
{
	UsageEnvironment& env(rtspServer->envir());
	ServerMediaSession* sms = ServerMediaSession::createNew(env, sessionName.c_str());
	if (sms != NULL)
	{
		sms->addSubsession(subSession);
		rtspServer->addServerMediaSession(sms);

		char* url = rtspServer->rtspURL(sms);
		if (url != NULL)
		{
			LOG(NOTICE) << "Play this stream using the URL \"" << url << "\"";
			delete[] url;			
		}
	}
}

std::string getRtpFormat(int format, bool muxTS)
{
	std::string rtpFormat;
	if (muxTS)
	{
		rtpFormat = "video/MP2T";
	}
	else
	{
		switch(format)
		{	
			case V4L2_PIX_FMT_H264 : rtpFormat = "video/H264"; break;
			case V4L2_PIX_FMT_JPEG : rtpFormat = "video/JPEG"; break;
#ifdef V4L2_PIX_FMT_VP8
			case V4L2_PIX_FMT_VP8  : rtpFormat = "video/VP8" ; break;
#endif
		}
	}
	
	return rtpFormat;
}

int decodeFormat(const char* fmt)
{
	char fourcc[4];
	memset(&fourcc, 0, sizeof(fourcc));
	if (fmt != NULL)
	{
		strncpy(fourcc, fmt, 4);	
	}
	return v4l2_fourcc(fourcc[0], fourcc[1], fourcc[2], fourcc[3]);
}

// -----------------------------------------
//    entry point
// -----------------------------------------
int main(int argc, char** argv) 
{
	// default parameters
	const char *dev_name = "/dev/video0";	
	int format = V4L2_PIX_FMT_H264;
	int width = 640;
	int height = 480;
	int queueSize = 10;
	int fps = 25;
	unsigned short rtspPort = 8554;
	unsigned short rtspOverHTTPPort = 0;
	bool multicast = false;
	int verbose = 0;
	std::string outputFile;
	V4l2DeviceFactory::IoType ioTypeIn  = V4l2DeviceFactory::IOTYPE_MMAP;
	V4l2DeviceFactory::IoType ioTypeOut = V4l2DeviceFactory::IOTYPE_MMAP;
	std::string url = "unicast";
	std::string murl = "multicast";
	bool useThread = true;
	std::string maddr;
	bool repeatConfig = true;
	int timeout = 65;
	bool muxTS = false;
	unsigned int hlsSegment = 0;

	// decode parameters
	int c = 0;     
	while ((c = getopt (argc, argv, "v::Q:O:" "I:P:p:m:u:M:ct:TS::" "rwsf::F:W:H:" "h")) != -1)
	{
		switch (c)
		{
			case 'v':	verbose    = 1; if (optarg && *optarg=='v') verbose++;  break;
			case 'Q':	queueSize  = atoi(optarg); break;
			case 'O':	outputFile = optarg; break;
			// RTSP/RTP
			case 'I':       ReceivingInterfaceAddr  = inet_addr(optarg); break;
			case 'P':	rtspPort                = atoi(optarg); break;
			case 'p':	rtspOverHTTPPort        = atoi(optarg); break;
			case 'u':	url                     = optarg; break;
			case 'm':	multicast = true; murl  = optarg; break;
			case 'M':	multicast = true; maddr = optarg; break;
			case 'c':	repeatConfig            = false; break;
			case 't':	timeout                 = atoi(optarg); break;
			case 'T':	muxTS                   = true; break;
			case 'S':	hlsSegment              = optarg ? atoi(optarg) : 5; muxTS=true; break;
			// V4L2
			case 'r':	ioTypeIn  = V4l2DeviceFactory::IOTYPE_READ; break;
			case 'w':	ioTypeOut = V4l2DeviceFactory::IOTYPE_READ; break;	
			case 's':	useThread =  false; break;
			case 'f':	format    = decodeFormat(optarg); break;
			case 'F':	fps       = atoi(optarg); break;
			case 'W':	width     = atoi(optarg); break;
			case 'H':	height    = atoi(optarg); break;

			case 'h':
			default:
			{
				std::cout << argv[0] << " [-v[v]] [-Q queueSize] [-O file]"                                        << std::endl;
				std::cout << "\t          [-I interface] [-P RTSP port] [-p RTSP/HTTP port] [-m multicast url] [-u unicast url] [-M multicast addr] [-c] [-t timeout] [-T] [-S[duration]]" << std::endl;
				std::cout << "\t          [-r] [-w] [-s] [-f[format] [-W width] [-H height] [-F fps] [device] [device]"         << std::endl;
				std::cout << "\t -v        : verbose"                                                               << std::endl;
				std::cout << "\t -vv       : very verbose"                                                          << std::endl;
				std::cout << "\t -Q length : Number of frame queue  (default "<< queueSize << ")"                   << std::endl;
				std::cout << "\t -O output : Copy captured frame to a file or a V4L2 device"                        << std::endl;
				std::cout << "\t RTSP/RTP options :"                                                                << std::endl;
				std::cout << "\t -I addr   : RTSP interface (default autodetect)"                                   << std::endl;
				std::cout << "\t -P port   : RTSP port (default "<< rtspPort << ")"                                 << std::endl;
				std::cout << "\t -p port   : RTSP over HTTP port (default "<< rtspOverHTTPPort << ")"               << std::endl;
				std::cout << "\t -u url    : unicast url (default " << url << ")"                                   << std::endl;
				std::cout << "\t -m url    : multicast url (default " << murl << ")"                                << std::endl;
				std::cout << "\t -M addr   : multicast group:port (default is random_address:20000)"                << std::endl;
				std::cout << "\t -c        : don't repeat config (default repeat config before IDR frame)"          << std::endl;
				std::cout << "\t -t timeout: RTCP expiration timeout in seconds (default " << timeout << ")"        << std::endl;
				std::cout << "\t -T        : send Transport Stream instead of elementary Stream"                    << std::endl;
				std::cout << "\t -S[duration]: enable HLS & MPEG-DASH with segment duration  in seconds (default 5)"     << std::endl;
				std::cout << "\t V4L2 options :"                                                                    << std::endl;
				std::cout << "\t -r        : V4L2 capture using read interface (default use memory mapped buffers)" << std::endl;
				std::cout << "\t -w        : V4L2 capture using write interface (default use memory mapped buffers)"<< std::endl;
				std::cout << "\t -s        : V4L2 capture using live555 mainloop (default use a reader thread)"     << std::endl;
				std::cout << "\t -f        : V4L2 capture using current capture format (-W,-H,-F are ignored)"      << std::endl;
				std::cout << "\t -fformat  : V4L2 capture using format (-W,-H,-F are used)"                         << std::endl;
				std::cout << "\t -W width  : V4L2 capture width (default "<< width << ")"                           << std::endl;
				std::cout << "\t -H height : V4L2 capture height (default "<< height << ")"                         << std::endl;
				std::cout << "\t -F fps    : V4L2 capture framerate (default "<< fps << ")"                         << std::endl;
				std::cout << "\t device    : V4L2 capture device (default "<< dev_name << ")"                        << std::endl;
				exit(0);
			}
		}
	}
	std::list<std::string> devList;
	while (optind<argc)
	{
		devList.push_back(argv[optind]);
		optind++;
	}
	if (devList.empty())
	{
		devList.push_back(dev_name);
	}

	// init logger
	initLogger(verbose);
     
	// create live555 environment
	TaskScheduler* scheduler = BasicTaskScheduler::createNew();
	UsageEnvironment* env = BasicUsageEnvironment::createNew(*scheduler);	

	// split multicast info
	std::istringstream is(maddr);
	std::string ip;
	getline(is, ip, ':');						
	struct in_addr destinationAddress;
	destinationAddress.s_addr = chooseRandomIPv4SSMAddress(*env);
	if (!ip.empty())
	{
		destinationAddress.s_addr = inet_addr(ip.c_str());
	}						
	
	std::string port;
	getline(is, port, ':');						
	unsigned short rtpPortNum = 20000;
	if (!port.empty())
	{
		rtpPortNum = atoi(port.c_str());
	}	
	unsigned short rtcpPortNum = rtpPortNum+1;
	unsigned char ttl = 5;
	
	// create RTSP server
	RTSPServer* rtspServer = createRTSPServer(*env, rtspPort, rtspOverHTTPPort, timeout, hlsSegment);
	if (rtspServer == NULL) 
	{
		LOG(ERROR) << "Failed to create RTSP server: " << env->getResultMsg();<commit_msg>fix #29: add authentification support<commit_after>/* ---------------------------------------------------------------------------
** This software is in the public domain, furnished "as is", without technical
** support, and with no warranty, express or implied, as to its usefulness for
** any purpose.
**
** main.cpp
** 
** V4L2 RTSP streamer                                                                 
**                                                                                    
** H264 capture using V4L2                                                            
** RTSP using live555                                                                 
**                                                                                    
** -------------------------------------------------------------------------*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <signal.h>
#include <sys/ioctl.h>

#include <sstream>

// libv4l2
#include <linux/videodev2.h>

// live555
#include <BasicUsageEnvironment.hh>
#include <GroupsockHelper.hh>

// project
#include "logger.h"

#include "V4l2Device.h"
#include "V4l2Capture.h"
#include "V4l2Output.h"

#include "H264_V4l2DeviceSource.h"
#include "ServerMediaSubsession.h"
#include "HTTPServer.h"

// -----------------------------------------
//    signal handler
// -----------------------------------------
char quit = 0;
void sighandler(int n)
{ 
	printf("SIGINT\n");
	quit =1;
}


// -----------------------------------------
//    create UserAuthenticationDatabase for RTSP server
// -----------------------------------------
UserAuthenticationDatabase* createUserAuthenticationDatabase(const std::list<std::string> & userPasswordList, const char* realm)
{
	UserAuthenticationDatabase* auth = NULL;
	if (userPasswordList.size() > 0)
	{
		auth = new UserAuthenticationDatabase(realm, (realm != NULL) );
		
		std::list<std::string>::const_iterator it;
		for (it = userPasswordList.begin(); it != userPasswordList.end(); ++it)
		{
			std::istringstream is(*it);
			std::string user;
			getline(is, user, ':');	
			std::string password;
			getline(is, password);	
			auth->addUserRecord(user.c_str(), password.c_str());
		}
	}
	
	return auth;
}

// -----------------------------------------
//    create RTSP server
// -----------------------------------------
RTSPServer* createRTSPServer(UsageEnvironment& env, unsigned short rtspPort, unsigned short rtspOverHTTPPort, int timeout, unsigned int hlsSegment, const std::list<std::string> & userPasswordList, const char* realm)
{
	UserAuthenticationDatabase* auth = createUserAuthenticationDatabase(userPasswordList, realm);
	RTSPServer* rtspServer = HTTPServer::createNew(env, rtspPort, auth, timeout, hlsSegment);
	if (rtspServer != NULL)
	{
		// set http tunneling
		if (rtspOverHTTPPort)
		{
			rtspServer->setUpTunnelingOverHTTP(rtspOverHTTPPort);
		}
	}
	return rtspServer;
}


// -----------------------------------------
//    create FramedSource server
// -----------------------------------------
FramedSource* createFramedSource(UsageEnvironment* env, const V4L2DeviceParameters& param, V4l2Capture* videoCapture, int outfd, int queueSize, bool useThread, bool repeatConfig, bool muxTS)
{
	FramedSource* source = NULL;
	int format = videoCapture->getFormat();
	if (format == V4L2_PIX_FMT_H264)
	{
		source = H264_V4L2DeviceSource::createNew(*env, param, videoCapture, outfd, queueSize, useThread, repeatConfig, muxTS);
		if (muxTS)
		{
			MPEG2TransportStreamFromESSource* muxer = MPEG2TransportStreamFromESSource::createNew(*env);
			muxer->addNewVideoSource(source, 5);
			source = muxer;
		}
	}
	else
	{
		source = V4L2DeviceSource::createNew(*env, param, videoCapture, outfd, queueSize, useThread);
	}
	return source;
}
	
// -----------------------------------------
//    add an RTSP session
// -----------------------------------------
void addSession(RTSPServer* rtspServer, const std::string & sessionName, ServerMediaSubsession *subSession)
{
	UsageEnvironment& env(rtspServer->envir());
	ServerMediaSession* sms = ServerMediaSession::createNew(env, sessionName.c_str());
	if (sms != NULL)
	{
		sms->addSubsession(subSession);
		rtspServer->addServerMediaSession(sms);

		char* url = rtspServer->rtspURL(sms);
		if (url != NULL)
		{
			LOG(NOTICE) << "Play this stream using the URL \"" << url << "\"";
			delete[] url;			
		}
	}
}

// -----------------------------------------
//    convert V4L2 pix format to RTP mime
// -----------------------------------------
std::string getRtpFormat(int format, bool muxTS)
{
	std::string rtpFormat;
	if (muxTS)
	{
		rtpFormat = "video/MP2T";
	}
	else
	{
		switch(format)
		{	
			case V4L2_PIX_FMT_H264 : rtpFormat = "video/H264"; break;
			case V4L2_PIX_FMT_JPEG : rtpFormat = "video/JPEG"; break;
#ifdef V4L2_PIX_FMT_VP8
			case V4L2_PIX_FMT_VP8  : rtpFormat = "video/VP8" ; break;
#endif
		}
	}
	
	return rtpFormat;
}

// -----------------------------------------
//    convert string format to fourcc 
// -----------------------------------------
int decodeFormat(const char* fmt)
{
	char fourcc[4];
	memset(&fourcc, 0, sizeof(fourcc));
	if (fmt != NULL)
	{
		strncpy(fourcc, fmt, 4);	
	}
	return v4l2_fourcc(fourcc[0], fourcc[1], fourcc[2], fourcc[3]);
}

// -------------------------------------------------------
//    decode multicast url <group>:<rtp_port>:<rtcp_port>
// -------------------------------------------------------
void decodeMulticastUrl(const std::string & maddr, in_addr & destinationAddress, unsigned short & rtpPortNum, unsigned short & rtcpPortNum)
{
	std::istringstream is(maddr);
	std::string ip;
	getline(is, ip, ':');						
	if (!ip.empty())
	{
		destinationAddress.s_addr = inet_addr(ip.c_str());
	}						
	
	std::string port;
	getline(is, port, ':');						
	rtpPortNum = 20000;
	if (!port.empty())
	{
		rtpPortNum = atoi(port.c_str());
	}	
	rtcpPortNum = rtpPortNum+1;
}

// -----------------------------------------
//    entry point
// -----------------------------------------
int main(int argc, char** argv) 
{
	// default parameters
	const char *dev_name = "/dev/video0";	
	int format = V4L2_PIX_FMT_H264;
	int width = 640;
	int height = 480;
	int queueSize = 10;
	int fps = 25;
	unsigned short rtspPort = 8554;
	unsigned short rtspOverHTTPPort = 0;
	bool multicast = false;
	int verbose = 0;
	std::string outputFile;
	V4l2DeviceFactory::IoType ioTypeIn  = V4l2DeviceFactory::IOTYPE_MMAP;
	V4l2DeviceFactory::IoType ioTypeOut = V4l2DeviceFactory::IOTYPE_MMAP;
	std::string url = "unicast";
	std::string murl = "multicast";
	bool useThread = true;
	std::string maddr;
	bool repeatConfig = true;
	int timeout = 65;
	bool muxTS = false;
	unsigned int hlsSegment = 0;
	const char* realm = NULL;
	std::list<std::string> userPasswordList;

	// decode parameters
	int c = 0;     
	while ((c = getopt (argc, argv, "v::Q:O:" "I:P:p:m:u:M:ct:TS::R:U:" "rwsf::F:W:H:" "h")) != -1)
	{
		switch (c)
		{
			case 'v':	verbose    = 1; if (optarg && *optarg=='v') verbose++;  break;
			case 'Q':	queueSize  = atoi(optarg); break;
			case 'O':	outputFile = optarg; break;
			
			// RTSP/RTP
			case 'I':       ReceivingInterfaceAddr  = inet_addr(optarg); break;
			case 'P':	rtspPort                = atoi(optarg); break;
			case 'p':	rtspOverHTTPPort        = atoi(optarg); break;
			case 'u':	url                     = optarg; break;
			case 'm':	multicast = true; murl  = optarg; break;
			case 'M':	multicast = true; maddr = optarg; break;
			case 'c':	repeatConfig            = false; break;
			case 't':	timeout                 = atoi(optarg); break;
			case 'T':	muxTS                   = true; break;
			case 'S':	hlsSegment              = optarg ? atoi(optarg) : 5; muxTS=true; break;
			case 'R':       realm                   = optarg; break;
			case 'U':       userPasswordList.push_back(optarg); break;
			
			// V4L2
			case 'r':	ioTypeIn  = V4l2DeviceFactory::IOTYPE_READ; break;
			case 'w':	ioTypeOut = V4l2DeviceFactory::IOTYPE_READ; break;	
			case 's':	useThread =  false; break;
			case 'f':	format    = decodeFormat(optarg); break;
			case 'F':	fps       = atoi(optarg); break;
			case 'W':	width     = atoi(optarg); break;
			case 'H':	height    = atoi(optarg); break;

			case 'h':
			default:
			{
				std::cout << argv[0] << " [-v[v]] [-Q queueSize] [-O file]"                                        << std::endl;
				std::cout << "\t          [-I interface] [-P RTSP port] [-p RTSP/HTTP port] [-m multicast url] [-u unicast url] [-M multicast addr] [-c] [-t timeout] [-T] [-S[duration]]" << std::endl;
				std::cout << "\t          [-r] [-w] [-s] [-f[format] [-W width] [-H height] [-F fps] [device] [device]"         << std::endl;
				std::cout << "\t -v        : verbose"                                                               << std::endl;
				std::cout << "\t -vv       : very verbose"                                                          << std::endl;
				std::cout << "\t -Q length : Number of frame queue  (default "<< queueSize << ")"                   << std::endl;
				std::cout << "\t -O output : Copy captured frame to a file or a V4L2 device"                        << std::endl;
				
				std::cout << "\t RTSP/RTP options :"                                                                << std::endl;
				std::cout << "\t -I addr   : RTSP interface (default autodetect)"                                   << std::endl;
				std::cout << "\t -P port   : RTSP port (default "<< rtspPort << ")"                                 << std::endl;
				std::cout << "\t -p port   : RTSP over HTTP port (default "<< rtspOverHTTPPort << ")"               << std::endl;
				std::cout << "\t -U user:password : RTSP user and password"                                         << std::endl;
				std::cout << "\t -R realm  : use md5 password '<PASSWORD>(<username>:<realm>:<password>')"                 << std::endl;
				std::cout << "\t -u url    : unicast url (default " << url << ")"                                   << std::endl;
				std::cout << "\t -m url    : multicast url (default " << murl << ")"                                << std::endl;
				std::cout << "\t -M addr   : multicast group:port (default is random_address:20000)"                << std::endl;
				std::cout << "\t -c        : don't repeat config (default repeat config before IDR frame)"          << std::endl;
				std::cout << "\t -t timeout: RTCP expiration timeout in seconds (default " << timeout << ")"        << std::endl;
				std::cout << "\t -T        : send Transport Stream instead of elementary Stream"                    << std::endl;				
				std::cout << "\t -S[duration]: enable HLS & MPEG-DASH with segment duration  in seconds (default 5)"<< std::endl;
				
				std::cout << "\t V4L2 options :"                                                                    << std::endl;
				std::cout << "\t -r        : V4L2 capture using read interface (default use memory mapped buffers)" << std::endl;
				std::cout << "\t -w        : V4L2 capture using write interface (default use memory mapped buffers)"<< std::endl;
				std::cout << "\t -s        : V4L2 capture using live555 mainloop (default use a reader thread)"     << std::endl;
				std::cout << "\t -f        : V4L2 capture using current capture format (-W,-H,-F are ignored)"      << std::endl;
				std::cout << "\t -fformat  : V4L2 capture using format (-W,-H,-F are used)"                         << std::endl;
				std::cout << "\t -W width  : V4L2 capture width (default "<< width << ")"                           << std::endl;
				std::cout << "\t -H height : V4L2 capture height (default "<< height << ")"                         << std::endl;
				std::cout << "\t -F fps    : V4L2 capture framerate (default "<< fps << ")"                         << std::endl;
				std::cout << "\t device    : V4L2 capture device (default "<< dev_name << ")"                       << std::endl;
				exit(0);
			}
		}
	}
	std::list<std::string> devList;
	while (optind<argc)
	{
		devList.push_back(argv[optind]);
		optind++;
	}
	if (devList.empty())
	{
		devList.push_back(dev_name);
	}

	// init logger
	initLogger(verbose);
     
	// create live555 environment
	TaskScheduler* scheduler = BasicTaskScheduler::createNew();
	UsageEnvironment* env = BasicUsageEnvironment::createNew(*scheduler);	

	// split multicast info
	struct in_addr destinationAddress;
	destinationAddress.s_addr = chooseRandomIPv4SSMAddress(*env);
	unsigned short rtpPortNum = 20000;
	unsigned short rtcpPortNum = rtpPortNum+1;
	unsigned char ttl = 5;
	decodeMulticastUrl(maddr, destinationAddress, rtpPortNum, rtcpPortNum);	
	
	// create RTSP server
	RTSPServer* rtspServer = createRTSPServer(*env, rtspPort, rtspOverHTTPPort, timeout, hlsSegment, userPasswordList, realm);
	if (rtspServer == NULL) 
	{
		LOG(ERROR) << "Failed to create RTSP server: " << env->getResultMsg();
{'fraction_non_alphanumeric': 0.19769657450141848, 'fraction_numerical': 0.01422703984418004, 'mean_word_length': 1.9485642946317103, 'pattern_counts': {'":': 0, '<': 374, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 58, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1143357', 'n_tokens_mistral': 7965, 'n_tokens_neox': 7339, 'n_words': 1864}