<commit_before>#include <functional>
#include <exception>
#include <memory>
#include <vector>
#include <array>
#include <utility>

#include <algorithm>

#include <stdio.h>

#ifdef _MSC_VER
#include <windows.h>
#include <winsock.h>
#pragma comment(lib,"wsock32.lib")
#else
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <errno.h>
#define closesocket(x) close(x)
#endif

namespace net11 {
	// the tcp class is a container for listeners and connections
	class tcp;
	// a single connection, protocols should inherit this class
	class connection;
	// a sink is a data receiver
	class sink;
	// buffer is a utility class built to pass along data as a memory fifo
	class buffer;

	// a utility sink that reads a full line
	class line_parser_sink;
	// a utility sink that parses RFC 822 headers
	class header_parser_sink;

	// utility functions to create producers from a string or vector
	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T * in_data);
	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T &in_data);

	// a utility function to give up a slice of cpu time
	void yield();

	class buffer {
		int m_cap;    // the total number of bytes in this buffer
		int m_bottom; // the bottom index, ie the first used data element
		int m_top;    // the top index, the first unused data element
		char *m_data; // the actual data
	public:
		buffer(int capacity) {
			m_cap=capacity;
			m_bottom=0;
			m_top=0;
			m_data=new char[capacity];
		}
		~buffer() {
			delete m_data;
		}
		// returns the number of bytes corrently in the buffer
		int usage() {
			return m_top-m_bottom;
		}
		// returns the number of bytes available to produce as a flat array
		int direct_avail() {
			return m_cap-m_top;
		}
		// returns the total number of bytes available to produce
		int total_avail() {
			return (m_cap-m_top)+(m_bottom);
		}
		// compacts the buffer to maximize the flatly available bytes
		int compact() {
			if (m_bottom==0)
				return direct_avail();
			int sz=usage();
			memmove(m_data,m_data+m_bottom,sz);
			m_bottom=0;
			m_top=sz;
			return direct_avail();
		}
		// consumes one byte from the currently available bytes
		char consume() {
			if (m_bottom>=m_top)
				throw std::out_of_range("no bytes to consume in buffer");
			return m_data[m_bottom++];
		}
		// returns the pointer to a number of bytes to consume directly.
		const char* to_consume() {
			return m_data+m_bottom;
		}
		// tells the buffer how many bytes was consumed
		void consumed(int amount) {
			if (usage()<amount || amount<0)
				throw std::invalid_argument("underflow");
			m_bottom+=amount;
		}
		// adds a byte to the buffer
		void produce(char c) {
			if (direct_avail()<1) {
				if (compact()<1) {
					throw std::out_of_range("no bytes available in buffer");
				}
			}
			m_data[m_top++]=c;
		}
		// adds as many bytes as possible from the source to this buffer
		void produce(buffer &source) {
			// how much do we want to copy if possible?
			int to_copy=source.usage();
			// now check the actual number of bytes we can copy
			if (to_copy>total_avail())
				to_copy=total_avail();
			produce(source,to_copy);
		}
		// copy the number of bytes from the source
		void produce(buffer &source,int to_copy) {
			// if we can fit it straight away copy it directly
			if (direct_avail()<to_copy) {
				// we can't copy it directly, then compact first before copy
				if (compact()<to_copy) {
					// still not enough space, fault here!
					throw std::invalid_argument("not enough space to take the copied bytes");
				}
			}
			// now copy the amount we can take
			memcpy(to_produce(),source.to_consume(),to_copy);
			produced(to_copy);
			source.consumed(to_copy);
		}
		// returns a the pointer to the free bytes to be written
		char* to_produce() {
			return m_data+m_top;
		}
		// tell the buffer how many bytes were actually written
		void produced(int amount) {
			if (direct_avail()<amount)
				throw std::invalid_argument("overflow");
			m_top+=amount;
		}
		// convert the buffer contents to a string
		std::string to_string() {
			std::string out;
			for (int i=m_bottom;i<m_top;i++) {
				out.push_back(m_data[i]);
			}
			return out;
		}
	};

	class sink {
	public:
		// implement this function to make a working sink
		virtual bool drain(buffer &buf)=0;
	};

	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T * in_data) {
		std::shared_ptr<int> off(new int);
		std::shared_ptr<T> data(in_data);
		*off=0;
		// return the actual producer function that writes out the contents on request
		return [data,off](buffer &ob){
			int dataleft=data->size()-*off; // dl is how much we have left to send
			int outleft=ob.compact();
			int to_copy=dataleft<outleft?dataleft:outleft;
			memcpy(ob.to_produce(),data->data()+*off,to_copy);
			ob.produced(to_copy);
			*off+=to_copy;<commit_msg>* use std::memcpy and std::memmove instead of the C counterparts<commit_after>#include <functional>
#include <exception>
#include <memory>
#include <vector>
#include <array>
#include <utility>

#include <algorithm>
#include <cstring>

//#include <stdio.h>

#ifdef _MSC_VER
#include <windows.h>
#include <winsock.h>
#pragma comment(lib,"wsock32.lib")
#else
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <sys/ioctl.h>
#include <fcntl.h>
#include <errno.h>
#define closesocket(x) close(x)
#endif

namespace net11 {
	// the tcp class is a container for listeners and connections
	class tcp;
	// a single connection, protocols should inherit this class
	class connection;
	// a sink is a data receiver
	class sink;
	// buffer is a utility class built to pass along data as a memory fifo
	class buffer;

	// a utility sink that reads a full line
	class line_parser_sink;
	// a utility sink that parses RFC 822 headers
	class header_parser_sink;

	// utility functions to create producers from a string or vector
	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T * in_data);
	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T &in_data);

	// a utility function to give up a slice of cpu time
	void yield();

	class buffer {
		int m_cap;    // the total number of bytes in this buffer
		int m_bottom; // the bottom index, ie the first used data element
		int m_top;    // the top index, the first unused data element
		char *m_data; // the actual data
	public:
		buffer(int capacity) {
			m_cap=capacity;
			m_bottom=0;
			m_top=0;
			m_data=new char[capacity];
		}
		~buffer() {
			delete m_data;
		}
		// returns the number of bytes corrently in the buffer
		int usage() {
			return m_top-m_bottom;
		}
		// returns the number of bytes available to produce as a flat array
		int direct_avail() {
			return m_cap-m_top;
		}
		// returns the total number of bytes available to produce
		int total_avail() {
			return (m_cap-m_top)+(m_bottom);
		}
		// compacts the buffer to maximize the flatly available bytes
		int compact() {
			if (m_bottom==0)
				return direct_avail();
			int sz=usage();
			std::memmove(m_data,m_data+m_bottom,sz);
			m_bottom=0;
			m_top=sz;
			return direct_avail();
		}
		// consumes one byte from the currently available bytes
		char consume() {
			if (m_bottom>=m_top)
				throw std::out_of_range("no bytes to consume in buffer");
			return m_data[m_bottom++];
		}
		// returns the pointer to a number of bytes to consume directly.
		const char* to_consume() {
			return m_data+m_bottom;
		}
		// tells the buffer how many bytes was consumed
		void consumed(int amount) {
			if (usage()<amount || amount<0)
				throw std::invalid_argument("underflow");
			m_bottom+=amount;
		}
		// adds a byte to the buffer
		void produce(char c) {
			if (direct_avail()<1) {
				if (compact()<1) {
					throw std::out_of_range("no bytes available in buffer");
				}
			}
			m_data[m_top++]=c;
		}
		// adds as many bytes as possible from the source to this buffer
		void produce(buffer &source) {
			// how much do we want to copy if possible?
			int to_copy=source.usage();
			// now check the actual number of bytes we can copy
			if (to_copy>total_avail())
				to_copy=total_avail();
			produce(source,to_copy);
		}
		// copy the number of bytes from the source
		void produce(buffer &source,int to_copy) {
			// if we can fit it straight away copy it directly
			if (direct_avail()<to_copy) {
				// we can't copy it directly, then compact first before copy
				if (compact()<to_copy) {
					// still not enough space, fault here!
					throw std::invalid_argument("not enough space to take the copied bytes");
				}
			}
			// now copy the amount we can take
			std::memcpy(to_produce(),source.to_consume(),to_copy);
			produced(to_copy);
			source.consumed(to_copy);
		}
		// returns a the pointer to the free bytes to be written
		char* to_produce() {
			return m_data+m_top;
		}
		// tell the buffer how many bytes were actually written
		void produced(int amount) {
			if (direct_avail()<amount)
				throw std::invalid_argument("overflow");
			m_top+=amount;
		}
		// convert the buffer contents to a string
		std::string to_string() {
			std::string out;
			for (int i=m_bottom;i<m_top;i++) {
				out.push_back(m_data[i]);
			}
			return out;
		}
	};

	class sink {
	public:
		// implement this function to make a working sink
		virtual bool drain(buffer &buf)=0;
	};

	template<typename T>
	std::function<bool(buffer &)> make_data_producer(T * in_data) {
		std::shared_ptr<int> off(new int);
		std::shared_ptr<T> data(in_data);
		*off=0;
		// return the actual producer function that writes out the contents on request
		return [data,off](buffer &ob){
			int dataleft=data->size()-*off; // dl is how much we have left to send
			int outleft=ob.compact();
			int to_copy=dataleft<outleft?dataleft:outleft;
			std::memcpy(ob.to_produce(),data->data()+*off,to_copy);
			ob.produced(to_copy);
			*off+=to_copy;
{'fraction_non_alphanumeric': 0.11510504829234293, 'fraction_numerical': 0.003186298914666932, 'mean_word_length': 3.6200551977920883, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4065877', 'n_tokens_mistral': 3794, 'n_tokens_neox': 3357, 'n_words': 1098}