<commit_before>/********************************************************************************
*                                                                               *
*                                Base i/o device                                *
*                                                                               *
*********************************************************************************
*        Copyright (C) 2003 by <NAME>.   All Rights Reserved.            *
*       NOTE THAT I DO NOT PERMIT ANY OF MY CODE TO BE PROMOTED TO THE GPL      *
*********************************************************************************
* This code is free software; you can redistribute it and/or modify it under    *
* the terms of the GNU Library General Public License v2.1 as published by the  *
* Free Software Foundation EXCEPT that clause 3 does not apply ie; you may not  *
* "upgrade" this code to the GPL without my prior written permission.           *
* Please consult the file "License_Addendum2.txt" accompanying this file.       *
*                                                                               *
* This code is distributed in the hope that it will be useful,                  *
* but WITHOUT ANY WARRANTY; without even the implied warranty of                *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                          *
*********************************************************************************
* $Id:                                                                          *
********************************************************************************/

#include "QIODeviceS.h"
#include "FXStream.h"
#include "FXACL.h"
#include "FXException.h"
#include "QTrans.h"
#include "QThread.h"
#include "FXSecure.h"
#include "QBuffer.h"
#include <qcstring.h>
#include <string.h>
#include "FXErrCodes.h"
#ifdef USE_POSIX
#include <unistd.h>
#else
#include "WindowsGubbins.h"
#endif
#include "FXMemDbg.h"
#if defined(DEBUG) && defined(FXMEMDBG_H)
static const char *_fxmemdbg_current_file_ = __FILE__;
#endif

namespace FX {

FXfval QIODevice::at() const
{
	return ioIndex;
}

bool QIODevice::at(FXfval newpos)
{
	ioIndex=newpos;
	return true;
}

bool QIODevice::atEnd() const
{
	return at()>=size();
}

const FXACL &QIODevice::permissions() const
{
	static QMutex lock;
	static FXACL perms;
	QMtxHold lh(lock);
	if(perms.count()) return perms;
	// Fake a default
	perms.append(FXACL::Entry(FXACLEntity::owner(), 0, FXACL::Permissions().setAll()));
	return perms;
}
void QIODevice::setPermissions(const FXACL &perms)
{
	FXERRG(QTrans::tr("QIODevice", "You cannot set the permissions on this device"), QIODEVICE_BADPERMISSIONS, 0);
}

FXuval QIODevice::readLine(char *data, FXuval maxlen)
{
	FXuval count=0;
	int c;
	do
	{
		c=getch();
		if(c>=0) data[count]=(char) c;
	} while(-1!=c && data[count++]!='\n');
	data[count]=0;
	return count;
}

int QIODevice::getch()
{
	char ret=0;
	if(readBlock(&ret, 1))
		return (int) ret;
	else
		return -1;
}

int QIODevice::putch(int c)
{
	char val=(char) c;
	if(writeBlock(&val, 1))
		return c;
	else
		return -1;
}



QIODevice::UnicodeType QIODevice::determineUnicodeType(FXuchar *data, FXuval len) throw()
{
#if 1
	// Always returns NoTranslation on v1.4.x based versions
	return NoTranslation;
#else
	QByteArray _data(data, len, true);
	QBuffer dev(_data);
	FXStream s(&dev);
	char buffer[12];
	int goods[6], bads[6];
	dev.open(IO_ReadOnly);
	for(int u=UTF32LE; u!=NoTranslation; u--)
	{
		int &good=goods[u], &bad=bads[u];
		FXuval inc=u & 6;
		if(!inc) inc=1;
		good=bad=0;
		s.setBigEndian(!(u & 1));
		for(FXuval n=0; n<len-8; n+=inc)
		{
			dev.at(n);
			if(4==inc)
			{
				FXwchar *c=(FXwchar *) buffer;
				s >> c[0];
				// By UTF-32 spec the top eleven bits must be clear
				if(c[0] & 0xFFE00000) bad+=inc; else good+=inc;
			}
			else if(2==inc)
			{
				FXushort *c=(FXushort *) buffer;
				s >> c[0] >> c[1];
				if(!c[0] || !isutfvalid((FXnchar *) c))
					bad+=inc;
				else
				{
					FXint len=wcinc((FXnchar *)(data+n), 0);
					good+=inc;
					if(len>2)
						good+=inc*4;
					if(!(c[0] & 0xff00))	// Prefer high byte empty
						good+=1;
					n+=len-inc;
				}
			}
			else
			{
				if(!data[n] || !isutfvalid((char *) data+n))
					bad+=inc;
				else
				{
					FXint len=wcinc((char *) data+n, 0);
					good+=inc;
					if(len>1)
						good+=inc<<(1+len);
					n+=len-inc;
				}
			}
		}
		//if(!bad) return (UnicodeType) u;
	}
	UnicodeType ret=NoTranslation;
	goods[ret]=0;
	bads[ret]=Generic::BiggestValue<int>::value;
	// Choose that with the lowest bads
	for(int n=0; n<=UTF32LE; n++)
	{
		if((double)bads[n]/(goods[n]+0.0000001)<0.1 && bads[n]<=bads[ret] && goods[n]>goods[ret]) ret=(QIODevice::UnicodeType) n;
	}
	return ret;
#endif
}

FXuval QIODevice::applyCRLF(FXuchar *output, const FXuchar *input, FXuval outputlen, FXuval &inputlen, QIODevice::CRLFType crlftype, QIODevice::UnicodeType utftype)
{
	if(Default==crlftype)
#ifdef WIN32
		crlftype=MSDOS;
#elif defined(USE_MACOSX)
		crlftype=MacOS;
#elif defined(USE_POSIX)
		crlftype=Unix;
#else
#error Unknown system
#endif
	if(Unix==crlftype && NoTranslation==utftype)
	{
		FXuval tocopy=FXMIN(outputlen, inputlen);
		memcpy(output, input, tocopy);
		return tocopy;
	}
	FXuval o=0, i=0;
	for(;;)
	{
		FXint inlen=1, outlen=1;
		/*if(UTF16 & utftype)
		{
			inlen=wclen((FXchar *) input+i);<commit_msg>Fast case applyCRLF didn't work<commit_after>/********************************************************************************
*                                                                               *
*                                Base i/o device                                *
*                                                                               *
*********************************************************************************
*        Copyright (C) 2003 by <NAME>.   All Rights Reserved.            *
*       NOTE THAT I DO NOT PERMIT ANY OF MY CODE TO BE PROMOTED TO THE GPL      *
*********************************************************************************
* This code is free software; you can redistribute it and/or modify it under    *
* the terms of the GNU Library General Public License v2.1 as published by the  *
* Free Software Foundation EXCEPT that clause 3 does not apply ie; you may not  *
* "upgrade" this code to the GPL without my prior written permission.           *
* Please consult the file "License_Addendum2.txt" accompanying this file.       *
*                                                                               *
* This code is distributed in the hope that it will be useful,                  *
* but WITHOUT ANY WARRANTY; without even the implied warranty of                *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.                          *
*********************************************************************************
* $Id:                                                                          *
********************************************************************************/

#include "QIODeviceS.h"
#include "FXStream.h"
#include "FXACL.h"
#include "FXException.h"
#include "QTrans.h"
#include "QThread.h"
#include "FXSecure.h"
#include "QBuffer.h"
#include <qcstring.h>
#include <string.h>
#include "FXErrCodes.h"
#ifdef USE_POSIX
#include <unistd.h>
#else
#include "WindowsGubbins.h"
#endif
#include "FXMemDbg.h"
#if defined(DEBUG) && defined(FXMEMDBG_H)
static const char *_fxmemdbg_current_file_ = __FILE__;
#endif

namespace FX {

FXfval QIODevice::at() const
{
	return ioIndex;
}

bool QIODevice::at(FXfval newpos)
{
	ioIndex=newpos;
	return true;
}

bool QIODevice::atEnd() const
{
	return at()>=size();
}

const FXACL &QIODevice::permissions() const
{
	static QMutex lock;
	static FXACL perms;
	QMtxHold lh(lock);
	if(perms.count()) return perms;
	// Fake a default
	perms.append(FXACL::Entry(FXACLEntity::owner(), 0, FXACL::Permissions().setAll()));
	return perms;
}
void QIODevice::setPermissions(const FXACL &perms)
{
	FXERRG(QTrans::tr("QIODevice", "You cannot set the permissions on this device"), QIODEVICE_BADPERMISSIONS, 0);
}

FXuval QIODevice::readLine(char *data, FXuval maxlen)
{
	FXuval count=0;
	int c;
	do
	{
		c=getch();
		if(c>=0) data[count]=(char) c;
	} while(-1!=c && data[count++]!='\n');
	data[count]=0;
	return count;
}

int QIODevice::getch()
{
	char ret=0;
	if(readBlock(&ret, 1))
		return (int) ret;
	else
		return -1;
}

int QIODevice::putch(int c)
{
	char val=(char) c;
	if(writeBlock(&val, 1))
		return c;
	else
		return -1;
}



QIODevice::UnicodeType QIODevice::determineUnicodeType(FXuchar *data, FXuval len) throw()
{
#if 1
	// Always returns NoTranslation on v1.4.x based versions
	return NoTranslation;
#else
	QByteArray _data(data, len, true);
	QBuffer dev(_data);
	FXStream s(&dev);
	char buffer[12];
	int goods[6], bads[6];
	dev.open(IO_ReadOnly);
	for(int u=UTF32LE; u!=NoTranslation; u--)
	{
		int &good=goods[u], &bad=bads[u];
		FXuval inc=u & 6;
		if(!inc) inc=1;
		good=bad=0;
		s.setBigEndian(!(u & 1));
		for(FXuval n=0; n<len-8; n+=inc)
		{
			dev.at(n);
			if(4==inc)
			{
				FXwchar *c=(FXwchar *) buffer;
				s >> c[0];
				// By UTF-32 spec the top eleven bits must be clear
				if(c[0] & 0xFFE00000) bad+=inc; else good+=inc;
			}
			else if(2==inc)
			{
				FXushort *c=(FXushort *) buffer;
				s >> c[0] >> c[1];
				if(!c[0] || !isutfvalid((FXnchar *) c))
					bad+=inc;
				else
				{
					FXint len=wcinc((FXnchar *)(data+n), 0);
					good+=inc;
					if(len>2)
						good+=inc*4;
					if(!(c[0] & 0xff00))	// Prefer high byte empty
						good+=1;
					n+=len-inc;
				}
			}
			else
			{
				if(!data[n] || !isutfvalid((char *) data+n))
					bad+=inc;
				else
				{
					FXint len=wcinc((char *) data+n, 0);
					good+=inc;
					if(len>1)
						good+=inc<<(1+len);
					n+=len-inc;
				}
			}
		}
		//if(!bad) return (UnicodeType) u;
	}
	UnicodeType ret=NoTranslation;
	goods[ret]=0;
	bads[ret]=Generic::BiggestValue<int>::value;
	// Choose that with the lowest bads
	for(int n=0; n<=UTF32LE; n++)
	{
		if((double)bads[n]/(goods[n]+0.0000001)<0.1 && bads[n]<=bads[ret] && goods[n]>goods[ret]) ret=(QIODevice::UnicodeType) n;
	}
	return ret;
#endif
}

FXuval QIODevice::applyCRLF(FXuchar *output, const FXuchar *input, FXuval outputlen, FXuval &inputlen, QIODevice::CRLFType crlftype, QIODevice::UnicodeType utftype)
{
	if(Default==crlftype)
#ifdef WIN32
		crlftype=MSDOS;
#elif defined(USE_MACOSX)
		crlftype=MacOS;
#elif defined(USE_POSIX)
		crlftype=Unix;
#else
#error Unknown system
#endif
	if(Unix==crlftype && NoTranslation==utftype)
	{
		inputlen=FXMIN(outputlen, inputlen);
		memcpy(output, input, inputlen);
		return inputlen;
	}
	FXuval o=0, i=0;
	for(;;)
	{
		FXint inlen=1, outlen=1;
		/*if(UTF16 & utftype)
		{
			inlen=wclen((FXchar *) input+i);
{'fraction_non_alphanumeric': 0.20786050373650705, 'fraction_numerical': 0.015130547098440816, 'mean_word_length': 2.964886613021214, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594498', 'n_tokens_mistral': 4229, 'n_tokens_neox': 3766, 'n_words': 904}