<commit_before>// SciTE - Scintilla based Text Editor
/** @file Cookie.cxx
 ** Examine start of files for coding cookies and type information.
 **/
// Copyright 2011 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>

#include <string>
#include <vector>

#include "Scintilla.h"

#include "GUI.h"

#include "StringHelpers.h"
#include "Cookie.h"

std::string ExtractLine(const char *buf, size_t length) {
	unsigned int endl = 0;
	if (length > 0) {
		while ((endl < length) && (buf[endl] != '\r') && (buf[endl] != '\n')) {
			endl++;
		}
		if (((endl + 1) < length) && (buf[endl] == '\r') && (buf[endl+1] == '\n')) {
			endl++;
		}
		if (endl < length) {
			endl++;
		}
	}
	return std::string(buf, 0, endl);
}

static const char codingCookie[] = "coding";

static bool isEncodingChar(char ch) {
	return (ch == '_') || (ch == '-') || (ch == '.') ||
	       (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
	       (ch >= '0' && ch <= '9');
}

static bool isSpaceChar(char ch) {
	return (ch == ' ') || (ch == '\t');
}

static UniMode CookieValue(const std::string &s) {
	size_t posCoding = s.find(codingCookie);
	if (posCoding != std::string::npos) {
		posCoding += strlen(codingCookie);
		if ((s[posCoding] == ':') || (s[posCoding] == '=')) {
			posCoding++;
			if ((s[posCoding] == '\"') || (s[posCoding] == '\'')) {
				posCoding++;
			}
			while ((posCoding < s.length()) &&
			        (isSpaceChar(s[posCoding]))) {
				posCoding++;
			}
			size_t endCoding = posCoding;
			while ((endCoding < s.length()) &&
			        (isEncodingChar(s[endCoding]))) {
				endCoding++;
			}
			std::string code(s, posCoding, endCoding-posCoding);
			LowerCaseAZ(code);
			if (code == "utf-8") {
				return uniCookie;
			}
		}
	}
	return uni8Bit;
}

UniMode CodingCookieValue(const char *buf, size_t length) {
	std::string l1 = ExtractLine(buf, length);
	UniMode unicodeMode = CookieValue(l1);
	if (unicodeMode == uni8Bit) {
		std::string l2 = ExtractLine(buf + l1.length(), length - l1.length());
		unicodeMode = CookieValue(l2);
	}
	return unicodeMode;
}

<commit_msg>Invoke the correct constructor. The only 3 argument std::string constructor is (const string& str, size_t pos, size_t len = npos). This requires a std::string first arg, not a char* so the char* is converted to std::string with the single argument constructor which depends on a NUL terminator. When the file being opened is larger than the buffer size (128K) there is no NUL and the call fails.<commit_after>// SciTE - Scintilla based Text Editor
/** @file Cookie.cxx
 ** Examine start of files for coding cookies and type information.
 **/
// Copyright 2011 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>

#include <string>
#include <vector>

#include "Scintilla.h"

#include "GUI.h"

#include "StringHelpers.h"
#include "Cookie.h"

std::string ExtractLine(const char *buf, size_t length) {
	unsigned int endl = 0;
	if (length > 0) {
		while ((endl < length) && (buf[endl] != '\r') && (buf[endl] != '\n')) {
			endl++;
		}
		if (((endl + 1) < length) && (buf[endl] == '\r') && (buf[endl+1] == '\n')) {
			endl++;
		}
		if (endl < length) {
			endl++;
		}
	}
	return std::string(buf, endl);
}

static const char codingCookie[] = "coding";

static bool isEncodingChar(char ch) {
	return (ch == '_') || (ch == '-') || (ch == '.') ||
	       (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
	       (ch >= '0' && ch <= '9');
}

static bool isSpaceChar(char ch) {
	return (ch == ' ') || (ch == '\t');
}

static UniMode CookieValue(const std::string &s) {
	size_t posCoding = s.find(codingCookie);
	if (posCoding != std::string::npos) {
		posCoding += strlen(codingCookie);
		if ((s[posCoding] == ':') || (s[posCoding] == '=')) {
			posCoding++;
			if ((s[posCoding] == '\"') || (s[posCoding] == '\'')) {
				posCoding++;
			}
			while ((posCoding < s.length()) &&
			        (isSpaceChar(s[posCoding]))) {
				posCoding++;
			}
			size_t endCoding = posCoding;
			while ((endCoding < s.length()) &&
			        (isEncodingChar(s[endCoding]))) {
				endCoding++;
			}
			std::string code(s, posCoding, endCoding-posCoding);
			LowerCaseAZ(code);
			if (code == "utf-8") {
				return uniCookie;
			}
		}
	}
	return uni8Bit;
}

UniMode CodingCookieValue(const char *buf, size_t length) {
	std::string l1 = ExtractLine(buf, length);
	UniMode unicodeMode = CookieValue(l1);
	if (unicodeMode == uni8Bit) {
		std::string l2 = ExtractLine(buf + l1.length(), length - l1.length());
		unicodeMode = CookieValue(l2);
	}
	return unicodeMode;
}


{'fraction_non_alphanumeric': 0.19358947144979835, 'fraction_numerical': 0.00912757376353216, 'mean_word_length': 3.5307692307692307, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704334', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1686, 'n_words': 454}