<commit_before>/*******************************************************************************
 * libproxy - A library for proxy configuration
 * Copyright (C) 2006 <NAME> <<EMAIL>>
 *
 * Based on work found in GLib GIO:
 * Copyright (C) 2006-2007 Red Hat, Inc.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 *
 ******************************************************************************/
#ifdef WIN32
#include <io.h>
#define open _open
#define O_RDONLY _O_RDONLY
#define close _close
#endif
#include <fcntl.h> // For ::open()
#include <cstring> // For memcpy()
#include <sstream> // For int/string conversion (using stringstream)
#include <cstdio>  // For sscanf()
#include <cstdlib>    // For atoi()
#include <sys/stat.h> // For stat()
#include <algorithm> // For transform()

#ifdef WIN32
#include <io.h>
#define pfsize(st) (st.st_size)
#define close _close
#define read _read
#define SHUT_RDWR SD_BOTH
#else
#define pfsize(st) (st.st_blksize * st.st_blocks)
#endif

#include "url.hpp"
using namespace libproxy;
using namespace std;

// This mime type should be reported by the web server
#define PAC_MIME_TYPE "application/x-ns-proxy-autoconfig"
// Fall back to checking for this mime type, which servers often report wrong
#define PAC_MIME_TYPE_FB "text/plain"

// This is the maximum pac size (to avoid memory attacks)
#define PAC_MAX_SIZE 102400

const string url::GENERIC_DELIMITERS(":/?#[]@");
const string url::SUBCOMPONENT_DELIMITERS("!$&'()*+,;=");
const string url::ALLOWED_IN_USERINFO_ELEMENT(url::SUBCOMPONENT_DELIMITERS);
const string url::ALLOWED_IN_USERINFO(url::ALLOWED_IN_USERINFO_ELEMENT + ":");
const string url::ALLOWED_IN_PATH_ELEMENT(url::SUBCOMPONENT_DELIMITERS + ":@");
const string url::ALLOWED_IN_PATH(url::ALLOWED_IN_PATH_ELEMENT + "/");

static inline int get_default_port(string scheme) {
	struct servent *serv;
	size_t plus = scheme.find('+');

	if (plus != string::npos)
		scheme = scheme.substr(plus + 1);

	if ((serv = getservbyname(scheme.c_str(), NULL)))
		return ntohs(serv->s_port);

	return 0;
}

template <class T>
static inline string to_string_ (const T& t) {
	stringstream ss;
	ss << t;
	return ss.str();
}

#define _copyaddr_t(type, addr) (sockaddr*) memcpy(new type, &(addr), sizeof(type))
static inline sockaddr* copyaddr(const struct sockaddr& addr) {
	switch (addr.sa_family) {
	case (AF_INET):
		return _copyaddr_t(sockaddr_in, addr);
	case (AF_INET6):
		return _copyaddr_t(sockaddr_in6, addr);
	default:
		return NULL;
	}
}

bool url::is_valid(const string url_) {
	bool rtv = true;

	try {
		url url(url_);
	} catch (parse_error&) {
		rtv = false;
	}

    return rtv;
}

string url::encode(const string &data, const string &valid_reserved) {
	ostringstream encoded;
	for (int i=0; data[i]; i++) {
		if (isalnum(data[i])<commit_msg>Don't read NULL character in url::encode<commit_after>/*******************************************************************************
 * libproxy - A library for proxy configuration
 * Copyright (C) 2006 <NAME> <<EMAIL>>
 *
 * Based on work found in GLib GIO:
 * Copyright (C) 2006-2007 Red Hat, Inc.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 *
 ******************************************************************************/
#ifdef WIN32
#include <io.h>
#define open _open
#define O_RDONLY _O_RDONLY
#define close _close
#endif
#include <fcntl.h> // For ::open()
#include <cstring> // For memcpy()
#include <sstream> // For int/string conversion (using stringstream)
#include <cstdio>  // For sscanf()
#include <cstdlib>    // For atoi()
#include <sys/stat.h> // For stat()
#include <algorithm> // For transform()

#ifdef WIN32
#include <io.h>
#define pfsize(st) (st.st_size)
#define close _close
#define read _read
#define SHUT_RDWR SD_BOTH
#else
#define pfsize(st) (st.st_blksize * st.st_blocks)
#endif

#include "url.hpp"
using namespace libproxy;
using namespace std;

// This mime type should be reported by the web server
#define PAC_MIME_TYPE "application/x-ns-proxy-autoconfig"
// Fall back to checking for this mime type, which servers often report wrong
#define PAC_MIME_TYPE_FB "text/plain"

// This is the maximum pac size (to avoid memory attacks)
#define PAC_MAX_SIZE 102400

const string url::GENERIC_DELIMITERS(":/?#[]@");
const string url::SUBCOMPONENT_DELIMITERS("!$&'()*+,;=");
const string url::ALLOWED_IN_USERINFO_ELEMENT(url::SUBCOMPONENT_DELIMITERS);
const string url::ALLOWED_IN_USERINFO(url::ALLOWED_IN_USERINFO_ELEMENT + ":");
const string url::ALLOWED_IN_PATH_ELEMENT(url::SUBCOMPONENT_DELIMITERS + ":@");
const string url::ALLOWED_IN_PATH(url::ALLOWED_IN_PATH_ELEMENT + "/");

static inline int get_default_port(string scheme) {
	struct servent *serv;
	size_t plus = scheme.find('+');

	if (plus != string::npos)
		scheme = scheme.substr(plus + 1);

	if ((serv = getservbyname(scheme.c_str(), NULL)))
		return ntohs(serv->s_port);

	return 0;
}

template <class T>
static inline string to_string_ (const T& t) {
	stringstream ss;
	ss << t;
	return ss.str();
}

#define _copyaddr_t(type, addr) (sockaddr*) memcpy(new type, &(addr), sizeof(type))
static inline sockaddr* copyaddr(const struct sockaddr& addr) {
	switch (addr.sa_family) {
	case (AF_INET):
		return _copyaddr_t(sockaddr_in, addr);
	case (AF_INET6):
		return _copyaddr_t(sockaddr_in6, addr);
	default:
		return NULL;
	}
}

bool url::is_valid(const string url_) {
	bool rtv = true;

	try {
		url url(url_);
	} catch (parse_error&) {
		rtv = false;
	}

    return rtv;
}

string url::encode(const string &data, const string &valid_reserved) {
	ostringstream encoded;
	for (unsigned int i=0; i < data.size(); i++) {
		if (isalnum(data[i])
{'fraction_non_alphanumeric': 0.15811234673510122, 'fraction_numerical': 0.01140575990875392, 'mean_word_length': 5.026632302405498, 'pattern_counts': {'":': 6, '<': 34, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435351', 'n_tokens_mistral': 2353, 'n_tokens_neox': 2265, 'n_words': 764}