<commit_before>// Copyright 2016 <NAME>
// MIT licensed (see LICENSE for details)

#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "url.h"

#define numberof(x) (sizeof(x) / sizeof(*(x)))

static void strlower(char *const str) {
	assert(str);
	for(size_t i = 0; '\0' != str[i]; i++) {
		char const c = str[i];
		if(c >= 'A' && c <= 'Z') str[i] += 'a'-'A'; // 0x20
	}
}
int url_parse(char const *const URL, url_t *const out) {
	assert(out);
	if(!URL) return -EINVAL;
	int len = 0;
	out->protocol[0] = '\0';
	out->host[0] = '\0';
	out->path[0] = '\0';
	sscanf(URL, PROTOCOL_FMT "://" HOST_FMT PATH_FMT "%n", out->protocol, out->host, out->path, &len);
	if(0 == len) return -EINVAL;
	if('\0' != URL[len] && '#' != URL[len]) return -EINVAL;
	if('\0' == out->protocol[0]) return -EINVAL;
	if('\0' == out->host[0]) return -EINVAL;
	if('\0' != out->path[0] && '/' != out->path[0]) return -EINVAL;
	strlower(out->protocol);
	strlower(out->host);
	return 0;
}
int url_format(url_t const *const URL, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!URL) return -EINVAL;
	int rc = snprintf(out, max, "%s://%s%s", URL->protocol, URL->host, URL->path);
	if(rc >= max) return -ENAMETOOLONG;
	if(rc < 0) return rc;
	return 0;
}

int host_parse(char const *const host, host_t *const out) {
	assert(out);
	if(!host) return -EINVAL;
	out->domain[0] = '\0';
	out->port = 0;
	sscanf(host, DOMAIN_FMT ":%u", out->domain, &out->port);
	if('\0' == out->domain[0]) return -EINVAL;
	if(out->port > UINT16_MAX) return -EINVAL;
	return 0;
}
int host_format(host_t const *const host, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!host) return -EINVAL;
	int rc = 0;
	if(host->port) {
		rc = snprintf(out, max, "%s:%u", host->domain, host->port);
	} else {
		rc = snprintf(out, max, "%s", host->domain);
	}
	if(rc >= max) return -ENAMETOOLONG;
	if(rc < 0) return rc;
	return 0;
}

#define SUBDOMAIN_FMT "%m[a-z0-9-]"
static int domain_reverse(char const *const domain, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!domain) return -EINVAL;
	char *d[8] = {};
	sscanf(domain,
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT,
		&d[0], &d[1], &d[2], &d[3],
		&d[4], &d[5], &d[6], &d[7]
	);
	strlcpy(out, "(", max);
	for(size_t i = numberof(d); i-- > 0;) {
		if(!d[i]) continue;
		strlcat(out, d[i], max);
		strlcat(out, ",", max);
		free(&d[i]); d[i] = NULL;
	}
	strlcat(out, ")", max);
	return 0;
}
int url_normalize_surt(char const *const URL, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!URL) return -EINVAL;<commit_msg>domain_reverse() is pretty broken.<commit_after>// Copyright 2016 <NAME>
// MIT licensed (see LICENSE for details)

#include <assert.h>
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "url.h"

#define numberof(x) (sizeof(x) / sizeof(*(x)))

static void strlower(char *const str) {
	assert(str);
	for(size_t i = 0; '\0' != str[i]; i++) {
		char const c = str[i];
		if(c >= 'A' && c <= 'Z') str[i] += 'a'-'A'; // 0x20
	}
}
int url_parse(char const *const URL, url_t *const out) {
	assert(out);
	if(!URL) return -EINVAL;
	int len = 0;
	out->protocol[0] = '\0';
	out->host[0] = '\0';
	out->path[0] = '\0';
	sscanf(URL, PROTOCOL_FMT "://" HOST_FMT PATH_FMT "%n", out->protocol, out->host, out->path, &len);
	if(0 == len) return -EINVAL;
	if('\0' != URL[len] && '#' != URL[len]) return -EINVAL;
	if('\0' == out->protocol[0]) return -EINVAL;
	if('\0' == out->host[0]) return -EINVAL;
	if('\0' != out->path[0] && '/' != out->path[0]) return -EINVAL;
	strlower(out->protocol);
	strlower(out->host);
	return 0;
}
int url_format(url_t const *const URL, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!URL) return -EINVAL;
	int rc = snprintf(out, max, "%s://%s%s", URL->protocol, URL->host, URL->path);
	if(rc >= max) return -ENAMETOOLONG;
	if(rc < 0) return rc;
	return 0;
}

int host_parse(char const *const host, host_t *const out) {
	assert(out);
	if(!host) return -EINVAL;
	out->domain[0] = '\0';
	out->port = 0;
	sscanf(host, DOMAIN_FMT ":%u", out->domain, &out->port);
	if('\0' == out->domain[0]) return -EINVAL;
	if(out->port > UINT16_MAX) return -EINVAL;
	return 0;
}
int host_format(host_t const *const host, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!host) return -EINVAL;
	int rc = 0;
	if(host->port) {
		rc = snprintf(out, max, "%s:%u", host->domain, host->port);
	} else {
		rc = snprintf(out, max, "%s", host->domain);
	}
	if(rc >= max) return -ENAMETOOLONG;
	if(rc < 0) return rc;
	return 0;
}

#define SUBDOMAIN_FMT "%m[a-z0-9-]"
static int domain_reverse(char const *const domain, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!domain) return -EINVAL;
	// TODO:
	// 1. Silently truncates domains with more than 8 levels
	// 2. Silently truncates broken domains
	// 3. Doesn't do much validation.
	char *d[8] = {};
	sscanf(domain,
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT "."
		SUBDOMAIN_FMT,
		&d[0], &d[1], &d[2], &d[3],
		&d[4], &d[5], &d[6], &d[7]
	);
	strlcpy(out, "(", max);
	for(size_t i = numberof(d); i-- > 0;) {
		if(!d[i]) continue;
		strlcat(out, d[i], max);
		strlcat(out, ",", max);
		free(&d[i]); d[i] = NULL;
	}
	strlcat(out, ")", max);
	return 0;
}
int url_normalize_surt(char const *const URL, char *const out, size_t const max) {
	assert(out);
	assert(max > 0);
	if(!URL) return -EINVAL;
{'fraction_non_alphanumeric': 0.21855921855921856, 'fraction_numerical': 0.020233734519448807, 'mean_word_length': 3.990426457789382, 'pattern_counts': {'":': 4, '<': 23, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7602072', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2451, 'n_words': 610}