<commit_before>/*
 * Copyright 2013-2017 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <ctype.h>
#include <glob.h>
#include <regex.h>
#include <sys/stat.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>
#include <assert.h>

#include "fnv1a.h"
#include "consistent-hash.h"
#include "server.h"
#include "queue.h"
#include "aggregator.h"
#include "allocator.h"
#include "relay.h"
#include "router.h"
#include "conffile.h"
#include "conffile.tab.h"

struct _router {
	cluster *clusters;
	route *routes;
	aggregator *aggregators;
	servers *srvrs;
	struct _router_collector {
		char *stub;
		int interval;
		char *prefix;
		col_mode mode;
	} collector;
	struct _router_parser_err {
		const char *msg;
		size_t line;
		size_t start;
		size_t stop;
	} parser_err;
	struct _router_conf {
		size_t queuesize;
		size_t batchsize;
		int maxstalls;
		unsigned short iotimeout;
		unsigned int sockbufsize;
	} conf;
	allocator *a;
};

/* declarations to avoid symbol/include hell when doing config.yy.h */
int router_yylex_init(void **);
void router_yy_scan_string(char *, void *);
int router_yylex_destroy(void *);

/* catch string used for aggrgator destinations */
#define STUB_AGGR   "_aggregator_stub_"
/* catch string used for collector output */
#define STUB_STATS  "_collector_stub_"

/**
 * Frees regexes from routes.
 */
static void
router_free_intern(route *routes)
{
	/* the only thing that isn't allocated using the allocators are the
	 * regexes */
	while (routes != NULL) {
		if (routes->matchtype == REGEX)
			regfree(&routes->rule);

		if (routes->next == NULL || routes->next->dests != routes->dests) {
			while (routes->dests != NULL) {
				if (routes->dests->cl->type == GROUP ||
						routes->dests->cl->type == AGGRSTUB ||
						routes->dests->cl->type == STATSTUB)
					router_free_intern(routes->dests->cl->members.routes);
				if (routes->dests->cl->type == VALIDATION)
					router_free_intern(
							routes->dests->cl->members.validation->rule);

				routes->dests = routes->dests->next;
			}
		}

		routes = routes->next;
	}
}

/**
 * Examines pattern and sets matchtype and rule or strmatch in route.
 */
static inline int
determine_if_regex(allocator *a, route *r, char *pat)
{
	/* try and see if we can avoid using a regex match, for
	 * it is simply very slow/expensive to do so: most of
	 * the time, people don't need fancy matching rules */
	char patbuf[8192];
	char *e = pat;
	char *pb = patbuf;
	char escape = 0;
	r->matchtype = CONTAINS;
	r->nmatch = 0;

	if (*e == '^' && r->matchtype == CONTAINS) {
		e++;
		r->matchtype = STARTS_WITH;
	}
	for (; *e != '\0'; e++) {
		switch (*e) {
			case '\\':
				if (escape)
					*pb++ = *e;
				escape = !escape;
				break;
			case '.':
			case '^':
			case '*':
			case '+':
				if (!escape)
					r->matchtype = REGEX;
				*pb++ = *e;
				escape = 0;
				break;
			case '$':
				if (!escape && e[1] == '\0') {
					if (r->matchtype == STARTS_WITH) {
						r->matchtype = MATCHES;
					} else {
						r->matchtype = ENDS_WITH;
					}
				} else {
					r->matchtype = REGEX;
				}
				escape = 0;
				break;
			default:
				if (
						!escape && (
							(*e == '_') || (*e == '-') ||
							(*e >= '0' && *e <= '9') ||
							(*e >= 'a' && *e <= 'z') ||
							(*e >= 'A' && *e <= 'Z')
						)
				   )
				{
					*pb++ = *e;
				} else {
					r->matchtype = REGEX;
				}
				escape = 0;
				break;
		}
		if (pb - patbuf == sizeof(patbuf))
			r->matchtype = REGEX;
		if (r->matchtype == REGEX)
			break;
	}
	if (r->matchtype != REGEX) {
		*pb = '\0';
		r->strmatch = ra_strdup(a, patbuf);
		r->pattern = ra_strdup(a, pat);
	} else {
		int ret = regcomp(&r->rule, pat, REG_EXTENDED);
		if (ret != 0)
			return ret;  /* allow use of regerror */
		r->strmatch = NULL;
		r->pattern = ra_strdup(a, pat);
		if (r->rule.re_nsub > 0) {
			/* we need +1 because position 0 contains the entire
			 * expression */
			r->nmatch = r->rule.re_nsub + 1;
			if (r->nmatch > RE_MAX_MATCHES) {
				logerr("determine_if_regex: too many match groups, "
						"please increase RE_MAX_MATCHES in router.h\n");
				return REG_ESPACE;  /* lie closest to the truth */
			}
		}
	}

	return 0;
}

static char serverip_server[256];  /* RFC1035 = 250 */
static const char *
serverip(server *s)
{
	const char *srvr = server_ip(s);

	if (strchr(srvr, ':') != NULL) {
		snprintf(serverip_server, sizeof(serverip_server), "[%s]", srvr);
		return serverip_server;
	}

	return srvr;
}


/**
 * Callback for the bison parser for yyerror
 */
void
router_yyerror(
		void *locptr,
		void *s,
		router *r,
		allocator *ra,
		allocator *a,
		const char *msg)
{
	(void)s;
	ROUTER_YYLTYPE *locp = (ROUTER_YYLTYPE *)locptr;
	regex_t re;
	size_t nmatch = 3;
	regmatch_t pmatch[3];
	char buf1[METRIC_BUFSIZ];
	char buf2[METRIC_BUFSIZ];
	char *dummy;
	char (*sa)[METRIC_BUFSIZ];
	char (*sb)[METRIC_BUFSIZ];
	char (*st)[METRIC_BUFSIZ];

	if (r->parser_err.msg != NULL)
		return;

	/* clean up the "value" types */
	if (regcomp(&re, "cr(STRING|COMMENT|INTVAL)", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	snprintf(buf1, sizeof(buf1), "%s", msg);
	sa = &buf1;
	sb = &buf2;
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "\\_1", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	/* clean up the keywords */
	if (regcomp(&re, "cr([A-Z]+)", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "'\\_1'", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	/* clean up bison speak */
	if (regcomp(&re, "\\$end", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "end of file", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	r->parser_err.msg = ra_strdup(a, *sa);
	if (locp != NULL) {
		r->parser_err.line = 1 + locp->first_line;
		r->parser_err.start = locp->first_column;
		if (locp->first_line == locp->last_line) {
			r->parser_err.stop = locp->last_column;
		} else {
			r->parser_err.stop = locp->first_column;
		}
	} else {
		r->parser_err.line = 0;
	}
}

/**
 * Parse ip string and validate it.  When it is a numeric address, write
 * the cannonical version in retip.
 */
char *
router_validate_address(
		router *rtr,
		char **retip, int *retport, void **retsaddr, void **rethint,
		char *ip, serv_ctype proto)
{
	int port = 2003;
	struct addrinfo *saddr = NULL;
	struct addrinfo hint;
	char sport[8];
	char hnbuf[256];
	char *p;
	char *lastcolon = NULL;

	for (p = ip; *p != '\0'; p++)
		if (*p == ':')
			lastcolon = p;

	if (*(p - 1) == ']')
		lastcolon = NULL;
	if (lastcolon != NULL) {
		char *endp = NULL;
		*lastcolon = '\0';
		port = (int)strtol(lastcolon + 1, &endp, 10);
		if (port < 1 || endp != p)
			return(ra_strdup(rtr->a, "invalid port number"));
	}
	if (*ip == '[') {
		ip++;
		if (lastcolon != NULL && *(lastcolon - 1) == ']') {
			*(lastcolon - 1) = '\0';
		} else if (lastcolon == NULL && *(p - 1) == ']') {
			*(p - 1) = '\0';
		} else {
			return(ra_strdup(rtr->a, "expected ']'"));
		}
	}

	/* try to see if this is a "numeric" IP address, in
	 * which case we take the cannonical representation so
	 * as to ensure (string) comparisons will match lateron */
	memset(&hint, 0, sizeof(hint));
	saddr = NULL;

	hint.ai_family = PF_UNSPEC;
	hint.ai_socktype = proto == CON_UDP ? SOCK_DGRAM : SOCK_STREAM;
	hint.ai_protocol = proto == CON_UDP ? IPPROTO_UDP : IPPROTO_TCP;
	hint.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
	snprintf(sport, sizeof(sport), "%u", port);

	if (getaddrinfo(ip, sport, &hint, &saddr) != 0) {
		int err;
		/* now resolve this the normal way to check validity */
		hint.ai_flags = AI_NUMERICSERV;
		if ((err = getaddrinfo(ip, sport, &hint, &saddr)) != 0) {
			char errmsg[512];
			snprintf(errmsg, sizeof(errmsg), "failed to resolve %s, port %s, "
					"proto %s: %s", ip, sport, proto == CON_UDP ? "udp" : "tcp",
					gai_strerror(err));
			return(ra_strdup(rtr->a, errmsg));
		}

		/* no ra_malloc, it gets owned by server */
		*rethint = malloc(sizeof(hint));
		if (*rethint == NULL)
			return(ra_strdup(rtr->a, "out of memory copying hint structure"));
		memcpy(*rethint, &hint, sizeof(hint));
	} else {
		/* serialise the IP address, to make sure we use cannonical form
		 * which we can then string-match lateron (to do duplicate
		 * detection) */
		*rethint = NULL;
		if (saddr->ai_family == AF_INET) {
			if (inet_ntop(saddr->ai_family,
						&((struct sockaddr_in *)saddr->ai_addr)->sin_addr,
						hnbuf, sizeof(hnbuf)) != NULL)
				ip = ra_strdup(rtr->a, hnbuf);
		} else if (saddr->ai_family == AF_INET6) {
			if (inet_ntop(saddr->ai_family,
						&((struct sockaddr_in6 *)saddr->ai_addr)->sin6_addr,
						hnbuf, sizeof(hnbuf)) != NULL)
				ip = ra_strdup(rtr->a, hnbuf);
		}
	}

	*retip = ip;<commit_msg>router_validate_address: do not tamper user input servers<commit_after>/*
 * Copyright 2013-2017 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <ctype.h>
#include <glob.h>
#include <regex.h>
#include <sys/stat.h>
#include <netdb.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>
#include <assert.h>

#include "fnv1a.h"
#include "consistent-hash.h"
#include "server.h"
#include "queue.h"
#include "aggregator.h"
#include "allocator.h"
#include "relay.h"
#include "router.h"
#include "conffile.h"
#include "conffile.tab.h"

struct _router {
	cluster *clusters;
	route *routes;
	aggregator *aggregators;
	servers *srvrs;
	struct _router_collector {
		char *stub;
		int interval;
		char *prefix;
		col_mode mode;
	} collector;
	struct _router_parser_err {
		const char *msg;
		size_t line;
		size_t start;
		size_t stop;
	} parser_err;
	struct _router_conf {
		size_t queuesize;
		size_t batchsize;
		int maxstalls;
		unsigned short iotimeout;
		unsigned int sockbufsize;
	} conf;
	allocator *a;
};

/* declarations to avoid symbol/include hell when doing config.yy.h */
int router_yylex_init(void **);
void router_yy_scan_string(char *, void *);
int router_yylex_destroy(void *);

/* catch string used for aggrgator destinations */
#define STUB_AGGR   "_aggregator_stub_"
/* catch string used for collector output */
#define STUB_STATS  "_collector_stub_"

/**
 * Frees regexes from routes.
 */
static void
router_free_intern(route *routes)
{
	/* the only thing that isn't allocated using the allocators are the
	 * regexes */
	while (routes != NULL) {
		if (routes->matchtype == REGEX)
			regfree(&routes->rule);

		if (routes->next == NULL || routes->next->dests != routes->dests) {
			while (routes->dests != NULL) {
				if (routes->dests->cl->type == GROUP ||
						routes->dests->cl->type == AGGRSTUB ||
						routes->dests->cl->type == STATSTUB)
					router_free_intern(routes->dests->cl->members.routes);
				if (routes->dests->cl->type == VALIDATION)
					router_free_intern(
							routes->dests->cl->members.validation->rule);

				routes->dests = routes->dests->next;
			}
		}

		routes = routes->next;
	}
}

/**
 * Examines pattern and sets matchtype and rule or strmatch in route.
 */
static inline int
determine_if_regex(allocator *a, route *r, char *pat)
{
	/* try and see if we can avoid using a regex match, for
	 * it is simply very slow/expensive to do so: most of
	 * the time, people don't need fancy matching rules */
	char patbuf[8192];
	char *e = pat;
	char *pb = patbuf;
	char escape = 0;
	r->matchtype = CONTAINS;
	r->nmatch = 0;

	if (*e == '^' && r->matchtype == CONTAINS) {
		e++;
		r->matchtype = STARTS_WITH;
	}
	for (; *e != '\0'; e++) {
		switch (*e) {
			case '\\':
				if (escape)
					*pb++ = *e;
				escape = !escape;
				break;
			case '.':
			case '^':
			case '*':
			case '+':
				if (!escape)
					r->matchtype = REGEX;
				*pb++ = *e;
				escape = 0;
				break;
			case '$':
				if (!escape && e[1] == '\0') {
					if (r->matchtype == STARTS_WITH) {
						r->matchtype = MATCHES;
					} else {
						r->matchtype = ENDS_WITH;
					}
				} else {
					r->matchtype = REGEX;
				}
				escape = 0;
				break;
			default:
				if (
						!escape && (
							(*e == '_') || (*e == '-') ||
							(*e >= '0' && *e <= '9') ||
							(*e >= 'a' && *e <= 'z') ||
							(*e >= 'A' && *e <= 'Z')
						)
				   )
				{
					*pb++ = *e;
				} else {
					r->matchtype = REGEX;
				}
				escape = 0;
				break;
		}
		if (pb - patbuf == sizeof(patbuf))
			r->matchtype = REGEX;
		if (r->matchtype == REGEX)
			break;
	}
	if (r->matchtype != REGEX) {
		*pb = '\0';
		r->strmatch = ra_strdup(a, patbuf);
		r->pattern = ra_strdup(a, pat);
	} else {
		int ret = regcomp(&r->rule, pat, REG_EXTENDED);
		if (ret != 0)
			return ret;  /* allow use of regerror */
		r->strmatch = NULL;
		r->pattern = ra_strdup(a, pat);
		if (r->rule.re_nsub > 0) {
			/* we need +1 because position 0 contains the entire
			 * expression */
			r->nmatch = r->rule.re_nsub + 1;
			if (r->nmatch > RE_MAX_MATCHES) {
				logerr("determine_if_regex: too many match groups, "
						"please increase RE_MAX_MATCHES in router.h\n");
				return REG_ESPACE;  /* lie closest to the truth */
			}
		}
	}

	return 0;
}

static char serverip_server[256];  /* RFC1035 = 250 */
static const char *
serverip(server *s)
{
	const char *srvr = server_ip(s);

	if (strchr(srvr, ':') != NULL) {
		snprintf(serverip_server, sizeof(serverip_server), "[%s]", srvr);
		return serverip_server;
	}

	return srvr;
}


/**
 * Callback for the bison parser for yyerror
 */
void
router_yyerror(
		void *locptr,
		void *s,
		router *r,
		allocator *ra,
		allocator *a,
		const char *msg)
{
	(void)s;
	ROUTER_YYLTYPE *locp = (ROUTER_YYLTYPE *)locptr;
	regex_t re;
	size_t nmatch = 3;
	regmatch_t pmatch[3];
	char buf1[METRIC_BUFSIZ];
	char buf2[METRIC_BUFSIZ];
	char *dummy;
	char (*sa)[METRIC_BUFSIZ];
	char (*sb)[METRIC_BUFSIZ];
	char (*st)[METRIC_BUFSIZ];

	if (r->parser_err.msg != NULL)
		return;

	/* clean up the "value" types */
	if (regcomp(&re, "cr(STRING|COMMENT|INTVAL)", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	snprintf(buf1, sizeof(buf1), "%s", msg);
	sa = &buf1;
	sb = &buf2;
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "\\_1", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	/* clean up the keywords */
	if (regcomp(&re, "cr([A-Z]+)", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "'\\_1'", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	/* clean up bison speak */
	if (regcomp(&re, "\\$end", REG_EXTENDED) != 0) {
		r->parser_err.msg = ra_strdup(a, msg);
		return;
	}
	while (regexec(&re, *sa, nmatch, pmatch, 0) == 0) {
		dummy = *sa + strlen(*sa);
		if (router_rewrite_metric(sb, &dummy,
					*sa, dummy, "end of file", nmatch, pmatch) == 0)
		{
			r->parser_err.msg = ra_strdup(a, msg);
			return;
		}
		st = sa;
		sa = sb;
		sb = st;
	}
	regfree(&re);

	r->parser_err.msg = ra_strdup(a, *sa);
	if (locp != NULL) {
		r->parser_err.line = 1 + locp->first_line;
		r->parser_err.start = locp->first_column;
		if (locp->first_line == locp->last_line) {
			r->parser_err.stop = locp->last_column;
		} else {
			r->parser_err.stop = locp->first_column;
		}
	} else {
		r->parser_err.line = 0;
	}
}

/**
 * Parse ip string and validate it.  When it is a numeric address, write
 * the cannonical version in retip.
 */
char *
router_validate_address(
		router *rtr,
		char **retip, int *retport, void **retsaddr, void **rethint,
		char *ip, serv_ctype proto)
{
	int port = 2003;
	struct addrinfo *saddr = NULL;
	struct addrinfo hint;
	char sport[8];
	char hnbuf[256];
	char *p;
	char *lastcolon = NULL;

	for (p = ip; *p != '\0'; p++)
		if (*p == ':')
			lastcolon = p;

	if (*(p - 1) == ']')
		lastcolon = NULL;
	if (lastcolon != NULL) {
		char *endp = NULL;
		*lastcolon = '\0';
		port = (int)strtol(lastcolon + 1, &endp, 10);
		if (port < 1 || endp != p)
			return(ra_strdup(rtr->a, "invalid port number"));
	}
	if (*ip == '[') {
		ip++;
		if (lastcolon != NULL && *(lastcolon - 1) == ']') {
			*(lastcolon - 1) = '\0';
		} else if (lastcolon == NULL && *(p - 1) == ']') {
			*(p - 1) = '\0';
		} else {
			return(ra_strdup(rtr->a, "expected ']'"));
		}
	}

	/* try to see if this is a "numeric" IP address, in
	 * which case we take the cannonical representation so
	 * as to ensure (string) comparisons will match lateron */
	memset(&hint, 0, sizeof(hint));
	saddr = NULL;

	hint.ai_family = PF_UNSPEC;
	hint.ai_socktype = proto == CON_UDP ? SOCK_DGRAM : SOCK_STREAM;
	hint.ai_protocol = proto == CON_UDP ? IPPROTO_UDP : IPPROTO_TCP;
	hint.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
	snprintf(sport, sizeof(sport), "%u", port);

	if (getaddrinfo(ip, sport, &hint, &saddr) != 0) {
		int err;
		/* now resolve this the normal way to check validity */
		hint.ai_flags = AI_NUMERICSERV;
		if ((err = getaddrinfo(ip, sport, &hint, &saddr)) != 0) {
			char errmsg[512];
			snprintf(errmsg, sizeof(errmsg), "failed to resolve %s, port %s, "
					"proto %s: %s", ip, sport, proto == CON_UDP ? "udp" : "tcp",
					gai_strerror(err));
			return(ra_strdup(rtr->a, errmsg));
		}

		/* no ra_malloc, it gets owned by server */
		*rethint = malloc(sizeof(hint));
		if (*rethint == NULL)
			return(ra_strdup(rtr->a, "out of memory copying hint structure"));
		memcpy(*rethint, &hint, sizeof(hint));
	}

	*retip = ip;
{'fraction_non_alphanumeric': 0.1696925933567155, 'fraction_numerical': 0.009954611099649268, 'mean_word_length': 3.1795645613278722, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 203, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2286247', 'n_tokens_mistral': 8552, 'n_tokens_neox': 7533, 'n_words': 1978}