<commit_before>// vim: noet ts=4 sw=4
#include <arpa/inet.h>
#include <netdb.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define DEBUG 0
#define SOCK_RECV_MAX 4096

const char FOURCHAN_API_HOST[] = "a.4cdn.org";
int main_sock_fd = 0;

const char API_REQUEST[] =
	"GET /b/catalog.json HTTP/1.1\r\n"
	"User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0\r\n"
	"Host: a.4cdn.org\r\n"
	"Accept text/html\r\n\r\n";

char *get_catalog(const int request_fd) {
	size_t msg_siz = SOCK_RECV_MAX;
	char *msg = malloc(SOCK_RECV_MAX);
	int num_bytes_read = 0;
	num_bytes_read = recv(request_fd, msg, SOCK_RECV_MAX, 0);

	/* 4Chan throws us data as chunk-encoded HTTP. Rad. */
	char *header_end = strstr(msg, "\r\n\r\n");
	/* This is where the data begins. */
	char *chunk_size_start = header_end + (sizeof(char) * 4);
	char *chunk_size_end = strstr(chunk_size_start, "\r\n");

	/* We cheat a little and set the first \r to a \0 so strtol will
	 * do the right thing. */
	chunk_size_start[chunk_size_end - chunk_size_start] = '\0';
	chunk_size_end = NULL;
	size_t chunk_size = strtol(chunk_size_start, NULL, 16);

	return msg;
}

int new_API_request() {
	struct addrinfo hints = {0};
	struct addrinfo *res = NULL;
	int request_fd;

	hints.ai_family = AF_INET;
	hints.ai_socktype = SOCK_STREAM;

	getaddrinfo(FOURCHAN_API_HOST, "80", &hints, &res);<commit_msg>Reads pseudo-intelligently now.<commit_after>// vim: noet ts=4 sw=4
#include <arpa/inet.h>
#include <netdb.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#define DEBUG 0
#define SOCK_RECV_MAX 4096

const char FOURCHAN_API_HOST[] = "a.4cdn.org";
int main_sock_fd = 0;

const char API_REQUEST[] =
	"GET /b/catalog.json HTTP/1.1\r\n"
	"User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0\r\n"
	"Host: a.4cdn.org\r\n"
	"Accept text/html\r\n\r\n";

char *get_catalog(const int request_fd) {
	char *msg = NULL;
	char *raw_buf = malloc(0);
	size_t buf_size = 0;
	int times_read = 0;

	fd_set chan_fds;
	FD_ZERO(&chan_fds);
	FD_SET(request_fd, &chan_fds);
	const int maxfd = request_fd;

	while (1) {
		times_read++;
		int num_bytes_read = 0;

		/* Wait for data to be read. */
		struct timeval tv = {2, 0};
		select(maxfd + 1, &chan_fds, NULL, NULL, &tv);

		int count;
		/* How many bytes should we read: */
		ioctl(request_fd, FIONREAD, &count);
		if (count <= 0)
			break;
		int old_offset = buf_size;
		buf_size += count;
		raw_buf = realloc(raw_buf, buf_size);
		printf("IOCTL: %i.\n", count);

		num_bytes_read = recv(request_fd, raw_buf + old_offset, count, 0);
	}
	printf("Full message is %s\n.", raw_buf);

	/* 4Chan throws us data as chunk-encoded HTTP. Rad. */
	char *header_end = strstr(raw_buf, "\r\n\r\n");
	/* This is where the data begins. */
	char *chunk_size_start = header_end + (sizeof(char) * 4);
	char *chunk_size_end = strstr(chunk_size_start, "\r\n");
	const int chunk_size_end_oft = chunk_size_end - chunk_size_start;

	/* We cheat a little and set the first \r to a \0 so strtol will
	 * do the right thing. */
	chunk_size_start[chunk_size_end_oft] = '\0';
	chunk_size_end = NULL;
	int chunk_size = strtol(chunk_size_start, NULL, 16);
	printf("Chunk size is %i.\n", chunk_size);

	/* So at this point we just read shit into an ever expanding buffer. */

	return msg;
}

int new_API_request() {
	struct addrinfo hints = {0};
	struct addrinfo *res = NULL;
	int request_fd;

	hints.ai_family = AF_INET;
	hints.ai_socktype = SOCK_STREAM;

	getaddrinfo(FOURCHAN_API_HOST, "80", &hints, &res);
{'fraction_non_alphanumeric': 0.14722598105548038, 'fraction_numerical': 0.02435723951285521, 'mean_word_length': 4.049180327868853, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '164087', 'n_tokens_mistral': 1618, 'n_tokens_neox': 1519, 'n_words': 388}