<commit_before>#include <string.h>

#include <bsd/string.h>
#include <bsd/sys/queue.h>

#include "common.h"
#include "sdn_sensor.h"

#include "dns.h"
#include "mappings.h" // NOTE: comes from spcdns

/*
 * Ethernet frame extractor function
 * Match raw traffic against pcap_chain
 * Relay matches to appropriate nm_queue
 */
int ss_extract_eth(ss_frame_t* fbuf) {
    ss_pcap_entry_t* pptr;
    ss_pcap_entry_t* ptmp;
    int rv;
    uint8_t* metadata;
    int mlength;
    ss_pcap_match_t match;
    
    rv = ss_pcap_match_prepare(&match, rte_pktmbuf_mtod(fbuf->mbuf, uint8_t*), rte_pktmbuf_pkt_len(fbuf->mbuf));
    if (rv) {
        RTE_LOG(ERR, SS, "pcap match prepare, rv %d\n", rv);
        goto error_out;
    }
    
    TAILQ_FOREACH_SAFE(pptr, &ss_conf->pcap_chain.pcap_list, entry, ptmp) {
        RTE_LOG(DEBUG, SS, "attempt match port %u frame direction %d against pcap rule %s\n",
            fbuf->data.port_id, fbuf->data.direction, pptr->name);
        rv = ss_pcap_match(pptr, &match);
        if (rv > 0) {
            // match
            RTE_LOG(INFO, SS, "successful match against pcap rule %s\n", pptr->name);
            metadata = ss_nn_queue_prepare_metadata("ethernet", &pptr->nn_queue, fbuf);
            // XXX: for now assume the output is C char*
            mlength = strlen((char*) metadata);
            //printf("metadata: %s\n", metadata);
            rv = ss_nn_queue_send(&pptr->nn_queue, metadata, mlength);
        }
        else if (rv == 0) {
            // no match
            RTE_LOG(DEBUG, SS, "failed match against pcap rule %s\n", pptr->name);
        }
        else {
            // error
            RTE_LOG(ERR, SS, "pcap match returned error %d\n", rv);
        }
    }
    
    return 0;
    
    error_out:
    RTE_LOG(ERR, SS, "match error port %u frame direction %d\n", fbuf->data.port_id, fbuf->data.direction);
    return -1;
}

int ss_extract_dns(ss_frame_t* fbuf) {
    ss_dns_entry_t* pptr;
    ss_dns_entry_t* ptmp;
    int rv;
    uint8_t* metadata;
    int mlength;
    
    dns_decoded_t   dns_info[DNS_DECODEBUF_4K];
    dns_query_t*    dns_query;
    dns_question_t* dns_question;
    enum dns_rcode  dns_rv;
    size_t          dns_info_size = sizeof(dns_info);
    
    RTE_LOG(INFO, SS, "decode udp dns packet\n");
    dns_rv = dns_decode(dns_info, &dns_info_size, (dns_packet_t *) ((uint8_t*) fbuf->udp + sizeof(udp_hdr_t)), fbuf->data.l4_length);
    if (dns_rv != RCODE_OKAY) {
        RTE_LOG(ERR, SS, "could not decode udp dns packet\n");
        rte_pktmbuf_dump(stderr, fbuf->mbuf, rte_pktmbuf_pkt_len(fbuf->mbuf));
        return -1;
    }
    dns_query    = (dns_query_t*) dns_info;
    dns_question = &dns_query->questions[0];
    if (dns_question == NULL) {
        RTE_LOG(ERR, SS, "dns question missing in query\n");
        return -1;
    }
    RTE_LOG(INFO, SS, "rx dns query for name [%s] type [%s] class [%s]\n",
        dns_question->name, dns_type_text(dns_question->type), dns_class_text(dns_question->class));
    
    strlcpy((char*) &fbuf->data.dns_name, dns_question->name, SS_DNS_NAME_MAX);
    
    TAILQ_FOREACH_SAFE(pptr, &ss_conf->dns_chain.dns_list, entry, ptmp) {
        RTE_LOG(INFO, SS, "successful match against dns rule %s\n", pptr->name);
        metadata = ss_nn_queue_prepare_metadata("dns", &pptr->nn_queue, fbuf);
        // XXX: for now assume the output is C char*
        mlength = strlen((char*) metadata);
        //printf("metadata: %s\n", metadata);
        rv = ss_nn_queue_send(&pptr->nn_queue, metadata, mlength);
    }
    
    return 0;
}<commit_msg>extractor.c: check ioc match tables, dns rules, dns answers<commit_after>#include <string.h>

#include <bsd/string.h>
#include <bsd/sys/queue.h>

#include <rte_memory.h>

#include "extractor.h"

#include "common.h"
#include "ioc.h"
#include "metadata.h"
#include "sdn_sensor.h"

// NOTE: this stuff comes from spcdns
#include "dns.h"
#include "mappings.h"

/*
 * Ethernet frame extractor function
 * Match raw traffic against pcap_chain
 * Relay matches to appropriate nm_queue
 */
int ss_extract_eth(ss_frame_t* fbuf) {
    ss_pcap_entry_t* pptr;
    ss_pcap_entry_t* ptmp;
    ss_ioc_entry_t* iptr;
    int rv;
    uint8_t* metadata;
    int mlength;
    ss_pcap_match_t match;
    
    rv = ss_pcap_match_prepare(&match, rte_pktmbuf_mtod(fbuf->mbuf, uint8_t*), rte_pktmbuf_pkt_len(fbuf->mbuf));
    if (rv) {
        RTE_LOG(ERR, SS, "pcap match prepare, rv %d\n", rv);
        goto error_out;
    }
    
    TAILQ_FOREACH_SAFE(pptr, &ss_conf->pcap_chain.pcap_list, entry, ptmp) {
        RTE_LOG(DEBUG, SS, "attempt match port %u frame direction %d against pcap rule %s\n",
            fbuf->data.port_id, fbuf->data.direction, pptr->name);
        rv = ss_pcap_match(pptr, &match);
        if (rv > 0) {
            // match
            RTE_LOG(INFO, SS, "successful match against pcap rule %s\n", pptr->name);
            metadata = ss_nn_queue_prepare_metadata("pcap", &pptr->nn_queue, fbuf, NULL);
            // XXX: for now assume the output is C char*
            mlength = strlen((char*) metadata);
            //printf("metadata: %s\n", metadata);
            rv = ss_nn_queue_send(&pptr->nn_queue, metadata, mlength);
        }
        else if (rv == 0) {
            // no match
            RTE_LOG(DEBUG, SS, "failed match against pcap rule %s\n", pptr->name);
        }
        else {
            // error
            RTE_LOG(ERR, SS, "pcap match returned error %d\n", rv);
        }
    }
    
    iptr = ss_ioc_metadata_match(&fbuf->data);
    if (iptr) {
        // match
        RTE_LOG(INFO, SS, "successful match against IOC id %lu\n", iptr->id);
        nn_queue_t* nn_queue = &ss_conf->ioc_files[iptr->file_id].nn_queue;
        metadata = ss_nn_queue_prepare_metadata("ioc_tables", nn_queue, fbuf, iptr);
        // XXX: for now assume the output is C char*
        mlength = strlen((char*) metadata);
        //printf("metadata: %s\n", metadata);
        rv = ss_nn_queue_send(nn_queue, metadata, mlength);
    }
    
    return 0;
    
    error_out:
    RTE_LOG(ERR, SS, "match error port %u frame direction %d\n", fbuf->data.port_id, fbuf->data.direction);
    return -1;
}

int ss_extract_dns(ss_frame_t* fbuf) {
    ss_dns_entry_t* pptr;
    ss_dns_entry_t* ptmp;
    int rv;
    uint8_t* metadata;
    int mlength;
    
    dns_decoded_t   dns_info[DNS_DECODEBUF_4K];
    dns_query_t*    dns_query;
    dns_question_t* dns_question;
    dns_answer_t*   dns_answer;
    ss_answer_t*    ss_answer;
    enum dns_rcode  dns_rv;
    size_t          dns_info_size = sizeof(dns_info);
    
    RTE_LOG(INFO, SS, "decode udp dns packet\n");
    dns_rv = dns_decode(dns_info, &dns_info_size, (dns_packet_t *) ((uint8_t*) fbuf->udp + sizeof(udp_hdr_t)), fbuf->data.l4_length);
    if (dns_rv != RCODE_OKAY) {
        RTE_LOG(ERR, SS, "could not decode udp dns packet\n");
        rte_pktmbuf_dump(stderr, fbuf->mbuf, rte_pktmbuf_pkt_len(fbuf->mbuf));
        return -1;
    }
    dns_query    = (dns_query_t*) dns_info;
    dns_question = &dns_query->questions[0];
    if (dns_question == NULL) {
        RTE_LOG(ERR, SS, "dns question missing in query\n");
        return -1;
    }
    
    RTE_LOG(INFO, SS, "rx dns query for name [%s] type [%s] class [%s]\n",
        dns_question->name, dns_type_text(dns_question->type), dns_class_text(dns_question->class));
    strlcpy((char*) &fbuf->data.dns_name, dns_question->name, SS_DNS_NAME_MAX);
    size_t ancount = dns_query->ancount;
    if (ancount > SS_DNS_RESULT_MAX) ancount = SS_DNS_RESULT_MAX;
    for (size_t i = 0; i < ancount; ++i) {
        dns_answer = &dns_query->answers[i];
        rv = ss_extract_dns_atype(&fbuf->data.dns_answers[i], dns_answer);
        if (rv) {
            RTE_LOG(ERR, SS, "rx dns query decode failure for name [%s] answer index [%zd]\n",
                dns_question->name, i);
        }
    }
    
    TAILQ_FOREACH_SAFE(pptr, &ss_conf->dns_chain.dns_list, entry, ptmp) {
        int is_match = 0;
        if (pptr->dns[0] && !strcasecmp(dns_question->name, pptr->dns)) {
            is_match = 1; goto done;
        }
        for (int i = 0; i < SS_DNS_RESULT_MAX; ++i) {
            ss_answer = &fbuf->data.dns_answers[i];
            switch (ss_answer->type) {
                case SS_TYPE_NAME: {
                    if (pptr->dns[0] && !strcasecmp((char*) ss_answer->payload, pptr->dns)) {
                        is_match = 1; goto done;
                    }
                }
                case SS_TYPE_IP: {
                    if (pptr->ip.family && !memcmp(ss_answer->payload, &pptr->ip, sizeof(ip_addr_t))) {
                        is_match = 1; goto done;
                    }
                }
                default: {
                    // fprintf(stderr, "unknown ss_answer type %d\n", dns_answer->type);
                    goto done;
                }
            }
        }
        done:
        if (!is_match) continue;
        RTE_LOG(INFO, SS, "successful match against dns rule %s\n", pptr->name);
        metadata = ss_nn_queue_prepare_metadata("dns", &pptr->nn_queue, fbuf, NULL);
        // XXX: for now assume the output is C string
        mlength = strlen((char*) metadata);
        //printf("metadata: %s\n", metadata);
        rv = ss_nn_queue_send(&pptr->nn_queue, metadata, mlength);
    }
    
    return 0;
}

int ss_extract_dns_atype(ss_answer_t* result, dns_answer_t* aptr) {
    char*        name  = (char*) result->payload;
    ip_addr_t*   ip    = (ip_addr_t*) result->payload;
    
    result->type = SS_TYPE_EMPTY;
    
    switch(aptr->generic.type) {
        case RR_NS: {
            result->type = SS_TYPE_NAME;
            strlcpy(name, aptr->ns.nsdname, sizeof(result->payload));
            break;
        }
        case RR_A: {
            result->type = SS_TYPE_IP;
            ip->family   = SS_AF_INET4;
            ip->prefix   = 32;
            ip->ip4_addr.addr = aptr->a.address;
            break;
        }
        case RR_AAAA: {
            result->type = SS_TYPE_IP;
            ip->family   = SS_AF_INET6;
            ip->prefix   = 128;
            rte_memcpy(ip->ip6_addr.addr, &aptr->aaaa.address, sizeof(ip->ip6_addr.addr));
            break;
        }
        case RR_CNAME: {
            result->type = SS_TYPE_NAME;
            strlcpy(name, aptr->cname.cname, sizeof(result->payload));
            break;
        }
        case RR_MX: {
            result->type = SS_TYPE_NAME;
            strlcpy(name, aptr->mx.exchange, sizeof(result->payload));
            break;
        }
        case RR_PTR: {
            result->type = SS_TYPE_NAME;
            strlcpy(name, aptr->ptr.ptr, sizeof(result->payload));
            break;
        }
        default: {
            fprintf(stderr, "unknown dns answer type %d\n", aptr->generic.type);
            memset(result, 0, sizeof(ss_answer_t));
            break;
        }
    }
    
    error_out:
    return -1;
}
{'fraction_non_alphanumeric': 0.12967877734053262, 'fraction_numerical': 0.0043927885055367434, 'mean_word_length': 2.256257449344458, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4690623', 'n_tokens_mistral': 4062, 'n_tokens_neox': 4025, 'n_words': 833}