<commit_before>#include "module.h"
#include "config.h"
#include <regex.h>
#include <curl/curl.h>
#include "utils.h"

enum { GOOGLE, DDG, MSDN, };

static void search_cmd(const char* chan, const char* name, const char* msg, int cmd);
static bool search_init(const IRCCoreCtx* _ctx);

const IRCModuleCtx irc_mod_ctx = {
    .name       = "search",
    .desc       = "Quick web search commands",
    .flags      = IRC_MOD_DEFAULT,
    .on_cmd     = &search_cmd,
    .on_init    = &search_init,
    .commands   = DEFINE_CMDS (
        [GOOGLE] = CONTROL_CHAR"g "CONTROL_CHAR"google",
        [DDG]    = CONTROL_CHAR"ddg "CONTROL_CHAR"duckduckgo",
        [MSDN]   = CONTROL_CHAR"msdn"
    )
};

static const IRCCoreCtx* ctx;

static regex_t ddg_result_regex;

static bool search_init(const IRCCoreCtx* _ctx){
    ctx = _ctx;

    bool ret = true;
    int reg_result = regcomp(&ddg_result_regex, "result__a.{1,10}href=\"([^\"]*)\"", REG_EXTENDED | REG_ICASE);
    if (reg_result != 0) {
        ret = false;
        char errbuf[256];
        regerror(reg_result, &ddg_result_regex, errbuf, 256);
        fprintf(stderr, "Couldn't compile search regex: %s\n", errbuf);
    }
    else {
        fprintf(stderr, "Search regex compiled succesfully\n");<commit_msg>Extend search regex to avoid ad results<commit_after>#include "module.h"
#include "config.h"
#include <regex.h>
#include <curl/curl.h>
#include "utils.h"

enum { GOOGLE, DDG, MSDN, };

static void search_cmd(const char* chan, const char* name, const char* msg, int cmd);
static bool search_init(const IRCCoreCtx* _ctx);

const IRCModuleCtx irc_mod_ctx = {
    .name       = "search",
    .desc       = "Quick web search commands",
    .flags      = IRC_MOD_DEFAULT,
    .on_cmd     = &search_cmd,
    .on_init    = &search_init,
    .commands   = DEFINE_CMDS (
        [GOOGLE] = CONTROL_CHAR"g "CONTROL_CHAR"google",
        [DDG]    = CONTROL_CHAR"ddg "CONTROL_CHAR"duckduckgo",
        [MSDN]   = CONTROL_CHAR"msdn"
    )
};

static const IRCCoreCtx* ctx;

static regex_t ddg_result_regex;

static bool search_init(const IRCCoreCtx* _ctx){
    ctx = _ctx;

    bool ret = true;
    int reg_result = regcomp(&ddg_result_regex, "[^a]>\\s+<h2 class=\"result__title\">\\s+<a rel=\"nofollow\"\\s+class=\"result__a.{1,10}href=\"([^\"]*)\"", REG_EXTENDED | REG_ICASE);
    if (reg_result != 0) {
        ret = false;
        char errbuf[256];
        regerror(reg_result, &ddg_result_regex, errbuf, 256);
        fprintf(stderr, "Couldn't compile search regex: %s\n", errbuf);
    }
    else {
        fprintf(stderr, "Search regex compiled succesfully\n");
{'fraction_non_alphanumeric': 0.13995398773006135, 'fraction_numerical': 0.008052147239263804, 'mean_word_length': 3.194533762057878, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2922495', 'n_tokens_mistral': 984, 'n_tokens_neox': 951, 'n_words': 217}