<commit_before>/* Copyright (c) 2007 The Kaphan Foundation
 *
 * Possession of a copy of this file grants no permission or license
 * to use, modify, or create derivate works.
 *
 * Please contact <EMAIL> for further information.
 */

#include <config.h>
#include <stdlib.h>
#include <dirent.h>
#include <getopt.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include "git_revision.h"
#include "logging.h"
#include "classification_engine.h"
#include "httpd.h"
#include "misc.h"
#include "feature_extractor.h"
#include "fetch_url.h"

static void print_help() {
	printf("Peerworks Classifier\n\n");
	printf("This version of the classifier will classify a\n single tag file or directory of tag files.\n\n");
	printf("Usage: classify <item_cache> <tag-file-url-or-directory>\n");
}

#define SHORT_OPTS "hv:"

static ItemCacheOptions item_cache_options = {60, 1000, 50};
static ItemCache *item_cache;
static TaggerCacheOptions tagger_cache_options;
static TaggerCache *tagger_cache;
static ClassificationEngineOptions ce_options = {1, 0.0, 10, NULL};
static ClassificationEngine *engine;

static int is_url(const char *path) {
  return path == strstr(path, "file:") || path == strstr(path, "http:");
}

static int start_classifier(char * corpus) {
	if (CLASSIFIER_OK != item_cache_create(&item_cache, corpus, &item_cache_options)) {
    fprintf(stderr, "Error opening classifier database file at %s: %s\n", corpus, item_cache_errmsg(item_cache));
    free_item_cache(item_cache);
    return EXIT_FAILURE;
  } else {
    item_cache_load(item_cache);
    tagger_cache = create_tagger_cache(item_cache, &tagger_cache_options);
    tagger_cache->tag_retriever = &fetch_url;
    tagger_cache->tag_index_retriever = &fetch_url;

    engine = create_classification_engine(item_cache, tagger_cache, &ce_options);
    return !ce_start(engine);
  }
}
<commit_msg>Tweak to classify default parameters<commit_after>/* Copyright (c) 2007 The Kaphan Foundation
 *
 * Possession of a copy of this file grants no permission or license
 * to use, modify, or create derivate works.
 *
 * Please contact <EMAIL> for further information.
 */

#include <config.h>
#include <stdlib.h>
#include <dirent.h>
#include <getopt.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include "git_revision.h"
#include "logging.h"
#include "classification_engine.h"
#include "httpd.h"
#include "misc.h"
#include "feature_extractor.h"
#include "fetch_url.h"

static void print_help() {
	printf("Peerworks Classifier\n\n");
	printf("This version of the classifier will classify a\n single tag file or directory of tag files.\n\n");
	printf("Usage: classify <item_cache> <tag-file-url-or-directory>\n");
}

#define SHORT_OPTS "hv:"

static ItemCacheOptions item_cache_options = {60, 1000, 0};
static ItemCache *item_cache;
static TaggerCacheOptions tagger_cache_options;
static TaggerCache *tagger_cache;
static ClassificationEngineOptions ce_options = {4, 0.9, 10, NULL};
static ClassificationEngine *engine;

static int is_url(const char *path) {
  return path == strstr(path, "file:") || path == strstr(path, "http:");
}

static int start_classifier(char * corpus) {
	if (CLASSIFIER_OK != item_cache_create(&item_cache, corpus, &item_cache_options)) {
    fprintf(stderr, "Error opening classifier database file at %s: %s\n", corpus, item_cache_errmsg(item_cache));
    free_item_cache(item_cache);
    return EXIT_FAILURE;
  } else {
    item_cache_load(item_cache);
    tagger_cache = create_tagger_cache(item_cache, &tagger_cache_options);
    tagger_cache->tag_retriever = &fetch_url;
    tagger_cache->tag_index_retriever = &fetch_url;

    engine = create_classification_engine(item_cache, tagger_cache, &ce_options);
    return !ce_start(engine);
  }
}

{'fraction_non_alphanumeric': 0.1228437009932044, 'fraction_numerical': 0.008625196027182437, 'mean_word_length': 5.586919104991394, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5646588', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1283, 'n_words': 338}