<commit_before>/**
 * parser.c -- web log parsing
 * Copyright (C) 2009-2014 by <NAME> <<EMAIL>>
 * GoAccess - An Ncurses apache weblog analyzer & interactive viewer
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of
 * the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * A copy of the GNU General Public License is attached to this
 * source distribution for its full text.
 *
 * Visit http://goaccess.prosoftcorp.com for new releases.
 */

/*
 * "_XOPEN_SOURCE" is required for the GNU libc to export "strptime(3)"
 * correctly.
 */
#define _LARGEFILE_SOURCE
#define _LARGEFILE64_SOURCE
#define _FILE_OFFSET_BITS 64

#define _XOPEN_SOURCE 700

#include <ctype.h>
#include <errno.h>

#if HAVE_CONFIG_H
#include <config.h>
#endif

#ifdef HAVE_NCURSESW_NCURSES_H
#include <ncursesw/ncurses.h>
#elif HAVE_NCURSES_NCURSES_H
#include <ncurses/ncurses.h>
#elif HAVE_NCURSES_H
#include <ncurses.h>
#elif HAVE_CURSES_H
#include <curses.h>
#endif

#include <arpa/inet.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>

#ifdef HAVE_LIBTOKYOCABINET
#include "tcabinet.h"
#else
#include "glibht.h"
#endif

#ifdef HAVE_LIBGEOIP
#include "geolocation.h"
#endif

#include "parser.h"

#include "browsers.h"
#include "goaccess.h"
#include "error.h"
#include "opesys.h"
#include "settings.h"
#include "util.h"
#include "xmalloc.h"

/* sort data ascending */
int
cmp_data_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->data, ib->data);
}

/* sort data descending */
int
cmp_data_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->data, ia->data);
}

/* sort raw data descending */
int
cmp_raw_data_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  return strcmp ((const char *) ib->key, (const char *) ia->key);
}

/* sort numeric descending */
int
cmp_num_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (int) (ib->hits - ia->hits);
}

/* sort raw numeric descending */
int
cmp_raw_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  return (int) (*(int *) ib->value - *(int *) ia->value);
}

/* sort raw numeric descending */
int
cmp_raw_os_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GOpeSys *aos = ia->value;
  GOpeSys *bos = ib->value;
  return (int) (bos->hits - aos->hits);
}

/* sort raw numeric descending */
int
cmp_raw_browser_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GBrowser *abro = ia->value;
  GBrowser *bbro = ib->value;
  return (int) (bbro->hits - abro->hits);
}

/* sort raw numeric descending */
#ifdef HAVE_LIBGEOIP
int
cmp_raw_geo_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GLocation *aloc = ia->value;
  GLocation *bloc = ib->value;
  return (int) (bloc->hits - aloc->hits);
}
#endif

/* sort numeric ascending */
int
cmp_num_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (int) (ia->hits - ib->hits);
}

/* sort bandwidth descending */
int
cmp_bw_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ib->bw - ia->bw);
}

/* sort bandwidth ascending */
int
cmp_bw_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ia->bw - ib->bw);
}

/* sort usec descending */
int
cmp_usec_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ib->usecs - ia->usecs);
}

/* sort usec ascending */
int
cmp_usec_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ia->usecs - ib->usecs);
}

/* sort protocol ascending */
int
cmp_proto_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->protocol, ib->protocol);
}

/* sort protocol descending */
int
cmp_proto_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->protocol, ia->protocol);
}

/* sort method ascending */
int
cmp_mthd_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->method, ib->method);
}

/* sort method descending */
int
cmp_mthd_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->method, ia->method);
}

/* allocate memory for ht raw data */
GRawData *
new_grawdata (void)
{
  GRawData *raw_data = xmalloc (sizeof (GRawData));
  memset (raw_data, 0, sizeof *raw_data);

  return raw_data;
}

/* allocate memory for raw data items */
GRawDataItem *
new_grawdata_item (unsigned int size)
{
  GRawDataItem *item = xcalloc (size, sizeof (GRawDataItem));
  return item;
}

/* free memory allocated in raw data */
void
free_raw_data (GRawData * raw_data)
{
#ifdef HAVE_LIBTOKYOCABINET
  int i;
  for (i = 0; i < raw_data->size; i++) {
    if (raw_data->items[i].key != NULL)
      free (raw_data->items[i].key);
    if (raw_data->items[i].value != NULL)
      free (raw_data->items[i].value);
  }
#endif
  free (raw_data->items);
  free (raw_data);
}

void
reset_struct (GLog * logger)
{
  logger->invalid = 0;
  logger->process = 0;
  logger->resp_size = 0LL;
}

GLog *
init_log (void)
{
  GLog *glog = xmalloc (sizeof (GLog));
  memset (glog, 0, sizeof *glog);

  return glog;
}

GLogItem *
init_log_item (GLog * logger)
{
  GLogItem *glog;
  logger->items = xmalloc (sizeof (GLogItem));
  glog = logger->items;
  memset (glog, 0, sizeof *glog);

  glog->agent = NULL;
  glog->date = NULL;
  glog->host = NULL;
  glog->ref = NULL;
  glog->method = NULL;
  glog->protocol = NULL;
  glog->req = NULL;
  glog->status = NULL;
  glog->resp_size = 0LL;
  glog->serve_time = 0;

  strncpy (glog->site, "", REF_SITE_LEN);
  glog->site[REF_SITE_LEN - 1] = '\0';

  return glog;
}

static void
free_logger (GLogItem * glog)
{
  if (glog->agent != NULL)
    free (glog->agent);
  if (glog->date != NULL)
    free (glog->date);
  if (glog->host != NULL)
    free (glog->host);
  if (glog->ref != NULL)
    free (glog->ref);
  if (glog->method != NULL)
    free (glog->method);
  if (glog->protocol != NULL)
    free (glog->protocol);
  if (glog->req != NULL)
    free (glog->req);
  if (glog->status != NULL)
    free (glog->status);
  free (glog);
}

#define BASE16_TO_10(x) (((x) >= '0' && (x) <= '9') ? ((x) - '0') : \
    (toupper((x)) - 'A' + 10))

static char *
decode_url (char *url)
{
  char *out, *ptr;
  const char *c;

  if ((url == NULL) || (*url == '\0'))
    return NULL;

  out = ptr = xstrdup (url);
  for (c = url; *c; c++) {
    if (*c != '%' || !isxdigit (c[1]) || !isxdigit (c[2]))
      *ptr++ = *c;
    else {
      *ptr++ = (BASE16_TO_10 (c[1]) * 16) + (BASE16_TO_10 (c[2]));
      c += 2;
    }
  }
  *ptr = 0;
  strip_newlines (out);

  return trim_str (out);
}

/* Process keyphrases from Google search, cache, and translate.
 * Note that the referer hasn't been decoded at the entry point
 * since there could be '&' within the search query. */
static int
process_keyphrases (char *ref)
{
  char *r, *ptr, *pch, *referer;
  int encoded = 0;

  if (!(strstr (ref, "http://www.google.")) &&
      !(strstr (ref, "http://webcache.googleusercontent.com/")) &&
      !(strstr (ref, "http://translate.googleusercontent.com/")))
    return 1;

  /* webcache.googleusercontent */
  if ((r = strstr (ref, "/+&")) != NULL)
    return 1;
  /* webcache.googleusercontent */
  else if ((r = strstr (ref, "/+")) != NULL)
    r += 2;
  /* webcache.googleusercontent */
  else if ((r = strstr (ref, "q=cache:")) != NULL) {
    pch = strchr (r, '+');
    if (pch)
      r += pch - r + 1;
  }
  /* www.google.* or translate.googleusercontent */
  else if ((r = strstr (ref, "&q=")) != NULL ||
           (r = strstr (ref, "?q=")) != NULL)
    r += 3;
  else if ((r = strstr (ref, "%26q%3D")) != NULL ||
           (r = strstr (ref, "%3Fq%3D")) != NULL)
    encoded = 1, r += 7;
  else
    return 1;

  if (!encoded && (ptr = strchr (r, '&')) != NULL)
    *ptr = '\0';
  else if (encoded && (ptr = strstr (r, "%26")) != NULL)
    *ptr = '\0';

  referer = decode_url (r);
  if (referer == NULL || *referer == '\0')
    return 1;

  referer = char_replace (referer, '+', ' ');
  process_generic_data (ht_keyphrases, trim_str (referer));
  free (referer);

  return 0;
}

/* extract the host part (i.e., www.google.com) from the referer */
static int
extract_referer_site (const char *referer, char *host)
{
  return (sscanf (referer, "%*[^/]%*[/]%[^/]", host) == 1);
}

/* process referer */
static void
process_referrers (char *referrer, char *site)
{
  char *ref;

  if (referrer == NULL)
    return;

  if (site != NULL && *site != '\0')
    process_generic_data (ht_referring_sites, site);
  process_keyphrases (referrer);

  ref = decode_url (referrer);
  if (ref == NULL || *ref == '\0')
    return;

  process_generic_data (ht_referrers, ref);
  free (ref);
}

/* process data based on a unique key, this includes the following
 * modules, VISITORS, BROWSERS, OS */
static void
process_unique_data (char *host, char *date, char *agent)
{
#ifdef HAVE_LIBGEOIP
  char city[CITY_LEN] = "";
  char continent[CONTINENT_LEN] = "";<commit_msg>Fixed issue where sscanf could not handle special chars within the referer.<commit_after>/**
 * parser.c -- web log parsing
 * Copyright (C) 2009-2014 by <NAME> <<EMAIL>>
 * GoAccess - An Ncurses apache weblog analyzer & interactive viewer
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of
 * the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * A copy of the GNU General Public License is attached to this
 * source distribution for its full text.
 *
 * Visit http://goaccess.prosoftcorp.com for new releases.
 */

/*
 * "_XOPEN_SOURCE" is required for the GNU libc to export "strptime(3)"
 * correctly.
 */
#define _LARGEFILE_SOURCE
#define _LARGEFILE64_SOURCE
#define _FILE_OFFSET_BITS 64

#define _XOPEN_SOURCE 700

#include <ctype.h>
#include <errno.h>

#if HAVE_CONFIG_H
#include <config.h>
#endif

#ifdef HAVE_NCURSESW_NCURSES_H
#include <ncursesw/ncurses.h>
#elif HAVE_NCURSES_NCURSES_H
#include <ncurses/ncurses.h>
#elif HAVE_NCURSES_H
#include <ncurses.h>
#elif HAVE_CURSES_H
#include <curses.h>
#endif

#include <arpa/inet.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>

#ifdef HAVE_LIBTOKYOCABINET
#include "tcabinet.h"
#else
#include "glibht.h"
#endif

#ifdef HAVE_LIBGEOIP
#include "geolocation.h"
#endif

#include "parser.h"

#include "browsers.h"
#include "goaccess.h"
#include "error.h"
#include "opesys.h"
#include "settings.h"
#include "util.h"
#include "xmalloc.h"

/* sort data ascending */
int
cmp_data_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->data, ib->data);
}

/* sort data descending */
int
cmp_data_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->data, ia->data);
}

/* sort raw data descending */
int
cmp_raw_data_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  return strcmp ((const char *) ib->key, (const char *) ia->key);
}

/* sort numeric descending */
int
cmp_num_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (int) (ib->hits - ia->hits);
}

/* sort raw numeric descending */
int
cmp_raw_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  return (int) (*(int *) ib->value - *(int *) ia->value);
}

/* sort raw numeric descending */
int
cmp_raw_os_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GOpeSys *aos = ia->value;
  GOpeSys *bos = ib->value;
  return (int) (bos->hits - aos->hits);
}

/* sort raw numeric descending */
int
cmp_raw_browser_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GBrowser *abro = ia->value;
  GBrowser *bbro = ib->value;
  return (int) (bbro->hits - abro->hits);
}

/* sort raw numeric descending */
#ifdef HAVE_LIBGEOIP
int
cmp_raw_geo_num_desc (const void *a, const void *b)
{
  const GRawDataItem *ia = a;
  const GRawDataItem *ib = b;
  GLocation *aloc = ia->value;
  GLocation *bloc = ib->value;
  return (int) (bloc->hits - aloc->hits);
}
#endif

/* sort numeric ascending */
int
cmp_num_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (int) (ia->hits - ib->hits);
}

/* sort bandwidth descending */
int
cmp_bw_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ib->bw - ia->bw);
}

/* sort bandwidth ascending */
int
cmp_bw_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ia->bw - ib->bw);
}

/* sort usec descending */
int
cmp_usec_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ib->usecs - ia->usecs);
}

/* sort usec ascending */
int
cmp_usec_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return (unsigned long long) (ia->usecs - ib->usecs);
}

/* sort protocol ascending */
int
cmp_proto_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->protocol, ib->protocol);
}

/* sort protocol descending */
int
cmp_proto_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->protocol, ia->protocol);
}

/* sort method ascending */
int
cmp_mthd_asc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ia->method, ib->method);
}

/* sort method descending */
int
cmp_mthd_desc (const void *a, const void *b)
{
  const GHolderItem *ia = a;
  const GHolderItem *ib = b;
  return strcmp (ib->method, ia->method);
}

/* allocate memory for ht raw data */
GRawData *
new_grawdata (void)
{
  GRawData *raw_data = xmalloc (sizeof (GRawData));
  memset (raw_data, 0, sizeof *raw_data);

  return raw_data;
}

/* allocate memory for raw data items */
GRawDataItem *
new_grawdata_item (unsigned int size)
{
  GRawDataItem *item = xcalloc (size, sizeof (GRawDataItem));
  return item;
}

/* free memory allocated in raw data */
void
free_raw_data (GRawData * raw_data)
{
#ifdef HAVE_LIBTOKYOCABINET
  int i;
  for (i = 0; i < raw_data->size; i++) {
    if (raw_data->items[i].key != NULL)
      free (raw_data->items[i].key);
    if (raw_data->items[i].value != NULL)
      free (raw_data->items[i].value);
  }
#endif
  free (raw_data->items);
  free (raw_data);
}

void
reset_struct (GLog * logger)
{
  logger->invalid = 0;
  logger->process = 0;
  logger->resp_size = 0LL;
}

GLog *
init_log (void)
{
  GLog *glog = xmalloc (sizeof (GLog));
  memset (glog, 0, sizeof *glog);

  return glog;
}

GLogItem *
init_log_item (GLog * logger)
{
  GLogItem *glog;
  logger->items = xmalloc (sizeof (GLogItem));
  glog = logger->items;
  memset (glog, 0, sizeof *glog);

  glog->agent = NULL;
  glog->date = NULL;
  glog->host = NULL;
  glog->ref = NULL;
  glog->method = NULL;
  glog->protocol = NULL;
  glog->req = NULL;
  glog->status = NULL;
  glog->resp_size = 0LL;
  glog->serve_time = 0;

  strncpy (glog->site, "", REF_SITE_LEN);
  glog->site[REF_SITE_LEN - 1] = '\0';

  return glog;
}

static void
free_logger (GLogItem * glog)
{
  if (glog->agent != NULL)
    free (glog->agent);
  if (glog->date != NULL)
    free (glog->date);
  if (glog->host != NULL)
    free (glog->host);
  if (glog->ref != NULL)
    free (glog->ref);
  if (glog->method != NULL)
    free (glog->method);
  if (glog->protocol != NULL)
    free (glog->protocol);
  if (glog->req != NULL)
    free (glog->req);
  if (glog->status != NULL)
    free (glog->status);
  free (glog);
}

#define BASE16_TO_10(x) (((x) >= '0' && (x) <= '9') ? ((x) - '0') : \
    (toupper((x)) - 'A' + 10))

static char *
decode_url (char *url)
{
  char *out, *ptr;
  const char *c;

  if ((url == NULL) || (*url == '\0'))
    return NULL;

  out = ptr = xstrdup (url);
  for (c = url; *c; c++) {
    if (*c != '%' || !isxdigit (c[1]) || !isxdigit (c[2]))
      *ptr++ = *c;
    else {
      *ptr++ = (BASE16_TO_10 (c[1]) * 16) + (BASE16_TO_10 (c[2]));
      c += 2;
    }
  }
  *ptr = 0;
  strip_newlines (out);

  return trim_str (out);
}

/* Process keyphrases from Google search, cache, and translate.
 * Note that the referer hasn't been decoded at the entry point
 * since there could be '&' within the search query. */
static int
process_keyphrases (char *ref)
{
  char *r, *ptr, *pch, *referer;
  int encoded = 0;

  if (!(strstr (ref, "http://www.google.")) &&
      !(strstr (ref, "http://webcache.googleusercontent.com/")) &&
      !(strstr (ref, "http://translate.googleusercontent.com/")))
    return 1;

  /* webcache.googleusercontent */
  if ((r = strstr (ref, "/+&")) != NULL)
    return 1;
  /* webcache.googleusercontent */
  else if ((r = strstr (ref, "/+")) != NULL)
    r += 2;
  /* webcache.googleusercontent */
  else if ((r = strstr (ref, "q=cache:")) != NULL) {
    pch = strchr (r, '+');
    if (pch)
      r += pch - r + 1;
  }
  /* www.google.* or translate.googleusercontent */
  else if ((r = strstr (ref, "&q=")) != NULL ||
           (r = strstr (ref, "?q=")) != NULL)
    r += 3;
  else if ((r = strstr (ref, "%26q%3D")) != NULL ||
           (r = strstr (ref, "%3Fq%3D")) != NULL)
    encoded = 1, r += 7;
  else
    return 1;

  if (!encoded && (ptr = strchr (r, '&')) != NULL)
    *ptr = '\0';
  else if (encoded && (ptr = strstr (r, "%26")) != NULL)
    *ptr = '\0';

  referer = decode_url (r);
  if (referer == NULL || *referer == '\0')
    return 1;

  referer = char_replace (referer, '+', ' ');
  process_generic_data (ht_keyphrases, trim_str (referer));
  free (referer);

  return 0;
}

static int
extract_referer_site (const char *referer, char *host)
{
  char *url, *begin, *end;
  int len = 0;

  if ((referer == NULL) || (*referer == '\0'))
    return 1;

  url = strdup (referer);
  if ((begin = strstr (url, "//")) == NULL)
    goto clean;

  begin += 2;
  if ((len = strlen (begin)) == 0)
    goto clean;

  if ((end = strchr (begin, '/')) != NULL)
    len = end - begin;

  if (len == 0)
    goto clean;

  if (len >= REF_SITE_LEN)
    len = REF_SITE_LEN - 1;

  memcpy (host, begin, len);
  host[len] = '\0';
  free (url);
  return 0;
clean:
  free (url);

  return 1;
}

/* process referer */
static void
process_referrers (char *referrer, char *site)
{
  char *ref;

  if (referrer == NULL)
    return;

  if (site != NULL && *site != '\0')
    process_generic_data (ht_referring_sites, site);
  process_keyphrases (referrer);

  ref = decode_url (referrer);
  if (ref == NULL || *ref == '\0')
    return;

  process_generic_data (ht_referrers, ref);
  free (ref);
}

/* process data based on a unique key, this includes the following
 * modules, VISITORS, BROWSERS, OS */
static void
process_unique_data (char *host, char *date, char *agent)
{
#ifdef HAVE_LIBGEOIP
  char city[CITY_LEN] = "";
  char continent[CONTINENT_LEN] = "";
{'fraction_non_alphanumeric': 0.15891566849328354, 'fraction_numerical': 0.008001551816109792, 'mean_word_length': 3.390461997019374, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 200, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928209', 'n_tokens_mistral': 8052, 'n_tokens_neox': 7842, 'n_words': 2364}