<commit_before>#!/usr/bin/env python

from symLogging import LogDebug, LogError, LogMessage, SetLoggingOptions, SetDebug, CheckDebug
from symFileManager import SymFileManager
from symbolicationRequest import SymbolicationRequest
from concurrent.futures import ProcessPoolExecutor as Pool

import sys
import os
import time
import json
import tempfile
import ConfigParser
from collections import OrderedDict as _default_dict
import tornado.gen
from tornado.ioloop import IOLoop
from tornado.web import Application, RequestHandler, url

# Report errors while symLogging is not configured yet
import logging

# .SYM cache manager
gSymFileManager = None

# Pool of symbolication workers
gPool = None

# Default config options
gOptions = {
  # IP address to listen on
  "hostname": "0.0.0.0",
  # TCP port to listen on
  "portNumber": 80,
  # Trace-level logging (verbose)
  "enableTracing": 0,
  # Fallback server if symbol is not found locally
  "remoteSymbolServer": "",
  # Maximum number of symbol files to keep in memory
  # "maxCacheEntries": 10 * 1000 * 1000,
  "maxCacheEntries": 100,
  # File in which to persist the list of most-recently-used symbols.
  "mruSymbolStateFile": os.path.join(tempfile.gettempdir(), "snappy-mru-symbols.json"),
  # Maximum number of symbol files to persist in the state file between runs.
  "maxMRUSymbolsPersist": 10,
  # Paths to .SYM files
  "symbolPaths": [
    # Default to empty so users don't have to list anything in their config
    # file to override the defaults.
  ],
  # URLs to symbol stores
  "symbolURLs": [
  ]
}

# Use a new class to make defaults case-sensitive
class CaseSensitiveConfigParser(ConfigParser.SafeConfigParser):
  superClass = ConfigParser.SafeConfigParser
  def __init__(self, defaults=None, dict_type=_default_dict,
                allow_no_value=False):
    self.optionxform = str
    self.superClass.__init__(self, defaults, dict_type, allow_no_value)

  def items(self, section, raw=False, vars=None):
    defaults = self.defaults()
    if vars is not None:
      defaults.update(vars)

    # Remove default items from the result
    return filter(
              lambda item: item[0] not in defaults,
              self.superClass.items(self, section, raw, vars))

def initializeSubprocess(options):
  global gSymFileManager

  # Setup logging in the child process
  SetLoggingOptions(options["Log"])

  # Create the .SYM cache manager singleton
  gSymFileManager = SymFileManager(options)

  # Prefetch recent symbols
  gSymFileManager.PrefetchRecentSymbolFiles()


def processSymbolicationRequest(rawRequest, remoteIp):
  decodedRequest = json.loads(rawRequest)
  request = SymbolicationRequest(gSymFileManager, decodedRequest, remoteIp)
  if not request.isValidRequest:
    LogDebug("Unable to parse request", remoteIp)
    return None

  response = { 'symbolicatedStacks': [] }<commit_msg>Ignore control-c in the symbolication subprocess.<commit_after>#!/usr/bin/env python

from symLogging import LogDebug, LogError, LogMessage, SetLoggingOptions, SetDebug, CheckDebug
from symFileManager import SymFileManager
from symbolicationRequest import SymbolicationRequest
from concurrent.futures import ProcessPoolExecutor as Pool

import sys
import os
import time
import json
import signal
import tempfile
import ConfigParser
from collections import OrderedDict as _default_dict
import tornado.gen
from tornado.ioloop import IOLoop
from tornado.web import Application, RequestHandler, url

# Report errors while symLogging is not configured yet
import logging

# .SYM cache manager
gSymFileManager = None

# Pool of symbolication workers
gPool = None

# Default config options
gOptions = {
  # IP address to listen on
  "hostname": "0.0.0.0",
  # TCP port to listen on
  "portNumber": 80,
  # Trace-level logging (verbose)
  "enableTracing": 0,
  # Fallback server if symbol is not found locally
  "remoteSymbolServer": "",
  # Maximum number of symbol files to keep in memory
  # "maxCacheEntries": 10 * 1000 * 1000,
  "maxCacheEntries": 100,
  # File in which to persist the list of most-recently-used symbols.
  "mruSymbolStateFile": os.path.join(tempfile.gettempdir(), "snappy-mru-symbols.json"),
  # Maximum number of symbol files to persist in the state file between runs.
  "maxMRUSymbolsPersist": 10,
  # Paths to .SYM files
  "symbolPaths": [
    # Default to empty so users don't have to list anything in their config
    # file to override the defaults.
  ],
  # URLs to symbol stores
  "symbolURLs": [
  ]
}

# Use a new class to make defaults case-sensitive
class CaseSensitiveConfigParser(ConfigParser.SafeConfigParser):
  superClass = ConfigParser.SafeConfigParser
  def __init__(self, defaults=None, dict_type=_default_dict,
                allow_no_value=False):
    self.optionxform = str
    self.superClass.__init__(self, defaults, dict_type, allow_no_value)

  def items(self, section, raw=False, vars=None):
    defaults = self.defaults()
    if vars is not None:
      defaults.update(vars)

    # Remove default items from the result
    return filter(
              lambda item: item[0] not in defaults,
              self.superClass.items(self, section, raw, vars))

def initializeSubprocess(options):
  global gSymFileManager

  # Ignore ctrl-c in the subprocess
  signal.signal(signal.SIGINT, signal.SIG_IGN)

  # Setup logging in the child process
  SetLoggingOptions(options["Log"])

  # Create the .SYM cache manager singleton
  gSymFileManager = SymFileManager(options)

  # Prefetch recent symbols
  gSymFileManager.PrefetchRecentSymbolFiles()


def processSymbolicationRequest(rawRequest, remoteIp):
  decodedRequest = json.loads(rawRequest)
  request = SymbolicationRequest(gSymFileManager, decodedRequest, remoteIp)
  if not request.isValidRequest:
    LogDebug("Unable to parse request", remoteIp)
    return None

  response = { 'symbolicatedStacks': [] }
{'fraction_non_alphanumeric': 0.07639836289222374, 'fraction_numerical': 0.007844474761255117, 'mean_word_length': 4.365965233302836, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1523224', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1663, 'n_words': 568}