<commit_before>#! /usr/bin/env python

import subprocess
import os.path
import threading
import time
import re

import logging

logger = logging.getLogger("processor")

import socorro.lib.util

import processor

#==========================================================
class ProcessorWithExternalBreakpad (processor.Processor):
  """
  """
#-----------------------------------------------------------------------------------------------------------------
  def __init__(self, config):
    super(ProcessorWithExternalBreakpad, self).__init__(config)

    #preprocess the breakpad_stackwalk command line
    # convert parameters of the form "$paramterName" into a python parameter of the form "%(paramterName)s"
    self.commandLine = re.compile(r'(\$(\w+))').sub(r'%(\2)s', config.stackwalkCommandLine)
    # convert parameters of the form "$(paramterName)" into a python parameter of the form "%(paramterName)s"
    self.commandLine = re.compile(r'(\$(\(\w+\)))').sub(r'%(\2)s', self.commandLine)
    # treat the "dumpfilePathname" as a special paramter by changing its syntax
    self.commandLine = self.commandLine.replace('%(dumpfilePathname)s', "DUMPFILEPATHNAME")
    # treat the "processorSymbolsPathnameList" as a special paramter by changing its syntax
    self.commandLine = self.commandLine.replace('%(processorSymbolsPathnameList)s', "SYMBOL_PATHS")
    # finally make the substitutions to make a real command out of the template
    self.commandLine = self.commandLine %  config

#-----------------------------------------------------------------------------------------------------------------
  def invokeBreakpadStackdump(self, dumpfilePathname):
    """ This function invokes breakpad_stackdump as an external process capturing and returning
          the text output of stdout.  This version represses the stderr output.

          input parameters:
            dumpfilePathname: the complete pathname of the dumpfile to be analyzed
    """
    symbol_path = ' '.join(['"%s"' % x for x in self.config.processorSymbolsPathnameList])
    #commandline = '"%s" %s "%s" %s 2>/dev/null' % (self.config.minidump_stackwalkPathname, "-m", dumpfilePathname, symbol_path)
    newCommandLine = self.commandLine.replace("DUMPFILEPATHNAME", dumpfilePathname)
    newCommandLine = newCommandLine.replace("SYMBOL_PATHS", symbol_path)
    logger.info("%s - invoking: %s", threading.currentThread().getName(), newCommandLine)
    subprocessHandle = subprocess.Popen(newCommandLine, shell=True, stdout=subprocess.PIPE)
    return (socorro.lib.util.CachingIterator(subprocessHandle.stdout), subprocessHandle)

<commit_msg>bug fixed to allow the processorSymbolsPathnameList to be specified as a list of strings or a single string with comma delimited components.<commit_after>#! /usr/bin/env python

import subprocess
import os.path
import threading
import time
import re

import logging

logger = logging.getLogger("processor")

import socorro.lib.util

import processor

#==========================================================
class ProcessorWithExternalBreakpad (processor.Processor):
  """
  """
#-----------------------------------------------------------------------------------------------------------------
  def __init__(self, config):
    super(ProcessorWithExternalBreakpad, self).__init__(config)

    #preprocess the breakpad_stackwalk command line
    # convert parameters of the form "$paramterName" into a python parameter of the form "%(paramterName)s"
    self.commandLine = re.compile(r'(\$(\w+))').sub(r'%(\2)s', config.stackwalkCommandLine)
    # convert parameters of the form "$(paramterName)" into a python parameter of the form "%(paramterName)s"
    self.commandLine = re.compile(r'(\$(\(\w+\)))').sub(r'%(\2)s', self.commandLine)
    # treat the "dumpfilePathname" as a special paramter by changing its syntax
    self.commandLine = self.commandLine.replace('%(dumpfilePathname)s', "DUMPFILEPATHNAME")
    # treat the "processorSymbolsPathnameList" as a special paramter by changing its syntax
    self.commandLine = self.commandLine.replace('%(processorSymbolsPathnameList)s', "SYMBOL_PATHS")
    # finally make the substitutions to make a real command out of the template
    self.commandLine = self.commandLine %  config

#-----------------------------------------------------------------------------------------------------------------
  def invokeBreakpadStackdump(self, dumpfilePathname):
    """ This function invokes breakpad_stackdump as an external process capturing and returning
          the text output of stdout.  This version represses the stderr output.

          input parameters:
            dumpfilePathname: the complete pathname of the dumpfile to be analyzed
    """
    if type(self.config.processorSymbolsPathnameList) is list:
      symbol_path = ' '.join(['"%s"' % x for x in self.config.processorSymbolsPathnameList])
    else:
      symbol_path = ' '.join(['"%s"' % x for x in self.config.processorSymbolsPathnameList.split(' ')])
    #commandline = '"%s" %s "%s" %s 2>/dev/null' % (self.config.minidump_stackwalkPathname, "-m", dumpfilePathname, symbol_path)
    newCommandLine = self.commandLine.replace("DUMPFILEPATHNAME", dumpfilePathname)
    newCommandLine = newCommandLine.replace("SYMBOL_PATHS", symbol_path)
    logger.info("%s - invoking: %s", threading.currentThread().getName(), newCommandLine)
    subprocessHandle = subprocess.Popen(newCommandLine, shell=True, stdout=subprocess.PIPE)
    return (socorro.lib.util.CachingIterator(subprocessHandle.stdout), subprocessHandle)


{'fraction_non_alphanumeric': 0.2018018018018018, 'fraction_numerical': 0.001081081081081081, 'mean_word_length': 3.202119606358819, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5885524', 'n_tokens_mistral': 1483, 'n_tokens_neox': 1443, 'n_words': 401}