<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The main engine, the backbone that glues plaso together.

This can currently be looked as an alpha stage.

This file contains the main engine used by plaso or the main glue
that holds everything in one place.
"""
import logging
import multiprocessing
import os
import pdb
import sys
import time
import traceback

import pytz
import pyvshadow

from plaso import preprocessors
from plaso import registry as reg_plugins

from plaso.lib import collector
from plaso.lib import errors
from plaso.lib import preprocess
from plaso.lib import queue
from plaso.lib import sleuthkit
from plaso.lib import storage
from plaso.lib import putils
from plaso.lib import vss
from plaso.lib import worker


__version__ = '1.0alpha'


def GetTimeZoneList():
  """Generates a list of all supported time zones."""
  yield 'local'
  for zone in pytz.all_timezones:
    yield zone


class Engine(object):
  """The main engine of plaso, the one that rules them all."""

  def __init__(self, config, event_filter=None):
    """Initialize the Plaso engine.

    Initializes some variables used in the tool as well
    as going over the configuration and veryfing each one.

    Args:
      config: A configuration object, either an optparse or an argparse one.
      event_filter: A filter object used to filter the output.

    Raises:
      errors.BadConfigOption: If the configuration options are wrong.
    """

    # TODO: Do something with the event_filter, that is
    # build an event filter that can be used to actually filter
    # out events and pass them to the needed modules for
    # additional processing.
    self.event_filter = event_filter

    self.worker_threads = []
    self.config = config

    # Do some initial verification here.
    if not os.path.isfile(config.filename) and not os.path.isdir(
        config.filename) and not (
            os.path.exists(config.filename) and config.image):
      raise errors.BadConfigOption(
          'File [%s] does not exist.' % config.filename)

    if os.path.isfile(config.output):
      logging.warning('Appending to an already existing file.')

    dirname = os.path.dirname(config.output)
    if not dirname:
      dirname = '.'

    if not os.access(dirname, os.W_OK):
      raise errors.BadConfigOption(
          'Unable to write to location: %s' % config.output)

    self.config.zone = pytz.timezone(config.tzone)

    if not hasattr(self.config, 'vss_stores'):
      self.config.vss_stores = None

    if self.config.image:
      self.config.preprocess = True

  def _GuessOS(self, col_obj):
    """Return a guess for the OS we are pre-processing."""
    try:
      if col_obj.FindPath('/(Windows|WINNT)/System32'):
        return 'Windows'
    except errors.PathNotFound:
      pass

    try:
      if col_obj.FindPath('/System/Library'):
        return 'OSX'
    except errors.PathNotFound:
      pass

    try:
      if col_obj.FindPath('/etc'):
        return 'Linux'
    except errors.PathNotFound:
      pass

    return 'None'

  def _PreProcess(self, pre_obj):
    """Run the preprocessors."""

    logging.info('Starting to collect pre-processing information.')
    logging.info('Filename: %s', self.config.filename)

    if self.config.image:
      ofs = self.config.image_offset_bytes or self.config.image_offset * 512
      pre_collector = preprocess.TSKFileCollector(
          pre_obj, self.config.filename, ofs)
    elif self.config.recursive:
      pre_collector = preprocess.FileSystemCollector(
          pre_obj, self.config.filename)
    else:
      return

    if not hasattr(self.config, 'os'):
      self.config.os = self._GuessOS(pre_collector)

    plugin_list = preprocessors.PreProcessList(pre_obj, pre_collector)

    for weight in plugin_list.GetWeightList(self.config.os):
      for plugin in plugin_list.GetWeight(self.config.os, weight):
        plugin.Run()

    # Set the timezone.
    if hasattr(pre_obj, 'time_zone_str'):
      logging.info(u'Timezone set to: %s', pre_obj.time_zone_str)
      try:<commit_msg>Code review: 7134048: These files are missing unit tes<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The main engine, the backbone that glues plaso together.

This can currently be looked as an alpha stage.

This file contains the main engine used by plaso or the main glue
that holds everything in one place.
"""
import logging
import multiprocessing
import os
import pdb
import sys
import time
import traceback

import pytz
import pyvshadow

from plaso import preprocessors
from plaso import registry as reg_plugins

from plaso.lib import collector
from plaso.lib import errors
from plaso.lib import preprocess
from plaso.lib import queue
from plaso.lib import sleuthkit
from plaso.lib import storage
from plaso.lib import putils
from plaso.lib import vss
from plaso.lib import worker


__version__ = '1.0alpha'


def GetTimeZoneList():
  """Generates a list of all supported time zones."""
  yield 'local'
  for zone in pytz.all_timezones:
    yield zone


class Engine(object):
  """The main engine of plaso, the one that rules them all."""

  def __init__(self, config, event_filter=None):
    """Initialize the Plaso engine.

    Initializes some variables used in the tool as well
    as going over the configuration and veryfing each one.

    Args:
      config: A configuration object, either an optparse or an argparse one.
      event_filter: A filter object used to filter the output.

    Raises:
      errors.BadConfigOption: If the configuration options are wrong.
    """

    # TODO: Do something with the event_filter, that is
    # build an event filter that can be used to actually filter
    # out events and pass them to the needed modules for
    # additional processing.
    self.event_filter = event_filter

    self.worker_threads = []
    self.config = config

    # Do some initial verification here.
    if not os.path.isfile(config.filename) and not os.path.isdir(
        config.filename) and not (
            os.path.exists(config.filename) and config.image):
      raise errors.BadConfigOption(
          'File [%s] does not exist.' % config.filename)

    if os.path.isfile(config.output):
      logging.warning('Appending to an already existing file.')

    dirname = os.path.dirname(config.output)
    if not dirname:
      dirname = '.'

    if not os.access(dirname, os.W_OK):
      raise errors.BadConfigOption(
          'Unable to write to location: %s' % config.output)

    self.config.zone = pytz.timezone(config.tzone)

    if not hasattr(self.config, 'vss_stores'):
      self.config.vss_stores = None

    if self.config.image:
      self.config.preprocess = True

  def _GuessOS(self, col_obj):
    """Return a guess for the OS we are pre-processing."""
    try:
      if col_obj.FindPath('/(Windows|WINNT)/System32'):
        return 'Windows'
    except errors.PathNotFound:
      pass

    try:
      if col_obj.FindPath('/System/Library'):
        return 'OSX'
    except errors.PathNotFound:
      pass

    try:
      if col_obj.FindPath('/etc'):
        return 'Linux'
    except errors.PathNotFound:
      pass

    return 'None'

  def _PreProcess(self, pre_obj):
    """Run the preprocessors."""

    logging.info('Starting to collect pre-processing information.')
    logging.info('Filename: %s', self.config.filename)

    if self.config.image:
      ofs = self.config.image_offset_bytes or self.config.image_offset * 512
      pre_collector = preprocess.TSKFileCollector(
          pre_obj, self.config.filename, ofs)
    elif self.config.recursive:
      pre_collector = preprocess.FileSystemCollector(
          pre_obj, self.config.filename)
    else:
      return

    if not hasattr(self.config, 'os'):
      self.config.os = self._GuessOS(pre_collector)

    plugin_list = preprocessors.PreProcessList(pre_obj, pre_collector)

    for weight in plugin_list.GetWeightList(self.config.os):
      for plugin in plugin_list.GetWeight(self.config.os, weight):
        try:
          plugin.Run()
        except errors.PreProcessFail as e:
          logging.warning('Unable to run preprocessor: %s, reason: %s',
                          plugin.__class__.__name__, e)

    # Set the timezone.
    if hasattr(pre_obj, 'time_zone_str'):
      logging.info(u'Timezone set to: %s', pre_obj.time_zone_str)
      try:
{'fraction_non_alphanumeric': 0.08570229728312179, 'fraction_numerical': 0.004091052134690024, 'mean_word_length': 3.150631258162821, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5471291', 'n_tokens_mistral': 2858, 'n_tokens_neox': 2732, 'n_words': 981}