<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This is a file collector for Plaso, finding files needed to be parsed.

The classes here are used to fill a queue of PathSpec protobufs that need
to be further processed by the tool.
"""
import hashlib
import logging
import os

import pytsk3
import pyvshadow

from plaso.lib import errors
from plaso.lib import event
from plaso.lib import pfile
from plaso.lib import queue
from plaso.lib import utils
from plaso.lib import vss


class PCollector(object):
  """The Plaso Collector that scans for files needed to be processed."""

  def __init__(self, proc_queue):
    """Initialize the Plaso Collector.

    Args:
      proc_queue: A Plaso queue like object to store PathSpec protobufs in.
    """
    self._queue = proc_queue
    self._fscache = pfile.FilesystemCache()
    self.parse_vss = False

  def CollectFromDir(self, directory):
    """Recursive traversal of a directory.

    Args:
      directory: The path to the directory to be parsed.
      to determine if they contain other files, read files within compressed.
    """
    for root, _, files in os.walk(directory):
      for filename in files:
        try:
          path = os.path.join(root, filename)
          if os.path.isfile(path):
            self.ProcessFile(path, self._queue)
        except IOError as e:
          logging.warning('Unable to further process %s:%s', filename, e)

  def CollectFromVss(self, image_path, store_nr, offset=0):
    """Collect files from a VSS image.

    Args:
      image_path: The path to the TSK image in the filesystem.
      store_nr: The store number for the VSS store.
      offset: An offset into the disk image to where the volume
      lies (sector size, not byte).
    """
    logging.debug('Collecting from VSS store %d', store_nr)

    try:
      fs = self._fscache.Open(image_path, offset, store_nr)
      self.ParseImageDir(fs, fs.fs.info.root_inum, '/')
    except errors.UnableToOpenFilesystem as e:
      logging.error('Unable to read filesystem: %s.', e)

    logging.debug('Collection from VSS store: %d COMPLETED.', store_nr)

  def CollectFromImage(self, image, offset=0):
    """Start the collector as an image collector.

    This collector goes over an image using pytsk.

    Args:
      image: The path to the image that needs to be opened.
      offset: An integer, a byte offset into the partition.
    """

    logging.debug(u'Collecting from an image file [%s]', image)

    # Check if we will in the future collect from VSS
    if self.parse_vss:
      self._hashlist = {}

    try:
      fs = self._fscache.Open(image, offset)
      # read the root dir, and move from there
      self.ParseImageDir(fs, fs.fs.info.root_inum, os.path.sep)
    except errors.UnableToOpenFilesystem as e:
      logging.error('Unable to read image [no collection] - %s.', e)

  def ParseImageDir(self, fs, cur_inode, path, retry=False):
    """A recursive traversal of a directory inside an image file.

    Recursive traversal through a directory that injects every file
    that is detected into the processing queue.

    Args:
      fs: A CacheFileSystem object that contains information about the
      filesystem being used to collect from.
      cur_inode: The inode number in which the directory begins.
      path: The full path name of the directory (string).
      retry: Boolean indicating this is the second attempt at parsing
      a directory.
    """
    try:
      directory = fs.fs.open_dir(inode=cur_inode)
    except IOError:
      logging.error(u'IOError while trying to open a directory: %s [%d]',
                    path, cur_inode)
      if not retry:
        logging.info(u'Reopening directory due to an IOError: %s', path)
        self.ParseImageDir(fs, cur_inode, path, True)
      return

    directories = []
    for f in directory:
      try:
        name = f.info.name.name
        if not f.info.meta:
          continue
        inode_addr = f.info.meta.addr
        f_type = f.info.meta.type
      except AttributeError as e:
        logging.error('[ParseImage] Problem reading file [%s], error: %s',
                      name, e)
        continue

      # List of files we do not want to process further.
      if name in ['$OrphanFiles', '.', '..']:
        continue

      # We only care about regular files and directories
      if f_type == pytsk3.TSK_FS_META_TYPE_DIR:
        directories.append((inode_addr, os.path.join(path, name)))
      elif f_type == pytsk3.TSK_FS_META_TYPE_REG:
        transfer_proto = event.EventPathSpec()
        if fs.store_nr >= 0:
          transfer_proto.type = 'VSS'
          transfer_proto.vss_store_number = fs.store_nr
        else:
          transfer_proto.type = 'TSK'
        transfer_proto.container_path = fs.path
        transfer_proto.image_offset = fs.offset
        transfer_proto.image_inode = inode_addr
        file_path = os.path.join(path, name)
        transfer_proto.file_path = utils.GetUnicodeString(file_path)

        # If we are dealing with a VSS we want to calculate a hash
        # value based on available timestamps and compare that to previously
        # calculated hash values, and only include the file into the queue if
        # the hash does not match.
        if self.parse_vss:
          hash_value = self.CalculateNTFSTimeHash(f.info.meta)

          if inode_addr in self._hashlist:
            if hash_value in self._hashlist[inode_addr]:
              continue<commit_msg>Code review: 7276044: .<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright 2012 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This is a file collector for Plaso, finding files needed to be parsed.

The classes here are used to fill a queue of PathSpec protobufs that need
to be further processed by the tool.
"""
import hashlib
import logging
import os

import pytsk3
import pyvshadow

from plaso.lib import errors
from plaso.lib import event
from plaso.lib import pfile
from plaso.lib import queue
from plaso.lib import utils
from plaso.lib import vss


class PCollector(object):
  """The Plaso Collector that scans for files needed to be processed."""

  def __init__(self, proc_queue):
    """Initialize the Plaso Collector.

    Args:
      proc_queue: A Plaso queue like object to store PathSpec protobufs in.
    """
    self._queue = proc_queue
    self._fscache = pfile.FilesystemCache()
    self.parse_vss = False

  def CollectFromDir(self, directory):
    """Recursive traversal of a directory.

    Args:
      directory: The path to the directory to be parsed.
      to determine if they contain other files, read files within compressed.
    """
    for root, _, files in os.walk(directory):
      for filename in files:
        try:
          path = os.path.join(root, filename)
          if os.path.isfile(path):
            self.ProcessFile(path, self._queue)
        except IOError as e:
          logging.warning('Unable to further process %s:%s', filename, e)

  def CollectFromVss(self, image_path, store_nr, offset=0):
    """Collect files from a VSS image.

    Args:
      image_path: The path to the TSK image in the filesystem.
      store_nr: The store number for the VSS store.
      offset: An offset into the disk image to where the volume
      lies (sector size, not byte).
    """
    logging.debug('Collecting from VSS store %d', store_nr)

    try:
      fs = self._fscache.Open(image_path, offset, store_nr)
      self.ParseImageDir(fs, fs.fs.info.root_inum, '/')
    except errors.UnableToOpenFilesystem as e:
      logging.error('Unable to read filesystem: %s.', e)

    logging.debug('Collection from VSS store: %d COMPLETED.', store_nr)

  def CollectFromImage(self, image, offset=0):
    """Start the collector as an image collector.

    This collector goes over an image using pytsk.

    Args:
      image: The path to the image that needs to be opened.
      offset: An integer, a byte offset into the partition.
    """

    logging.debug(u'Collecting from an image file [%s]', image)

    # Check if we will in the future collect from VSS
    if self.parse_vss:
      self._hashlist = {}

    try:
      fs = self._fscache.Open(image, offset)
      # read the root dir, and move from there
      self.ParseImageDir(fs, fs.fs.info.root_inum, os.path.sep)
    except errors.UnableToOpenFilesystem as e:
      logging.error('Unable to read image [no collection] - %s.', e)

  def ParseImageDir(self, fs, cur_inode, path, retry=False):
    """A recursive traversal of a directory inside an image file.

    Recursive traversal through a directory that injects every file
    that is detected into the processing queue.

    Args:
      fs: A CacheFileSystem object that contains information about the
      filesystem being used to collect from.
      cur_inode: The inode number in which the directory begins.
      path: The full path name of the directory (string).
      retry: Boolean indicating this is the second attempt at parsing
      a directory.
    """
    try:
      directory = fs.fs.open_dir(inode=cur_inode)
    except IOError:
      logging.error(u'IOError while trying to open a directory: %s [%d]',
                    path, cur_inode)
      if not retry:
        logging.info(u'Reopening directory due to an IOError: %s', path)
        self.ParseImageDir(fs, cur_inode, path, True)
      return

    directories = []
    for f in directory:
      try:
        tsk_name = f.info.name
        if not tsk_name:
          continue
        # TODO: Add a test that tests the reason for this. Why is this?
        # If an inode/file gets reallocated yet is still listed inside
        # a directory node there may be issues (reparsing a file, ending
        # up in an endless loop, etc.).
        if tsk_name.flags == pytsk3.TSK_FS_NAME_FLAG_UNALLOC:
          # TODO: Perhaps add a direct parsing of timestamps here to include
          # in the timeline.
          continue
        if not f.info.meta:
          continue
        name_str = tsk_name.name
        inode_addr = f.info.meta.addr
        f_type = f.info.meta.type
      except AttributeError as e:
        logging.error('[ParseImage] Problem reading file [%s], error: %s',
                      name_str, e)
        continue

      # List of files we do not want to process further.
      if name_str in ['$OrphanFiles', '.', '..']:
        continue

      # We only care about regular files and directories
      if f_type == pytsk3.TSK_FS_META_TYPE_DIR:
        directories.append((inode_addr, os.path.join(path, name_str)))
      elif f_type == pytsk3.TSK_FS_META_TYPE_REG:
        transfer_proto = event.EventPathSpec()
        if fs.store_nr >= 0:
          transfer_proto.type = 'VSS'
          transfer_proto.vss_store_number = fs.store_nr
        else:
          transfer_proto.type = 'TSK'
        transfer_proto.container_path = fs.path
        transfer_proto.image_offset = fs.offset
        transfer_proto.image_inode = inode_addr
        file_path = os.path.join(path, name_str)
        transfer_proto.file_path = utils.GetUnicodeString(file_path)

        # If we are dealing with a VSS we want to calculate a hash
        # value based on available timestamps and compare that to previously
        # calculated hash values, and only include the file into the queue if
        # the hash does not match.
        if self.parse_vss:
          hash_value = self.CalculateNTFSTimeHash(f.info.meta)

          if inode_addr in self._hashlist:
            if hash_value in self._hashlist[inode_addr]:
              continue
{'fraction_non_alphanumeric': 0.07911567311488353, 'fraction_numerical': 0.0030003947887879984, 'mean_word_length': 2.6970227670753064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2707830', 'n_tokens_mistral': 3703, 'n_tokens_neox': 3605, 'n_words': 1390}