<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Script to extract common Windows artifacts from source image and its shadow copies."""

from __future__ import print_function
import hashlib
import itertools
import logging
import os
import sys

import artifacts
import artifact_selector
import vsm

from datetime import datetime as dt
from dfvfs.helpers import volume_scanner
from dfvfs.resolver import resolver
from win32file import SetFileTime, CreateFileW, CloseHandle
from win32file import GENERIC_WRITE, FILE_SHARE_WRITE
from win32file import OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL

LOG_FILE = ''


class ArtifactExtractor(volume_scanner.VolumeScanner):
    """Class that extracts common Windows artifacts."""

    _READ_BUFFER_SIZE = 32768  # Class constant that defines the default read buffer size
    _extracted = {}

    @staticmethod
    def _preserve_timestamps(file_entry, output_path):
        """Obtain and set (to preserve) original timestamps of exported files."""

        accessed = created = modified = dt.now()
        stat_object = file_entry.GetStat()

        if stat_object.atime:
            if stat_object.atime_nano:
                accessed = dt.fromtimestamp((float(str(stat_object.atime) + '.' + str(stat_object.atime_nano))))
            else:
                accessed = dt.fromtimestamp(stat_object.atime)

        if stat_object.crtime:
            if stat_object.crtime_nano:
                created = dt.fromtimestamp((float(str(stat_object.crtime) + '.' + str(stat_object.crtime_nano))))
            else:
                created = dt.fromtimestamp(stat_object.crtime)

        if stat_object.mtime:
            if stat_object.mtime_nano:
                modified = dt.fromtimestamp((float(str(stat_object.mtime) + '.' + str(stat_object.mtime_nano))))
            else:
                modified = dt.fromtimestamp(stat_object.mtime)

        handle = CreateFileW(output_path, GENERIC_WRITE, FILE_SHARE_WRITE, None, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
                             None)
        SetFileTime(handle, created, accessed, modified)  # does not seem to preserve nano precision of timestamps
        CloseHandle(handle)

    def _check_unique(self, file_entry, md5):
        """Check if file of the same hash has been previously extracted."""

        if file_entry.path_spec.location in self._extracted:
            if md5 in self._extracted[file_entry.path_spec.location]:
                return False
            else:
                self._extracted[file_entry.path_spec.location].append(md5)
                return True
        else:
            self._extracted[file_entry.path_spec.location] = [md5]
            return True

    def export_file(self, file_entry, output_path, recursive=False, string_to_match=None):
        """Export file to specified output path."""

        md5_obj = hashlib.md5()

        if len(output_path) > 254:
            output_path = output_path[:251] + output_path[-4:]

        if file_entry.IsDirectory():
            if not os.path.exists(os.path.dirname(output_path)):
                os.makedirs(os.path.dirname(output_path))

            for sub_file in file_entry.sub_file_entries:
                if recursive and sub_file.IsDirectory():
                    self.export_file(sub_file, os.path.join(output_path, sub_file.name), True, string_to_match)
                elif not sub_file.IsDirectory():
                    self.export_file(sub_file, os.path.join(output_path, sub_file.name), False, string_to_match)
        elif file_entry.IsFile():
            if not os.path.exists(os.path.dirname(output_path)):
                os.makedirs(os.path.dirname(output_path))

            if string_to_match is not None and string_to_match.lower() not in file_entry.name.lower():
                return
            stat_object = file_entry.GetStat()
            if stat_object.size <= 0 and file_entry.path_spec.data_stream is None:  # empty file
                logging.info(u"Empty File:\t{}".format(file_entry.path_spec.location))
                return<commit_msg>Revert "Fixed minor bug"<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""Script to extract common Windows artifacts from source image and its shadow copies."""

from __future__ import print_function
import hashlib
import itertools
import logging
import os
import sys

import artifacts
import artifact_selector
import vsm

from datetime import datetime as dt
from dfvfs.helpers import volume_scanner
from dfvfs.resolver import resolver
from win32file import SetFileTime, CreateFileW, CloseHandle
from win32file import GENERIC_WRITE, FILE_SHARE_WRITE
from win32file import OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL

LOG_FILE = ''


class ArtifactExtractor(volume_scanner.VolumeScanner):
    """Class that extracts common Windows artifacts."""

    _READ_BUFFER_SIZE = 32768  # Class constant that defines the default read buffer size
    _extracted = {}

    @staticmethod
    def _preserve_timestamps(file_entry, output_path):
        """Obtain and set (to preserve) original timestamps of exported files."""

        accessed = created = modified = dt.now()
        stat_object = file_entry.GetStat()

        if stat_object.atime:
            if stat_object.atime_nano:
                accessed = dt.fromtimestamp((float(str(stat_object.atime) + '.' + str(stat_object.atime_nano))))
            else:
                accessed = dt.fromtimestamp(stat_object.atime)

        if stat_object.crtime:
            if stat_object.crtime_nano:
                created = dt.fromtimestamp((float(str(stat_object.crtime) + '.' + str(stat_object.crtime_nano))))
            else:
                created = dt.fromtimestamp(stat_object.crtime)

        if stat_object.mtime:
            if stat_object.mtime_nano:
                modified = dt.fromtimestamp((float(str(stat_object.mtime) + '.' + str(stat_object.mtime_nano))))
            else:
                modified = dt.fromtimestamp(stat_object.mtime)

        handle = CreateFileW(output_path, GENERIC_WRITE, FILE_SHARE_WRITE, None, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
                             None)
        SetFileTime(handle, created, accessed, modified)  # does not seem to preserve nano precision of timestamps
        CloseHandle(handle)

    def _check_unique(self, file_entry, md5):
        """Check if file of the same hash has been previously extracted."""

        if file_entry.path_spec.location in self._extracted:
            if md5 in self._extracted[file_entry.path_spec.location]:
                return False
            else:
                self._extracted[file_entry.path_spec.location].append(md5)
                return True
        else:
            self._extracted[file_entry.path_spec.location] = [md5]
            return True

    def export_file(self, file_entry, output_path, recursive=False, string_to_match=None):
        """Export file to specified output path."""

        md5_obj = hashlib.md5()

        if len(output_path) > 254:
            output_path = output_path[:251] + output_path[-4:]

        if file_entry.IsDirectory():
            for sub_file in file_entry.sub_file_entries:
                if recursive and sub_file.IsDirectory():
                    self.export_file(sub_file, os.path.join(output_path, sub_file.name), True, string_to_match)
                elif not sub_file.IsDirectory():
                    self.export_file(sub_file, os.path.join(output_path, sub_file.name), False, string_to_match)
        elif file_entry.IsFile():
            if not os.path.exists(os.path.dirname(output_path)):
                os.makedirs(os.path.dirname(output_path))

            if string_to_match is not None and string_to_match.lower() not in file_entry.name.lower():
                return
            stat_object = file_entry.GetStat()
            if stat_object.size <= 0 and file_entry.path_spec.data_stream is None:  # empty file
                logging.info(u"Empty File:\t{}".format(file_entry.path_spec.location))
                return
{'fraction_non_alphanumeric': 0.08740167311774254, 'fraction_numerical': 0.006492695717318017, 'mean_word_length': 2.6310063463281956, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5177308', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2273, 'n_words': 540}