<commit_before># Copyright (C) 2010-2015 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import re

from lib.cuckoo.common.abstracts import Processing
from lib.cuckoo.common.objects import File
from lib.cuckoo.common.whitelist import is_whitelisted_domain

HTTP_REGEX = (
    "(https?://)(["
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]"
    "]|[a-zA-Z0-9\\.-]+)"
    "(:\\d+)?(/[a-zA-Z0-9_:%?=/\\.-]+)"
)

class ProcessMemory(Processing):
    """Analyze process memory dumps."""

    def run(self):
        """Run analysis.
        @return: structured results.
        """
        self.key = "procmemory"
        results = []

        if os.path.exists(self.pmemory_path):
            for dmp in os.listdir(self.pmemory_path):
                dmp_path = os.path.join(self.pmemory_path, dmp)
                dmp_file = File(dmp_path)

                # Let's hope the file is not too big.
                buf = open(dmp_path, "rb").read()
                urls = set()
                for url in re.findall(HTTP_REGEX, buf):
                    if not is_whitelisted_domain(url[1]):
                        urls.add("".join(url))

                proc = dict(
                    file=dmp_path,
                    pid=int(os.path.basename(dmp_path).split("-")[0]),
                    yara=dmp_file.get_yara("memory"),
                    urls=list(urls),
                )

                results.append(proc)

        return results<commit_msg>process memory dump analysis improvements<commit_after># Copyright (C) 2010-2015 Cuckoo Foundation.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import mmap
import os
import re
import struct

from lib.cuckoo.common.abstracts import Processing
from lib.cuckoo.common.objects import File
from lib.cuckoo.common.whitelist import is_whitelisted_domain

HTTP_REGEX = (
    "(https?://)(["
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]\\."
    "[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]"
    "]|[a-zA-Z0-9\\.-]+)"
    "(:\\d+)?(/[a-zA-Z0-9_:%?=/\\.-]+)"
)

class ProcessMemory(Processing):
    """Analyze process memory dumps."""
    def read_dump(self, filepath):
        f = open(filepath, "rb")

        while True:
            buf = f.read(24)
            if not buf:
                break

            row = struct.unpack("QIIII", buf)
            addr, size, state, typ, protect = row

            yield {
                "addr": "0x%x" % addr,
                "end": "0x%x" % (addr + size),
                "size": size,
                "type": typ,
                "protect": protect,
                "offset": f.tell(),
            }

            f.seek(size, 1)

    def extract_urls(self, filepath):
        # http://stackoverflow.com/a/454589
        urls = set()
        f = open(filepath, "rb")
        m = mmap.mmap(f.fileno(), 0, access=mmap.PROT_READ)

        for url in re.findall(HTTP_REGEX, m):
            if not is_whitelisted_domain(url[1]):
                urls.add("".join(url))

        return urls

    def run(self):
        """Run analysis.
        @return: structured results.
        """
        self.key = "procmemory"
        results = []

        if os.path.exists(self.pmemory_path):
            for dmp in os.listdir(self.pmemory_path):
                dump_path = os.path.join(self.pmemory_path, dmp)
                dump_file = File(dump_path)

                if "-" in os.path.basename(dump_path):
                    pid = int(os.path.basename(dump_path).split("-")[0])
                else:
                    pid = int(os.path.basename(dump_path).split(".")[0])

                proc = dict(
                    file=dump_path, pid=pid,
                    yara=dump_file.get_yara("memory"),
                    urls=list(self.extract_urls(dump_path)),
                    space=list(self.read_dump(dump_path)),
                )

                results.append(proc)

        return results
{'fraction_non_alphanumeric': 0.17871071346639136, 'fraction_numerical': 0.044276210139940354, 'mean_word_length': 1.7984595635430038, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2114515', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1569, 'n_words': 265}