<commit_before># Performs raw decompression of various compression algorithms (currently, only deflate).

import os
import lzma
import struct
import binwalk.core.C
import binwalk.core.compat
import binwalk.core.common
from binwalk.core.module import Option, Kwarg, Module

class LZMAHeader(object):
    def __init__(self, **kwargs):
        for (k,v) in binwalk.core.compat.iterator(kwargs):
            setattr(self, k, v)

class LZMA(object):

    DESCRIPTION = "Raw LZMA compression stream"
    COMMON_PROPERTIES = [0x5D, 0x6E]
    MAX_PROP = ((4 * 5 + 4) * 9 + 8)
    BLOCK_SIZE = 32*1024

    def __init__(self, module):
        self.module = module
        self.properties = None

        self.build_properties()
        self.build_dictionaries()
        self.build_headers()

        # Add an extraction rule
        if self.module.extractor.enabled:
            self.module.extractor.add_rule(regex='^%s' % self.DESCRIPTION.lower(), extension="7z", cmd=self.extractor)

    def extractor(self, file_name):
        # Open and read the file containing the raw compressed data.
        # This is not terribly efficient, especially for large files...
        compressed_data = binwalk.core.common.BlockFile(file_name).read()

        # Re-run self.decompress to detect the properties for this compressed data (stored in self.properties)
        if self.decompress(compressed_data[:self.BLOCK_SIZE]):
            # Build an LZMA header on top of the raw compressed data and write it back to disk.
            # Header consists of the detected properties values, the largest possible dictionary size,
            # and a fake output file size field.
            header = chr(self.properties) + self.dictionaries[-1] + ("\xFF" * 8)
            binwalk.core.common.BlockFile(file_name, "wb").write(header + compressed_data)

            # Try to extract it with all the normal lzma extractors until one works
            for exrule in self.module.extractor.match("lzma compressed data"):
                if self.module.extractor.execute(exrule['cmd'], file_name) == True:
                    break

    def build_property(self, pb, lp, lc):
        prop = (((pb * 5) + lp) * 9) + lc
        if prop > self.MAX_PROP:
            prop = None
        return int(prop)

    def parse_property(self, prop):
        prop = int(ord(prop))

        if prop > self.MAX_PROP:
            return None

        pb = prop / (9 * 5);<commit_msg>Fixed compression module exception bug<commit_after># Performs raw decompression of various compression algorithms (currently, only deflate).

import os
import lzma
import struct
import binwalk.core.C
import binwalk.core.compat
import binwalk.core.common
from binwalk.core.module import Option, Kwarg, Module

class LZMAHeader(object):
    def __init__(self, **kwargs):
        for (k,v) in binwalk.core.compat.iterator(kwargs):
            setattr(self, k, v)

class LZMA(object):

    DESCRIPTION = "Raw LZMA compression stream"
    COMMON_PROPERTIES = [0x5D, 0x6E]
    MAX_PROP = ((4 * 5 + 4) * 9 + 8)
    BLOCK_SIZE = 32*1024

    def __init__(self, module):
        self.module = module
        self.properties = None

        self.build_properties()
        self.build_dictionaries()
        self.build_headers()

        # Add an extraction rule
        if self.module.extractor.enabled:
            self.module.extractor.add_rule(regex='^%s' % self.DESCRIPTION.lower(), extension="7z", cmd=self.extractor)

    def extractor(self, file_name):
        # Open and read the file containing the raw compressed data.
        # This is not terribly efficient, especially for large files...
        compressed_data = binwalk.core.common.BlockFile(file_name).read()

        # Re-run self.decompress to detect the properties for this compressed data (stored in self.properties)
        if self.decompress(compressed_data[:self.BLOCK_SIZE]):
            # Build an LZMA header on top of the raw compressed data and write it back to disk.
            # Header consists of the detected properties values, the largest possible dictionary size,
            # and a fake output file size field.
            header = chr(self.properties) + self.dictionaries[-1] + ("\xFF" * 8)
            binwalk.core.common.BlockFile(file_name, "wb").write(header + compressed_data)

            # Try to extract it with all the normal lzma extractors until one works
            for exrule in self.module.extractor.match("lzma compressed data"):
                if self.module.extractor.execute(exrule['cmd'], file_name) == True:
                    break

    def build_property(self, pb, lp, lc):
        prop = (((pb * 5) + lp) * 9) + lc
        if prop > self.MAX_PROP:
            return None
        return int(prop)

    def parse_property(self, prop):
        prop = int(ord(prop))

        if prop > self.MAX_PROP:
            return None

        pb = prop / (9 * 5);
{'fraction_non_alphanumeric': 0.09678739512993656, 'fraction_numerical': 0.009003478616738286, 'mean_word_length': 2.567883211678832, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4534918', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1381, 'n_words': 439}