<commit_before>require "base64"
require "stringio"

module LastPass
    class Parser
        class << self
            def parse blob
                parser = Parser.new blob
                parser.send :parse # to avoid exposing the private 'parse' method

                parser
            end
        end

        attr_reader :chunks

        private

        def initialize blob
            @blob = blob
        end

        # Does all the parsing
        def parse
            @chunks = parse_chunks extract_chunks decode_blob @blob
        end

        # Decodes the blob form base64 to raw
        def decode_blob blob
            if not String === blob
                raise ArgumentError, 'Blob should be a string'
            end

            if blob[0, 4] != 'TFBB'
                raise ArgumentError, 'Blob doesn\'t seem to be base64 encoded'
            end

            decode_base64 blob
        end

        # Splits the blob into the chunks grouped by id
        def extract_chunks blob
            chunks = Hash.new { |hash, key| hash[key] = [] }

            StringIO.open blob do |stream|
                while !stream.eof?
                    chunk = read_chunk stream
                    chunks[chunk[:id]] << chunk[:payload]
                end
            end

            chunks
        end

        # Iterates over the chunks in the stream
        def each_chunk stream
            while !stream.eof?
                yield read_chunk stream
            end
        end

        def parse_chunks raw_chunks
            parsed_chunks = {}

            raw_chunks.each do |id, chunks|
                parse_method = "parse_chunk_#{id}"
                if respond_to? parse_method, true
                    parsed_chunks[id] = chunks.map do |chunk|
                        StringIO.open chunk do |stream|
                            send parse_method, stream
                        end
                    end
                end
            end

            parsed_chunks
        end

        #
        # IO
        #

        def read_chunk stream
            # LastPass blob chunk is made up of 4-byte ID, 4-byte size and payload of that size
            # Example:
            #   0000: 'IDID'
            #   0004: 4
            #   0008: 0xDE 0xAD 0xBE 0xEF
            #   000C: --- Next chunk ---
            id = stream.read 4
            size = read_uint32 stream
            payload = stream.read size

            {:id => id, :size => size, :payload => payload}
        end

        def read_item stream
            # An item in an itemized chunk is made up of a size and the payload
            # Example:
            #   0000: 4
            #   0004: 0xDE 0xAD 0xBE 0xEF
            #   0008: --- Next item ---
            size = read_uint32 stream
            payload = stream.read size

            {:size => size, :payload => payload}
        end

        def read_uint32 stream
            stream.read(4).unpack('N').first
        end

        #
        # Decoders
        #

        # Allowed encodings:
        #  - nil or :plain
        #  - :base64
        def decode data, encoding = nil
            encoding.nil? || encoding == :plain ? data : send("decode_#{encoding}", data)
        end

        def decode_base64 data
            Base64.decode64 data
        end

        #
        # Parsing
        #

        # Generic itemized chunk parser.  Info parameter should look like this:
        # [
        #   {:name => 'item_name1'},
        #   {:name => 'item_name2', :encoding => :hex},
        #   {:name => 'item_name3', :encoding => :aes256}
        # ]
        def parse_itemized_chunk stream, info
            chunk = {}

            info.each do |item_info|
                chunk[item_info[:name]] = parse_item stream, item_info[:encoding]
            end

            chunk
        end<commit_msg>Refactoring and cleanup.<commit_after>require "base64"
require "stringio"

module LastPass
    class Parser
        class << self
            def parse blob
                parser = Parser.new blob
                parser.send :parse # to avoid exposing the private 'parse' method

                parser
            end
        end

        attr_reader :chunks

        private

        def initialize blob
            @blob = blob
        end

        # Does all the parsing
        def parse
            @chunks = parse_chunks extract_chunks decode_blob @blob
        end

        # Decodes the blob form base64 to raw
        def decode_blob blob
            if not String === blob
                raise ArgumentError, 'Blob should be a string'
            end

            if blob[0, 4] != 'TFBB'
                raise ArgumentError, 'Blob doesn\'t seem to be base64 encoded'
            end

            decode_base64 blob
        end

        # Splits the blob into the chunks grouped by id
        def extract_chunks blob
            chunks = Hash.new { |hash, key| hash[key] = [] }

            StringIO.open blob do |stream|
                while !stream.eof?
                    chunk = read_chunk stream
                    chunks[chunk[:id]] << chunk[:payload]
                end
            end

            chunks
        end

        # Iterates over the chunks in the stream
        def each_chunk stream
            while !stream.eof?
                yield read_chunk stream
            end
        end

        def parse_chunks raw_chunks
            parsed_chunks = {}

            raw_chunks.each do |id, chunks|
                parse_method = "parse_chunk_#{id}"
                if respond_to? parse_method, true
                    parsed_chunks[id] = chunks.map do |chunk|
                        StringIO.open chunk do |stream|
                            send parse_method, stream
                        end
                    end
                end
            end

            parsed_chunks
        end

        #
        # IO
        #

        def read_chunk stream
            # LastPass blob chunk is made up of 4-byte ID, 4-byte size and payload of that size
            # Example:
            #   0000: 'IDID'
            #   0004: 4
            #   0008: 0xDE 0xAD 0xBE 0xEF
            #   000C: --- Next chunk ---
            id = stream.read 4
            size = read_uint32 stream
            payload = stream.read size

            {:id => id, :size => size, :payload => payload}
        end

        def read_item stream
            # An item in an itemized chunk is made up of a size and the payload
            # Example:
            #   0000: 4
            #   0004: 0xDE 0xAD 0xBE 0xEF
            #   0008: --- Next item ---
            size = read_uint32 stream
            payload = stream.read size

            {:size => size, :payload => payload}
        end

        def read_uint32 stream
            stream.read(4).unpack('N').first
        end

        #
        # Decoders
        #

        # Allowed encodings:
        #  - nil or :plain
        #  - :base64
        def decode data, encoding = nil
            if encoding.nil? || encoding == :plain
                data
            else
                send "decode_#{encoding}", data
            end
        end

        def decode_base64 data
            # TODO: Check for input validity
            Base64.decode64 data
        end

        #
        # Parsing
        #

        # Generic itemized chunk parser.  Info parameter should look like this:
        # [
        #   {:name => 'item_name1'},
        #   {:name => 'item_name2', :encoding => :hex},
        #   {:name => 'item_name3', :encoding => :aes256}
        # ]
        def parse_itemized_chunk stream, info
            chunk = {}

            info.each do |item_info|
                chunk[item_info[:name]] = parse_item stream, item_info[:encoding]
            end

            chunk
        end
{'fraction_non_alphanumeric': 0.06983275884080174, 'fraction_numerical': 0.018128430997063703, 'mean_word_length': 1.1616997792494481, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6922450', 'n_tokens_mistral': 2171, 'n_tokens_neox': 1987, 'n_words': 656}