<commit_before>class SpriteInfoExtractor
  class CreateCodeReadError < StandardError ; end
  
  def self.get_gfx_and_palette_and_sprite_from_create_code(create_code_pointer, fs, overlay_to_load, reused_info, ptr_to_ptr_to_files_to_load=nil)
    # This function attempts to find the enemy/object's gfx files, palette pointer, and sprite file.
    # It first looks in the list of files to load for that enemy/object (if given).
    # If any are missing after looking there, it then looks in the create code for pointers that look like they could be the pointers we want.
    
    puts "create code: %08X" % create_code_pointer
    
    if overlay_to_load.is_a?(Integer)
      fs.load_overlay(overlay_to_load)
    elsif overlay_to_load.is_a?(Array)
      overlay_to_load.each do |overlay|
        fs.load_overlay(overlay)
      end
    end
    
    if GAME == "por"
      fs.load_overlay(4)
    end
    
    init_code_pointer      = reused_info[:init_code] || create_code_pointer
    gfx_sheet_ptr_index    = reused_info[:gfx_sheet_ptr_index] || 0
    palette_offset         = reused_info[:palette_offset] || 0
    palette_list_ptr_index = reused_info[:palette_list_ptr_index] || 0
    sprite_ptr_index       = reused_info[:sprite_ptr_index] || 0
    
    gfx_files_to_load = []
    sprite_files_to_load = []
    skeleton_files_to_load = []
    palette_pointer_to_load = nil
    if ptr_to_ptr_to_files_to_load
      pointer_to_start_of_file_index_list = fs.read(ptr_to_ptr_to_files_to_load, 4).unpack("V").first
      
      i = 0
      while true
        file_index_or_palette_pointer, file_data_type = fs.read(pointer_to_start_of_file_index_list+i*8, 8).unpack("VV")
        puts "%08X %08X" % [file_index_or_palette_pointer, file_data_type]
        if file_index_or_palette_pointer == 0xFFFFFFFF
          # End of list.
          break
        end
        if file_data_type == 1 || file_data_type == 2
          file_index = file_index_or_palette_pointer
          file = fs.files_by_index[file_index]
          
          if file_data_type == 1
            render_mode = fs.read(file[:ram_start_offset]+1, 1).unpack("C").first
            canvas_width = fs.read(file[:ram_start_offset]+2, 1).unpack("C").first
            gfx_files_to_load << {file: file, render_mode: render_mode, canvas_width: canvas_width}
          elsif file_data_type == 2
            if file[:file_path] =~ /\/so2?\/.+\.dat/
              sprite_files_to_load << file
            elsif file[:file_path] =~ /\/jnt\/.+\.jnt/
              skeleton_files_to_load << file
            else
              puts file
            end
          end
        elsif file_data_type == 3
          palette_pointer_to_load = file_index_or_palette_pointer
        else
          raise CreateCodeReadError.new("Unknown file data type: #{file_data_type}")
        end
        
        i += 1
      end
      
      if gfx_files_to_load.empty? && sprite_files_to_load.empty?
        raise CreateCodeReadError.new("No gfx files or sprite files to load found")
      end
      if gfx_files_to_load.empty?
        raise CreateCodeReadError.new("No gfx files to load found")
      end
      if sprite_files_to_load.empty?
        #raise CreateCodeReadError.new("No sprite file to load found")
      end
      
      if gfx_files_to_load.length > 0 && sprite_files_to_load.length > 0 && palette_pointer_to_load
        gfx_file_pointers = gfx_files_to_load.map{|file| file[:file][:ram_start_offset]}
        sprite_file_pointer = sprite_files_to_load.first[:ram_start_offset]
        
        return [gfx_file_pointers, palette_pointer_to_load, palette_offset, sprite_file_pointer, skeleton_files_to_load.first]
      end
    end
    
    
    
    possible_gfx_pointers = []
    gfx_page_pointer = nil
    list_of_gfx_page_pointers_wrapper_pointer = nil
    possible_palette_pointers = []
    palette_pointer = nil
    possible_sprite_pointers = []
    sprite_file_pointer = nil
    
    data = fs.read(init_code_pointer, 4*1000, allow_length_to_exceed_end_of_file: true)
    
    data.unpack("V*").each_with_index do |word, i|
      if (0x02000000..0x02FFFFFF).include?(word)
        possible_gfx_pointers << word
        possible_palette_pointers << word
        possible_sprite_pointers << word
      end
    end
    
    
    
    if possible_gfx_pointers.empty? && gfx_files_to_load.empty?
      raise CreateCodeReadError.new("Failed to find any possible enemy gfx pointers.")
    end
    
    valid_gfx_pointers = possible_gfx_pointers.select do |pointer|
      header_vals = fs.read(pointer, 4).unpack("C*") rescue next
      data = fs.read(pointer+4, 4).unpack("V").first
      if data >= 0x02000000 && data < 0x03000000
        # There's a chance this might just be something that looks like a pointer (like palette data), so check to make sure it really is one.
        possible_gfx_page_pointer = fs.read(data, 4).unpack("V").first rescue next
        if possible_gfx_page_pointer >= 0x02000000 && possible_gfx_page_pointer < 0x03000000
          # List of GFX pages
          header_vals.all?{|val| val < 0x50} && (1..2).include?(header_vals[1])
        else
          false
        end
      elsif data == 0x10
        # Just one GFX page, not a list
        header_vals[0] == 0 && (1..2).include?(header_vals[1]) && header_vals[2] == 0x10 && header_vals[3] == 0
      elsif data == 0x20
        # Canvas width is doubled.
        header_vals[0] == 0 && (1..2).include?(header_vals[1]) && header_vals[2] == 0x20 && header_vals[3] == 0
      else
        false
      end
    end
    possible_palette_pointers -= valid_gfx_pointers
    
    if gfx_files_to_load.empty?
      if valid_gfx_pointers.empty?
        raise CreateCodeReadError.new("Failed to find any valid enemy gfx pointers.")
      end
      if gfx_sheet_ptr_index >= valid_gfx_pointers.length
        raise CreateCodeReadError.new("Failed to find enough valid enemy gfx pointers to match the reused enemy gfx sheet index. (#{valid_gfx_pointers.length} found, #{gfx_sheet_ptr_index+1} needed.)")
      end
      
      gfx_pointer = valid_gfx_pointers[gfx_sheet_ptr_index]
      
      data = fs.read(gfx_pointer+4, 4).unpack("V").first
      if data >= 0x02000000 && data < 0x03000000
        _, _, number_of_gfx_pages, _ = fs.read(list_of_gfx_page_pointers_wrapper_pointer, 1).unpack("C*")
        pointer_to_list_of_gfx_file_pointers = data
        
        gfx_file_pointers = fs.read(pointer_to_list_of_gfx_file_pointers, 4*number_of_gfx_pages).unpack("V*")
      else
        gfx_file_pointers = [gfx_pointer]
      end
    end
    
    
    
    if possible_palette_pointers.empty?<commit_msg>Sprite info extractor: Fix bug extracting gfx pointers<commit_after>class SpriteInfoExtractor
  class CreateCodeReadError < StandardError ; end
  
  def self.get_gfx_and_palette_and_sprite_from_create_code(create_code_pointer, fs, overlay_to_load, reused_info, ptr_to_ptr_to_files_to_load=nil)
    # This function attempts to find the enemy/object's gfx files, palette pointer, and sprite file.
    # It first looks in the list of files to load for that enemy/object (if given).
    # If any are missing after looking there, it then looks in the create code for pointers that look like they could be the pointers we want.
    
    puts "create code: %08X" % create_code_pointer
    
    if overlay_to_load.is_a?(Integer)
      fs.load_overlay(overlay_to_load)
    elsif overlay_to_load.is_a?(Array)
      overlay_to_load.each do |overlay|
        fs.load_overlay(overlay)
      end
    end
    
    if GAME == "por"
      fs.load_overlay(4)
    end
    
    init_code_pointer      = reused_info[:init_code] || create_code_pointer
    gfx_sheet_ptr_index    = reused_info[:gfx_sheet_ptr_index] || 0
    palette_offset         = reused_info[:palette_offset] || 0
    palette_list_ptr_index = reused_info[:palette_list_ptr_index] || 0
    sprite_ptr_index       = reused_info[:sprite_ptr_index] || 0
    
    gfx_files_to_load = []
    sprite_files_to_load = []
    skeleton_files_to_load = []
    palette_pointer_to_load = nil
    if ptr_to_ptr_to_files_to_load
      pointer_to_start_of_file_index_list = fs.read(ptr_to_ptr_to_files_to_load, 4).unpack("V").first
      
      i = 0
      while true
        file_index_or_palette_pointer, file_data_type = fs.read(pointer_to_start_of_file_index_list+i*8, 8).unpack("VV")
        puts "%08X %08X" % [file_index_or_palette_pointer, file_data_type]
        if file_index_or_palette_pointer == 0xFFFFFFFF
          # End of list.
          break
        end
        if file_data_type == 1 || file_data_type == 2
          file_index = file_index_or_palette_pointer
          file = fs.files_by_index[file_index]
          
          if file_data_type == 1
            render_mode = fs.read(file[:ram_start_offset]+1, 1).unpack("C").first
            canvas_width = fs.read(file[:ram_start_offset]+2, 1).unpack("C").first
            gfx_files_to_load << {file: file, render_mode: render_mode, canvas_width: canvas_width}
          elsif file_data_type == 2
            if file[:file_path] =~ /\/so2?\/.+\.dat/
              sprite_files_to_load << file
            elsif file[:file_path] =~ /\/jnt\/.+\.jnt/
              skeleton_files_to_load << file
            else
              puts file
            end
          end
        elsif file_data_type == 3
          palette_pointer_to_load = file_index_or_palette_pointer
        else
          raise CreateCodeReadError.new("Unknown file data type: #{file_data_type}")
        end
        
        i += 1
      end
      
      if gfx_files_to_load.empty? && sprite_files_to_load.empty?
        raise CreateCodeReadError.new("No gfx files or sprite files to load found")
      end
      if gfx_files_to_load.empty?
        raise CreateCodeReadError.new("No gfx files to load found")
      end
      if sprite_files_to_load.empty?
        #raise CreateCodeReadError.new("No sprite file to load found")
      end
      
      if gfx_files_to_load.length > 0 && sprite_files_to_load.length > 0 && palette_pointer_to_load
        gfx_file_pointers = gfx_files_to_load.map{|file| file[:file][:ram_start_offset]}
        sprite_file_pointer = sprite_files_to_load.first[:ram_start_offset]
        
        return [gfx_file_pointers, palette_pointer_to_load, palette_offset, sprite_file_pointer, skeleton_files_to_load.first]
      end
    end
    
    
    
    possible_gfx_pointers = []
    gfx_page_pointer = nil
    list_of_gfx_page_pointers_wrapper_pointer = nil
    possible_palette_pointers = []
    palette_pointer = nil
    possible_sprite_pointers = []
    sprite_file_pointer = nil
    
    data = fs.read(init_code_pointer, 4*1000, allow_length_to_exceed_end_of_file: true)
    
    data.unpack("V*").each_with_index do |word, i|
      if (0x02000000..0x02FFFFFF).include?(word)
        possible_gfx_pointers << word
        possible_palette_pointers << word
        possible_sprite_pointers << word
      end
    end
    
    
    
    if possible_gfx_pointers.empty? && gfx_files_to_load.empty?
      raise CreateCodeReadError.new("Failed to find any possible enemy gfx pointers.")
    end
    
    valid_gfx_pointers = possible_gfx_pointers.select do |pointer|
      header_vals = fs.read(pointer, 4).unpack("C*") rescue next
      data = fs.read(pointer+4, 4).unpack("V").first
      if data >= 0x02000000 && data < 0x03000000
        # There's a chance this might just be something that looks like a pointer (like palette data), so check to make sure it really is one.
        possible_gfx_page_pointer = fs.read(data, 4).unpack("V").first rescue next
        if possible_gfx_page_pointer >= 0x02000000 && possible_gfx_page_pointer < 0x03000000
          # List of GFX pages
          header_vals.all?{|val| val < 0x50} && (1..2).include?(header_vals[1])
        else
          false
        end
      elsif data == 0x10
        # Just one GFX page, not a list
        header_vals[0] == 0 && (1..2).include?(header_vals[1]) && header_vals[2] == 0x10 && header_vals[3] == 0
      elsif data == 0x20
        # Canvas width is doubled.
        header_vals[0] == 0 && (1..2).include?(header_vals[1]) && header_vals[2] == 0x20 && header_vals[3] == 0
      else
        false
      end
    end
    possible_palette_pointers -= valid_gfx_pointers
    
    if gfx_files_to_load.empty?
      if valid_gfx_pointers.empty?
        raise CreateCodeReadError.new("Failed to find any valid enemy gfx pointers.")
      end
      if gfx_sheet_ptr_index >= valid_gfx_pointers.length
        raise CreateCodeReadError.new("Failed to find enough valid enemy gfx pointers to match the reused enemy gfx sheet index. (#{valid_gfx_pointers.length} found, #{gfx_sheet_ptr_index+1} needed.)")
      end
      
      gfx_pointer = valid_gfx_pointers[gfx_sheet_ptr_index]
      
      data = fs.read(gfx_pointer+4, 4).unpack("V").first
      if data >= 0x02000000 && data < 0x03000000
        _, _, number_of_gfx_pages, _ = fs.read(gfx_pointer, 4).unpack("C*")
        pointer_to_list_of_gfx_file_pointers = data
        
        gfx_file_pointers = fs.read(pointer_to_list_of_gfx_file_pointers, 4*number_of_gfx_pages).unpack("V*")
      else
        gfx_file_pointers = [gfx_pointer]
      end
    end
    
    
    
    if possible_palette_pointers.empty?
{'fraction_non_alphanumeric': 0.0866189219751225, 'fraction_numerical': 0.02156049754994346, 'mean_word_length': 2.781641961231471, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6251282', 'n_tokens_mistral': 4606, 'n_tokens_neox': 4428, 'n_words': 912}