<commit_before>
class TextDatabase
  class StringDatabaseTooLargeError < StandardError ; end
  
  attr_reader :fs,
              :text_list
  
  def initialize(fs)
    @fs = fs
    @text_list = TextList.new(fs) # Lazy loader
  end
  
  def write_to_rom
    # Writes all text back into ROM, repointing everything in case any strings are longer than they were originally.
    
    # For DoS, note that you can't go past where strings would normally end in RAM for the original game - that would either overwrite or be overwritten by other data.
    # However, this function packs the new strings in more snugly than they were originally in DoS, leaving no free space between them. This means that there is a significant amount of extra room for increasing the size of strings at the end. But not unlimited, so raise an error if even that extra space isn't enough to hold the new strings.
    
    # For OoE, the strings are already packed snugly, so we can't gain any space that way. But we can expand the overlay file here since nothing immediately follows it in RAM.
    
    # For PoR, the strings are split across two different overlay files, so we have to handle those separately.
    
    if REGION == :jp || GAME == "hod"
      header_footer_length = 4
    else
      header_footer_length = 3
    end
    
    overlays = TEXT_REGIONS_OVERLAYS.values.uniq
    overlays.each do |overlay|
      fs.load_overlay(overlay) if overlay
      
      text_list_for_overlay = text_list.select{|text| text.overlay_id == overlay}
      
      next_string_ram_pointer = STRING_DATABASE_START_OFFSET
      should_write_to_end_of_file = false
      text_list_for_overlay.each do |text|
        if next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ALLOWABLE_END_OFFSET
          # Writing strings past this point would result in something being overwritten, so raise an error.
          
          raise StringDatabaseTooLargeError.new
        end
        
        region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
        
        if GAME == "ooe" && next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ORIGINAL_END_OFFSET
          # Reached the end of where strings were in the original game, but in OoE we can expand the file.
          
          should_write_to_end_of_file = true
          next_string_ram_pointer = fs.expand_file_and_get_end_of_file_ram_address(text.string_ram_pointer, text.encoded_string.length + header_footer_length)
        end
        
        # System strings and AoS strings must be aligned to the nearest 4 bytes or they won't be displayed.
        region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
        if GAME == "aos" || (region_name == "System" && next_string_ram_pointer % 4 != 0)
          next_string_ram_pointer = ((next_string_ram_pointer / 4) * 4) + 4
        end
        
        text.string_ram_pointer = next_string_ram_pointer
        
        if should_write_to_end_of_file
          next_string_ram_pointer = fs.expand_file_and_get_end_of_file_ram_address(text.string_ram_pointer, text.encoded_string.length + header_footer_length)
        else
          next_string_ram_pointer += text.encoded_string.length + header_footer_length
        end
      end
    end
    
    overlays.each do |overlay|
      fs.load_overlay(overlay)
      <commit_msg>Fix bug when expanding text overlay for OoE that would waste space<commit_after>
class TextDatabase
  class StringDatabaseTooLargeError < StandardError ; end
  
  attr_reader :fs,
              :text_list
  
  def initialize(fs)
    @fs = fs
    @text_list = TextList.new(fs) # Lazy loader
  end
  
  def write_to_rom
    # Writes all text back into ROM, repointing everything in case any strings are longer than they were originally.
    
    # For DoS, note that you can't go past where strings would normally end in RAM for the original game - that would either overwrite or be overwritten by other data.
    # However, this function packs the new strings in more snugly than they were originally in DoS, leaving no free space between them. This means that there is a significant amount of extra room for increasing the size of strings at the end. But not unlimited, so raise an error if even that extra space isn't enough to hold the new strings.
    
    # For OoE, the strings are already packed snugly, so we can't gain any space that way. But we can expand the overlay file here since nothing immediately follows it in RAM.
    
    # For PoR, the strings are split across two different overlay files, so we have to handle those separately.
    
    if REGION == :jp || GAME == "hod"
      header_footer_length = 4
    else
      header_footer_length = 3
    end
    
    overlays = TEXT_REGIONS_OVERLAYS.values.uniq
    overlays.each do |overlay|
      fs.load_overlay(overlay) if overlay
      
      text_list_for_overlay = text_list.select{|text| text.overlay_id == overlay}
      
      next_string_ram_pointer = STRING_DATABASE_START_OFFSET
      writing_to_end_of_file = false
      text_list_for_overlay.each do |text|
        if next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ALLOWABLE_END_OFFSET
          # Writing strings past this point would result in something being overwritten, so raise an error.
          raise StringDatabaseTooLargeError.new
        end
        
        region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
        
        if !writing_to_end_of_file && GAME == "ooe" && next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ORIGINAL_END_OFFSET
          # Reached the end of where strings were in the original game, but in OoE we can expand the file.
          writing_to_end_of_file = true
        end
        if writing_to_end_of_file
          next_string_ram_pointer = fs.expand_file_and_get_end_of_file_ram_address(text.string_ram_pointer, text.encoded_string.length + header_footer_length)
        end
        
        # System strings and AoS strings must be aligned to the nearest 4 bytes or they won't be displayed.
        region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
        if GAME == "aos" || (region_name == "System" && next_string_ram_pointer % 4 != 0)
          next_string_ram_pointer = ((next_string_ram_pointer / 4) * 4) + 4
        end
        
        text.string_ram_pointer = next_string_ram_pointer
        
        if !writing_to_end_of_file
          next_string_ram_pointer += text.encoded_string.length + header_footer_length
        end
      end
    end
    
    overlays.each do |overlay|
      fs.load_overlay(overlay)
      
{'fraction_non_alphanumeric': 0.05555555555555555, 'fraction_numerical': 0.0029472443265546712, 'mean_word_length': 2.969005847953216, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1424073', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1949, 'n_words': 656}