<commit_before>#include "util/bitmap.h"
#include <fstream>
#include <iostream>
#include <algorithm>
#include <cctype>
#include <cstring>
#include <ostream>
#include <sstream>
#include <iostream>

// To aid in filesize
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "util/funcs.h"
#include "util/file-system.h"
#include "util/timedifference.h"

#include "mugen_animation.h"
#include "mugen_background.h"
#include "mugen_item.h"
#include "mugen_item_content.h"
#include "mugen_section.h"
#include "mugen_menu.h"
#include "mugen_sound.h"
#include "mugen_reader.h"
#include "mugen_sprite.h"
#include "globals.h"
#include "state.h"

#include "mugen_util.h"
#include "ast/all.h"
#include "parser/all.h"

using namespace std;

namespace PaintownUtil = ::Util;

static int lowerCase( int c ){ return tolower( c );}

const std::string Mugen::Util::fixCase( const std::string &str ){
    std::string tempStr = str;
    transform( tempStr.begin(), tempStr.end(), tempStr.begin(), lowerCase );
    return tempStr;
}

const std::string Mugen::Util::removeSpaces( const std::string &str ){
    std::string tempStr = str;
    if( tempStr.find(' ') != std::string::npos ){
	Global::debug(2) << "Removing spaces from: " << tempStr << endl;
	for( int i = tempStr.size()-1; i>-1; --i){
	    if( tempStr[i] == ' ' )tempStr.erase( tempStr.begin()+i );
	    else if( tempStr[i] == '\t' )tempStr.erase( tempStr.begin()+i );
	}
    }
    return tempStr;
}

const std::string Mugen::Util::invertSlashes( const std::string &str ){
    std::string tempStr = str;
    if( tempStr.find('\\') != std::string::npos ){
	for( int i = tempStr.size()-1; i>-1; --i){
	    if( tempStr[i] == '\\' )tempStr[i] = '/';
	}
    }
    return tempStr;
}
/*
std:string getHeadDir( const std::string & dir ){
    return dir.substr( ( dir.find_lastof( '/' ) != std::string::npos ? dir.find_lastof( '/' ) : 0 ), lastslash,dir.size() );
}*/

const std::string Mugen::Util::fixFileName( const std::string &dir, std::string str ){
    Global::debug(2) << "Current File: " << str << endl;
    // Temp fix until the lexer fixes this crap
    str = Mugen::Util::removeSpaces(str);
    // Fixes windows paths
    str = Mugen::Util::invertSlashes(str);
    // Lets check if we need to fix anything first
    Global::debug(2) << "Checking for file in " << (dir+str) << endl;
    if( ::Util::exists( dir + str ) == false ){
	Global::debug(2) << "Couldn't find file: " << str << endl;
	std::string returnString = "";
	std::vector< string > files = ::Util::getFiles( dir, "*" );
	Global::debug(2) << "Correcting file: " << str << ", in directory: "<< dir <<".\nGot " << files.size() << " files." << endl;
	for( unsigned int i = 0; i < files.size(); ++i ){
	    std::string temp = files[i].c_str();
	    temp = Mugen::Util::fixCase( temp );
	    if( std::string( dir + str ) == temp ){
		// We got number one chinese retaurant
		returnString = files[i];
		break;
	    }
	}
	Global::debug(2) << "Corrected file: " << returnString << endl;
	return returnString;
    }
    return std::string(dir + str);
}


/* this is basename() */
const std::string Mugen::Util::stripDir( const std::string &str ){ 
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
	size_t rem = temp.find_last_of( "/" );
	if( rem != std::string::npos ){
	    return str.substr(rem+1,str.size());
	}
	rem = temp.find_last_of( "\\" );
	if( rem != std::string::npos ){
	    return str.substr(rem+1,str.size());
	}
    }
    return str; 
}


/* this is dirname() */
const std::string Mugen::Util::getFileDir( const std::string &str ){
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
	size_t rem = temp.find_last_of( "/" );
	if( rem != std::string::npos ){
	    return str.substr(0,rem+1);
	}
	rem = temp.find_last_of( "\\" );
	if( rem != std::string::npos ){
	    return str.substr(0,rem+1);
	}
    }
    return "";
}

// If you use this, please delete the item after you use it, this isn't java ok
MugenItemContent *Mugen::Util::parseOpt( const std::string &opt ){
    std::string contentHolder = "";
    MugenItemContent *temp = new MugenItemContent();
    const char * ignored = " \r\n";
    Global::debug(2) << "Parsing string to ItemContent: " << opt << endl;
    for( unsigned int i = 0; i < opt.size(); ++i ){
	if( opt[i] == ';' )break;
	if( opt[i] == ' ' ){
	    if( !contentHolder.empty() ) *temp << contentHolder;
	    Global::debug(3) << "Got content: " << contentHolder << endl;
	    contentHolder = "";
	}
	// We got one push back the other and reset the holder to get the next
	else if( opt[i] == ',' ){
	    if( !contentHolder.empty() ) *temp << contentHolder;
	    Global::debug(3) << "Got content: " << contentHolder << endl;
	    contentHolder = "";
	}
	//Start grabbing our item
	else if (! strchr(ignored, opt[i])){
		contentHolder += opt[i];
	}
    }
    if( !contentHolder.empty() ){
	*temp << contentHolder;
	Global::debug(3) << "Got content: " << contentHolder << endl;
    }
    return temp;
}

/* PCX HEADER */
typedef struct {
    unsigned char manufacturer; /* Constant Flag, 10 = ZSoft .pcx */
    unsigned char version;      /* Version information */
    unsigned char encoding;     /* 1 = .PCX run length encoding */
    unsigned char BitsPerPixel; /* Number of bits to represent a pixel */
    short Xmin;                 /* Image Dimensions */
    short Ymin;
    short Xmax;
    short Ymax;
    unsigned short HDpi;         /* Horizontal Resolution of image in DPI */
    unsigned short VDpi;         /* Vertical Resolution of image in DPI */
    unsigned char colormap[48];  /* Color palette setting */
    unsigned char reserved;      /* Should be set to 0. */
    unsigned char NPlanes;       /* Number of color planes */
    unsigned short BytesPerLine; /* Number of bytes to allocate for a scanline plane. */
    unsigned short PaletteInfo;  /* How to interpret palette: 1 = Color/BW,
				    2 = Grayscale (ignored in PB IV/ IV +) */
    unsigned short HscreenSize;  /* Horizontal screen size in pixels. */
    unsigned short VscreenSize;  /* Vertical screen size in pixels. */
    char filler[54];             /* Blank to fill out 128 byte header. */
} pcx_header;
/* Source: ZSoft Corporation's PCX File Format Technical Reference Manual, Revision 5. */


bool Mugen::Util::readPalette(const string &filename, unsigned char *pal){
    unsigned char colorsave[3]; // rgb pal save
    FILE *act_file = fopen( filename.c_str(), "rb" );
    if( !act_file ){
	Global::debug(1) << "Unable to open ACT file: " << filename << endl;
	if( act_file )fclose( act_file );
	pal = 0;
	return false;
    }
    else{
	fseek( act_file, 0, SEEK_END );
	// Then it must be an act file
	if( ftell(act_file) == 768 ){ 
	    for( int i=0; i<256; i++ ){
		fseek( act_file, -3*(i+1), SEEK_END );
		// stupid fread
		size_t bleh = fread( colorsave, 3, 1, act_file );
		bleh = bleh;
		pal[3*i]=colorsave[0];
		pal[3*i+1]=colorsave[1];
		pal[3*i+2]=colorsave[2];
	    }
	    Global::debug(1) << "Applying palette from ACT file: " << filename << endl;
	    if( act_file )fclose( act_file );
	    return true;
	}
	// 128-byte header + 768-byte palette + 0x0C byte = minimum 8-bit PCX file size.
	else if( ftell(act_file) < 897 ){ 
            Global::debug(1) << "File " << filename << " is not a valid palette file." << endl;
	    if( act_file )fclose( act_file );
	    pal = 0;
	    return false;
        }
	// we'll assume it's a PCX file, for now.
        else{ 
            fseek( act_file, 0, SEEK_SET );
	    pcx_header pcxhead;
            size_t bleh = fread( &pcxhead, sizeof(pcx_header), 1, act_file );
            // Check to see if the PCX file uses an 8-bit palette.
            if( (pcxhead.manufacturer == 10) && (pcxhead.version >= 5) && (pcxhead.BitsPerPixel == 8) && (pcxhead.NPlanes == 1) ){
                fseek( act_file, -769, SEEK_END);
		// No need to define another variable; colorsave will do just fine.
                bleh = fread( colorsave, 1, 1, act_file ); 
                if( colorsave[0] == 12 ){
                    fseek( act_file, -768, SEEK_END );
                    bleh = fread( pal, 768, 1, act_file );
                    Global::debug(1) << "Applying palette from PCX file: " << filename << endl;
		    return true;
                }
                else{
                    Global::debug(1) << "File " << filename << " is not a valid palette file. (Must be ACT or 8-bit PCX.)";
		    if( act_file )fclose( act_file );
		    pal =0;
		    return false;
                }
            }
            // Add support for JASC and RIFF palette files later... 
            // Minimum JASC PAL size = 1,813 bytes (carriage returns are necessary). 
            else{
		Global::debug(1) << "File " << filename << " is not a valid palette file. (Must be ACT or 8-bit PCX.)";
		if( act_file )fclose( act_file );
		pal = 0;
		return false;;
            }
        }
    }
    if( act_file )fclose( act_file );
    pal = 0;
    return false;
}

/* maybe move this to Filesystem */
static int computeFileSize(const string & path){
    // Lets get the filesize
    FILE * stream = fopen(path.c_str(), "r" );
    fseek(stream, 0, SEEK_END);
    int filesize = ftell(stream);
    fclose(stream);
    return filesize;
}

namespace Mugen{
    namespace Util{

class SffReader{
public:
    SffReader(const string & filename, const string & palette):
    filename(filename),
    currentSprite(0){
        /* 16 skips the header stuff */
        location = 16;
        sffStream.open(filename.c_str(), ios::binary);
        if (!sffStream){
            throw MugenException("Could not open SFF file: '" + filename + "'");
        }

        filesize = computeFileSize(filename);

        /* Lets go ahead and skip the crap -> (Elecbyte signature and version)
         * start at the 16th byte
         */
        sffStream.seekg(location,ios::beg);
        /* FIXME: change these to uint32 or whatever */
        unsigned long totalGroups = 0;
        totalImages = 0;
        unsigned long suboffset = 0;
        unsigned long subhead = 0;
        unsigned int sharedPal = 0;

        /* this probably isn't endian safe.. */
        sffStream.read((char *)&totalGroups, sizeof(totalGroups));
        sffStream.read((char *)&totalImages, sizeof(totalImages));
        sffStream.read((char *)&suboffset, sizeof(suboffset));
        sffStream.read((char *)&subhead, sizeof(subhead));
        sffStream.read((char *)&sharedPal, sizeof(bool));
        if (sharedPal && sharedPal != 1){
            sharedPal = 0;
        }

        location = suboffset;

        if (location < 512 || location > 2147482717 ){
            location = 512;
        }

        Global::debug(2) << "Got Total Groups: " << totalGroups << ", Total Images: " << totalImages << ", Next Location in file: " << location << endl;

        // MugenSprite *spriteIndex[totalImages + 1];
        spriteIndex = new MugenSprite*[totalImages + 1];

        // Palette related
        useact = false;

        //unsigned char colorsave[3]; // rgb pal save

        // Load in first palette
        if (Mugen::Util::readPalette(palette, palsave1)){
            useact = true;
        }

        /*
        if (location < 512 || location > 2147482717){
            location = 512;
        } else {
            location = suboffset;
        }
        */
    }

    virtual ~SffReader(){
        sffStream.close();
        delete[] spriteIndex;
    }

    MugenSprite * readSprite(){
        bool islinked = false;
        if (location > filesize){
            throw MugenException("Error in SFF file: " + filename + ". Offset of image beyond the end of the file.");
        }

        MugenSprite * sprite = new MugenSprite();//readSprite(ifile, location);    
        sprite->read(sffStream, location);

        if (sprite->getLength() == 0){ // Lets get the linked sprite
            // This is linked
            islinked = true;
            /* Lets check if this is a duplicate sprite if so copy it
             * if prev is larger than index then this file is corrupt */
            if (sprite->getPrevious() >= currentSprite){
                throw MugenException("Error in SFF file: " + filename + ". Incorrect reference to sprite.");
            }

            if (sprite->getPrevious() > 32767){
                throw MugenException("Error in SFF file: " + filename + ". Reference too large in image.");
            }

            while (sprite->getLength() == 0){
                const MugenSprite * temp = spriteIndex[sprite->getPrevious()];
                if (!temp){
                    throw MugenException("Error in SFF file: " + filename + ". Referenced sprite is NULL.");
                }

                // Seek to the location of the pcx data
                sffStream.seekg(temp->getLocation() + 32, ios::beg);
                sprite->setLocation(temp->getLocation());
                sprite->setLength(temp->getNext() - temp->getLocation() - 32);

                if ((sprite->getPrevious() <= temp->getPrevious()) &&
                    ((sprite->getPrevious() != 0) || (currentSprite == 0)) &&
                    temp->getLength()==0){

                    std::ostringstream st;
                    st << "Image " << currentSprite << "(" << sprite->getGroupNumber() << "," << sprite->getImageNumber() << ") : circular definition or forward linking. Aborting.\n"; 
                    throw MugenException( st.str() );
                }
                else{
                    if(sprite->getLength() == 0){
                        sprite->setPrevious(temp->getPrevious());
                    } else {
                        sprite->setNewLength(temp->getNext() - temp->getLocation() -32);
                        sprite->setRealLength(sprite->getNewLength());
                    }
                }

                Global::debug(2) << "Referenced Sprite Location: " << temp->getLocation() << " | Group: " << temp->getGroupNumber() << " | Sprite: " << temp->getGroupNumber() << " | at index: " << sprite->getPrevious() << endl;
            }
        }

        try{
            sprite->loadPCX(sffStream, islinked, useact, palsave1);

            // Add to our lists
            spriteIndex[currentSprite] = sprite;
            // Check if the sprite exists already so we can delete it and overwrite
            /*
            Mugen::SpriteMap::iterator first_it = sprites.find(sprite->getGroupNumber());
            if (first_it != sprites.end()){
                std::map< unsigned int, MugenSprite * >::iterator it = first_it->second.find(sprite->getImageNumber());
                if (it != first_it->second.end()){
                    delete it->second;
                }
            }
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = sprite;
            */
            location = sprite->getNext();
        } catch (const LoadException & le){
            /* ignore this error?? */
            location = sprite->getNext();
            // Global::debug(0) << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << endl;
            // sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = 0;
            spriteIndex[currentSprite] = 0;
            delete sprite;
            ostringstream out;
            out << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << ": " << le.getReason() << endl;
            throw MugenException(out.str());
        }
        /*
        // Dump to file just so we can test the pcx in something else
        if( Global::getDebug() == 3 ){
        std::ostringstream st;
        st << "pcxdump/g" << sprite->getGroupNumber() << "i" << sprite->getImageNumber() << ".pcx";
        FILE *pcx;
        if( (pcx = fopen( st.str().c_str(), "wb" )) != NULL ){
        size_t bleh = fwrite( sprite->pcx, sprite->newlength, 1, pcx );
        bleh = bleh;
        fclose( pcx );
        }
        }
        */
        // Set the next file location

        if (!location){
            Global::debug(1) << "End of Sprites or File. Continuing...." << endl;
            throw MugenException("No sprites left");
        }

        Global::debug(2) << "Index: " << currentSprite << ", Location: " << sprite->getLocation()  << ", Next Sprite: "  << sprite->getNext() << ", Length: " << sprite->getRealLength() << ", x|y: " << sprite->getX() << "|" << sprite->getY() << ", Group|Image Number: " << sprite->getGroupNumber() << "|" << sprite->getImageNumber() << ", Prev: " << sprite->getPrevious() << ", Same Pal: " << sprite->getSamePalette() << ", Comments: " << sprite->getComments() << endl;

    }

    bool moreSprites(){
        return currentSprite < totalImages;
    }

protected:
    const string filename;
    ifstream sffStream;
    unsigned long currentSprite;
    int totalSprites;
    MugenSprite ** spriteIndex;
    bool useact;
    int filesize;
    int location;
    unsigned long totalImages;
    unsigned char palsave1[768]; // First image palette
};

    }
}

void Mugen::Util::readSprites(const string & filename, const string & palette, Mugen::SpriteMap & sprites) throw (MugenException){
    /* 16 skips the header stuff */
    int location = 16;
    ifstream ifile;
    ifile.open(filename.c_str(), ios::binary);
    if (!ifile){
	throw MugenException("Could not open SFF file: '" + filename + "'");
    }
    
    int filesize = computeFileSize(filename);

    /* Lets go ahead and skip the crap -> (Elecbyte signature and version)
     * start at the 16th byte
     */
    ifile.seekg(location,ios::beg);
    unsigned long totalGroups = 0;
    unsigned long totalImages = 0;
    unsigned long suboffset = 0;
    unsigned long subhead = 0;
    unsigned int sharedPal = 0;
    
    /* this probably isn't endian safe.. */
    ifile.read((char *)&totalGroups, sizeof(unsigned long));
    ifile.read((char *)&totalImages, sizeof(unsigned long));
    ifile.read((char *)&suboffset, sizeof(unsigned long));
    ifile.read((char *)&subhead, sizeof(unsigned long));
    ifile.read((char *)&sharedPal, sizeof(bool));
    if( sharedPal && sharedPal != 1 )sharedPal = 0;
    location = suboffset;
    if( location < 512 || location > 2147482717 )location = 512;
    
    Global::debug(2) << "Got Total Groups: " << totalGroups << ", Total Images: " << totalImages << ", Next Location in file: " << location << endl;

    MugenSprite *spriteIndex[totalImages + 1];
    
    // Palette related
    bool useact = false;
    
    //unsigned char colorsave[3]; // rgb pal save
    unsigned char palsave1[768]; // First image palette
    
    // Load in first palette
    if (Mugen::Util::readPalette(palette,palsave1)){
	useact = true;
    }
    
    if( location < 512 || location > 2147482717 )location = 512;
    else location = suboffset;
    
    for (unsigned int i = 0; i < totalImages; ++i){
        bool islinked = false;
	if (location > filesize){
	    throw MugenException("Error in SFF file: " + filename + ". Offset of image beyond the end of the file.");
	}
	
        MugenSprite * sprite = new MugenSprite();//readSprite(ifile, location);    
	sprite->read(ifile,location);
	
	if (sprite->getLength() == 0){ // Lets get the linked sprite
	    // This is linked
	    islinked = true;
	    /* Lets check if this is a duplicate sprite if so copy it
	    * if prev is larger than index then this file is corrupt */
	    if( sprite->getPrevious() >= i ) throw MugenException("Error in SFF file: " + filename + ". Incorrect reference to sprite.");
	    if( sprite->getPrevious() > 32767 ) throw MugenException("Error in SFF file: " + filename + ". Reference too large in image.");
	    while( sprite->getLength() == 0 ){
		const MugenSprite *temp = spriteIndex[sprite->getPrevious()];
		if( !temp ) throw MugenException("Error in SFF file: " + filename + ". Referenced sprite is NULL.");
		
		// Seek to the location of the pcx data
		ifile.seekg(temp->getLocation() + 32, ios::beg);
		sprite->setLocation(temp->getLocation());
		sprite->setLength(temp->getNext() - temp->getLocation() - 32);
		
		if( (sprite->getPrevious() <= temp->getPrevious()) && ((sprite->getPrevious() != 0) || (i == 0)) && temp->getLength()==0 ) {
		    std::ostringstream st;
		    st << "Image " << i << "(" << sprite->getGroupNumber() << "," << sprite->getImageNumber() << ") : circular definition or forward linking. Aborting.\n"; 
		    throw MugenException( st.str() );
		}
		else{
		    if(sprite->getLength() == 0){
			sprite->setPrevious(temp->getPrevious());
		    } else {
			sprite->setNewLength(temp->getNext() - temp->getLocation() -32);
			sprite->setRealLength(sprite->getNewLength());
		    }
		}
		
		Global::debug(2) << "Referenced Sprite Location: " << temp->getLocation() << " | Group: " << temp->getGroupNumber() << " | Sprite: " << temp->getGroupNumber() << " | at index: " << sprite->getPrevious() << endl;
	    }
	}
	/*
	// Read in pcx header
	pcx_header pcxhead;
	ifile.read( (char *)&pcxhead.manufacturer, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.version, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.encoding, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.BitsPerPixel, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.Xmin, sizeof(short) );
	ifile.read( (char *)&pcxhead.Ymin, sizeof(short) );
	ifile.read( (char *)&pcxhead.Xmax, sizeof(short));
	ifile.read( (char *)&pcxhead.Ymax, sizeof(short) );
	ifile.read( (char *)&pcxhead.HDpi, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.VDpi, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.colormap, sizeof(unsigned char) * 48);
	ifile.read( (char *)&pcxhead.reserved, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.NPlanes, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.BytesPerLine, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.PaletteInfo, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.HscreenSize, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.VscreenSize, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.filler, sizeof(char) * 54 );
	
	// Now read in the pcx
	// Seek to the location of the pcx data
	ifile.seekg(sprite->location + 32, ios::beg);
	if( sprite->samePalette ){
	    // Lets give it space for the palette
	    Global::debug(1) << "This sprite is less that 768 or has a shared palette - Group: " << sprite->groupNumber << " | Image: " << sprite->imageNumber << endl;
	    sprite->newlength += 768;
	    sprite->pcx = new char[sprite->newlength];
	}
	else sprite->pcx = new char[sprite->reallength];
	ifile.read((char *)sprite->pcx, sprite->reallength);
	
	if ( !islinked ){
	    if ( !useact){
		if ( sprite->samePalette ){
		    memcpy( sprite->pcx + (sprite->reallength), palsave1, 768);
		    Global::debug(1) << "Applying 1st palette to Sprite: " << sprite->imageNumber << " in Group: " << sprite->groupNumber << endl;
		} else {
		    memcpy( palsave1, sprite->pcx+(sprite->reallength)-768, 768);
		}
	    } else {
		// Replace all palettes with the one supplied in act
		if ( sprite->samePalette){
		    memcpy( sprite->pcx + (sprite->reallength), palsave1, 768);
		} else {
		    if (!(sprite->groupNumber == 9000 && sprite->imageNumber == 1)){
			memcpy( sprite->pcx + (sprite->reallength)-768, palsave1, 768);
		    } 
		}
	    }
	}
	*/
	
        try{
            sprite->loadPCX(ifile, islinked, useact, palsave1);
            
            // Add to our lists
            spriteIndex[i] = sprite;
            // Check if the sprite exists already so we can delete it and overwrite
            Mugen::SpriteMap::iterator first_it = sprites.find(sprite->getGroupNumber());
            if (first_it != sprites.end()){
                std::map< unsigned int, MugenSprite * >::iterator it = first_it->second.find(sprite->getImageNumber());
                if (it != first_it->second.end()){
                    delete it->second;
                }
            }
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = sprite;
            location = sprite->getNext();
        } catch (const LoadException & le){
            /* ignore this error?? */
            location = sprite->getNext();
            Global::debug(0) << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << endl;
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = 0;
            spriteIndex[i] = 0;
            delete sprite;
        }
	/*
	// Dump to file just so we can test the pcx in something else
	if( Global::getDebug() == 3 ){
	    std::ostringstream st;
	    st << "pcxdump/g" << sprite->getGroupNumber() << "i" << sprite->getImageNumber() << ".pcx";
	    FILE *pcx;
	    if( (pcx = fopen( st.str().c_str(), "wb" )) != NULL ){
		size_t bleh = fwrite( sprite->pcx, sprite->newlength, 1, pcx );
		bleh = bleh;
		fclose( pcx );
	    }
	}
	*/
	// Set the next file location
	
	if( !location ){
	    Global::debug(1) << "End of Sprites or File. Continuing...." << endl;
	    break;
	}
	
	Global::debug(2) << "Index: " << i << ", Location: " << sprite->getLocation()  << ", Next Sprite: "  << sprite->getNext() << ", Length: " << sprite->getRealLength() << ", x|y: " << sprite->getX() << "|" << sprite->getY() << ", Group|Image Number: " << sprite->getGroupNumber() << "|" << sprite->getImageNumber() << ", Prev: " << sprite->getPrevious() << ", Same Pal: " << sprite->getSamePalette() << ", Comments: " << sprite->getComments() << endl;
	
    }

    ifile.close();
    
    Global::debug(1) << "Got Total Sprites: " << totalImages << endl;
    
    //return sprites;
}

void Mugen::Util::readSounds(const string & filename, std::map<unsigned int,std::map<unsigned int, MugenSound *> > & sounds) throw (MugenException){
    /* 16 skips the header stuff */
    int location = 16;
    ifstream ifile;
    ifile.open(filename.c_str(), ios::binary);
    if (!ifile){<commit_msg>use the sff reader class<commit_after>#include "util/bitmap.h"
#include <fstream>
#include <iostream>
#include <algorithm>
#include <cctype>
#include <cstring>
#include <ostream>
#include <sstream>
#include <iostream>

// To aid in filesize
#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "util/funcs.h"
#include "util/file-system.h"
#include "util/timedifference.h"

#include "mugen_animation.h"
#include "mugen_background.h"
#include "mugen_item.h"
#include "mugen_item_content.h"
#include "mugen_section.h"
#include "mugen_menu.h"
#include "mugen_sound.h"
#include "mugen_reader.h"
#include "mugen_sprite.h"
#include "globals.h"
#include "state.h"

#include "mugen_util.h"
#include "ast/all.h"
#include "parser/all.h"

using namespace std;

namespace PaintownUtil = ::Util;

static int lowerCase( int c ){ return tolower( c );}

const std::string Mugen::Util::fixCase( const std::string &str ){
    std::string tempStr = str;
    transform( tempStr.begin(), tempStr.end(), tempStr.begin(), lowerCase );
    return tempStr;
}

const std::string Mugen::Util::removeSpaces( const std::string &str ){
    std::string tempStr = str;
    if( tempStr.find(' ') != std::string::npos ){
	Global::debug(2) << "Removing spaces from: " << tempStr << endl;
	for( int i = tempStr.size()-1; i>-1; --i){
	    if( tempStr[i] == ' ' )tempStr.erase( tempStr.begin()+i );
	    else if( tempStr[i] == '\t' )tempStr.erase( tempStr.begin()+i );
	}
    }
    return tempStr;
}

const std::string Mugen::Util::invertSlashes( const std::string &str ){
    std::string tempStr = str;
    if( tempStr.find('\\') != std::string::npos ){
	for( int i = tempStr.size()-1; i>-1; --i){
	    if( tempStr[i] == '\\' )tempStr[i] = '/';
	}
    }
    return tempStr;
}
/*
std:string getHeadDir( const std::string & dir ){
    return dir.substr( ( dir.find_lastof( '/' ) != std::string::npos ? dir.find_lastof( '/' ) : 0 ), lastslash,dir.size() );
}*/

const std::string Mugen::Util::fixFileName( const std::string &dir, std::string str ){
    Global::debug(2) << "Current File: " << str << endl;
    // Temp fix until the lexer fixes this crap
    str = Mugen::Util::removeSpaces(str);
    // Fixes windows paths
    str = Mugen::Util::invertSlashes(str);
    // Lets check if we need to fix anything first
    Global::debug(2) << "Checking for file in " << (dir+str) << endl;
    if( ::Util::exists( dir + str ) == false ){
	Global::debug(2) << "Couldn't find file: " << str << endl;
	std::string returnString = "";
	std::vector< string > files = ::Util::getFiles( dir, "*" );
	Global::debug(2) << "Correcting file: " << str << ", in directory: "<< dir <<".\nGot " << files.size() << " files." << endl;
	for( unsigned int i = 0; i < files.size(); ++i ){
	    std::string temp = files[i].c_str();
	    temp = Mugen::Util::fixCase( temp );
	    if( std::string( dir + str ) == temp ){
		// We got number one chinese retaurant
		returnString = files[i];
		break;
	    }
	}
	Global::debug(2) << "Corrected file: " << returnString << endl;
	return returnString;
    }
    return std::string(dir + str);
}


/* this is basename() */
const std::string Mugen::Util::stripDir( const std::string &str ){ 
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
	size_t rem = temp.find_last_of( "/" );
	if( rem != std::string::npos ){
	    return str.substr(rem+1,str.size());
	}
	rem = temp.find_last_of( "\\" );
	if( rem != std::string::npos ){
	    return str.substr(rem+1,str.size());
	}
    }
    return str; 
}


/* this is dirname() */
const std::string Mugen::Util::getFileDir( const std::string &str ){
    std::string temp = str;
    if( str.find( "/") != std::string::npos || str.find( "\\") != std::string::npos ){
	size_t rem = temp.find_last_of( "/" );
	if( rem != std::string::npos ){
	    return str.substr(0,rem+1);
	}
	rem = temp.find_last_of( "\\" );
	if( rem != std::string::npos ){
	    return str.substr(0,rem+1);
	}
    }
    return "";
}

// If you use this, please delete the item after you use it, this isn't java ok
MugenItemContent *Mugen::Util::parseOpt( const std::string &opt ){
    std::string contentHolder = "";
    MugenItemContent *temp = new MugenItemContent();
    const char * ignored = " \r\n";
    Global::debug(2) << "Parsing string to ItemContent: " << opt << endl;
    for( unsigned int i = 0; i < opt.size(); ++i ){
	if( opt[i] == ';' )break;
	if( opt[i] == ' ' ){
	    if( !contentHolder.empty() ) *temp << contentHolder;
	    Global::debug(3) << "Got content: " << contentHolder << endl;
	    contentHolder = "";
	}
	// We got one push back the other and reset the holder to get the next
	else if( opt[i] == ',' ){
	    if( !contentHolder.empty() ) *temp << contentHolder;
	    Global::debug(3) << "Got content: " << contentHolder << endl;
	    contentHolder = "";
	}
	//Start grabbing our item
	else if (! strchr(ignored, opt[i])){
		contentHolder += opt[i];
	}
    }
    if( !contentHolder.empty() ){
	*temp << contentHolder;
	Global::debug(3) << "Got content: " << contentHolder << endl;
    }
    return temp;
}

/* PCX HEADER */
typedef struct {
    unsigned char manufacturer; /* Constant Flag, 10 = ZSoft .pcx */
    unsigned char version;      /* Version information */
    unsigned char encoding;     /* 1 = .PCX run length encoding */
    unsigned char BitsPerPixel; /* Number of bits to represent a pixel */
    short Xmin;                 /* Image Dimensions */
    short Ymin;
    short Xmax;
    short Ymax;
    unsigned short HDpi;         /* Horizontal Resolution of image in DPI */
    unsigned short VDpi;         /* Vertical Resolution of image in DPI */
    unsigned char colormap[48];  /* Color palette setting */
    unsigned char reserved;      /* Should be set to 0. */
    unsigned char NPlanes;       /* Number of color planes */
    unsigned short BytesPerLine; /* Number of bytes to allocate for a scanline plane. */
    unsigned short PaletteInfo;  /* How to interpret palette: 1 = Color/BW,
				    2 = Grayscale (ignored in PB IV/ IV +) */
    unsigned short HscreenSize;  /* Horizontal screen size in pixels. */
    unsigned short VscreenSize;  /* Vertical screen size in pixels. */
    char filler[54];             /* Blank to fill out 128 byte header. */
} pcx_header;
/* Source: ZSoft Corporation's PCX File Format Technical Reference Manual, Revision 5. */


bool Mugen::Util::readPalette(const string &filename, unsigned char *pal){
    unsigned char colorsave[3]; // rgb pal save
    FILE *act_file = fopen( filename.c_str(), "rb" );
    if( !act_file ){
	Global::debug(1) << "Unable to open ACT file: " << filename << endl;
	if( act_file )fclose( act_file );
	pal = 0;
	return false;
    }
    else{
	fseek( act_file, 0, SEEK_END );
	// Then it must be an act file
	if( ftell(act_file) == 768 ){ 
	    for( int i=0; i<256; i++ ){
		fseek( act_file, -3*(i+1), SEEK_END );
		// stupid fread
		size_t bleh = fread( colorsave, 3, 1, act_file );
		bleh = bleh;
		pal[3*i]=colorsave[0];
		pal[3*i+1]=colorsave[1];
		pal[3*i+2]=colorsave[2];
	    }
	    Global::debug(1) << "Applying palette from ACT file: " << filename << endl;
	    if( act_file )fclose( act_file );
	    return true;
	}
	// 128-byte header + 768-byte palette + 0x0C byte = minimum 8-bit PCX file size.
	else if( ftell(act_file) < 897 ){ 
            Global::debug(1) << "File " << filename << " is not a valid palette file." << endl;
	    if( act_file )fclose( act_file );
	    pal = 0;
	    return false;
        }
	// we'll assume it's a PCX file, for now.
        else{ 
            fseek( act_file, 0, SEEK_SET );
	    pcx_header pcxhead;
            size_t bleh = fread( &pcxhead, sizeof(pcx_header), 1, act_file );
            // Check to see if the PCX file uses an 8-bit palette.
            if( (pcxhead.manufacturer == 10) && (pcxhead.version >= 5) && (pcxhead.BitsPerPixel == 8) && (pcxhead.NPlanes == 1) ){
                fseek( act_file, -769, SEEK_END);
		// No need to define another variable; colorsave will do just fine.
                bleh = fread( colorsave, 1, 1, act_file ); 
                if( colorsave[0] == 12 ){
                    fseek( act_file, -768, SEEK_END );
                    bleh = fread( pal, 768, 1, act_file );
                    Global::debug(1) << "Applying palette from PCX file: " << filename << endl;
		    return true;
                }
                else{
                    Global::debug(1) << "File " << filename << " is not a valid palette file. (Must be ACT or 8-bit PCX.)";
		    if( act_file )fclose( act_file );
		    pal =0;
		    return false;
                }
            }
            // Add support for JASC and RIFF palette files later... 
            // Minimum JASC PAL size = 1,813 bytes (carriage returns are necessary). 
            else{
		Global::debug(1) << "File " << filename << " is not a valid palette file. (Must be ACT or 8-bit PCX.)";
		if( act_file )fclose( act_file );
		pal = 0;
		return false;;
            }
        }
    }
    if( act_file )fclose( act_file );
    pal = 0;
    return false;
}

/* maybe move this to Filesystem */
static int computeFileSize(const string & path){
    // Lets get the filesize
    FILE * stream = fopen(path.c_str(), "r" );
    fseek(stream, 0, SEEK_END);
    int filesize = ftell(stream);
    fclose(stream);
    return filesize;
}

namespace Mugen{
    namespace Util{

class SffReader{
public:
    SffReader(const string & filename, const string & palette):
    filename(filename),
    currentSprite(0){
        /* 16 skips the header stuff */
        location = 16;
        sffStream.open(filename.c_str(), ios::binary);
        if (!sffStream){
            throw MugenException("Could not open SFF file: '" + filename + "'");
        }

        filesize = computeFileSize(filename);

        /* Lets go ahead and skip the crap -> (Elecbyte signature and version)
         * start at the 16th byte
         */
        sffStream.seekg(location,ios::beg);
        /* FIXME: change these to uint32 or whatever */
        unsigned long totalGroups = 0;
        totalImages = 0;
        unsigned long suboffset = 0;
        unsigned long subhead = 0;
        unsigned int sharedPal = 0;

        /* this probably isn't endian safe.. */
        sffStream.read((char *)&totalGroups, sizeof(totalGroups));
        sffStream.read((char *)&totalImages, sizeof(totalImages));
        sffStream.read((char *)&suboffset, sizeof(suboffset));
        sffStream.read((char *)&subhead, sizeof(subhead));
        sffStream.read((char *)&sharedPal, sizeof(bool));
        if (sharedPal && sharedPal != 1){
            sharedPal = 0;
        }

        location = suboffset;

        if (location < 512 || location > 2147482717 ){
            location = 512;
        }

        Global::debug(2) << "Got Total Groups: " << totalGroups << ", Total Images: " << totalImages << ", Next Location in file: " << location << endl;

        // MugenSprite *spriteIndex[totalImages + 1];
        spriteIndex = new MugenSprite*[totalImages + 1];

        // Palette related
        useact = false;

        //unsigned char colorsave[3]; // rgb pal save

        // Load in first palette
        if (Mugen::Util::readPalette(palette, palsave1)){
            useact = true;
        }

        /*
        if (location < 512 || location > 2147482717){
            location = 512;
        } else {
            location = suboffset;
        }
        */
    }

    virtual ~SffReader(){
        sffStream.close();
        delete[] spriteIndex;
    }

    MugenSprite * readSprite(){
        bool islinked = false;
        if (location > filesize){
            throw MugenException("Error in SFF file: " + filename + ". Offset of image beyond the end of the file.");
        }

        MugenSprite * sprite = new MugenSprite();//readSprite(ifile, location);    
        sprite->read(sffStream, location);

        if (sprite->getLength() == 0){ // Lets get the linked sprite
            // This is linked
            islinked = true;
            /* Lets check if this is a duplicate sprite if so copy it
             * if prev is larger than index then this file is corrupt */
            if (sprite->getPrevious() >= currentSprite){
                throw MugenException("Error in SFF file: " + filename + ". Incorrect reference to sprite.");
            }

            if (sprite->getPrevious() > 32767){
                throw MugenException("Error in SFF file: " + filename + ". Reference too large in image.");
            }

            while (sprite->getLength() == 0){
                const MugenSprite * temp = spriteIndex[sprite->getPrevious()];
                if (!temp){
                    throw MugenException("Error in SFF file: " + filename + ". Referenced sprite is NULL.");
                }

                // Seek to the location of the pcx data
                sffStream.seekg(temp->getLocation() + 32, ios::beg);
                sprite->setLocation(temp->getLocation());
                sprite->setLength(temp->getNext() - temp->getLocation() - 32);

                if ((sprite->getPrevious() <= temp->getPrevious()) &&
                    ((sprite->getPrevious() != 0) || (currentSprite == 0)) &&
                    temp->getLength()==0){

                    std::ostringstream st;
                    st << "Image " << currentSprite << "(" << sprite->getGroupNumber() << "," << sprite->getImageNumber() << ") : circular definition or forward linking. Aborting.\n"; 
                    throw MugenException( st.str() );
                }
                else{
                    if(sprite->getLength() == 0){
                        sprite->setPrevious(temp->getPrevious());
                    } else {
                        sprite->setNewLength(temp->getNext() - temp->getLocation() -32);
                        sprite->setRealLength(sprite->getNewLength());
                    }
                }

                Global::debug(2) << "Referenced Sprite Location: " << temp->getLocation() << " | Group: " << temp->getGroupNumber() << " | Sprite: " << temp->getGroupNumber() << " | at index: " << sprite->getPrevious() << endl;
            }
        }

        try{
            sprite->loadPCX(sffStream, islinked, useact, palsave1);

            // Add to our lists
            spriteIndex[currentSprite] = sprite;
            // Check if the sprite exists already so we can delete it and overwrite
            /*
            Mugen::SpriteMap::iterator first_it = sprites.find(sprite->getGroupNumber());
            if (first_it != sprites.end()){
                std::map< unsigned int, MugenSprite * >::iterator it = first_it->second.find(sprite->getImageNumber());
                if (it != first_it->second.end()){
                    delete it->second;
                }
            }
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = sprite;
            */
            location = sprite->getNext();

            if (!location){
                Global::debug(1) << "End of Sprites or File. Continuing...." << endl;
                throw MugenException("No sprites left");
            }

            Global::debug(2) << "Index: " << currentSprite << ", Location: " << sprite->getLocation()  << ", Next Sprite: "  << sprite->getNext() << ", Length: " << sprite->getRealLength() << ", x|y: " << sprite->getX() << "|" << sprite->getY() << ", Group|Image Number: " << sprite->getGroupNumber() << "|" << sprite->getImageNumber() << ", Prev: " << sprite->getPrevious() << ", Same Pal: " << sprite->getSamePalette() << ", Comments: " << sprite->getComments() << endl;

            currentSprite += 1;
            return sprite;
        } catch (const LoadException & le){
            /* ignore this error?? */
            location = sprite->getNext();
            // Global::debug(0) << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << endl;
            // sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = 0;
            spriteIndex[currentSprite] = 0;
            currentSprite += 1;
            delete sprite;
            ostringstream out;
            out << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << ": " << le.getReason() << endl;
            throw MugenException(out.str());
        }
        /*
        // Dump to file just so we can test the pcx in something else
        if( Global::getDebug() == 3 ){
        std::ostringstream st;
        st << "pcxdump/g" << sprite->getGroupNumber() << "i" << sprite->getImageNumber() << ".pcx";
        FILE *pcx;
        if( (pcx = fopen( st.str().c_str(), "wb" )) != NULL ){
        size_t bleh = fwrite( sprite->pcx, sprite->newlength, 1, pcx );
        bleh = bleh;
        fclose( pcx );
        }
        }
        */
        // Set the next file location
    }

    bool moreSprites(){
        return currentSprite < totalImages;
    }

protected:
    const string filename;
    ifstream sffStream;
    unsigned long currentSprite;
    int totalSprites;
    MugenSprite ** spriteIndex;
    bool useact;
    int filesize;
    int location;
    unsigned long totalImages;
    unsigned char palsave1[768]; // First image palette
};

    }
}

void Mugen::Util::readSprites(const string & filename, const string & palette, Mugen::SpriteMap & sprites) throw (MugenException){
    SffReader reader(filename, palette);
    while (reader.moreSprites()){
        try{
            MugenSprite * sprite = reader.readSprite();

            Mugen::SpriteMap::iterator first_it = sprites.find(sprite->getGroupNumber());
            if (first_it != sprites.end()){
                std::map< unsigned int, MugenSprite * >::iterator it = first_it->second.find(sprite->getImageNumber());
                if (it != first_it->second.end()){
                    delete it->second;
                }
            }
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = sprite;
        } catch (const MugenException & e){
            Global::debug(0) << e.getReason() << endl;
        }
    }
}

#if 0
void Mugen::Util::readSprites2(const string & filename, const string & palette, Mugen::SpriteMap & sprites) throw (MugenException){
    /* 16 skips the header stuff */
    int location = 16;
    ifstream ifile;
    ifile.open(filename.c_str(), ios::binary);
    if (!ifile){
	throw MugenException("Could not open SFF file: '" + filename + "'");
    }
    
    int filesize = computeFileSize(filename);

    /* Lets go ahead and skip the crap -> (Elecbyte signature and version)
     * start at the 16th byte
     */
    ifile.seekg(location,ios::beg);
    unsigned long totalGroups = 0;
    unsigned long totalImages = 0;
    unsigned long suboffset = 0;
    unsigned long subhead = 0;
    unsigned int sharedPal = 0;
    
    /* this probably isn't endian safe.. */
    ifile.read((char *)&totalGroups, sizeof(unsigned long));
    ifile.read((char *)&totalImages, sizeof(unsigned long));
    ifile.read((char *)&suboffset, sizeof(unsigned long));
    ifile.read((char *)&subhead, sizeof(unsigned long));
    ifile.read((char *)&sharedPal, sizeof(bool));
    if( sharedPal && sharedPal != 1 )sharedPal = 0;
    location = suboffset;
    if( location < 512 || location > 2147482717 )location = 512;
    
    Global::debug(2) << "Got Total Groups: " << totalGroups << ", Total Images: " << totalImages << ", Next Location in file: " << location << endl;

    MugenSprite *spriteIndex[totalImages + 1];
    
    // Palette related
    bool useact = false;
    
    //unsigned char colorsave[3]; // rgb pal save
    unsigned char palsave1[768]; // First image palette
    
    // Load in first palette
    if (Mugen::Util::readPalette(palette,palsave1)){
	useact = true;
    }
    
    if( location < 512 || location > 2147482717 )location = 512;
    else location = suboffset;
    
    for (unsigned int i = 0; i < totalImages; ++i){
        bool islinked = false;
	if (location > filesize){
	    throw MugenException("Error in SFF file: " + filename + ". Offset of image beyond the end of the file.");
	}
	
        MugenSprite * sprite = new MugenSprite();//readSprite(ifile, location);    
	sprite->read(ifile,location);
	
	if (sprite->getLength() == 0){ // Lets get the linked sprite
	    // This is linked
	    islinked = true;
	    /* Lets check if this is a duplicate sprite if so copy it
	    * if prev is larger than index then this file is corrupt */
	    if( sprite->getPrevious() >= i ) throw MugenException("Error in SFF file: " + filename + ". Incorrect reference to sprite.");
	    if( sprite->getPrevious() > 32767 ) throw MugenException("Error in SFF file: " + filename + ". Reference too large in image.");
	    while( sprite->getLength() == 0 ){
		const MugenSprite *temp = spriteIndex[sprite->getPrevious()];
		if( !temp ) throw MugenException("Error in SFF file: " + filename + ". Referenced sprite is NULL.");
		
		// Seek to the location of the pcx data
		ifile.seekg(temp->getLocation() + 32, ios::beg);
		sprite->setLocation(temp->getLocation());
		sprite->setLength(temp->getNext() - temp->getLocation() - 32);
		
		if( (sprite->getPrevious() <= temp->getPrevious()) && ((sprite->getPrevious() != 0) || (i == 0)) && temp->getLength()==0 ) {
		    std::ostringstream st;
		    st << "Image " << i << "(" << sprite->getGroupNumber() << "," << sprite->getImageNumber() << ") : circular definition or forward linking. Aborting.\n"; 
		    throw MugenException( st.str() );
		}
		else{
		    if(sprite->getLength() == 0){
			sprite->setPrevious(temp->getPrevious());
		    } else {
			sprite->setNewLength(temp->getNext() - temp->getLocation() -32);
			sprite->setRealLength(sprite->getNewLength());
		    }
		}
		
		Global::debug(2) << "Referenced Sprite Location: " << temp->getLocation() << " | Group: " << temp->getGroupNumber() << " | Sprite: " << temp->getGroupNumber() << " | at index: " << sprite->getPrevious() << endl;
	    }
	}
	/*
	// Read in pcx header
	pcx_header pcxhead;
	ifile.read( (char *)&pcxhead.manufacturer, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.version, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.encoding, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.BitsPerPixel, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.Xmin, sizeof(short) );
	ifile.read( (char *)&pcxhead.Ymin, sizeof(short) );
	ifile.read( (char *)&pcxhead.Xmax, sizeof(short));
	ifile.read( (char *)&pcxhead.Ymax, sizeof(short) );
	ifile.read( (char *)&pcxhead.HDpi, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.VDpi, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.colormap, sizeof(unsigned char) * 48);
	ifile.read( (char *)&pcxhead.reserved, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.NPlanes, sizeof(unsigned char) );
	ifile.read( (char *)&pcxhead.BytesPerLine, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.PaletteInfo, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.HscreenSize, sizeof(unsigned short));
	ifile.read( (char *)&pcxhead.VscreenSize, sizeof(unsigned short) );
	ifile.read( (char *)&pcxhead.filler, sizeof(char) * 54 );
	
	// Now read in the pcx
	// Seek to the location of the pcx data
	ifile.seekg(sprite->location + 32, ios::beg);
	if( sprite->samePalette ){
	    // Lets give it space for the palette
	    Global::debug(1) << "This sprite is less that 768 or has a shared palette - Group: " << sprite->groupNumber << " | Image: " << sprite->imageNumber << endl;
	    sprite->newlength += 768;
	    sprite->pcx = new char[sprite->newlength];
	}
	else sprite->pcx = new char[sprite->reallength];
	ifile.read((char *)sprite->pcx, sprite->reallength);
	
	if ( !islinked ){
	    if ( !useact){
		if ( sprite->samePalette ){
		    memcpy( sprite->pcx + (sprite->reallength), palsave1, 768);
		    Global::debug(1) << "Applying 1st palette to Sprite: " << sprite->imageNumber << " in Group: " << sprite->groupNumber << endl;
		} else {
		    memcpy( palsave1, sprite->pcx+(sprite->reallength)-768, 768);
		}
	    } else {
		// Replace all palettes with the one supplied in act
		if ( sprite->samePalette){
		    memcpy( sprite->pcx + (sprite->reallength), palsave1, 768);
		} else {
		    if (!(sprite->groupNumber == 9000 && sprite->imageNumber == 1)){
			memcpy( sprite->pcx + (sprite->reallength)-768, palsave1, 768);
		    } 
		}
	    }
	}
	*/
	
        try{
            sprite->loadPCX(ifile, islinked, useact, palsave1);
            
            // Add to our lists
            spriteIndex[i] = sprite;
            // Check if the sprite exists already so we can delete it and overwrite
            Mugen::SpriteMap::iterator first_it = sprites.find(sprite->getGroupNumber());
            if (first_it != sprites.end()){
                std::map< unsigned int, MugenSprite * >::iterator it = first_it->second.find(sprite->getImageNumber());
                if (it != first_it->second.end()){
                    delete it->second;
                }
            }
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = sprite;
            location = sprite->getNext();
        } catch (const LoadException & le){
            /* ignore this error?? */
            location = sprite->getNext();
            Global::debug(0) << "Could not load sprite " << sprite->getGroupNumber() << ", " << sprite->getImageNumber() << endl;
            sprites[sprite->getGroupNumber()][sprite->getImageNumber()] = 0;
            spriteIndex[i] = 0;
            delete sprite;
        }
	/*
	// Dump to file just so we can test the pcx in something else
	if( Global::getDebug() == 3 ){
	    std::ostringstream st;
	    st << "pcxdump/g" << sprite->getGroupNumber() << "i" << sprite->getImageNumber() << ".pcx";
	    FILE *pcx;
	    if( (pcx = fopen( st.str().c_str(), "wb" )) != NULL ){
		size_t bleh = fwrite( sprite->pcx, sprite->newlength, 1, pcx );
		bleh = bleh;
		fclose( pcx );
	    }
	}
	*/
	// Set the next file location
	
	if( !location ){
	    Global::debug(1) << "End of Sprites or File. Continuing...." << endl;
	    break;
	}
	
	Global::debug(2) << "Index: " << i << ", Location: " << sprite->getLocation()  << ", Next Sprite: "  << sprite->getNext() << ", Length: " << sprite->getRealLength() << ", x|y: " << sprite->getX() << "|" << sprite->getY() << ", Group|Image Number: " << sprite->getGroupNumber() << "|" << sprite->getImageNumber() << ", Prev: " << sprite->getPrevious() << ", Same Pal: " << sprite->getSamePalette() << ", Comments: " << sprite->getComments() << endl;
	
    }

    ifile.close();
    
    Global::debug(1) << "Got Total Sprites: " << totalImages << endl;
    
    //return sprites;
}
#endif

void Mugen::Util::readSounds(const string & filename, std::map<unsigned int,std::map<unsigned int, MugenSound *> > & sounds) throw (MugenException){
    /* 16 skips the header stuff */
    int location = 16;
    ifstream ifile;
    ifile.open(filename.c_str(), ios::binary);
    if (!ifile){
{'fraction_non_alphanumeric': 0.15948471549636803, 'fraction_numerical': 0.012238952784503631, 'mean_word_length': 2.56689831995142, 'pattern_counts': {'":': 2, '<': 836, '<?xml version=': 0, '>': 359, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 4, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7555490', 'n_tokens_mistral': 17737, 'n_tokens_neox': 16377, 'n_words': 4421}