<commit_before>package CustomOreGen.Util;

import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.regex.Pattern;

import cpw.mods.fml.common.FMLLog;

import net.minecraft.block.Block;
import net.minecraft.item.ItemStack;
import net.minecraftforge.oredict.OreDictionary;
import CustomOreGen.Server.DistributionSettingMap.Copyable;

public class BlockDescriptor implements Copyable<BlockDescriptor>
{
    protected LinkedList<Descriptor> _descriptors = new LinkedList();
    protected Map<Integer,Float> _matches = new Hashtable();
    protected boolean _compiled = false;
    protected float[] _fastMatch = new float[256];

    public BlockDescriptor()
    {
        this.clear();
    }

    public BlockDescriptor(String descriptor)
    {
        this.set(descriptor);
    }

    public void copyFrom(BlockDescriptor source)
    {
        this._descriptors = new LinkedList(source._descriptors);
        this._matches = new Hashtable(source._matches);
        this._compiled = source._compiled;
        this._fastMatch = (float[])source._fastMatch.clone();
    }

    public BlockDescriptor set(String descriptor)
    {
        this.clear();

        if (descriptor != null)
        {
            this._descriptors.add(new Descriptor(descriptor, 1.0F, false));
        }

        return this;
    }

    public BlockDescriptor add(String descriptor)
    {
        return this.add(descriptor, 1.0F, false);
    }

    public BlockDescriptor add(String descriptor, float weight) {
    	return this.add(descriptor, weight, false);
    }
    
    public BlockDescriptor add(String descriptor, float weight, boolean describesOre)
    {
        if (descriptor != null && weight != 0.0F)
        {
            this._compiled = false;
            this._descriptors.add(new Descriptor(descriptor, weight, describesOre));
        }

        return this;
    }

    public BlockDescriptor clear()
    {
        this._compiled = false;
        this._descriptors.clear();
        return this;
    }

    public List getDescriptors()
    {
        return Collections.unmodifiableList(this._descriptors);
    }

    protected void add(int blockID, int metaData, float weight)
    {
        if (weight != 0.0F)
        {
            Integer key = Integer.valueOf(blockID << Short.SIZE | metaData & Short.MAX_VALUE);
            Float currentValue = this._matches.get(key);

            if (currentValue != null)
            {
                weight += currentValue;
            }

            this._matches.put(key, weight);

            if (blockID >= 0 && blockID < this._fastMatch.length)
            {
                if (metaData == OreDictionary.WILDCARD_VALUE && !Float.isNaN(this._fastMatch[blockID]))
                {
                    this._fastMatch[blockID] += weight;
                }
                else
                {
                    this._fastMatch[blockID] = Float.NaN;
                }
            }
        }
    }

    protected float[] regexMatch(String id, String name, boolean describesOre)
    {
        float[] weights = new float[Short.SIZE + 1];
        
        for (Descriptor desc : this._descriptors) {
        	if (desc.describesOre != describesOre)
        		continue;
        	if ((id == null || !desc.getPattern().matcher(id).matches()) && (name == null || !desc.getPattern().matcher(name).matches()))
            {
                for (int m = 0; m < Short.SIZE; ++m)
                {
                    if (id != null && desc.getPattern().matcher(id + ":" + m).matches() || name != null && desc.getPattern().matcher(name + ":" + m).matches())
                    {
                        ++desc.matches;
                        weights[m] += desc.weight;
                    }
                }
            }
            else
            {
                ++desc.matches;
                weights[Short.SIZE] += desc.weight;
            }
        }

        return weights;
    }

    protected void compileMatches()
    {
        if (!this._compiled)
        {
            this._compiled = true;
            this._matches.clear();
            Arrays.fill(this._fastMatch, 0.0F);
            
            for (Descriptor desc : this._descriptors) {
            	desc.matches = 0;
            }
            
            float[] var10 = this.regexMatch("0", "air", false);
            this.add(0, OreDictionary.WILDCARD_VALUE, var10[Short.SIZE]);
            
            for (Block block : Block.blocksList) {
            	if (block != null && block.blockID != 0)
                {
                    String id = Integer.toString(block.blockID);
                    String name = block.getUnlocalizedName() == null ? null : block.getUnlocalizedName().replace("tile.", "");
                    float[] weights = this.regexMatch(id, name, false);
                    this.add(block.blockID, OreDictionary.WILDCARD_VALUE, weights[Short.SIZE]);

                    for (int m = 0; m < Short.SIZE; ++m)
                    {
                        this.add(block.blockID, m, weights[m]);
                    }
                }
            }
            
            for (String oreName : OreDictionary.getOreNames()) {
            	float[] weights = this.regexMatch(Integer.toString(OreDictionary.getOreID(oreName)), oreName, true);
            	for (ItemStack ore : OreDictionary.getOres(oreName)) {
            		boolean isBlock = ore.itemID < Block.blocksList.length;
            		if (isBlock) {
            			this.add(ore.itemID, ore.getItemDamage(), weights[Short.SIZE]);
                	}
            	}
            }
        }
    }

    public Map getMatches()
    {
        this.compileMatches();
        return Collections.unmodifiableMap(this._matches);
    }

    public float getWeight_fast(int blockID)
    {
        this.compileMatches();
        return blockID >= 0 && blockID < this._fastMatch.length ? this._fastMatch[blockID] : Float.NaN;
    }

    public float getWeight(int blockID, int metaData)<commit_msg>optimize ore dictionary processing for <ReplacesOre>; should be massive speed-up when used with GT<commit_after>package CustomOreGen.Util;

import java.util.Arrays;
import java.util.Collections;
import java.util.Hashtable;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.regex.Pattern;

import cpw.mods.fml.common.FMLLog;

import net.minecraft.block.Block;
import net.minecraft.item.ItemStack;
import net.minecraftforge.oredict.OreDictionary;
import CustomOreGen.CustomOreGenBase;
import CustomOreGen.Server.DistributionSettingMap.Copyable;

public class BlockDescriptor implements Copyable<BlockDescriptor>
{
    protected LinkedList<Descriptor> _descriptors = new LinkedList();
    protected Map<Integer,Float> _matches = new Hashtable();
    protected boolean _compiled = false;
    protected float[] _fastMatch = new float[256];

    public BlockDescriptor()
    {
        this.clear();
    }

    public BlockDescriptor(String descriptor)
    {
        this.set(descriptor);
    }

    public void copyFrom(BlockDescriptor source)
    {
        this._descriptors = new LinkedList(source._descriptors);
        this._matches = new Hashtable(source._matches);
        this._compiled = source._compiled;
        this._fastMatch = (float[])source._fastMatch.clone();
    }

    public BlockDescriptor set(String descriptor)
    {
        this.clear();

        if (descriptor != null)
        {
            this._descriptors.add(new Descriptor(descriptor, 1.0F, false));
        }

        return this;
    }

    public BlockDescriptor add(String descriptor)
    {
        return this.add(descriptor, 1.0F, false);
    }

    public BlockDescriptor add(String descriptor, float weight) {
    	return this.add(descriptor, weight, false);
    }
    
    public BlockDescriptor add(String descriptor, float weight, boolean describesOre)
    {
        if (descriptor != null && weight != 0.0F)
        {
            this._compiled = false;
            this._descriptors.add(new Descriptor(descriptor, weight, describesOre));
        }

        return this;
    }

    public BlockDescriptor clear()
    {
        this._compiled = false;
        this._descriptors.clear();
        return this;
    }

    public List getDescriptors()
    {
        return Collections.unmodifiableList(this._descriptors);
    }

    protected void add(int blockID, int metaData, float weight)
    {
        if (weight != 0.0F)
        {
            Integer key = Integer.valueOf(blockID << Short.SIZE | metaData & Short.MAX_VALUE);
            Float currentValue = this._matches.get(key);

            if (currentValue != null)
            {
                weight += currentValue;
            }

            this._matches.put(key, weight);

            if (blockID >= 0 && blockID < this._fastMatch.length)
            {
                if (metaData == OreDictionary.WILDCARD_VALUE && !Float.isNaN(this._fastMatch[blockID]))
                {
                    this._fastMatch[blockID] += weight;
                }
                else
                {
                    this._fastMatch[blockID] = Float.NaN;
                }
            }
        }
    }

    protected float[] regexMatch(String id, String name)
    {
        float[] weights = new float[Short.SIZE + 1];
        
        for (Descriptor desc : this._descriptors) {
        	if (desc.describesOre)
        		continue;
        	if ((id == null || !desc.getPattern().matcher(id).matches()) && (name == null || !desc.getPattern().matcher(name).matches()))
            {
                for (int m = 0; m < Short.SIZE; ++m)
                {
                    if (id != null && desc.getPattern().matcher(id + ":" + m).matches() || name != null && desc.getPattern().matcher(name + ":" + m).matches())
                    {
                        ++desc.matches;
                        weights[m] += desc.weight;
                    }
                }
            }
            else
            {
                ++desc.matches;
                weights[Short.SIZE] += desc.weight;
            }
        }

        return weights;
    }

    protected void compileMatches()
    {
        if (!this._compiled)
        {
            this._compiled = true;
            this._matches.clear();
            Arrays.fill(this._fastMatch, 0.0F);
            
            for (Descriptor desc : this._descriptors) {
            	desc.matches = 0;
            }
            
            float[] var10 = this.regexMatch("0", "air");
            this.add(0, OreDictionary.WILDCARD_VALUE, var10[Short.SIZE]);
            
            for (Block block : Block.blocksList) {
            	if (block != null && block.blockID != 0)
                {
                    String id = Integer.toString(block.blockID);
                    String name = block.getUnlocalizedName() == null ? null : block.getUnlocalizedName().replace("tile.", "");
                    float[] weights = this.regexMatch(id, name);
                    this.add(block.blockID, OreDictionary.WILDCARD_VALUE, weights[Short.SIZE]);

                    for (int m = 0; m < Short.SIZE; ++m)
                    {
                        this.add(block.blockID, m, weights[m]);
                    }
                }
            }
         
            for (Descriptor desc : this._descriptors) {
        		if (!desc.describesOre)
            		continue;
        		for (ItemStack ore : OreDictionary.getOres(desc.description)) {
        			boolean isBlock = ore.itemID < Block.blocksList.length;
        			if (isBlock) {
        				this.add(ore.itemID, ore.getItemDamage(), desc.weight);
        			}
        		}
        	}
        }
    }

    public Map getMatches()
    {
        this.compileMatches();
        return Collections.unmodifiableMap(this._matches);
    }

    public float getWeight_fast(int blockID)
    {
        this.compileMatches();
        return blockID >= 0 && blockID < this._fastMatch.length ? this._fastMatch[blockID] : Float.NaN;
    }

    public float getWeight(int blockID, int metaData)
{'fraction_non_alphanumeric': 0.1111111111111111, 'fraction_numerical': 0.004242127590145211, 'mean_word_length': 1.7697695436059648, 'pattern_counts': {'":': 4, '<': 24, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7139006', 'n_tokens_mistral': 3464, 'n_tokens_neox': 3336, 'n_words': 689}