<commit_before>/**
 * OWASP Enterprise Security API (ESAPI)
 * 
 * This file is part of the Open Web Application Security Project (OWASP)
 * Enterprise Security API (ESAPI) project. For details, please see
 * <a href="http://www.owasp.org/index.php/ESAPI">http://www.owasp.org/index.php/ESAPI</a>.
 *
 * Copyright (c) 2007 - The OWASP Foundation
 * 
 * The ESAPI is published by OWASP under the BSD license. You should read and accept the
 * LICENSE before you use, modify, and/or redistribute this software.
 * 
 * @author <NAME> <a href="http://www.aspectsecurity.com">Aspect Security</a>
 * @created 2007
 */
package org.owasp.esapi.codecs;

import java.util.HashMap;
import java.util.Map;

/**
 * Implementation of the Codec interface for HTML entity encoding.
 * 
 * @author <NAME> (jeff.williams .at. aspectsecurity.com) <a
 *         href="http://www.aspectsecurity.com">Aspect Security</a>
 * @since June 1, 2007
 * @see org.owasp.esapi.Encoder
 */
public class HTMLEntityCodec extends Codec {
	
	private static HashMap<Character,String> characterToEntityMap;

	private static HashTrie<Character> entityToCharacterMap;

	static {
		initializeMaps();
	}
	
    /**
     *
     */
    public HTMLEntityCodec() {
	}

	/**
	 * {@inheritDoc}
	 * 
     * Encodes a Character for safe use in an HTML entity field.
     * @param immune
     */
	public String encodeCharacter( char[] immune, Character c ) {

		// check for immune characters
		if ( containsCharacter(c, immune ) ) {
			return ""+c;
		}
		
		// check for alphanumeric characters
		String hex = Codec.getHexForNonAlphanumeric(c);
		if ( hex == null ) {
			return ""+c;
		}
		
		// check for illegal characters
		if ( ( c <= 0x1f && c != '\t' && c != '\n' && c != '\r' ) || ( c >= 0x7f && c <= 0x9f ) ) {
			return( " " );
		}
		
		// check if there's a defined entity
		String entityName = (String) characterToEntityMap.get(c);
		if (entityName != null) {
			return "&" + entityName + ";";
		}
		
		// return the hex entity as suggested in the spec
		return "&#x" + hex + ";";
	}
	
	/**
	 * {@inheritDoc}
	 * 
	 * Returns the decoded version of the character starting at index, or
	 * null if no decoding is possible.
	 * 
	 * Formats all are legal both with and without semi-colon, upper/lower case:
	 *   &#dddd;
	 *   &#xhhhh;
	 *   &name;
	 */
	public Character decodeCharacter( PushbackString input ) {
		input.mark();
		Character first = input.next();
		if ( first == null ) {
			input.reset();
			return null;
		}
		
		// if this is not an encoded character, return null
		if (first != '&' ) {
			input.reset();
			return null;
		}
		
		// test for numeric encodings
		Character second = input.next();
		if ( second == null ) {
			input.reset();
			return null;
		}
		
		if (second == '#' ) {
			// handle numbers
			Character c = getNumericEntity( input );
			if ( c != null ) return c;
		} else if ( Character.isLetter( second.charValue() ) ) {
			// handle entities
			input.pushback( second );
			Character c = getNamedEntity( input );
			if ( c != null ) return c;
		}
		input.reset();
		return null;
	}
	
	/**
	 * getNumericEntry checks input to see if it is a numeric entity
	 * 
	 * @param input
	 * 			The input to test for being a numeric entity
	 *  
	 * @return
	 * 			null if input is null, the character of input after decoding
	 */
	private Character getNumericEntity( PushbackString input ) {
		Character first = input.peek();
		if ( first == null ) return null;

		if (first == 'x' || first == 'X' ) {
			input.next();
			return parseHex( input );
		}
		return parseNumber( input );
	}

	/**
	 * Parse a decimal number, such as those from JavaScript's String.fromCharCode(value)
	 * 
	 * @param input
	 * 			decimal encoded string, such as 65
	 * @return
	 * 			character representation of this decimal value, e.g. A 
	 * @throws NumberFormatException
	 */
	private Character parseNumber( PushbackString input ) {
		StringBuilder sb = new StringBuilder();
		while( input.hasNext() ) {
			Character c = input.peek();
			
			// if character is a digit then add it on and keep going
			if ( Character.isDigit( c.charValue() ) ) {
				sb.append( c );
				input.next();
				
			// if character is a semi-colon, eat it and quit
			} else if (c == ';' ) {
				input.next();
				break;
				
			// otherwise just quit
			} else {
				break;
			}
		}
		try {
			int i = Integer.parseInt(sb.toString());
            if (Character.isValidCodePoint(i)) {
                return (char) i;
            }
		} catch( NumberFormatException e ) {
			// throw an exception for malformed entity?
		}
			return null;
		}
	
	/**
	 * Parse a hex encoded entity
	 * 
	 * @param input
	 * 			Hex encoded input (such as 437ae;)
	 * @return
	 * 			A single character from the string
	 * @throws NumberFormatException
	 */
	private Character parseHex( PushbackString input ) {
		StringBuilder sb = new StringBuilder();
		while( input.hasNext() ) {
			Character c = input.peek();
			
			// if character is a hex digit then add it on and keep going
			if ( "0123456789ABCDEFabcdef".indexOf(c) != -1 ) {
				sb.append( c );
				input.next();
				
			// if character is a semi-colon, eat it and quit
			} else if (c == ';' ) {
				input.next();
				break;
				
			// otherwise just quit
			} else {
				break;
			}
		}
		try {
			int i = Integer.parseInt(sb.toString(), 16);
            if (Character.isValidCodePoint(i)) {
                return (char) i;
            }
		} catch( NumberFormatException e ) {
			// throw an exception for malformed entity?
		}
			return null;
		}
	
	/**
	 * 
	 * Returns the decoded version of the character starting at index, or
	 * null if no decoding is possible.
	 * 
	 * Formats all are legal both with and without semi-colon, upper/lower case:
	 *   &aa;
	 *   &aaa;
	 *   &aaaa;
	 *   &aaaaa;
	 *   &aaaaaa;
	 *   &aaaaaaa;
	 *
	 * @param input
	 * 		A string containing a named entity like &quot;
	 * @return
	 * 		Returns the decoded version of the character starting at index, or null if no decoding is possible.
	 */
	private Character getNamedEntity( PushbackString input ) {
		StringBuilder possible = new StringBuilder();
		Map.Entry<CharSequence,Character> entry;
		int len;
		
		// kludge around PushbackString....
		len = Math.min(input.remainder().length(), entityToCharacterMap.getMaxKeyLength());
		for(int i=0;i<len;i++)
			possible.append(Character.toLowerCase(input.next()));

		// look up the longest match
		entry = entityToCharacterMap.getLongestMatch(possible);
		if(entry == null)
			return null;	// no match, caller will reset input

		// fixup input
		input.reset();
		input.next();	// read &
		len = entry.getKey().length();	// what matched's length
		for(int i=0;i<len;i++)
			input.next();

		// check for a trailing semicolen
		if(input.peek(';'))
			input.next();

		return entry.getValue();
	}

	/**
	 * Initialize the entityNames array with all possible named entities
	 */
	private static void initializeMaps() {
		String[] entityNames = { "quot"
		/* 34 : quotation mark */, "amp"
		/* 38 : ampersand */, "lt"
		/* 60 : less-than sign */, "gt"
		/* 62 : greater-than sign */, "nbsp"
		/* 160 : no-break space */, "iexcl"
		/* 161 : inverted exclamation mark */, "cent"
		/* 162 : cent sign */, "pound"
		/* 163 : pound sign */, "curren"
		/* 164 : currency sign */, "yen"<commit_msg>synchronizing the loading of one of the codecs static hashes<commit_after>/**
 * OWASP Enterprise Security API (ESAPI)
 * 
 * This file is part of the Open Web Application Security Project (OWASP)
 * Enterprise Security API (ESAPI) project. For details, please see
 * <a href="http://www.owasp.org/index.php/ESAPI">http://www.owasp.org/index.php/ESAPI</a>.
 *
 * Copyright (c) 2007 - The OWASP Foundation
 * 
 * The ESAPI is published by OWASP under the BSD license. You should read and accept the
 * LICENSE before you use, modify, and/or redistribute this software.
 * 
 * @author <NAME> <a href="http://www.aspectsecurity.com">Aspect Security</a>
 * @created 2007
 */
package org.owasp.esapi.codecs;

import java.util.HashMap;
import java.util.Map;

/**
 * Implementation of the Codec interface for HTML entity encoding.
 * 
 * @author <NAME> (jeff.williams .at. aspectsecurity.com) <a
 *         href="http://www.aspectsecurity.com">Aspect Security</a>
 * @since June 1, 2007
 * @see org.owasp.esapi.Encoder
 */
public class HTMLEntityCodec extends Codec {
	
	private static HashMap<Character,String> characterToEntityMap;

	private static HashTrie<Character> entityToCharacterMap;

	static {
		initializeMaps();
	}
	
    /**
     *
     */
    public HTMLEntityCodec() {
	}

	/**
	 * {@inheritDoc}
	 * 
     * Encodes a Character for safe use in an HTML entity field.
     * @param immune
     */
	public String encodeCharacter( char[] immune, Character c ) {

		// check for immune characters
		if ( containsCharacter(c, immune ) ) {
			return ""+c;
		}
		
		// check for alphanumeric characters
		String hex = Codec.getHexForNonAlphanumeric(c);
		if ( hex == null ) {
			return ""+c;
		}
		
		// check for illegal characters
		if ( ( c <= 0x1f && c != '\t' && c != '\n' && c != '\r' ) || ( c >= 0x7f && c <= 0x9f ) ) {
			return( " " );
		}
		
		// check if there's a defined entity
		String entityName = (String) characterToEntityMap.get(c);
		if (entityName != null) {
			return "&" + entityName + ";";
		}
		
		// return the hex entity as suggested in the spec
		return "&#x" + hex + ";";
	}
	
	/**
	 * {@inheritDoc}
	 * 
	 * Returns the decoded version of the character starting at index, or
	 * null if no decoding is possible.
	 * 
	 * Formats all are legal both with and without semi-colon, upper/lower case:
	 *   &#dddd;
	 *   &#xhhhh;
	 *   &name;
	 */
	public Character decodeCharacter( PushbackString input ) {
		input.mark();
		Character first = input.next();
		if ( first == null ) {
			input.reset();
			return null;
		}
		
		// if this is not an encoded character, return null
		if (first != '&' ) {
			input.reset();
			return null;
		}
		
		// test for numeric encodings
		Character second = input.next();
		if ( second == null ) {
			input.reset();
			return null;
		}
		
		if (second == '#' ) {
			// handle numbers
			Character c = getNumericEntity( input );
			if ( c != null ) return c;
		} else if ( Character.isLetter( second.charValue() ) ) {
			// handle entities
			input.pushback( second );
			Character c = getNamedEntity( input );
			if ( c != null ) return c;
		}
		input.reset();
		return null;
	}
	
	/**
	 * getNumericEntry checks input to see if it is a numeric entity
	 * 
	 * @param input
	 * 			The input to test for being a numeric entity
	 *  
	 * @return
	 * 			null if input is null, the character of input after decoding
	 */
	private Character getNumericEntity( PushbackString input ) {
		Character first = input.peek();
		if ( first == null ) return null;

		if (first == 'x' || first == 'X' ) {
			input.next();
			return parseHex( input );
		}
		return parseNumber( input );
	}

	/**
	 * Parse a decimal number, such as those from JavaScript's String.fromCharCode(value)
	 * 
	 * @param input
	 * 			decimal encoded string, such as 65
	 * @return
	 * 			character representation of this decimal value, e.g. A 
	 * @throws NumberFormatException
	 */
	private Character parseNumber( PushbackString input ) {
		StringBuilder sb = new StringBuilder();
		while( input.hasNext() ) {
			Character c = input.peek();
			
			// if character is a digit then add it on and keep going
			if ( Character.isDigit( c.charValue() ) ) {
				sb.append( c );
				input.next();
				
			// if character is a semi-colon, eat it and quit
			} else if (c == ';' ) {
				input.next();
				break;
				
			// otherwise just quit
			} else {
				break;
			}
		}
		try {
			int i = Integer.parseInt(sb.toString());
            if (Character.isValidCodePoint(i)) {
                return (char) i;
            }
		} catch( NumberFormatException e ) {
			// throw an exception for malformed entity?
		}
			return null;
		}
	
	/**
	 * Parse a hex encoded entity
	 * 
	 * @param input
	 * 			Hex encoded input (such as 437ae;)
	 * @return
	 * 			A single character from the string
	 * @throws NumberFormatException
	 */
	private Character parseHex( PushbackString input ) {
		StringBuilder sb = new StringBuilder();
		while( input.hasNext() ) {
			Character c = input.peek();
			
			// if character is a hex digit then add it on and keep going
			if ( "0123456789ABCDEFabcdef".indexOf(c) != -1 ) {
				sb.append( c );
				input.next();
				
			// if character is a semi-colon, eat it and quit
			} else if (c == ';' ) {
				input.next();
				break;
				
			// otherwise just quit
			} else {
				break;
			}
		}
		try {
			int i = Integer.parseInt(sb.toString(), 16);
            if (Character.isValidCodePoint(i)) {
                return (char) i;
            }
		} catch( NumberFormatException e ) {
			// throw an exception for malformed entity?
		}
			return null;
		}
	
	/**
	 * 
	 * Returns the decoded version of the character starting at index, or
	 * null if no decoding is possible.
	 * 
	 * Formats all are legal both with and without semi-colon, upper/lower case:
	 *   &aa;
	 *   &aaa;
	 *   &aaaa;
	 *   &aaaaa;
	 *   &aaaaaa;
	 *   &aaaaaaa;
	 *
	 * @param input
	 * 		A string containing a named entity like &quot;
	 * @return
	 * 		Returns the decoded version of the character starting at index, or null if no decoding is possible.
	 */
	private Character getNamedEntity( PushbackString input ) {
		StringBuilder possible = new StringBuilder();
		Map.Entry<CharSequence,Character> entry;
		int len;
		
		// kludge around PushbackString....
		len = Math.min(input.remainder().length(), entityToCharacterMap.getMaxKeyLength());
		for(int i=0;i<len;i++)
			possible.append(Character.toLowerCase(input.next()));

		// look up the longest match
		entry = entityToCharacterMap.getLongestMatch(possible);
		if(entry == null)
			return null;	// no match, caller will reset input

		// fixup input
		input.reset();
		input.next();	// read &
		len = entry.getKey().length();	// what matched's length
		for(int i=0;i<len;i++)
			input.next();

		// check for a trailing semicolen
		if(input.peek(';'))
			input.next();

		return entry.getValue();
	}

	/**
	 * Initialize the entityNames array with all possible named entities
	 */
	private static synchronized void initializeMaps() {
		String[] entityNames = { "quot"
		/* 34 : quotation mark */, "amp"
		/* 38 : ampersand */, "lt"
		/* 60 : less-than sign */, "gt"
		/* 62 : greater-than sign */, "nbsp"
		/* 160 : no-break space */, "iexcl"
		/* 161 : inverted exclamation mark */, "cent"
		/* 162 : cent sign */, "pound"
		/* 163 : pound sign */, "curren"
		/* 164 : currency sign */, "yen"
{'fraction_non_alphanumeric': 0.13119891008174386, 'fraction_numerical': 0.008446866485013624, 'mean_word_length': 2.6583603289309745, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4765209', 'n_tokens_mistral': 5367, 'n_tokens_neox': 4520, 'n_words': 1481}