<commit_before>/*******************************************************************************
 * Copyright (c) 2009 LegSem.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser Public License v2.1
 * which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 * 
 * Contributors:
 *     LegSem - initial API and implementation
 ******************************************************************************/
lexer grammar CobolStructureLexer;
/*------------------------------------------------------------------
 * Lexer grammar for COBOL structures.
 * Built from a subset of IBM Entreprise COBOL V3R4
 *------------------------------------------------------------------*/

/*------------------------------------------------------------------
 * Java overrides
 *------------------------------------------------------------------*/
@header {
package com.legstar.cobol;
import java.util.HashMap;
import java.util.Map;
}

@members {
    /** Keeps track of the last COBOL keyword recognized. This helps
        disambiguate lexing rules. */
    private int lastKeyword = PERIOD;
    
    /** True when a picture string is being built (potentially from multiple parts). */
    private boolean pictureStarted;

    /** Map to help with COBOL keyword recognition.*/
    private static Map <String, Integer> KEYWORDS_MAP = new HashMap <String, Integer>()
    {{
        put("RENAMES", RENAMES_KEYWORD);
        put("THROUGH", THROUGH_KEYWORD);
        put("THRU", THROUGH_KEYWORD);
        put("REDEFINES", REDEFINES_KEYWORD);
        put("BLANK", BLANK_KEYWORD);
        put("WHEN", Token.SKIP_TOKEN.getType());
        put("EXTERNAL", EXTERNAL_KEYWORD);
        put("GLOBAL", GLOBAL_KEYWORD);
        put("GROUP-USAGE", GROUP_USAGE_KEYWORD);
        put("IS", Token.SKIP_TOKEN.getType());
        put("ARE", Token.SKIP_TOKEN.getType());
        put("NATIONAL", NATIONAL_KEYWORD);
        put("JUSTIFIED", JUSTIFIED_KEYWORD);
        put("JUST", JUSTIFIED_KEYWORD);
        put("RIGHT", RIGHT_KEYWORD);
        put("OCCURS", OCCURS_KEYWORD);
        put("TIMES", Token.SKIP_TOKEN.getType());
        put("TO", TO_KEYWORD);
        put("ASCENDING", ASCENDING_KEYWORD);
        put("DESCENDING", DESCENDING_KEYWORD);
        put("KEY", KEY_KEYWORD);
        put("INDEXED", INDEXED_KEYWORD);
        put("BY", Token.SKIP_TOKEN.getType());
        put("PICTURE", PICTURE_KEYWORD);
        put("PIC", PICTURE_KEYWORD);
        put("DEPENDING", DEPENDING_KEYWORD);
        put("ON", Token.SKIP_TOKEN.getType());
        put("SIGN", Token.SKIP_TOKEN.getType());
        put("LEADING", SIGN_LEADING_KEYWORD);
        put("TRAILING", SIGN_TRAILING_KEYWORD);
        put("SEPARATE", SEPARATE_KEYWORD);
        put("CHARACTER", Token.SKIP_TOKEN.getType());
        put("SYNCHRONIZED", SYNCHRONIZED_KEYWORD);
        put("SYNC", SYNCHRONIZED_KEYWORD);
        put("LEFT", LEFT_KEYWORD);
        put("USAGE", USAGE_KEYWORD);
        put("COMPUTATIONAL-1", SINGLE_FLOAT_KEYWORD);
        put("COMP-1", SINGLE_FLOAT_KEYWORD);
        put("COMPUTATIONAL-2", DOUBLE_FLOAT_KEYWORD);
        put("COMP-2", DOUBLE_FLOAT_KEYWORD);
        put("COMPUTATIONAL-5", NATIVE_BINARY_KEYWORD);
        put("COMP-5", NATIVE_BINARY_KEYWORD);
        put("COMPUTATIONAL-3", PACKED_DECIMAL_KEYWORD);
        put("COMP-3", PACKED_DECIMAL_KEYWORD);
        put("PACKED-DECIMAL", PACKED_DECIMAL_KEYWORD);
        put("COMPUTATIONAL", BINARY_KEYWORD);
        put("COMP", BINARY_KEYWORD);
        put("BINARY", BINARY_KEYWORD);
        put("DISPLAY-1", DISPLAY_1_KEYWORD);
        put("DISPLAY", DISPLAY_KEYWORD);
        put("INDEX", INDEX_KEYWORD);
        put("POINTER", POINTER_KEYWORD);
        put("PROCEDURE-POINTER", PROCEDURE_POINTER_KEYWORD);
        put("FUNCTION-POINTER", FUNCTION_POINTER_KEYWORD);
        put("VALUES", VALUE_KEYWORD);
        put("VALUE", VALUE_KEYWORD);
        put("DATE", Token.SKIP_TOKEN.getType());
        put("FORMAT", DATE_FORMAT_KEYWORD);
        put("ZEROES", ZERO_CONSTANT);
        put("ZEROS", ZERO_CONSTANT);
        put("ZERO", ZERO_CONSTANT);<commit_msg>Fixing bug 26. COMP-4 not supported.<commit_after>/*******************************************************************************
 * Copyright (c) 2009 LegSem.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the GNU Lesser Public License v2.1
 * which accompanies this distribution, and is available at
 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 * 
 * Contributors:
 *     LegSem - initial API and implementation
 ******************************************************************************/
lexer grammar CobolStructureLexer;
/*------------------------------------------------------------------
 * Lexer grammar for COBOL structures.
 * Built from a subset of IBM Entreprise COBOL V3R4
 *------------------------------------------------------------------*/

/*------------------------------------------------------------------
 * Java overrides
 *------------------------------------------------------------------*/
@header {
package com.legstar.cobol;
import java.util.HashMap;
import java.util.Map;
}

@members {
    /** Keeps track of the last COBOL keyword recognized. This helps
        disambiguate lexing rules. */
    private int lastKeyword = PERIOD;
    
    /** True when a picture string is being built (potentially from multiple parts). */
    private boolean pictureStarted;

    /** Map to help with COBOL keyword recognition.*/
    private static Map <String, Integer> KEYWORDS_MAP = new HashMap <String, Integer>()
    {{
        put("RENAMES", RENAMES_KEYWORD);
        put("THROUGH", THROUGH_KEYWORD);
        put("THRU", THROUGH_KEYWORD);
        put("REDEFINES", REDEFINES_KEYWORD);
        put("BLANK", BLANK_KEYWORD);
        put("WHEN", Token.SKIP_TOKEN.getType());
        put("EXTERNAL", EXTERNAL_KEYWORD);
        put("GLOBAL", GLOBAL_KEYWORD);
        put("GROUP-USAGE", GROUP_USAGE_KEYWORD);
        put("IS", Token.SKIP_TOKEN.getType());
        put("ARE", Token.SKIP_TOKEN.getType());
        put("NATIONAL", NATIONAL_KEYWORD);
        put("JUSTIFIED", JUSTIFIED_KEYWORD);
        put("JUST", JUSTIFIED_KEYWORD);
        put("RIGHT", RIGHT_KEYWORD);
        put("OCCURS", OCCURS_KEYWORD);
        put("TIMES", Token.SKIP_TOKEN.getType());
        put("TO", TO_KEYWORD);
        put("ASCENDING", ASCENDING_KEYWORD);
        put("DESCENDING", DESCENDING_KEYWORD);
        put("KEY", KEY_KEYWORD);
        put("INDEXED", INDEXED_KEYWORD);
        put("BY", Token.SKIP_TOKEN.getType());
        put("PICTURE", PICTURE_KEYWORD);
        put("PIC", PICTURE_KEYWORD);
        put("DEPENDING", DEPENDING_KEYWORD);
        put("ON", Token.SKIP_TOKEN.getType());
        put("SIGN", Token.SKIP_TOKEN.getType());
        put("LEADING", SIGN_LEADING_KEYWORD);
        put("TRAILING", SIGN_TRAILING_KEYWORD);
        put("SEPARATE", SEPARATE_KEYWORD);
        put("CHARACTER", Token.SKIP_TOKEN.getType());
        put("SYNCHRONIZED", SYNCHRONIZED_KEYWORD);
        put("SYNC", SYNCHRONIZED_KEYWORD);
        put("LEFT", LEFT_KEYWORD);
        put("USAGE", USAGE_KEYWORD);
        put("COMPUTATIONAL-1", SINGLE_FLOAT_KEYWORD);
        put("COMP-1", SINGLE_FLOAT_KEYWORD);
        put("COMPUTATIONAL-2", DOUBLE_FLOAT_KEYWORD);
        put("COMP-2", DOUBLE_FLOAT_KEYWORD);
        put("COMPUTATIONAL-5", NATIVE_BINARY_KEYWORD);
        put("COMP-5", NATIVE_BINARY_KEYWORD);
        put("COMPUTATIONAL-3", PACKED_DECIMAL_KEYWORD);
        put("COMP-3", PACKED_DECIMAL_KEYWORD);
        put("PACKED-DECIMAL", PACKED_DECIMAL_KEYWORD);
        put("COMPUTATIONAL", BINARY_KEYWORD);
        put("COMP", BINARY_KEYWORD);
        put("COMPUTATIONAL-4", BINARY_KEYWORD);
        put("COMP-4", BINARY_KEYWORD);
        put("BINARY", BINARY_KEYWORD);
        put("DISPLAY-1", DISPLAY_1_KEYWORD);
        put("DISPLAY", DISPLAY_KEYWORD);
        put("INDEX", INDEX_KEYWORD);
        put("POINTER", POINTER_KEYWORD);
        put("PROCEDURE-POINTER", PROCEDURE_POINTER_KEYWORD);
        put("FUNCTION-POINTER", FUNCTION_POINTER_KEYWORD);
        put("VALUES", VALUE_KEYWORD);
        put("VALUE", VALUE_KEYWORD);
        put("DATE", Token.SKIP_TOKEN.getType());
        put("FORMAT", DATE_FORMAT_KEYWORD);
        put("ZEROES", ZERO_CONSTANT);
        put("ZEROS", ZERO_CONSTANT);
        put("ZERO", ZERO_CONSTANT);
{'fraction_non_alphanumeric': 0.2218948856940844, 'fraction_numerical': 0.005302851755833137, 'mean_word_length': 2.637805400771539, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6983704', 'n_tokens_mistral': 2613, 'n_tokens_neox': 2577, 'n_words': 488}