<commit_before>/*
 * Copyright 2017 <NAME>. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

#include <stdint.h>

#ifndef _TYPES_H
#define _TYPES_H

#define ERROR__INTERNAL -1  // internal error
#define ERROR__STACK    -2
#define ERROR__TOKEN    -3
#define ERROR__CLOSE    -4
#define ERROR__SYNTAX   -5
#define ERROR__TODO     -9

#define __STACK_SIZE      256  // stack size used by token
#define __STACK_SIZE_BITS 8    // bits needed for __STACK_SIZE

typedef struct {
  char *p;
  int len;
  int line_no;
  uint8_t type : 5;
} token;

// empty: will not contain text
#define TOKEN_EOF       0

// fixed: will always be the same, or in the same set
#define TOKEN_SEMICOLON 1   // might be blank for ASI
#define TOKEN_COMMA     2
#define TOKEN_SPREAD    3
#define TOKEN_DOT       4
#define TOKEN_OP        5   // can include 'in', 'instanceof'
#define TOKEN_ARROW     6
#define TOKEN_COLON     7   // used in label
#define TOKEN_TERNARY   8   // starts ternary block, ends with ':'
#define TOKEN_BRACE     9
#define TOKEN_ARRAY     10
#define TOKEN_PAREN     11
#define TOKEN_T_BRACE   12  // '${' within template literal
#define TOKEN_CLOSE     13  // '}', ']', ')' or ':'

// variable: could be anything
#define TOKEN_COMMENT   14
#define TOKEN_STRING    15
#define TOKEN_REGEXP    16
#define TOKEN_NUMBER    17
#define TOKEN_SYMBOL    18
#define TOKEN_KEYWORD   19
#define TOKEN_LABEL     20  // to the left of a ':', e.g. 'foo:'

// literal: internal use except for reporting ambiguous tokens
#define TOKEN_LIT       21  // symbol, keyword or label
#define TOKEN_SLASH     22  // ambigous slash
#define TOKEN_INTERNAL  23<commit_msg>revamp extra types<commit_after>/*
 * Copyright 2017 <NAME>. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

#include <stdint.h>

#ifndef _TYPES_H
#define _TYPES_H

#define ERROR__INTERNAL -1  // internal error
#define ERROR__STACK    -2
#define ERROR__TOKEN    -3
#define ERROR__CLOSE    -4
#define ERROR__SYNTAX   -5
#define ERROR__TODO     -9

#define __STACK_SIZE      256  // stack size used by token
#define __STACK_SIZE_BITS 8    // bits needed for __STACK_SIZE

typedef struct {
  char *p;
  int len;
  int line_no;
  uint8_t type : 5;
  uint8_t flag : 3;
} token;

// empty: will not contain text
#define TOKEN_EOF       0

// fixed: will always be the same, or in the same set
#define TOKEN_SEMICOLON 1   // might be blank for ASI
#define TOKEN_COMMA     2
#define TOKEN_SPREAD    3
#define TOKEN_DOT       4
#define TOKEN_OP        5   // can include 'in', 'instanceof'
#define TOKEN_ARROW     6
#define TOKEN_COLON     7   // used in label
#define TOKEN_BLOCK     8   // block-like {
#define TOKEN_DICT      9   // dict-like {
#define TOKEN_ARRAY     10
#define TOKEN_PAREN     11
#define TOKEN_T_BRACE   12  // '${' within template literal
#define TOKEN_TERNARY   13  // starts ternary block, ends with ':'
#define TOKEN_CLOSE     14  // '}', ']', ')' or ':'

// variable: could be anything
#define TOKEN_COMMENT   15
#define TOKEN_STRING    16
#define TOKEN_REGEXP    17  // literal "/foo/", not "new RegExp('foo')"
#define TOKEN_NUMBER    18
#define TOKEN_SYMBOL    19
#define TOKEN_KEYWORD   20
#define TOKEN_LABEL     21  // to the left of a ':', e.g. 'foo:'

// internal/ambiguous tokens
#define TOKEN_LIT       28  // symbol, keyword or label
#define TOKEN_BRACE     29  // ambigous brace '{'
#define TOKEN_SLASH     30  // ambigous slash that is op or regexp
#define TOKEN_INTERNAL  31
{'fraction_non_alphanumeric': 0.08418254319893664, 'fraction_numerical': 0.02702702702702703, 'mean_word_length': 3.0712353471596034, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '149382', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1447, 'n_words': 562}