<commit_before>/*
 * tokenizer.c - simple tokenizer for shell input
 *
 * This source code is licensed under the MIT License.
 * See the file COPYING for more details.
 *
 * @author: <NAME> <<EMAIL>>
 */

#include <pwd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#include "parser.h"

static inline const char _getc(char* input);
static token_t *_init_token(tokenizer_t *t);
static void _append_token(token_t *token, const char *c);
static const token_t *_scan_word(tokenizer_t *t);
static const token_t *_scan_num(tokenizer_t *t);
static const token_t *_scan_env(tokenizer_t *t);
static const token_t *_scan_home(tokenizer_t *t);
static const token_t *_scan_env_assignment(tokenizer_t *t);
static const token_t *_scan_redirect_in(tokenizer_t *t);
static const token_t *_scan_redirect_out(tokenizer_t *t);
// static const token_t *_next_token(tokenizer_t *t);

/*
 * _getc - get a character from input
 */
static inline const char _getc(char *input)
{
    const char c = input[0];
    int index;
    char *substring;
    /*
    index = strspn(input, &c);
    substring = input + index;
    */
    if (c != '\0' || c != '\n') {
        substring = input + 1;
        strncpy(input, substring, ELEMENT_MAX);
    }

    return c;
}

/**
 * init_tokenizer - Initialize and set up scanninig from input.
 * @input: input from prompt
 */
tokenizer_t *init_tokenizer(const char *input)
{
    int i = 0;
    tokenizer_t *t = (tokenizer_t *) malloc(sizeof(tokenizer_t));

    // t->p = 0;
    // for (i = 0; i < PIPE_MAX; i++)
    //     t->command[i].command_flag = false;
    strncpy(t->input, input, INPUT_MAX);
    // Read first character from input.
    t->c = _getc(t->input);
    // _append_token(&(t->token), &(t->c));
    next_token(t);
    
    return t;
}
/*
 * _init_token - Initialize token
 */
static token_t *_init_token(tokenizer_t *t)
{
    t->token.spec = ERROR;
    memset((void *) t->token.element, '\0', sizeof(char) * ELEMENT_MAX);
    
    return &(t->token);
}

/**
 * current_token - Get current token.
 * @t: Token information and next character.
 */
const token_t *current_token(tokenizer_t *t)
{
    return &(t->token);
}

/*
 * _append_token - append a character to the tail of token's element
 */
static void _append_token(token_t *token, const char *c)
{
    if (strlen(token->element) < ELEMENT_MAX) {
        strncat(token->element, c, 1);
    } else {
        fprintf(stderr, "error: length of command element is too long.");
        exit(-1);
    }
}

/*
 * _scan_word - Parse <word>
 */
static const token_t *_scan_word(tokenizer_t *t)
{
    switch (t->c) {
    case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
    case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
    case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':
    case 'v': case 'w': case 'x': case 'y': case 'z': 
    case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
    case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N':
    case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':
    case 'V': case 'W': case 'X': case 'Y': case 'Z': 
    case '!': case '"': case '#': case '%': case '\'': case '(': case ')':
    case '*': case '+': case ',': case '-': case '.': case '/': case ':':
    case ';': case '?': case '@': case '[': case ']': case '&': case '\\':
    case '^': case '_': case '`': case '{': case '|': case '}':
        _append_token(&(t->token), &(t->c));
        t->c = _getc(t->input);
        _scan_word(t);
        break;
    case '$':
        _scan_env(t);
        t->c = _getc(t->input);
        break;
    case '~':
        _scan_home(t);
        t->c = _getc(t->input);
        break;<commit_msg>[Fix] not to expand ~ in the middle of a word<commit_after>/*
 * tokenizer.c - simple tokenizer for shell input
 *
 * This source code is licensed under the MIT License.
 * See the file COPYING for more details.
 *
 * @author: <NAME> <<EMAIL>>
 */

#include <pwd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>

#include "parser.h"

static inline const char _getc(char* input);
static token_t *_init_token(tokenizer_t *t);
static void _append_token(token_t *token, const char *c);
static const token_t *_scan_word(tokenizer_t *t);
static const token_t *_scan_num(tokenizer_t *t);
static const token_t *_scan_env(tokenizer_t *t);
static const token_t *_scan_home(tokenizer_t *t);
static const token_t *_scan_env_assignment(tokenizer_t *t);
static const token_t *_scan_redirect_in(tokenizer_t *t);
static const token_t *_scan_redirect_out(tokenizer_t *t);
// static const token_t *_next_token(tokenizer_t *t);

/*
 * _getc - get a character from input
 */
static inline const char _getc(char *input)
{
    const char c = input[0];
    int index;
    char *substring;
    /*
    index = strspn(input, &c);
    substring = input + index;
    */
    if (c != '\0' || c != '\n') {
        substring = input + 1;
        strncpy(input, substring, ELEMENT_MAX);
    }

    return c;
}

/**
 * init_tokenizer - Initialize and set up scanninig from input.
 * @input: input from prompt
 */
tokenizer_t *init_tokenizer(const char *input)
{
    int i = 0;
    tokenizer_t *t = (tokenizer_t *) malloc(sizeof(tokenizer_t));

    // t->p = 0;
    // for (i = 0; i < PIPE_MAX; i++)
    //     t->command[i].command_flag = false;
    strncpy(t->input, input, INPUT_MAX);
    // Read first character from input.
    t->c = _getc(t->input);
    // _append_token(&(t->token), &(t->c));
    next_token(t);
    
    return t;
}
/*
 * _init_token - Initialize token
 */
static token_t *_init_token(tokenizer_t *t)
{
    t->token.spec = ERROR;
    memset((void *) t->token.element, '\0', sizeof(char) * ELEMENT_MAX);
    
    return &(t->token);
}

/**
 * current_token - Get current token.
 * @t: Token information and next character.
 */
const token_t *current_token(tokenizer_t *t)
{
    return &(t->token);
}

/*
 * _append_token - append a character to the tail of token's element
 */
static void _append_token(token_t *token, const char *c)
{
    if (strlen(token->element) < ELEMENT_MAX) {
        strncat(token->element, c, 1);
    } else {
        fprintf(stderr, "error: length of command element is too long.");
        exit(-1);
    }
}

/*
 * _scan_word - Parse <word>
 */
static const token_t *_scan_word(tokenizer_t *t)
{
    switch (t->c) {
    case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g':
    case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n':
    case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u':
    case 'v': case 'w': case 'x': case 'y': case 'z': 
    case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G':
    case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N':
    case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U':
    case 'V': case 'W': case 'X': case 'Y': case 'Z': 
    case '!': case '"': case '#': case '%': case '\'': case '(': case ')':
    case '*': case '+': case ',': case '-': case '.': case '/': case ':':
    case ';': case '?': case '@': case '[': case ']': case '&': case '\\':
    case '^': case '_': case '`': case '{': case '|': case '}': case '~':
        _append_token(&(t->token), &(t->c));
        t->c = _getc(t->input);
        _scan_word(t);
        break;
    case '$':
        _scan_env(t);
        t->c = _getc(t->input);
        break;
{'fraction_non_alphanumeric': 0.18738428987040465, 'fraction_numerical': 0.0023803226659613857, 'mean_word_length': 3.0400641025641026, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3247621', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2777, 'n_words': 822}