<commit_before>#include <errno.h>
#include <limits.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <unistd.h>

#include "config.h"
#include "ignore.h"
#include "options.h"
#include "lang.h"
#include "log.h"
#include "util.h"

const char *color_line_number = "\033[1;33m"; /* yellow with black background */
const char *color_match = "\033[30;43m";      /* black with yellow background */
const char *color_path = "\033[1;32m";        /* bold green */

/* TODO: try to obey out_fd? */
void usage(void) {
    printf("\n");
    printf("Usage: ag [OPTIONS] PATTERN [PATH]\n\n");

    printf("  Recursively search for PATTERN in PATH.\n");
    printf("  Like grep or ack, but faster.\n\n");

    printf("Example:\n  ag -i foo /bar/\n\n");

    printf("\
Output Options:\n\
     --ackmate            Print results in AckMate-parseable format\n\
  -A --after [LINES]      Print lines before match (Default: 2)\n\
  -B --before [LINES]     Print lines after match (Default: 2)\n\
     --[no]break          Print newlines between matches in different files\n\
                          (Enabled by default)\n\
     --[no]color          Print color codes in results (Enabled by default)\n\
     --color-line-number  Color codes for line numbers (Default: 1;33)\n\
     --color-match        Color codes for result match numbers (Default: 30;43)\n\
     --color-path         Color codes for path names (Default: 1;32)\n\
     --column             Print column numbers in results\n\
     --[no]heading\n\
     --line-numbers       Print line numbers even for streams\n\
  -C --context [LINES]    Print lines before and after matches (Default: 2)\n\
     --[no]group          Same as --[no]break --[no]heading\n\
  -g PATTERN              Print filenames matching PATTERN\n\
  -l --files-with-matches Only print filenames that contain matches\n\
                          (don't print the matching lines)\n\
  -L --files-without-matches\n\
                          Only print filenames that don't contain matches\n\
     --no-numbers         Don't print line numbers\n\
     --print-long-lines   Print matches on very long lines (Default: >2k characters)\n\
     --stats              Print stats (files scanned, time taken, etc.)\n\
\n\
Search Options:\n\
  -a --all-types          Search all files (doesn't include hidden files\n\
                          or patterns from ignore files)\n\
  -D --debug              Ridiculous debugging (probably not useful)\n\
     --depth NUM          Search up to NUM directories deep (Default: 25)\n\
  -f --follow             Follow symlinks\n\
  -G --file-search-regex  PATTERN Limit search to filenames matching PATTERN\n\
     --hidden             Search hidden files (obeys .*ignore files)\n\
  -i --ignore-case        Match case insensitively\n\
     --ignore PATTERN     Ignore files/directories matching PATTERN\n\
                          (literal file/directory names also allowed)\n\
     --ignore-dir NAME    Alias for --ignore for compatibility with ack.\n\
  -m --max-count NUM      Skip the rest of a file after NUM matches (Default: 10,000)\n\
  -p --path-to-agignore STRING\n\
                          Use .agignore file at STRING\n\
  -Q --literal            Don't parse PATTERN as a regular expression\n\
  -s --case-sensitive     Match case sensitively (Enabled by default)\n\
  -S --smart-case         Match case insensitively unless PATTERN contains\n\
                          uppercase characters\n\
     --search-binary      Search binary files for matches\n\
  -t --all-text           Search all text files (doesn't include hidden files)\n\
  -u --unrestricted       Search all files (ignore .agignore, .gitignore, etc.;\n\
                          searches binary and hidden files as well)\n\
  -U --skip-vcs-ignores   Ignore VCS ignore files\n\
                          (.gitignore, .hgignore, .svnignore; still obey .agignore)\n\
  -v --invert-match\n\
  -w --word-regexp        Only match whole words\n\
  -z --search-zip         Search contents of compressed (e.g., gzip) files\n\
\n");
}

void print_version(void) {
    printf("ag version %s\n", PACKAGE_VERSION);
}

void init_options(void) {
    memset(&opts, 0, sizeof(opts));
    opts.casing = CASE_SENSITIVE;
#ifdef _WIN32
    opts.color = getenv("ANSICON") ? TRUE : FALSE;
#else
    opts.color = TRUE;
#endif
    opts.max_matches_per_file = 10000;
    opts.max_search_depth = 25;
    opts.print_break = TRUE;
    opts.print_heading = TRUE;
    opts.print_line_numbers = TRUE;
    opts.recurse_dirs = TRUE;
    opts.color_path = ag_strdup(color_path);
    opts.color_match = ag_strdup(color_match);
    opts.color_line_number = ag_strdup(color_line_number);
}

void cleanup_options(void) {
    free(opts.color_path);
    free(opts.color_match);
    free(opts.color_line_number);

    if (opts.query) {
        free(opts.query);
    }

    pcre_free(opts.re);
    if (opts.re_extra) {
        /* Using pcre_free_study on pcre_extra* can segfault on some versions of PCRE */
        pcre_free(opts.re_extra);
    }

    if (opts.ackmate_dir_filter) {
        pcre_free(opts.ackmate_dir_filter);
    }
    if (opts.ackmate_dir_filter_extra) {
        pcre_free(opts.ackmate_dir_filter_extra);
    }

    if (opts.file_search_regex) {
        pcre_free(opts.file_search_regex);
    }
    if (opts.file_search_regex_extra) {
        pcre_free(opts.file_search_regex_extra);
    }
}

void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
    int ch;
    int i;
    int path_len = 0;
    int useless = 0;
    int group = 1;
    int help = 0;
    int version = 0;
    int opt_index = 0;
    const char *home_dir = getenv("HOME");
    char *ignore_file_path = NULL;
    int needs_query = 1;
    struct stat statbuf;
    int rv;

    size_t longopts_len, full_len;
    option_t *longopts;
    char *lang_regex = NULL;

    init_options();

    option_t base_longopts[] = {
        { "ackmate", no_argument, &opts.ackmate, 1 },
        { "ackmate-dir-filter", required_argument, NULL, 0 },
        { "after", required_argument, NULL, 'A' },
        { "all-text", no_argument, NULL, 't' },
        { "all-types", no_argument, NULL, 'a' },
        { "before", required_argument, NULL, 'B' },
        { "break", no_argument, &opts.print_break, 1 },
        { "case-sensitive", no_argument, NULL, 's' },
        { "color", no_argument, &opts.color, 1 },
        { "color-path", required_argument, NULL, 0 },
        { "color-match", required_argument, NULL, 0 },
        { "color-line-number", required_argument, NULL, 0 },
        { "column", no_argument, &opts.column, 1 },
        { "context", optional_argument, NULL, 'C' },
        { "debug", no_argument, NULL, 'D' },
        { "depth", required_argument, NULL, 0 },
        { "file-search-regex", required_argument, NULL, 'G' },
        { "files-with-matches", no_argument, NULL, 'l' },
        { "files-without-matches", no_argument, NULL, 'L' },
        { "follow", no_argument, &opts.follow_symlinks, 1 },
        { "group", no_argument, &group, 1 },
        { "heading", no_argument, &opts.print_heading, 1 },
        { "help", no_argument, NULL, 'h' },
        { "hidden", no_argument, &opts.search_hidden_files, 1 },
        { "ignore", required_argument, NULL, 0 },
        { "ignore-dir", required_argument, NULL, 0 },
        { "ignore-case", no_argument, NULL, 'i' },
        { "invert-match", no_argument, &opts.invert_match, 1 },
        { "line-numbers", no_argument, &opts.print_line_numbers, 2 },
        { "literal", no_argument, NULL, 'Q' },
        { "match", no_argument, &useless, 0 },
        { "max-count", required_argument, NULL, 'm' },
        { "no-numbers", no_argument, NULL, 0 },
        { "no-recurse", no_argument, NULL, 'n' },
        { "nobreak", no_argument, &opts.print_break, 0 },
        { "nocolor", no_argument, &opts.color, 0 },
        { "nofollow", no_argument, &opts.follow_symlinks, 0 },
        { "nogroup", no_argument, &group, 0 },
        { "noheading", no_argument, &opts.print_heading, 0 },
        { "nopager", no_argument, NULL, 0 },
        { "pager", required_argument, NULL, 0 },
        { "parallel", no_argument, &opts.parallel, 1 },
        { "path-to-agignore", required_argument, NULL, 'p' },
        { "print-long-lines", no_argument, &opts.print_long_lines, 1 },
        { "recurse", no_argument, NULL, 'r' },
        { "search-binary", no_argument, &opts.search_binary_files, 1 },
        { "search-zip", no_argument, &opts.search_zip_files, 1 },
        { "search-files", no_argument, &opts.search_stream, 0 },
        { "silent", no_argument, NULL, 0 },
        { "skip-vcs-ignores", no_argument, NULL, 'U' },
        { "smart-case", no_argument, NULL, 'S' },
        { "stats", no_argument, &opts.stats, 1 },
        { "unrestricted", no_argument, NULL, 'u' },
        { "version", no_argument, &version, 1 },
        { "word-regexp", no_argument, NULL, 'w' },
        { "workers", required_argument, NULL, 0 },
    };

    longopts_len = (sizeof(base_longopts) / sizeof(option_t));
    full_len = (longopts_len + LANG_COUNT + 1);
    longopts = ag_malloc(full_len * sizeof(option_t));
    memcpy(longopts, base_longopts, sizeof(base_longopts));

    for (i = 0; i < LANG_COUNT; i++) {
        option_t opt = { langs[i].name, no_argument, NULL, 0 };
        longopts[i + longopts_len] = opt;
    }
    longopts[full_len - 1] = (option_t) { NULL, 0, NULL, 0 };

    if (argc < 2) {
        usage();
        cleanup_ignore(root_ignores);
        cleanup_options();
        exit(1);
    }

    rv = fstat(fileno(stdin), &statbuf);
    if (rv == 0) {
        if (S_ISFIFO(statbuf.st_mode)) {
            opts.search_stream = 1;
        }
    }

    /* If we're not outputting to a terminal. change output to:<commit_msg>Cast const char* to char* to avoid compiler warning<commit_after>#include <errno.h>
#include <limits.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <unistd.h>

#include "config.h"
#include "ignore.h"
#include "options.h"
#include "lang.h"
#include "log.h"
#include "util.h"

const char *color_line_number = "\033[1;33m"; /* yellow with black background */
const char *color_match = "\033[30;43m";      /* black with yellow background */
const char *color_path = "\033[1;32m";        /* bold green */

/* TODO: try to obey out_fd? */
void usage(void) {
    printf("\n");
    printf("Usage: ag [OPTIONS] PATTERN [PATH]\n\n");

    printf("  Recursively search for PATTERN in PATH.\n");
    printf("  Like grep or ack, but faster.\n\n");

    printf("Example:\n  ag -i foo /bar/\n\n");

    printf("\
Output Options:\n\
     --ackmate            Print results in AckMate-parseable format\n\
  -A --after [LINES]      Print lines before match (Default: 2)\n\
  -B --before [LINES]     Print lines after match (Default: 2)\n\
     --[no]break          Print newlines between matches in different files\n\
                          (Enabled by default)\n\
     --[no]color          Print color codes in results (Enabled by default)\n\
     --color-line-number  Color codes for line numbers (Default: 1;33)\n\
     --color-match        Color codes for result match numbers (Default: 30;43)\n\
     --color-path         Color codes for path names (Default: 1;32)\n\
     --column             Print column numbers in results\n\
     --[no]heading\n\
     --line-numbers       Print line numbers even for streams\n\
  -C --context [LINES]    Print lines before and after matches (Default: 2)\n\
     --[no]group          Same as --[no]break --[no]heading\n\
  -g PATTERN              Print filenames matching PATTERN\n\
  -l --files-with-matches Only print filenames that contain matches\n\
                          (don't print the matching lines)\n\
  -L --files-without-matches\n\
                          Only print filenames that don't contain matches\n\
     --no-numbers         Don't print line numbers\n\
     --print-long-lines   Print matches on very long lines (Default: >2k characters)\n\
     --stats              Print stats (files scanned, time taken, etc.)\n\
\n\
Search Options:\n\
  -a --all-types          Search all files (doesn't include hidden files\n\
                          or patterns from ignore files)\n\
  -D --debug              Ridiculous debugging (probably not useful)\n\
     --depth NUM          Search up to NUM directories deep (Default: 25)\n\
  -f --follow             Follow symlinks\n\
  -G --file-search-regex  PATTERN Limit search to filenames matching PATTERN\n\
     --hidden             Search hidden files (obeys .*ignore files)\n\
  -i --ignore-case        Match case insensitively\n\
     --ignore PATTERN     Ignore files/directories matching PATTERN\n\
                          (literal file/directory names also allowed)\n\
     --ignore-dir NAME    Alias for --ignore for compatibility with ack.\n\
  -m --max-count NUM      Skip the rest of a file after NUM matches (Default: 10,000)\n\
  -p --path-to-agignore STRING\n\
                          Use .agignore file at STRING\n\
  -Q --literal            Don't parse PATTERN as a regular expression\n\
  -s --case-sensitive     Match case sensitively (Enabled by default)\n\
  -S --smart-case         Match case insensitively unless PATTERN contains\n\
                          uppercase characters\n\
     --search-binary      Search binary files for matches\n\
  -t --all-text           Search all text files (doesn't include hidden files)\n\
  -u --unrestricted       Search all files (ignore .agignore, .gitignore, etc.;\n\
                          searches binary and hidden files as well)\n\
  -U --skip-vcs-ignores   Ignore VCS ignore files\n\
                          (.gitignore, .hgignore, .svnignore; still obey .agignore)\n\
  -v --invert-match\n\
  -w --word-regexp        Only match whole words\n\
  -z --search-zip         Search contents of compressed (e.g., gzip) files\n\
\n");
}

void print_version(void) {
    printf("ag version %s\n", PACKAGE_VERSION);
}

void init_options(void) {
    memset(&opts, 0, sizeof(opts));
    opts.casing = CASE_SENSITIVE;
#ifdef _WIN32
    opts.color = getenv("ANSICON") ? TRUE : FALSE;
#else
    opts.color = TRUE;
#endif
    opts.max_matches_per_file = 10000;
    opts.max_search_depth = 25;
    opts.print_break = TRUE;
    opts.print_heading = TRUE;
    opts.print_line_numbers = TRUE;
    opts.recurse_dirs = TRUE;
    opts.color_path = ag_strdup(color_path);
    opts.color_match = ag_strdup(color_match);
    opts.color_line_number = ag_strdup(color_line_number);
}

void cleanup_options(void) {
    free(opts.color_path);
    free(opts.color_match);
    free(opts.color_line_number);

    if (opts.query) {
        free(opts.query);
    }

    pcre_free(opts.re);
    if (opts.re_extra) {
        /* Using pcre_free_study on pcre_extra* can segfault on some versions of PCRE */
        pcre_free(opts.re_extra);
    }

    if (opts.ackmate_dir_filter) {
        pcre_free(opts.ackmate_dir_filter);
    }
    if (opts.ackmate_dir_filter_extra) {
        pcre_free(opts.ackmate_dir_filter_extra);
    }

    if (opts.file_search_regex) {
        pcre_free(opts.file_search_regex);
    }
    if (opts.file_search_regex_extra) {
        pcre_free(opts.file_search_regex_extra);
    }
}

void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
    int ch;
    int i;
    int path_len = 0;
    int useless = 0;
    int group = 1;
    int help = 0;
    int version = 0;
    int opt_index = 0;
    const char *home_dir = getenv("HOME");
    char *ignore_file_path = NULL;
    int needs_query = 1;
    struct stat statbuf;
    int rv;

    size_t longopts_len, full_len;
    option_t *longopts;
    char *lang_regex = NULL;

    init_options();

    option_t base_longopts[] = {
        { "ackmate", no_argument, &opts.ackmate, 1 },
        { "ackmate-dir-filter", required_argument, NULL, 0 },
        { "after", required_argument, NULL, 'A' },
        { "all-text", no_argument, NULL, 't' },
        { "all-types", no_argument, NULL, 'a' },
        { "before", required_argument, NULL, 'B' },
        { "break", no_argument, &opts.print_break, 1 },
        { "case-sensitive", no_argument, NULL, 's' },
        { "color", no_argument, &opts.color, 1 },
        { "color-path", required_argument, NULL, 0 },
        { "color-match", required_argument, NULL, 0 },
        { "color-line-number", required_argument, NULL, 0 },
        { "column", no_argument, &opts.column, 1 },
        { "context", optional_argument, NULL, 'C' },
        { "debug", no_argument, NULL, 'D' },
        { "depth", required_argument, NULL, 0 },
        { "file-search-regex", required_argument, NULL, 'G' },
        { "files-with-matches", no_argument, NULL, 'l' },
        { "files-without-matches", no_argument, NULL, 'L' },
        { "follow", no_argument, &opts.follow_symlinks, 1 },
        { "group", no_argument, &group, 1 },
        { "heading", no_argument, &opts.print_heading, 1 },
        { "help", no_argument, NULL, 'h' },
        { "hidden", no_argument, &opts.search_hidden_files, 1 },
        { "ignore", required_argument, NULL, 0 },
        { "ignore-dir", required_argument, NULL, 0 },
        { "ignore-case", no_argument, NULL, 'i' },
        { "invert-match", no_argument, &opts.invert_match, 1 },
        { "line-numbers", no_argument, &opts.print_line_numbers, 2 },
        { "literal", no_argument, NULL, 'Q' },
        { "match", no_argument, &useless, 0 },
        { "max-count", required_argument, NULL, 'm' },
        { "no-numbers", no_argument, NULL, 0 },
        { "no-recurse", no_argument, NULL, 'n' },
        { "nobreak", no_argument, &opts.print_break, 0 },
        { "nocolor", no_argument, &opts.color, 0 },
        { "nofollow", no_argument, &opts.follow_symlinks, 0 },
        { "nogroup", no_argument, &group, 0 },
        { "noheading", no_argument, &opts.print_heading, 0 },
        { "nopager", no_argument, NULL, 0 },
        { "pager", required_argument, NULL, 0 },
        { "parallel", no_argument, &opts.parallel, 1 },
        { "path-to-agignore", required_argument, NULL, 'p' },
        { "print-long-lines", no_argument, &opts.print_long_lines, 1 },
        { "recurse", no_argument, NULL, 'r' },
        { "search-binary", no_argument, &opts.search_binary_files, 1 },
        { "search-zip", no_argument, &opts.search_zip_files, 1 },
        { "search-files", no_argument, &opts.search_stream, 0 },
        { "silent", no_argument, NULL, 0 },
        { "skip-vcs-ignores", no_argument, NULL, 'U' },
        { "smart-case", no_argument, NULL, 'S' },
        { "stats", no_argument, &opts.stats, 1 },
        { "unrestricted", no_argument, NULL, 'u' },
        { "version", no_argument, &version, 1 },
        { "word-regexp", no_argument, NULL, 'w' },
        { "workers", required_argument, NULL, 0 },
    };

    longopts_len = (sizeof(base_longopts) / sizeof(option_t));
    full_len = (longopts_len + LANG_COUNT + 1);
    longopts = ag_malloc(full_len * sizeof(option_t));
    memcpy(longopts, base_longopts, sizeof(base_longopts));

    for (i = 0; i < LANG_COUNT; i++) {
        option_t opt = { (char *) langs[i].name, no_argument, NULL, 0 };
        longopts[i + longopts_len] = opt;
    }
    longopts[full_len - 1] = (option_t) { NULL, 0, NULL, 0 };

    if (argc < 2) {
        usage();
        cleanup_ignore(root_ignores);
        cleanup_options();
        exit(1);
    }

    rv = fstat(fileno(stdin), &statbuf);
    if (rv == 0) {
        if (S_ISFIFO(statbuf.st_mode)) {
            opts.search_stream = 1;
        }
    }

    /* If we're not outputting to a terminal. change output to:
{'fraction_non_alphanumeric': 0.14235109559232834, 'fraction_numerical': 0.010323364202216486, 'mean_word_length': 2.3421275156434973, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123567', 'n_tokens_mistral': 6746, 'n_tokens_neox': 6427, 'n_words': 1700}