<commit_before>/***************************************************************
 *
 *    ASSETS
 *
 *    This program crawls a directory tree and
 *    makes a record of all assets it finds.
 *
 *    This file is the main file.
 *
 *    Author <NAME>
 *    Copyright 2014 <NAME>
 *    License MIT (included with this source code)
 *
 **************************************************************/


/*  ------------------------------------------------------------
 *
 *  IMPORT LIBRARIES
 * 
 *  ------------------------------------------------------------
 */

// The standard C library.
#include <stdio.h>

// For things like `exit(0)`.
#include <stdlib.h>

// For using the `stat()` function.
#include <sys/stat.h>

// For working with strings, e.g., `strcat()`.
#include <string.h>

// Our own utilities are defined in utilities.h.
#include "utilities.h"

// Our tools for processing files are defined in processing.h.
#include "processing.h"

// Our tools for logging/writing output are defined in logging.h.
#include "logging.h"


/*  ------------------------------------------------------------
 *
 *  DEF/CONSTANTS
 *
 *  ------------------------------------------------------------
 */
#define MAX_PATH_LENGTH 1024


/*  ------------------------------------------------------------
 *
 *  NON-CONSTANT VARIABLES THAT CAN BE SET
 *
 *  ------------------------------------------------------------
 */
int has_blacklist_additions = 0;
char *blacklist_additions;
char *blacklist;


/*  ------------------------------------------------------------
 *
 *  FUNCTION PROTOTYPES
 * 
 *  ------------------------------------------------------------
 */
void print_usage();


/*  ------------------------------------------------------------
 *
 *  FUNCTION DEFINITIONS
 * 
 *  ------------------------------------------------------------
 */

/*
 *  Print usage instructions.
 *
 *  @return void
 */
void print_usage() {
  puts("");
  puts("Assets 0.1");
  puts("----------");
  puts("This program crawls a folder and generates a dictionary");
  puts("of all assets it finds, noting the path, type of file, etc.");
  puts("");
  puts("Usage: assets <folder> [<output-file>] [options]");
  puts(" <folder>      : is the folder to crawl for assets");
  puts(" <output-file> : is where to save the dictionary");
  puts("");
  puts("Options:");
  puts("--cachebust     : renames files with cachebusting names");
  puts("--base64 <size> : base64 encode files smaller than <size> bytes");
  puts("--ignore file1,file2,file3 : ignore the specified files"); 
  puts("");
  puts("Example: assets . assets.json");
  puts("-- This will crawl the current directory (\".\")");
  puts("   and create a dictionary of all assets it finds");
  puts("   in a file called \"assets.json\".");
  puts("");
}



/*  ------------------------------------------------------------
 *
 *  MAIN FUNCTION
 *
 *  ------------------------------------------------------------
 *
 *  This is the MAIN entry point into the program,
 *  meaning that when the program runs,
 *  this function gets executed first.
 *
 *  @param int number_of_arguments The number of arguments.
 *  @param char *argument[] The list of arguments.
 *  @return int A status code.
 */
int main(int number_of_arguments, char *argument[]) {

  // Are there at least two arguments? If not, print the help.
  if (number_of_arguments < 2) {
    print_usage();
  } 

  // Otherwise, we can proceed.
  else {

    // We want to find certain things in the arguments.
    // Here are some flags that specify whether we find a blacklist, 
    // a directory to crawl, and an output file to write to.
    int has_folder_to_crawl = 0;
    int has_output_file = 0;

    // We'll store the path to the folder to crawl here:
    char folder_to_crawl[MAX_PATH_LENGTH];

    // Now we can process each argument.
    int i;
    for (i = 1; i < number_of_arguments; i++) {

      // Is this argument the optional "--cachebust"?
      if (strcmp(argument[i], "--cachebust") == 0) {
        set_cachebust(1);
      }

      // Is this argument the optional "--ignore"? 
      else if (strcmp(argument[i], "--ignore") == 0) {

        // The string of items to blacklist/ignore will be the next argument.
        blacklist_additions = argument[i + 1];
        has_blacklist_additions = 1;

        // Increment the counter so the next iteration of the loop
        // won't try to process the blacklist string.
        i++;<commit_msg>Updated a comment in assets.c<commit_after>/***************************************************************
 *
 *    ASSETS
 *
 *    This program crawls a directory tree and
 *    makes a record of all assets it finds.
 *
 *    This file is the main file.
 *
 *    Author <NAME>
 *    Copyright 2014 <NAME>
 *    License MIT (included with this source code)
 *
 **************************************************************/


/*  ------------------------------------------------------------
 *
 *  IMPORT LIBRARIES
 * 
 *  ------------------------------------------------------------
 */

// The standard C library.
#include <stdio.h>

// For things like `exit(0)`.
#include <stdlib.h>

// For using the `stat()` function.
#include <sys/stat.h>

// For working with strings, e.g., `strcat()`.
#include <string.h>

// Our own utilities are defined in utilities.h.
#include "utilities.h"

// Our tools for processing files are defined in processing.h.
#include "processing.h"

// Our tools for logging/writing output are defined in logging.h.
#include "logging.h"


/*  ------------------------------------------------------------
 *
 *  DEF/CONSTANTS
 *
 *  ------------------------------------------------------------
 */
#define MAX_PATH_LENGTH 1024


/*  ------------------------------------------------------------
 *
 *  NON-CONSTANT VARIABLES THAT CAN BE SET
 *
 *  ------------------------------------------------------------
 */
int has_blacklist_additions = 0;
char *blacklist_additions;
char *blacklist;


/*  ------------------------------------------------------------
 *
 *  FUNCTION PROTOTYPES
 * 
 *  ------------------------------------------------------------
 */
void print_usage();


/*  ------------------------------------------------------------
 *
 *  FUNCTION DEFINITIONS
 * 
 *  ------------------------------------------------------------
 */

/*
 *  Print usage instructions.
 *
 *  @return void
 */
void print_usage() {
  puts("");
  puts("Assets 0.1");
  puts("----------");
  puts("This program crawls a folder and generates a dictionary");
  puts("of all assets it finds, noting the path, type of file, etc.");
  puts("");
  puts("Usage: assets <folder> [<output-file>] [options]");
  puts(" <folder>      : is the folder to crawl for assets");
  puts(" <output-file> : is where to save the dictionary");
  puts("");
  puts("Options:");
  puts("--cachebust     : renames files with cachebusting names");
  puts("--base64 <size> : base64 encode files smaller than <size> bytes");
  puts("--ignore file1,file2,file3 : ignore the specified files"); 
  puts("");
  puts("Example: assets . assets.json");
  puts("-- This will crawl the current directory (\".\")");
  puts("   and create a dictionary of all assets it finds");
  puts("   in a file called \"assets.json\".");
  puts("");
}



/*  ------------------------------------------------------------
 *
 *  MAIN FUNCTION
 *
 *  ------------------------------------------------------------
 *
 *  This is the MAIN entry point into the program,
 *  meaning that when the program runs,
 *  this function gets executed first.
 *
 *  @param int number_of_arguments The number of arguments.
 *  @param char *argument[] The list of arguments.
 *  @return int A status code.
 */
int main(int number_of_arguments, char *argument[]) {

  // Are there at least two arguments? If not, print the help.
  if (number_of_arguments < 2) {
    print_usage();
  } 

  // Otherwise, we can proceed.
  else {

    // We want to find certain things in the arguments.
    // Here are some flags that specify whether we find 
    // a directory to crawl, or an output file to write to.
    int has_folder_to_crawl = 0;
    int has_output_file = 0;

    // We'll store the path to the folder to crawl here:
    char folder_to_crawl[MAX_PATH_LENGTH];

    // Now we can process each argument.
    int i;
    for (i = 1; i < number_of_arguments; i++) {

      // Is this argument the optional "--cachebust"?
      if (strcmp(argument[i], "--cachebust") == 0) {
        set_cachebust(1);
      }

      // Is this argument the optional "--ignore"? 
      else if (strcmp(argument[i], "--ignore") == 0) {

        // The string of items to blacklist/ignore will be the next argument.
        blacklist_additions = argument[i + 1];
        has_blacklist_additions = 1;

        // Increment the counter so the next iteration of the loop
        // won't try to process the blacklist string.
        i++;
{'fraction_non_alphanumeric': 0.29719416386083053, 'fraction_numerical': 0.006285072951739619, 'mean_word_length': 1.5994749124854142, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3162315', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2338, 'n_words': 819}