<commit_before>/*
Advanced Computing Center for Research and Education Proprietary License
Version 1.0 (April 2006)

Copyright (c) 2006, Advanced Computing Center for Research and Education,
 Vanderbilt University, All rights reserved.

This Work is the sole and exclusive property of the Advanced Computing Center
for Research and Education department at Vanderbilt University.  No right to
disclose or otherwise disseminate any of the information contained herein is
granted by virtue of your possession of this software except in accordance with
the terms and conditions of a separate License Agreement entered into with
Vanderbilt University.

THE AUTHOR OR COPYRIGHT HOLDERS PROVIDES THE "WORK" ON AN "AS IS" BASIS,
WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, TITLE, FITNESS FOR A PARTICULAR
PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Vanderbilt University
Advanced Computing Center for Research and Education
230 Appleton Place
Nashville, TN 37203
http://www.accre.vanderbilt.edu
*/

#define _log_module_index 201

#include <assert.h>
#include "exnode.h"
#include "log.h"
#include "iniparse.h"
#include "type_malloc.h"
#include "thread_pool.h"
#include "lio.h"
#include "string_token.h"

#define MAX_SET 1000


//*************************************************************************
// load_file - Loads a file from disk
//*************************************************************************

void load_file(char *fname, char **val, int *v_size)
{
  FILE *fd;
  int i;

  *v_size = 0;  *val = NULL;

  fd = fopen(fname, "r");
  if (fd == NULL) {
     info_printf(lio_ifd, 0, "ERROR opeing file=%s!  Exiting\n", fname);
     exit(1);
  }
  fseek(fd, 0, SEEK_END);

  i = ftell(fd);
  type_malloc(*val, char, i+1);
  (*val)[i] = 0;
  *v_size = i;

  fseek(fd, 0, SEEK_SET);
  if (fread(*val, i, 1, fd) != 1) { //**
     info_printf(lio_ifd, 0, "ERROR reading file=%s! Exiting\n", fname);
     exit(1);
  }
  fclose(fd);

//info_printf(lio_ifd, 0, "fname=%s size=%d val=%s\n", fname, i, *val);

}

//*************************************************************************
//*************************************************************************

int main(int argc, char **argv)
{
  int i, j, rg_mode, start_option, start_index, fin, ftype, prefix_len;
  lio_path_tuple_t tuple;
  os_regex_table_t *rp_single, *ro_single, *attr_regex;
  os_object_iter_t *it;
  os_attr_iter_t *ait;
  char *bstate, *fname;
  char *key[MAX_SET];
  char *val[MAX_SET];
  int v_size[MAX_SET];
  int n_keys, n_keys_al;
  int max_attr = 1024*1024;
  char *new_obj_fmt = "object=%s\\n";
  char *end_obj_fmt = "\\n";
  char *attr_fmt = "\t%s=%s\\n";
  char *attr_sep = "";

  int recurse_depth = 0;
  int obj_types = OS_OBJECT_FILE;

//estr = argv2format(argv[1]);
//printf("argv[1]=!%s! str=!%s!\n", argv[1], estr);
  if (argc < 2) {
     printf("\n");
     printf("lio_getattr LIO_COMMON_OPTIONS [-rd recurse_depth] [-t object_types] [-vmax max_val] PRINT_OPTIONS [-ga attr_glob | -ra attr_regex | -al key1,key2,...keyN] LIO_PATH_OPTIONS\n");
     lio_print_options(stdout);
     lio_print_path_options(stdout);
     printf("    PRINT_OPTIONS: [-new_obj new_obj_fmt] [-end_obj end_obj_fmt] [-attr_fmt attr_fmt] [-attr_sep attr_sep]\n");
     printf("       -list                 - Prints object and attibute names but not their values\n");
     printf("       -list1                - Prints 1 line per file only listing the attributes. Not their values\n");
     printf("       -single               - Prints 1 line per file with all attributes space separated\n");
     printf("       -new_obj new_obj_fmt  - New object C format string.  Default is \"%s\"\n", new_obj_fmt);
     printf("       -end_obj end_obj_fmt  - End object C format string.  Default is \"%s\"\n", end_obj_fmt);
     printf("       -attr_fmt attr_fmt    - Attribute C format string.  Default is \"%s\"\n", attr_fmt);
     printf("       -attr_sep attr_sep    - Attribute separator C format string.  Default is \"%s\"\n", attr_sep);
     printf("\n");
     printf("       The format strings are string together according to the following code snippet:\n");
     printf("           printf(new_obj_fmt, object_name);\n");
     printf("           printf(attr_fmt, key[0], val[0]);\n");
     printf("           for (i=1; i<nattr; i++) {\n");
     printf("             printf(attr_sep);\n");
     printf("             printf(attr_fmt, key[i], val[i]);\n");
     printf("           }\n");
     printf("           printf(end_obj_fmt, object_name);\n");
     printf("\n");
     printf("    -rd recurse_depth  - Max recursion depth on directories. Defaults to %d\n", recurse_depth);
     printf("    -t  object_types   - Types of objects to list bitwise OR of 1=Files, 2=Directories, 4=symlink, 8=hardlink.  Default is %d.\n", obj_types);
     printf("    -vmax max_val      - Max size of attribute value allowed.  Default value is %d\n", max_attr);
     printf("    -ga attr_glob      - Attribute glob string\n");
     printf("    -ra attr_regex     - Attribute regex string\n");
     printf("    -al key1,key2,...  - Comma separated list of keys to retrieve\n");
     return(1);
  }

  lio_init(&argc, &argv);


  //*** Parse the args
  rg_mode = 0;
  rp_single = ro_single = attr_regex = NULL;
  n_keys_al = 0;

  rg_mode = lio_parse_path_options(&argc, argv, lio_gc->auto_translate, &tuple, &rp_single, &ro_single);

  i=1;
  do {
     start_option = i;

     if (strcmp(argv[i], "-rd") == 0) { //** Recurse depth
        i++;
        recurse_depth = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-t") == 0) {  //** Object types
        i++;
        obj_types = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-vmax") == 0) {  //** MAx attribute size
        i++;
        max_attr = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-new_obj") == 0) {  //**  New object format
        i++;
        new_obj_fmt = argv[i]; i++;
     } else if (strcmp(argv[i], "-end_obj") == 0) {  //**  End object format
        i++;
        end_obj_fmt = argv[i]; i++;
     } else if (strcmp(argv[i], "-attr_fmt") == 0) {  //**  Attribute format
        i++;
        attr_fmt = argv[i]; i++;<commit_msg>Make the default attribute size be what's specified in the config<commit_after>/*
Advanced Computing Center for Research and Education Proprietary License
Version 1.0 (April 2006)

Copyright (c) 2006, Advanced Computing Center for Research and Education,
 Vanderbilt University, All rights reserved.

This Work is the sole and exclusive property of the Advanced Computing Center
for Research and Education department at Vanderbilt University.  No right to
disclose or otherwise disseminate any of the information contained herein is
granted by virtue of your possession of this software except in accordance with
the terms and conditions of a separate License Agreement entered into with
Vanderbilt University.

THE AUTHOR OR COPYRIGHT HOLDERS PROVIDES THE "WORK" ON AN "AS IS" BASIS,
WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT
LIMITED TO THE WARRANTIES OF MERCHANTABILITY, TITLE, FITNESS FOR A PARTICULAR
PURPOSE, AND NON-INFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Vanderbilt University
Advanced Computing Center for Research and Education
230 Appleton Place
Nashville, TN 37203
http://www.accre.vanderbilt.edu
*/

#define _log_module_index 201

#include <assert.h>
#include "exnode.h"
#include "log.h"
#include "iniparse.h"
#include "type_malloc.h"
#include "thread_pool.h"
#include "lio.h"
#include "string_token.h"

#define MAX_SET 1000


//*************************************************************************
// load_file - Loads a file from disk
//*************************************************************************

void load_file(char *fname, char **val, int *v_size)
{
  FILE *fd;
  int i;

  *v_size = 0;  *val = NULL;

  fd = fopen(fname, "r");
  if (fd == NULL) {
     info_printf(lio_ifd, 0, "ERROR opeing file=%s!  Exiting\n", fname);
     exit(1);
  }
  fseek(fd, 0, SEEK_END);

  i = ftell(fd);
  type_malloc(*val, char, i+1);
  (*val)[i] = 0;
  *v_size = i;

  fseek(fd, 0, SEEK_SET);
  if (fread(*val, i, 1, fd) != 1) { //**
     info_printf(lio_ifd, 0, "ERROR reading file=%s! Exiting\n", fname);
     exit(1);
  }
  fclose(fd);

//info_printf(lio_ifd, 0, "fname=%s size=%d val=%s\n", fname, i, *val);

}

//*************************************************************************
//*************************************************************************

int main(int argc, char **argv)
{
  int i, j, rg_mode, start_option, start_index, fin, ftype, prefix_len;
  lio_path_tuple_t tuple;
  os_regex_table_t *rp_single, *ro_single, *attr_regex;
  os_object_iter_t *it;
  os_attr_iter_t *ait;
  char *bstate, *fname;
  char *key[MAX_SET];
  char *val[MAX_SET];
  int v_size[MAX_SET];
  int n_keys, n_keys_al;
  int max_attr = 1024*1024;
  char *new_obj_fmt = "object=%s\\n";
  char *end_obj_fmt = "\\n";
  char *attr_fmt = "\t%s=%s\\n";
  char *attr_sep = "";

  int recurse_depth = 0;
  int obj_types = OS_OBJECT_FILE;

//estr = argv2format(argv[1]);
//printf("argv[1]=!%s! str=!%s!\n", argv[1], estr);
  if (argc < 2) {
     printf("\n");
     printf("lio_getattr LIO_COMMON_OPTIONS [-rd recurse_depth] [-t object_types] [-vmax max_val] PRINT_OPTIONS [-ga attr_glob | -ra attr_regex | -al key1,key2,...keyN] LIO_PATH_OPTIONS\n");
     lio_print_options(stdout);
     lio_print_path_options(stdout);
     printf("    PRINT_OPTIONS: [-new_obj new_obj_fmt] [-end_obj end_obj_fmt] [-attr_fmt attr_fmt] [-attr_sep attr_sep]\n");
     printf("       -list                 - Prints object and attibute names but not their values\n");
     printf("       -list1                - Prints 1 line per file only listing the attributes. Not their values\n");
     printf("       -single               - Prints 1 line per file with all attributes space separated\n");
     printf("       -new_obj new_obj_fmt  - New object C format string.  Default is \"%s\"\n", new_obj_fmt);
     printf("       -end_obj end_obj_fmt  - End object C format string.  Default is \"%s\"\n", end_obj_fmt);
     printf("       -attr_fmt attr_fmt    - Attribute C format string.  Default is \"%s\"\n", attr_fmt);
     printf("       -attr_sep attr_sep    - Attribute separator C format string.  Default is \"%s\"\n", attr_sep);
     printf("\n");
     printf("       The format strings are string together according to the following code snippet:\n");
     printf("           printf(new_obj_fmt, object_name);\n");
     printf("           printf(attr_fmt, key[0], val[0]);\n");
     printf("           for (i=1; i<nattr; i++) {\n");
     printf("             printf(attr_sep);\n");
     printf("             printf(attr_fmt, key[i], val[i]);\n");
     printf("           }\n");
     printf("           printf(end_obj_fmt, object_name);\n");
     printf("\n");
     printf("    -rd recurse_depth  - Max recursion depth on directories. Defaults to %d\n", recurse_depth);
     printf("    -t  object_types   - Types of objects to list bitwise OR of 1=Files, 2=Directories, 4=symlink, 8=hardlink.  Default is %d.\n", obj_types);
     printf("    -vmax max_val      - Max size of attribute value allowed.  Default value is %d\n", max_attr);
     printf("    -ga attr_glob      - Attribute glob string\n");
     printf("    -ra attr_regex     - Attribute regex string\n");
     printf("    -al key1,key2,...  - Comma separated list of keys to retrieve\n");
     return(1);
  }

  lio_init(&argc, &argv);

   max_attr = lio_gc->max_attr;

  //*** Parse the args
  rg_mode = 0;
  rp_single = ro_single = attr_regex = NULL;
  n_keys_al = 0;

  rg_mode = lio_parse_path_options(&argc, argv, lio_gc->auto_translate, &tuple, &rp_single, &ro_single);

  i=1;
  do {
     start_option = i;

     if (strcmp(argv[i], "-rd") == 0) { //** Recurse depth
        i++;
        recurse_depth = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-t") == 0) {  //** Object types
        i++;
        obj_types = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-vmax") == 0) {  //** MAx attribute size
        i++;
        max_attr = atoi(argv[i]); i++;
     } else if (strcmp(argv[i], "-new_obj") == 0) {  //**  New object format
        i++;
        new_obj_fmt = argv[i]; i++;
     } else if (strcmp(argv[i], "-end_obj") == 0) {  //**  End object format
        i++;
        end_obj_fmt = argv[i]; i++;
     } else if (strcmp(argv[i], "-attr_fmt") == 0) {  //**  Attribute format
        i++;
        attr_fmt = argv[i]; i++;
{'fraction_non_alphanumeric': 0.17746629901960784, 'fraction_numerical': 0.011488970588235295, 'mean_word_length': 3.192999357739242, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6776304', 'n_tokens_mistral': 4492, 'n_tokens_neox': 4238, 'n_words': 1339}