<commit_before>/*
 * Copyright (c) 2007, Novell Inc.
 *
 * This program is licensed under the BSD license, read LICENSE.BSD
 * for further information
 */

#include <sys/types.h>
#include <limits.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "pool.h"
#include "repo.h"
#include "tools_util.h"
#include "repo_susetags.h"

struct parsedata {
  char *kind;
  Repo *repo;
  Repodata *data;
  struct parsedata_common common;
  int last_found_source;
  char **share_with;
  int nshare;
  Id (*dirs)[3]; // dirid, size, nfiles
  int ndirs;
  Id langcache[ID_NUM_INTERNAL];
  int lineno;
};

static char *flagtab[] = {
  ">",
  "=",
  ">=",
  "<",
  "!=",
  "<="
};


static Id
langtag(struct parsedata *pd, Id tag, const char *language)
{
  if (language && !language[0])
    language = 0;
  if (!language || tag >= ID_NUM_INTERNAL)
    return pool_id2langid(pd->repo->pool, tag, language, 1);
  return pool_id2langid(pd->repo->pool, tag, language, 1);
  if (!pd->langcache[tag])
    pd->langcache[tag] = pool_id2langid(pd->repo->pool, tag, language, 1);
  return pd->langcache[tag];
}

/*
 * adddep
 * create and add dependency
 */

static unsigned int
adddep(Pool *pool, struct parsedata *pd, unsigned int olddeps, char *line, Id marker, char *kind)
{
  int i, flags;
  Id id, evrid;
  char *sp[4];

  if (line[6] == '/')
    {
      /* A file dependency. Do not try to parse it */
      id = str2id(pool, line + 6, 1);
    }
  else
    {
      i = split(line + 6, sp, 4); /* name, <op>, evr, ? */
      if (i != 1 && i != 3) /* expect either 'name' or 'name' <op> 'evr' */
        {
	  pool_debug(pool, SAT_FATAL, "susetags: bad dependency line: %d: %s\n", pd->lineno, line);
          exit(1);
        }
      if (kind)
        id = str2id(pool, join2(kind, ":", sp[0]), 1);
      else
        id = str2id(pool, sp[0], 1);
      if (i == 3)
        {
          evrid = makeevr(pool, sp[2]);
          for (flags = 0; flags < 6; flags++)
            if (!strcmp(sp[1], flagtab[flags]))
              break;
          if (flags == 6)
            {
	      pool_debug(pool, SAT_FATAL, "susetags: unknown relation in %d: '%s'\n", pd->lineno, sp[1]);
              exit(1);
            }
          id = rel2id(pool, id, evrid, flags + 1, 1);
        }
    }
  return repo_addid_dep(pd->repo, olddeps, id, marker);
}


/*
 * add_source
 *
 */

static void
add_source(struct parsedata *pd, char *line, Solvable *s, Id handle)
{
  Repo *repo = s->repo;
  Pool *pool = repo->pool;
  char *sp[5];

  if (split(line, sp, 5) != 4)
    {
      pool_debug(pool, SAT_FATAL, "susetags: bad source line: %d: %s\n", pd->lineno, line);
      exit(1);
    }

  Id name = str2id(pool, sp[0], 1);
  Id evr = makeevr(pool, join2(sp[1], "-", sp[2]));
  Id arch = str2id(pool, sp[3], 1);
  /* XXX: could record a dep here, depends on where we want to store the data */
  if (name == s->name)
    repodata_set_void(pd->data, handle, SOLVABLE_SOURCENAME);
  else
    repodata_set_id(pd->data, handle, SOLVABLE_SOURCENAME, name);
  if (evr == s->evr)
    repodata_set_void(pd->data, handle, SOLVABLE_SOURCEEVR);
  else
    repodata_set_id(pd->data, handle, SOLVABLE_SOURCEEVR, evr);
  repodata_set_constantid(pd->data, handle, SOLVABLE_SOURCEARCH, arch);
}

/*
 * add_dirline
 * add a line with directory information
 *
 */

static void
add_dirline(struct parsedata *pd, char *line)
{
  char *sp[6];
  if (split(line, sp, 6) != 5)
    return;
  pd->dirs = sat_extend(pd->dirs, pd->ndirs, 1, sizeof(pd->dirs[0]), 31);
  long filesz = strtol(sp[1], 0, 0);
  filesz += strtol(sp[2], 0, 0);
  long filenum = strtol(sp[3], 0, 0);
  filenum += strtol(sp[4], 0, 0);
  /* hack: we know that there's room for a / */
  if (*sp[0] != '/')
    *--sp[0] = '/';
  unsigned dirid = repodata_str2dir(pd->data, sp[0], 1);
#if 0
fprintf(stderr, "%s -> %d\n", sp[0], dirid);
#endif
  pd->dirs[pd->ndirs][0] = dirid;
  pd->dirs[pd->ndirs][1] = filesz;
  pd->dirs[pd->ndirs][2] = filenum;
  pd->ndirs++;
}

static void
set_checksum(struct parsedata *pd, Repodata *data, Id handle, Id keyname, char *line)
{
  char *sp[3];
  int l;
  Id type;
  if (split(line, sp, 3) != 2)
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: bad checksum line: %d: %s\n", pd->lineno, line);
      exit(1);
    }
  if (!strcasecmp(sp[0], "sha1"))
    l = SIZEOF_SHA1 * 2, type = REPOKEY_TYPE_SHA1;
  else if (!strcasecmp(sp[0], "md5"))
    l = SIZEOF_MD5 * 2, type = REPOKEY_TYPE_MD5;
  else
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: unknown checksum type: %d: %s\n", pd->lineno, sp[0]);
      exit(1);
    }
  if (strlen(sp[1]) != l)
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: bad checksum length: %d: for %s: %s\n", pd->lineno, sp[0], sp[1]);
      exit(1);
    }
  repodata_set_checksum(data, handle, keyname, type, sp[1]);
}<commit_msg>- also support sha256 in susetags<commit_after>/*
 * Copyright (c) 2007, Novell Inc.
 *
 * This program is licensed under the BSD license, read LICENSE.BSD
 * for further information
 */

#include <sys/types.h>
#include <limits.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "pool.h"
#include "repo.h"
#include "tools_util.h"
#include "repo_susetags.h"

struct parsedata {
  char *kind;
  Repo *repo;
  Repodata *data;
  struct parsedata_common common;
  int last_found_source;
  char **share_with;
  int nshare;
  Id (*dirs)[3]; // dirid, size, nfiles
  int ndirs;
  Id langcache[ID_NUM_INTERNAL];
  int lineno;
};

static char *flagtab[] = {
  ">",
  "=",
  ">=",
  "<",
  "!=",
  "<="
};


static Id
langtag(struct parsedata *pd, Id tag, const char *language)
{
  if (language && !language[0])
    language = 0;
  if (!language || tag >= ID_NUM_INTERNAL)
    return pool_id2langid(pd->repo->pool, tag, language, 1);
  return pool_id2langid(pd->repo->pool, tag, language, 1);
  if (!pd->langcache[tag])
    pd->langcache[tag] = pool_id2langid(pd->repo->pool, tag, language, 1);
  return pd->langcache[tag];
}

/*
 * adddep
 * create and add dependency
 */

static unsigned int
adddep(Pool *pool, struct parsedata *pd, unsigned int olddeps, char *line, Id marker, char *kind)
{
  int i, flags;
  Id id, evrid;
  char *sp[4];

  if (line[6] == '/')
    {
      /* A file dependency. Do not try to parse it */
      id = str2id(pool, line + 6, 1);
    }
  else
    {
      i = split(line + 6, sp, 4); /* name, <op>, evr, ? */
      if (i != 1 && i != 3) /* expect either 'name' or 'name' <op> 'evr' */
        {
	  pool_debug(pool, SAT_FATAL, "susetags: bad dependency line: %d: %s\n", pd->lineno, line);
          exit(1);
        }
      if (kind)
        id = str2id(pool, join2(kind, ":", sp[0]), 1);
      else
        id = str2id(pool, sp[0], 1);
      if (i == 3)
        {
          evrid = makeevr(pool, sp[2]);
          for (flags = 0; flags < 6; flags++)
            if (!strcmp(sp[1], flagtab[flags]))
              break;
          if (flags == 6)
            {
	      pool_debug(pool, SAT_FATAL, "susetags: unknown relation in %d: '%s'\n", pd->lineno, sp[1]);
              exit(1);
            }
          id = rel2id(pool, id, evrid, flags + 1, 1);
        }
    }
  return repo_addid_dep(pd->repo, olddeps, id, marker);
}


/*
 * add_source
 *
 */

static void
add_source(struct parsedata *pd, char *line, Solvable *s, Id handle)
{
  Repo *repo = s->repo;
  Pool *pool = repo->pool;
  char *sp[5];

  if (split(line, sp, 5) != 4)
    {
      pool_debug(pool, SAT_FATAL, "susetags: bad source line: %d: %s\n", pd->lineno, line);
      exit(1);
    }

  Id name = str2id(pool, sp[0], 1);
  Id evr = makeevr(pool, join2(sp[1], "-", sp[2]));
  Id arch = str2id(pool, sp[3], 1);
  /* XXX: could record a dep here, depends on where we want to store the data */
  if (name == s->name)
    repodata_set_void(pd->data, handle, SOLVABLE_SOURCENAME);
  else
    repodata_set_id(pd->data, handle, SOLVABLE_SOURCENAME, name);
  if (evr == s->evr)
    repodata_set_void(pd->data, handle, SOLVABLE_SOURCEEVR);
  else
    repodata_set_id(pd->data, handle, SOLVABLE_SOURCEEVR, evr);
  repodata_set_constantid(pd->data, handle, SOLVABLE_SOURCEARCH, arch);
}

/*
 * add_dirline
 * add a line with directory information
 *
 */

static void
add_dirline(struct parsedata *pd, char *line)
{
  char *sp[6];
  if (split(line, sp, 6) != 5)
    return;
  pd->dirs = sat_extend(pd->dirs, pd->ndirs, 1, sizeof(pd->dirs[0]), 31);
  long filesz = strtol(sp[1], 0, 0);
  filesz += strtol(sp[2], 0, 0);
  long filenum = strtol(sp[3], 0, 0);
  filenum += strtol(sp[4], 0, 0);
  /* hack: we know that there's room for a / */
  if (*sp[0] != '/')
    *--sp[0] = '/';
  unsigned dirid = repodata_str2dir(pd->data, sp[0], 1);
#if 0
fprintf(stderr, "%s -> %d\n", sp[0], dirid);
#endif
  pd->dirs[pd->ndirs][0] = dirid;
  pd->dirs[pd->ndirs][1] = filesz;
  pd->dirs[pd->ndirs][2] = filenum;
  pd->ndirs++;
}

static void
set_checksum(struct parsedata *pd, Repodata *data, Id handle, Id keyname, char *line)
{
  char *sp[3];
  int l;
  Id type;
  if (split(line, sp, 3) != 2)
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: bad checksum line: %d: %s\n", pd->lineno, line);
      exit(1);
    }
  if (!strcasecmp(sp[0], "sha1"))
    l = SIZEOF_SHA1 * 2, type = REPOKEY_TYPE_SHA1;
  else if (!strcasecmp(sp[0], "sha256"))
    l = SIZEOF_SHA256 * 2, type = REPOKEY_TYPE_SHA256;
  else if (!strcasecmp(sp[0], "md5"))
    l = SIZEOF_MD5 * 2, type = REPOKEY_TYPE_MD5;
  else
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: unknown checksum type: %d: %s\n", pd->lineno, sp[0]);
      exit(1);
    }
  if (strlen(sp[1]) != l)
    {
      pool_debug(pd->repo->pool, SAT_FATAL, "susetags: bad checksum length: %d: for %s: %s\n", pd->lineno, sp[0], sp[1]);
      exit(1);
    }
  repodata_set_checksum(data, handle, keyname, type, sp[1]);
}
{'fraction_non_alphanumeric': 0.18120805369127516, 'fraction_numerical': 0.022574740695546065, 'mean_word_length': 2.9705288655631814, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5750818', 'n_tokens_mistral': 4091, 'n_tokens_neox': 3944, 'n_words': 1004}