<commit_before>/*
 * Copyright (c) 2007-2012, Novell Inc.
 *
 * This program is licensed under the BSD license, read LICENSE.BSD
 * for further information
 */

/*
 * repo_rpmdb
 *
 * convert rpm db to repo
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <assert.h>
#include <stdint.h>
#include <errno.h>

#include <rpm/rpmio.h>
#include <rpm/rpmpgp.h>
#ifndef RPM5
#include <rpm/header.h>
#endif
#include <rpm/rpmdb.h>

#ifndef DB_CREATE
# if defined(SUSE) || defined(HAVE_RPM_DB_H)
#  include <rpm/db.h>
# else
#  include <db.h>
# endif
#endif

#include "pool.h"
#include "repo.h"
#include "hash.h"
#include "util.h"
#include "queue.h"
#include "chksum.h"
#include "repo_rpmdb.h"
#include "repo_solv.h"

/* 3: added triggers */
/* 4: fixed triggers */
/* 5: fixed checksum copying */
#define RPMDB_COOKIE_VERSION 5

#define TAG_NAME		1000
#define TAG_VERSION		1001
#define TAG_RELEASE		1002
#define TAG_EPOCH		1003
#define TAG_SUMMARY		1004
#define TAG_DESCRIPTION		1005
#define TAG_BUILDTIME		1006
#define TAG_BUILDHOST		1007
#define TAG_INSTALLTIME		1008
#define TAG_SIZE                1009
#define TAG_DISTRIBUTION	1010
#define TAG_VENDOR		1011
#define TAG_LICENSE		1014
#define TAG_PACKAGER		1015
#define TAG_GROUP		1016
#define TAG_URL			1020
#define TAG_ARCH		1022
#define TAG_FILESIZES		1028
#define TAG_FILEMODES		1030
#define TAG_FILEMD5S		1035
#define TAG_FILELINKTOS		1036
#define TAG_FILEFLAGS		1037
#define TAG_SOURCERPM		1044
#define TAG_PROVIDENAME		1047
#define TAG_REQUIREFLAGS	1048
#define TAG_REQUIRENAME		1049
#define TAG_REQUIREVERSION	1050
#define TAG_NOSOURCE		1051
#define TAG_NOPATCH		1052
#define TAG_CONFLICTFLAGS	1053
#define TAG_CONFLICTNAME	1054
#define TAG_CONFLICTVERSION	1055
#define TAG_TRIGGERNAME		1066
#define TAG_TRIGGERVERSION	1067
#define TAG_TRIGGERFLAGS	1068
#define TAG_CHANGELOGTIME	1080
#define TAG_CHANGELOGNAME	1081
#define TAG_CHANGELOGTEXT	1082
#define TAG_OBSOLETENAME	1090
#define TAG_FILEDEVICES		1095
#define TAG_FILEINODES		1096
#define TAG_SOURCEPACKAGE	1106
#define TAG_PROVIDEFLAGS	1112
#define TAG_PROVIDEVERSION	1113
#define TAG_OBSOLETEFLAGS	1114
#define TAG_OBSOLETEVERSION	1115
#define TAG_DIRINDEXES		1116
#define TAG_BASENAMES		1117
#define TAG_DIRNAMES		1118
#define TAG_PAYLOADFORMAT	1124
#define TAG_PATCHESNAME         1133
#define TAG_FILECOLORS		1140
#define TAG_SUGGESTSNAME	1156
#define TAG_SUGGESTSVERSION	1157
#define TAG_SUGGESTSFLAGS	1158
#define TAG_ENHANCESNAME	1159
#define TAG_ENHANCESVERSION	1160
#define TAG_ENHANCESFLAGS	1161

/* rpm5 tags */
#define TAG_DISTEPOCH		1218

/* rpm4 tags */
#define TAG_LONGFILESIZES	5008
#define TAG_LONGSIZE		5009

/* signature tags */
#define	TAG_SIGBASE		256
#define TAG_SIGMD5		(TAG_SIGBASE + 5)
#define TAG_SHA1HEADER		(TAG_SIGBASE + 13)

#define SIGTAG_SIZE		1000
#define SIGTAG_PGP		1002	/* RSA signature */
#define SIGTAG_MD5		1004	/* header+payload md5 checksum */
#define SIGTAG_GPG		1005	/* DSA signature */

#define DEP_LESS		(1 << 1)
#define DEP_GREATER		(1 << 2)
#define DEP_EQUAL		(1 << 3)
#define DEP_STRONG		(1 << 27)
#define DEP_PRE_IN		((1 << 6) | (1 << 9) | (1 << 10))
#define DEP_PRE_UN		((1 << 6) | (1 << 11) | (1 << 12))

#define FILEFLAG_GHOST		(1 <<  6)


#ifdef RPM5
# define RPM_INDEX_SIZE 4
#else
# define RPM_INDEX_SIZE 8
#endif


typedef struct rpmhead {
  int cnt;
  int dcnt;
  unsigned char *dp;
  int forcebinary;		/* sigh, see rh#478907 */
  unsigned char data[1];
} RpmHead;


static inline unsigned char *
headfindtag(RpmHead *h, int tag)
{
  unsigned int i;
  unsigned char *d, taga[4];
  d = h->dp - 16;
  taga[0] = tag >> 24;
  taga[1] = tag >> 16;
  taga[2] = tag >> 8;
  taga[3] = tag;
  for (i = 0; i < h->cnt; i++, d -= 16)
    if (d[3] == taga[3] && d[2] == taga[2] && d[1] == taga[1] && d[0] == taga[0])
      return d;
  return 0;
}

static int
headexists(RpmHead *h, int tag)
{
  return headfindtag(h, tag) ? 1 : 0;
}

static unsigned int *
headint32array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o, *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 4)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned int));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 4)
    r[o] = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
  return r;
}

/* returns the first entry of an integer array */
static unsigned int
headint32(RpmHead *h, int tag)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 4)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (i == 0 || o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  return d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
}

static unsigned long long *
headint64array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o;
  unsigned long long *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 5)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 8 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned long long));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 8)
    {
      unsigned int x = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
      r[o] = (unsigned long long)x << 32 | (d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7]);
    }
  return r;
}

/* returns the first entry of an 64bit integer array */
static unsigned long long
headint64(RpmHead *h, int tag)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 5)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (i == 0 || o + 8 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  i = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
  return (unsigned long long)i << 32 | (d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7]);
}

static unsigned int *
headint16array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o, *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 3)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned int));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 2)
    r[o] = d[0] << 8 | d[1];
  return r;
}

static char *
headstring(RpmHead *h, int tag)
{
  unsigned int o;
  unsigned char *d = headfindtag(h, tag);
  /* 6: STRING, 9: I18NSTRING */
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || (d[7] != 6 && d[7] != 9))
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  if (o >= h->dcnt)
    return 0;
  return (char *)h->dp + o;
}

static char **
headstringarray(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  char **r;

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 8)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  r = solv_calloc(i ? i : 1, sizeof(char *));
  if (cnt)
    *cnt = i;
  d = h->dp + o;
  for (o = 0; o < i; o++)
    {
      r[o] = (char *)d;
      if (o + 1 < i)
        d += strlen((char *)d) + 1;
      if (d >= h->dp + h->dcnt)
        {
          solv_free(r);
          return 0;
        }
    }
  return r;
}

static unsigned char *
headbinary(RpmHead *h, int tag, unsigned int *sizep)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 7)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o > h->dcnt || o + i < o || o + i > h->dcnt)
    return 0;
  if (sizep)
    *sizep = i;
  return h->dp + o;
}

static char *headtoevr(RpmHead *h)
{
  unsigned int epoch;
  char *version, *v;
  char *release;
  char *evr;
  char *distepoch;

  version  = headstring(h, TAG_VERSION);
  release  = headstring(h, TAG_RELEASE);
  epoch = headint32(h, TAG_EPOCH);
  if (!version || !release)
    {
      fprintf(stderr, "headtoevr: bad rpm header\n");
      return 0;
    }
  for (v = version; *v >= '0' && *v <= '9'; v++)
    ;
  if (epoch || (v != version && *v == ':'))
    {
      char epochbuf[11];        /* 32bit decimal will fit in */
      sprintf(epochbuf, "%u", epoch);
      evr = solv_malloc(strlen(epochbuf) + 1 + strlen(version) + 1 + strlen(release) + 1);
      sprintf(evr, "%s:%s-%s", epochbuf, version, release);
    }
  else
    {
      evr = solv_malloc(strlen(version) + 1 + strlen(release) + 1);
      sprintf(evr, "%s-%s", version, release);
    }
  distepoch = headstring(h, TAG_DISTEPOCH);
  if (distepoch && *distepoch)
    {
      int l = strlen(evr);
      evr = solv_realloc(evr, l + strlen(distepoch) + 2);
      evr[l++] = ':';
      strcpy(evr + l, distepoch);
    }
  return evr;
}


static void
setutf8string(Repodata *repodata, Id handle, Id tag, const char *str)
{
  if (str[solv_validutf8(str)])
    {
      char *ustr = solv_latin1toutf8(str);	/* not utf8, assume latin1 */
      repodata_set_str(repodata, handle, tag, ustr);
      solv_free(ustr);
    }
  else
    repodata_set_str(repodata, handle, tag, str);
}


#define MAKEDEPS_FILTER_WEAK	(1 << 0)
#define MAKEDEPS_FILTER_STRONG	(1 << 1)
#define MAKEDEPS_NO_RPMLIB	(1 << 2)

/*
 * strong: 0: ignore strongness
 *         1: filter to strong
 *         2: filter to weak
 */
static unsigned int
makedeps(Pool *pool, Repo *repo, RpmHead *rpmhead, int tagn, int tagv, int tagf, int flags)
{
  char **n, **v;
  unsigned int *f;
  int i, cc, nc, vc, fc;
  int haspre, premask;
  unsigned int olddeps;
  Id *ida;
  int strong;

  strong = flags & (MAKEDEPS_FILTER_STRONG|MAKEDEPS_FILTER_WEAK);
  n = headstringarray(rpmhead, tagn, &nc);
  if (!n || !nc)
    return 0;
  vc = fc = 0;
  v = headstringarray(rpmhead, tagv, &vc);
  f = headint32array(rpmhead, tagf, &fc);
  if (!v || !f || nc != vc || nc != fc)
    {
      char *pkgname = rpm_query(rpmhead, 0);
      pool_error(pool, 0, "bad dependency entries for %s: %d %d %d", pkgname ? pkgname : "<NULL>", nc, vc, fc);
      solv_free(pkgname);
      solv_free(n);
      solv_free(v);
      solv_free(f);
      return 0;
    }

  cc = nc;
  haspre = 0;	/* add no prereq marker */
  premask = DEP_PRE_IN | DEP_PRE_UN;
  if (flags)
    {
      /* we do filtering */
      cc = 0;
      for (i = 0; i < nc; i++)
	{
	  if (strong && (f[i] & DEP_STRONG) != (strong == MAKEDEPS_FILTER_WEAK ? 0 : DEP_STRONG))
	    continue;
	  if ((flags & MAKEDEPS_NO_RPMLIB) != 0)
	    if (!strncmp(n[i], "rpmlib(", 7))
	      continue;
	  if ((f[i] & premask) != 0)
	    haspre = 1;
	  cc++;
	}
    }
  else if (tagn == TAG_REQUIRENAME)
    {
      /* no filtering, just look for the first prereq */
      for (i = 0; i < nc; i++)
	if ((f[i] & premask) != 0)
	  {
	    haspre = 1;
	    break;
	  }
    }
  if (cc == 0)
    {
      solv_free(n);
      solv_free(v);
      solv_free(f);
      return 0;
    }
  cc += haspre;		/* add slot for the prereq marker */
  olddeps = repo_reserve_ids(repo, 0, cc);
  ida = repo->idarraydata + olddeps;
  for (i = 0; ; i++)
    {
      if (i == nc)
	{
	  if (haspre != 1)
	    break;
	  haspre = 2;	/* pass two: prereqs */
	  i = 0;
	  *ida++ = SOLVABLE_PREREQMARKER;
	}
      if (strong && (f[i] & DEP_STRONG) != (strong == MAKEDEPS_FILTER_WEAK ? 0 : DEP_STRONG))
	continue;
      if (haspre == 1 && (f[i] & premask) != 0)
	continue;
      if (haspre == 2 && (f[i] & premask) == 0)
	continue;
      if ((flags & MAKEDEPS_NO_RPMLIB) != 0)
	if (!strncmp(n[i], "rpmlib(", 7))
	  continue;
      if (f[i] & (DEP_LESS|DEP_GREATER|DEP_EQUAL))
	{
	  Id name, evr;
	  int flags = 0;
	  if ((f[i] & DEP_LESS) != 0)
	    flags |= REL_LT;
	  if ((f[i] & DEP_EQUAL) != 0)
	    flags |= REL_EQ;
	  if ((f[i] & DEP_GREATER) != 0)
	    flags |= REL_GT;
	  name = pool_str2id(pool, n[i], 1);
	  if (v[i][0] == '0' && v[i][1] == ':' && v[i][2])
	    evr = pool_str2id(pool, v[i] + 2, 1);
	  else
	    evr = pool_str2id(pool, v[i], 1);
	  *ida++ = pool_rel2id(pool, name, evr, flags, 1);
	}
      else
        *ida++ = pool_str2id(pool, n[i], 1);
    }
  *ida++ = 0;
  repo->idarraysize += cc + 1;
  solv_free(n);
  solv_free(v);
  solv_free(f);
  return olddeps;
}


static void
adddudata(Repodata *data, Id handle, RpmHead *rpmhead, char **dn, unsigned int *di, int fc, int dc)
{
  Id did;
  int i, fszc;
  unsigned int *fkb, *fn, *fsz, *fm, *fino;
  unsigned long long *fsz64;
  unsigned int inotest[256], inotestok;

  if (!fc)
    return;
  if ((fsz64 = headint64array(rpmhead, TAG_LONGFILESIZES, &fszc)) != 0)
    {
      /* convert to kbyte */
      fsz = solv_malloc2(fszc, sizeof(*fsz));
      for (i = 0; i < fszc; i++)
        fsz[i] = fsz64[i] ? fsz64[i] / 1024 + 1 : 0;
      solv_free(fsz64);
    }
  else if ((fsz = headint32array(rpmhead, TAG_FILESIZES, &fszc)) != 0)
    {
      /* convert to kbyte */
      for (i = 0; i < fszc; i++)
        if (fsz[i])
	  fsz[i] = fsz[i] / 1024 + 1;
    }
  else
    return;
  if (fc != fszc)
    {
      solv_free(fsz);
      return;
    }

  /* stupid rpm records sizes of directories, so we have to check the mode */
  fm = headint16array(rpmhead, TAG_FILEMODES, &fszc);
  if (!fm || fc != fszc)
    {
      solv_free(fsz);
      solv_free(fm);
      return;
    }
  fino = headint32array(rpmhead, TAG_FILEINODES, &fszc);
  if (!fino || fc != fszc)
    {
      solv_free(fsz);
      solv_free(fm);
      solv_free(fino);
      return;
    }

  /* kill hardlinked entries */
  inotestok = 0;
  if (fc < sizeof(inotest))
    {
      /* quick test just hashing the inode numbers */
      memset(inotest, 0, sizeof(inotest));
      for (i = 0; i < fc; i++)
	{
	  int off, bit;
	  if (fsz[i] == 0 || !S_ISREG(fm[i]))
	    continue;	/* does not matter */
	  off = (fino[i] >> 5) & (sizeof(inotest)/sizeof(*inotest) - 1);
	  bit = 1 << (fino[i] & 31);
	  if ((inotest[off] & bit) != 0)
	    break;
	  inotest[off] |= bit;
	}
      if (i == fc)
	inotestok = 1;	/* no conflict found */
    }
  if (!inotestok)
    {
      /* hardlinked files are possible, check ino/dev pairs */
      unsigned int *fdev = headint32array(rpmhead, TAG_FILEDEVICES, &fszc);
      unsigned int *fx, j;
      unsigned int mask, hash, hh;
      if (!fdev || fc != fszc)
	{
	  solv_free(fsz);
	  solv_free(fm);
	  solv_free(fdev);
	  solv_free(fino);
	  return;
	}
      mask = fc;
      while ((mask & (mask - 1)) != 0)
	mask = mask & (mask - 1);
      mask <<= 2;
      if (mask > sizeof(inotest)/sizeof(*inotest))
        fx = solv_calloc(mask, sizeof(unsigned int));
      else
	{
	  fx = inotest;
	  memset(fx, 0, mask * sizeof(unsigned int));
	}
      mask--;
      for (i = 0; i < fc; i++)
	{
	  if (fsz[i] == 0 || !S_ISREG(fm[i]))
	    continue;
	  hash = (fino[i] + fdev[i] * 31) & mask;
          hh = 7;
	  while ((j = fx[hash]) != 0)
	    {
	      if (fino[j - 1] == fino[i] && fdev[j - 1] == fdev[i])
		{
		  fsz[i] = 0;	/* kill entry */
		  break;
		}
	      hash = (hash + hh++) & mask;
	    }
	  if (!j)
	    fx[hash] = i + 1;
	}
      if (fx != inotest)
        solv_free(fx);
      solv_free(fdev);
    }
  solv_free(fino);

  /* sum up inode count and kbytes for each directory */
  fn = solv_calloc(dc, sizeof(unsigned int));
  fkb = solv_calloc(dc, sizeof(unsigned int));
  for (i = 0; i < fc; i++)
    {
      if (di[i] >= dc)
	continue;	/* corrupt entry */
      fn[di[i]]++;
      if (fsz[i] == 0 || !S_ISREG(fm[i]))
	continue;
      fkb[di[i]] += fsz[i];
    }
  solv_free(fsz);
  solv_free(fm);
  /* commit */
  for (i = 0; i < dc; i++)
    {
      if (!fn[i])
	continue;
      if (!*dn[i])
	{
	  Solvable *s = data->repo->pool->solvables + handle;
          if (s->arch == ARCH_SRC || s->arch == ARCH_NOSRC)
	    did = repodata_str2dir(data, "/usr/src", 1);
	  else
	    continue;	/* work around rpm bug */
	}
      else
        did = repodata_str2dir(data, dn[i], 1);
      repodata_add_dirnumnum(data, handle, SOLVABLE_DISKUSAGE, did, fkb[i], fn[i]);
    }
  solv_free(fn);
  solv_free(fkb);
}

static void
addfilelist(Repodata *data, Id handle, RpmHead *rpmhead)
{
  char **bn;
  char **dn;
  unsigned int *di;
  int bnc, dnc, dic;
  int i;
  Id lastdid = 0;
  unsigned int lastdii = -1;

  if (!data)
    return;
  bn = headstringarray(rpmhead, TAG_BASENAMES, &bnc);
  if (!bn)
    return;
  dn = headstringarray(rpmhead, TAG_DIRNAMES, &dnc);
  if (!dn)
    {
      solv_free(bn);
      return;
    }
  di = headint32array(rpmhead, TAG_DIRINDEXES, &dic);
  if (!di)
    {
      solv_free(bn);
      solv_free(dn);
      return;
    }
  if (bnc != dic)
    {
      pool_error(data->repo->pool, 0, "bad filelist");
      return;
    }

  adddudata(data, handle, rpmhead, dn, di, bnc, dnc);

  for (i = 0; i < bnc; i++)
    {
      Id did;
      char *b = bn[i];

      if (di[i] == lastdii)
	did = lastdid;
      else
	{
	  if (di[i] >= dnc)
	    continue;	/* corrupt entry */
	  did = repodata_str2dir(data, dn[di[i]], 1);
	  if (!did)
	    did = repodata_str2dir(data, "/", 1);
	  lastdid = did;
	  lastdii = di[i];
	}
      if (b && *b == '/')	/* work around rpm bug */
	b++;
      repodata_add_dirstr(data, handle, SOLVABLE_FILELIST, did, b);
    }
  solv_free(bn);
  solv_free(dn);
  solv_free(di);
}

static void
addchangelog(Repodata *data, Id handle, RpmHead *rpmhead)
{
  char **cn;
  char **cx;
  unsigned int *ct;
  int i, cnc, cxc, ctc;
  Queue hq;

  ct = headint32array(rpmhead, TAG_CHANGELOGTIME, &ctc);
  cx = headstringarray(rpmhead, TAG_CHANGELOGTEXT, &cxc);
  cn = headstringarray(rpmhead, TAG_CHANGELOGNAME, &cnc);
  if (!ct || !cx || !cn || !ctc || ctc != cxc || ctc != cnc)
    {
      solv_free(ct);
      solv_free(cx);
      solv_free(cn);
      return;
    }
  queue_init(&hq);
  for (i = 0; i < ctc; i++)
    {
      Id h = repodata_new_handle(data);
      if (ct[i])
        repodata_set_num(data, h, SOLVABLE_CHANGELOG_TIME, ct[i]);
      if (cn[i])
        setutf8string(data, h, SOLVABLE_CHANGELOG_AUTHOR, cn[i]);
      if (cx[i])
        setutf8string(data, h, SOLVABLE_CHANGELOG_TEXT, cx[i]);
      queue_push(&hq, h);
    }
  for (i = 0; i < hq.count; i++)
    repodata_add_flexarray(data, handle, SOLVABLE_CHANGELOG, hq.elements[i]);
  queue_free(&hq);
  solv_free(ct);
  solv_free(cx);
  solv_free(cn);
}

static void
set_description_author(Repodata *data, Id handle, char *str)
{
  char *aut, *p;
  for (aut = str; (aut = strchr(aut, '\n')) != 0; aut++)
    if (!strncmp(aut, "\nAuthors:\n--------\n", 19))
      break;
  if (aut)
    {
      /* oh my, found SUSE special author section */
      int l = aut - str;
      str = solv_strdup(str);
      aut = str + l;
      str[l] = 0;
      while (l > 0 && str[l - 1] == '\n')
	str[--l] = 0;
      if (l)
	setutf8string(data, handle, SOLVABLE_DESCRIPTION, str);
      p = aut + 19;
      aut = str;	/* copy over */
      while (*p == ' ' || *p == '\n')
	p++;
      while (*p)
	{
	  if (*p == '\n')
	    {
	      *aut++ = *p++;
	      while (*p == ' ')
		p++;
	      continue;
	    }
	  *aut++ = *p++;
	}
      while (aut != str && aut[-1] == '\n')
	aut--;
      *aut = 0;
      if (*str)
	setutf8string(data, handle, SOLVABLE_AUTHORS, str);
      free(str);
    }
  else if (*str)
    setutf8string(data, handle, SOLVABLE_DESCRIPTION, str);
}

static int
rpm2solv(Pool *pool, Repo *repo, Repodata *data, Solvable *s, RpmHead *rpmhead, int flags)
{
  char *name;
  char *evr;
  char *sourcerpm;

  name = headstring(rpmhead, TAG_NAME);
  if (!name)
    {
      pool_error(pool, 0, "package has no name");
      return 0;
    }
  if (!strcmp(name, "gpg-pubkey"))
    return 0;
  s->name = pool_str2id(pool, name, 1);
  sourcerpm = headstring(rpmhead, TAG_SOURCERPM);
  if (sourcerpm || (rpmhead->forcebinary && !headexists(rpmhead, TAG_SOURCEPACKAGE)))
    s->arch = pool_str2id(pool, headstring(rpmhead, TAG_ARCH), 1);
  else
    {
      if (headexists(rpmhead, TAG_NOSOURCE) || headexists(rpmhead, TAG_NOPATCH))
        s->arch = ARCH_NOSRC;
      else
        s->arch = ARCH_SRC;
    }
  if (!s->arch)
    s->arch = ARCH_NOARCH;
  evr = headtoevr(rpmhead);
  s->evr = pool_str2id(pool, evr, 1);
  s->vendor = pool_str2id(pool, headstring(rpmhead, TAG_VENDOR), 1);

  s->provides = makedeps(pool, repo, rpmhead, TAG_PROVIDENAME, TAG_PROVIDEVERSION, TAG_PROVIDEFLAGS, 0);
  if (s->arch != ARCH_SRC && s->arch != ARCH_NOSRC)
    s->provides = repo_addid_dep(repo, s->provides, pool_rel2id(pool, s->name, s->evr, REL_EQ, 1), 0);
  s->requires = makedeps(pool, repo, rpmhead, TAG_REQUIRENAME, TAG_REQUIREVERSION, TAG_REQUIREFLAGS, (flags & RPM_ADD_NO_RPMLIBREQS) ? MAKEDEPS_NO_RPMLIB : 0);
  s->conflicts = makedeps(pool, repo, rpmhead, TAG_CONFLICTNAME, TAG_CONFLICTVERSION, TAG_CONFLICTFLAGS, 0);
  s->obsoletes = makedeps(pool, repo, rpmhead, TAG_OBSOLETENAME, TAG_OBSOLETEVERSION, TAG_OBSOLETEFLAGS, 0);

  s->recommends = makedeps(pool, repo, rpmhead, TAG_SUGGESTSNAME, TAG_SUGGESTSVERSION, TAG_SUGGESTSFLAGS, MAKEDEPS_FILTER_STRONG);
  s->suggests = makedeps(pool, repo, rpmhead, TAG_SUGGESTSNAME, TAG_SUGGESTSVERSION, TAG_SUGGESTSFLAGS, MAKEDEPS_FILTER_WEAK);
  s->supplements = makedeps(pool, repo, rpmhead, TAG_ENHANCESNAME, TAG_ENHANCESVERSION, TAG_ENHANCESFLAGS, MAKEDEPS_FILTER_STRONG);
  s->enhances  = makedeps(pool, repo, rpmhead, TAG_ENHANCESNAME, TAG_ENHANCESVERSION, TAG_ENHANCESFLAGS, MAKEDEPS_FILTER_WEAK);
  s->supplements = repo_fix_supplements(repo, s->provides, s->supplements, 0);
  s->conflicts = repo_fix_conflicts(repo, s->conflicts);

  if (data)
    {
      Id handle;
      char *str;
      unsigned int u32;
      unsigned long long u64;

      handle = s - pool->solvables;
      str = headstring(rpmhead, TAG_SUMMARY);
      if (str)
        setutf8string(data, handle, SOLVABLE_SUMMARY, str);
      str = headstring(rpmhead, TAG_DESCRIPTION);<commit_msg>adapt to new rpm tags for weak dependencies<commit_after>/*
 * Copyright (c) 2007-2012, Novell Inc.
 *
 * This program is licensed under the BSD license, read LICENSE.BSD
 * for further information
 */

/*
 * repo_rpmdb
 *
 * convert rpm db to repo
 *
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <limits.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <assert.h>
#include <stdint.h>
#include <errno.h>

#include <rpm/rpmio.h>
#include <rpm/rpmpgp.h>
#ifndef RPM5
#include <rpm/header.h>
#endif
#include <rpm/rpmdb.h>

#ifndef DB_CREATE
# if defined(SUSE) || defined(HAVE_RPM_DB_H)
#  include <rpm/db.h>
# else
#  include <db.h>
# endif
#endif

#include "pool.h"
#include "repo.h"
#include "hash.h"
#include "util.h"
#include "queue.h"
#include "chksum.h"
#include "repo_rpmdb.h"
#include "repo_solv.h"

/* 3: added triggers */
/* 4: fixed triggers */
/* 5: fixed checksum copying */
#define RPMDB_COOKIE_VERSION 5

#define TAG_NAME		1000
#define TAG_VERSION		1001
#define TAG_RELEASE		1002
#define TAG_EPOCH		1003
#define TAG_SUMMARY		1004
#define TAG_DESCRIPTION		1005
#define TAG_BUILDTIME		1006
#define TAG_BUILDHOST		1007
#define TAG_INSTALLTIME		1008
#define TAG_SIZE		1009
#define TAG_DISTRIBUTION	1010
#define TAG_VENDOR		1011
#define TAG_LICENSE		1014
#define TAG_PACKAGER		1015
#define TAG_GROUP		1016
#define TAG_URL			1020
#define TAG_ARCH		1022
#define TAG_FILESIZES		1028
#define TAG_FILEMODES		1030
#define TAG_FILEMD5S		1035
#define TAG_FILELINKTOS		1036
#define TAG_FILEFLAGS		1037
#define TAG_SOURCERPM		1044
#define TAG_PROVIDENAME		1047
#define TAG_REQUIREFLAGS	1048
#define TAG_REQUIRENAME		1049
#define TAG_REQUIREVERSION	1050
#define TAG_NOSOURCE		1051
#define TAG_NOPATCH		1052
#define TAG_CONFLICTFLAGS	1053
#define TAG_CONFLICTNAME	1054
#define TAG_CONFLICTVERSION	1055
#define TAG_TRIGGERNAME		1066
#define TAG_TRIGGERVERSION	1067
#define TAG_TRIGGERFLAGS	1068
#define TAG_CHANGELOGTIME	1080
#define TAG_CHANGELOGNAME	1081
#define TAG_CHANGELOGTEXT	1082
#define TAG_OBSOLETENAME	1090
#define TAG_FILEDEVICES		1095
#define TAG_FILEINODES		1096
#define TAG_SOURCEPACKAGE	1106
#define TAG_PROVIDEFLAGS	1112
#define TAG_PROVIDEVERSION	1113
#define TAG_OBSOLETEFLAGS	1114
#define TAG_OBSOLETEVERSION	1115
#define TAG_DIRINDEXES		1116
#define TAG_BASENAMES		1117
#define TAG_DIRNAMES		1118
#define TAG_PAYLOADFORMAT	1124
#define TAG_PATCHESNAME		1133
#define TAG_FILECOLORS		1140
#define TAG_OLDSUGGESTSNAME	1156
#define TAG_OLDSUGGESTSVERSION	1157
#define TAG_OLDSUGGESTSFLAGS	1158
#define TAG_OLDENHANCESNAME	1159
#define TAG_OLDENHANCESVERSION	1160
#define TAG_OLDENHANCESFLAGS	1161

/* rpm5 tags */
#define TAG_DISTEPOCH		1218

/* rpm4 tags */
#define TAG_LONGFILESIZES	5008
#define TAG_LONGSIZE		5009
#define TAG_RECOMMENDNAME	5046
#define TAG_RECOMMENDVERSION	5047
#define TAG_RECOMMENDFLAGS	5048
#define TAG_SUGGESTNAME		5049
#define TAG_SUGGESTVERSION	5050
#define TAG_SUGGESTFLAGS	5051
#define TAG_SUPPLEMENTNAME	5052
#define TAG_SUPPLEMENTVERSION	5053
#define TAG_SUPPLEMENTFLAGS	5054
#define TAG_ENHANCENAME		5055
#define TAG_ENHANCEVERSION	5056
#define TAG_ENHANCEFLAGS	5057

/* signature tags */
#define	TAG_SIGBASE		256
#define TAG_SIGMD5		(TAG_SIGBASE + 5)
#define TAG_SHA1HEADER		(TAG_SIGBASE + 13)

#define SIGTAG_SIZE		1000
#define SIGTAG_PGP		1002	/* RSA signature */
#define SIGTAG_MD5		1004	/* header+payload md5 checksum */
#define SIGTAG_GPG		1005	/* DSA signature */

#define DEP_LESS		(1 << 1)
#define DEP_GREATER		(1 << 2)
#define DEP_EQUAL		(1 << 3)
#define DEP_STRONG		(1 << 27)
#define DEP_PRE_IN		((1 << 6) | (1 << 9) | (1 << 10))
#define DEP_PRE_UN		((1 << 6) | (1 << 11) | (1 << 12))

#define FILEFLAG_GHOST		(1 <<  6)


#ifdef RPM5
# define RPM_INDEX_SIZE 4	/* just the rpmdbid */
#else
# define RPM_INDEX_SIZE 8	/* rpmdbid + array index */
#endif


typedef struct rpmhead {
  int cnt;
  int dcnt;
  unsigned char *dp;
  int forcebinary;		/* sigh, see rh#478907 */
  unsigned char data[1];
} RpmHead;


static inline unsigned char *
headfindtag(RpmHead *h, int tag)
{
  unsigned int i;
  unsigned char *d, taga[4];
  d = h->dp - 16;
  taga[0] = tag >> 24;
  taga[1] = tag >> 16;
  taga[2] = tag >> 8;
  taga[3] = tag;
  for (i = 0; i < h->cnt; i++, d -= 16)
    if (d[3] == taga[3] && d[2] == taga[2] && d[1] == taga[1] && d[0] == taga[0])
      return d;
  return 0;
}

static int
headexists(RpmHead *h, int tag)
{
  return headfindtag(h, tag) ? 1 : 0;
}

static unsigned int *
headint32array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o, *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 4)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned int));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 4)
    r[o] = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
  return r;
}

/* returns the first entry of an integer array */
static unsigned int
headint32(RpmHead *h, int tag)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 4)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (i == 0 || o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  return d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
}

static unsigned long long *
headint64array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o;
  unsigned long long *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 5)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 8 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned long long));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 8)
    {
      unsigned int x = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
      r[o] = (unsigned long long)x << 32 | (d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7]);
    }
  return r;
}

/* returns the first entry of an 64bit integer array */
static unsigned long long
headint64(RpmHead *h, int tag)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 5)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (i == 0 || o + 8 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  i = d[0] << 24 | d[1] << 16 | d[2] << 8 | d[3];
  return (unsigned long long)i << 32 | (d[4] << 24 | d[5] << 16 | d[6] << 8 | d[7]);
}

static unsigned int *
headint16array(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o, *r;
  unsigned char *d = headfindtag(h, tag);

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 3)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o + 4 * i > h->dcnt)
    return 0;
  d = h->dp + o;
  r = solv_calloc(i ? i : 1, sizeof(unsigned int));
  if (cnt)
    *cnt = i;
  for (o = 0; o < i; o++, d += 2)
    r[o] = d[0] << 8 | d[1];
  return r;
}

static char *
headstring(RpmHead *h, int tag)
{
  unsigned int o;
  unsigned char *d = headfindtag(h, tag);
  /* 6: STRING, 9: I18NSTRING */
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || (d[7] != 6 && d[7] != 9))
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  if (o >= h->dcnt)
    return 0;
  return (char *)h->dp + o;
}

static char **
headstringarray(RpmHead *h, int tag, int *cnt)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  char **r;

  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 8)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  r = solv_calloc(i ? i : 1, sizeof(char *));
  if (cnt)
    *cnt = i;
  d = h->dp + o;
  for (o = 0; o < i; o++)
    {
      r[o] = (char *)d;
      if (o + 1 < i)
        d += strlen((char *)d) + 1;
      if (d >= h->dp + h->dcnt)
        {
          solv_free(r);
          return 0;
        }
    }
  return r;
}

static unsigned char *
headbinary(RpmHead *h, int tag, unsigned int *sizep)
{
  unsigned int i, o;
  unsigned char *d = headfindtag(h, tag);
  if (!d || d[4] != 0 || d[5] != 0 || d[6] != 0 || d[7] != 7)
    return 0;
  o = d[8] << 24 | d[9] << 16 | d[10] << 8 | d[11];
  i = d[12] << 24 | d[13] << 16 | d[14] << 8 | d[15];
  if (o > h->dcnt || o + i < o || o + i > h->dcnt)
    return 0;
  if (sizep)
    *sizep = i;
  return h->dp + o;
}

static char *headtoevr(RpmHead *h)
{
  unsigned int epoch;
  char *version, *v;
  char *release;
  char *evr;
  char *distepoch;

  version  = headstring(h, TAG_VERSION);
  release  = headstring(h, TAG_RELEASE);
  epoch = headint32(h, TAG_EPOCH);
  if (!version || !release)
    {
      fprintf(stderr, "headtoevr: bad rpm header\n");
      return 0;
    }
  for (v = version; *v >= '0' && *v <= '9'; v++)
    ;
  if (epoch || (v != version && *v == ':'))
    {
      char epochbuf[11];        /* 32bit decimal will fit in */
      sprintf(epochbuf, "%u", epoch);
      evr = solv_malloc(strlen(epochbuf) + 1 + strlen(version) + 1 + strlen(release) + 1);
      sprintf(evr, "%s:%s-%s", epochbuf, version, release);
    }
  else
    {
      evr = solv_malloc(strlen(version) + 1 + strlen(release) + 1);
      sprintf(evr, "%s-%s", version, release);
    }
  distepoch = headstring(h, TAG_DISTEPOCH);
  if (distepoch && *distepoch)
    {
      int l = strlen(evr);
      evr = solv_realloc(evr, l + strlen(distepoch) + 2);
      evr[l++] = ':';
      strcpy(evr + l, distepoch);
    }
  return evr;
}


static void
setutf8string(Repodata *repodata, Id handle, Id tag, const char *str)
{
  if (str[solv_validutf8(str)])
    {
      char *ustr = solv_latin1toutf8(str);	/* not utf8, assume latin1 */
      repodata_set_str(repodata, handle, tag, ustr);
      solv_free(ustr);
    }
  else
    repodata_set_str(repodata, handle, tag, str);
}


/*
 * strong: 0: ignore strongness
 *         1: filter to strong
 *         2: filter to weak
 */
static unsigned int
makedeps(Pool *pool, Repo *repo, RpmHead *rpmhead, int tagn, int tagv, int tagf, int flags)
{
  char **n, **v;
  unsigned int *f;
  int i, cc, nc, vc, fc;
  int haspre, premask;
  unsigned int olddeps;
  Id *ida;
  int strong = 0;

  n = headstringarray(rpmhead, tagn, &nc);
  if (!n)
    {
      switch (tagn)
	{
	case TAG_SUGGESTNAME:
	  tagn = TAG_OLDSUGGESTSNAME;
	  tagv = TAG_OLDSUGGESTSVERSION;
	  tagf = TAG_OLDSUGGESTSFLAGS;
	  strong = -1;
	  break;
	case TAG_ENHANCENAME:
	  tagn = TAG_OLDENHANCESNAME;
	  tagv = TAG_OLDENHANCESVERSION;
	  tagf = TAG_OLDENHANCESFLAGS;
	  strong = -1;
	  break;
	case TAG_RECOMMENDNAME:
	  tagn = TAG_OLDSUGGESTSNAME;
	  tagv = TAG_OLDSUGGESTSVERSION;
	  tagf = TAG_OLDSUGGESTSFLAGS;
	  strong = 1;
	  break;
	case TAG_SUPPLEMENTNAME:
	  tagn = TAG_OLDENHANCESNAME;
	  tagv = TAG_OLDENHANCESVERSION;
	  tagf = TAG_OLDENHANCESFLAGS;
	  strong = 1;
	  break;
	default:
	  return 0;
	}
      n = headstringarray(rpmhead, tagn, &nc);
    }
  if (!n || !nc)
    return 0;
  vc = fc = 0;
  v = headstringarray(rpmhead, tagv, &vc);
  f = headint32array(rpmhead, tagf, &fc);
  if (!v || !f || nc != vc || nc != fc)
    {
      char *pkgname = rpm_query(rpmhead, 0);
      pool_error(pool, 0, "bad dependency entries for %s: %d %d %d", pkgname ? pkgname : "<NULL>", nc, vc, fc);
      solv_free(pkgname);
      solv_free(n);
      solv_free(v);
      solv_free(f);
      return 0;
    }

  cc = nc;
  haspre = 0;	/* add no prereq marker */
  premask = tagn == TAG_REQUIRENAME ? DEP_PRE_IN | DEP_PRE_UN : 0;
  if ((flags & RPM_ADD_NO_RPMLIBREQS) || strong)
    {
      /* we do filtering */
      cc = 0;
      for (i = 0; i < nc; i++)
	{
	  if (strong && (f[i] & DEP_STRONG) != (strong < 0 ? 0 : DEP_STRONG))
	    continue;
	  if ((flags & RPM_ADD_NO_RPMLIBREQS) != 0)
	    if (!strncmp(n[i], "rpmlib(", 7))
	      continue;
	  if ((f[i] & premask) != 0)
	    haspre = 1;
	  cc++;
	}
    }
  else if (premask)
    {
      /* no filtering, just look for the first prereq */
      for (i = 0; i < nc; i++)
	if ((f[i] & premask) != 0)
	  {
	    haspre = 1;
	    break;
	  }
    }
  if (cc == 0)
    {
      solv_free(n);
      solv_free(v);
      solv_free(f);
      return 0;
    }
  cc += haspre;		/* add slot for the prereq marker */
  olddeps = repo_reserve_ids(repo, 0, cc);
  ida = repo->idarraydata + olddeps;
  for (i = 0; ; i++)
    {
      if (i == nc)
	{
	  if (haspre != 1)
	    break;
	  haspre = 2;	/* pass two: prereqs */
	  i = 0;
	  *ida++ = SOLVABLE_PREREQMARKER;
	}
      if (strong && (f[i] & DEP_STRONG) != (strong < 0 ? 0 : DEP_STRONG))
	continue;
      if (haspre)
	{
	  if (haspre == 1 && (f[i] & premask) != 0)
	    continue;
	  if (haspre == 2 && (f[i] & premask) == 0)
	    continue;
	}
      if ((flags & RPM_ADD_NO_RPMLIBREQS) != 0)
	if (!strncmp(n[i], "rpmlib(", 7))
	  continue;
      if (f[i] & (DEP_LESS|DEP_GREATER|DEP_EQUAL))
	{
	  Id name, evr;
	  int fl = 0;
	  if ((f[i] & DEP_LESS) != 0)
	    fl |= REL_LT;
	  if ((f[i] & DEP_EQUAL) != 0)
	    fl |= REL_EQ;
	  if ((f[i] & DEP_GREATER) != 0)
	    fl |= REL_GT;
	  name = pool_str2id(pool, n[i], 1);
	  if (v[i][0] == '0' && v[i][1] == ':' && v[i][2])
	    evr = pool_str2id(pool, v[i] + 2, 1);
	  else
	    evr = pool_str2id(pool, v[i], 1);
	  *ida++ = pool_rel2id(pool, name, evr, fl, 1);
	}
      else
        *ida++ = pool_str2id(pool, n[i], 1);
    }
  *ida++ = 0;
  repo->idarraysize += cc + 1;
  solv_free(n);
  solv_free(v);
  solv_free(f);
  return olddeps;
}


static void
adddudata(Repodata *data, Id handle, RpmHead *rpmhead, char **dn, unsigned int *di, int fc, int dc)
{
  Id did;
  int i, fszc;
  unsigned int *fkb, *fn, *fsz, *fm, *fino;
  unsigned long long *fsz64;
  unsigned int inotest[256], inotestok;

  if (!fc)
    return;
  if ((fsz64 = headint64array(rpmhead, TAG_LONGFILESIZES, &fszc)) != 0)
    {
      /* convert to kbyte */
      fsz = solv_malloc2(fszc, sizeof(*fsz));
      for (i = 0; i < fszc; i++)
        fsz[i] = fsz64[i] ? fsz64[i] / 1024 + 1 : 0;
      solv_free(fsz64);
    }
  else if ((fsz = headint32array(rpmhead, TAG_FILESIZES, &fszc)) != 0)
    {
      /* convert to kbyte */
      for (i = 0; i < fszc; i++)
        if (fsz[i])
	  fsz[i] = fsz[i] / 1024 + 1;
    }
  else
    return;
  if (fc != fszc)
    {
      solv_free(fsz);
      return;
    }

  /* stupid rpm records sizes of directories, so we have to check the mode */
  fm = headint16array(rpmhead, TAG_FILEMODES, &fszc);
  if (!fm || fc != fszc)
    {
      solv_free(fsz);
      solv_free(fm);
      return;
    }
  fino = headint32array(rpmhead, TAG_FILEINODES, &fszc);
  if (!fino || fc != fszc)
    {
      solv_free(fsz);
      solv_free(fm);
      solv_free(fino);
      return;
    }

  /* kill hardlinked entries */
  inotestok = 0;
  if (fc < sizeof(inotest))
    {
      /* quick test just hashing the inode numbers */
      memset(inotest, 0, sizeof(inotest));
      for (i = 0; i < fc; i++)
	{
	  int off, bit;
	  if (fsz[i] == 0 || !S_ISREG(fm[i]))
	    continue;	/* does not matter */
	  off = (fino[i] >> 5) & (sizeof(inotest)/sizeof(*inotest) - 1);
	  bit = 1 << (fino[i] & 31);
	  if ((inotest[off] & bit) != 0)
	    break;
	  inotest[off] |= bit;
	}
      if (i == fc)
	inotestok = 1;	/* no conflict found */
    }
  if (!inotestok)
    {
      /* hardlinked files are possible, check ino/dev pairs */
      unsigned int *fdev = headint32array(rpmhead, TAG_FILEDEVICES, &fszc);
      unsigned int *fx, j;
      unsigned int mask, hash, hh;
      if (!fdev || fc != fszc)
	{
	  solv_free(fsz);
	  solv_free(fm);
	  solv_free(fdev);
	  solv_free(fino);
	  return;
	}
      mask = fc;
      while ((mask & (mask - 1)) != 0)
	mask = mask & (mask - 1);
      mask <<= 2;
      if (mask > sizeof(inotest)/sizeof(*inotest))
        fx = solv_calloc(mask, sizeof(unsigned int));
      else
	{
	  fx = inotest;
	  memset(fx, 0, mask * sizeof(unsigned int));
	}
      mask--;
      for (i = 0; i < fc; i++)
	{
	  if (fsz[i] == 0 || !S_ISREG(fm[i]))
	    continue;
	  hash = (fino[i] + fdev[i] * 31) & mask;
          hh = 7;
	  while ((j = fx[hash]) != 0)
	    {
	      if (fino[j - 1] == fino[i] && fdev[j - 1] == fdev[i])
		{
		  fsz[i] = 0;	/* kill entry */
		  break;
		}
	      hash = (hash + hh++) & mask;
	    }
	  if (!j)
	    fx[hash] = i + 1;
	}
      if (fx != inotest)
        solv_free(fx);
      solv_free(fdev);
    }
  solv_free(fino);

  /* sum up inode count and kbytes for each directory */
  fn = solv_calloc(dc, sizeof(unsigned int));
  fkb = solv_calloc(dc, sizeof(unsigned int));
  for (i = 0; i < fc; i++)
    {
      if (di[i] >= dc)
	continue;	/* corrupt entry */
      fn[di[i]]++;
      if (fsz[i] == 0 || !S_ISREG(fm[i]))
	continue;
      fkb[di[i]] += fsz[i];
    }
  solv_free(fsz);
  solv_free(fm);
  /* commit */
  for (i = 0; i < dc; i++)
    {
      if (!fn[i])
	continue;
      if (!*dn[i])
	{
	  Solvable *s = data->repo->pool->solvables + handle;
          if (s->arch == ARCH_SRC || s->arch == ARCH_NOSRC)
	    did = repodata_str2dir(data, "/usr/src", 1);
	  else
	    continue;	/* work around rpm bug */
	}
      else
        did = repodata_str2dir(data, dn[i], 1);
      repodata_add_dirnumnum(data, handle, SOLVABLE_DISKUSAGE, did, fkb[i], fn[i]);
    }
  solv_free(fn);
  solv_free(fkb);
}

static void
addfilelist(Repodata *data, Id handle, RpmHead *rpmhead)
{
  char **bn;
  char **dn;
  unsigned int *di;
  int bnc, dnc, dic;
  int i;
  Id lastdid = 0;
  unsigned int lastdii = -1;

  if (!data)
    return;
  bn = headstringarray(rpmhead, TAG_BASENAMES, &bnc);
  if (!bn)
    return;
  dn = headstringarray(rpmhead, TAG_DIRNAMES, &dnc);
  if (!dn)
    {
      solv_free(bn);
      return;
    }
  di = headint32array(rpmhead, TAG_DIRINDEXES, &dic);
  if (!di)
    {
      solv_free(bn);
      solv_free(dn);
      return;
    }
  if (bnc != dic)
    {
      pool_error(data->repo->pool, 0, "bad filelist");
      return;
    }

  adddudata(data, handle, rpmhead, dn, di, bnc, dnc);

  for (i = 0; i < bnc; i++)
    {
      Id did;
      char *b = bn[i];

      if (di[i] == lastdii)
	did = lastdid;
      else
	{
	  if (di[i] >= dnc)
	    continue;	/* corrupt entry */
	  did = repodata_str2dir(data, dn[di[i]], 1);
	  if (!did)
	    did = repodata_str2dir(data, "/", 1);
	  lastdid = did;
	  lastdii = di[i];
	}
      if (b && *b == '/')	/* work around rpm bug */
	b++;
      repodata_add_dirstr(data, handle, SOLVABLE_FILELIST, did, b);
    }
  solv_free(bn);
  solv_free(dn);
  solv_free(di);
}

static void
addchangelog(Repodata *data, Id handle, RpmHead *rpmhead)
{
  char **cn;
  char **cx;
  unsigned int *ct;
  int i, cnc, cxc, ctc;
  Queue hq;

  ct = headint32array(rpmhead, TAG_CHANGELOGTIME, &ctc);
  cx = headstringarray(rpmhead, TAG_CHANGELOGTEXT, &cxc);
  cn = headstringarray(rpmhead, TAG_CHANGELOGNAME, &cnc);
  if (!ct || !cx || !cn || !ctc || ctc != cxc || ctc != cnc)
    {
      solv_free(ct);
      solv_free(cx);
      solv_free(cn);
      return;
    }
  queue_init(&hq);
  for (i = 0; i < ctc; i++)
    {
      Id h = repodata_new_handle(data);
      if (ct[i])
        repodata_set_num(data, h, SOLVABLE_CHANGELOG_TIME, ct[i]);
      if (cn[i])
        setutf8string(data, h, SOLVABLE_CHANGELOG_AUTHOR, cn[i]);
      if (cx[i])
        setutf8string(data, h, SOLVABLE_CHANGELOG_TEXT, cx[i]);
      queue_push(&hq, h);
    }
  for (i = 0; i < hq.count; i++)
    repodata_add_flexarray(data, handle, SOLVABLE_CHANGELOG, hq.elements[i]);
  queue_free(&hq);
  solv_free(ct);
  solv_free(cx);
  solv_free(cn);
}

static void
set_description_author(Repodata *data, Id handle, char *str)
{
  char *aut, *p;
  for (aut = str; (aut = strchr(aut, '\n')) != 0; aut++)
    if (!strncmp(aut, "\nAuthors:\n--------\n", 19))
      break;
  if (aut)
    {
      /* oh my, found SUSE special author section */
      int l = aut - str;
      str = solv_strdup(str);
      aut = str + l;
      str[l] = 0;
      while (l > 0 && str[l - 1] == '\n')
	str[--l] = 0;
      if (l)
	setutf8string(data, handle, SOLVABLE_DESCRIPTION, str);
      p = aut + 19;
      aut = str;	/* copy over */
      while (*p == ' ' || *p == '\n')
	p++;
      while (*p)
	{
	  if (*p == '\n')
	    {
	      *aut++ = *p++;
	      while (*p == ' ')
		p++;
	      continue;
	    }
	  *aut++ = *p++;
	}
      while (aut != str && aut[-1] == '\n')
	aut--;
      *aut = 0;
      if (*str)
	setutf8string(data, handle, SOLVABLE_AUTHORS, str);
      free(str);
    }
  else if (*str)
    setutf8string(data, handle, SOLVABLE_DESCRIPTION, str);
}

static int
rpm2solv(Pool *pool, Repo *repo, Repodata *data, Solvable *s, RpmHead *rpmhead, int flags)
{
  char *name;
  char *evr;
  char *sourcerpm;

  name = headstring(rpmhead, TAG_NAME);
  if (!name)
    {
      pool_error(pool, 0, "package has no name");
      return 0;
    }
  if (!strcmp(name, "gpg-pubkey"))
    return 0;
  s->name = pool_str2id(pool, name, 1);
  sourcerpm = headstring(rpmhead, TAG_SOURCERPM);
  if (sourcerpm || (rpmhead->forcebinary && !headexists(rpmhead, TAG_SOURCEPACKAGE)))
    s->arch = pool_str2id(pool, headstring(rpmhead, TAG_ARCH), 1);
  else
    {
      if (headexists(rpmhead, TAG_NOSOURCE) || headexists(rpmhead, TAG_NOPATCH))
        s->arch = ARCH_NOSRC;
      else
        s->arch = ARCH_SRC;
    }
  if (!s->arch)
    s->arch = ARCH_NOARCH;
  evr = headtoevr(rpmhead);
  s->evr = pool_str2id(pool, evr, 1);
  s->vendor = pool_str2id(pool, headstring(rpmhead, TAG_VENDOR), 1);

  s->provides = makedeps(pool, repo, rpmhead, TAG_PROVIDENAME, TAG_PROVIDEVERSION, TAG_PROVIDEFLAGS, 0);
  if (s->arch != ARCH_SRC && s->arch != ARCH_NOSRC)
    s->provides = repo_addid_dep(repo, s->provides, pool_rel2id(pool, s->name, s->evr, REL_EQ, 1), 0);
  s->requires = makedeps(pool, repo, rpmhead, TAG_REQUIRENAME, TAG_REQUIREVERSION, TAG_REQUIREFLAGS, flags);
  s->conflicts = makedeps(pool, repo, rpmhead, TAG_CONFLICTNAME, TAG_CONFLICTVERSION, TAG_CONFLICTFLAGS, 0);
  s->obsoletes = makedeps(pool, repo, rpmhead, TAG_OBSOLETENAME, TAG_OBSOLETEVERSION, TAG_OBSOLETEFLAGS, 0);

  s->recommends = makedeps(pool, repo, rpmhead, TAG_RECOMMENDNAME, TAG_RECOMMENDVERSION, TAG_RECOMMENDFLAGS, 0);
  s->suggests = makedeps(pool, repo, rpmhead, TAG_SUGGESTNAME, TAG_SUGGESTVERSION, TAG_SUGGESTFLAGS, 0);
  s->supplements = makedeps(pool, repo, rpmhead, TAG_SUPPLEMENTNAME, TAG_SUPPLEMENTVERSION, TAG_SUPPLEMENTFLAGS, 0);
  s->enhances  = makedeps(pool, repo, rpmhead, TAG_ENHANCENAME, TAG_ENHANCEVERSION, TAG_ENHANCEFLAGS, 0);

  s->supplements = repo_fix_supplements(repo, s->provides, s->supplements, 0);
  s->conflicts = repo_fix_conflicts(repo, s->conflicts);

  if (data)
    {
      Id handle;
      char *str;
      unsigned int u32;
      unsigned long long u64;

      handle = s - pool->solvables;
      str = headstring(rpmhead, TAG_SUMMARY);
      if (str)
        setutf8string(data, handle, SOLVABLE_SUMMARY, str);
      str = headstring(rpmhead, TAG_DESCRIPTION);
{'fraction_non_alphanumeric': 0.1710026564798349, 'fraction_numerical': 0.04140595841840655, 'mean_word_length': 2.7825942534462715, 'pattern_counts': {'":': 0, '<': 403, '<?xml version=': 0, '>': 199, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6490906', 'n_tokens_mistral': 21303, 'n_tokens_neox': 19430, 'n_words': 4962}