<commit_before>/**************************************************************************
 *
 * Copyright 2011 <NAME> <<EMAIL>>
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/

/**
 * This module uploads user buffers and translates the vertex buffers which
 * contain incompatible vertices (i.e. not supported by the driver/hardware)
 * into compatible ones, based on the Gallium CAPs.
 *
 * It does not upload index buffers.
 *
 * The module heavily uses bitmasks to represent per-buffer and
 * per-vertex-element flags to avoid looping over the list of buffers just
 * to see if there's a non-zero stride, or user buffer, or unsupported format,
 * etc.
 *
 * There are 3 categories of vertex elements, which are processed separately:
 * - per-vertex attribs (stride != 0, instance_divisor == 0)
 * - instanced attribs (stride != 0, instance_divisor > 0)
 * - constant attribs (stride == 0)
 *
 * All needed uploads and translations are performed every draw command, but
 * only the subset of vertices needed for that draw command is uploaded or
 * translated. (the module never translates whole buffers)
 *
 *
 * The module consists of two main parts:
 *
 *
 * 1) Translate (u_vbuf_translate_begin/end)
 *
 * This is pretty much a vertex fetch fallback. It translates vertices from
 * one vertex buffer to another in an unused vertex buffer slot. It does
 * whatever is needed to make the vertices readable by the hardware (changes
 * vertex formats and aligns offsets and strides). The translate module is
 * used here.
 *
 * Each of the 3 categories is translated to a separate buffer.
 * Only the [min_index, max_index] range is translated. For instanced attribs,
 * the range is [start_instance, start_instance+instance_count]. For constant
 * attribs, the range is [0, 1].
 *
 *
 * 2) User buffer uploading (u_vbuf_upload_buffers)
 *
 * Only the [min_index, max_index] range is uploaded (just like Translate)
 * with a single memcpy.
 *
 * This method works best for non-indexed draw operations or indexed draw
 * operations where the [min_index, max_index] range is not being way bigger
 * than the vertex count.
 *
 * If the range is too big (e.g. one triangle with indices {0, 1, 10000}),
 * the per-vertex attribs are uploaded via the translate module, all packed
 * into one vertex buffer, and the indexed draw call is turned into
 * a non-indexed one in the process. This adds additional complexity
 * to the translate part, but it prevents bad apps from bringing your frame
 * rate down.
 *
 *
 * If there is nothing to do, it forwards every command to the driver.
 * The module also has its own CSO cache of vertex element states.
 */

#include "util/u_vbuf.h"

#include "util/u_dump.h"
#include "util/u_format.h"
#include "util/u_inlines.h"
#include "util/u_memory.h"
#include "util/u_upload_mgr.h"
#include "translate/translate.h"
#include "translate/translate_cache.h"
#include "cso_cache/cso_cache.h"
#include "cso_cache/cso_hash.h"

struct u_vbuf_elements {
   unsigned count;
   struct pipe_vertex_element ve[PIPE_MAX_ATTRIBS];

   unsigned src_format_size[PIPE_MAX_ATTRIBS];

   /* If (velem[i].src_format != native_format[i]), the vertex buffer
    * referenced by the vertex element cannot be used for rendering and
    * its vertex data must be translated to native_format[i]. */
   enum pipe_format native_format[PIPE_MAX_ATTRIBS];
   unsigned native_format_size[PIPE_MAX_ATTRIBS];

   /* Which buffers are used by the vertex element state. */
   uint32_t used_vb_mask;
   /* This might mean two things:
    * - src_format != native_format, as discussed above.
    * - src_offset % 4 != 0 (if the caps don't allow such an offset). */
   uint32_t incompatible_elem_mask; /* each bit describes a corresp. attrib  */
   /* Which buffer has at least one vertex element referencing it
    * incompatible. */
   uint32_t incompatible_vb_mask_any;
   /* Which buffer has all vertex elements referencing it incompatible. */
   uint32_t incompatible_vb_mask_all;
   /* Which buffer has at least one vertex element referencing it
    * compatible. */
   uint32_t compatible_vb_mask_any;
   /* Which buffer has all vertex elements referencing it compatible. */
   uint32_t compatible_vb_mask_all;

   /* Which buffer has at least one vertex element referencing it
    * non-instanced. */
   uint32_t noninstance_vb_mask_any;

   void *driver_cso;
};

enum {
   VB_VERTEX = 0,
   VB_INSTANCE = 1,
   VB_CONST = 2,
   VB_NUM = 3
};

struct u_vbuf {
   struct u_vbuf_caps caps;

   struct pipe_context *pipe;
   struct translate_cache *translate_cache;
   struct cso_cache *cso_cache;
   struct u_upload_mgr *uploader;

   /* This is what was set in set_vertex_buffers.
    * May contain user buffers. */
   struct pipe_vertex_buffer vertex_buffer[PIPE_MAX_ATTRIBS];
   uint32_t enabled_vb_mask;

   /* Saved vertex buffer. */
   unsigned aux_vertex_buffer_slot;
   struct pipe_vertex_buffer aux_vertex_buffer_saved;

   /* Vertex buffers for the driver.
    * There are usually no user buffers. */
   struct pipe_vertex_buffer real_vertex_buffer[PIPE_MAX_ATTRIBS];
   uint32_t dirty_real_vb_mask; /* which buffers are dirty since the last
                                   call of set_vertex_buffers */

   /* The index buffer. */
   struct pipe_index_buffer index_buffer;

   /* Vertex elements. */
   struct u_vbuf_elements *ve, *ve_saved;

   /* Vertex elements used for the translate fallback. */
   struct pipe_vertex_element fallback_velems[PIPE_MAX_ATTRIBS];
   /* If non-NULL, this is a vertex element state used for the translate
    * fallback and therefore used for rendering too. */
   boolean using_translate;
   /* The vertex buffer slot index where translated vertices have been
    * stored in. */
   unsigned fallback_vbs[VB_NUM];

   /* Which buffer is a user buffer. */
   uint32_t user_vb_mask; /* each bit describes a corresp. buffer */
   /* Which buffer is incompatible (unaligned). */
   uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
   /* Which buffer has a non-zero stride. */
   uint32_t nonzero_stride_vb_mask; /* each bit describes a corresp. buffer */
};

static void *
u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
                              const struct pipe_vertex_element *attribs);
static void u_vbuf_delete_vertex_elements(struct u_vbuf *mgr, void *cso);

static const struct {
   enum pipe_format from, to;
} vbuf_format_fallbacks[] = {
   { PIPE_FORMAT_R32_FIXED,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_FIXED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_FIXED,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_FIXED,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_FLOAT,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_FLOAT,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_FLOAT,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_FLOAT,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R64_FLOAT,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R64G64_FLOAT,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R64G64B64_FLOAT,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R64G64B64A64_FLOAT,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_UNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_UNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_UNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_UNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_SNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_SNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_SNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_SNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_USCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_USCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_USCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_SSCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_SSCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_SSCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_UNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_UNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_UNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_UNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_SNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_SNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_SNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_SNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_USCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_USCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_USCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_SSCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_SSCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_SSCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_UNORM,             PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_UNORM,           PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_UNORM,         PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_UNORM,       PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_SNORM,             PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_SNORM,           PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_SNORM,         PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_SNORM,       PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_USCALED,           PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_USCALED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_USCALED,       PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_USCALED,     PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_SSCALED,           PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_SSCALED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_SSCALED,       PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_SSCALED,     PIPE_FORMAT_R32G32B32A32_FLOAT },
};

boolean u_vbuf_get_caps(struct pipe_screen *screen, struct u_vbuf_caps *caps)
{
   unsigned i;
   boolean fallback = FALSE;

   /* I'd rather have a bitfield of which formats are supported and a static
    * table of the translations indexed by format, but since we don't have C99
    * we can't easily make a sparsely-populated table indexed by format.  So,
    * we construct the sparse table here.
    */
   for (i = 0; i < PIPE_FORMAT_COUNT; i++)
      caps->format_translation[i] = i;

   for (i = 0; i < Elements(vbuf_format_fallbacks); i++) {
      enum pipe_format format = vbuf_format_fallbacks[i].from;

      if (!screen->is_format_supported(screen, format, PIPE_BUFFER, 0,
                                       PIPE_BIND_VERTEX_BUFFER)) {
         caps->format_translation[format] = vbuf_format_fallbacks[i].to;
         fallback = TRUE;
      }
   }

   caps->buffer_offset_unaligned =
      !screen->get_param(screen,
                         PIPE_CAP_VERTEX_BUFFER_OFFSET_4BYTE_ALIGNED_ONLY);
   caps->buffer_stride_unaligned =
     !screen->get_param(screen,
                        PIPE_CAP_VERTEX_BUFFER_STRIDE_4BYTE_ALIGNED_ONLY);
   caps->velem_src_offset_unaligned =
      !screen->get_param(screen,
                         PIPE_CAP_VERTEX_ELEMENT_SRC_OFFSET_4BYTE_ALIGNED_ONLY);
   caps->user_vertex_buffers =
      screen->get_param(screen, PIPE_CAP_USER_VERTEX_BUFFERS);

   if (!caps->buffer_offset_unaligned ||
       !caps->buffer_stride_unaligned ||
       !caps->velem_src_offset_unaligned ||
       !caps->user_vertex_buffers) {
      fallback = TRUE;
   }

   return fallback;
}

struct u_vbuf *
u_vbuf_create(struct pipe_context *pipe,
              struct u_vbuf_caps *caps, unsigned aux_vertex_buffer_index)
{
   struct u_vbuf *mgr = CALLOC_STRUCT(u_vbuf);

   mgr->caps = *caps;
   mgr->aux_vertex_buffer_slot = aux_vertex_buffer_index;
   mgr->pipe = pipe;
   mgr->cso_cache = cso_cache_create();
   mgr->translate_cache = translate_cache_create();
   memset(mgr->fallback_vbs, ~0, sizeof(mgr->fallback_vbs));

   mgr->uploader = u_upload_create(pipe, 1024 * 1024, 4,
                                   PIPE_BIND_VERTEX_BUFFER);

   return mgr;
}

/* u_vbuf uses its own caching for vertex elements, because it needs to keep
 * its own preprocessed state per vertex element CSO. */
static struct u_vbuf_elements *
u_vbuf_set_vertex_elements_internal(struct u_vbuf *mgr, unsigned count,
                                    const struct pipe_vertex_element *states)
{
   struct pipe_context *pipe = mgr->pipe;
   unsigned key_size, hash_key;
   struct cso_hash_iter iter;
   struct u_vbuf_elements *ve;
   struct cso_velems_state velems_state;

   /* need to include the count into the stored state data too. */
   key_size = sizeof(struct pipe_vertex_element) * count + sizeof(unsigned);
   velems_state.count = count;
   memcpy(velems_state.velems, states,
          sizeof(struct pipe_vertex_element) * count);
   hash_key = cso_construct_key((void*)&velems_state, key_size);
   iter = cso_find_state_template(mgr->cso_cache, hash_key, CSO_VELEMENTS,
                                  (void*)&velems_state, key_size);

   if (cso_hash_iter_is_null(iter)) {
      struct cso_velements *cso = MALLOC_STRUCT(cso_velements);
      memcpy(&cso->state, &velems_state, key_size);
      cso->data = u_vbuf_create_vertex_elements(mgr, count, states);
      cso->delete_state = (cso_state_callback)u_vbuf_delete_vertex_elements;
      cso->context = (void*)mgr;

      iter = cso_insert_state(mgr->cso_cache, hash_key, CSO_VELEMENTS, cso);
      ve = cso->data;
   } else {
      ve = ((struct cso_velements *)cso_hash_iter_data(iter))->data;
   }

   assert(ve);

   if (ve != mgr->ve)
      pipe->bind_vertex_elements_state(pipe, ve->driver_cso);

   return ve;
}

void u_vbuf_set_vertex_elements(struct u_vbuf *mgr, unsigned count,
                               const struct pipe_vertex_element *states)
{
   mgr->ve = u_vbuf_set_vertex_elements_internal(mgr, count, states);
}

void u_vbuf_destroy(struct u_vbuf *mgr)
{
   struct pipe_screen *screen = mgr->pipe->screen;
   unsigned i;
   unsigned num_vb = screen->get_shader_param(screen, PIPE_SHADER_VERTEX,
                                              PIPE_SHADER_CAP_MAX_INPUTS);

   mgr->pipe->set_index_buffer(mgr->pipe, NULL);
   pipe_resource_reference(&mgr->index_buffer.buffer, NULL);

   mgr->pipe->set_vertex_buffers(mgr->pipe, 0, num_vb, NULL);

   for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
      pipe_resource_reference(&mgr->vertex_buffer[i].buffer, NULL);
   }
   for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
      pipe_resource_reference(&mgr->real_vertex_buffer[i].buffer, NULL);
   }
   pipe_resource_reference(&mgr->aux_vertex_buffer_saved.buffer, NULL);

   translate_cache_destroy(mgr->translate_cache);
   u_upload_destroy(mgr->uploader);
   cso_cache_delete(mgr->cso_cache);
   FREE(mgr);
}

static enum pipe_error
u_vbuf_translate_buffers(struct u_vbuf *mgr, struct translate_key *key,
                         unsigned vb_mask, unsigned out_vb,
                         int start_vertex, unsigned num_vertices,
                         int start_index, unsigned num_indices, int min_index,
                         boolean unroll_indices)
{
   struct translate *tr;
   struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS] = {0};
   struct pipe_resource *out_buffer = NULL;
   uint8_t *out_map;
   unsigned out_offset, mask;

   /* Get a translate object. */
   tr = translate_cache_find(mgr->translate_cache, key);

   /* Map buffers we want to translate. */
   mask = vb_mask;
   while (mask) {
      struct pipe_vertex_buffer *vb;
      unsigned offset;
      uint8_t *map;
      unsigned i = u_bit_scan(&mask);

      vb = &mgr->vertex_buffer[i];
      offset = vb->buffer_offset + vb->stride * start_vertex;

      if (vb->user_buffer) {
         map = (uint8_t*)vb->user_buffer + offset;
      } else {
         unsigned size = vb->stride ? num_vertices * vb->stride
                                    : sizeof(double)*4;

         if (offset+size > vb->buffer->width0) {
            size = vb->buffer->width0 - offset;
         }

         map = pipe_buffer_map_range(mgr->pipe, vb->buffer, offset, size,
                                     PIPE_TRANSFER_READ, &vb_transfer[i]);
      }

      /* Subtract min_index so that indexing with the index buffer works. */
      if (unroll_indices) {
         map -= (ptrdiff_t)vb->stride * min_index;
      }

      tr->set_buffer(tr, i, map, vb->stride, ~0);
   }

   /* Translate. */
   if (unroll_indices) {
      struct pipe_index_buffer *ib = &mgr->index_buffer;
      struct pipe_transfer *transfer = NULL;
      unsigned offset = ib->offset + start_index * ib->index_size;
      uint8_t *map;

      assert((ib->buffer || ib->user_buffer) && ib->index_size);

      /* Create and map the output buffer. */
      u_upload_alloc(mgr->uploader, 0,
                     key->output_stride * num_indices,
                     &out_offset, &out_buffer,
                     (void**)&out_map);
      if (!out_buffer)
         return PIPE_ERROR_OUT_OF_MEMORY;

      if (ib->user_buffer) {
         map = (uint8_t*)ib->user_buffer + offset;
      } else {
         map = pipe_buffer_map_range(mgr->pipe, ib->buffer, offset,
                                     num_indices * ib->index_size,
                                     PIPE_TRANSFER_READ, &transfer);
      }

      switch (ib->index_size) {
      case 4:
         tr->run_elts(tr, (unsigned*)map, num_indices, 0, 0, out_map);
         break;
      case 2:
         tr->run_elts16(tr, (uint16_t*)map, num_indices, 0, 0, out_map);
         break;
      case 1:
         tr->run_elts8(tr, map, num_indices, 0, 0, out_map);
         break;
      }

      if (transfer) {
         pipe_buffer_unmap(mgr->pipe, transfer);
      }
   } else {
      /* Create and map the output buffer. */
      u_upload_alloc(mgr->uploader,
                     key->output_stride * start_vertex,
                     key->output_stride * num_vertices,
                     &out_offset, &out_buffer,
                     (void**)&out_map);
      if (!out_buffer)
         return PIPE_ERROR_OUT_OF_MEMORY;

      out_offset -= key->output_stride * start_vertex;

      tr->run(tr, 0, num_vertices, 0, 0, out_map);
   }

   /* Unmap all buffers. */
   mask = vb_mask;
   while (mask) {
      unsigned i = u_bit_scan(&mask);

      if (vb_transfer[i]) {
         pipe_buffer_unmap(mgr->pipe, vb_transfer[i]);
      }
   }

   /* Setup the new vertex buffer. */
   mgr->real_vertex_buffer[out_vb].buffer_offset = out_offset;
   mgr->real_vertex_buffer[out_vb].stride = key->output_stride;

   /* Move the buffer reference. */
   pipe_resource_reference(
      &mgr->real_vertex_buffer[out_vb].buffer, NULL);
   mgr->real_vertex_buffer[out_vb].buffer = out_buffer;

   return PIPE_OK;
}

static boolean
u_vbuf_translate_find_free_vb_slots(struct u_vbuf *mgr,
                                    unsigned mask[VB_NUM])
{
   unsigned type;
   unsigned fallback_vbs[VB_NUM];
   /* Set the bit for each buffer which is incompatible, or isn't set. */
   uint32_t unused_vb_mask =
      mgr->ve->incompatible_vb_mask_all | mgr->incompatible_vb_mask |
      ~mgr->enabled_vb_mask;

   memset(fallback_vbs, ~0, sizeof(fallback_vbs));

   /* Find free slots for each type if needed. */
   for (type = 0; type < VB_NUM; type++) {
      if (mask[type]) {
         uint32_t index;

         if (!unused_vb_mask) {
            return FALSE;
         }

         index = ffs(unused_vb_mask) - 1;
         fallback_vbs[type] = index;
         /*printf("found slot=%i for type=%i\n", index, type);*/
      }
   }

   for (type = 0; type < VB_NUM; type++) {
      if (mask[type]) {
         mgr->dirty_real_vb_mask |= 1 << fallback_vbs[type];<commit_msg>u_vbuf: fix vb slot assignment for translated buffers<commit_after>/**************************************************************************
 *
 * Copyright 2011 <NAME> <<EMAIL>>
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/

/**
 * This module uploads user buffers and translates the vertex buffers which
 * contain incompatible vertices (i.e. not supported by the driver/hardware)
 * into compatible ones, based on the Gallium CAPs.
 *
 * It does not upload index buffers.
 *
 * The module heavily uses bitmasks to represent per-buffer and
 * per-vertex-element flags to avoid looping over the list of buffers just
 * to see if there's a non-zero stride, or user buffer, or unsupported format,
 * etc.
 *
 * There are 3 categories of vertex elements, which are processed separately:
 * - per-vertex attribs (stride != 0, instance_divisor == 0)
 * - instanced attribs (stride != 0, instance_divisor > 0)
 * - constant attribs (stride == 0)
 *
 * All needed uploads and translations are performed every draw command, but
 * only the subset of vertices needed for that draw command is uploaded or
 * translated. (the module never translates whole buffers)
 *
 *
 * The module consists of two main parts:
 *
 *
 * 1) Translate (u_vbuf_translate_begin/end)
 *
 * This is pretty much a vertex fetch fallback. It translates vertices from
 * one vertex buffer to another in an unused vertex buffer slot. It does
 * whatever is needed to make the vertices readable by the hardware (changes
 * vertex formats and aligns offsets and strides). The translate module is
 * used here.
 *
 * Each of the 3 categories is translated to a separate buffer.
 * Only the [min_index, max_index] range is translated. For instanced attribs,
 * the range is [start_instance, start_instance+instance_count]. For constant
 * attribs, the range is [0, 1].
 *
 *
 * 2) User buffer uploading (u_vbuf_upload_buffers)
 *
 * Only the [min_index, max_index] range is uploaded (just like Translate)
 * with a single memcpy.
 *
 * This method works best for non-indexed draw operations or indexed draw
 * operations where the [min_index, max_index] range is not being way bigger
 * than the vertex count.
 *
 * If the range is too big (e.g. one triangle with indices {0, 1, 10000}),
 * the per-vertex attribs are uploaded via the translate module, all packed
 * into one vertex buffer, and the indexed draw call is turned into
 * a non-indexed one in the process. This adds additional complexity
 * to the translate part, but it prevents bad apps from bringing your frame
 * rate down.
 *
 *
 * If there is nothing to do, it forwards every command to the driver.
 * The module also has its own CSO cache of vertex element states.
 */

#include "util/u_vbuf.h"

#include "util/u_dump.h"
#include "util/u_format.h"
#include "util/u_inlines.h"
#include "util/u_memory.h"
#include "util/u_upload_mgr.h"
#include "translate/translate.h"
#include "translate/translate_cache.h"
#include "cso_cache/cso_cache.h"
#include "cso_cache/cso_hash.h"

struct u_vbuf_elements {
   unsigned count;
   struct pipe_vertex_element ve[PIPE_MAX_ATTRIBS];

   unsigned src_format_size[PIPE_MAX_ATTRIBS];

   /* If (velem[i].src_format != native_format[i]), the vertex buffer
    * referenced by the vertex element cannot be used for rendering and
    * its vertex data must be translated to native_format[i]. */
   enum pipe_format native_format[PIPE_MAX_ATTRIBS];
   unsigned native_format_size[PIPE_MAX_ATTRIBS];

   /* Which buffers are used by the vertex element state. */
   uint32_t used_vb_mask;
   /* This might mean two things:
    * - src_format != native_format, as discussed above.
    * - src_offset % 4 != 0 (if the caps don't allow such an offset). */
   uint32_t incompatible_elem_mask; /* each bit describes a corresp. attrib  */
   /* Which buffer has at least one vertex element referencing it
    * incompatible. */
   uint32_t incompatible_vb_mask_any;
   /* Which buffer has all vertex elements referencing it incompatible. */
   uint32_t incompatible_vb_mask_all;
   /* Which buffer has at least one vertex element referencing it
    * compatible. */
   uint32_t compatible_vb_mask_any;
   /* Which buffer has all vertex elements referencing it compatible. */
   uint32_t compatible_vb_mask_all;

   /* Which buffer has at least one vertex element referencing it
    * non-instanced. */
   uint32_t noninstance_vb_mask_any;

   void *driver_cso;
};

enum {
   VB_VERTEX = 0,
   VB_INSTANCE = 1,
   VB_CONST = 2,
   VB_NUM = 3
};

struct u_vbuf {
   struct u_vbuf_caps caps;

   struct pipe_context *pipe;
   struct translate_cache *translate_cache;
   struct cso_cache *cso_cache;
   struct u_upload_mgr *uploader;

   /* This is what was set in set_vertex_buffers.
    * May contain user buffers. */
   struct pipe_vertex_buffer vertex_buffer[PIPE_MAX_ATTRIBS];
   uint32_t enabled_vb_mask;

   /* Saved vertex buffer. */
   unsigned aux_vertex_buffer_slot;
   struct pipe_vertex_buffer aux_vertex_buffer_saved;

   /* Vertex buffers for the driver.
    * There are usually no user buffers. */
   struct pipe_vertex_buffer real_vertex_buffer[PIPE_MAX_ATTRIBS];
   uint32_t dirty_real_vb_mask; /* which buffers are dirty since the last
                                   call of set_vertex_buffers */

   /* The index buffer. */
   struct pipe_index_buffer index_buffer;

   /* Vertex elements. */
   struct u_vbuf_elements *ve, *ve_saved;

   /* Vertex elements used for the translate fallback. */
   struct pipe_vertex_element fallback_velems[PIPE_MAX_ATTRIBS];
   /* If non-NULL, this is a vertex element state used for the translate
    * fallback and therefore used for rendering too. */
   boolean using_translate;
   /* The vertex buffer slot index where translated vertices have been
    * stored in. */
   unsigned fallback_vbs[VB_NUM];

   /* Which buffer is a user buffer. */
   uint32_t user_vb_mask; /* each bit describes a corresp. buffer */
   /* Which buffer is incompatible (unaligned). */
   uint32_t incompatible_vb_mask; /* each bit describes a corresp. buffer */
   /* Which buffer has a non-zero stride. */
   uint32_t nonzero_stride_vb_mask; /* each bit describes a corresp. buffer */
};

static void *
u_vbuf_create_vertex_elements(struct u_vbuf *mgr, unsigned count,
                              const struct pipe_vertex_element *attribs);
static void u_vbuf_delete_vertex_elements(struct u_vbuf *mgr, void *cso);

static const struct {
   enum pipe_format from, to;
} vbuf_format_fallbacks[] = {
   { PIPE_FORMAT_R32_FIXED,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_FIXED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_FIXED,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_FIXED,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_FLOAT,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_FLOAT,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_FLOAT,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_FLOAT,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R64_FLOAT,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R64G64_FLOAT,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R64G64B64_FLOAT,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R64G64B64A64_FLOAT,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_UNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_UNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_UNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_UNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_SNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_SNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_SNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_SNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_USCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_USCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_USCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R32_SSCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R32G32_SSCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R32G32B32_SSCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R32G32B32A32_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_UNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_UNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_UNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_UNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_SNORM,            PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_SNORM,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_SNORM,      PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_SNORM,   PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_USCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_USCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_USCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_USCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R16_SSCALED,          PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R16G16_SSCALED,       PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R16G16B16_SSCALED,    PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R16G16B16A16_SSCALED, PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_UNORM,             PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_UNORM,           PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_UNORM,         PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_UNORM,       PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_SNORM,             PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_SNORM,           PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_SNORM,         PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_SNORM,       PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_USCALED,           PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_USCALED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_USCALED,       PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_USCALED,     PIPE_FORMAT_R32G32B32A32_FLOAT },
   { PIPE_FORMAT_R8_SSCALED,           PIPE_FORMAT_R32_FLOAT },
   { PIPE_FORMAT_R8G8_SSCALED,         PIPE_FORMAT_R32G32_FLOAT },
   { PIPE_FORMAT_R8G8B8_SSCALED,       PIPE_FORMAT_R32G32B32_FLOAT },
   { PIPE_FORMAT_R8G8B8A8_SSCALED,     PIPE_FORMAT_R32G32B32A32_FLOAT },
};

boolean u_vbuf_get_caps(struct pipe_screen *screen, struct u_vbuf_caps *caps)
{
   unsigned i;
   boolean fallback = FALSE;

   /* I'd rather have a bitfield of which formats are supported and a static
    * table of the translations indexed by format, but since we don't have C99
    * we can't easily make a sparsely-populated table indexed by format.  So,
    * we construct the sparse table here.
    */
   for (i = 0; i < PIPE_FORMAT_COUNT; i++)
      caps->format_translation[i] = i;

   for (i = 0; i < Elements(vbuf_format_fallbacks); i++) {
      enum pipe_format format = vbuf_format_fallbacks[i].from;

      if (!screen->is_format_supported(screen, format, PIPE_BUFFER, 0,
                                       PIPE_BIND_VERTEX_BUFFER)) {
         caps->format_translation[format] = vbuf_format_fallbacks[i].to;
         fallback = TRUE;
      }
   }

   caps->buffer_offset_unaligned =
      !screen->get_param(screen,
                         PIPE_CAP_VERTEX_BUFFER_OFFSET_4BYTE_ALIGNED_ONLY);
   caps->buffer_stride_unaligned =
     !screen->get_param(screen,
                        PIPE_CAP_VERTEX_BUFFER_STRIDE_4BYTE_ALIGNED_ONLY);
   caps->velem_src_offset_unaligned =
      !screen->get_param(screen,
                         PIPE_CAP_VERTEX_ELEMENT_SRC_OFFSET_4BYTE_ALIGNED_ONLY);
   caps->user_vertex_buffers =
      screen->get_param(screen, PIPE_CAP_USER_VERTEX_BUFFERS);

   if (!caps->buffer_offset_unaligned ||
       !caps->buffer_stride_unaligned ||
       !caps->velem_src_offset_unaligned ||
       !caps->user_vertex_buffers) {
      fallback = TRUE;
   }

   return fallback;
}

struct u_vbuf *
u_vbuf_create(struct pipe_context *pipe,
              struct u_vbuf_caps *caps, unsigned aux_vertex_buffer_index)
{
   struct u_vbuf *mgr = CALLOC_STRUCT(u_vbuf);

   mgr->caps = *caps;
   mgr->aux_vertex_buffer_slot = aux_vertex_buffer_index;
   mgr->pipe = pipe;
   mgr->cso_cache = cso_cache_create();
   mgr->translate_cache = translate_cache_create();
   memset(mgr->fallback_vbs, ~0, sizeof(mgr->fallback_vbs));

   mgr->uploader = u_upload_create(pipe, 1024 * 1024, 4,
                                   PIPE_BIND_VERTEX_BUFFER);

   return mgr;
}

/* u_vbuf uses its own caching for vertex elements, because it needs to keep
 * its own preprocessed state per vertex element CSO. */
static struct u_vbuf_elements *
u_vbuf_set_vertex_elements_internal(struct u_vbuf *mgr, unsigned count,
                                    const struct pipe_vertex_element *states)
{
   struct pipe_context *pipe = mgr->pipe;
   unsigned key_size, hash_key;
   struct cso_hash_iter iter;
   struct u_vbuf_elements *ve;
   struct cso_velems_state velems_state;

   /* need to include the count into the stored state data too. */
   key_size = sizeof(struct pipe_vertex_element) * count + sizeof(unsigned);
   velems_state.count = count;
   memcpy(velems_state.velems, states,
          sizeof(struct pipe_vertex_element) * count);
   hash_key = cso_construct_key((void*)&velems_state, key_size);
   iter = cso_find_state_template(mgr->cso_cache, hash_key, CSO_VELEMENTS,
                                  (void*)&velems_state, key_size);

   if (cso_hash_iter_is_null(iter)) {
      struct cso_velements *cso = MALLOC_STRUCT(cso_velements);
      memcpy(&cso->state, &velems_state, key_size);
      cso->data = u_vbuf_create_vertex_elements(mgr, count, states);
      cso->delete_state = (cso_state_callback)u_vbuf_delete_vertex_elements;
      cso->context = (void*)mgr;

      iter = cso_insert_state(mgr->cso_cache, hash_key, CSO_VELEMENTS, cso);
      ve = cso->data;
   } else {
      ve = ((struct cso_velements *)cso_hash_iter_data(iter))->data;
   }

   assert(ve);

   if (ve != mgr->ve)
      pipe->bind_vertex_elements_state(pipe, ve->driver_cso);

   return ve;
}

void u_vbuf_set_vertex_elements(struct u_vbuf *mgr, unsigned count,
                               const struct pipe_vertex_element *states)
{
   mgr->ve = u_vbuf_set_vertex_elements_internal(mgr, count, states);
}

void u_vbuf_destroy(struct u_vbuf *mgr)
{
   struct pipe_screen *screen = mgr->pipe->screen;
   unsigned i;
   unsigned num_vb = screen->get_shader_param(screen, PIPE_SHADER_VERTEX,
                                              PIPE_SHADER_CAP_MAX_INPUTS);

   mgr->pipe->set_index_buffer(mgr->pipe, NULL);
   pipe_resource_reference(&mgr->index_buffer.buffer, NULL);

   mgr->pipe->set_vertex_buffers(mgr->pipe, 0, num_vb, NULL);

   for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
      pipe_resource_reference(&mgr->vertex_buffer[i].buffer, NULL);
   }
   for (i = 0; i < PIPE_MAX_ATTRIBS; i++) {
      pipe_resource_reference(&mgr->real_vertex_buffer[i].buffer, NULL);
   }
   pipe_resource_reference(&mgr->aux_vertex_buffer_saved.buffer, NULL);

   translate_cache_destroy(mgr->translate_cache);
   u_upload_destroy(mgr->uploader);
   cso_cache_delete(mgr->cso_cache);
   FREE(mgr);
}

static enum pipe_error
u_vbuf_translate_buffers(struct u_vbuf *mgr, struct translate_key *key,
                         unsigned vb_mask, unsigned out_vb,
                         int start_vertex, unsigned num_vertices,
                         int start_index, unsigned num_indices, int min_index,
                         boolean unroll_indices)
{
   struct translate *tr;
   struct pipe_transfer *vb_transfer[PIPE_MAX_ATTRIBS] = {0};
   struct pipe_resource *out_buffer = NULL;
   uint8_t *out_map;
   unsigned out_offset, mask;

   /* Get a translate object. */
   tr = translate_cache_find(mgr->translate_cache, key);

   /* Map buffers we want to translate. */
   mask = vb_mask;
   while (mask) {
      struct pipe_vertex_buffer *vb;
      unsigned offset;
      uint8_t *map;
      unsigned i = u_bit_scan(&mask);

      vb = &mgr->vertex_buffer[i];
      offset = vb->buffer_offset + vb->stride * start_vertex;

      if (vb->user_buffer) {
         map = (uint8_t*)vb->user_buffer + offset;
      } else {
         unsigned size = vb->stride ? num_vertices * vb->stride
                                    : sizeof(double)*4;

         if (offset+size > vb->buffer->width0) {
            size = vb->buffer->width0 - offset;
         }

         map = pipe_buffer_map_range(mgr->pipe, vb->buffer, offset, size,
                                     PIPE_TRANSFER_READ, &vb_transfer[i]);
      }

      /* Subtract min_index so that indexing with the index buffer works. */
      if (unroll_indices) {
         map -= (ptrdiff_t)vb->stride * min_index;
      }

      tr->set_buffer(tr, i, map, vb->stride, ~0);
   }

   /* Translate. */
   if (unroll_indices) {
      struct pipe_index_buffer *ib = &mgr->index_buffer;
      struct pipe_transfer *transfer = NULL;
      unsigned offset = ib->offset + start_index * ib->index_size;
      uint8_t *map;

      assert((ib->buffer || ib->user_buffer) && ib->index_size);

      /* Create and map the output buffer. */
      u_upload_alloc(mgr->uploader, 0,
                     key->output_stride * num_indices,
                     &out_offset, &out_buffer,
                     (void**)&out_map);
      if (!out_buffer)
         return PIPE_ERROR_OUT_OF_MEMORY;

      if (ib->user_buffer) {
         map = (uint8_t*)ib->user_buffer + offset;
      } else {
         map = pipe_buffer_map_range(mgr->pipe, ib->buffer, offset,
                                     num_indices * ib->index_size,
                                     PIPE_TRANSFER_READ, &transfer);
      }

      switch (ib->index_size) {
      case 4:
         tr->run_elts(tr, (unsigned*)map, num_indices, 0, 0, out_map);
         break;
      case 2:
         tr->run_elts16(tr, (uint16_t*)map, num_indices, 0, 0, out_map);
         break;
      case 1:
         tr->run_elts8(tr, map, num_indices, 0, 0, out_map);
         break;
      }

      if (transfer) {
         pipe_buffer_unmap(mgr->pipe, transfer);
      }
   } else {
      /* Create and map the output buffer. */
      u_upload_alloc(mgr->uploader,
                     key->output_stride * start_vertex,
                     key->output_stride * num_vertices,
                     &out_offset, &out_buffer,
                     (void**)&out_map);
      if (!out_buffer)
         return PIPE_ERROR_OUT_OF_MEMORY;

      out_offset -= key->output_stride * start_vertex;

      tr->run(tr, 0, num_vertices, 0, 0, out_map);
   }

   /* Unmap all buffers. */
   mask = vb_mask;
   while (mask) {
      unsigned i = u_bit_scan(&mask);

      if (vb_transfer[i]) {
         pipe_buffer_unmap(mgr->pipe, vb_transfer[i]);
      }
   }

   /* Setup the new vertex buffer. */
   mgr->real_vertex_buffer[out_vb].buffer_offset = out_offset;
   mgr->real_vertex_buffer[out_vb].stride = key->output_stride;

   /* Move the buffer reference. */
   pipe_resource_reference(
      &mgr->real_vertex_buffer[out_vb].buffer, NULL);
   mgr->real_vertex_buffer[out_vb].buffer = out_buffer;

   return PIPE_OK;
}

static boolean
u_vbuf_translate_find_free_vb_slots(struct u_vbuf *mgr,
                                    unsigned mask[VB_NUM])
{
   unsigned type;
   unsigned fallback_vbs[VB_NUM];
   /* Set the bit for each buffer which is incompatible, or isn't set. */
   uint32_t unused_vb_mask =
      mgr->ve->incompatible_vb_mask_all | mgr->incompatible_vb_mask |
      ~mgr->enabled_vb_mask;

   memset(fallback_vbs, ~0, sizeof(fallback_vbs));

   /* Find free slots for each type if needed. */
   for (type = 0; type < VB_NUM; type++) {
      if (mask[type]) {
         uint32_t index;

         if (!unused_vb_mask) {
            return FALSE;
         }

         index = ffs(unused_vb_mask) - 1;
         fallback_vbs[type] = index;
         unused_vb_mask &= ~(1 << index);
         /*printf("found slot=%i for type=%i\n", index, type);*/
      }
   }

   for (type = 0; type < VB_NUM; type++) {
      if (mask[type]) {
         mgr->dirty_real_vb_mask |= 1 << fallback_vbs[type];
{'fraction_non_alphanumeric': 0.0931433866110725, 'fraction_numerical': 0.0310632383599722, 'mean_word_length': 3.161461345671872, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 219, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5882352', 'n_tokens_mistral': 15391, 'n_tokens_neox': 14497, 'n_words': 3341}