<commit_before>/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vlib/vlib.h>
#include <vnet/l2/feat_bitmap.h>
#include <vnet/l2/l2_rw.h>

l2_rw_main_t l2_rw_main;

vlib_node_registration_t l2_rw_node;

typedef struct {
  u32 sw_if_index;
  u32 classify_table_index;
  u32 rewrite_entry_index;
} l2_rw_trace_t;

static u8 *format_l2_rw_entry (u8 * s, va_list *args)
{
  l2_rw_entry_t *e = va_arg (*args, l2_rw_entry_t *);
  l2_rw_main_t *rw = &l2_rw_main;
  s = format (s, "%d -  mask:%U value:%U\n",
              e - rw->entries,
              format_hex_bytes, e->mask, e->rewrite_n_vectors * sizeof(u32x4),
              format_hex_bytes, e->value, e->rewrite_n_vectors * sizeof(u32x4));
  s = format (s, "      hits:%d skip_bytes:%d",
                e->hit_count, e->skip_n_vectors * sizeof(u32x4));
  return s;
}

static u8 *format_l2_rw_config (u8 * s, va_list *args)
{
  l2_rw_config_t *c = va_arg (*args, l2_rw_config_t *);
  return format(s, "table-index:%d miss-index:%d",
                c->table_index,
                c->miss_index);
}

/* packet trace format function */
static u8 *format_l2_rw_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  l2_rw_trace_t * t = va_arg (*args, l2_rw_trace_t *);
  return format (s, "l2-rw: sw_if_index %d, table %d, entry %d",
              t->sw_if_index, t->classify_table_index,
              t->rewrite_entry_index);
}

always_inline l2_rw_config_t *l2_rw_get_config(u32 sw_if_index)
{
  l2_rw_main_t *rw = &l2_rw_main;
  if (PREDICT_FALSE(!clib_bitmap_get(rw->configs_bitmap, sw_if_index))) {
    vec_validate(rw->configs, sw_if_index);
    rw->configs[sw_if_index].table_index = ~0;
    rw->configs[sw_if_index].miss_index = ~0;
    rw->configs_bitmap = clib_bitmap_set(rw->configs_bitmap, sw_if_index, 1);
  }
  return &rw->configs[sw_if_index];
}

static_always_inline void l2_rw_rewrite(l2_rw_entry_t *rwe, u8 *h)
{
  if (U32X4_ALIGNED(h)) {
    u32x4 *d = ((u32x4 *) h) + rwe->skip_n_vectors;
    switch(rwe->rewrite_n_vectors) {
      case 5:
        d[4] = (d[4] & ~rwe->mask[4]) | rwe->value[4];
        /* FALLTHROUGH */
      case 4:
        d[3] = (d[3] & ~rwe->mask[3]) | rwe->value[3];
        /* FALLTHROUGH */
      case 3:
        d[2] = (d[2] & ~rwe->mask[2]) | rwe->value[2];
        /* FALLTHROUGH */
      case 2:
        d[1] = (d[1] & ~rwe->mask[1]) | rwe->value[1];
        /* FALLTHROUGH */
      case 1:
        d[0] = (d[0] & ~rwe->mask[0]) | rwe->value[0];
        break;
      default:
        abort();
    }
  } else {
    u64 *d = ((u64 *) h) + rwe->skip_n_vectors * 2;
    switch(rwe->rewrite_n_vectors) {
      case 5:
        d[8] = (d[8] & ~(((u64*)rwe->mask)[8])) | (((u64*)rwe->value)[8]);
        d[9] = (d[9] & ~(((u64*)rwe->mask)[9])) | (((u64*)rwe->value)[9]);
        /* FALLTHROUGH */
      case 4:
        d[6] = (d[6] & ~(((u64*)rwe->mask)[6])) | (((u64*)rwe->value)[6]);
        d[7] = (d[7] & ~(((u64*)rwe->mask)[7])) | (((u64*)rwe->value)[7]);
        /* FALLTHROUGH */
      case 3:
        d[4] = (d[4] & ~(((u64*)rwe->mask)[4])) | (((u64*)rwe->value)[4]);
        d[5] = (d[5] & ~(((u64*)rwe->mask)[5])) | (((u64*)rwe->value)[5]);
        /* FALLTHROUGH */
      case 2:
        d[2] = (d[2] & ~(((u64*)rwe->mask)[2])) | (((u64*)rwe->value)[2]);
        d[3] = (d[3] & ~(((u64*)rwe->mask)[3])) | (((u64*)rwe->value)[3]);
        /* FALLTHROUGH */
      case 1:
        d[0] = (d[0] & ~(((u64*)rwe->mask)[0])) | (((u64*)rwe->value)[0]);
        d[1] = (d[1] & ~(((u64*)rwe->mask)[1])) | (((u64*)rwe->value)[1]);
        break;
      default:
        abort();
    }
  }
}

static uword
l2_rw_node_fn(vlib_main_t * vm,
              vlib_node_runtime_t * node,
              vlib_frame_t * frame)
{
  l2_rw_main_t *rw = &l2_rw_main;
  u32 n_left_from, * from, * to_next, next_index;
  //vlib_node_t *n = vlib_get_node(vm, node->node_index);
  vnet_classify_main_t *vcm = &vnet_classify_main;
  f64 now = vlib_time_now(vlib_get_main());
  u32 prefetch_size = 0;

  from = vlib_frame_vector_args (frame);
  n_left_from = frame->n_vectors; /* number of packets to process */
  next_index = node->cached_next_index;

  while (n_left_from > 0)
  {
    u32 n_left_to_next;

    /* get space to enqueue frame to graph node "next_index" */
    vlib_get_next_frame (vm, node, next_index,
                         to_next, n_left_to_next);

    while (n_left_from >= 4 && n_left_to_next >= 2)
    {
      u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0;
      u32 bi1, next1, sw_if_index1, feature_bitmap1, rwe_index1;
      vlib_buffer_t *b0, *b1;
      ethernet_header_t *h0, *h1;
      l2_rw_config_t *config0, *config1;
      u64 hash0, hash1;
      vnet_classify_table_t *t0, *t1;
      vnet_classify_entry_t *e0, *e1;
      l2_rw_entry_t *rwe0, *rwe1;

      {
        vlib_buffer_t * p2, * p3;
        p2 = vlib_get_buffer (vm, from[2]);
        p3 = vlib_get_buffer (vm, from[3]);

        vlib_prefetch_buffer_header (p2, LOAD);
        vlib_prefetch_buffer_header (p3, LOAD);
        CLIB_PREFETCH (p2->data, prefetch_size, LOAD);
        CLIB_PREFETCH (p3->data, prefetch_size, LOAD);
      }

      bi0 = from[0];
      bi1 = from[1];
      to_next[0] = bi0;
      to_next[1] = bi1;
      from += 2;
      to_next += 2;
      n_left_from -= 2;
      n_left_to_next -= 2;

      b0 = vlib_get_buffer(vm, bi0);
      b1 = vlib_get_buffer(vm, bi1);
      h0 = vlib_buffer_get_current(b0);
      h1 = vlib_buffer_get_current(b1);

      sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
      sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
      config0 = l2_rw_get_config(sw_if_index0); //TODO: check sw_if_index0 value
      config1 = l2_rw_get_config(sw_if_index1); //TODO: check sw_if_index0 value
      t0 = pool_elt_at_index(vcm->tables, config0->table_index);
      t1 = pool_elt_at_index(vcm->tables, config1->table_index);
      prefetch_size = (t1->skip_n_vectors + t1->match_n_vectors)*sizeof(u32x4);

      hash0 = vnet_classify_hash_packet(t0, (u8 *)h0);
      hash1 = vnet_classify_hash_packet(t1, (u8 *)h1);
      e0 = vnet_classify_find_entry(t0, (u8 *) h0, hash0, now);
      e1 = vnet_classify_find_entry(t1, (u8 *) h1, hash1, now);
<commit_msg>l2_rw preload starts where data is<commit_after>/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vlib/vlib.h>
#include <vnet/l2/feat_bitmap.h>
#include <vnet/l2/l2_rw.h>

l2_rw_main_t l2_rw_main;

vlib_node_registration_t l2_rw_node;

typedef struct {
  u32 sw_if_index;
  u32 classify_table_index;
  u32 rewrite_entry_index;
} l2_rw_trace_t;

static u8 *format_l2_rw_entry (u8 * s, va_list *args)
{
  l2_rw_entry_t *e = va_arg (*args, l2_rw_entry_t *);
  l2_rw_main_t *rw = &l2_rw_main;
  s = format (s, "%d -  mask:%U value:%U\n",
              e - rw->entries,
              format_hex_bytes, e->mask, e->rewrite_n_vectors * sizeof(u32x4),
              format_hex_bytes, e->value, e->rewrite_n_vectors * sizeof(u32x4));
  s = format (s, "      hits:%d skip_bytes:%d",
                e->hit_count, e->skip_n_vectors * sizeof(u32x4));
  return s;
}

static u8 *format_l2_rw_config (u8 * s, va_list *args)
{
  l2_rw_config_t *c = va_arg (*args, l2_rw_config_t *);
  return format(s, "table-index:%d miss-index:%d",
                c->table_index,
                c->miss_index);
}

/* packet trace format function */
static u8 *format_l2_rw_trace (u8 * s, va_list * args)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  l2_rw_trace_t * t = va_arg (*args, l2_rw_trace_t *);
  return format (s, "l2-rw: sw_if_index %d, table %d, entry %d",
              t->sw_if_index, t->classify_table_index,
              t->rewrite_entry_index);
}

always_inline l2_rw_config_t *l2_rw_get_config(u32 sw_if_index)
{
  l2_rw_main_t *rw = &l2_rw_main;
  if (PREDICT_FALSE(!clib_bitmap_get(rw->configs_bitmap, sw_if_index))) {
    vec_validate(rw->configs, sw_if_index);
    rw->configs[sw_if_index].table_index = ~0;
    rw->configs[sw_if_index].miss_index = ~0;
    rw->configs_bitmap = clib_bitmap_set(rw->configs_bitmap, sw_if_index, 1);
  }
  return &rw->configs[sw_if_index];
}

static_always_inline void l2_rw_rewrite(l2_rw_entry_t *rwe, u8 *h)
{
  if (U32X4_ALIGNED(h)) {
    u32x4 *d = ((u32x4 *) h) + rwe->skip_n_vectors;
    switch(rwe->rewrite_n_vectors) {
      case 5:
        d[4] = (d[4] & ~rwe->mask[4]) | rwe->value[4];
        /* FALLTHROUGH */
      case 4:
        d[3] = (d[3] & ~rwe->mask[3]) | rwe->value[3];
        /* FALLTHROUGH */
      case 3:
        d[2] = (d[2] & ~rwe->mask[2]) | rwe->value[2];
        /* FALLTHROUGH */
      case 2:
        d[1] = (d[1] & ~rwe->mask[1]) | rwe->value[1];
        /* FALLTHROUGH */
      case 1:
        d[0] = (d[0] & ~rwe->mask[0]) | rwe->value[0];
        break;
      default:
        abort();
    }
  } else {
    u64 *d = ((u64 *) h) + rwe->skip_n_vectors * 2;
    switch(rwe->rewrite_n_vectors) {
      case 5:
        d[8] = (d[8] & ~(((u64*)rwe->mask)[8])) | (((u64*)rwe->value)[8]);
        d[9] = (d[9] & ~(((u64*)rwe->mask)[9])) | (((u64*)rwe->value)[9]);
        /* FALLTHROUGH */
      case 4:
        d[6] = (d[6] & ~(((u64*)rwe->mask)[6])) | (((u64*)rwe->value)[6]);
        d[7] = (d[7] & ~(((u64*)rwe->mask)[7])) | (((u64*)rwe->value)[7]);
        /* FALLTHROUGH */
      case 3:
        d[4] = (d[4] & ~(((u64*)rwe->mask)[4])) | (((u64*)rwe->value)[4]);
        d[5] = (d[5] & ~(((u64*)rwe->mask)[5])) | (((u64*)rwe->value)[5]);
        /* FALLTHROUGH */
      case 2:
        d[2] = (d[2] & ~(((u64*)rwe->mask)[2])) | (((u64*)rwe->value)[2]);
        d[3] = (d[3] & ~(((u64*)rwe->mask)[3])) | (((u64*)rwe->value)[3]);
        /* FALLTHROUGH */
      case 1:
        d[0] = (d[0] & ~(((u64*)rwe->mask)[0])) | (((u64*)rwe->value)[0]);
        d[1] = (d[1] & ~(((u64*)rwe->mask)[1])) | (((u64*)rwe->value)[1]);
        break;
      default:
        abort();
    }
  }
}

static uword
l2_rw_node_fn(vlib_main_t * vm,
              vlib_node_runtime_t * node,
              vlib_frame_t * frame)
{
  l2_rw_main_t *rw = &l2_rw_main;
  u32 n_left_from, * from, * to_next, next_index;
  //vlib_node_t *n = vlib_get_node(vm, node->node_index);
  vnet_classify_main_t *vcm = &vnet_classify_main;
  f64 now = vlib_time_now(vlib_get_main());
  u32 prefetch_size = 0;

  from = vlib_frame_vector_args (frame);
  n_left_from = frame->n_vectors; /* number of packets to process */
  next_index = node->cached_next_index;

  while (n_left_from > 0)
  {
    u32 n_left_to_next;

    /* get space to enqueue frame to graph node "next_index" */
    vlib_get_next_frame (vm, node, next_index,
                         to_next, n_left_to_next);

    while (n_left_from >= 4 && n_left_to_next >= 2)
    {
      u32 bi0, next0, sw_if_index0, feature_bitmap0, rwe_index0;
      u32 bi1, next1, sw_if_index1, feature_bitmap1, rwe_index1;
      vlib_buffer_t *b0, *b1;
      ethernet_header_t *h0, *h1;
      l2_rw_config_t *config0, *config1;
      u64 hash0, hash1;
      vnet_classify_table_t *t0, *t1;
      vnet_classify_entry_t *e0, *e1;
      l2_rw_entry_t *rwe0, *rwe1;

      {
        vlib_buffer_t * p2, * p3;
        p2 = vlib_get_buffer (vm, from[2]);
        p3 = vlib_get_buffer (vm, from[3]);

        vlib_prefetch_buffer_header (p2, LOAD);
        vlib_prefetch_buffer_header (p3, LOAD);
        CLIB_PREFETCH (vlib_buffer_get_current(p2), prefetch_size, LOAD);
        CLIB_PREFETCH (vlib_buffer_get_current(p3), prefetch_size, LOAD);
      }

      bi0 = from[0];
      bi1 = from[1];
      to_next[0] = bi0;
      to_next[1] = bi1;
      from += 2;
      to_next += 2;
      n_left_from -= 2;
      n_left_to_next -= 2;

      b0 = vlib_get_buffer(vm, bi0);
      b1 = vlib_get_buffer(vm, bi1);
      h0 = vlib_buffer_get_current(b0);
      h1 = vlib_buffer_get_current(b1);

      sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
      sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
      config0 = l2_rw_get_config(sw_if_index0); //TODO: check sw_if_index0 value
      config1 = l2_rw_get_config(sw_if_index1); //TODO: check sw_if_index0 value
      t0 = pool_elt_at_index(vcm->tables, config0->table_index);
      t1 = pool_elt_at_index(vcm->tables, config1->table_index);
      prefetch_size = (t1->skip_n_vectors + t1->match_n_vectors)*sizeof(u32x4);

      hash0 = vnet_classify_hash_packet(t0, (u8 *)h0);
      hash1 = vnet_classify_hash_packet(t1, (u8 *)h1);
      e0 = vnet_classify_find_entry(t0, (u8 *) h0, hash0, now);
      e1 = vnet_classify_find_entry(t1, (u8 *) h1, hash1, now);

{'fraction_non_alphanumeric': 0.1645560460235907, 'fraction_numerical': 0.043490845936753744, 'mean_word_length': 2.9598853868194843, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397747', 'n_tokens_mistral': 6000, 'n_tokens_neox': 5797, 'n_words': 1081}