<commit_before>/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2019 Intel Corporation
 */

#include "ice_rxtx_vec_common.h"

#include <tmmintrin.h>

#ifndef __INTEL_COMPILER
#pragma GCC diagnostic ignored "-Wcast-qual"
#endif

static inline __m128i
ice_flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
{
#define FDID_MIS_MAGIC 0xFFFFFFFF
	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
			PKT_RX_FDIR_ID);
	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
			fdir_mis_mask);
	/* this XOR op results to bit-reverse the fdir_mask */
	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);

	return fdir_flags;
}

static inline void
ice_rxq_rearm(struct ice_rx_queue *rxq)
{
	int i;
	uint16_t rx_id;
	volatile union ice_rx_flex_desc *rxdp;
	struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
	struct rte_mbuf *mb0, *mb1;
	__m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
					  RTE_PKTMBUF_HEADROOM);
	__m128i dma_addr0, dma_addr1;

	rxdp = rxq->rx_ring + rxq->rxrearm_start;

	/* Pull 'n' more MBUFs into the software ring */
	if (rte_mempool_get_bulk(rxq->mp,
				 (void *)rxep,
				 ICE_RXQ_REARM_THRESH) < 0) {
		if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
		    rxq->nb_rx_desc) {
			dma_addr0 = _mm_setzero_si128();
			for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
				rxep[i].mbuf = &rxq->fake_mbuf;
				_mm_store_si128((__m128i *)&rxdp[i].read,
						dma_addr0);
			}
		}
		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
			ICE_RXQ_REARM_THRESH;
		return;
	}

	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
	for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
		__m128i vaddr0, vaddr1;

		mb0 = rxep[0].mbuf;
		mb1 = rxep[1].mbuf;

		/* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
				 offsetof(struct rte_mbuf, buf_addr) + 8);
		vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
		vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);

		/* convert pa to dma_addr hdr/data */
		dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
		dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);

		/* add headroom to pa values */
		dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
		dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);

		/* flush desc with pa dma_addr */
		_mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
		_mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
	}

	rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
		rxq->rxrearm_start = 0;

	rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;

	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
			   (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));

	/* Update the tail pointer on the NIC */
	ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
}

static inline void
ice_rx_desc_to_olflags_v(struct ice_rx_queue *rxq, __m128i descs[4],
			 struct rte_mbuf **rx_pkts)
{
	const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);
	__m128i rearm0, rearm1, rearm2, rearm3;

	__m128i tmp_desc, flags, rss_vlan;

	/* mask everything except checksum, RSS and VLAN flags.
	 * bit6:4 for checksum.
	 * bit12 for RSS indication.
	 * bit13 for VLAN indication.
	 */
	const __m128i desc_mask = _mm_set_epi32(0x3070, 0x3070,
						0x3070, 0x3070);

	const __m128i cksum_mask = _mm_set_epi32(PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD);

	/* map the checksum, rss and vlan fields to the checksum, rss
	 * and vlan flag
	 */
	const __m128i cksum_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
			/* shift right 1 bit to make sure it not exceed 255 */
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
			 PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
			 PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |
			 PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |
			 PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1);

	const __m128i rss_vlan_flags = _mm_set_epi8(0, 0, 0, 0,
			0, 0, 0, 0,
			0, 0, 0, 0,
			PKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
			PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
			PKT_RX_RSS_HASH, 0);

	/* merge 4 descriptors */
	flags = _mm_unpackhi_epi32(descs[0], descs[1]);
	tmp_desc = _mm_unpackhi_epi32(descs[2], descs[3]);
	tmp_desc = _mm_unpacklo_epi64(flags, tmp_desc);
	tmp_desc = _mm_and_si128(flags, desc_mask);

	/* checksum flags */
	tmp_desc = _mm_srli_epi32(tmp_desc, 4);
	flags = _mm_shuffle_epi8(cksum_flags, tmp_desc);
	/* then we shift left 1 bit */
	flags = _mm_slli_epi32(flags, 1);
	/* we need to mask out the reduntant bits introduced by RSS or<commit_msg>net/ice: fix Rx offload flags in SSE path<commit_after>/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2019 Intel Corporation
 */

#include "ice_rxtx_vec_common.h"

#include <tmmintrin.h>

#ifndef __INTEL_COMPILER
#pragma GCC diagnostic ignored "-Wcast-qual"
#endif

static inline __m128i
ice_flex_rxd_to_fdir_flags_vec(const __m128i fdir_id0_3)
{
#define FDID_MIS_MAGIC 0xFFFFFFFF
	RTE_BUILD_BUG_ON(PKT_RX_FDIR != (1 << 2));
	RTE_BUILD_BUG_ON(PKT_RX_FDIR_ID != (1 << 13));
	const __m128i pkt_fdir_bit = _mm_set1_epi32(PKT_RX_FDIR |
			PKT_RX_FDIR_ID);
	/* desc->flow_id field == 0xFFFFFFFF means fdir mismatch */
	const __m128i fdir_mis_mask = _mm_set1_epi32(FDID_MIS_MAGIC);
	__m128i fdir_mask = _mm_cmpeq_epi32(fdir_id0_3,
			fdir_mis_mask);
	/* this XOR op results to bit-reverse the fdir_mask */
	fdir_mask = _mm_xor_si128(fdir_mask, fdir_mis_mask);
	const __m128i fdir_flags = _mm_and_si128(fdir_mask, pkt_fdir_bit);

	return fdir_flags;
}

static inline void
ice_rxq_rearm(struct ice_rx_queue *rxq)
{
	int i;
	uint16_t rx_id;
	volatile union ice_rx_flex_desc *rxdp;
	struct ice_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
	struct rte_mbuf *mb0, *mb1;
	__m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM,
					  RTE_PKTMBUF_HEADROOM);
	__m128i dma_addr0, dma_addr1;

	rxdp = rxq->rx_ring + rxq->rxrearm_start;

	/* Pull 'n' more MBUFs into the software ring */
	if (rte_mempool_get_bulk(rxq->mp,
				 (void *)rxep,
				 ICE_RXQ_REARM_THRESH) < 0) {
		if (rxq->rxrearm_nb + ICE_RXQ_REARM_THRESH >=
		    rxq->nb_rx_desc) {
			dma_addr0 = _mm_setzero_si128();
			for (i = 0; i < ICE_DESCS_PER_LOOP; i++) {
				rxep[i].mbuf = &rxq->fake_mbuf;
				_mm_store_si128((__m128i *)&rxdp[i].read,
						dma_addr0);
			}
		}
		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
			ICE_RXQ_REARM_THRESH;
		return;
	}

	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
	for (i = 0; i < ICE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
		__m128i vaddr0, vaddr1;

		mb0 = rxep[0].mbuf;
		mb1 = rxep[1].mbuf;

		/* load buf_addr(lo 64bit) and buf_iova(hi 64bit) */
		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
				 offsetof(struct rte_mbuf, buf_addr) + 8);
		vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
		vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);

		/* convert pa to dma_addr hdr/data */
		dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
		dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);

		/* add headroom to pa values */
		dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
		dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);

		/* flush desc with pa dma_addr */
		_mm_store_si128((__m128i *)&rxdp++->read, dma_addr0);
		_mm_store_si128((__m128i *)&rxdp++->read, dma_addr1);
	}

	rxq->rxrearm_start += ICE_RXQ_REARM_THRESH;
	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
		rxq->rxrearm_start = 0;

	rxq->rxrearm_nb -= ICE_RXQ_REARM_THRESH;

	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
			   (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));

	/* Update the tail pointer on the NIC */
	ICE_PCI_REG_WC_WRITE(rxq->qrx_tail, rx_id);
}

static inline void
ice_rx_desc_to_olflags_v(struct ice_rx_queue *rxq, __m128i descs[4],
			 struct rte_mbuf **rx_pkts)
{
	const __m128i mbuf_init = _mm_set_epi64x(0, rxq->mbuf_initializer);
	__m128i rearm0, rearm1, rearm2, rearm3;

	__m128i tmp_desc, flags, rss_vlan;

	/* mask everything except checksum, RSS and VLAN flags.
	 * bit6:4 for checksum.
	 * bit12 for RSS indication.
	 * bit13 for VLAN indication.
	 */
	const __m128i desc_mask = _mm_set_epi32(0x3070, 0x3070,
						0x3070, 0x3070);

	const __m128i cksum_mask = _mm_set_epi32(PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD,
						 PKT_RX_IP_CKSUM_MASK |
						 PKT_RX_L4_CKSUM_MASK |
						 PKT_RX_EIP_CKSUM_BAD);

	/* map the checksum, rss and vlan fields to the checksum, rss
	 * and vlan flag
	 */
	const __m128i cksum_flags = _mm_set_epi8(0, 0, 0, 0, 0, 0, 0, 0,
			/* shift right 1 bit to make sure it not exceed 255 */
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
			 PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
			 PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |
			 PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_GOOD |
			 PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_GOOD) >> 1,
			(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD) >> 1,
			(PKT_RX_L4_CKSUM_GOOD | PKT_RX_IP_CKSUM_GOOD) >> 1);

	const __m128i rss_vlan_flags = _mm_set_epi8(0, 0, 0, 0,
			0, 0, 0, 0,
			0, 0, 0, 0,
			PKT_RX_RSS_HASH | PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
			PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED,
			PKT_RX_RSS_HASH, 0);

	/* merge 4 descriptors */
	flags = _mm_unpackhi_epi32(descs[0], descs[1]);
	tmp_desc = _mm_unpackhi_epi32(descs[2], descs[3]);
	tmp_desc = _mm_unpacklo_epi64(flags, tmp_desc);
	tmp_desc = _mm_and_si128(tmp_desc, desc_mask);

	/* checksum flags */
	tmp_desc = _mm_srli_epi32(tmp_desc, 4);
	flags = _mm_shuffle_epi8(cksum_flags, tmp_desc);
	/* then we shift left 1 bit */
	flags = _mm_slli_epi32(flags, 1);
	/* we need to mask out the reduntant bits introduced by RSS or
{'fraction_non_alphanumeric': 0.11095674724460498, 'fraction_numerical': 0.051495785866444384, 'mean_word_length': 4.380169407075237, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2471809', 'n_tokens_mistral': 5892, 'n_tokens_neox': 5052, 'n_words': 880}