<commit_before>/*
 * Copyright (c) 2006 Oracle.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <rdma/rdma_cm.h>

#include "rds.h"
#include "ib.h"

static struct kmem_cache *rds_ib_incoming_slab;
static struct kmem_cache *rds_ib_frag_slab;
static atomic_t	rds_ib_allocation = ATOMIC_INIT(0);

void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
{
	struct rds_ib_recv_work *recv;
	u32 i;

	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
		struct ib_sge *sge;

		recv->r_ibinc = NULL;
		recv->r_frag = NULL;

		recv->r_wr.next = NULL;
		recv->r_wr.wr_id = i;
		recv->r_wr.sg_list = recv->r_sge;
		recv->r_wr.num_sge = RDS_IB_RECV_SGE;

		sge = &recv->r_sge[0];
		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
		sge->length = sizeof(struct rds_header);
		sge->lkey = ic->i_mr->lkey;

		sge = &recv->r_sge[1];
		sge->addr = 0;
		sge->length = RDS_FRAG_SIZE;
		sge->lkey = ic->i_mr->lkey;
	}
}

/*
 * The entire 'from' list, including the from element itself, is put on
 * to the tail of the 'to' list.
 */
static void list_splice_entire_tail(struct list_head *from,
				    struct list_head *to)
{
	struct list_head *from_last = from->prev;

	list_splice_tail(from_last, to);
	list_add_tail(from_last, to);
}

static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache)
{
	struct list_head *tmp;

	tmp = xchg(&cache->xfer, NULL);
	if (tmp) {
		if (cache->ready)
			list_splice_entire_tail(tmp, cache->ready);
		else
			cache->ready = tmp;
	}
}

static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache)
{
	struct rds_ib_cache_head *head;
	int cpu;

	cache->percpu = alloc_percpu(struct rds_ib_cache_head);
	if (!cache->percpu)
	       return -ENOMEM;

	for_each_possible_cpu(cpu) {
		head = per_cpu_ptr(cache->percpu, cpu);
		head->first = NULL;
		head->count = 0;
	}
	cache->xfer = NULL;
	cache->ready = NULL;

	return 0;
}

int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic)
{
	int ret;

	ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs);
	if (!ret) {
		ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags);
		if (ret)
			free_percpu(ic->i_cache_incs.percpu);
	}

	return ret;
}

static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,
					  struct list_head *caller_list)
{
	struct rds_ib_cache_head *head;
	int cpu;

	for_each_possible_cpu(cpu) {
		head = per_cpu_ptr(cache->percpu, cpu);
		if (head->first) {
			list_splice_entire_tail(head->first, caller_list);
			head->first = NULL;
		}
	}

	if (cache->ready) {
		list_splice_entire_tail(cache->ready, caller_list);
		cache->ready = NULL;
	}
}

void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
{
	struct rds_ib_incoming *inc;
	struct rds_ib_incoming *inc_tmp;
	struct rds_page_frag *frag;
	struct rds_page_frag *frag_tmp;
	LIST_HEAD(list);

	rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
	rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list);
	free_percpu(ic->i_cache_incs.percpu);

	list_for_each_entry_safe(inc, inc_tmp, &list, ii_cache_entry) {
		list_del(&inc->ii_cache_entry);
		WARN_ON(!list_empty(&inc->ii_frags));
		kmem_cache_free(rds_ib_incoming_slab, inc);
	}

	rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
	rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);
	free_percpu(ic->i_cache_frags.percpu);

	list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
		list_del(&frag->f_cache_entry);
		WARN_ON(!list_empty(&frag->f_item));
		kmem_cache_free(rds_ib_frag_slab, frag);
	}
}

/* fwd decl */
static void rds_ib_recv_cache_put(struct list_head *new_item,
				  struct rds_ib_refill_cache *cache);
static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);


/* Recycle frag and attached recv buffer f_sg */
static void rds_ib_frag_free(struct rds_ib_connection *ic,
			     struct rds_page_frag *frag)
{
	rdsdebug("frag %p page %p\n", frag, sg_page(&frag->f_sg));

	rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags);
}

/* Recycle inc after freeing attached frags */
void rds_ib_inc_free(struct rds_incoming *inc)
{
	struct rds_ib_incoming *ibinc;
	struct rds_page_frag *frag;
	struct rds_page_frag *pos;
	struct rds_ib_connection *ic = inc->i_conn->c_transport_data;

	ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);

	/* Free attached frags */
	list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
		list_del_init(&frag->f_item);
		rds_ib_frag_free(ic, frag);
	}
	BUG_ON(!list_empty(&ibinc->ii_frags));

	rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
	rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs);
}

static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
				  struct rds_ib_recv_work *recv)
{
	if (recv->r_ibinc) {
		rds_inc_put(&recv->r_ibinc->ii_inc);
		recv->r_ibinc = NULL;
	}
	if (recv->r_frag) {
		ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
		rds_ib_frag_free(ic, recv->r_frag);
		recv->r_frag = NULL;
	}
}

void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
{
	u32 i;

	for (i = 0; i < ic->i_recv_ring.w_nr; i++)
		rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
}

static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,
						     gfp_t slab_mask)
{
	struct rds_ib_incoming *ibinc;
	struct list_head *cache_item;
	int avail_allocs;

	cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs);
	if (cache_item) {
		ibinc = container_of(cache_item, struct rds_ib_incoming, ii_cache_entry);
	} else {
		avail_allocs = atomic_add_unless(&rds_ib_allocation,
						 1, rds_ib_sysctl_max_recv_allocation);
		if (!avail_allocs) {
			rds_ib_stats_inc(s_ib_rx_alloc_limit);
			return NULL;
		}
		ibinc = kmem_cache_alloc(rds_ib_incoming_slab, slab_mask);
		if (!ibinc) {
			atomic_dec(&rds_ib_allocation);
			return NULL;
		}
	}
	INIT_LIST_HEAD(&ibinc->ii_frags);
	rds_inc_init(&ibinc->ii_inc, ic->conn, ic->conn->c_faddr);

	return ibinc;
}

static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,
						    gfp_t slab_mask, gfp_t page_mask)
{
	struct rds_page_frag *frag;
	struct list_head *cache_item;
	int ret;

	cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags);
	if (cache_item) {
		frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
	} else {
		frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
		if (!frag)
			return NULL;

		sg_init_table(&frag->f_sg, 1);
		ret = rds_page_remainder_alloc(&frag->f_sg,
					       RDS_FRAG_SIZE, page_mask);
		if (ret) {
			kmem_cache_free(rds_ib_frag_slab, frag);
			return NULL;
		}
	}

	INIT_LIST_HEAD(&frag->f_item);

	return frag;
}

static int rds_ib_recv_refill_one(struct rds_connection *conn,
				  struct rds_ib_recv_work *recv, gfp_t gfp)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct ib_sge *sge;
	int ret = -ENOMEM;
	gfp_t slab_mask = GFP_NOWAIT;
	gfp_t page_mask = GFP_NOWAIT;

	if (gfp & __GFP_WAIT) {
		slab_mask = GFP_KERNEL;
		page_mask = GFP_HIGHUSER;
	}

	if (!ic->i_cache_incs.ready)
		rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
	if (!ic->i_cache_frags.ready)
		rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);

	/*
	 * ibinc was taken from recv if recv contained the start of a message.
	 * recvs that were continuations will still have this allocated.
	 */
	if (!recv->r_ibinc) {
		recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);
		if (!recv->r_ibinc)
			goto out;
	}

	WARN_ON(recv->r_frag); /* leak! */
	recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);
	if (!recv->r_frag)
		goto out;

	ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,
			    1, DMA_FROM_DEVICE);
	WARN_ON(ret != 1);

	sge = &recv->r_sge[0];
	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
	sge->length = sizeof(struct rds_header);

	sge = &recv->r_sge[1];
	sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg);
	sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg);

	ret = 0;
out:
	return ret;
}

static int acquire_refill(struct rds_connection *conn)
{
	return test_and_set_bit(RDS_RECV_REFILL, &conn->c_flags) == 0;
}

static void release_refill(struct rds_connection *conn)
{
	clear_bit(RDS_RECV_REFILL, &conn->c_flags);

	/* We don't use wait_on_bit()/wake_up_bit() because our waking is in a
	 * hot path and finding waiters is very rare.  We don't want to walk
	 * the system-wide hashed waitqueue buckets in the fast path only to
	 * almost never find waiters.
	 */
	if (waitqueue_active(&conn->c_waitq))
		wake_up_all(&conn->c_waitq);
}

/*
 * This tries to allocate and post unused work requests after making sure that
 * they have all the allocations they need to queue received fragments into
 * sockets.
 *
 * -1 is returned if posting fails due to temporary resource exhaustion.
 */
void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct rds_ib_recv_work *recv;
	struct ib_recv_wr *failed_wr;
	unsigned int posted = 0;
	int ret = 0;
	int can_wait = gfp & __GFP_WAIT;
	u32 pos;

	/* the goal here is to just make sure that someone, somewhere
	 * is posting buffers.  If we can't get the refill lock,
	 * let them do their thing
	 */
	if (!acquire_refill(conn))
		return;

	while ((prefill || rds_conn_up(conn)) &&
	       rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
		if (pos >= ic->i_recv_ring.w_nr) {
			printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
					pos);
			break;
		}
<commit_msg>rds: Fix improper gfp_t usage.<commit_after>/*
 * Copyright (c) 2006 Oracle.  All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <rdma/rdma_cm.h>

#include "rds.h"
#include "ib.h"

static struct kmem_cache *rds_ib_incoming_slab;
static struct kmem_cache *rds_ib_frag_slab;
static atomic_t	rds_ib_allocation = ATOMIC_INIT(0);

void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
{
	struct rds_ib_recv_work *recv;
	u32 i;

	for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
		struct ib_sge *sge;

		recv->r_ibinc = NULL;
		recv->r_frag = NULL;

		recv->r_wr.next = NULL;
		recv->r_wr.wr_id = i;
		recv->r_wr.sg_list = recv->r_sge;
		recv->r_wr.num_sge = RDS_IB_RECV_SGE;

		sge = &recv->r_sge[0];
		sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
		sge->length = sizeof(struct rds_header);
		sge->lkey = ic->i_mr->lkey;

		sge = &recv->r_sge[1];
		sge->addr = 0;
		sge->length = RDS_FRAG_SIZE;
		sge->lkey = ic->i_mr->lkey;
	}
}

/*
 * The entire 'from' list, including the from element itself, is put on
 * to the tail of the 'to' list.
 */
static void list_splice_entire_tail(struct list_head *from,
				    struct list_head *to)
{
	struct list_head *from_last = from->prev;

	list_splice_tail(from_last, to);
	list_add_tail(from_last, to);
}

static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache)
{
	struct list_head *tmp;

	tmp = xchg(&cache->xfer, NULL);
	if (tmp) {
		if (cache->ready)
			list_splice_entire_tail(tmp, cache->ready);
		else
			cache->ready = tmp;
	}
}

static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache)
{
	struct rds_ib_cache_head *head;
	int cpu;

	cache->percpu = alloc_percpu(struct rds_ib_cache_head);
	if (!cache->percpu)
	       return -ENOMEM;

	for_each_possible_cpu(cpu) {
		head = per_cpu_ptr(cache->percpu, cpu);
		head->first = NULL;
		head->count = 0;
	}
	cache->xfer = NULL;
	cache->ready = NULL;

	return 0;
}

int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic)
{
	int ret;

	ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs);
	if (!ret) {
		ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags);
		if (ret)
			free_percpu(ic->i_cache_incs.percpu);
	}

	return ret;
}

static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,
					  struct list_head *caller_list)
{
	struct rds_ib_cache_head *head;
	int cpu;

	for_each_possible_cpu(cpu) {
		head = per_cpu_ptr(cache->percpu, cpu);
		if (head->first) {
			list_splice_entire_tail(head->first, caller_list);
			head->first = NULL;
		}
	}

	if (cache->ready) {
		list_splice_entire_tail(cache->ready, caller_list);
		cache->ready = NULL;
	}
}

void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
{
	struct rds_ib_incoming *inc;
	struct rds_ib_incoming *inc_tmp;
	struct rds_page_frag *frag;
	struct rds_page_frag *frag_tmp;
	LIST_HEAD(list);

	rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
	rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list);
	free_percpu(ic->i_cache_incs.percpu);

	list_for_each_entry_safe(inc, inc_tmp, &list, ii_cache_entry) {
		list_del(&inc->ii_cache_entry);
		WARN_ON(!list_empty(&inc->ii_frags));
		kmem_cache_free(rds_ib_incoming_slab, inc);
	}

	rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
	rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);
	free_percpu(ic->i_cache_frags.percpu);

	list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
		list_del(&frag->f_cache_entry);
		WARN_ON(!list_empty(&frag->f_item));
		kmem_cache_free(rds_ib_frag_slab, frag);
	}
}

/* fwd decl */
static void rds_ib_recv_cache_put(struct list_head *new_item,
				  struct rds_ib_refill_cache *cache);
static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);


/* Recycle frag and attached recv buffer f_sg */
static void rds_ib_frag_free(struct rds_ib_connection *ic,
			     struct rds_page_frag *frag)
{
	rdsdebug("frag %p page %p\n", frag, sg_page(&frag->f_sg));

	rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags);
}

/* Recycle inc after freeing attached frags */
void rds_ib_inc_free(struct rds_incoming *inc)
{
	struct rds_ib_incoming *ibinc;
	struct rds_page_frag *frag;
	struct rds_page_frag *pos;
	struct rds_ib_connection *ic = inc->i_conn->c_transport_data;

	ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);

	/* Free attached frags */
	list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
		list_del_init(&frag->f_item);
		rds_ib_frag_free(ic, frag);
	}
	BUG_ON(!list_empty(&ibinc->ii_frags));

	rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
	rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs);
}

static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
				  struct rds_ib_recv_work *recv)
{
	if (recv->r_ibinc) {
		rds_inc_put(&recv->r_ibinc->ii_inc);
		recv->r_ibinc = NULL;
	}
	if (recv->r_frag) {
		ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
		rds_ib_frag_free(ic, recv->r_frag);
		recv->r_frag = NULL;
	}
}

void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
{
	u32 i;

	for (i = 0; i < ic->i_recv_ring.w_nr; i++)
		rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
}

static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,
						     gfp_t slab_mask)
{
	struct rds_ib_incoming *ibinc;
	struct list_head *cache_item;
	int avail_allocs;

	cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs);
	if (cache_item) {
		ibinc = container_of(cache_item, struct rds_ib_incoming, ii_cache_entry);
	} else {
		avail_allocs = atomic_add_unless(&rds_ib_allocation,
						 1, rds_ib_sysctl_max_recv_allocation);
		if (!avail_allocs) {
			rds_ib_stats_inc(s_ib_rx_alloc_limit);
			return NULL;
		}
		ibinc = kmem_cache_alloc(rds_ib_incoming_slab, slab_mask);
		if (!ibinc) {
			atomic_dec(&rds_ib_allocation);
			return NULL;
		}
	}
	INIT_LIST_HEAD(&ibinc->ii_frags);
	rds_inc_init(&ibinc->ii_inc, ic->conn, ic->conn->c_faddr);

	return ibinc;
}

static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,
						    gfp_t slab_mask, gfp_t page_mask)
{
	struct rds_page_frag *frag;
	struct list_head *cache_item;
	int ret;

	cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags);
	if (cache_item) {
		frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
	} else {
		frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
		if (!frag)
			return NULL;

		sg_init_table(&frag->f_sg, 1);
		ret = rds_page_remainder_alloc(&frag->f_sg,
					       RDS_FRAG_SIZE, page_mask);
		if (ret) {
			kmem_cache_free(rds_ib_frag_slab, frag);
			return NULL;
		}
	}

	INIT_LIST_HEAD(&frag->f_item);

	return frag;
}

static int rds_ib_recv_refill_one(struct rds_connection *conn,
				  struct rds_ib_recv_work *recv, gfp_t gfp)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct ib_sge *sge;
	int ret = -ENOMEM;
	gfp_t slab_mask = GFP_NOWAIT;
	gfp_t page_mask = GFP_NOWAIT;

	if (gfp & __GFP_WAIT) {
		slab_mask = GFP_KERNEL;
		page_mask = GFP_HIGHUSER;
	}

	if (!ic->i_cache_incs.ready)
		rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
	if (!ic->i_cache_frags.ready)
		rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);

	/*
	 * ibinc was taken from recv if recv contained the start of a message.
	 * recvs that were continuations will still have this allocated.
	 */
	if (!recv->r_ibinc) {
		recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);
		if (!recv->r_ibinc)
			goto out;
	}

	WARN_ON(recv->r_frag); /* leak! */
	recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);
	if (!recv->r_frag)
		goto out;

	ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,
			    1, DMA_FROM_DEVICE);
	WARN_ON(ret != 1);

	sge = &recv->r_sge[0];
	sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
	sge->length = sizeof(struct rds_header);

	sge = &recv->r_sge[1];
	sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg);
	sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg);

	ret = 0;
out:
	return ret;
}

static int acquire_refill(struct rds_connection *conn)
{
	return test_and_set_bit(RDS_RECV_REFILL, &conn->c_flags) == 0;
}

static void release_refill(struct rds_connection *conn)
{
	clear_bit(RDS_RECV_REFILL, &conn->c_flags);

	/* We don't use wait_on_bit()/wake_up_bit() because our waking is in a
	 * hot path and finding waiters is very rare.  We don't want to walk
	 * the system-wide hashed waitqueue buckets in the fast path only to
	 * almost never find waiters.
	 */
	if (waitqueue_active(&conn->c_waitq))
		wake_up_all(&conn->c_waitq);
}

/*
 * This tries to allocate and post unused work requests after making sure that
 * they have all the allocations they need to queue received fragments into
 * sockets.
 *
 * -1 is returned if posting fails due to temporary resource exhaustion.
 */
void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp)
{
	struct rds_ib_connection *ic = conn->c_transport_data;
	struct rds_ib_recv_work *recv;
	struct ib_recv_wr *failed_wr;
	unsigned int posted = 0;
	int ret = 0;
	bool can_wait = !!(gfp & __GFP_WAIT);
	u32 pos;

	/* the goal here is to just make sure that someone, somewhere
	 * is posting buffers.  If we can't get the refill lock,
	 * let them do their thing
	 */
	if (!acquire_refill(conn))
		return;

	while ((prefill || rds_conn_up(conn)) &&
	       rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
		if (pos >= ic->i_recv_ring.w_nr) {
			printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
					pos);
			break;
		}

{'fraction_non_alphanumeric': 0.11493518349461385, 'fraction_numerical': 0.002921307285010042, 'mean_word_length': 4.538169868554095, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 265, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4964946', 'n_tokens_mistral': 9213, 'n_tokens_neox': 8721, 'n_words': 1922}