scst/www/sgv_big_order_alloc.diff

In the pass-through mode (i.e. using the pass-through device handlers
scst_disk, scst_tape, etc) SCSI commands, coming from remote initiators,
are passed to local SCSI hardware on target as is, without any
modifications. As any other hardware, the local SCSI hardware can not
handle commands with amount of data and/or segments count in
scatter-gather array bigger some values. If you have this issue you will
see symptoms like small transfers work well, but large ones stall and
messages like: "Unable to complete command due to SG IO count
limitation" are printed in the kernel logs.

This is proposed patch to solve that. It allows SGV cache do allocation
of pages with order > 0, i.e. more than 1 page per SG entry.

Compile tested only.

Index: scst/include/scst.h
===================================================================
--- scst/include/scst.h	(revision 558)
+++ scst/include/scst.h	(working copy)
@@ -2649,12 +2649,13 @@ struct sgv_pool *sgv_pool_create(const c
 void sgv_pool_destroy(struct sgv_pool *pool);
 
 void sgv_pool_set_allocator(struct sgv_pool *pool,
-	struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *),
-	void (*free_pages_fn)(struct scatterlist *, int, void *));
+	struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *, int),
+	void (*free_pages_fn)(struct scatterlist *, int, void *, int));
 
 struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
 	gfp_t gfp_mask, int flags, int *count,
-	struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv);
+	struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv,
+	int max_sg_count);
 void sgv_pool_free(struct sgv_pool_obj *sgv, struct scst_mem_lim *mem_lim);
 
 void *sgv_get_priv(struct sgv_pool_obj *sgv);
Index: scst/src/scst_mem.h
===================================================================
--- scst/src/scst_mem.h	(revision 558)
+++ scst/src/scst_mem.h	(working copy)
@@ -36,6 +36,8 @@ struct sgv_pool_obj {
 	/* if <0 - pages, >0 - order */
 	int order_or_pages;
 
+	int alloc_order;
+
 	struct {
 		/* jiffies, protected by pool_mgr_lock */
 		unsigned long time_stamp;
@@ -67,9 +69,9 @@ struct sgv_pool_cache_acc {
 
 struct sgv_pool_alloc_fns {
 	struct page *(*alloc_pages_fn)(struct scatterlist *sg, gfp_t gfp_mask,
-		void *priv);
+		void *priv, int alloc_order);
 	void (*free_pages_fn)(struct scatterlist *sg, int sg_count,
-		void *priv);
+		void *priv, int alloc_order);
 };
 
 struct sgv_pool {
Index: scst/src/scst_lib.c
===================================================================
--- scst/src/scst_lib.c	(revision 558)
+++ scst/src/scst_lib.c	(working copy)
@@ -1663,34 +1663,18 @@ int scst_alloc_space(struct scst_cmd *cm
 		flags |= SCST_POOL_ALLOC_NO_CACHED;
 
 	cmd->sg = sgv_pool_alloc(tgt_dev->pool, cmd->bufflen, gfp_mask, flags,
-			&cmd->sg_cnt, &cmd->sgv, &cmd->dev->dev_mem_lim, NULL);
+			&cmd->sg_cnt, &cmd->sgv, &cmd->dev->dev_mem_lim, NULL,
+			tgt_dev->max_sg_cnt);
 	if (cmd->sg == NULL)
 		goto out;
 
-	if (unlikely(cmd->sg_cnt > tgt_dev->max_sg_cnt)) {
-		static int ll;
-		if (ll < 10) {
-			PRINT_INFO("Unable to complete command due to "
-				"SG IO count limitation (requested %d, "
-				"available %d, tgt lim %d)", cmd->sg_cnt,
-				tgt_dev->max_sg_cnt, cmd->tgt->sg_tablesize);
-			ll++;
-		}
-		goto out_sg_free;
-	}
+	EXTRACHECKS_BUG_ON(cmd->sg_cnt > tgt_dev->max_sg_cnt);
 
 	res = 0;
 
 out:
 	TRACE_EXIT();
 	return res;
-
-out_sg_free:
-	sgv_pool_free(cmd->sgv, &cmd->dev->dev_mem_lim);
-	cmd->sgv = NULL;
-	cmd->sg = NULL;
-	cmd->sg_cnt = 0;
-	goto out;
 }
 
 void scst_release_space(struct scst_cmd *cmd)
Index: scst/src/scst_mem.c
===================================================================
--- scst/src/scst_mem.c	(revision 558)
+++ scst/src/scst_mem.c	(working copy)
@@ -118,7 +118,7 @@ out_head:
 }
 
 static void scst_free_sys_sg_entries(struct scatterlist *sg, int sg_count,
-	void *priv)
+	void *priv, int alloc_order)
 {
 	int i;
 
@@ -134,7 +134,7 @@ static void scst_free_sys_sg_entries(str
 			(unsigned long)p, len, pages);
 
 		while (pages > 0) {
-			int order = 0;
+			int order = alloc_order;
 
 /*
  * __free_pages() doesn't like freeing pages with not that order with
@@ -159,9 +159,9 @@ static void scst_free_sys_sg_entries(str
 }
 
 static struct page *scst_alloc_sys_pages(struct scatterlist *sg,
-	gfp_t gfp_mask, void *priv)
+	gfp_t gfp_mask, void *priv, int alloc_order)
 {
-	struct page *page = alloc_pages(gfp_mask, 0);
+	struct page *page = alloc_pages(gfp_mask, alloc_order);
 
 	sg_set_page(sg, page, PAGE_SIZE, 0);
 	TRACE_MEM("page=%p, sg=%p, priv=%p", page, sg, priv);
@@ -174,10 +174,10 @@ static struct page *scst_alloc_sys_pages
 
 static int scst_alloc_sg_entries(struct scatterlist *sg, int pages,
 	gfp_t gfp_mask, int clustered, struct trans_tbl_ent *trans_tbl,
-	const struct sgv_pool_alloc_fns *alloc_fns, void *priv)
+	const struct sgv_pool_alloc_fns *alloc_fns, void *priv, int alloc_order)
 {
 	int sg_count = 0;
-	int pg, i, j;
+	int pg, i, j, pg_inc = 1 << alloc_order;
 	int merged = -1;
 
 	TRACE_MEM("pages=%d, clustered=%d", pages, clustered);
@@ -189,7 +189,7 @@ static int scst_alloc_sg_entries(struct 
 	gfp_mask |= __GFP_ZERO;
 #endif
 
-	for (pg = 0; pg < pages; pg++) {
+	for (pg = 0; pg < pages; pg += pg_inc) {
 		void *rc;
 #ifdef CONFIG_SCST_DEBUG_OOM
 		if (((gfp_mask & __GFP_NOFAIL) != __GFP_NOFAIL) &&
@@ -198,7 +198,7 @@ static int scst_alloc_sg_entries(struct 
 		else
 #endif
 			rc = alloc_fns->alloc_pages_fn(&sg[sg_count], gfp_mask,
-				priv);
+				priv, alloc_order);
 		if (rc == NULL)
 			goto out_no_mem;
 		if (clustered) {
@@ -229,7 +229,7 @@ out:
 	return sg_count;
 
 out_no_mem:
-	alloc_fns->free_pages_fn(sg, sg_count, priv);
+	alloc_fns->free_pages_fn(sg, sg_count, priv, alloc_order);
 	sg_count = 0;
 	goto out;
 }
@@ -292,7 +292,7 @@ static void sgv_dtor_and_free(struct sgv
 {
 	if (obj->sg_count != 0) {
 		obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries,
-			obj->sg_count, obj->allocator_priv);
+			obj->sg_count, obj->allocator_priv, obj->alloc_order);
 	}
 	if (obj->sg_entries != obj->sg_entries_data) {
 		if (obj->trans_tbl !=
@@ -308,6 +308,36 @@ static void sgv_dtor_and_free(struct sgv
 	return;
 }
 
+static struct sgv_pool_obj *sgv_pool_cached_create(struct sgv_pool *pool,
+	int order, gfp_t gfp_mask, bool locked)
+{
+	struct sgv_pool_obj *obj;
+	int pages = 1 << order;
+
+	if (!locked)
+		spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
+
+	pool->acc.cached_entries++;
+	pool->acc.cached_pages += pages;
+
+	spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
+
+	obj = kmem_cache_alloc(pool->caches[order],
+		gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA));
+	if (likely(obj)) {
+		memset(obj, 0, sizeof(*obj));
+		obj->order_or_pages = order;
+		obj->owner_pool = pool;
+	} else {
+		spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
+		pool->acc.cached_entries--;
+		pool->acc.cached_pages -= pages;
+		spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
+	}
+
+	return obj;
+}
+
 static struct sgv_pool_obj *sgv_pool_cached_get(struct sgv_pool *pool,
 	int order, gfp_t gfp_mask)
 {
@@ -332,23 +362,7 @@ static struct sgv_pool_obj *sgv_pool_cac
 		goto out;
 	}
 
-	pool->acc.cached_entries++;
-	pool->acc.cached_pages += pages;
-
-	spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
-
-	obj = kmem_cache_alloc(pool->caches[order],
-		gfp_mask & ~(__GFP_HIGHMEM|GFP_DMA));
-	if (likely(obj)) {
-		memset(obj, 0, sizeof(*obj));
-		obj->order_or_pages = order;
-		obj->owner_pool = pool;
-	} else {
-		spin_lock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
-		pool->acc.cached_entries--;
-		pool->acc.cached_pages -= pages;
-		spin_unlock_bh(&sgv_pools_mgr.mgr.pool_mgr_lock);
-	}
+	obj = sgv_pool_cached_create(pool, order, gfp_mask, true);
 
 out:
 	return obj;
@@ -546,12 +560,13 @@ static void scst_uncheck_allowed_mem(str
 
 struct scatterlist *sgv_pool_alloc(struct sgv_pool *pool, unsigned int size,
 	gfp_t gfp_mask, int flags, int *count,
-	struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv)
+	struct sgv_pool_obj **sgv, struct scst_mem_lim *mem_lim, void *priv,
+	int max_sg_count)
 {
 	struct sgv_pool_obj *obj;
 	int order, pages, cnt;
 	struct scatterlist *res = NULL;
-	int pages_to_alloc;
+	int pages_to_alloc, alloc_order;
 	struct kmem_cache *cache;
 	int no_cached = flags & SCST_POOL_ALLOC_NO_CACHED;
 	bool allowed_mem_checked = false, hiwmk_checked = false;
@@ -605,7 +620,23 @@ struct scatterlist *sgv_pool_alloc(struc
 		if (obj->sg_count != 0) {
 			TRACE_MEM("Cached sgv_obj %p", obj);
 			EXTRACHECKS_BUG_ON(obj->order_or_pages != order);
-			atomic_inc(&pool->cache_acc[order].hit_alloc);
+
+			if (unlikely(max_sg_count < obj->sg_count)) {
+				TRACE_MEM("Too many SG entries %d (max %d)",
+					obj->sg_count, max_sg_count);
+
+				sgv_pool_cached_put(obj);
+
+				obj = sgv_pool_cached_create(pool, order,
+						gfp_mask, false);
+				if (obj == NULL) {
+					TRACE(TRACE_OUT_OF_MEM, "Allocation of "
+						"sgv_pool_obj failed (size %d)",
+						size);
+					goto out_fail;
+				}
+			} else
+				atomic_inc(&pool->cache_acc[order].hit_alloc);
 			goto success;
 		}
 
@@ -682,15 +713,27 @@ struct scatterlist *sgv_pool_alloc(struc
 		TRACE_MEM("Big or no_cached sgv_obj %p (size %d)", obj,	sz);
 	}
 
-	obj->sg_count = scst_alloc_sg_entries(obj->sg_entries,
-		pages_to_alloc, gfp_mask, pool->clustered, obj->trans_tbl,
-		&pool->alloc_fns, priv);
-	if (unlikely(obj->sg_count <= 0)) {
-		obj->sg_count = 0;
-		if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache)
-			goto out_return1;
-		else
-			goto out_fail_free_sg_entries;
+	alloc_order = 0;
+	while (1) {
+		obj->sg_count = scst_alloc_sg_entries(obj->sg_entries,
+			pages_to_alloc, gfp_mask, pool->clustered,
+			obj->trans_tbl, &pool->alloc_fns, priv, alloc_order);
+		if (unlikely(obj->sg_count <= 0)) {
+			obj->sg_count = 0;
+			if ((flags & SCST_POOL_RETURN_OBJ_ON_ALLOC_FAIL) && cache)
+				goto out_return1;
+			else
+				goto out_fail_free_sg_entries;
+		}
+		obj->alloc_order = alloc_order;
+
+		if (max_sg_count >= obj->sg_count)
+			break;
+
+		obj->owner_pool->alloc_fns.free_pages_fn(obj->sg_entries,
+			obj->sg_count, obj->allocator_priv,
+			obj->alloc_order);
+		alloc_order++;
 	}
 
 	if (cache) {
@@ -815,7 +858,7 @@ void sgv_pool_free(struct sgv_pool_obj *
 		sgv_pool_cached_put(sgv);
 	} else {
 		sgv->owner_pool->alloc_fns.free_pages_fn(sgv->sg_entries,
-			sgv->sg_count, sgv->allocator_priv);
+			sgv->sg_count, sgv->allocator_priv, sgv->alloc_order);
 		pages = (sgv->sg_count != 0) ? -sgv->order_or_pages : 0;
 		kfree(sgv);
 		sgv_pool_hiwmk_uncheck(pages);
@@ -861,7 +904,7 @@ struct scatterlist *scst_alloc(int size,
 	 * So, always don't use clustering.
 	 */
 	*count = scst_alloc_sg_entries(res, pages, gfp_mask, 0, NULL,
-			&sys_alloc_fns, NULL);
+			&sys_alloc_fns, NULL, 0);
 	if (*count <= 0)
 		goto out_free;
 
@@ -888,7 +931,7 @@ void scst_free(struct scatterlist *sg, i
 
 	sgv_pool_hiwmk_uncheck(count);
 
-	scst_free_sys_sg_entries(sg, count, NULL);
+	scst_free_sys_sg_entries(sg, count, NULL, 0);
 	kfree(sg);
 	return;
 }
@@ -1060,8 +1103,8 @@ void sgv_pool_deinit(struct sgv_pool *po
 }
 
 void sgv_pool_set_allocator(struct sgv_pool *pool,
-	struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *),
-	void (*free_pages_fn)(struct scatterlist *, int, void *))
+	struct page *(*alloc_pages_fn)(struct scatterlist *, gfp_t, void *, int),
+	void (*free_pages_fn)(struct scatterlist *, int, void *, int))
 {
 	pool->alloc_fns.alloc_pages_fn = alloc_pages_fn;
 	pool->alloc_fns.free_pages_fn = free_pages_fn;
Index: scst/src/dev_handlers/scst_user.c
===================================================================
--- scst/src/dev_handlers/scst_user.c	(revision 559)
+++ scst/src/dev_handlers/scst_user.c	(working copy)
@@ -168,9 +168,9 @@ static int dev_user_disk_done(struct scs
 static int dev_user_tape_done(struct scst_cmd *cmd);
 
 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
-	gfp_t gfp_mask, void *priv);
+	gfp_t gfp_mask, void *priv, int alloc_order);
 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
-				     void *priv);
+				     void *priv, int alloc_order);
 
 static void dev_user_add_to_ready(struct scst_user_cmd *ucmd);
 
@@ -368,7 +368,7 @@ static void dev_user_free_ucmd(struct sc
 }
 
 static struct page *dev_user_alloc_pages(struct scatterlist *sg,
-	gfp_t gfp_mask, void *priv)
+	gfp_t gfp_mask, void *priv, int alloc_order)
 {
 	struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 	int offset = 0;
@@ -377,8 +377,11 @@ static struct page *dev_user_alloc_pages
 
 	/* *sg supposed to be zeroed */
 
-	TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d", ucmd,
-		ucmd->ubuff, ucmd->cur_data_page);
+	TRACE_MEM("ucmd %p, ubuff %lx, ucmd->cur_data_page %d, alloc_order %d",
+		ucmd, ucmd->ubuff, ucmd->cur_data_page, alloc_order);
+
+	if (unlikely(alloc_order != 0))
+		goto out;
 
 	if (ucmd->cur_data_page == 0) {
 		TRACE_MEM("ucmd->first_page_offset %d",
@@ -468,7 +471,7 @@ static void __dev_user_free_sg_entries(s
 }
 
 static void dev_user_free_sg_entries(struct scatterlist *sg, int sg_count,
-	void *priv)
+	void *priv, int alloc_order)
 {
 	struct scst_user_cmd *ucmd = (struct scst_user_cmd *)priv;
 
@@ -537,7 +540,8 @@ static int dev_user_alloc_sg(struct scst
 	ucmd->buff_cached = cached_buff;
 
 	cmd->sg = sgv_pool_alloc(dev->pool, bufflen, gfp_mask, flags,
-			&cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd);
+			&cmd->sg_cnt, &ucmd->sgv, &dev->udev_mem_lim, ucmd,
+			cmd->tgt_dev->max_sg_cnt);
 	if (cmd->sg != NULL) {
 		struct scst_user_cmd *buf_ucmd =
 			(struct scst_user_cmd *)sgv_get_priv(ucmd->sgv);
@@ -559,21 +563,7 @@ static int dev_user_alloc_sg(struct scst
 			"last_len %d, l %d)", ucmd, cached_buff, ucmd->ubuff,
 			last_len, cmd->sg[cmd->sg_cnt-1].length);
 
-		if (unlikely(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt)) {
-			static int ll;
-			if (ll < 10) {
-				PRINT_INFO("Unable to complete command due to "
-					"SG IO count limitation (requested %d, "
-					"available %d, tgt lim %d)",
-					cmd->sg_cnt,
-					cmd->tgt_dev->max_sg_cnt,
-					cmd->tgt->sg_tablesize);
-				ll++;
-			}
-			cmd->sg = NULL;
-			/* sgv will be freed in dev_user_free_sgv() */
-			res = -1;
-		}
+		EXTRACHECKS_BUG_ON(cmd->sg_cnt > cmd->tgt_dev->max_sg_cnt);
 	} else {
 		TRACE_MEM("Buf not alloced (ucmd %p, h %d, buff_cached, %d, "
 			"sg_cnt %d, ubuff %lx, sgv %p", ucmd, ucmd->h,