scst/scst/src/scst_mem.h

/*
 *  scst_mem.h
 *
 *  Copyright (C) 2006 - 2018 Vladislav Bolkhovitin <vst@vlnb.net>
 *  Copyright (C) 2007 - 2018 Western Digital Corporation
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation, version 2
 *  of the License.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 */

#include <linux/scatterlist.h>
#include <linux/workqueue.h>

#define SGV_POOL_ELEMENTS	11

/*
 * sg_num is indexed by the page number, pg_count is indexed by the sg number.
 * Made in one entry to simplify the code (eg all sizeof(*) parts) and save
 * some CPU cache for non-clustered case.
 */
struct trans_tbl_ent {
	unsigned short sg_num;
	unsigned short pg_count;
};

/*
 * SGV pool object
 */
struct sgv_pool_obj {
	int cache_num;
	int pages;

	/* jiffies, protected by sgv_pool_lock */
	unsigned long time_stamp;

	struct list_head recycling_list_entry;
	struct list_head sorted_recycling_list_entry;

	struct sgv_pool *owner_pool;
	int orig_sg;
	int orig_length;
	int sg_count;
	void *allocator_priv;
	struct trans_tbl_ent *trans_tbl;
	struct scatterlist *sg_entries;
	struct scatterlist sg_entries_data[0];
};

/*
 * SGV pool statistics accounting structure
 */
struct sgv_pool_cache_acc {
	atomic_t total_alloc, hit_alloc;
	atomic_t merged;
};

/*
 * SGV pool allocation functions
 */
struct sgv_pool_alloc_fns {
	struct page *(*alloc_pages_fn)(struct scatterlist *sg, gfp_t gfp_mask,
		void *priv);
	void (*free_pages_fn)(struct scatterlist *sg, int sg_count,
		void *priv);
}
#ifdef CONSTIFY_PLUGIN
/* Avoid that the Grsecurity gcc constify_plugin constifies this structure. */
__attribute__((no_const))
#endif
;

/*
 * SGV pool
 */
struct sgv_pool {
	enum sgv_clustering_types clustering_type;
	int single_alloc_pages;
	int max_cached_pages;

	struct sgv_pool_alloc_fns alloc_fns;

	/* <=4K, <=8, <=16, <=32, <=64, <=128, <=256, <=512, <=1024, <=2048 */
	struct kmem_cache *caches[SGV_POOL_ELEMENTS];

	spinlock_t sgv_pool_lock; /* outer lock for sgv_pools_lock! */

	int purge_interval;

	/* Protected by sgv_pool_lock, if necessary */
	unsigned int purge_work_scheduled:1;

	/* Protected by sgv_pool_lock */
	struct list_head sorted_recycling_list;

	int inactive_cached_pages; /* protected by sgv_pool_lock */

	/* Protected by sgv_pool_lock */
	struct list_head recycling_lists[SGV_POOL_ELEMENTS];

	int cached_pages, cached_entries; /* protected by sgv_pool_lock */

	struct sgv_pool_cache_acc cache_acc[SGV_POOL_ELEMENTS];

#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 20))
	struct delayed_work sgv_purge_work;
#else
	struct work_struct sgv_purge_work;
#endif

	atomic_t big_alloc, big_pages, big_merged;
	atomic_t other_alloc, other_pages, other_merged;

	atomic_t sgv_pool_ref;

	int max_caches;

	/* SCST_MAX_NAME + few more bytes to match scst_user expectations */
	char cache_names[SGV_POOL_ELEMENTS][SCST_MAX_NAME + 10];
	char name[SCST_MAX_NAME + 10];

	struct mm_struct *owner_mm;

	struct list_head sgv_pools_list_entry;

	struct kobject sgv_kobj;

	/* sysfs release completion */
	struct completion *sgv_kobj_release_cmpl;
};

extern bool scst_force_global_sgv_pool;

static inline struct scatterlist *sgv_pool_sg(struct sgv_pool_obj *obj)
{
	return obj->sg_entries;
}

int scst_sgv_pools_init(unsigned long mem_hwmark, unsigned long mem_lwmark);
void scst_sgv_pools_deinit(void);


void scst_sgv_pool_use_norm(struct scst_tgt_dev *tgt_dev);
void scst_sgv_pool_use_norm_clust(struct scst_tgt_dev *tgt_dev);
void scst_sgv_pool_use_dma(struct scst_tgt_dev *tgt_dev);