XDMA driver refresh

XDMA/linux-kernel/RELEASE

+Release: 2019.2
+===============
+
+Change list:
+- Updated data rate for performance run. Now it will print data rate based on the size.
+- remove BUG_ON, return proper error code instead
+- Streaming mode: enable credit mechanism by default 
+- Streaming mode: Do not read more than user supplied buffer size on C2H
+- Streaming mode: Added support for Async-IO for both streaming and MM transfers
+- fixed performance appliaction crash

XDMA/linux-kernel/include/libxdma_api.h

@@ -133,6 +133,14 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask);
 ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
 
+ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
+			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
+
+
+ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
+			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
+
+			
 
 /////////////////////missing API////////////////////
 

XDMA/linux-kernel/libxdma/libxdma.h

@@ -31,7 +31,7 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/workqueue.h>
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 #include <linux/swait.h>
 #endif
 /*
@@ -83,6 +83,7 @@
 #define XDMA_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
 #define XDMA_CTRL_NON_INCR_ADDR			(1UL << 25)
 #define XDMA_CTRL_POLL_MODE_WB			(1UL << 26)
+#define XDMA_CTRL_STM_MODE_WB			(1UL << 27)
 
 /* bits of the SG DMA status register */
 #define XDMA_STAT_BUSY			(1UL << 0)
@@ -247,6 +248,23 @@ enum dev_capabilities {
 
 /* SECTION: Structure definitions */
 
+struct xdma_io_cb {
+	void __user *buf;
+	size_t len;
+	void *private;
+	unsigned int pages_nr;
+	struct sg_table sgt;
+	struct page **pages;
+	/** total data size */
+	unsigned int count;
+	/** MM only, DDR/BRAM memory addr */
+	u64 ep_addr;
+	/** write: if write to the device */
+	struct xdma_request_cb *req;
+	u8 write:1;
+	void (*io_done)(unsigned long cb_hndl, int err);
+};
+
 struct config_regs {
 	u32 identifier;
 	u32 reserved_1[4];
@@ -392,11 +410,14 @@ struct sw_desc {
 struct xdma_transfer {
 	struct list_head entry;		/* queue of non-completed transfers */
 	struct xdma_desc *desc_virt;	/* virt addr of the 1st descriptor */
+	struct xdma_result *res_virt; /* virt addr of result for c2h streaming */
+	dma_addr_t res_bus;			  /* bus addr for result descriptors */
 	dma_addr_t desc_bus;		/* bus addr of the first descriptor */
 	int desc_adjacent;		/* adjacent descriptors at desc_bus */
 	int desc_num;			/* number of descriptors in transfer */
+	int desc_index;			/* index for first descriptor in transfer */
 	enum dma_data_direction dir;
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head wq;
 #else
 	wait_queue_head_t wq;		/* wait queue for transfer completion */
@@ -409,6 +430,7 @@ struct xdma_transfer {
 	int last_in_request;		/* flag if last within request */
 	unsigned int len;
 	struct sg_table *sgt;
+	struct xdma_io_cb *cb;
 };
 
 struct xdma_request_cb {
@@ -416,7 +438,8 @@ struct xdma_request_cb {
 	unsigned int total_len;
 	u64 ep_addr;
 
-	struct xdma_transfer xfer;
+	struct xdma_transfer tfer[2]; /* Use two transfers in case single request needs to be split */
+	struct xdma_io_cb *cb;
 
 	unsigned int sw_desc_idx;
 	unsigned int sw_desc_cnt;
@@ -450,7 +473,8 @@ struct xdma_engine {
 	int max_extra_adj;	/* descriptor prefetch capability */
 	int desc_dequeued;	/* num descriptors of completed transfers */
 	u32 status;		/* last known status of device */
-	u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */
+	/* only used for MSIX mode to store per-engine interrupt mask value */
+	u32 interrupt_enable_mask_value;
 
 	/* Transfer list management */
 	struct list_head transfer_list;	/* queue of transfers */
@@ -460,6 +484,8 @@ struct xdma_engine {
 	dma_addr_t cyclic_result_bus;	/* bus addr for transfer */
 	struct xdma_request_cb *cyclic_req;
 	struct sg_table cyclic_sgt;
+    u8 *perf_buf_virt;
+    dma_addr_t perf_buf_bus; /* bus address */
 	u8 eop_found; /* used only for cyclic(rx:c2h) */
 	int eop_count;
 	int rx_tail;	/* follows the HW */
@@ -474,7 +500,7 @@ struct xdma_engine {
 	dma_addr_t poll_mode_bus;	/* bus addr for descriptor writeback */
 
 	/* Members associated with interrupt mode support */
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head shutdown_wq;
 #else
 	wait_queue_head_t shutdown_wq;	/* wait queue for shutdown sync */
@@ -488,14 +514,23 @@ struct xdma_engine {
 	spinlock_t desc_lock;		/* protects concurrent access */
 	dma_addr_t desc_bus;
 	struct xdma_desc *desc;
+	int desc_idx;			/* current descriptor index */
+	int desc_used;			/* total descriptors used */
 
 	/* for performance test support */
 	struct xdma_performance_ioctl *xdma_perf;	/* perf test control */
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head xdma_perf_wq;
 #else
 	wait_queue_head_t xdma_perf_wq;	/* Perf test sync */
 #endif
+
+	struct xdma_kthread *cmplthp;
+	/* completion status thread list for the queue */
+	struct list_head cmplthp_list;
+	/* pending work thread list */
+	/* cpu attached to intr_work */
+	unsigned int intr_work_cpu;
 };
 
 struct xdma_user_irq {
@@ -525,7 +560,7 @@ struct xdma_dev {
 	unsigned int flags;
 
 	/* PCIe BAR management */
-	void *__iomem bar[XDMA_BAR_NUM];	/* addresses for mapped BARs */
+	void __iomem *bar[XDMA_BAR_NUM];	/* addresses for mapped BARs */
 	int user_bar_idx;	/* BAR index of user logic */
 	int config_bar_idx;	/* BAR index of XDMA config logic */
 	int bypass_bar_idx;	/* BAR index of XDMA bypass logic */
@@ -541,7 +576,7 @@ struct xdma_dev {
 	int irq_line;		/* flag if irq allocated successfully */
 	int msi_enabled;	/* flag if msi was enabled for the device */
 	int msix_enabled;	/* flag if msi-x was enabled for the device */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0)
+#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE
 	struct msix_entry entry[32];	/* msi-x vector/entry table */
 #endif
 	struct xdma_user_irq user_irq[16];	/* user IRQ management */
@@ -621,8 +656,8 @@ void get_perf_stats(struct xdma_engine *engine);
 
 int xdma_cyclic_transfer_setup(struct xdma_engine *engine);
 int xdma_cyclic_transfer_teardown(struct xdma_engine *engine);
-ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t,
-			 int);
+ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine,
+		char __user *buf, size_t count, int timeout_ms);
 int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg);
-
+int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count);
 #endif /* XDMA_LIB_H */

XDMA/linux-kernel/libxdma/version.h

@@ -20,9 +20,9 @@
 #ifndef __XDMA_VERSION_H__
 #define __XDMA_VERSION_H__
 
-#define DRV_MOD_MAJOR		2018
-#define DRV_MOD_MINOR		3
-#define DRV_MOD_PATCHLEVEL	41
+#define DRV_MOD_MAJOR		2019
+#define DRV_MOD_MINOR		2
+#define DRV_MOD_PATCHLEVEL	42
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/Makefile

@@ -18,7 +18,7 @@ EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS)
 #EXTRA_CFLAGS += -DINTERNAL_TESTING
 
 ifneq ($(KERNELRELEASE),)
-	$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o
+	$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o
 	obj-m := $(TARGET_MODULE).o
 else
 	BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build

XDMA/linux-kernel/xdma/cdev_bypass.c

@@ -20,7 +20,7 @@
 #include "libxdma_api.h"
 #include "xdma_cdev.h"
 
-#define write_register(v,mem,off) iowrite32(v, mem)
+#define write_register(v, mem, off) iowrite32(v, mem)
 
 static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
 		size_t *buf_offset, size_t buf_size)
@@ -29,8 +29,15 @@ static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
 	int copy_err;
 	int rc = 0;
 
-	BUG_ON(!buf);
-	BUG_ON(!buf_offset);
+	if (!buf) {
+		pr_err("Invalid user buffer\n");
+		return -EINVAL;
+	}
+
+	if (!buf_offset) {
+		pr_err("Invalid user buffer offset\n");
+		return -EINVAL;
+	}
 
 	/* Fill user buffer with descriptor data */
 	for (i = 0; i < transfer->desc_num; i++) {
@@ -71,7 +78,7 @@ static ssize_t char_bypass_read(struct file *file, char __user *buf,
 	xdev = xcdev->xdev;
 	engine = xcdev->engine;
 
-	dbg_sg("In char_bypass_read()\n");
+	dbg_sg("In %s()\n", __func__);
 
 	if (count & 3) {
 		dbg_sg("Buffer size must be a multiple of 4 bytes\n");
@@ -114,7 +121,7 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
 
 	u32 desc_data;
-	u32 *bypass_addr;
+	void __iomem *bypass_addr;
 	size_t buf_offset = 0;
 	int rc = 0;
 	int copy_err;
@@ -140,18 +147,21 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
 		return -ENODEV;
 	}
 
-	dbg_sg("In char_bypass_write()\n");
+	dbg_sg("In %s()\n", __func__);
 
 	spin_lock(&engine->lock);
 
 	/* Write descriptor data to the bypass BAR */
-	bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx];
-	bypass_addr += engine->bypass_offset;
+	bypass_addr = xdev->bar[xdev->bypass_bar_idx];
+	bypass_addr = (void __iomem *)(
+			(u32 __iomem *)bypass_addr + engine->bypass_offset
+			);
 	while (buf_offset < count) {
 		copy_err = copy_from_user(&desc_data, &buf[buf_offset],
 			sizeof(u32));
 		if (!copy_err) {
-			write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset);
+			write_register(desc_data, bypass_addr,
+					bypass_addr - engine->bypass_offset);
 			buf_offset += sizeof(u32);
 			rc = buf_offset;
 		} else {

XDMA/linux-kernel/xdma/cdev_ctrl.c

@@ -32,7 +32,7 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
 {
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data;
 	struct xdma_dev *xdev;
-	void *reg;
+	void __iomem *reg;
 	u32 w;
 	int rv;
 
@@ -48,8 +48,8 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
 	reg = xdev->bar[xcdev->bar] + *pos;
 	//w = read_register(reg);
 	w = ioread32(reg);
-	dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg,
-		(long)count, (int)*pos, w);
+	dbg_sg("%s(@%p, count=%ld, pos=%d) value = 0x%08x\n",
+			__func__, reg, (long)count, (int)*pos, w);
 	rv = copy_to_user(buf, &w, 4);
 	if (rv)
 		dbg_sg("Copy to userspace failed but continuing\n");
@@ -63,7 +63,7 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
 {
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
 	struct xdma_dev *xdev;
-	void *reg;
+	void __iomem *reg;
 	u32 w;
 	int rv;
 
@@ -79,12 +79,11 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
 	/* first address is BAR base plus file position offset */
 	reg = xdev->bar[xcdev->bar] + *pos;
 	rv = copy_from_user(&w, buf, 4);
-	if (rv) {
+	if (rv)
 		pr_info("copy from user failed %d/4, but continuing.\n", rv);
-	}
 
-	dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg,
-		(long)count, (int)*pos);
+	dbg_sg("%s(0x%08x @%p, count=%ld, pos=%d)\n",
+			__func__, w, reg, (long)count, (int)*pos);
 	//write_register(w, reg);
 	iowrite32(w, reg);
 	*pos += 4;
@@ -158,7 +157,7 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	switch (cmd) {
 	case XDMA_IOCINFO:
-		if (copy_from_user((void *)&ioctl_obj, (void *) arg,
+		if (copy_from_user((void *)&ioctl_obj, (void __user *) arg,
 			 sizeof(struct xdma_ioc_base))) {
 			pr_err("copy_from_user failed.\n");
 			return -EFAULT;

XDMA/linux-kernel/xdma/cdev_sgdma.c

@@ -19,16 +19,116 @@
 
 #define pr_fmt(fmt)     KBUILD_MODNAME ":%s: " fmt, __func__
 
+#include <linux/types.h>
 #include <asm/cacheflush.h>
+#include <linux/slab.h>
+#include <linux/aio.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/kthread.h>
+#include <linux/version.h>
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+#include <linux/uio.h>
+#endif
 #include "libxdma_api.h"
 #include "xdma_cdev.h"
 #include "cdev_sgdma.h"
+#include "xdma_thread.h"
 
 /* Module Parameters */
 unsigned int sgdma_timeout = 10;
 module_param(sgdma_timeout, uint, 0644);
 MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec.");
 
+
+extern struct kmem_cache *cdev_cache;
+static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write);
+
+
+static void async_io_handler(unsigned long  cb_hndl, int err)
+{
+	struct xdma_cdev *xcdev;
+	struct xdma_engine *engine;
+	struct xdma_dev *xdev;
+	struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl;
+	struct cdev_async_io *caio = (struct cdev_async_io *)cb->private;
+	ssize_t numbytes = 0;
+	ssize_t res, res2;
+	int lock_stat;
+	int rv;
+
+	if (NULL == caio) {
+		pr_err("Invalid work struct\n");
+		return;
+	}
+
+	xcdev = (struct xdma_cdev *)caio->iocb->ki_filp->private_data;
+
+	rv = xcdev_check(__func__, xcdev, 1);
+		if (rv < 0)
+			return;
+
+	/* Safeguarding for cancel requests */
+	lock_stat = spin_trylock(&caio->lock);
+	if (!lock_stat) {
+		pr_err("caio lock not acquired\n");
+		goto skip_dev_lock;
+	}
+
+	if (false != caio->cancel) {
+		pr_err("skipping aio\n");
+		goto skip_tran;
+	}
+
+
+	engine = xcdev->engine;
+	xdev = xcdev->xdev;
+
+	if (!err)
+		numbytes = xdma_xfer_completion((void *)cb, xdev, engine->channel, cb->write, cb->ep_addr, &cb->sgt,
+				0, sgdma_timeout * 1000);
+
+	char_sgdma_unmap_user_buf(cb, cb->write);
+
+	caio->res2 |= (err < 0) ? err : 0;
+	if (caio->res2)
+		caio->err_cnt++;
+
+	caio->cmpl_cnt++;
+	caio->res += numbytes;
+
+
+	if (caio->cmpl_cnt == caio->req_cnt)
+	{
+		res = caio->res;
+		res2 = caio->res2;
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
+		caio->iocb->ki_complete(caio->iocb, res, res2);
+#else
+		aio_complete(caio->iocb, res, res2);
+#endif
+skip_tran:
+		spin_unlock(&caio->lock);
+		kmem_cache_free(cdev_cache, caio);
+		kfree(cb);
+		return;
+	}
+	else
+	{
+		spin_unlock(&caio->lock);
+		return;
+
+	}
+skip_dev_lock:
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
+	caio->iocb->ki_complete(caio->iocb, numbytes, -EBUSY);
+#else
+	aio_complete(caio->iocb, numbytes, -EBUSY);
+#endif
+	kmem_cache_free(cdev_cache, caio);
+}
+
+
 /*
  * character device file operations for SG DMA engine
  */
@@ -52,10 +152,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence)
 	if (newpos < 0)
 		return -EINVAL;
 	file->f_pos = newpos;
-	dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos);
+	dbg_fops("%s: pos=%lld\n", __func__, (signed long long)newpos);
 
 #if 0
-	pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n",
+	pr_err("0x%p, off %lld, whence %d -> pos %lld.\n",
 		file, (signed long long)off, whence, (signed long long)off);
 #endif
 
@@ -80,7 +180,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence)
 static int check_transfer_align(struct xdma_engine *engine,
 	const char __user *buf, size_t count, loff_t pos, int sync)
 {
-	BUG_ON(!engine);
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
 
 	/* AXI ST or AXI MM non-incremental addressing mode? */
 	if (engine->non_incr_addr) {
@@ -171,17 +274,16 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
 {
 	struct sg_table *sgt = &cb->sgt;
 	unsigned long len = cb->len;
-	char *buf = cb->buf;
+	void __user *buf = cb->buf;
 	struct scatterlist *sg;
-	unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) -
+	unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE - 1) -
 				 ((unsigned long)buf & PAGE_MASK))
 				>> PAGE_SHIFT;
 	int i;
 	int rv;
 
-	if (pages_nr == 0) {
+	if (pages_nr == 0)
 		return -EINVAL;
-	}
 
 	if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) {
 		pr_err("sgl OOM.\n");
@@ -225,7 +327,8 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
 	sg = sgt->sgl;
 	for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) {
 		unsigned int offset = offset_in_page(buf);
-		unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
+		unsigned int nbytes =
+			min_t(unsigned int, PAGE_SIZE - offset, len);
 
 		flush_dcache_page(cb->pages[i]);
 		sg_set_page(sg, cb->pages[i], nbytes, offset);
@@ -234,7 +337,10 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
 		len -= nbytes;
 	}
 
-	BUG_ON(len);
+	if (len) {
+		pr_err("Invalid user buffer length. Cannot map to sgl\n");
+		return -EINVAL;
+	}
 	cb->pages_nr = pages_nr;
 	return 0;
 
@@ -244,7 +350,7 @@ err_out:
 	return rv;
 }
 
-static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
+static ssize_t char_sgdma_read_write(struct file *file, const char __user *buf,
 		size_t count, loff_t *pos, bool write)
 {
 	int rv;
@@ -278,8 +384,10 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
 	}
 
 	memset(&cb, 0, sizeof(struct xdma_io_cb));
-	cb.buf = buf;
+	cb.buf = (char __user *)buf;
 	cb.len = count;
+	cb.ep_addr = (u64)*pos;
+	cb.write = write;
 	rv = char_sgdma_map_user_buf_to_sgl(&cb, write);
 	if (rv < 0)
 		return rv;
@@ -296,41 +404,183 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
 static ssize_t char_sgdma_write(struct file *file, const char __user *buf,
 		size_t count, loff_t *pos)
 {
-        return char_sgdma_read_write(file, (char *)buf, count, pos, 1);
+	return char_sgdma_read_write(file, buf, count, pos, 1);
 }
 
 static ssize_t char_sgdma_read(struct file *file, char __user *buf,
 		size_t count, loff_t *pos)
 {
-	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
+	return char_sgdma_read_write(file, buf, count, pos, 0);
+}
+
+static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io,
+                              unsigned long count, loff_t pos)
+{
+	struct xdma_cdev *xcdev = (struct xdma_cdev *)iocb->ki_filp->private_data;
+	struct cdev_async_io *caio;
 	struct xdma_engine *engine;
+	struct xdma_dev *xdev;
 	int rv;
+	unsigned long i;
 
-	rv = xcdev_check(__func__, xcdev, 1);
-	if (rv < 0)
+	if (!xcdev) {
+		pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
+		        iocb->ki_filp, (u64)count, (u64)pos, 1);
+		return -EINVAL;
+	}
+
+ 	engine = xcdev->engine;
+	xdev = xcdev->xdev;
+
+	if (engine->dir != DMA_TO_DEVICE) {
+		pr_err("r/w mismatch. WRITE, dir %d.\n",
+			engine->dir);
+		return -EINVAL;
+	}
+
+	caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
+	memset(caio, 0, sizeof(struct cdev_async_io));
+
+	caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL);
+
+	spin_lock_init(&caio->lock);
+	iocb->private = caio;
+	caio->iocb = iocb;
+	caio->write = true;
+	caio->cancel = false;
+	caio->req_cnt = count;
+
+	for (i = 0; i < count; i++) {
+
+		memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb));
+
+		caio->cb[i].buf = io[i].iov_base;
+		caio->cb[i].len = io[i].iov_len;
+		caio->cb[i].ep_addr = (u64)pos;
+		caio->cb[i].write = true;
+		caio->cb[i].private = caio;
+		caio->cb[i].io_done = async_io_handler;
+		rv = check_transfer_align(engine, caio->cb[i].buf, caio->cb[i].len, pos, 1);
+		if (rv) {
+			pr_info("Invalid transfer alignment detected\n");
+			kmem_cache_free(cdev_cache, caio);
+ 			return rv;
+		}
+
+		rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true);
+		if (rv < 0) {
 			return rv;
+		}
+
+		rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, engine->channel, caio->cb[i].write, caio->cb[i].ep_addr, &caio->cb[i].sgt,
+									0, sgdma_timeout * 1000);
+ 	}
+
+	if (engine->cmplthp)
+		xdma_kthread_wakeup(engine->cmplthp);
+
+	return -EIOCBQUEUED;
+}
+
+
+static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io,
+                             unsigned long count, loff_t pos)
+{
+
+	struct xdma_cdev *xcdev = (struct xdma_cdev *)iocb->ki_filp->private_data;
+	struct cdev_async_io *caio;
+	struct xdma_engine *engine;
+	struct xdma_dev *xdev;
+	int rv;
+	unsigned long i;
+
+	if (!xcdev) {
+		pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
+		        iocb->ki_filp, (u64)count, (u64)pos, 1);
+		return -EINVAL;
+	}
 
 	engine = xcdev->engine;
+	xdev = xcdev->xdev;
 
-	if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
-		rv = xdma_cyclic_transfer_setup(engine);
-		if (rv < 0 && rv != -EBUSY)
+	if (engine->dir != DMA_FROM_DEVICE) {
+		pr_err("r/w mismatch. READ, dir %d.\n",
+			engine->dir);
+		return -EINVAL;
+	}
+
+	caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
+	memset(caio, 0, sizeof(struct cdev_async_io));
+
+	caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL);
+
+	spin_lock_init(&caio->lock);
+	iocb->private = caio;
+	caio->iocb = iocb;
+	caio->write = false;
+	caio->cancel = false;
+	caio->req_cnt = count;
+
+	for (i = 0; i < count; i++) {
+
+		memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb));
+
+		caio->cb[i].buf = io[i].iov_base;
+		caio->cb[i].len = io[i].iov_len;
+		caio->cb[i].ep_addr = (u64)pos;
+		caio->cb[i].write = false;
+		caio->cb[i].private = caio;
+		caio->cb[i].io_done = async_io_handler;
+
+		rv = check_transfer_align(engine, caio->cb[i].buf, caio->cb[i].len, pos, 1);
+		if (rv) {
+			pr_info("Invalid transfer alignment detected\n");
+			kmem_cache_free(cdev_cache, caio);
 			return rv;
-		/* 600 sec. timeout */
-		return xdma_engine_read_cyclic(engine, buf, count, 600000);
 		}
 
-        return char_sgdma_read_write(file, (char *)buf, count, pos, 0);
+		rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true);
+		if (rv < 0) {
+			return rv;
+		}
+
+		rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, engine->channel, caio->cb[i].write, caio->cb[i].ep_addr, &caio->cb[i].sgt,
+											0, sgdma_timeout * 1000);
+	}
+
+	if (engine->cmplthp)
+		xdma_kthread_wakeup(engine->cmplthp);
+
+	return -EIOCBQUEUED;
 }
 
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+static ssize_t cdev_write_iter(struct kiocb *iocb, struct iov_iter *io)
+{
+	return cdev_aio_write(iocb, io->iov, io->nr_segs, io->iov_offset);
+}
+
+static ssize_t cdev_read_iter(struct kiocb *iocb, struct iov_iter *io)
+{
+	return cdev_aio_read(iocb, io->iov, io->nr_segs, io->iov_offset);
+}
+#endif
+
 static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
 {
 	int rv;
 	struct xdma_dev *xdev;
 
-        BUG_ON(!engine);
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
+
 	xdev = engine->xdev;
-        BUG_ON(!xdev);
+	if (!xdev) {
+		pr_err("Invalid xdev\n");
+		return -EINVAL;
+	}
 
 	/* performance measurement already running on this engine? */
 	if (engine->xdma_perf) {
@@ -345,7 +595,7 @@ static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
 		return -ENOMEM;
 
 	rv = copy_from_user(engine->xdma_perf,
-                (struct xdma_performance_ioctl *)arg,
+		(struct xdma_performance_ioctl __user *)arg,
 		sizeof(struct xdma_performance_ioctl));
 
 	if (rv < 0) {
@@ -361,14 +611,15 @@ static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
 	enable_perf(engine);
 	dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size);
 	/* initialize wait queue */
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	init_swait_queue_head(&engine->xdma_perf_wq);
 #else
 	init_waitqueue_head(&engine->xdma_perf_wq);
 #endif
-        xdma_performance_submit(xdev, engine);
-
-        return 0;
+	rv = xdma_performance_submit(xdev, engine);
+	if (rv < 0)
+		pr_err("Failed to submit dma performance\n");
+	return rv;
 }
 
 static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
@@ -376,6 +627,11 @@ static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
 	struct xdma_transfer *transfer = NULL;
 	int rv;
 
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
+
 	dbg_perf("IOCTL_XDMA_PERF_STOP\n");
 
 	/* no performance measurement running on this engine? */
@@ -386,23 +642,22 @@ static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
 
 	/* stop measurement */
 	transfer = engine_cyclic_stop(engine);
+	if (!transfer) {
+		pr_err("Failed to stop cyclic transfer\n");
+		return -EINVAL;
+	}
 	dbg_perf("Waiting for measurement to stop\n");
 
-        if (engine->xdma_perf) {
 	get_perf_stats(engine);
 
 	rv = copy_to_user((void __user *)arg, engine->xdma_perf,
 			sizeof(struct xdma_performance_ioctl));
 	if (rv) {
 		dbg_perf("Error copying result to user\n");
-                        return -EINVAL;
+		return rv;
 	}
 
-		if (transfer)
 	kfree(transfer);
-        } else {
-                dbg_perf("engine->xdma_perf == NULL?\n");
-        }
 
 	kfree(engine->xdma_perf);
 	engine->xdma_perf = NULL;
@@ -414,7 +669,10 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg)
 {
 	int rc;
 
-        BUG_ON(!engine);
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
 
 	dbg_perf("IOCTL_XDMA_PERF_GET\n");
 
@@ -425,7 +683,7 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg)
 			sizeof(struct xdma_performance_ioctl));
 		if (rc) {
 			dbg_perf("Error copying result to user\n");
-                        return -EINVAL;
+			return rc;
 		}
 	} else {
 		dbg_perf("engine->xdma_perf == NULL?\n");
@@ -445,7 +703,10 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg)
 	int rv;
 	unsigned long src;
 
-	BUG_ON(!engine);
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
 	src = !!engine->non_incr_addr;
 
 	dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n");
@@ -456,7 +717,10 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg)
 
 static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg)
 {
-	BUG_ON(!engine);
+	if (!engine) {
+		pr_err("Invalid DMA engine\n");
+		return -EINVAL;
+	}
 
 	dbg_perf("IOCTL_XDMA_ALIGN_GET\n");
 	return put_user(engine->addr_align, (int __user *)arg);
@@ -519,7 +783,6 @@ static int char_sgdma_open(struct inode *inode, struct file *file)
 	if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
 		if (engine->device_open == 1)
 			return -EBUSY;
-		else
 		engine->device_open = 1;
 	}
 
@@ -551,7 +814,17 @@ static const struct file_operations sgdma_fops = {
 	.open = char_sgdma_open,
 	.release = char_sgdma_close,
 	.write = char_sgdma_write,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+	.write_iter = cdev_write_iter,
+#else
+	.aio_write = cdev_aio_write,
+#endif
 	.read = char_sgdma_read,
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
+	.read_iter = cdev_read_iter,
+#else
+	.aio_read = cdev_aio_read,
+#endif
 	.unlocked_ioctl = char_sgdma_ioctl,
 	.llseek = char_sgdma_llseek,
 };

XDMA/linux-kernel/xdma/cdev_sgdma.h

@@ -46,8 +46,7 @@
  * _IOC_SIZE(nr)	    returns size
  */
 
-struct xdma_performance_ioctl
-{
+struct xdma_performance_ioctl {
 	/* IOCTL_XDMA_IOCTL_Vx */
 	uint32_t version;
 	uint32_t transfer_size;

XDMA/linux-kernel/xdma/cdev_xvc.c

@@ -32,30 +32,30 @@
 
 #ifdef __REG_DEBUG__
 /* SECTION: Function definitions */
-inline void __write_register(const char *fn, u32 value, void *base,
+inline void __write_register(const char *fn, u32 value, void __iomem *base,
 				unsigned int off)
 {
 	pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
 	iowrite32(value, base + off);
 }
 
-inline u32 __read_register(const char *fn, void *base, unsigned int off)
+inline u32 __read_register(const char *fn, void __iomem *base, unsigned int off)
 {
 	u32 v = ioread32(base + off);
 
 	pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
 	return v;
 }
-#define write_register(v,base,off) __write_register(__func__, v, base, off)
-#define read_register(base,off) __read_register(__func__, base, off)
+#define write_register(v, base, off) __write_register(__func__, v, base, off)
+#define read_register(base, off) __read_register(__func__, base, off)
 
 #else
-#define write_register(v,base,off)	iowrite32(v, (base) + (off))
-#define read_register(base,off)		ioread32((base) + (off))
+#define write_register(v, base, off)	iowrite32(v, (base) + (off))
+#define read_register(base, off)		ioread32((base) + (off))
 #endif /* #ifdef __REG_DEBUG__ */
 
 
-static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
+static int xvc_shift_bits(void __iomem *base, u32 tms_bits, u32 tdi_bits,
 			u32 *tdo_bits)
 {
 	u32 control;
@@ -109,6 +109,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
 		return rv;
+
 	xdev = xcdev->xdev;
 
 	if (cmd != XDMA_IOCXVC) {
@@ -135,7 +136,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	total_bits = xvc_obj.length;
 	total_bytes = (total_bits + 7) >> 3;
 
-	buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL);
+	buffer = (unsigned char *)kmalloc(total_bytes * 3, GFP_KERNEL);
 	if (!buffer) {
 		pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n",
 			3 * total_bytes, opcode, total_bits, total_bytes);
@@ -146,12 +147,16 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	tdi_buf = tms_buf + total_bytes;
 	tdo_buf = tdi_buf + total_bytes;
 
-	rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes);
+	rv = copy_from_user((void *)tms_buf,
+			(const char __user *)xvc_obj.tms_buf,
+			total_bytes);
 	if (rv) {
 		pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes);
 		goto cleanup;
 	}
-	rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes);
+	rv = copy_from_user((void *)tdi_buf,
+			(const char __user *)xvc_obj.tdi_buf,
+			total_bytes);
 	if (rv) {
 		pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes);
 		goto cleanup;
@@ -162,7 +167,8 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	iobase = xdev->bar[xcdev->bar] + xcdev->base;
 	/* set length register to 32 initially if more than one
-	 * word-transaction is to be done */
+	 * word-transaction is to be done
+	 */
 	if (total_bits >= 32)
 		write_register(0x20, iobase, XVC_BAR_LENGTH_REG);
 
@@ -186,33 +192,33 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		/* Shift data out and copy to output buffer */
 		rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store);
 		if (rv < 0)
-			goto cleanup;
+			break;
 
 		memcpy(tdo_buf + bytes, &tdo_store, shift_bytes);
 	}
 
+	if (rv < 0)
+		goto unlock;
+
 	/* if testing bar access swap tdi and tdo bufferes to "loopback" */
 	if (opcode == 0x2) {
-		char *tmp = tdo_buf;
+		unsigned char *tmp = tdo_buf;
 
 		tdo_buf = tdi_buf;
 		tdi_buf = tmp;
 	}
 
-	rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes);
-	if (rv) {
+	rv = copy_to_user(xvc_obj.tdo_buf, (const void *)tdo_buf, total_bytes);
+	if (rv)
 		pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes);
-		rv = -EFAULT;
-		goto cleanup;
-	}
-
-cleanup:
-	if (buffer)
-		kfree(buffer);
 
+unlock:
 	mmiowb();
 	spin_unlock(&xcdev->lock);
 
+cleanup:
+	kfree(buffer);
+
 	return rv;
 }
 

XDMA/linux-kernel/xdma/cdev_xvc.h

@@ -33,9 +33,9 @@
 struct xvc_ioc {
 	unsigned int opcode;
 	unsigned int length;
-	unsigned char *tms_buf;
-	unsigned char *tdi_buf;
-	unsigned char *tdo_buf;
+	const char __user *tms_buf;
+	const char __user *tdi_buf;
+	void __user *tdo_buf;
 };
 
 #define XDMA_IOCXVC	_IOWR(XVC_MAGIC, 1, struct xvc_ioc)

XDMA/linux-kernel/xdma/version.h

@@ -20,9 +20,9 @@
 #ifndef __XDMA_VERSION_H__
 #define __XDMA_VERSION_H__
 
-#define DRV_MOD_MAJOR		2018
-#define DRV_MOD_MINOR		3
-#define DRV_MOD_PATCHLEVEL	50
+#define DRV_MOD_MAJOR		2019
+#define DRV_MOD_MINOR		2
+#define DRV_MOD_PATCHLEVEL	51
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/xdma_cdev.c

@@ -21,7 +21,9 @@
 
 #include "xdma_cdev.h"
 
-struct class *g_xdma_class;
+static struct class *g_xdma_class;
+
+struct kmem_cache *cdev_cache;
 
 enum cdev_type {
 	CHAR_USER,
@@ -75,16 +77,18 @@ static inline int xpdev_flag_test(struct xdma_pci_dev *xpdev,
 }
 
 #ifdef __XDMA_SYSFS__
-ssize_t show_device_numbers(struct device *dev, struct device_attribute *attr,
+ssize_t xdma_dev_instance_show(struct device *dev,
+		struct device_attribute *attr,
 		char *buf)
 {
-	struct xdma_pci_dev *xpdev = (struct xdma_pci_dev *)dev_get_drvdata(dev);
+	struct xdma_pci_dev *xpdev =
+		(struct xdma_pci_dev *)dev_get_drvdata(dev);
 
 	return snprintf(buf, PAGE_SIZE, "%d\t%d\n",
 			xpdev->major, xpdev->xdev->idx);
 }
 
-static DEVICE_ATTR(xdma_dev_instance, S_IRUGO, show_device_numbers, NULL);
+static DEVICE_ATTR_RO(xdma_dev_instance);
 #endif
 
 static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type)
@@ -98,7 +102,10 @@ static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type)
 	case CHAR_XDMA_C2H:
 	case CHAR_BYPASS_H2C:
 	case CHAR_BYPASS_C2H:
-		BUG_ON(!engine);
+		if (!engine) {
+			pr_err("Invalid DMA engine\n");
+			return rv;
+		}
 		rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type],
 			xdev->idx, engine->channel);
 		break;
@@ -142,6 +149,7 @@ int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine)
 
 	if (check_engine) {
 		struct xdma_engine *engine = xcdev->engine;
+
 		if (!engine || engine->magic != MAGIC_ENGINE) {
 			pr_info("%s, engine 0x%p, magic 0x%lx.\n", fname,
 				engine, engine ? engine->magic : 0xFFFFFFFF);
@@ -158,7 +166,11 @@ int char_open(struct inode *inode, struct file *file)
 
 	/* pointer to containing structure of the character device inode */
 	xcdev = container_of(inode->i_cdev, struct xdma_cdev, cdev);
-	BUG_ON(xcdev->magic != MAGIC_CHAR);
+	if (xcdev->magic != MAGIC_CHAR) {
+		pr_err("xcdev 0x%p inode 0x%lx magic mismatch 0x%lx\n",
+			xcdev, inode->i_ino, xcdev->magic);
+		return -EINVAL;
+	}
 	/* create a reference to our char device in the opened file */
 	file->private_data = xcdev;
 
@@ -173,13 +185,30 @@ int char_close(struct inode *inode, struct file *file)
 	struct xdma_dev *xdev;
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
 
-	BUG_ON(!xcdev);
-	BUG_ON(xcdev->magic != MAGIC_CHAR);
+	if (!xcdev) {
+		pr_err("char device with inode 0x%lx xcdev NULL\n",
+			inode->i_ino);
+		return -EINVAL;
+	}
+
+	if (xcdev->magic != MAGIC_CHAR) {
+		pr_err("xcdev 0x%p magic mismatch 0x%lx\n",
+				xcdev, xcdev->magic);
+		return -EINVAL;
+	}
 
 	/* fetch device specific data stored earlier during open */
 	xdev = xcdev->xdev;
-	BUG_ON(!xdev);
-	BUG_ON(xdev->magic != MAGIC_DEVICE);
+	if (!xdev) {
+		pr_err("char device with inode 0x%lx xdev NULL\n",
+			inode->i_ino);
+		return -EINVAL;
+	}
+
+	if (xdev->magic != MAGIC_DEVICE) {
+		pr_err("xdev 0x%p magic mismatch 0x%lx\n", xdev, xdev->magic);
+		return -EINVAL;
+	}
 
 	return 0;
 }
@@ -218,15 +247,27 @@ static int destroy_xcdev(struct xdma_cdev *cdev)
 {
 	if (!cdev) {
 		pr_warn("cdev NULL.\n");
-		return 0;
+		return -EINVAL;
 	}
 	if (cdev->magic != MAGIC_CHAR) {
 		pr_warn("cdev 0x%p magic mismatch 0x%lx\n", cdev, cdev->magic);
-		return 0;
+		return -EINVAL;
+	}
+
+	if (!cdev->xdev) {
+		pr_err("xdev NULL\n");
+		return -EINVAL;
+	}
+
+	if (!g_xdma_class) {
+		pr_err("g_xdma_class NULL\n");
+		return -EINVAL;
+	}
+
+	if (!cdev->sys_device) {
+		pr_err("cdev sys_device NULL\n");
+		return -EINVAL;
 	}
-	BUG_ON(!cdev->xdev);
-	BUG_ON(!g_xdma_class);
-	BUG_ON(!cdev->sys_device);
 
 	if (cdev->sys_device)
 		device_destroy(g_xdma_class, cdev->cdevno);
@@ -343,52 +384,85 @@ unregister_region:
 
 void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev)
 {
-	int i;
-
+	int i = 0;
+	int rv;
 #ifdef __XDMA_SYSFS__
 	device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance);
 #endif
 
 	if (xpdev_flag_test(xpdev, XDF_CDEV_SG)) {
 		/* iterate over channels */
-		for (i = 0; i < xpdev->h2c_channel_max; i++)
+		for (i = 0; i < xpdev->h2c_channel_max; i++) {
 			/* remove SG DMA character device */
-			destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]);
-		for (i = 0; i < xpdev->c2h_channel_max; i++)
-			destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]);
+			rv = destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]);
+			if (rv < 0)
+				pr_err("Failed to destroy h2c xcdev %d error :0x%x\n",
+						i, rv);
+		}
+		for (i = 0; i < xpdev->c2h_channel_max; i++) {
+			rv = destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]);
+			if (rv < 0)
+				pr_err("Failed to destroy c2h xcdev %d error 0x%x\n",
+						i, rv);
+		}
 	}
 
 	if (xpdev_flag_test(xpdev, XDF_CDEV_EVENT)) {
-		for (i = 0; i < xpdev->user_max; i++)
-			destroy_xcdev(&xpdev->events_cdev[i]);
+		for (i = 0; i < xpdev->user_max; i++) {
+			rv = destroy_xcdev(&xpdev->events_cdev[i]);
+			if (rv < 0)
+				pr_err("Failed to destroy cdev event %d error 0x%x\n",
+					i, rv);
+		}
 	}
 
 	/* remove control character device */
 	if (xpdev_flag_test(xpdev, XDF_CDEV_CTRL)) {
-		destroy_xcdev(&xpdev->ctrl_cdev);
+		rv = destroy_xcdev(&xpdev->ctrl_cdev);
+		if (rv < 0)
+			pr_err("Failed to destroy cdev ctrl event %d error 0x%x\n",
+				i, rv);
 	}
 
 	/* remove user character device */
 	if (xpdev_flag_test(xpdev, XDF_CDEV_USER)) {
-		destroy_xcdev(&xpdev->user_cdev);
+		rv = destroy_xcdev(&xpdev->user_cdev);
+		if (rv < 0)
+			pr_err("Failed to destroy user cdev %d error 0x%x\n",
+				i, rv);
 	}
 
 	if (xpdev_flag_test(xpdev, XDF_CDEV_XVC)) {
-		destroy_xcdev(&xpdev->xvc_cdev);
+		rv = destroy_xcdev(&xpdev->xvc_cdev);
+		if (rv < 0)
+			pr_err("Failed to destroy xvc cdev %d error 0x%x\n",
+				i, rv);
 	}
 
 	if (xpdev_flag_test(xpdev, XDF_CDEV_BYPASS)) {
 		/* iterate over channels */
-		for (i = 0; i < xpdev->h2c_channel_max; i++)
+		for (i = 0; i < xpdev->h2c_channel_max; i++) {
 			/* remove DMA Bypass character device */
-			destroy_xcdev(&xpdev->bypass_h2c_cdev[i]);
-		for (i = 0; i < xpdev->c2h_channel_max; i++)
-			destroy_xcdev(&xpdev->bypass_c2h_cdev[i]);
-		destroy_xcdev(&xpdev->bypass_cdev_base);
+			rv = destroy_xcdev(&xpdev->bypass_h2c_cdev[i]);
+			if (rv < 0)
+				pr_err("Failed to destroy bypass h2c cdev %d error 0x%x\n",
+					i, rv);
+		}
+		for (i = 0; i < xpdev->c2h_channel_max; i++) {
+			rv = destroy_xcdev(&xpdev->bypass_c2h_cdev[i]);
+			if (rv < 0)
+				pr_err("Failed to destroy bypass c2h %d error 0x%x\n",
+					i, rv);
+		}
+		rv = destroy_xcdev(&xpdev->bypass_cdev_base);
+		if (rv < 0)
+			pr_err("Failed to destroy base cdev\n");
 	}
 
 	if (xpdev->major)
-		unregister_chrdev_region(MKDEV(xpdev->major, XDMA_MINOR_BASE), XDMA_MINOR_COUNT);
+		unregister_chrdev_region(
+				MKDEV(xpdev->major, XDMA_MINOR_BASE),
+				XDMA_MINOR_COUNT);
 }
 
 int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
@@ -450,7 +524,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
 
 	/* ??? Bypass */
 	/* Initialize Bypass Character Device */
-	if (xdev->bypass_bar_idx > 0){
+	if (xdev->bypass_bar_idx > 0) {
 		for (i = 0; i < xpdev->h2c_channel_max; i++) {
 			engine = &xdev->engine_h2c[i];
 
@@ -515,7 +589,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
 	rv = device_create_file(&xpdev->pdev->dev,
 				&dev_attr_xdma_dev_instance);
 	if (rv) {
-		pr_err("Failed to create device file \n");
+		pr_err("Failed to create device file\n");
 		goto fail;
 	}
 #endif
@@ -536,11 +610,29 @@ int xdma_cdev_init(void)
 		return -1;
 	}
 
+    /* using kmem_cache_create to enable sequential cleanup */
+    cdev_cache = kmem_cache_create("cdev_cache",
+                                   sizeof(struct cdev_async_io),
+                                   0,
+                                   SLAB_HWCACHE_ALIGN,
+                                   NULL);
+    if (!cdev_cache) {
+    	pr_info("memory allocation for cdev_cache failed. OOM\n");
+    	return -ENOMEM;
+    }
+
+   	xdma_threads_create(8);
+
 	return 0;
 }
 
 void xdma_cdev_cleanup(void)
 {
+	if (cdev_cache)
+		kmem_cache_destroy(cdev_cache);
+
 	if (g_xdma_class)
 		class_destroy(g_xdma_class);
+
+	xdma_threads_destroy();
 }

XDMA/linux-kernel/xdma/xdma_cdev.h

@@ -35,13 +35,13 @@ int xdma_cdev_init(void);
 
 int char_open(struct inode *inode, struct file *file);
 int char_close(struct inode *inode, struct file *file);
-int xcdev_check(const char *, struct xdma_cdev *, bool);
-
+int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine);
 void cdev_ctrl_init(struct xdma_cdev *xcdev);
 void cdev_xvc_init(struct xdma_cdev *xcdev);
 void cdev_event_init(struct xdma_cdev *xcdev);
 void cdev_sgdma_init(struct xdma_cdev *xcdev);
 void cdev_bypass_init(struct xdma_cdev *xcdev);
+long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 
 void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev);
 int xpdev_create_interfaces(struct xdma_pci_dev *xpdev);

XDMA/linux-kernel/xdma/xdma_mod.c

@@ -40,10 +40,10 @@ static char version[] =
 MODULE_AUTHOR("Xilinx, Inc.");
 MODULE_DESCRIPTION(DRV_MODULE_DESC);
 MODULE_VERSION(DRV_MODULE_VERSION);
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual BSD/GPL");
 
 /* SECTION: Module global variables */
-static int xpdev_cnt = 0;
+static int xpdev_cnt;
 
 static const struct pci_device_id pci_ids[] = {
 	{ PCI_DEVICE(0x10ee, 0x903f), },
@@ -154,14 +154,28 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max,
 			&xpdev->h2c_channel_max, &xpdev->c2h_channel_max);
-	if (!hndl){
+	if (!hndl) {
 		rv = -EINVAL;
 		goto err_out;
 	}
 
-	BUG_ON(xpdev->user_max > MAX_USER_IRQ);
-	BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX);
-	BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX);
+	if (xpdev->user_max > MAX_USER_IRQ) {
+		pr_err("Maximum users limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
+
+	if (xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) {
+		pr_err("Maximun H2C channel limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
+
+	if (xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) {
+		pr_err("Maximun C2H channel limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
 
 	if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max)
 		pr_warn("NO engine found!\n");
@@ -181,7 +195,12 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		rv =  -EINVAL;
 		goto err_out;
 	}
-	BUG_ON(hndl != xdev );
+
+	if (hndl != xdev) {
+		pr_err("xdev handle mismatch\n");
+		rv =  -EINVAL;
+		goto err_out;
+	}
 
 	pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n",
 		dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev,
@@ -270,7 +289,7 @@ static void xdma_error_resume(struct pci_dev *pdev)
 	pci_cleanup_aer_uncorrect_error_status(pdev);
 }
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
 static void xdma_reset_prepare(struct pci_dev *pdev)
 {
 	struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
@@ -287,7 +306,7 @@ static void xdma_reset_done(struct pci_dev *pdev)
 	xdma_device_online(pdev, xpdev->xdev);
 }
 
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
 static void xdma_reset_notify(struct pci_dev *pdev, bool prepare)
 {
 	struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
@@ -305,10 +324,10 @@ static const struct pci_error_handlers xdma_err_handler = {
 	.error_detected	= xdma_error_detected,
 	.slot_reset	= xdma_slot_reset,
 	.resume		= xdma_error_resume,
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
 	.reset_prepare	= xdma_reset_prepare,
 	.reset_done	= xdma_reset_done,
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
 	.reset_notify	= xdma_reset_notify,
 #endif
 };
@@ -324,8 +343,6 @@ static struct pci_driver pci_driver = {
 static int __init xdma_mod_init(void)
 {
 	int rv;
-	extern unsigned int desc_blen_max;
-	extern unsigned int sgdma_timeout;
 
 	pr_info("%s", version);
 

XDMA/linux-kernel/xdma/xdma_mod.h

@@ -44,14 +44,19 @@
 #include <linux/splice.h>
 #include <linux/version.h>
 #include <linux/uio.h>
+#include <linux/spinlock_types.h>
 
 #include "libxdma.h"
+#include "xdma_thread.h"
 
 #define MAGIC_ENGINE	0xEEEEEEEEUL
 #define MAGIC_DEVICE	0xDDDDDDDDUL
 #define MAGIC_CHAR	0xCCCCCCCCUL
 #define MAGIC_BITSTREAM 0xBBBBBBBBUL
 
+extern unsigned int desc_blen_max;
+extern unsigned int sgdma_timeout;
+
 struct xdma_cdev {
 	unsigned long magic;		/* structure ID for sanity checks */
 	struct xdma_pci_dev *xpdev;
@@ -94,12 +99,19 @@ struct xdma_pci_dev {
 	void *data;
 };
 
-struct xdma_io_cb {
-	void __user *buf;
-	size_t len;
-	unsigned int pages_nr;
-	struct sg_table sgt;
-	struct page **pages;
+struct cdev_async_io {
+	struct kiocb *iocb;
+	struct xdma_io_cb* cb;
+	bool write;
+	bool cancel;
+	int cmpl_cnt;
+	int req_cnt;
+	spinlock_t lock;
+	struct work_struct wrk_itm;
+	struct cdev_async_io *next;
+	ssize_t res;
+	ssize_t res2;
+	int err_cnt;
 };
 
 #endif /* ifndef __XDMA_MODULE_H__ */