Merge branch 'master' of https://github.com/Xilinx/dma_ip_drivers

README.md

@@ -8,9 +8,7 @@ Both the linux kernel driver and the DPDK driver can be run on a PCI Express roo
 
 ### Getting Started
 
-* [QDMA DPDK Reference Driver User Guide](http://htmlpreview.github.io/?https://github.com/Xilinx/dma_ip_drivers/blob/master/QDMA/DPDK/docs/git_doc/html/index.html)
-
-* [QDMA Linux Kernel Reference Driver User Guide](http://htmlpreview.github.io/?https://github.com/Xilinx/dma_ip_drivers/blob/master/QDMA/linux-kernel/docs/git_doc/html/index.html)
+* [QDMA Reference Drivers Comprehensive documentation](https://xilinx.github.io/dma_ip_drivers/)
 
 ## Xilinx-VSEC (XVSEC)
 

XDMA/linux-kernel/RELEASE

+Release: 2019.2
+===============
+
+Change list:
+- Updated data rate for performance run. Now it will print data rate based on the size.
+- remove BUG_ON, return proper error code instead
+- Streaming mode: enable credit mechanism by default 
+- Streaming mode: Do not read more than user supplied buffer size on C2H
+- Streaming mode: Added support for Async-IO for both streaming and MM transfers
+- fixed performance appliaction crash

XDMA/linux-kernel/include/libxdma_api.h

@@ -132,6 +132,14 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask);
  */
 ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
+
+ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
+			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
+
+
+ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
+			struct sg_table *sgt, bool dma_mapped, int timeout_ms);
+
 			
 
 /////////////////////missing API////////////////////

XDMA/linux-kernel/libxdma/Makefile

@@ -13,7 +13,7 @@ ifneq ($(KERNELRELEASE),)
 else
 	BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build
 	PWD:=$(shell pwd)
-all : 
+all :
 	$(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules
 
 clean:

XDMA/linux-kernel/libxdma/libxdma.h

@@ -31,11 +31,11 @@
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/workqueue.h>
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 #include <linux/swait.h>
 #endif
 /*
- *  if the config bar is fixed, the driver does not neeed to search through 
+ *  if the config bar is fixed, the driver does not neeed to search through
  *  all of the bars
  */
 //#define XDMA_CONFIG_BAR_NUM	1
@@ -69,7 +69,7 @@
 #define XDMA_TRANSFER_MAX_DESC (2048)
 
 /* maximum size of a single DMA transfer descriptor */
-#define XDMA_DESC_BLEN_BITS 	28
+#define XDMA_DESC_BLEN_BITS	28
 #define XDMA_DESC_BLEN_MAX	((1 << (XDMA_DESC_BLEN_BITS)) - 1)
 
 /* bits of the SG DMA control register */
@@ -83,6 +83,7 @@
 #define XDMA_CTRL_IE_DESC_ERROR			(0x1FUL << 19)
 #define XDMA_CTRL_NON_INCR_ADDR			(1UL << 25)
 #define XDMA_CTRL_POLL_MODE_WB			(1UL << 26)
+#define XDMA_CTRL_STM_MODE_WB			(1UL << 27)
 
 /* bits of the SG DMA status register */
 #define XDMA_STAT_BUSY			(1UL << 0)
@@ -138,7 +139,7 @@
 /* all combined */
 #define XDMA_STAT_H2C_ERR_MASK	\
 	(XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
-	 XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK) 
+	 XDMA_STAT_H2C_R_ERR_MASK | XDMA_STAT_H2C_W_ERR_MASK)
 
 #define XDMA_STAT_C2H_ERR_MASK	\
 	(XDMA_STAT_COMMON_ERR_MASK | XDMA_STAT_DESC_ERR_MASK | \
@@ -161,7 +162,7 @@
 #define XDMA_ID_C2H 0x1fc1U
 
 /* for C2H AXI-ST mode */
-#define CYCLIC_RX_PAGES_MAX	256	
+#define CYCLIC_RX_PAGES_MAX	256
 
 #define LS_BYTE_MASK 0x000000FFUL
 
@@ -247,6 +248,23 @@ enum dev_capabilities {
 
 /* SECTION: Structure definitions */
 
+struct xdma_io_cb {
+	void __user *buf;
+	size_t len;
+	void *private;
+	unsigned int pages_nr;
+	struct sg_table sgt;
+	struct page **pages;
+	/** total data size */
+	unsigned int count;
+	/** MM only, DDR/BRAM memory addr */
+	u64 ep_addr;
+	/** write: if write to the device */
+	struct xdma_request_cb *req;
+	u8 write:1;
+	void (*io_done)(unsigned long cb_hndl, int err);
+};
+
 struct config_regs {
 	u32 identifier;
 	u32 reserved_1[4];
@@ -392,11 +410,14 @@ struct sw_desc {
 struct xdma_transfer {
 	struct list_head entry;		/* queue of non-completed transfers */
 	struct xdma_desc *desc_virt;	/* virt addr of the 1st descriptor */
+	struct xdma_result *res_virt; /* virt addr of result for c2h streaming */
+	dma_addr_t res_bus;			  /* bus addr for result descriptors */
 	dma_addr_t desc_bus;		/* bus addr of the first descriptor */
 	int desc_adjacent;		/* adjacent descriptors at desc_bus */
 	int desc_num;			/* number of descriptors in transfer */
+	int desc_index;			/* index for first descriptor in transfer */
 	enum dma_data_direction dir;
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head wq;
 #else
 	wait_queue_head_t wq;		/* wait queue for transfer completion */
@@ -409,6 +430,7 @@ struct xdma_transfer {
 	int last_in_request;		/* flag if last within request */
 	unsigned int len;
 	struct sg_table *sgt;
+	struct xdma_io_cb *cb;
 };
 
 struct xdma_request_cb {
@@ -416,7 +438,8 @@ struct xdma_request_cb {
 	unsigned int total_len;
 	u64 ep_addr;
 
-	struct xdma_transfer xfer;
+	struct xdma_transfer tfer[2]; /* Use two transfers in case single request needs to be split */
+	struct xdma_io_cb *cb;
 
 	unsigned int sw_desc_idx;
 	unsigned int sw_desc_cnt;
@@ -450,7 +473,8 @@ struct xdma_engine {
 	int max_extra_adj;	/* descriptor prefetch capability */
 	int desc_dequeued;	/* num descriptors of completed transfers */
 	u32 status;		/* last known status of device */
-	u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */
+	/* only used for MSIX mode to store per-engine interrupt mask value */
+	u32 interrupt_enable_mask_value;
 
 	/* Transfer list management */
 	struct list_head transfer_list;	/* queue of transfers */
@@ -458,8 +482,10 @@ struct xdma_engine {
 	/* Members applicable to AXI-ST C2H (cyclic) transfers */
 	struct xdma_result *cyclic_result;
 	dma_addr_t cyclic_result_bus;	/* bus addr for transfer */
-	struct xdma_request_cb *cyclic_req; 
-	struct sg_table cyclic_sgt; 
+	struct xdma_request_cb *cyclic_req;
+	struct sg_table cyclic_sgt;
+    u8 *perf_buf_virt;
+    dma_addr_t perf_buf_bus; /* bus address */
 	u8 eop_found; /* used only for cyclic(rx:c2h) */
 	int eop_count;
 	int rx_tail;	/* follows the HW */
@@ -474,7 +500,7 @@ struct xdma_engine {
 	dma_addr_t poll_mode_bus;	/* bus addr for descriptor writeback */
 
 	/* Members associated with interrupt mode support */
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head shutdown_wq;
 #else
 	wait_queue_head_t shutdown_wq;	/* wait queue for shutdown sync */
@@ -488,14 +514,23 @@ struct xdma_engine {
 	spinlock_t desc_lock;		/* protects concurrent access */
 	dma_addr_t desc_bus;
 	struct xdma_desc *desc;
+	int desc_idx;			/* current descriptor index */
+	int desc_used;			/* total descriptors used */
 
 	/* for performance test support */
 	struct xdma_performance_ioctl *xdma_perf;	/* perf test control */
-#if	LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
+#if	KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
 	struct swait_queue_head xdma_perf_wq;
 #else
 	wait_queue_head_t xdma_perf_wq;	/* Perf test sync */
 #endif
+
+	struct xdma_kthread *cmplthp;
+	/* completion status thread list for the queue */
+	struct list_head cmplthp_list;
+	/* pending work thread list */
+	/* cpu attached to intr_work */
+	unsigned int intr_work_cpu;
 };
 
 struct xdma_user_irq {
@@ -506,14 +541,14 @@ struct xdma_user_irq {
 	wait_queue_head_t events_wq;	/* wait queue to sync waiting threads */
 	irq_handler_t handler;
 
-	void *dev;	
+	void *dev;
 };
 
 /* XDMA PCIe device specific book-keeping */
 #define XDEV_FLAG_OFFLINE	0x1
 struct xdma_dev {
 	struct list_head list_head;
-        struct list_head rcu_node;
+	struct list_head rcu_node;
 
 	unsigned long magic;		/* structure ID for sanity checks */
 	struct pci_dev *pdev;	/* pci device struct from probe() */
@@ -525,7 +560,7 @@ struct xdma_dev {
 	unsigned int flags;
 
 	/* PCIe BAR management */
-	void *__iomem bar[XDMA_BAR_NUM];	/* addresses for mapped BARs */
+	void __iomem *bar[XDMA_BAR_NUM];	/* addresses for mapped BARs */
 	int user_bar_idx;	/* BAR index of user logic */
 	int config_bar_idx;	/* BAR index of XDMA config logic */
 	int bypass_bar_idx;	/* BAR index of XDMA bypass logic */
@@ -541,7 +576,7 @@ struct xdma_dev {
 	int irq_line;		/* flag if irq allocated successfully */
 	int msi_enabled;	/* flag if msi was enabled for the device */
 	int msix_enabled;	/* flag if msi-x was enabled for the device */
-#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0)
+#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE
 	struct msix_entry entry[32];	/* msi-x vector/entry table */
 #endif
 	struct xdma_user_irq user_irq[16];	/* user IRQ management */
@@ -621,8 +656,8 @@ void get_perf_stats(struct xdma_engine *engine);
 
 int xdma_cyclic_transfer_setup(struct xdma_engine *engine);
 int xdma_cyclic_transfer_teardown(struct xdma_engine *engine);
-ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t,
-			 int);
+ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine,
+		char __user *buf, size_t count, int timeout_ms);
 int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg);
-
+int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count);
 #endif /* XDMA_LIB_H */

XDMA/linux-kernel/libxdma/version.h

@@ -20,9 +20,9 @@
 #ifndef __XDMA_VERSION_H__
 #define __XDMA_VERSION_H__
 
-#define DRV_MOD_MAJOR		2018
-#define DRV_MOD_MINOR		3
-#define DRV_MOD_PATCHLEVEL	41
+#define DRV_MOD_MAJOR		2019
+#define DRV_MOD_MINOR		2
+#define DRV_MOD_PATCHLEVEL	42
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/Makefile

@@ -18,12 +18,12 @@ EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS)
 #EXTRA_CFLAGS += -DINTERNAL_TESTING
 
 ifneq ($(KERNELRELEASE),)
-	$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o
+	$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o
 	obj-m := $(TARGET_MODULE).o
 else
 	BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build
 	PWD:=$(shell pwd)
-all : 
+all :
 	$(MAKE) -C $(BUILDSYSTEM_DIR) M=$(PWD) modules
 
 clean:

XDMA/linux-kernel/xdma/cdev_bypass.c

@@ -20,7 +20,7 @@
 #include "libxdma_api.h"
 #include "xdma_cdev.h"
 
-#define write_register(v,mem,off) iowrite32(v, mem)
+#define write_register(v, mem, off) iowrite32(v, mem)
 
 static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
 		size_t *buf_offset, size_t buf_size)
@@ -29,8 +29,15 @@ static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
 	int copy_err;
 	int rc = 0;
 
-	BUG_ON(!buf);
-	BUG_ON(!buf_offset);
+	if (!buf) {
+		pr_err("Invalid user buffer\n");
+		return -EINVAL;
+	}
+
+	if (!buf_offset) {
+		pr_err("Invalid user buffer offset\n");
+		return -EINVAL;
+	}
 
 	/* Fill user buffer with descriptor data */
 	for (i = 0; i < transfer->desc_num; i++) {
@@ -71,7 +78,7 @@ static ssize_t char_bypass_read(struct file *file, char __user *buf,
 	xdev = xcdev->xdev;
 	engine = xcdev->engine;
 
-	dbg_sg("In char_bypass_read()\n");
+	dbg_sg("In %s()\n", __func__);
 
 	if (count & 3) {
 		dbg_sg("Buffer size must be a multiple of 4 bytes\n");
@@ -114,7 +121,7 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
 
 	u32 desc_data;
-	u32 *bypass_addr;
+	void __iomem *bypass_addr;
 	size_t buf_offset = 0;
 	int rc = 0;
 	int copy_err;
@@ -140,18 +147,21 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
 		return -ENODEV;
 	}
 
-	dbg_sg("In char_bypass_write()\n");
+	dbg_sg("In %s()\n", __func__);
 
 	spin_lock(&engine->lock);
 
 	/* Write descriptor data to the bypass BAR */
-	bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx];
-	bypass_addr += engine->bypass_offset;
+	bypass_addr = xdev->bar[xdev->bypass_bar_idx];
+	bypass_addr = (void __iomem *)(
+			(u32 __iomem *)bypass_addr + engine->bypass_offset
+			);
 	while (buf_offset < count) {
 		copy_err = copy_from_user(&desc_data, &buf[buf_offset],
 			sizeof(u32));
 		if (!copy_err) {
-			write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset);
+			write_register(desc_data, bypass_addr,
+					bypass_addr - engine->bypass_offset);
 			buf_offset += sizeof(u32);
 			rc = buf_offset;
 		} else {
@@ -183,5 +193,5 @@ static const struct file_operations bypass_fops = {
 
 void cdev_bypass_init(struct xdma_cdev *xcdev)
 {
-        cdev_init(&xcdev->cdev, &bypass_fops);
+	cdev_init(&xcdev->cdev, &bypass_fops);
 }

XDMA/linux-kernel/xdma/cdev_ctrl.c

@@ -32,13 +32,13 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
 {
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data;
 	struct xdma_dev *xdev;
-	void *reg;
+	void __iomem *reg;
 	u32 w;
 	int rv;
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 	xdev = xcdev->xdev;
 
 	/* only 32-bit aligned and 32-bit multiples */
@@ -48,8 +48,8 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
 	reg = xdev->bar[xcdev->bar] + *pos;
 	//w = read_register(reg);
 	w = ioread32(reg);
-	dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg,
-		(long)count, (int)*pos, w);
+	dbg_sg("%s(@%p, count=%ld, pos=%d) value = 0x%08x\n",
+			__func__, reg, (long)count, (int)*pos, w);
 	rv = copy_to_user(buf, &w, 4);
 	if (rv)
 		dbg_sg("Copy to userspace failed but continuing\n");
@@ -63,13 +63,13 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
 {
 	struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
 	struct xdma_dev *xdev;
-	void *reg;
+	void __iomem *reg;
 	u32 w;
 	int rv;
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 	xdev = xcdev->xdev;
 
 	/* only 32-bit aligned and 32-bit multiples */
@@ -79,12 +79,11 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
 	/* first address is BAR base plus file position offset */
 	reg = xdev->bar[xcdev->bar] + *pos;
 	rv = copy_from_user(&w, buf, 4);
-	if (rv) {
+	if (rv)
 		pr_info("copy from user failed %d/4, but continuing.\n", rv);
-	}
 
-	dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg,
-		(long)count, (int)*pos);
+	dbg_sg("%s(0x%08x @%p, count=%ld, pos=%d)\n",
+			__func__, w, reg, (long)count, (int)*pos);
 	//write_register(w, reg);
 	iowrite32(w, reg);
 	*pos += 4;
@@ -129,7 +128,7 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 
 	xdev = xcdev->xdev;
 	if (!xdev) {
@@ -158,7 +157,7 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	switch (cmd) {
 	case XDMA_IOCINFO:
-		if (copy_from_user((void *)&ioctl_obj, (void *) arg,
+		if (copy_from_user((void *)&ioctl_obj, (void __user *) arg,
 			 sizeof(struct xdma_ioc_base))) {
 			pr_err("copy_from_user failed.\n");
 			return -EFAULT;
@@ -196,7 +195,7 @@ int bridge_mmap(struct file *file, struct vm_area_struct *vma)
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 	xdev = xcdev->xdev;
 
 	off = vma->vm_pgoff << PAGE_SHIFT;

XDMA/linux-kernel/xdma/cdev_ctrl.h

@@ -60,14 +60,14 @@ struct xdma_ioc_base {
 };
 
 struct xdma_ioc_info {
-        struct xdma_ioc_base	base;
-        unsigned short		vendor;
-        unsigned short		device;
-        unsigned short		subsystem_vendor;
-        unsigned short		subsystem_device;
-        unsigned int		dma_engine_version;
-        unsigned int		driver_version;
-        unsigned long long 	feature_id;
+	struct xdma_ioc_base	base;
+	unsigned short		vendor;
+	unsigned short		device;
+	unsigned short		subsystem_vendor;
+	unsigned short		subsystem_device;
+	unsigned int		dma_engine_version;
+	unsigned int		driver_version;
+	unsigned long long	feature_id;
 	unsigned short		domain;
 	unsigned char		bus;
 	unsigned char		dev;

XDMA/linux-kernel/xdma/cdev_events.c

@@ -35,7 +35,7 @@ static ssize_t char_events_read(struct file *file, char __user *buf,
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 	user_irq = xcdev->user_irq;
 	if (!user_irq) {
 		pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev);
@@ -84,7 +84,7 @@ static unsigned int char_events_poll(struct file *file, poll_table *wait)
 
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
-		return rv;	
+		return rv;
 	user_irq = xcdev->user_irq;
 	if (!user_irq) {
 		pr_info("xcdev 0x%p, user_irq NULL.\n", xcdev);

XDMA/linux-kernel/xdma/cdev_sgdma.h

@@ -46,17 +46,16 @@
  * _IOC_SIZE(nr)	    returns size
  */
 
-struct xdma_performance_ioctl
-{
-        /* IOCTL_XDMA_IOCTL_Vx */
-        uint32_t version;
-        uint32_t transfer_size;
-        /* measurement */
-        uint32_t stopped;
-        uint32_t iterations;
-        uint64_t clock_cycle_count;
-        uint64_t data_cycle_count;
-        uint64_t pending_count;
+struct xdma_performance_ioctl {
+	/* IOCTL_XDMA_IOCTL_Vx */
+	uint32_t version;
+	uint32_t transfer_size;
+	/* measurement */
+	uint32_t stopped;
+	uint32_t iterations;
+	uint64_t clock_cycle_count;
+	uint64_t data_cycle_count;
+	uint64_t pending_count;
 };
 
 

XDMA/linux-kernel/xdma/cdev_xvc.c

@@ -32,30 +32,30 @@
 
 #ifdef __REG_DEBUG__
 /* SECTION: Function definitions */
-inline void __write_register(const char *fn, u32 value, void *base,
+inline void __write_register(const char *fn, u32 value, void __iomem *base,
 				unsigned int off)
 {
-        pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
-        iowrite32(value, base + off);
+	pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
+	iowrite32(value, base + off);
 }
 
-inline u32 __read_register(const char *fn, void *base, unsigned int off)
+inline u32 __read_register(const char *fn, void __iomem *base, unsigned int off)
 {
 	u32 v = ioread32(base + off);
 
-        pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
-        return v;
+	pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
+	return v;
 }
-#define write_register(v,base,off) __write_register(__func__, v, base, off)
-#define read_register(base,off) __read_register(__func__, base, off)
+#define write_register(v, base, off) __write_register(__func__, v, base, off)
+#define read_register(base, off) __read_register(__func__, base, off)
 
 #else
-#define write_register(v,base,off)	iowrite32(v, (base) + (off))
-#define read_register(base,off)		ioread32((base) + (off))
+#define write_register(v, base, off)	iowrite32(v, (base) + (off))
+#define read_register(base, off)		ioread32((base) + (off))
 #endif /* #ifdef __REG_DEBUG__ */
 
 
-static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
+static int xvc_shift_bits(void __iomem *base, u32 tms_bits, u32 tdi_bits,
 			u32 *tdo_bits)
 {
 	u32 control;
@@ -92,7 +92,7 @@ static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
 
 static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
-        struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data;
+	struct xdma_cdev *xcdev = (struct xdma_cdev *)filp->private_data;
 	struct xdma_dev *xdev;
 	struct xvc_ioc xvc_obj;
 	unsigned int opcode;
@@ -109,6 +109,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	rv = xcdev_check(__func__, xcdev, 0);
 	if (rv < 0)
 		return rv;
+
 	xdev = xcdev->xdev;
 
 	if (cmd != XDMA_IOCXVC) {
@@ -135,7 +136,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	total_bits = xvc_obj.length;
 	total_bytes = (total_bits + 7) >> 3;
 
-	buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL);
+	buffer = (unsigned char *)kmalloc(total_bytes * 3, GFP_KERNEL);
 	if (!buffer) {
 		pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n",
 			3 * total_bytes, opcode, total_bits, total_bytes);
@@ -146,12 +147,16 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 	tdi_buf = tms_buf + total_bytes;
 	tdo_buf = tdi_buf + total_bytes;
 
-	rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes);
+	rv = copy_from_user((void *)tms_buf,
+			(const char __user *)xvc_obj.tms_buf,
+			total_bytes);
 	if (rv) {
 		pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes);
 		goto cleanup;
 	}
-	rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes);
+	rv = copy_from_user((void *)tdi_buf,
+			(const char __user *)xvc_obj.tdi_buf,
+			total_bytes);
 	if (rv) {
 		pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes);
 		goto cleanup;
@@ -162,7 +167,8 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 
 	iobase = xdev->bar[xcdev->bar] + xcdev->base;
 	/* set length register to 32 initially if more than one
-	 * word-transaction is to be done */
+	 * word-transaction is to be done
+	 */
 	if (total_bits >= 32)
 		write_register(0x20, iobase, XVC_BAR_LENGTH_REG);
 
@@ -173,7 +179,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		u32 tms_store = 0;
 		u32 tdi_store = 0;
 		u32 tdo_store = 0;
-		
+
 		if (bits_left < 32) {
 			/* set number of bits to shift out */
 			write_register(bits_left, iobase, XVC_BAR_LENGTH_REG);
@@ -186,33 +192,33 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 		/* Shift data out and copy to output buffer */
 		rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store);
 		if (rv < 0)
-			goto cleanup;
+			break;
 
 		memcpy(tdo_buf + bytes, &tdo_store, shift_bytes);
 	}
 
+	if (rv < 0)
+		goto unlock;
+
 	/* if testing bar access swap tdi and tdo bufferes to "loopback" */
 	if (opcode == 0x2) {
-		char *tmp = tdo_buf;
+		unsigned char *tmp = tdo_buf;
 
 		tdo_buf = tdi_buf;
 		tdi_buf = tmp;
 	}
 
-	rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes);
-	if (rv) {
+	rv = copy_to_user(xvc_obj.tdo_buf, (const void *)tdo_buf, total_bytes);
+	if (rv)
 		pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes);
-		rv = -EFAULT;
-		goto cleanup;
-	}
-
-cleanup:
-	if (buffer)
-		kfree(buffer);
 
+unlock:
 	mmiowb();
 	spin_unlock(&xcdev->lock);
 
+cleanup:
+	kfree(buffer);
+
 	return rv;
 }
 
@@ -220,10 +226,10 @@ cleanup:
  * character device file operations for the XVC
  */
 static const struct file_operations xvc_fops = {
-        .owner = THIS_MODULE,
-        .open = char_open,
-        .release = char_close,
-        .unlocked_ioctl = xvc_ioctl,
+	.owner = THIS_MODULE,
+	.open = char_open,
+	.release = char_close,
+	.unlocked_ioctl = xvc_ioctl,
 };
 
 void cdev_xvc_init(struct xdma_cdev *xcdev)

XDMA/linux-kernel/xdma/cdev_xvc.h

@@ -33,9 +33,9 @@
 struct xvc_ioc {
 	unsigned int opcode;
 	unsigned int length;
-	unsigned char *tms_buf;
-	unsigned char *tdi_buf;
-	unsigned char *tdo_buf;
+	const char __user *tms_buf;
+	const char __user *tdi_buf;
+	void __user *tdo_buf;
 };
 
 #define XDMA_IOCXVC	_IOWR(XVC_MAGIC, 1, struct xvc_ioc)

XDMA/linux-kernel/xdma/version.h

@@ -20,9 +20,9 @@
 #ifndef __XDMA_VERSION_H__
 #define __XDMA_VERSION_H__
 
-#define DRV_MOD_MAJOR		2018
-#define DRV_MOD_MINOR		3
-#define DRV_MOD_PATCHLEVEL	50
+#define DRV_MOD_MAJOR		2019
+#define DRV_MOD_MINOR		2
+#define DRV_MOD_PATCHLEVEL	51
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/xdma_cdev.h

@@ -35,13 +35,13 @@ int xdma_cdev_init(void);
 
 int char_open(struct inode *inode, struct file *file);
 int char_close(struct inode *inode, struct file *file);
-int xcdev_check(const char *, struct xdma_cdev *, bool);
-
+int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine);
 void cdev_ctrl_init(struct xdma_cdev *xcdev);
 void cdev_xvc_init(struct xdma_cdev *xcdev);
 void cdev_event_init(struct xdma_cdev *xcdev);
 void cdev_sgdma_init(struct xdma_cdev *xcdev);
 void cdev_bypass_init(struct xdma_cdev *xcdev);
+long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 
 void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev);
 int xpdev_create_interfaces(struct xdma_pci_dev *xpdev);

XDMA/linux-kernel/xdma/xdma_mod.c

@@ -40,51 +40,51 @@ static char version[] =
 MODULE_AUTHOR("Xilinx, Inc.");
 MODULE_DESCRIPTION(DRV_MODULE_DESC);
 MODULE_VERSION(DRV_MODULE_VERSION);
-MODULE_LICENSE("GPL");
+MODULE_LICENSE("Dual BSD/GPL");
 
 /* SECTION: Module global variables */
-static int xpdev_cnt = 0;
+static int xpdev_cnt;
 
 static const struct pci_device_id pci_ids[] = {
 	{ PCI_DEVICE(0x10ee, 0x903f), },
 	{ PCI_DEVICE(0x10ee, 0x9038), },
 	{ PCI_DEVICE(0x10ee, 0x9028), },
-        { PCI_DEVICE(0x10ee, 0x9018), },
+	{ PCI_DEVICE(0x10ee, 0x9018), },
 	{ PCI_DEVICE(0x10ee, 0x9034), },
 	{ PCI_DEVICE(0x10ee, 0x9024), },
-        { PCI_DEVICE(0x10ee, 0x9014), },
+	{ PCI_DEVICE(0x10ee, 0x9014), },
 	{ PCI_DEVICE(0x10ee, 0x9032), },
 	{ PCI_DEVICE(0x10ee, 0x9022), },
-        { PCI_DEVICE(0x10ee, 0x9012), },
+	{ PCI_DEVICE(0x10ee, 0x9012), },
 	{ PCI_DEVICE(0x10ee, 0x9031), },
 	{ PCI_DEVICE(0x10ee, 0x9021), },
-        { PCI_DEVICE(0x10ee, 0x9011), },
+	{ PCI_DEVICE(0x10ee, 0x9011), },
 
 	{ PCI_DEVICE(0x10ee, 0x8011), },
 	{ PCI_DEVICE(0x10ee, 0x8012), },
-        { PCI_DEVICE(0x10ee, 0x8014), },
-        { PCI_DEVICE(0x10ee, 0x8018), },
-        { PCI_DEVICE(0x10ee, 0x8021), },
-        { PCI_DEVICE(0x10ee, 0x8022), },
-        { PCI_DEVICE(0x10ee, 0x8024), },
-        { PCI_DEVICE(0x10ee, 0x8028), },
-        { PCI_DEVICE(0x10ee, 0x8031), },
-        { PCI_DEVICE(0x10ee, 0x8032), },
-        { PCI_DEVICE(0x10ee, 0x8034), },
-        { PCI_DEVICE(0x10ee, 0x8038), },
-
-        { PCI_DEVICE(0x10ee, 0x7011), },
-        { PCI_DEVICE(0x10ee, 0x7012), },
-        { PCI_DEVICE(0x10ee, 0x7014), },
-        { PCI_DEVICE(0x10ee, 0x7018), },
-        { PCI_DEVICE(0x10ee, 0x7021), },
-        { PCI_DEVICE(0x10ee, 0x7022), },
-        { PCI_DEVICE(0x10ee, 0x7024), },
+	{ PCI_DEVICE(0x10ee, 0x8014), },
+	{ PCI_DEVICE(0x10ee, 0x8018), },
+	{ PCI_DEVICE(0x10ee, 0x8021), },
+	{ PCI_DEVICE(0x10ee, 0x8022), },
+	{ PCI_DEVICE(0x10ee, 0x8024), },
+	{ PCI_DEVICE(0x10ee, 0x8028), },
+	{ PCI_DEVICE(0x10ee, 0x8031), },
+	{ PCI_DEVICE(0x10ee, 0x8032), },
+	{ PCI_DEVICE(0x10ee, 0x8034), },
+	{ PCI_DEVICE(0x10ee, 0x8038), },
+
+	{ PCI_DEVICE(0x10ee, 0x7011), },
+	{ PCI_DEVICE(0x10ee, 0x7012), },
+	{ PCI_DEVICE(0x10ee, 0x7014), },
+	{ PCI_DEVICE(0x10ee, 0x7018), },
+	{ PCI_DEVICE(0x10ee, 0x7021), },
+	{ PCI_DEVICE(0x10ee, 0x7022), },
+	{ PCI_DEVICE(0x10ee, 0x7024), },
 	{ PCI_DEVICE(0x10ee, 0x7028), },
-        { PCI_DEVICE(0x10ee, 0x7031), },
-        { PCI_DEVICE(0x10ee, 0x7032), },
-        { PCI_DEVICE(0x10ee, 0x7034), },
-        { PCI_DEVICE(0x10ee, 0x7038), },
+	{ PCI_DEVICE(0x10ee, 0x7031), },
+	{ PCI_DEVICE(0x10ee, 0x7032), },
+	{ PCI_DEVICE(0x10ee, 0x7034), },
+	{ PCI_DEVICE(0x10ee, 0x7038), },
 
 	{ PCI_DEVICE(0x10ee, 0x6828), },
 	{ PCI_DEVICE(0x10ee, 0x6830), },
@@ -125,7 +125,7 @@ static void xpdev_free(struct xdma_pci_dev *xpdev)
 
 static struct xdma_pci_dev *xpdev_alloc(struct pci_dev *pdev)
 {
-	struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL);	
+	struct xdma_pci_dev *xpdev = kmalloc(sizeof(*xpdev), GFP_KERNEL);
 
 	if (!xpdev)
 		return NULL;
@@ -154,14 +154,28 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max,
 			&xpdev->h2c_channel_max, &xpdev->c2h_channel_max);
-	if (!hndl){
-		rv =  -EINVAL;
+	if (!hndl) {
+		rv = -EINVAL;
 		goto err_out;
 	}
 
-	BUG_ON(xpdev->user_max > MAX_USER_IRQ);
-	BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX);
-	BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX);
+	if (xpdev->user_max > MAX_USER_IRQ) {
+		pr_err("Maximum users limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
+
+	if (xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) {
+		pr_err("Maximun H2C channel limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
+
+	if (xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) {
+		pr_err("Maximun C2H channel limit reached\n");
+		rv = -EINVAL;
+		goto err_out;
+	}
 
 	if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max)
 		pr_warn("NO engine found!\n");
@@ -181,7 +195,12 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 		rv =  -EINVAL;
 		goto err_out;
 	}
-	BUG_ON(hndl != xdev );
+
+	if (hndl != xdev) {
+		pr_err("xdev handle mismatch\n");
+		rv =  -EINVAL;
+		goto err_out;
+	}
 
 	pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n",
 		dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev,
@@ -198,7 +217,7 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	return 0;
 
-err_out:	
+err_out:
 	pr_err("pdev 0x%p, err %d.\n", pdev, rv);
 	xpdev_free(xpdev);
 	return rv;
@@ -219,7 +238,7 @@ static void remove_one(struct pci_dev *pdev)
 		pdev, xpdev, xpdev->xdev);
 	xpdev_free(xpdev);
 
-        dev_set_drvdata(&pdev->dev, NULL);
+	dev_set_drvdata(&pdev->dev, NULL);
 }
 
 static pci_ers_result_t xdma_error_detected(struct pci_dev *pdev,
@@ -270,7 +289,7 @@ static void xdma_error_resume(struct pci_dev *pdev)
 	pci_cleanup_aer_uncorrect_error_status(pdev);
 }
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
 static void xdma_reset_prepare(struct pci_dev *pdev)
 {
 	struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
@@ -287,7 +306,7 @@ static void xdma_reset_done(struct pci_dev *pdev)
 	xdma_device_online(pdev, xpdev->xdev);
 }
 
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
 static void xdma_reset_notify(struct pci_dev *pdev, bool prepare)
 {
 	struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
@@ -305,10 +324,10 @@ static const struct pci_error_handlers xdma_err_handler = {
 	.error_detected	= xdma_error_detected,
 	.slot_reset	= xdma_slot_reset,
 	.resume		= xdma_error_resume,
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
+#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
 	.reset_prepare	= xdma_reset_prepare,
 	.reset_done	= xdma_reset_done,
-#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
+#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
 	.reset_notify	= xdma_reset_notify,
 #endif
 };
@@ -324,8 +343,6 @@ static struct pci_driver pci_driver = {
 static int __init xdma_mod_init(void)
 {
 	int rv;
-	extern unsigned int desc_blen_max;
-	extern unsigned int sgdma_timeout;
 
 	pr_info("%s", version);
 

XDMA/linux-kernel/xdma/xdma_mod.h

@@ -44,14 +44,19 @@
 #include <linux/splice.h>
 #include <linux/version.h>
 #include <linux/uio.h>
+#include <linux/spinlock_types.h>
 
 #include "libxdma.h"
+#include "xdma_thread.h"
 
 #define MAGIC_ENGINE	0xEEEEEEEEUL
 #define MAGIC_DEVICE	0xDDDDDDDDUL
 #define MAGIC_CHAR	0xCCCCCCCCUL
 #define MAGIC_BITSTREAM 0xBBBBBBBBUL
 
+extern unsigned int desc_blen_max;
+extern unsigned int sgdma_timeout;
+
 struct xdma_cdev {
 	unsigned long magic;		/* structure ID for sanity checks */
 	struct xdma_pci_dev *xpdev;
@@ -94,12 +99,19 @@ struct xdma_pci_dev {
 	void *data;
 };
 
-struct xdma_io_cb {
-	void __user *buf;
-	size_t len;
-	unsigned int pages_nr;
-	struct sg_table sgt;
-	struct page **pages;
+struct cdev_async_io {
+	struct kiocb *iocb;
+	struct xdma_io_cb* cb;
+	bool write;
+	bool cancel;
+	int cmpl_cnt;
+	int req_cnt;
+	spinlock_t lock;
+	struct work_struct wrk_itm;
+	struct cdev_async_io *next;
+	ssize_t res;
+	ssize_t res2;
+	int err_cnt;
 };
 
 #endif /* ifndef __XDMA_MODULE_H__ */

XDMA/linux-kernel/xdma/xdma_thread.c

+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present,  Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#define pr_fmt(fmt)	KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include "xdma_thread.h"
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+
+
+/* ********************* global variables *********************************** */
+static struct xdma_kthread *cs_threads;
+static unsigned int thread_cnt;
+
+
+/* ********************* static function definitions ************************ */
+static int xdma_thread_cmpl_status_pend(struct list_head *work_item)
+{
+	struct xdma_engine *engine = list_entry(work_item, struct xdma_engine,
+						cmplthp_list);
+	int pend = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->lock, flags);
+		pend = !list_empty(&engine->transfer_list);
+	spin_unlock_irqrestore(&engine->lock, flags);
+
+	return pend;
+}
+
+static int xdma_thread_cmpl_status_proc(struct list_head *work_item)
+{
+	struct xdma_engine *engine;
+	struct xdma_transfer * transfer;
+
+	engine = list_entry(work_item, struct xdma_engine, cmplthp_list);
+	transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
+			entry);
+	engine_service_poll(engine, transfer->desc_num);
+	return 0;
+}
+
+
+
+static inline int xthread_work_pending(struct xdma_kthread *thp)
+{
+	struct list_head *work_item, *next;
+
+	/* any work items assigned to this thread? */
+	if (list_empty(&thp->work_list))
+		return 0;
+
+
+	/* any work item has pending work to do? */
+	list_for_each_safe(work_item, next, &thp->work_list) {
+		if (thp->fpending && thp->fpending(work_item))
+			return 1;
+
+	}
+	return 0;
+}
+
+static inline void xthread_reschedule(struct xdma_kthread *thp)
+{
+	if (thp->timeout) {
+		pr_debug_thread("%s rescheduling for %u seconds",
+				thp->name, thp->timeout);
+		wait_event_interruptible_timeout(thp->waitq, thp->schedule,
+					      msecs_to_jiffies(thp->timeout));
+	} else {
+		pr_debug_thread("%s rescheduling", thp->name);
+		wait_event_interruptible(thp->waitq, thp->schedule);
+	}
+}
+
+static int xthread_main(void *data)
+{
+	struct xdma_kthread *thp = (struct xdma_kthread *)data;
+
+	pr_debug_thread("%s UP.\n", thp->name);
+
+	disallow_signal(SIGPIPE);
+
+	if (thp->finit)
+		thp->finit(thp);
+
+
+	while (!kthread_should_stop()) {
+
+		struct list_head *work_item, *next;
+
+		pr_debug_thread("%s interruptible\n", thp->name);
+
+		/* any work to do? */
+		lock_thread(thp);
+		if (!xthread_work_pending(thp)) {
+			unlock_thread(thp);
+			xthread_reschedule(thp);
+			lock_thread(thp);
+		}
+		thp->schedule = 0;
+
+		if (thp->work_cnt) {
+			pr_debug_thread("%s processing %u work items\n",
+					thp->name, thp->work_cnt);
+			/* do work */
+			list_for_each_safe(work_item, next, &thp->work_list) {
+				thp->fproc(work_item);
+			}
+		}
+		unlock_thread(thp);
+		schedule();
+	}
+
+	pr_debug_thread("%s, work done.\n", thp->name);
+
+	if (thp->fdone)
+		thp->fdone(thp);
+
+	pr_debug_thread("%s, exit.\n", thp->name);
+	return 0;
+}
+
+
+int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
+{
+	int len;
+
+	if (thp->task) {
+		pr_warn("kthread %s task already running?\n", thp->name);
+		return -EINVAL;
+	}
+
+	len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id);
+	if (len < 0)
+		return -EINVAL;
+
+	thp->id = id;
+
+	spin_lock_init(&thp->lock);
+	INIT_LIST_HEAD(&thp->work_list);
+	init_waitqueue_head(&thp->waitq);
+
+	thp->task = kthread_create_on_node(xthread_main, (void *)thp,
+					cpu_to_node(thp->cpu), "%s", thp->name);
+	if (IS_ERR(thp->task)) {
+		pr_err("kthread %s, create task failed: 0x%lx\n",
+			thp->name, (unsigned long)IS_ERR(thp->task));
+		thp->task = NULL;
+		return -EFAULT;
+	}
+
+	kthread_bind(thp->task, thp->cpu);
+
+	pr_debug_thread("kthread 0x%p, %s, cpu %u, task 0x%p.\n",
+		thp, thp->name, thp->cpu, thp->task);
+
+	wake_up_process(thp->task);
+	return 0;
+}
+
+
+int xdma_kthread_stop(struct xdma_kthread *thp)
+{
+	int rv;
+
+	if (!thp->task) {
+		pr_debug_thread("kthread %s, already stopped.\n", thp->name);
+		return 0;
+	}
+
+	thp->schedule = 1;
+	rv = kthread_stop(thp->task);
+	if (rv < 0) {
+		pr_warn("kthread %s, stop err %d.\n", thp->name, rv);
+		return rv;
+	}
+
+	pr_debug_thread("kthread %s, 0x%p, stopped.\n", thp->name, thp->task);
+	thp->task = NULL;
+
+	return 0;
+}
+
+
+
+void xdma_thread_remove_work(struct xdma_engine *engine)
+{
+	struct xdma_kthread *cmpl_thread;
+	unsigned long flags;
+
+	spin_lock_irqsave(&engine->lock, flags);
+	cmpl_thread = engine->cmplthp;
+	engine->cmplthp = NULL;
+
+//	pr_debug("%s removing from thread %s, %u.\n",
+//		descq->conf.name, cmpl_thread ? cmpl_thread->name : "?",
+//		cpu_idx);
+
+	spin_unlock_irqrestore(&engine->lock, flags);
+
+#if 0
+	if (cpu_idx < cpu_count) {
+		spin_lock(&qcnt_lock);
+		per_cpu_qcnt[cpu_idx]--;
+		spin_unlock(&qcnt_lock);
+	}
+#endif
+
+	if (cmpl_thread) {
+		lock_thread(cmpl_thread);
+		list_del(&engine->cmplthp_list);
+		cmpl_thread->work_cnt--;
+		unlock_thread(cmpl_thread);
+	}
+
+
+}
+
+void xdma_thread_add_work(struct xdma_engine *engine)
+{
+	struct xdma_kthread *thp = cs_threads;
+	unsigned int v = 0;
+	int i, idx = thread_cnt;
+	unsigned long flags;
+
+
+	/* Polled mode only */
+	for (i = 0; i < thread_cnt; i++, thp++) {
+		lock_thread(thp);
+		if (idx == thread_cnt) {
+			v = thp->work_cnt;
+			idx = i;
+		} else if (!thp->work_cnt) {
+			idx = i;
+			unlock_thread(thp);
+			break;
+		} else if (thp->work_cnt < v)
+			idx = i;
+		unlock_thread(thp);
+	}
+
+	thp = cs_threads + idx;
+	lock_thread(thp);
+	list_add_tail(&engine->cmplthp_list, &thp->work_list);
+	engine->intr_work_cpu = idx;
+	thp->work_cnt++;
+	unlock_thread(thp);
+
+	pr_info("%s 0x%p assigned to cmpl status thread %s,%u.\n",
+		engine->name, engine, thp->name, thp->work_cnt);
+
+
+	spin_lock_irqsave(&engine->lock, flags);
+	engine->cmplthp = thp;
+	spin_unlock_irqrestore(&engine->lock, flags);
+
+}
+
+int xdma_threads_create(unsigned int num_threads)
+{
+	struct xdma_kthread *thp;
+	int i;
+	int rv;
+
+	if (thread_cnt) {
+		pr_warn("threads already created!");
+		return 0;
+	}
+
+	pr_info("xdma_threads_create\n");
+
+	thread_cnt = num_threads;
+
+	cs_threads = kzalloc(thread_cnt * sizeof(struct xdma_kthread),
+					GFP_KERNEL);
+	if (!cs_threads)
+		return -ENOMEM;
+
+	/* N dma writeback monitoring threads */
+	thp = cs_threads;
+	for (i = 0; i < thread_cnt; i++, thp++) {
+		thp->cpu = i;
+		thp->timeout = 0;
+		thp->fproc = xdma_thread_cmpl_status_proc;
+		thp->fpending = xdma_thread_cmpl_status_pend;
+		rv = xdma_kthread_start(thp, "cmpl_status_th", i);
+		if (rv < 0)
+			goto cleanup_threads;
+	}
+
+	return 0;
+
+cleanup_threads:
+	kfree(cs_threads);
+	cs_threads = NULL;
+	thread_cnt = 0;
+
+	return rv;
+}
+
+void xdma_threads_destroy(void)
+{
+	int i;
+	struct xdma_kthread *thp;
+
+	if (!thread_cnt)
+		return;
+
+	/* N dma writeback monitoring threads */
+	thp = cs_threads;
+	for (i = 0; i < thread_cnt; i++, thp++)
+		if (thp->fproc)
+			xdma_kthread_stop(thp);
+
+	kfree(cs_threads);
+	cs_threads = NULL;
+	thread_cnt = 0;
+}

XDMA/linux-kernel/xdma/xdma_thread.h

+/*
+ * This file is part of the Xilinx DMA IP Core driver for Linux
+ *
+ * Copyright (c) 2017-present,  Xilinx, Inc.
+ * All rights reserved.
+ *
+ * This source code is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ */
+
+#ifndef __XDMA_KTHREAD_H__
+#define __XDMA_KTHREAD_H__
+/**
+ * @file
+ * @brief This file contains the declarations for xdma kernel threads
+ *
+ */
+#include <linux/version.h>
+#include <linux/spinlock.h>
+#include <linux/kthread.h>
+#include <linux/cpuset.h>
+#include <linux/signal.h>
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/errno.h>
+#include "libxdma.h"
+
+#ifdef DEBUG_THREADS
+#define lock_thread(thp)	\
+	do { \
+		pr_debug("locking thp %s ...\n", (thp)->name); \
+		spin_lock(&(thp)->lock); \
+	} while (0)
+
+#define unlock_thread(thp)	\
+	do { \
+		pr_debug("unlock thp %s ...\n", (thp)->name); \
+		spin_unlock(&(thp)->lock); \
+	} while (0)
+
+#define xdma_kthread_wakeup(thp)	\
+	do { \
+		pr_info("signaling thp %s ...\n", (thp)->name); \
+		wake_up_process((thp)->task); \
+	} while (0)
+
+#define pr_debug_thread(fmt, ...) pr_info(fmt, __VA_ARGS__)
+
+#else
+/** lock thread macro */
+#define lock_thread(thp)		spin_lock(&(thp)->lock)
+/** un lock thread macro */
+#define unlock_thread(thp)		spin_unlock(&(thp)->lock)
+#define xdma_kthread_wakeup(thp) \
+	do { \
+		thp->schedule = 1; \
+		wake_up_interruptible(&thp->waitq); \
+	} while (0)
+/** pr_debug_thread */
+#define pr_debug_thread(fmt, ...)
+#endif
+
+/**
+ * @struct - xdma_kthread
+ * @brief	xdma thread book keeping parameters
+ */
+struct xdma_kthread {
+	/**  thread lock*/
+	spinlock_t lock;
+	/**  name of the thread */
+	char name[16];
+	/**  cpu number for which the thread associated with */
+	unsigned short cpu;
+	/**  thread id */
+	unsigned short id;
+	/**  thread sleep timeout value */
+	unsigned int timeout;
+	/**  flags for thread */
+	unsigned long flag;
+	/**  thread wait queue */
+	wait_queue_head_t waitq;
+	/* flag to indicate scheduling of thread */
+	unsigned int schedule;
+	/**  kernel task structure associated with thread*/
+	struct task_struct *task;
+	/**  thread work list count */
+	unsigned int work_cnt;
+	/**  thread work list count */
+	struct list_head work_list;
+	/**  thread initialization handler */
+	int (*finit)(struct xdma_kthread *);
+	/**  thread pending handler */
+	int (*fpending)(struct list_head *);
+	/**  thread peocessing handler */
+	int (*fproc)(struct list_head *);
+	/**  thread done handler */
+	int (*fdone)(struct xdma_kthread *);
+};
+
+
+/*****************************************************************************/
+/**
+ * xdma_threads_create() - create xdma threads
+*********/
+int xdma_threads_create(unsigned int num_threads);
+
+/*****************************************************************************/
+/**
+ * xdma_threads_destroy() - destroy all the xdma threads created
+ *                          during system initialization
+ *
+ * @return	none
+ *****************************************************************************/
+void xdma_threads_destroy(void);
+
+/*****************************************************************************/
+/**
+ * xdma_thread_remove_work() - handler to remove the attached work thread
+ *
+ * @param[in]	engine:	pointer to xdma_engine
+ *
+ * @return	none
+ *****************************************************************************/
+void xdma_thread_remove_work(struct xdma_engine *engine);
+
+/*****************************************************************************/
+/**
+ * xdma_thread_add_work() - handler to add a work thread
+ *
+ * @param[in]	engine:	pointer to xdma_engine
+ *
+ * @return	none
+ *****************************************************************************/
+void xdma_thread_add_work(struct xdma_engine *engine);
+
+#endif /* #ifndef __XDMA_KTHREAD_H__ */