Merge pull request #131 from karenx-xilinx/master

XDMA: reg_rw fix and MM aperture update

XDMA/linux-kernel/tools/dma_from_device.c

@@ -24,7 +24,9 @@
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
-#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "../xdma/cdev_sgdma.h"
 
 #include "dma_utils.c"
 
@@ -177,7 +179,6 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 	size_t out_offset = 0;
 	size_t bytes_done = 0;
 	uint64_t i;
-	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
 	struct timespec ts_start, ts_end;
@@ -230,34 +231,38 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 
 	for (i = 0; i < count; i++) {
 		rc = clock_gettime(CLOCK_MONOTONIC, &ts_start);
-		if (apt_loop) {
-			uint64_t j;
-			uint64_t len = size;
-			char *buf = buffer;
+		if (aperture) {
+			struct xdma_aperture_ioctl io;
 
-			bytes_done = 0;
-			for (j = 0; j < apt_loop; j++, len -= aperture, buf += aperture) {
-				uint64_t bytes = (len > aperture) ? aperture : len,
-				rc = read_to_buffer(devname, fpga_fd, buf,
-						bytes, addr);
-				if (rc < 0)
-					goto out;
+			io.buffer = (unsigned long)buffer;
+			io.len = size;
+			io.ep_addr = addr;
+			io.aperture = aperture;
+			io.done = 0UL;
 
-				if (!underflow && rc < bytes)
-					underflow = 1;
-				bytes_done += rc;
+			rc = ioctl(fpga_fd, IOCTL_XDMA_APERTURE_R, &io);
+			if (rc < 0 || io.error) {
+				fprintf(stderr,
+					"#%d: aperture R failed %d,%d.\n",
+					i, rc, io.error);
+				goto out;
 			}
+
+			bytes_done = io.done;
 		} else {
 			rc = read_to_buffer(devname, fpga_fd, buffer, size, addr);
 			if (rc < 0)
 				goto out;
 			bytes_done = rc;
 
-			if (!underflow && bytes_done < size)
-				underflow = 1;
 		}
 		clock_gettime(CLOCK_MONOTONIC, &ts_end);
 
+		if (bytes_done < size) {
+			fprintf(stderr, "#%d: underflow %ld/%ld.\n",
+				i, bytes_done, size);
+			underflow = 1;
+		}
 
 		/* subtract the start time from the end time */
 		timespec_sub(&ts_end, &ts_start);

XDMA/linux-kernel/tools/dma_to_device.c

@@ -25,7 +25,9 @@
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
-#include <unistd.h>
+#include <sys/ioctl.h>
+
+#include "../xdma/cdev_sgdma.h"
 
 #include "dma_utils.c"
 
@@ -179,7 +181,6 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 	ssize_t rc;
 	size_t bytes_done = 0;
 	size_t out_offset = 0;
-	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
 	struct timespec ts_start, ts_end;
@@ -243,24 +244,24 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 		/* write buffer to AXI MM address using SGDMA */
 		rc = clock_gettime(CLOCK_MONOTONIC, &ts_start);
 
-		if (apt_loop) {
-			uint64_t j;
-			uint64_t len = size;
-		       	char *buf = buffer;
-
-			bytes_done = 0;
-			for (j = 0; j < apt_loop; j++, len -= aperture,
-					buf += aperture) {
-				uint64_t bytes = (len > aperture) ? aperture : len,
-				rc = write_from_buffer(devname, fpga_fd, buf,
-							bytes, addr);
-				if (rc < 0)
-					goto out;
-
-				bytes_done += rc;
-				if (!underflow && rc < bytes)
-					underflow = 1;
+		if (aperture) {
+			struct xdma_aperture_ioctl io;
+
+			io.buffer = (unsigned long)buffer;
+			io.len = size;
+			io.ep_addr = addr;
+			io.aperture = aperture;
+			io.done = 0UL;
+
+			rc = ioctl(fpga_fd, IOCTL_XDMA_APERTURE_W, &io);
+			if (rc < 0 || io.error) {
+				fprintf(stdout,
+					"#%d: aperture W ioctl failed %d,%d.\n",
+					i, rc, io.error);
+				goto out;
 			}
+
+			bytes_done = io.done;
 		} else {
 			rc = write_from_buffer(devname, fpga_fd, buffer, size,
 				      	 	addr);
@@ -268,11 +269,16 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 				goto out;
 
 			bytes_done = rc;
-			if (!underflow && bytes_done < size)
-				underflow = 1;
 		}
 
 		rc = clock_gettime(CLOCK_MONOTONIC, &ts_end);
+
+		if (bytes_done < size) {
+			printf("#%d: underflow %ld/%ld.\n",
+				i, bytes_done, size);
+			underflow = 1;
+		}
+
 		/* subtract the start time from the end time */
 		timespec_sub(&ts_end, &ts_start);
 		total_time += ts_end.tv_nsec;

XDMA/linux-kernel/tools/reg_rw.c

@@ -39,17 +39,14 @@
 
 #define FATAL do { fprintf(stderr, "Error at line %d, file %s (%d) [%s]\n", __LINE__, __FILE__, errno, strerror(errno)); exit(1); } while(0)
 
-#define MAP_SIZE (32*1024UL)
-#define MAP_MASK (MAP_SIZE - 1)
-
 int main(int argc, char **argv)
 {
 	int fd;
-	void *map_base, *virt_addr;
+	void *map;
 	uint32_t read_result, writeval;
 	off_t target;
 	/* access width */
-	int access_width = 'w';
+	char access_width = 'w';
 	char *device;
 
 	/* not enough arguments given? */
@@ -64,20 +61,14 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	printf("argc = %d\n", argc);
-
 	device = strdup(argv[1]);
-	printf("device: %s\n", device);
 	target = strtoul(argv[2], 0, 0);
-	printf("address: 0x%08x\n", (unsigned int)target);
-
-	printf("access type: %s\n", argc >= 4 ? "write" : "read");
+	printf("device: %s, address: 0x%08x, access %s: %s.\n",
+		device, (unsigned int)target, argc >= 4 ? "write" : "read");
 
 	/* data given? */
-	if (argc >= 4) {
-		printf("access width given.\n");
+	if (argc >= 4)
 		access_width = tolower(argv[3][0]);
-	}
 	printf("access width: ");
 	if (access_width == 'b')
 		printf("byte (8-bits)\n");
@@ -86,7 +77,7 @@ int main(int argc, char **argv)
 	else if (access_width == 'w')
 		printf("word (32-bits)\n");
 	else {
-		printf("word (32-bits)\n");
+		printf("default to word (32-bits)\n");
 		access_width = 'w';
 	}
 
@@ -95,42 +86,38 @@ int main(int argc, char **argv)
 	printf("character device %s opened.\n", argv[1]);
 	fflush(stdout);
 
-	/* map one page */
-	map_base = mmap(0, MAP_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-	if (map_base == (void *)-1)
+	map = mmap(0, 4, PROT_READ | PROT_WRITE, MAP_SHARED, fd, target);
+	if (map == (void *)-1)
 		FATAL;
-	printf("Memory mapped at address %p.\n", map_base);
+	printf("Memory 0x%lx mapped at address %p.\n", target, map);
 	fflush(stdout);
 
-	/* calculate the virtual address to be accessed */
-	virt_addr = map_base + target;
 	/* read only */
 	if (argc <= 4) {
-		//printf("Read from address %p.\n", virt_addr); 
 		switch (access_width) {
 		case 'b':
-			read_result = *((uint8_t *) virt_addr);
+			read_result = *((uint8_t *) map);
 			printf
 			    ("Read 8-bits value at address 0x%08x (%p): 0x%02x\n",
-			     (unsigned int)target, virt_addr,
+			     (unsigned int)target, map,
 			     (unsigned int)read_result);
 			break;
 		case 'h':
-			read_result = *((uint16_t *) virt_addr);
+			read_result = *((uint16_t *) map);
 			/* swap 16-bit endianess if host is not little-endian */
 			read_result = ltohs(read_result);
 			printf
 			    ("Read 16-bit value at address 0x%08x (%p): 0x%04x\n",
-			     (unsigned int)target, virt_addr,
+			     (unsigned int)target, map,
 			     (unsigned int)read_result);
 			break;
 		case 'w':
-			read_result = *((uint32_t *) virt_addr);
+			read_result = *((uint32_t *) map);
 			/* swap 32-bit endianess if host is not little-endian */
 			read_result = ltohl(read_result);
 			printf
 			    ("Read 32-bit value at address 0x%08x (%p): 0x%08x\n",
-			     (unsigned int)target, virt_addr,
+			     (unsigned int)target, map,
 			     (unsigned int)read_result);
 			return (int)read_result;
 			break;
@@ -148,11 +135,11 @@ int main(int argc, char **argv)
 		case 'b':
 			printf("Write 8-bits value 0x%02x to 0x%08x (0x%p)\n",
 			       (unsigned int)writeval, (unsigned int)target,
-			       virt_addr);
-			*((uint8_t *) virt_addr) = writeval;
+			       map);
+			*((uint8_t *) map) = writeval;
 #if 0
 			if (argc > 4) {
-				read_result = *((uint8_t *) virt_addr);
+				read_result = *((uint8_t *) map);
 				printf("Written 0x%02x; readback 0x%02x\n",
 				       writeval, read_result);
 			}
@@ -161,13 +148,13 @@ int main(int argc, char **argv)
 		case 'h':
 			printf("Write 16-bits value 0x%04x to 0x%08x (0x%p)\n",
 			       (unsigned int)writeval, (unsigned int)target,
-			       virt_addr);
+			       map);
 			/* swap 16-bit endianess if host is not little-endian */
 			writeval = htols(writeval);
-			*((uint16_t *) virt_addr) = writeval;
+			*((uint16_t *) map) = writeval;
 #if 0
 			if (argc > 4) {
-				read_result = *((uint16_t *) virt_addr);
+				read_result = *((uint16_t *) map);
 				printf("Written 0x%04x; readback 0x%04x\n",
 				       writeval, read_result);
 			}
@@ -176,13 +163,13 @@ int main(int argc, char **argv)
 		case 'w':
 			printf("Write 32-bits value 0x%08x to 0x%08x (0x%p)\n",
 			       (unsigned int)writeval, (unsigned int)target,
-			       virt_addr);
+			       map);
 			/* swap 32-bit endianess if host is not little-endian */
 			writeval = htoll(writeval);
-			*((uint32_t *) virt_addr) = writeval;
+			*((uint32_t *) map) = writeval;
 #if 0
 			if (argc > 4) {
-				read_result = *((uint32_t *) virt_addr);
+				read_result = *((uint32_t *) map);
 				printf("Written 0x%08x; readback 0x%08x\n",
 				       writeval, read_result);
 			}
@@ -191,7 +178,7 @@ int main(int argc, char **argv)
 		}
 		fflush(stdout);
 	}
-	if (munmap(map_base, MAP_SIZE) == -1)
+	if (munmap(map, 4) == -1)
 		FATAL;
 	close(fd);
 	return 0;

XDMA/linux-kernel/xdma/Makefile

 SHELL = /bin/bash
+#
+# optional makefile parameters:
+# - DEBUG=<0|1>,	enable verbose debug print-out in the driver
+# - xvc_bar_num=,	xvc pci bar #
+# - xvc_bar_offset=,	xvc register base offset
+# 
 ifneq ($(xvc_bar_num),)
 	XVC_FLAGS += -D__XVC_BAR_NUM__=$(xvc_bar_num)
 endif
@@ -14,7 +20,9 @@ topdir := $(shell cd $(src)/.. && pwd)
 TARGET_MODULE:=xdma
 
 EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS)
-#EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__
+ifeq ($(DEBUG),1)
+	EXTRA_CFLAGS += -D__LIBXDMA_DEBUG__
+endif
 #EXTRA_CFLAGS += -DINTERNAL_TESTING
 
 ifneq ($(KERNELRELEASE),)

XDMA/linux-kernel/xdma/cdev_ctrl.c

@@ -216,8 +216,7 @@ int bridge_mmap(struct file *file, struct vm_area_struct *vma)
 	dbg_sg("mmap(): cdev->bar = %d\n", xcdev->bar);
 	dbg_sg("mmap(): xdev = 0x%p\n", xdev);
 	dbg_sg("mmap(): pci_dev = 0x%08lx\n", (unsigned long)xdev->pdev);
-
-	dbg_sg("off = 0x%lx\n", off);
+	dbg_sg("off = 0x%lx, vsize 0x%lu, psize 0x%lu.\n", off, vsize, psize);
 	dbg_sg("start = 0x%llx\n",
 		(unsigned long long)pci_resource_start(xdev->pdev,
 		xcdev->bar));

XDMA/linux-kernel/xdma/cdev_sgdma.c

@@ -731,6 +731,71 @@ static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg)
 	return put_user(engine->addr_align, (int __user *)arg);
 }
 
+
+static int ioctl_do_aperture_dma(struct xdma_engine *engine, unsigned long arg,
+				bool write)
+{
+	struct xdma_aperture_ioctl io;
+	struct xdma_io_cb cb;
+	ssize_t res;
+	int rv;
+
+	rv = copy_from_user(&io, (struct xdma_aperture_ioctl __user *)arg,
+				sizeof(struct xdma_aperture_ioctl));
+	if (rv < 0) {
+		dbg_tfr("%s failed to copy from user space 0x%lx\n",
+			engine->name, arg);
+		return -EINVAL;
+	}
+
+	dbg_tfr("%s, W %d, buf 0x%lx,%lu, ep %llu, aperture %u.\n",
+		engine->name, write, io.buffer, io.len, io.ep_addr,
+		io.aperture);
+
+	if ((write && engine->dir != DMA_TO_DEVICE) ||
+	    (!write && engine->dir != DMA_FROM_DEVICE)) {
+		pr_err("r/w mismatch. W %d, dir %d.\n", write, engine->dir);
+		return -EINVAL;
+	}
+
+	rv = check_transfer_align(engine, (char *)io.buffer, io.len,
+				io.ep_addr, 1);
+	if (rv) {
+		pr_info("Invalid transfer alignment detected\n");
+		return rv;
+	}
+
+	memset(&cb, 0, sizeof(struct xdma_io_cb));
+	cb.buf = (char __user *)io.buffer;
+	cb.len = io.len;
+	cb.ep_addr = io.ep_addr;
+	cb.write = write;
+	rv = char_sgdma_map_user_buf_to_sgl(&cb, write);
+	if (rv < 0)
+		return rv;
+
+	io.error = 0;
+	res = xdma_xfer_aperture(engine, write, io.ep_addr, io.aperture,
+				&cb.sgt, 0, write ? h2c_timeout * 1000 :
+						c2h_timeout * 1000);
+
+	char_sgdma_unmap_user_buf(&cb, write);
+	if (res < 0)
+		io.error = res;
+	else
+		io.done = res;
+
+	rv = copy_to_user((struct xdma_aperture_ioctl __user *)arg, &io,
+				sizeof(struct xdma_aperture_ioctl));
+	if (rv < 0) {
+		dbg_tfr("%s failed to copy to user space 0x%lx, %ld\n",
+			engine->name, arg, res);
+		return -EINVAL;
+	}
+
+	return io.error;
+}
+	
 static long char_sgdma_ioctl(struct file *file, unsigned int cmd,
 		unsigned long arg)
 {
@@ -766,6 +831,12 @@ static long char_sgdma_ioctl(struct file *file, unsigned int cmd,
 	case IOCTL_XDMA_ALIGN_GET:
 		rv = ioctl_do_align_get(engine, arg);
 		break;
+	case IOCTL_XDMA_APERTURE_R:
+		rv = ioctl_do_aperture_dma(engine, arg, 0);
+		break;
+	case IOCTL_XDMA_APERTURE_W:
+		rv = ioctl_do_aperture_dma(engine, arg, 1);
+		break;
 	default:
 		dbg_perf("Unsupported operation\n");
 		rv = -EINVAL;

XDMA/linux-kernel/xdma/cdev_sgdma.h

@@ -37,7 +37,7 @@
  *
  * _IO(type,nr)		    no arguments
  * _IOR(type,nr,datatype)   read data from driver
- * _IOW(type,nr.datatype)   write data to driver
+ * _IOW(type,nr,datatype)   write data to driver
  * _IORW(type,nr,datatype)  read/write data
  *
  * _IOC_DIR(nr)		    returns direction
@@ -58,6 +58,14 @@ struct xdma_performance_ioctl {
 	uint64_t pending_count;
 };
 
+struct xdma_aperture_ioctl {
+	uint64_t ep_addr;
+	unsigned int aperture;
+	unsigned long buffer;
+	unsigned long len;
+	int error;
+	unsigned long done;
+};
 
 
 /* IOCTL codes */
@@ -68,5 +76,7 @@ struct xdma_performance_ioctl {
 #define IOCTL_XDMA_ADDRMODE_SET _IOW('q', 4, int)
 #define IOCTL_XDMA_ADDRMODE_GET _IOR('q', 5, int)
 #define IOCTL_XDMA_ALIGN_GET    _IOR('q', 6, int)
+#define IOCTL_XDMA_APERTURE_R   _IOW('q', 7, struct xdma_aperture_ioctl *)
+#define IOCTL_XDMA_APERTURE_W   _IOW('q', 8, struct xdma_aperture_ioctl *)
 
 #endif /* _XDMA_IOCALLS_POSIX_H_ */

XDMA/linux-kernel/xdma/libxdma.c

@@ -3147,6 +3147,304 @@ static struct xdma_request_cb *xdma_init_request(struct sg_table *sgt,
 	return req;
 }
 
+ssize_t xdma_xfer_aperture(struct xdma_engine *engine, bool write, u64 ep_addr,
+			unsigned int aperture, struct sg_table *sgt,
+			bool dma_mapped, int timeout_ms)
+{
+	struct xdma_dev *xdev;
+	struct xdma_request_cb *req = NULL;
+	struct scatterlist *sg;
+	enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	unsigned int maxlen = min_t(unsigned int, aperture, desc_blen_max);
+	unsigned int sg_max;
+	unsigned int tlen = 0;
+	u64 ep_addr_max = ep_addr + aperture - 1;
+	ssize_t done = 0;
+	int i, rv = 0;
+
+	if (!engine) {
+		pr_err("dma engine NULL\n");
+		return -EINVAL;
+	}
+
+	if (engine->magic != MAGIC_ENGINE) {
+		pr_err("%s has invalid magic number %lx\n", engine->name,
+		       engine->magic);
+		return -EINVAL;
+	}
+
+	xdev = engine->xdev;
+	if (xdma_device_flag_check(xdev, XDEV_FLAG_OFFLINE)) {
+		pr_info("xdev 0x%p, offline.\n", xdev);
+		return -EBUSY;
+	}
+
+	/* check the direction */
+	if (engine->dir != dir) {
+		pr_info("0x%p, %s, W %d, 0x%x/0x%x mismatch.\n", engine,
+			engine->name, write, engine->dir, dir);
+		return -EINVAL;
+	}
+
+	if (engine->streaming) {
+		pr_info("%s aperture not supported in ST.\n", engine->name);
+		return -EINVAL;
+	}
+
+	if (!dma_mapped) {
+		sgt->nents = pci_map_sg(xdev->pdev, sgt->sgl, sgt->orig_nents,
+					dir);
+		if (!sgt->nents) {
+			pr_info("map sgl failed, sgt 0x%p.\n", sgt);
+			return -EIO;
+		}
+	} else if (!sgt->nents) {
+		pr_err("sg table has invalid number of entries 0x%p.\n", sgt);
+		return -EIO;
+	}
+	sg_max = sgt->nents;
+
+	req = kzalloc(sizeof(struct xdma_request_cb), GFP_KERNEL);
+	if (!req) {
+		rv = -ENOMEM;
+		goto unmap_sgl;
+	}
+	memset(req, 0, sizeof(struct xdma_request_cb));
+	req->sgt = sgt;
+	req->ep_addr = ep_addr;
+	req->aperture = aperture;
+	req->sg = sgt->sgl;
+
+	for (i = 0, sg = sgt->sgl; i < sgt->nents; i++, sg = sg_next(sg))
+		tlen += sg_dma_len(sg);
+	req->total_len = tlen;
+
+	dbg_tfr("%s, aperture: sg cnt %u.\n", engine->name, sgt->nents);
+
+	mutex_lock(&engine->desc_lock);
+
+	while (req->offset < req->total_len) {
+		unsigned long flags;
+		struct xdma_transfer *xfer = &req->tfer[0];
+		unsigned int sg_offset = req->sg_offset;
+		unsigned int desc_idx, desc_max, desc_cnt = 0;
+		struct xdma_desc *desc_virt;
+		dma_addr_t desc_bus;
+
+		/* initialize transfer */
+		memset(xfer, 0, sizeof(struct xdma_transfer));
+#if HAS_SWAKE_UP
+		init_swait_queue_head(&xfer->wq);
+#else
+		init_waitqueue_head(&xfer->wq);
+#endif
+		xfer->dir = engine->dir;
+		if (!dma_mapped)
+			xfer->flags = XFER_FLAG_NEED_UNMAP;
+
+		spin_lock_irqsave(&engine->lock, flags);
+
+		desc_idx = engine->desc_idx;
+		desc_max = engine->desc_max;
+
+		xfer->desc_virt = desc_virt = engine->desc + desc_idx;
+		xfer->res_virt = engine->cyclic_result + desc_idx;
+		xfer->desc_bus = desc_bus = engine->desc_bus +
+					(sizeof(struct xdma_desc) * desc_idx);
+		xfer->res_bus = engine->cyclic_result_bus +
+					(sizeof(struct xdma_result) * desc_idx);
+		xfer->desc_index = desc_idx;
+
+		/* build transfer */
+		sg = req->sg;
+		ep_addr = req->ep_addr + (req->offset & (aperture - 1));
+		i = req->sg_idx;
+		
+		for (sg = req->sg; i < sg_max && desc_idx < desc_max;
+			i++, sg = sg_next(sg)) {
+			dma_addr_t addr = sg_dma_address(sg);
+
+			tlen = sg_dma_len(sg);
+			if (sg_offset) {
+				tlen -= sg_offset;
+				addr += sg_offset;
+			}
+
+			while (tlen) {
+				unsigned int len = min_t(unsigned int, tlen,
+								 maxlen);
+
+				if (ep_addr > ep_addr_max)
+					ep_addr = req->ep_addr;
+
+				if ((ep_addr + len) > ep_addr_max)
+					len = ep_addr_max - ep_addr + 1;
+
+				xdma_desc_set(engine->desc + desc_idx, addr,
+						ep_addr, len, dir);
+
+	                	dbg_desc("sg %d -> desc %u: ep 0x%llx, 0x%llx + %u \n",
+					i, desc_idx, ep_addr, addr, len);
+
+				sg_offset += len;
+				req->offset += len;
+				xfer->len += len;
+				ep_addr += len;
+				addr += len;
+				tlen -= len;
+				
+				desc_idx++;
+				desc_cnt++;
+				if (desc_idx == desc_max)
+					break;
+			}
+
+			if (!tlen)
+				sg_offset = 0;
+			else
+				break;
+		}
+		
+		req->sg_offset = sg_offset;
+		req->sg_idx = i;
+
+		xfer->desc_adjacent = desc_cnt;
+		xfer->desc_num = desc_cnt;
+		engine->desc_used += desc_cnt;
+
+		/* create the desc linked list */
+		for (i = 0; i < (desc_cnt - 1); i++, desc_virt++) {
+			desc_bus += sizeof(struct xdma_desc);
+			/* singly-linked list uses bus addresses */
+			desc_virt->next_lo = cpu_to_le32(PCI_DMA_L(desc_bus));
+                	desc_virt->next_hi = cpu_to_le32(PCI_DMA_H(desc_bus));
+                	desc_virt->control = cpu_to_le32(DESC_MAGIC);
+		}
+		/* terminate the last descriptor next pointer */
+		desc_virt->next_lo = cpu_to_le32(0);
+		desc_virt->next_hi = cpu_to_le32(0);
+		desc_virt->control = cpu_to_le32(DESC_MAGIC |
+					XDMA_DESC_STOPPED | XDMA_DESC_EOP |
+					XDMA_DESC_COMPLETED);
+
+		xfer->desc_cmpl_th = desc_cnt;
+
+		/* fill in adjacent numbers */
+		for (i = 0; i < desc_cnt; i++) {
+			u32 next_adj = xdma_get_next_adj(desc_cnt - i - 1,
+					(xfer->desc_virt + i)->next_lo);
+
+	                dbg_desc("set next adj at idx %d to %u\n", i, next_adj);
+        	        xdma_desc_adjacent(xfer->desc_virt + i, next_adj);
+        	}
+
+		engine->desc_idx = (engine->desc_idx + desc_cnt) % desc_max;
+		spin_unlock_irqrestore(&engine->lock, flags);
+
+		/* last transfer for the given request? */
+		if (req->offset == req->total_len) {
+			xfer->last_in_request = 1;
+			xfer->sgt = sgt;
+		}
+
+		dbg_tfr("xfer %u,%u/%u, ep 0x%llx/0x%x, done %ld, sg %u/%u, desc %u.\n",
+			xfer->len, req->offset, req->total_len, req->ep_addr,
+			req->aperture, done, req->sg_idx, sg_max, desc_cnt);
+
+		rv = transfer_queue(engine, xfer);
+		if (rv < 0) {
+			mutex_unlock(&engine->desc_lock);
+			pr_info("unable to submit %s, %d.\n", engine->name, rv);
+			goto unmap_sgl;
+		}
+
+		if (engine->cmplthp)
+			xdma_kthread_wakeup(engine->cmplthp);
+
+		if (timeout_ms > 0)
+			xlx_wait_event_interruptible_timeout(xfer->wq,
+				(xfer->state != TRANSFER_STATE_SUBMITTED),
+				msecs_to_jiffies(timeout_ms));
+		else
+			xlx_wait_event_interruptible(xfer->wq,
+				(xfer->state != TRANSFER_STATE_SUBMITTED));
+
+		spin_lock_irqsave(&engine->lock, flags);
+
+		switch (xfer->state) {
+		case TRANSFER_STATE_COMPLETED:
+			spin_unlock_irqrestore(&engine->lock, flags);
+
+			rv = 0;
+			dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n",
+				xfer, xfer->len, req->ep_addr - xfer->len,
+				done);
+
+			done += xfer->len;
+
+			break;
+		case TRANSFER_STATE_FAILED:
+			pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", xfer,
+				xfer->len, req->ep_addr - xfer->len);
+			spin_unlock_irqrestore(&engine->lock, flags);
+
+#ifdef __LIBXDMA_DEBUG__
+			transfer_dump(xfer);
+			sgt_dump(sgt);
+#endif
+			rv = -EIO;
+			break;
+		default:
+			/* transfer can still be in-flight */
+			pr_info("xfer 0x%p,%u, s 0x%x timed out, ep 0x%llx.\n",
+				xfer, xfer->len, xfer->state, req->ep_addr);
+			rv = engine_status_read(engine, 0, 1);
+			if (rv < 0) {
+				pr_err("Failed to read engine status\n");
+			} else if (rv == 0) {
+				//engine_status_dump(engine);
+				rv = transfer_abort(engine, xfer);
+				if (rv < 0) {
+					pr_err("Failed to stop engine\n");
+				} else if (rv == 0) {
+					rv = xdma_engine_stop(engine);
+					if (rv < 0)
+						pr_err("Failed to stop engine\n");
+				}
+			}
+			spin_unlock_irqrestore(&engine->lock, flags);
+
+#ifdef __LIBXDMA_DEBUG__
+			transfer_dump(xfer);
+			sgt_dump(sgt);
+#endif
+			rv = -ERESTARTSYS;
+			break;
+		}
+
+		engine->desc_used -= xfer->desc_num;
+		transfer_destroy(xdev, xfer);
+
+		if (rv < 0) {
+			mutex_unlock(&engine->desc_lock);
+			goto unmap_sgl;
+		}
+	} /* while (sg) */
+	mutex_unlock(&engine->desc_lock);
+
+unmap_sgl:
+	if (!dma_mapped && sgt->nents) {
+		pci_unmap_sg(xdev->pdev, sgt->sgl, sgt->orig_nents, dir);
+		sgt->nents = 0;
+	}
+
+	if (req)
+		xdma_request_free(req);
+
+	/* as long as some data is processed, return the count */
+	return done ? done : rv;
+}
+
 ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 			 struct sg_table *sgt, bool dma_mapped, int timeout_ms)
 {

XDMA/linux-kernel/xdma/libxdma.h

@@ -63,9 +63,6 @@
  */
 //#define XDMA_CONFIG_BAR_NUM	1
 
-/* Switch debug printing on/off */
-#define XDMA_DEBUG 0
-
 /* SECTION: Preprocessor macros/constants */
 #define XDMA_BAR_NUM (6)
 
@@ -463,8 +460,15 @@ struct xdma_transfer {
 
 struct xdma_request_cb {
 	struct sg_table *sgt;
-	unsigned int total_len;
 	u64 ep_addr;
+	unsigned int aperture;
+
+	unsigned int total_len;
+	unsigned int offset;
+
+	struct scatterlist *sg;
+	unsigned int sg_idx;
+	unsigned int sg_offset;
 
 	/* Use two transfers in case single request needs to be split */
 	struct xdma_transfer tfer[2];
@@ -678,4 +682,8 @@ void get_perf_stats(struct xdma_engine *engine);
 
 int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg);
 int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count);
+
+ssize_t xdma_xfer_aperture(struct xdma_engine *engine, bool write, u64 ep_addr,
+			unsigned int aperture, struct sg_table *sgt,
+			bool dma_mapped, int timeout_ms);
 #endif /* XDMA_LIB_H */

XDMA/linux-kernel/xdma/version.h

@@ -22,7 +22,7 @@
 
 #define DRV_MOD_MAJOR		2020
 #define DRV_MOD_MINOR		2
-#define DRV_MOD_PATCHLEVEL	0
+#define DRV_MOD_PATCHLEVEL	2
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \