XDMA v2020.1.08: allow ST c2h to end a dma transfer when EOP is received

XDMA/linux-kernel/RELEASE

+v2020.1.08
+===============
+- replaced module parameter "sgdma_timeout" to "h2c_timeout" and "c2h_timeout"
+  for H2C and C2H channels.
+  value of 0 means no timeout: wait forever for the dma completion.
+- added new "-e" option to dma_from_device
+  this is for streaming mode only, when -e is set, the driver will end the dma
+     and return the data when an EOP (end-of-packet) is received or the
+     specified bytes of data is received. 
+  without "-e" option, the driver will end the dma when the specified bytes of
+     data is received.
+- added gen4 device ids
+- fixed next adjacent descriptors when dma_alloc_coherent doesn't return a
+  page-aligned address
+
+v2020.1.06
+===============
+- added memory aperture support (-k) option in dma_from_device and dma_to_device.
+- fixed holding spinlock while doing wait_event_interruptible_xxx
+- kernel 5.0 support
+- fixed next adjacent descriptors crossing the 4K boundary
+
 Release: 2019.2
 ===============
 

XDMA/linux-kernel/tools/dma_from_device.c

@@ -40,6 +40,7 @@ static struct option const long_opts[] = {
 	{"offset", required_argument, NULL, 'o'},
 	{"count", required_argument, NULL, 'c'},
 	{"file", required_argument, NULL, 'f'},
+	{"eop_flush", no_argument, NULL, 'e'},
 	{"help", no_argument, NULL, 'h'},
 	{"verbose", no_argument, NULL, 'v'},
 	{0, 0, 0, 0}
@@ -48,7 +49,7 @@ static struct option const long_opts[] = {
 static int test_dma(char *devname, uint64_t addr, uint64_t aperture, 
 		uint64_t size, uint64_t offset, uint64_t count,
 		char *ofname);
-static int no_write = 0;
+static int eop_flush = 0;
 
 static void usage(const char *name)
 {
@@ -80,12 +81,25 @@ static void usage(const char *name)
 		"  -%c (--%s) file to write the data of the transfers\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+	fprintf(stdout,
+		 "  -%c (--%s) end dma when ST end-of-packet(eop) is rcved\n",
+		long_opts[i].val, long_opts[i].name);
+	fprintf(stdout,
+		 "\t\t* streaming only, ignored for memory-mapped channels\n");
+	fprintf(stdout,
+		 "\t\t* acutal # of bytes dma'ed could be smaller than specified\n");
+	i++;
 	fprintf(stdout, "  -%c (--%s) print usage help and exit\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
 	fprintf(stdout, "  -%c (--%s) verbose output\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+
+	fprintf(stdout, "\nReturn code:\n");
+	fprintf(stdout, "  0: all bytes were dma'ed successfully\n");
+	fprintf(stdout, "     * with -e set, the bytes dma'ed could be smaller\n");
+	fprintf(stdout, "  < 0: error\n\n");
 }
 
 int main(int argc, char *argv[])
@@ -99,7 +113,7 @@ int main(int argc, char *argv[])
 	uint64_t count = COUNT_DEFAULT;
 	char *ofname = NULL;
 
-	while ((cmd_opt = getopt_long(argc, argv, "vhxc:f:d:a:k:s:o:", long_opts,
+	while ((cmd_opt = getopt_long(argc, argv, "vhec:f:d:a:k:s:o:", long_opts,
 			    NULL)) != -1) {
 		switch (cmd_opt) {
 		case 0:
@@ -133,12 +147,12 @@ int main(int argc, char *argv[])
 			ofname = strdup(optarg);
 			break;
 			/* print usage help and exit */
-    		case 'x':
-			no_write++;
-			break;
 		case 'v':
 			verbose = 1;
 			break;
+		case 'e':
+			eop_flush = 1;
+			break;
 		case 'h':
 		default:
 			usage(argv[0]);
@@ -159,19 +173,30 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t size, uint64_t offset, uint64_t count,
 			char *ofname)
 {
-	ssize_t rc;
+	ssize_t rc = 0;
+	size_t out_offset = 0;
+	size_t bytes_done = 0;
 	uint64_t i;
 	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
 	struct timespec ts_start, ts_end;
 	int out_fd = -1;
-	int fpga_fd = open(devname, O_RDWR | O_NONBLOCK);
+	int fpga_fd;
 	long total_time = 0;
 	float result;
 	float avg_time = 0;
 	int underflow = 0;
 
+	/*
+	 * use O_TRUNC to indicate to the driver to flush the data up based on
+	 * EOP (end-of-packet), streaming mode only
+	 */
+	if (eop_flush)
+		fpga_fd = open(devname, O_RDWR | O_TRUNC);
+	else
+		fpga_fd = open(devname, O_RDWR);
+
 	if (fpga_fd < 0) {
                 fprintf(stderr, "unable to open device %s, %d.\n",
                         devname, fpga_fd);
@@ -210,6 +235,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t len = size;
 			char *buf = buffer;
 
+			bytes_done = 0;
 			for (j = 0; j < apt_loop; j++, len -= aperture, buf += aperture) {
 				uint64_t bytes = (len > aperture) ? aperture : len,
 				rc = read_to_buffer(devname, fpga_fd, buf,
@@ -219,13 +245,15 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 
 				if (!underflow && rc < bytes)
 					underflow = 1;
+				bytes_done += rc;
 			}
 		} else {
 			rc = read_to_buffer(devname, fpga_fd, buffer, size, addr);
 			if (rc < 0)
 				goto out;
+			bytes_done = rc;
 
-			if (!underflow && rc < size)
+			if (!underflow && bytes_done < size)
 				underflow = 1;
 		}
 		clock_gettime(CLOCK_MONOTONIC, &ts_end);
@@ -238,14 +266,15 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 		if (verbose)
 		fprintf(stdout,
 			"#%lu: CLOCK_MONOTONIC %ld.%09ld sec. read %ld/%ld bytes\n",
-			i, ts_end.tv_sec, ts_end.tv_nsec, rc, size);
+			i, ts_end.tv_sec, ts_end.tv_nsec, bytes_done, size);
 
 		/* file argument given? */
-		if ((out_fd >= 0) & (no_write == 0)) {
+		if (out_fd >= 0) {
 			rc = write_from_buffer(ofname, out_fd, buffer,
-					 size, i*size);
-			if (rc < 0)
+					 bytes_done, out_offset);
+			if (rc < 0 || rc < bytes_done)
 				goto out;
+			out_offset += bytes_done;
 		}
 	}
 
@@ -256,9 +285,13 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			printf("** Avg time device %s, total time %ld nsec, avg_time = %f, size = %lu, BW = %f \n",
 				devname, total_time, avg_time, size, result);
 		printf("%s ** Average BW = %lu, %f\n", devname, size, result);
-	}
-
 		rc = 0;
+	} else if (eop_flush) {
+		/* allow underflow with -e option */
+		rc = 0;
+	} else 
+		rc = -EIO;
+
 out:
 	close(fpga_fd);
 	if (out_fd >= 0)

XDMA/linux-kernel/tools/dma_to_device.c

@@ -93,6 +93,10 @@ static void usage(const char *name)
 	fprintf(stdout, "  -%c (--%s) verbose output\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+
+	fprintf(stdout, "\nReturn code:\n");
+	fprintf(stdout, "  0: all bytes were dma'ed successfully\n");
+	fprintf(stdout, "  < 0: error\n\n");
 }
 
 int main(int argc, char *argv[])
@@ -173,6 +177,8 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 {
 	uint64_t i;
 	ssize_t rc;
+	size_t bytes_done = 0;
+	size_t out_offset = 0;
 	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
@@ -229,7 +235,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 
 	if (infile_fd >= 0) {
 		rc = read_to_buffer(infname, infile_fd, buffer, size, 0);
-		if (rc < 0)
+		if (rc < 0 || rc < size)
 			goto out;
 	}
 
@@ -242,6 +248,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t len = size;
 		       	char *buf = buffer;
 
+			bytes_done = 0;
 			for (j = 0; j < apt_loop; j++, len -= aperture,
 					buf += aperture) {
 				uint64_t bytes = (len > aperture) ? aperture : len,
@@ -250,6 +257,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 				if (rc < 0)
 					goto out;
 
+				bytes_done += rc;
 				if (!underflow && rc < bytes)
 					underflow = 1;
 			}
@@ -259,7 +267,8 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			if (rc < 0)
 				goto out;
 
-			if (!underflow && rc < size)
+			bytes_done = rc;
+			if (!underflow && bytes_done < size)
 				underflow = 1;
 		}
 
@@ -275,9 +284,10 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			
 		if (outfile_fd >= 0) {
 			rc = write_from_buffer(ofname, outfile_fd, buffer,
-						 size, i * size);
-			if (rc < 0)
+						 bytes_done, out_offset);
+			if (rc < 0 || rc < bytes_done)
 				goto out;
+			out_offset += bytes_done;
 		}
 	}
 
@@ -289,7 +299,6 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			devname, total_time, avg_time, size, result);
 		printf("%s ** Average BW = %lu, %f\n", devname, size, result);
 	}
-	rc = 0;
 
 out:
 	close(fpga_fd);
@@ -299,5 +308,8 @@ out:
 		close(outfile_fd);
 	free(allocated);
 
+	if (rc < 0)
 		return rc;
+	/* treat underflow as error */
+	return underflow ? -EIO : 0;
 }

XDMA/linux-kernel/tools/dma_utils.c

@@ -47,6 +47,7 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 	uint64_t count = 0;
 	char *buf = buffer;
 	off_t offset = base;
+	int loop = 0;
 
 	while (count < size) {
 		uint64_t bytes = size - count;
@@ -67,7 +68,7 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 		/* read data from file into memory buffer */
 		rc = read(fd, buf, bytes);
 		if (rc < 0) {
-			fprintf(stderr, "%s, read 0x%lx @ 0x%lx failed %d.\n",
+			fprintf(stderr, "%s, read 0x%lx @ 0x%lx failed %ld.\n",
 				fname, bytes, offset, rc);
 			perror("read file");
 			return -EIO;
@@ -75,19 +76,19 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 
 		count += rc;
 		if (rc != bytes) {
-			fprintf(stderr, "%s, read underflow 0x%lx/0x%lx, off 0x%lx.\n",
+			fprintf(stderr, "%s, read underflow 0x%lx/0x%lx @ 0x%lx.\n",
 				fname, rc, bytes, offset);
 			break;
 		}
 
 		buf += bytes;
 		offset += bytes;
+		loop++;
 	}
 
-	if (count != size)
-		fprintf(stderr, "%s, read underflow 0x%lx != 0x%lx.\n",
+	if (count != size && loop)
+		fprintf(stderr, "%s, read underflow 0x%lx/0x%lx.\n",
 			fname, count, size);
-
 	return count;
 }
 
@@ -98,6 +99,7 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 	uint64_t count = 0;
 	char *buf = buffer;
 	off_t offset = base;
+	int loop = 0;
 
 	while (count < size) {
 		uint64_t bytes = size - count;
@@ -118,7 +120,7 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 		/* write data to file from memory buffer */
 		rc = write(fd, buf, bytes);
 		if (rc < 0) {
-			fprintf(stderr, "%s, write 0x%lx @ 0x%lx failed %d.\n",
+			fprintf(stderr, "%s, write 0x%lx @ 0x%lx failed %ld.\n",
 				fname, bytes, offset, rc);
 			perror("write file");
 			return -EIO;
@@ -126,16 +128,18 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 
 		count += rc;
 		if (rc != bytes) {
-			fprintf(stderr, "%s, write underflow 0x%lx/0x%lx, off 0x%lx.\n",
+			fprintf(stderr, "%s, write underflow 0x%lx/0x%lx @ 0x%lx.\n",
 				fname, rc, bytes, offset);
 			break;
 		}
 		buf += bytes;
 		offset += bytes;
+
+		loop++;
 	}	
 
-	if (count != size)
-		fprintf(stderr, "%s, write underflow 0x%lx != 0x%lx.\n",
+	if (count != size && loop)
+		fprintf(stderr, "%s, write underflow 0x%lx/0x%lx.\n",
 			fname, count, size);
 
 	return count;
@@ -177,4 +181,3 @@ void timespec_sub(struct timespec *t1, struct timespec *t2)
 		t1->tv_nsec += 1000000000;
 	}
 }
-

XDMA/linux-kernel/xdma/cdev_sgdma.c

@@ -485,7 +485,6 @@ static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io,
 	return -EIOCBQUEUED;
 }
 
-
 static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io,
 				unsigned long count, loff_t pos)
 {
@@ -790,6 +789,8 @@ static int char_sgdma_open(struct inode *inode, struct file *file)
 		if (engine->device_open == 1)
 			return -EBUSY;
 		engine->device_open = 1;
+
+		engine->eop_flush = (file->f_flags & O_TRUNC) ? 1 : 0;
 	}
 
 	return 0;

XDMA/linux-kernel/xdma/libxdma.h

@@ -431,6 +431,8 @@ struct sw_desc {
 };
 
 /* Describes a (SG DMA) single transfer for the engine */
+#define XFER_FLAG_NEED_UNMAP		0x1
+#define XFER_FLAG_ST_C2H_EOP_RCVED	0x2	/* ST c2h only */ 
 struct xdma_transfer {
 	struct list_head entry;		/* queue of non-completed transfers */
 	struct xdma_desc *desc_virt;	/* virt addr of the 1st descriptor */
@@ -440,6 +442,8 @@ struct xdma_transfer {
 	int desc_adjacent;		/* adjacent descriptors at desc_bus */
 	int desc_num;			/* number of descriptors in transfer */
 	int desc_index;			/* index for 1st desc. in transfer */
+	int desc_cmpl;			/* completed descriptors */
+	int desc_cmpl_th;		/* completed descriptor threshold */
 	enum dma_data_direction dir;
 #if	HAS_SWAKE_UP
 	struct swait_queue_head wq;
@@ -449,7 +453,6 @@ struct xdma_transfer {
 
 	enum transfer_state state;	/* state of the transfer */
 	unsigned int flags;
-#define XFER_FLAG_NEED_UNMAP	0x1
 	int cyclic;			/* flag if transfer is cyclic */
 	int last_in_request;		/* flag if last within request */
 	unsigned int len;
@@ -477,8 +480,6 @@ struct xdma_engine {
 	struct xdma_dev *xdev;	/* parent device */
 	char name[16];		/* name of this engine */
 	int version;		/* version of this engine */
-	//dev_t cdevno;		/* character device major:minor */
-	//struct cdev cdev;	/* character device (embedded struct) */
 
 	/* HW register address offsets */
 	struct engine_regs *regs;		/* Control reg BAR offset */
@@ -488,14 +489,17 @@ struct xdma_engine {
 	/* Engine state, configuration and flags */
 	enum shutdown_state shutdown;	/* engine shutdown mode */
 	enum dma_data_direction dir;
-	int device_open;	/* flag if engine node open, ST mode only */
-	int running;		/* flag if the driver started engine */
-	int non_incr_addr;	/* flag if non-incremental addressing used */
-	int streaming;
-	int addr_align;		/* source/dest alignment in bytes */
-	int len_granularity;	/* transfer length multiple */
-	int addr_bits;		/* HW datapath address width */
-	int channel;		/* engine indices */
+	u8 addr_align;		/* source/dest alignment in bytes */
+	u8 len_granularity;	/* transfer length multiple */
+	u8 addr_bits;		/* HW datapath address width */
+	u8 channel:2;		/* engine indices */
+	u8 streaming:1;
+	u8 device_open:1;	/* flag if engine node open, ST mode only */
+	u8 running:1;		/* flag if the driver started engine */
+	u8 non_incr_addr:1;	/* flag if non-incremental addressing used */
+	u8 eop_flush:1;		/* st c2h only, flush up the data with eop */
+	u8 filler:1;
+
 	int max_extra_adj;	/* descriptor prefetch capability */
 	int desc_dequeued;	/* num descriptors of completed transfers */
 	u32 status;		/* last known status of device */

XDMA/linux-kernel/xdma/version.h

@@ -22,7 +22,7 @@
 
 #define DRV_MOD_MAJOR		2020
 #define DRV_MOD_MINOR		1
-#define DRV_MOD_PATCHLEVEL	07
+#define DRV_MOD_PATCHLEVEL	08
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/xdma_cdev.c

@@ -619,8 +619,6 @@ int xdma_cdev_init(void)
 		return -ENOMEM;
 	}
 
-	xdma_threads_create(num_online_cpus());
-
 	return 0;
 }
 
@@ -631,6 +629,4 @@ void xdma_cdev_cleanup(void)
 
 	if (g_xdma_class)
 		class_destroy(g_xdma_class);
-
-	xdma_threads_destroy();
 }

XDMA/linux-kernel/xdma/xdma_thread.c

@@ -53,12 +53,11 @@ static int xdma_thread_cmpl_status_proc(struct list_head *work_item)
 	engine = list_entry(work_item, struct xdma_engine, cmplthp_list);
 	transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
 			entry);
-	engine_service_poll(engine, transfer->desc_num);
+	if (transfer)
+		engine_service_poll(engine, transfer->desc_cmpl_th);
 	return 0;
 }
 
-
-
 static inline int xthread_work_pending(struct xdma_kthread *thp)
 {
 	struct list_head *work_item, *next;
@@ -67,7 +66,6 @@ static inline int xthread_work_pending(struct xdma_kthread *thp)
 	if (list_empty(&thp->work_list))
 		return 0;
 
-
 	/* any work item has pending work to do? */
 	list_for_each_safe(work_item, next, &thp->work_list) {
 		if (thp->fpending && thp->fpending(work_item))
@@ -101,7 +99,6 @@ static int xthread_main(void *data)
 	if (thp->finit)
 		thp->finit(thp);
 
-
 	while (!kthread_should_stop()) {
 
 		struct list_head *work_item, *next;
@@ -150,8 +147,10 @@ int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
 	}
 
 	len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id);
-	if (len < 0)
+	if (len < 0) {
+		pr_err("thread %d, error in snprintf name %s.\n", id, name);
 		return -EINVAL;
+	}
 
 	thp->id = id;
 
@@ -180,7 +179,6 @@ int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
 	return 0;
 }
 
-
 int xdma_kthread_stop(struct xdma_kthread *thp)
 {
 	int rv;
@@ -234,8 +232,6 @@ void xdma_thread_remove_work(struct xdma_engine *engine)
 		cmpl_thread->work_cnt--;
 		unlock_thread(cmpl_thread);
 	}
-
-
 }
 
 void xdma_thread_add_work(struct xdma_engine *engine)
@@ -275,7 +271,6 @@ void xdma_thread_add_work(struct xdma_engine *engine)
 	spin_lock_irqsave(&engine->lock, flags);
 	engine->cmplthp = thp;
 	spin_unlock_irqrestore(&engine->lock, flags);
-
 }
 
 int xdma_threads_create(unsigned int num_threads)
@@ -289,12 +284,12 @@ int xdma_threads_create(unsigned int num_threads)
 		return 0;
 	}
 
-	pr_info("xdma_threads_create\n");
-
 	cs_threads = kzalloc(num_threads * sizeof(struct xdma_kthread),
 					GFP_KERNEL);
-	if (!cs_threads)
+	if (!cs_threads) {
+		pr_err("OOM, # threads %u.\n", num_threads);
 		return -ENOMEM;
+	}
 
 	/* N dma writeback monitoring threads */
 	thp = cs_threads;