XDMA v2020.1.08: allow ST c2h to end a dma transfer when EOP is received

XDMA/linux-kernel/RELEASE

+v2020.1.08
+===============
+- replaced module parameter "sgdma_timeout" to "h2c_timeout" and "c2h_timeout"
+  for H2C and C2H channels.
+  value of 0 means no timeout: wait forever for the dma completion.
+- added new "-e" option to dma_from_device
+  this is for streaming mode only, when -e is set, the driver will end the dma
+     and return the data when an EOP (end-of-packet) is received or the
+     specified bytes of data is received. 
+  without "-e" option, the driver will end the dma when the specified bytes of
+     data is received.
+- added gen4 device ids
+- fixed next adjacent descriptors when dma_alloc_coherent doesn't return a
+  page-aligned address
+
+v2020.1.06
+===============
+- added memory aperture support (-k) option in dma_from_device and dma_to_device.
+- fixed holding spinlock while doing wait_event_interruptible_xxx
+- kernel 5.0 support
+- fixed next adjacent descriptors crossing the 4K boundary
+
 Release: 2019.2
 ===============
 

XDMA/linux-kernel/tools/dma_from_device.c

@@ -40,6 +40,7 @@ static struct option const long_opts[] = {
 	{"offset", required_argument, NULL, 'o'},
 	{"count", required_argument, NULL, 'c'},
 	{"file", required_argument, NULL, 'f'},
+	{"eop_flush", no_argument, NULL, 'e'},
 	{"help", no_argument, NULL, 'h'},
 	{"verbose", no_argument, NULL, 'v'},
 	{0, 0, 0, 0}
@@ -48,7 +49,7 @@ static struct option const long_opts[] = {
 static int test_dma(char *devname, uint64_t addr, uint64_t aperture, 
 		uint64_t size, uint64_t offset, uint64_t count,
 		char *ofname);
-static int no_write = 0;
+static int eop_flush = 0;
 
 static void usage(const char *name)
 {
@@ -80,12 +81,25 @@ static void usage(const char *name)
 		"  -%c (--%s) file to write the data of the transfers\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+	fprintf(stdout,
+		 "  -%c (--%s) end dma when ST end-of-packet(eop) is rcved\n",
+		long_opts[i].val, long_opts[i].name);
+	fprintf(stdout,
+		 "\t\t* streaming only, ignored for memory-mapped channels\n");
+	fprintf(stdout,
+		 "\t\t* acutal # of bytes dma'ed could be smaller than specified\n");
+	i++;
 	fprintf(stdout, "  -%c (--%s) print usage help and exit\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
 	fprintf(stdout, "  -%c (--%s) verbose output\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+
+	fprintf(stdout, "\nReturn code:\n");
+	fprintf(stdout, "  0: all bytes were dma'ed successfully\n");
+	fprintf(stdout, "     * with -e set, the bytes dma'ed could be smaller\n");
+	fprintf(stdout, "  < 0: error\n\n");
 }
 
 int main(int argc, char *argv[])
@@ -99,7 +113,7 @@ int main(int argc, char *argv[])
 	uint64_t count = COUNT_DEFAULT;
 	char *ofname = NULL;
 
-	while ((cmd_opt = getopt_long(argc, argv, "vhxc:f:d:a:k:s:o:", long_opts,
+	while ((cmd_opt = getopt_long(argc, argv, "vhec:f:d:a:k:s:o:", long_opts,
 			    NULL)) != -1) {
 		switch (cmd_opt) {
 		case 0:
@@ -133,12 +147,12 @@ int main(int argc, char *argv[])
 			ofname = strdup(optarg);
 			break;
 			/* print usage help and exit */
-    		case 'x':
-			no_write++;
-			break;
 		case 'v':
 			verbose = 1;
 			break;
+		case 'e':
+			eop_flush = 1;
+			break;
 		case 'h':
 		default:
 			usage(argv[0]);
@@ -159,19 +173,30 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t size, uint64_t offset, uint64_t count,
 			char *ofname)
 {
-	ssize_t rc;
+	ssize_t rc = 0;
+	size_t out_offset = 0;
+	size_t bytes_done = 0;
 	uint64_t i;
 	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
 	struct timespec ts_start, ts_end;
 	int out_fd = -1;
-	int fpga_fd = open(devname, O_RDWR | O_NONBLOCK);
+	int fpga_fd;
 	long total_time = 0;
 	float result;
 	float avg_time = 0;
 	int underflow = 0;
 
+	/*
+	 * use O_TRUNC to indicate to the driver to flush the data up based on
+	 * EOP (end-of-packet), streaming mode only
+	 */
+	if (eop_flush)
+		fpga_fd = open(devname, O_RDWR | O_TRUNC);
+	else
+		fpga_fd = open(devname, O_RDWR);
+
 	if (fpga_fd < 0) {
                 fprintf(stderr, "unable to open device %s, %d.\n",
                         devname, fpga_fd);
@@ -210,6 +235,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t len = size;
 			char *buf = buffer;
 
+			bytes_done = 0;
 			for (j = 0; j < apt_loop; j++, len -= aperture, buf += aperture) {
 				uint64_t bytes = (len > aperture) ? aperture : len,
 				rc = read_to_buffer(devname, fpga_fd, buf,
@@ -219,13 +245,15 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 
 				if (!underflow && rc < bytes)
 					underflow = 1;
+				bytes_done += rc;
 			}
 		} else {
 			rc = read_to_buffer(devname, fpga_fd, buffer, size, addr);
 			if (rc < 0)
 				goto out;
+			bytes_done = rc;
 
-			if (!underflow && rc < size)
+			if (!underflow && bytes_done < size)
 				underflow = 1;
 		}
 		clock_gettime(CLOCK_MONOTONIC, &ts_end);
@@ -238,14 +266,15 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 		if (verbose)
 		fprintf(stdout,
 			"#%lu: CLOCK_MONOTONIC %ld.%09ld sec. read %ld/%ld bytes\n",
-			i, ts_end.tv_sec, ts_end.tv_nsec, rc, size);
+			i, ts_end.tv_sec, ts_end.tv_nsec, bytes_done, size);
 
 		/* file argument given? */
-		if ((out_fd >= 0) & (no_write == 0)) {
+		if (out_fd >= 0) {
 			rc = write_from_buffer(ofname, out_fd, buffer,
-					 size, i*size);
-			if (rc < 0)
+					 bytes_done, out_offset);
+			if (rc < 0 || rc < bytes_done)
 				goto out;
+			out_offset += bytes_done;
 		}
 	}
 
@@ -256,9 +285,13 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			printf("** Avg time device %s, total time %ld nsec, avg_time = %f, size = %lu, BW = %f \n",
 				devname, total_time, avg_time, size, result);
 		printf("%s ** Average BW = %lu, %f\n", devname, size, result);
-	}
-
 		rc = 0;
+	} else if (eop_flush) {
+		/* allow underflow with -e option */
+		rc = 0;
+	} else 
+		rc = -EIO;
+
 out:
 	close(fpga_fd);
 	if (out_fd >= 0)

XDMA/linux-kernel/tools/dma_to_device.c

@@ -93,6 +93,10 @@ static void usage(const char *name)
 	fprintf(stdout, "  -%c (--%s) verbose output\n",
 		long_opts[i].val, long_opts[i].name);
 	i++;
+
+	fprintf(stdout, "\nReturn code:\n");
+	fprintf(stdout, "  0: all bytes were dma'ed successfully\n");
+	fprintf(stdout, "  < 0: error\n\n");
 }
 
 int main(int argc, char *argv[])
@@ -173,6 +177,8 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 {
 	uint64_t i;
 	ssize_t rc;
+	size_t bytes_done = 0;
+	size_t out_offset = 0;
 	uint64_t apt_loop = aperture ? (size + aperture - 1) / aperture : 0;
 	char *buffer = NULL;
 	char *allocated = NULL;
@@ -229,7 +235,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 
 	if (infile_fd >= 0) {
 		rc = read_to_buffer(infname, infile_fd, buffer, size, 0);
-		if (rc < 0)
+		if (rc < 0 || rc < size)
 			goto out;
 	}
 
@@ -242,6 +248,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			uint64_t len = size;
 		       	char *buf = buffer;
 
+			bytes_done = 0;
 			for (j = 0; j < apt_loop; j++, len -= aperture,
 					buf += aperture) {
 				uint64_t bytes = (len > aperture) ? aperture : len,
@@ -250,6 +257,7 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 				if (rc < 0)
 					goto out;
 
+				bytes_done += rc;
 				if (!underflow && rc < bytes)
 					underflow = 1;
 			}
@@ -259,7 +267,8 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			if (rc < 0)
 				goto out;
 
-			if (!underflow && rc < size)
+			bytes_done = rc;
+			if (!underflow && bytes_done < size)
 				underflow = 1;
 		}
 
@@ -275,9 +284,10 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			
 		if (outfile_fd >= 0) {
 			rc = write_from_buffer(ofname, outfile_fd, buffer,
-						 size, i * size);
-			if (rc < 0)
+						 bytes_done, out_offset);
+			if (rc < 0 || rc < bytes_done)
 				goto out;
+			out_offset += bytes_done;
 		}
 	}
 
@@ -289,7 +299,6 @@ static int test_dma(char *devname, uint64_t addr, uint64_t aperture,
 			devname, total_time, avg_time, size, result);
 		printf("%s ** Average BW = %lu, %f\n", devname, size, result);
 	}
-	rc = 0;
 
 out:
 	close(fpga_fd);
@@ -299,5 +308,8 @@ out:
 		close(outfile_fd);
 	free(allocated);
 
+	if (rc < 0)
 		return rc;
+	/* treat underflow as error */
+	return underflow ? -EIO : 0;
 }

XDMA/linux-kernel/tools/dma_utils.c

@@ -47,6 +47,7 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 	uint64_t count = 0;
 	char *buf = buffer;
 	off_t offset = base;
+	int loop = 0;
 
 	while (count < size) {
 		uint64_t bytes = size - count;
@@ -67,7 +68,7 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 		/* read data from file into memory buffer */
 		rc = read(fd, buf, bytes);
 		if (rc < 0) {
-			fprintf(stderr, "%s, read 0x%lx @ 0x%lx failed %d.\n",
+			fprintf(stderr, "%s, read 0x%lx @ 0x%lx failed %ld.\n",
 				fname, bytes, offset, rc);
 			perror("read file");
 			return -EIO;
@@ -75,19 +76,19 @@ ssize_t read_to_buffer(char *fname, int fd, char *buffer, uint64_t size,
 
 		count += rc;
 		if (rc != bytes) {
-			fprintf(stderr, "%s, read underflow 0x%lx/0x%lx, off 0x%lx.\n",
+			fprintf(stderr, "%s, read underflow 0x%lx/0x%lx @ 0x%lx.\n",
 				fname, rc, bytes, offset);
 			break;
 		}
 
 		buf += bytes;
 		offset += bytes;
+		loop++;
 	}
 
-	if (count != size)
-		fprintf(stderr, "%s, read underflow 0x%lx != 0x%lx.\n",
+	if (count != size && loop)
+		fprintf(stderr, "%s, read underflow 0x%lx/0x%lx.\n",
 			fname, count, size);
-
 	return count;
 }
 
@@ -98,6 +99,7 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 	uint64_t count = 0;
 	char *buf = buffer;
 	off_t offset = base;
+	int loop = 0;
 
 	while (count < size) {
 		uint64_t bytes = size - count;
@@ -118,7 +120,7 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 		/* write data to file from memory buffer */
 		rc = write(fd, buf, bytes);
 		if (rc < 0) {
-			fprintf(stderr, "%s, write 0x%lx @ 0x%lx failed %d.\n",
+			fprintf(stderr, "%s, write 0x%lx @ 0x%lx failed %ld.\n",
 				fname, bytes, offset, rc);
 			perror("write file");
 			return -EIO;
@@ -126,16 +128,18 @@ ssize_t write_from_buffer(char *fname, int fd, char *buffer, uint64_t size,
 
 		count += rc;
 		if (rc != bytes) {
-			fprintf(stderr, "%s, write underflow 0x%lx/0x%lx, off 0x%lx.\n",
+			fprintf(stderr, "%s, write underflow 0x%lx/0x%lx @ 0x%lx.\n",
 				fname, rc, bytes, offset);
 			break;
 		}
 		buf += bytes;
 		offset += bytes;
+
+		loop++;
 	}	
 
-	if (count != size)
-		fprintf(stderr, "%s, write underflow 0x%lx != 0x%lx.\n",
+	if (count != size && loop)
+		fprintf(stderr, "%s, write underflow 0x%lx/0x%lx.\n",
 			fname, count, size);
 
 	return count;
@@ -177,4 +181,3 @@ void timespec_sub(struct timespec *t1, struct timespec *t2)
 		t1->tv_nsec += 1000000000;
 	}
 }
-

XDMA/linux-kernel/xdma/cdev_sgdma.c

@@ -485,7 +485,6 @@ static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io,
 	return -EIOCBQUEUED;
 }
 
-
 static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io,
 				unsigned long count, loff_t pos)
 {
@@ -790,6 +789,8 @@ static int char_sgdma_open(struct inode *inode, struct file *file)
 		if (engine->device_open == 1)
 			return -EBUSY;
 		engine->device_open = 1;
+
+		engine->eop_flush = (file->f_flags & O_TRUNC) ? 1 : 0;
 	}
 
 	return 0;

XDMA/linux-kernel/xdma/libxdma.c

@@ -95,12 +95,20 @@ static DEFINE_SPINLOCK(xdev_rcu_lock);
 #define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member)
 #endif
 
-static inline void xdev_list_add(struct xdma_dev *xdev)
+static inline int xdev_list_add(struct xdma_dev *xdev)
 {
 	mutex_lock(&xdev_mutex);
-	if (list_empty(&xdev_list))
+	if (list_empty(&xdev_list)) {
 		xdev->idx = 0;
-	else {
+		if (poll_mode) {
+			int rv = xdma_threads_create(xdev->h2c_channel_max +
+					xdev->c2h_channel_max);
+			if (rv < 0) {
+				mutex_unlock(&xdev_mutex);
+				return rv;
+			}
+		}
+	} else {
 		struct xdma_dev *last;
 
 		last = list_last_entry(&xdev_list, struct xdma_dev, list_head);
@@ -115,6 +123,8 @@ static inline void xdev_list_add(struct xdma_dev *xdev)
 	spin_lock(&xdev_rcu_lock);
 	list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list);
 	spin_unlock(&xdev_rcu_lock);
+
+	return 0;
 }
 
 #undef list_last_entry
@@ -123,6 +133,8 @@ static inline void xdev_list_remove(struct xdma_dev *xdev)
 {
 	mutex_lock(&xdev_mutex);
 	list_del(&xdev->list_head);
+	if (poll_mode && list_empty(&xdev_list))
+		xdma_threads_destroy();
 	mutex_unlock(&xdev_mutex);
 
 	spin_lock(&xdev_rcu_lock);
@@ -526,6 +538,10 @@ static int xdma_engine_stop(struct xdma_engine *engine)
 	}
 	dbg_tfr("%s(engine=%p)\n", __func__, engine);
 
+	if (enable_credit_mp && engine->streaming &&
+	    engine->dir == DMA_FROM_DEVICE)
+		write_register(0, &engine->sgdma_regs->credits, 0);
+
 	w = 0;
 	w |= (u32)XDMA_CTRL_IE_DESC_ALIGN_MISMATCH;
 	w |= (u32)XDMA_CTRL_IE_MAGIC_STOPPED;
@@ -871,7 +887,8 @@ static int engine_err_handle(struct xdma_engine *engine,
 	if (engine->status & XDMA_STAT_BUSY) {
 		value = read_register(&engine->regs->status);
 		if ((value & XDMA_STAT_BUSY))
-			printk_ratelimited(KERN_INFO "%s has errors but is still BUSY\n",
+			printk_ratelimited(KERN_INFO
+					"%s has errors but is still BUSY\n",
 					engine->name);
 	}
 
@@ -908,6 +925,7 @@ engine_service_final_transfer(struct xdma_engine *engine,
 			*pdesc_completed);
 		return NULL;
 	}
+
 	if (((engine->dir == DMA_FROM_DEVICE) &&
 	     (engine->status & XDMA_STAT_C2H_ERR_MASK)) ||
 	    ((engine->dir == DMA_TO_DEVICE) &&
@@ -925,10 +943,42 @@ engine_service_final_transfer(struct xdma_engine *engine,
 
 	/* the engine stopped on current transfer? */
 	if (*pdesc_completed < transfer->desc_num) {
+		if (engine->eop_flush) {
+			/* check if eop received */
+			struct xdma_result *result = transfer->res_virt;
+			int i;
+			int max = *pdesc_completed;
+
+			for (i = 0; i < max; i++) {
+				if ((result[i].status & RX_STATUS_EOP) != 0) {
+					transfer->flags |=
+						XFER_FLAG_ST_C2H_EOP_RCVED;
+					break;
+				}
+			}
+
+			transfer->desc_cmpl += *pdesc_completed;
+			if (!(transfer->flags & XFER_FLAG_ST_C2H_EOP_RCVED)) {
+				return NULL;
+			}
+
+			/* mark transfer as successfully completed */
+			engine_service_shutdown(engine);
+
+			transfer->state = TRANSFER_STATE_COMPLETED;
+
+			engine->desc_dequeued += transfer->desc_cmpl;
+
+		} else {
 			transfer->state = TRANSFER_STATE_FAILED;
 			pr_info("%s, xfer 0x%p, stopped half-way, %d/%d.\n",
 				engine->name, transfer, *pdesc_completed,
 				transfer->desc_num);
+
+			/* add dequeued number of descriptors during this run */
+			engine->desc_dequeued += transfer->desc_num;
+			transfer->desc_cmpl = *pdesc_completed;
+		}
 	} else {
 		dbg_tfr("engine %s completed transfer\n", engine->name);
 		dbg_tfr("Completed transfer ID = 0x%p\n", transfer);
@@ -944,13 +994,14 @@ engine_service_final_transfer(struct xdma_engine *engine,
 		}
 		/* mark transfer as successfully completed */
 		transfer->state = TRANSFER_STATE_COMPLETED;
+		transfer->desc_cmpl = transfer->desc_num;
+		/* add dequeued number of descriptors during this run */
+		engine->desc_dequeued += transfer->desc_num;
 	}
 
 transfer_del:
 	/* remove completed transfer from list */
 	list_del(engine->transfer_list.next);
-	/* add to dequeued number of descriptors during this run */
-	engine->desc_dequeued += transfer->desc_num;
 
 	/*
 	 * Complete transfer - sets transfer to NULL if an asynchronous
@@ -1021,13 +1072,8 @@ static int engine_service_resume(struct xdma_engine *engine)
 			dbg_tfr("no pending transfers, %s engine stays idle.\n",
 				engine->name);
 		}
-	} else {
-		/* engine is still running? */
-		if (list_empty(&engine->transfer_list)) {
-			pr_warn("no queued transfers but %s engine running!\n",
-				engine->name);
-			WARN_ON(1);
-		}
+	} else if (list_empty(&engine->transfer_list)) {
+		engine_service_shutdown(engine);
 	}
 	return 0;
 }
@@ -1046,23 +1092,18 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback)
 	u32 desc_count = desc_writeback & WB_COUNT_MASK;
 	u32 err_flag = desc_writeback & WB_ERR_MASK;
 	int rv = 0;
-	struct xdma_poll_wb *wb_data;
 
 	if (!engine) {
 		pr_err("dma engine NULL\n");
 		return -EINVAL;
 	}
 
-	/* If polling detected an error, signal to the caller */
-	if (err_flag)
-		rv = -1;
-
 	/* Service the engine */
 	if (!engine->running) {
 		dbg_tfr("Engine was not running!!! Clearing status\n");
 		rv = engine_status_read(engine, 1, 0);
 		if (rv < 0) {
-			pr_err("Failed to read engine status\n");
+			pr_err("%s failed to read status\n", engine->name);
 			return rv;
 		}
 		return 0;
@@ -1086,13 +1127,14 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback)
 	 * shut down
 	 */
 	if ((engine->running && !(engine->status & XDMA_STAT_BUSY)) ||
-	    (desc_count != 0)) {
+	    (!engine->eop_flush && desc_count != 0)) {
 		rv = engine_service_shutdown(engine);
 		if (rv < 0) {
 			pr_err("Failed to shutdown engine\n");
 			return rv;
 		}
 	}
+
 	/*
 	 * If called from the ISR, or if an error occurred, the descriptor
 	 * count will be zero.  In this scenario, read the descriptor count
@@ -1101,7 +1143,12 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback)
 	 */
 	if (!desc_count)
 		desc_count = read_register(&engine->regs->completed_desc_count);
-	dbg_tfr("desc_count = %d\n", desc_count);
+	dbg_tfr("%s wb 0x%x, desc_count %u, err %u, dequeued %u.\n",
+		engine->name, desc_writeback, desc_count, err_flag,
+		engine->desc_dequeued);
+
+	if (!desc_count)
+		goto done;
 
 	/* transfers on queue? */
 	if (!list_empty(&engine->transfer_list)) {
@@ -1135,18 +1182,16 @@ static int engine_service(struct xdma_engine *engine, int desc_writeback)
 	 */
 	transfer = engine_service_final_transfer(engine, transfer, &desc_count);
 
-	/* Before starting engine again, clear the writeback data */
-	if (poll_mode) {
-		wb_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt;
-		wb_data->completed_desc_count = 0;
-	}
-
 	/* Restart the engine following the servicing */
+	if (!engine->eop_flush) {
 		rv = engine_service_resume(engine);
 		if (rv < 0)
 			pr_err("Failed to resume engine\n");
+	}
 
-	return rv;
+done:
+	/* If polling detected an error, signal to the caller */
+	return err_flag ? -1 : 0;
 }
 
 /* engine_service_work */
@@ -1214,6 +1259,9 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine,
 	while (expected_wb != 0) {
 		desc_wb = wb_data->completed_desc_count;
 
+		if (desc_wb)
+			wb_data->completed_desc_count = 0;
+
 		if (desc_wb & WB_ERR_MASK)
 			break;
 		else if (desc_wb >= expected_wb)
@@ -1248,7 +1296,6 @@ static u32 engine_service_wb_monitor(struct xdma_engine *engine,
 int engine_service_poll(struct xdma_engine *engine,
 			       u32 expected_desc_count)
 {
-	struct xdma_poll_wb *writeback_data;
 	u32 desc_wb = 0;
 	unsigned long flags;
 	int rv = 0;
@@ -1264,13 +1311,6 @@ int engine_service_poll(struct xdma_engine *engine,
 		return -EINVAL;
 	}
 
-	writeback_data = (struct xdma_poll_wb *)engine->poll_mode_addr_virt;
-
-	if ((expected_desc_count & WB_COUNT_MASK) != expected_desc_count) {
-		dbg_tfr("Queued descriptor count is larger than supported\n");
-		return -1;
-	}
-
 	/*
 	 * Poll the writeback location for the expected number of
 	 * descriptors / error events This loop is skipped for cyclic mode,
@@ -1279,6 +1319,8 @@ int engine_service_poll(struct xdma_engine *engine,
 	 */
 
 	desc_wb = engine_service_wb_monitor(engine, expected_desc_count);
+	if (!desc_wb)
+		return 0;
 
 	spin_lock_irqsave(&engine->lock, flags);
 	dbg_tfr("%s service.\n", engine->name);
@@ -2957,6 +2999,14 @@ static int transfer_init(struct xdma_engine *engine,
 	control |= XDMA_DESC_COMPLETED;
 	xdma_desc_control_set(xfer->desc_virt + last, control);
 
+	if (engine->eop_flush) {
+		for (i = 0; i < last; i++)
+			xdma_desc_control_set(xfer->desc_virt + i,
+					XDMA_DESC_COMPLETED);
+		xfer->desc_cmpl_th = 1;
+	} else
+		xfer->desc_cmpl_th = desc_max;
+
 	xfer->desc_num = desc_max;
 	engine->desc_idx = (engine->desc_idx + desc_max) %
 					XDMA_TRANSFER_MAX_DESC;
@@ -3101,7 +3151,6 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 	int nents;
 	enum dma_data_direction dir = write ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
 	struct xdma_request_cb *req = NULL;
-	struct xdma_result *result;
 
 	if (!dev_hndl)
 		return -EINVAL;
@@ -3213,26 +3262,9 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 			goto unmap_sgl;
 		}
 
-		/*
-		 * When polling, determine how many descriptors have been queued
-		 * on the engine to determine the writeback value expected
-		 */
-		if (poll_mode) {
-			unsigned int desc_count;
+		if (engine->cmplthp)
+			xdma_kthread_wakeup(engine->cmplthp);
 
-			spin_lock_irqsave(&engine->lock, flags);
-			desc_count = xfer->desc_num;
-			spin_unlock_irqrestore(&engine->lock, flags);
-			dbg_tfr("%s poll desc_count=%d\n", engine->name,
-				desc_count);
-			rv = engine_service_poll(engine, desc_count);
-			if (rv < 0) {
-				mutex_unlock(&engine->desc_lock);
-				pr_err("Failed to service polling\n");
-				goto unmap_sgl;
-			}
-
-		} else {
 		if (timeout_ms > 0)
 			xlx_wait_event_interruptible_timeout(xfer->wq,
 				(xfer->state != TRANSFER_STATE_SUBMITTED),
@@ -3240,7 +3272,6 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 		else
 			xlx_wait_event_interruptible(xfer->wq,
 				(xfer->state != TRANSFER_STATE_SUBMITTED));
-		}
 
 		spin_lock_irqsave(&engine->lock, flags);
 
@@ -3248,8 +3279,7 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 		case TRANSFER_STATE_COMPLETED:
 			spin_unlock_irqrestore(&engine->lock, flags);
 
-			result = xfer->res_virt;
-
+			rv = 0;
 			dbg_tfr("transfer %p, %u, ep 0x%llx compl, +%lu.\n",
 				xfer, xfer->len, req->ep_addr - xfer->len,
 				done);
@@ -3257,12 +3287,17 @@ ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
 			/* For C2H streaming use writeback results */
 			if (engine->streaming &&
 			    engine->dir == DMA_FROM_DEVICE) {
-				for (i = 0; i < xfer->desc_num; i++)
+				struct xdma_result *result = xfer->res_virt;
+
+				for (i = 0; i < xfer->desc_cmpl; i++)
 					done += result[i].length;
+
+				/* finish the whole request */
+				if (engine->eop_flush)
+					nents = 0;
 			} else
 				done += xfer->len;
 
-			rv = 0;
 			break;
 		case TRANSFER_STATE_FAILED:
 			pr_info("xfer 0x%p,%u, failed, ep 0x%llx.\n", xfer,
@@ -4053,7 +4088,6 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max,
 	xdev->c2h_channel_max = *c2h_channel_max;
 
 	xdma_device_flag_set(xdev, XDEV_FLAG_OFFLINE);
-	xdev_list_add(xdev);
 
 	if (xdev->user_max == 0 || xdev->user_max > MAX_USER_IRQ)
 		xdev->user_max = MAX_USER_IRQ;
@@ -4064,6 +4098,10 @@ void *xdma_device_open(const char *mname, struct pci_dev *pdev, int *user_max,
 	    xdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX)
 		xdev->c2h_channel_max = XDMA_CHANNEL_NUM_MAX;
 
+	rv = xdev_list_add(xdev);
+	if (rv < 0)
+		goto free_xdev;
+
 	rv = pci_enable_device(pdev);
 	if (rv) {
 		dbg_init("pci_enable_device() failed, %d.\n", rv);
@@ -4145,6 +4183,7 @@ err_regions:
 		pci_disable_device(pdev);
 err_enable:
 	xdev_list_remove(xdev);
+free_xdev:
 	kfree(xdev);
 	return NULL;
 }

XDMA/linux-kernel/xdma/libxdma.h

@@ -431,6 +431,8 @@ struct sw_desc {
 };
 
 /* Describes a (SG DMA) single transfer for the engine */
+#define XFER_FLAG_NEED_UNMAP		0x1
+#define XFER_FLAG_ST_C2H_EOP_RCVED	0x2	/* ST c2h only */ 
 struct xdma_transfer {
 	struct list_head entry;		/* queue of non-completed transfers */
 	struct xdma_desc *desc_virt;	/* virt addr of the 1st descriptor */
@@ -440,6 +442,8 @@ struct xdma_transfer {
 	int desc_adjacent;		/* adjacent descriptors at desc_bus */
 	int desc_num;			/* number of descriptors in transfer */
 	int desc_index;			/* index for 1st desc. in transfer */
+	int desc_cmpl;			/* completed descriptors */
+	int desc_cmpl_th;		/* completed descriptor threshold */
 	enum dma_data_direction dir;
 #if	HAS_SWAKE_UP
 	struct swait_queue_head wq;
@@ -449,7 +453,6 @@ struct xdma_transfer {
 
 	enum transfer_state state;	/* state of the transfer */
 	unsigned int flags;
-#define XFER_FLAG_NEED_UNMAP	0x1
 	int cyclic;			/* flag if transfer is cyclic */
 	int last_in_request;		/* flag if last within request */
 	unsigned int len;
@@ -477,8 +480,6 @@ struct xdma_engine {
 	struct xdma_dev *xdev;	/* parent device */
 	char name[16];		/* name of this engine */
 	int version;		/* version of this engine */
-	//dev_t cdevno;		/* character device major:minor */
-	//struct cdev cdev;	/* character device (embedded struct) */
 
 	/* HW register address offsets */
 	struct engine_regs *regs;		/* Control reg BAR offset */
@@ -488,14 +489,17 @@ struct xdma_engine {
 	/* Engine state, configuration and flags */
 	enum shutdown_state shutdown;	/* engine shutdown mode */
 	enum dma_data_direction dir;
-	int device_open;	/* flag if engine node open, ST mode only */
-	int running;		/* flag if the driver started engine */
-	int non_incr_addr;	/* flag if non-incremental addressing used */
-	int streaming;
-	int addr_align;		/* source/dest alignment in bytes */
-	int len_granularity;	/* transfer length multiple */
-	int addr_bits;		/* HW datapath address width */
-	int channel;		/* engine indices */
+	u8 addr_align;		/* source/dest alignment in bytes */
+	u8 len_granularity;	/* transfer length multiple */
+	u8 addr_bits;		/* HW datapath address width */
+	u8 channel:2;		/* engine indices */
+	u8 streaming:1;
+	u8 device_open:1;	/* flag if engine node open, ST mode only */
+	u8 running:1;		/* flag if the driver started engine */
+	u8 non_incr_addr:1;	/* flag if non-incremental addressing used */
+	u8 eop_flush:1;		/* st c2h only, flush up the data with eop */
+	u8 filler:1;
+
 	int max_extra_adj;	/* descriptor prefetch capability */
 	int desc_dequeued;	/* num descriptors of completed transfers */
 	u32 status;		/* last known status of device */

XDMA/linux-kernel/xdma/version.h

@@ -22,7 +22,7 @@
 
 #define DRV_MOD_MAJOR		2020
 #define DRV_MOD_MINOR		1
-#define DRV_MOD_PATCHLEVEL	07
+#define DRV_MOD_PATCHLEVEL	08
 
 #define DRV_MODULE_VERSION      \
 	__stringify(DRV_MOD_MAJOR) "." \

XDMA/linux-kernel/xdma/xdma_cdev.c

@@ -619,8 +619,6 @@ int xdma_cdev_init(void)
 		return -ENOMEM;
 	}
 
-	xdma_threads_create(num_online_cpus());
-
 	return 0;
 }
 
@@ -631,6 +629,4 @@ void xdma_cdev_cleanup(void)
 
 	if (g_xdma_class)
 		class_destroy(g_xdma_class);
-
-	xdma_threads_destroy();
 }

XDMA/linux-kernel/xdma/xdma_thread.c

@@ -53,12 +53,11 @@ static int xdma_thread_cmpl_status_proc(struct list_head *work_item)
 	engine = list_entry(work_item, struct xdma_engine, cmplthp_list);
 	transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
 			entry);
-	engine_service_poll(engine, transfer->desc_num);
+	if (transfer)
+		engine_service_poll(engine, transfer->desc_cmpl_th);
 	return 0;
 }
 
-
-
 static inline int xthread_work_pending(struct xdma_kthread *thp)
 {
 	struct list_head *work_item, *next;
@@ -67,7 +66,6 @@ static inline int xthread_work_pending(struct xdma_kthread *thp)
 	if (list_empty(&thp->work_list))
 		return 0;
 
-
 	/* any work item has pending work to do? */
 	list_for_each_safe(work_item, next, &thp->work_list) {
 		if (thp->fpending && thp->fpending(work_item))
@@ -101,7 +99,6 @@ static int xthread_main(void *data)
 	if (thp->finit)
 		thp->finit(thp);
 
-
 	while (!kthread_should_stop()) {
 
 		struct list_head *work_item, *next;
@@ -150,8 +147,10 @@ int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
 	}
 
 	len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id);
-	if (len < 0)
+	if (len < 0) {
+		pr_err("thread %d, error in snprintf name %s.\n", id, name);
 		return -EINVAL;
+	}
 
 	thp->id = id;
 
@@ -180,7 +179,6 @@ int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
 	return 0;
 }
 
-
 int xdma_kthread_stop(struct xdma_kthread *thp)
 {
 	int rv;
@@ -234,8 +232,6 @@ void xdma_thread_remove_work(struct xdma_engine *engine)
 		cmpl_thread->work_cnt--;
 		unlock_thread(cmpl_thread);
 	}
-
-
 }
 
 void xdma_thread_add_work(struct xdma_engine *engine)
@@ -275,7 +271,6 @@ void xdma_thread_add_work(struct xdma_engine *engine)
 	spin_lock_irqsave(&engine->lock, flags);
 	engine->cmplthp = thp;
 	spin_unlock_irqrestore(&engine->lock, flags);
-
 }
 
 int xdma_threads_create(unsigned int num_threads)
@@ -289,12 +284,12 @@ int xdma_threads_create(unsigned int num_threads)
 		return 0;
 	}
 
-	pr_info("xdma_threads_create\n");
-
 	cs_threads = kzalloc(num_threads * sizeof(struct xdma_kthread),
 					GFP_KERNEL);
-	if (!cs_threads)
+	if (!cs_threads) {
+		pr_err("OOM, # threads %u.\n", num_threads);
 		return -ENOMEM;
+	}
 
 	/* N dma writeback monitoring threads */
 	thp = cs_threads;