Skip to content

D
dma_ip_drivers
Commit a0df2803 authored by sujathabanoth-xlnx's avatar sujathabanoth-xlnx
Browse files
Options

Merge branch 'master' of https://github.com/Xilinx/dma_ip_drivers

parents c0fcc39e e316ccf3
Show whitespace changes
Inline Side-by-side
Showing with 3089 additions and 1186 deletions
README.md
View file @ a0df2803
......@@ -8,9 +8,7 @@ Both the linux kernel driver and the DPDK driver can be run on a PCI Express roo
### Getting Started
* [QDMA DPDK Reference Driver User Guide](http://htmlpreview.github.io/?https://github.com/Xilinx/dma_ip_drivers/blob/master/QDMA/DPDK/docs/git_doc/html/index.html)
* [QDMA Linux Kernel Reference Driver User Guide](http://htmlpreview.github.io/?https://github.com/Xilinx/dma_ip_drivers/blob/master/QDMA/linux-kernel/docs/git_doc/html/index.html)
* [QDMA Reference Drivers Comprehensive documentation](https://xilinx.github.io/dma_ip_drivers/)
## Xilinx-VSEC (XVSEC)
......
XDMA/linux-kernel/RELEASE 0 → 100644
View file @ a0df2803
Release: 2019.2
===============
Change list:
- Updated data rate for performance run. Now it will print data rate based on the size.
- remove BUG_ON, return proper error code instead
- Streaming mode: enable credit mechanism by default
- Streaming mode: Do not read more than user supplied buffer size on C2H
- Streaming mode: Added support for Async-IO for both streaming and MM transfers
- fixed performance appliaction crash
XDMA/linux-kernel/include/libxdma_api.h
View file @ a0df2803
......@@ -133,6 +133,14 @@ int xdma_user_isr_disable(void *dev_hndl, unsigned int mask);
ssize_t xdma_xfer_submit(void *dev_hndl, int channel, bool write, u64 ep_addr,
struct sg_table *sgt, bool dma_mapped, int timeout_ms);
ssize_t xdma_xfer_submit_nowait(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
struct sg_table *sgt, bool dma_mapped, int timeout_ms);
ssize_t xdma_xfer_completion(void *cb_hndl, void *dev_hndl, int channel, bool write, u64 ep_addr,
struct sg_table *sgt, bool dma_mapped, int timeout_ms);
/////////////////////missing API////////////////////
......
XDMA/linux-kernel/libxdma/libxdma.c
View file @ a0df2803
This source diff could not be displayed because it is too large. You can view the blob instead.
XDMA/linux-kernel/libxdma/libxdma.h
View file @ a0df2803
......@@ -31,7 +31,7 @@
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/workqueue.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
#include <linux/swait.h>
#endif
/*
......@@ -83,6 +83,7 @@
#define XDMA_CTRL_IE_DESC_ERROR (0x1FUL << 19)
#define XDMA_CTRL_NON_INCR_ADDR (1UL << 25)
#define XDMA_CTRL_POLL_MODE_WB (1UL << 26)
#define XDMA_CTRL_STM_MODE_WB (1UL << 27)
/* bits of the SG DMA status register */
#define XDMA_STAT_BUSY (1UL << 0)
......@@ -247,6 +248,23 @@ enum dev_capabilities {
/* SECTION: Structure definitions */
struct xdma_io_cb {
void __user *buf;
size_t len;
void *private;
unsigned int pages_nr;
struct sg_table sgt;
struct page **pages;
/** total data size */
unsigned int count;
/** MM only, DDR/BRAM memory addr */
u64 ep_addr;
/** write: if write to the device */
struct xdma_request_cb *req;
u8 write:1;
void (*io_done)(unsigned long cb_hndl, int err);
};
struct config_regs {
u32 identifier;
u32 reserved_1[4];
......@@ -392,11 +410,14 @@ struct sw_desc {
struct xdma_transfer {
struct list_head entry; /* queue of non-completed transfers */
struct xdma_desc *desc_virt; /* virt addr of the 1st descriptor */
struct xdma_result *res_virt; /* virt addr of result for c2h streaming */
dma_addr_t res_bus; /* bus addr for result descriptors */
dma_addr_t desc_bus; /* bus addr of the first descriptor */
int desc_adjacent; /* adjacent descriptors at desc_bus */
int desc_num; /* number of descriptors in transfer */
int desc_index; /* index for first descriptor in transfer */
enum dma_data_direction dir;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
struct swait_queue_head wq;
#else
wait_queue_head_t wq; /* wait queue for transfer completion */
......@@ -409,6 +430,7 @@ struct xdma_transfer {
int last_in_request; /* flag if last within request */
unsigned int len;
struct sg_table *sgt;
struct xdma_io_cb *cb;
};
struct xdma_request_cb {
......@@ -416,7 +438,8 @@ struct xdma_request_cb {
unsigned int total_len;
u64 ep_addr;
struct xdma_transfer xfer;
struct xdma_transfer tfer[2]; /* Use two transfers in case single request needs to be split */
struct xdma_io_cb *cb;
unsigned int sw_desc_idx;
unsigned int sw_desc_cnt;
......@@ -450,7 +473,8 @@ struct xdma_engine {
int max_extra_adj; /* descriptor prefetch capability */
int desc_dequeued; /* num descriptors of completed transfers */
u32 status; /* last known status of device */
u32 interrupt_enable_mask_value;/* only used for MSIX mode to store per-engine interrupt mask value */
/* only used for MSIX mode to store per-engine interrupt mask value */
u32 interrupt_enable_mask_value;
/* Transfer list management */
struct list_head transfer_list; /* queue of transfers */
......@@ -460,6 +484,8 @@ struct xdma_engine {
dma_addr_t cyclic_result_bus; /* bus addr for transfer */
struct xdma_request_cb *cyclic_req;
struct sg_table cyclic_sgt;
u8 *perf_buf_virt;
dma_addr_t perf_buf_bus; /* bus address */
u8 eop_found; /* used only for cyclic(rx:c2h) */
int eop_count;
int rx_tail; /* follows the HW */
......@@ -474,7 +500,7 @@ struct xdma_engine {
dma_addr_t poll_mode_bus; /* bus addr for descriptor writeback */
/* Members associated with interrupt mode support */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
struct swait_queue_head shutdown_wq;
#else
wait_queue_head_t shutdown_wq; /* wait queue for shutdown sync */
......@@ -488,14 +514,23 @@ struct xdma_engine {
spinlock_t desc_lock; /* protects concurrent access */
dma_addr_t desc_bus;
struct xdma_desc *desc;
int desc_idx; /* current descriptor index */
int desc_used; /* total descriptors used */
/* for performance test support */
struct xdma_performance_ioctl *xdma_perf; /* perf test control */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
struct swait_queue_head xdma_perf_wq;
#else
wait_queue_head_t xdma_perf_wq; /* Perf test sync */
#endif
struct xdma_kthread *cmplthp;
/* completion status thread list for the queue */
struct list_head cmplthp_list;
/* pending work thread list */
/* cpu attached to intr_work */
unsigned int intr_work_cpu;
};
struct xdma_user_irq {
......@@ -525,7 +560,7 @@ struct xdma_dev {
unsigned int flags;
/* PCIe BAR management */
void *__iomem bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */
void __iomem *bar[XDMA_BAR_NUM]; /* addresses for mapped BARs */
int user_bar_idx; /* BAR index of user logic */
int config_bar_idx; /* BAR index of XDMA config logic */
int bypass_bar_idx; /* BAR index of XDMA bypass logic */
......@@ -541,7 +576,7 @@ struct xdma_dev {
int irq_line; /* flag if irq allocated successfully */
int msi_enabled; /* flag if msi was enabled for the device */
int msix_enabled; /* flag if msi-x was enabled for the device */
#if LINUX_VERSION_CODE < KERNEL_VERSION(4,12,0)
#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE
struct msix_entry entry[32]; /* msi-x vector/entry table */
#endif
struct xdma_user_irq user_irq[16]; /* user IRQ management */
......@@ -621,8 +656,8 @@ void get_perf_stats(struct xdma_engine *engine);
int xdma_cyclic_transfer_setup(struct xdma_engine *engine);
int xdma_cyclic_transfer_teardown(struct xdma_engine *engine);
ssize_t xdma_engine_read_cyclic(struct xdma_engine *, char __user *, size_t,
int);
ssize_t xdma_engine_read_cyclic(struct xdma_engine *engine,
char __user *buf, size_t count, int timeout_ms);
int engine_addrmode_set(struct xdma_engine *engine, unsigned long arg);
int engine_service_poll(struct xdma_engine *engine, u32 expected_desc_count);
#endif /* XDMA_LIB_H */
XDMA/linux-kernel/libxdma/version.h
View file @ a0df2803
......@@ -20,9 +20,9 @@
#ifndef __XDMA_VERSION_H__
#define __XDMA_VERSION_H__
#define DRV_MOD_MAJOR 2018
#define DRV_MOD_MINOR 3
#define DRV_MOD_PATCHLEVEL 41
#define DRV_MOD_MAJOR 2019
#define DRV_MOD_MINOR 2
#define DRV_MOD_PATCHLEVEL 42
#define DRV_MODULE_VERSION \
__stringify(DRV_MOD_MAJOR) "." \
......
XDMA/linux-kernel/xdma/Makefile
View file @ a0df2803
......@@ -18,7 +18,7 @@ EXTRA_CFLAGS := -I$(topdir)/include $(XVC_FLAGS)
#EXTRA_CFLAGS += -DINTERNAL_TESTING
ifneq ($(KERNELRELEASE),)
$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o
$(TARGET_MODULE)-objs := libxdma.o xdma_cdev.o cdev_ctrl.o cdev_events.o cdev_sgdma.o cdev_xvc.o cdev_bypass.o xdma_mod.o xdma_thread.o
obj-m := $(TARGET_MODULE).o
else
BUILDSYSTEM_DIR:=/lib/modules/$(shell uname -r)/build
......
XDMA/linux-kernel/xdma/cdev_bypass.c
View file @ a0df2803
......@@ -20,7 +20,7 @@
#include "libxdma_api.h"
#include "xdma_cdev.h"
#define write_register(v,mem,off) iowrite32(v, mem)
#define write_register(v, mem, off) iowrite32(v, mem)
static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
size_t *buf_offset, size_t buf_size)
......@@ -29,8 +29,15 @@ static int copy_desc_data(struct xdma_transfer *transfer, char __user *buf,
int copy_err;
int rc = 0;
BUG_ON(!buf);
BUG_ON(!buf_offset);
if (!buf) {
pr_err("Invalid user buffer\n");
return -EINVAL;
}
if (!buf_offset) {
pr_err("Invalid user buffer offset\n");
return -EINVAL;
}
/* Fill user buffer with descriptor data */
for (i = 0; i < transfer->desc_num; i++) {
......@@ -71,7 +78,7 @@ static ssize_t char_bypass_read(struct file *file, char __user *buf,
xdev = xcdev->xdev;
engine = xcdev->engine;
dbg_sg("In char_bypass_read()\n");
dbg_sg("In %s()\n", __func__);
if (count & 3) {
dbg_sg("Buffer size must be a multiple of 4 bytes\n");
......@@ -114,7 +121,7 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
u32 desc_data;
u32 *bypass_addr;
void __iomem *bypass_addr;
size_t buf_offset = 0;
int rc = 0;
int copy_err;
......@@ -140,18 +147,21 @@ static ssize_t char_bypass_write(struct file *file, const char __user *buf,
return -ENODEV;
}
dbg_sg("In char_bypass_write()\n");
dbg_sg("In %s()\n", __func__);
spin_lock(&engine->lock);
/* Write descriptor data to the bypass BAR */
bypass_addr = (u32 *)xdev->bar[xdev->bypass_bar_idx];
bypass_addr += engine->bypass_offset;
bypass_addr = xdev->bar[xdev->bypass_bar_idx];
bypass_addr = (void __iomem *)(
(u32 __iomem *)bypass_addr + engine->bypass_offset
);
while (buf_offset < count) {
copy_err = copy_from_user(&desc_data, &buf[buf_offset],
sizeof(u32));
if (!copy_err) {
write_register(desc_data, bypass_addr, bypass_addr - engine->bypass_offset);
write_register(desc_data, bypass_addr,
bypass_addr - engine->bypass_offset);
buf_offset += sizeof(u32);
rc = buf_offset;
} else {
......
XDMA/linux-kernel/xdma/cdev_ctrl.c
View file @ a0df2803
......@@ -32,7 +32,7 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
{
struct xdma_cdev *xcdev = (struct xdma_cdev *)fp->private_data;
struct xdma_dev *xdev;
void *reg;
void __iomem *reg;
u32 w;
int rv;
......@@ -48,8 +48,8 @@ static ssize_t char_ctrl_read(struct file *fp, char __user *buf, size_t count,
reg = xdev->bar[xcdev->bar] + *pos;
//w = read_register(reg);
w = ioread32(reg);
dbg_sg("char_ctrl_read(@%p, count=%ld, pos=%d) value = 0x%08x\n", reg,
(long)count, (int)*pos, w);
dbg_sg("%s(@%p, count=%ld, pos=%d) value = 0x%08x\n",
__func__, reg, (long)count, (int)*pos, w);
rv = copy_to_user(buf, &w, 4);
if (rv)
dbg_sg("Copy to userspace failed but continuing\n");
......@@ -63,7 +63,7 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
{
struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
struct xdma_dev *xdev;
void *reg;
void __iomem *reg;
u32 w;
int rv;
......@@ -79,12 +79,11 @@ static ssize_t char_ctrl_write(struct file *file, const char __user *buf,
/* first address is BAR base plus file position offset */
reg = xdev->bar[xcdev->bar] + *pos;
rv = copy_from_user(&w, buf, 4);
if (rv) {
if (rv)
pr_info("copy from user failed %d/4, but continuing.\n", rv);
}
dbg_sg("char_ctrl_write(0x%08x @%p, count=%ld, pos=%d)\n", w, reg,
(long)count, (int)*pos);
dbg_sg("%s(0x%08x @%p, count=%ld, pos=%d)\n",
__func__, w, reg, (long)count, (int)*pos);
//write_register(w, reg);
iowrite32(w, reg);
*pos += 4;
......@@ -158,7 +157,7 @@ long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
switch (cmd) {
case XDMA_IOCINFO:
if (copy_from_user((void *)&ioctl_obj, (void *) arg,
if (copy_from_user((void *)&ioctl_obj, (void __user *) arg,
sizeof(struct xdma_ioc_base))) {
pr_err("copy_from_user failed.\n");
return -EFAULT;
......
XDMA/linux-kernel/xdma/cdev_sgdma.c
View file @ a0df2803
......@@ -19,16 +19,116 @@
#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
#include <linux/types.h>
#include <asm/cacheflush.h>
#include <linux/slab.h>
#include <linux/aio.h>
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/kthread.h>
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
#include <linux/uio.h>
#endif
#include "libxdma_api.h"
#include "xdma_cdev.h"
#include "cdev_sgdma.h"
#include "xdma_thread.h"
/* Module Parameters */
unsigned int sgdma_timeout = 10;
module_param(sgdma_timeout, uint, 0644);
MODULE_PARM_DESC(sgdma_timeout, "timeout in seconds for sgdma, default is 10 sec.");
extern struct kmem_cache *cdev_cache;
static void char_sgdma_unmap_user_buf(struct xdma_io_cb *cb, bool write);
static void async_io_handler(unsigned long cb_hndl, int err)
{
struct xdma_cdev *xcdev;
struct xdma_engine *engine;
struct xdma_dev *xdev;
struct xdma_io_cb *cb = (struct xdma_io_cb *)cb_hndl;
struct cdev_async_io *caio = (struct cdev_async_io *)cb->private;
ssize_t numbytes = 0;
ssize_t res, res2;
int lock_stat;
int rv;
if (NULL == caio) {
pr_err("Invalid work struct\n");
return;
}
xcdev = (struct xdma_cdev *)caio->iocb->ki_filp->private_data;
rv = xcdev_check(__func__, xcdev, 1);
if (rv < 0)
return;
/* Safeguarding for cancel requests */
lock_stat = spin_trylock(&caio->lock);
if (!lock_stat) {
pr_err("caio lock not acquired\n");
goto skip_dev_lock;
}
if (false != caio->cancel) {
pr_err("skipping aio\n");
goto skip_tran;
}
engine = xcdev->engine;
xdev = xcdev->xdev;
if (!err)
numbytes = xdma_xfer_completion((void *)cb, xdev, engine->channel, cb->write, cb->ep_addr, &cb->sgt,
0, sgdma_timeout * 1000);
char_sgdma_unmap_user_buf(cb, cb->write);
caio->res2 |= (err < 0) ? err : 0;
if (caio->res2)
caio->err_cnt++;
caio->cmpl_cnt++;
caio->res += numbytes;
if (caio->cmpl_cnt == caio->req_cnt)
{
res = caio->res;
res2 = caio->res2;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
caio->iocb->ki_complete(caio->iocb, res, res2);
#else
aio_complete(caio->iocb, res, res2);
#endif
skip_tran:
spin_unlock(&caio->lock);
kmem_cache_free(cdev_cache, caio);
kfree(cb);
return;
}
else
{
spin_unlock(&caio->lock);
return;
}
skip_dev_lock:
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)
caio->iocb->ki_complete(caio->iocb, numbytes, -EBUSY);
#else
aio_complete(caio->iocb, numbytes, -EBUSY);
#endif
kmem_cache_free(cdev_cache, caio);
}
/*
* character device file operations for SG DMA engine
*/
......@@ -52,10 +152,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence)
if (newpos < 0)
return -EINVAL;
file->f_pos = newpos;
dbg_fops("char_sgdma_llseek: pos=%lld\n", (signed long long)newpos);
dbg_fops("%s: pos=%lld\n", __func__, (signed long long)newpos);
#if 0
pr_err("0x%p, off 0x%lld, whence %d -> pos %lld.\n",
pr_err("0x%p, off %lld, whence %d -> pos %lld.\n",
file, (signed long long)off, whence, (signed long long)off);
#endif
......@@ -80,7 +180,10 @@ static loff_t char_sgdma_llseek(struct file *file, loff_t off, int whence)
static int check_transfer_align(struct xdma_engine *engine,
const char __user *buf, size_t count, loff_t pos, int sync)
{
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
/* AXI ST or AXI MM non-incremental addressing mode? */
if (engine->non_incr_addr) {
......@@ -171,17 +274,16 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
{
struct sg_table *sgt = &cb->sgt;
unsigned long len = cb->len;
char *buf = cb->buf;
void __user *buf = cb->buf;
struct scatterlist *sg;
unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE -1) -
unsigned int pages_nr = (((unsigned long)buf + len + PAGE_SIZE - 1) -
((unsigned long)buf & PAGE_MASK))
>> PAGE_SHIFT;
int i;
int rv;
if (pages_nr == 0) {
if (pages_nr == 0)
return -EINVAL;
}
if (sg_alloc_table(sgt, pages_nr, GFP_KERNEL)) {
pr_err("sgl OOM.\n");
......@@ -225,7 +327,8 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
sg = sgt->sgl;
for (i = 0; i < pages_nr; i++, sg = sg_next(sg)) {
unsigned int offset = offset_in_page(buf);
unsigned int nbytes = min_t(unsigned int, PAGE_SIZE - offset, len);
unsigned int nbytes =
min_t(unsigned int, PAGE_SIZE - offset, len);
flush_dcache_page(cb->pages[i]);
sg_set_page(sg, cb->pages[i], nbytes, offset);
......@@ -234,7 +337,10 @@ static int char_sgdma_map_user_buf_to_sgl(struct xdma_io_cb *cb, bool write)
len -= nbytes;
}
BUG_ON(len);
if (len) {
pr_err("Invalid user buffer length. Cannot map to sgl\n");
return -EINVAL;
}
cb->pages_nr = pages_nr;
return 0;
......@@ -244,7 +350,7 @@ err_out:
return rv;
}
static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
static ssize_t char_sgdma_read_write(struct file *file, const char __user *buf,
size_t count, loff_t *pos, bool write)
{
int rv;
......@@ -278,8 +384,10 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
}
memset(&cb, 0, sizeof(struct xdma_io_cb));
cb.buf = buf;
cb.buf = (char __user *)buf;
cb.len = count;
cb.ep_addr = (u64)*pos;
cb.write = write;
rv = char_sgdma_map_user_buf_to_sgl(&cb, write);
if (rv < 0)
return rv;
......@@ -296,41 +404,183 @@ static ssize_t char_sgdma_read_write(struct file *file, char __user *buf,
static ssize_t char_sgdma_write(struct file *file, const char __user *buf,
size_t count, loff_t *pos)
{
return char_sgdma_read_write(file, (char *)buf, count, pos, 1);
return char_sgdma_read_write(file, buf, count, pos, 1);
}
static ssize_t char_sgdma_read(struct file *file, char __user *buf,
size_t count, loff_t *pos)
{
struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
return char_sgdma_read_write(file, buf, count, pos, 0);
}
static ssize_t cdev_aio_write(struct kiocb *iocb, const struct iovec *io,
unsigned long count, loff_t pos)
{
struct xdma_cdev *xcdev = (struct xdma_cdev *)iocb->ki_filp->private_data;
struct cdev_async_io *caio;
struct xdma_engine *engine;
struct xdma_dev *xdev;
int rv;
unsigned long i;
rv = xcdev_check(__func__, xcdev, 1);
if (rv < 0)
if (!xcdev) {
pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
iocb->ki_filp, (u64)count, (u64)pos, 1);
return -EINVAL;
}
engine = xcdev->engine;
xdev = xcdev->xdev;
if (engine->dir != DMA_TO_DEVICE) {
pr_err("r/w mismatch. WRITE, dir %d.\n",
engine->dir);
return -EINVAL;
}
caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
memset(caio, 0, sizeof(struct cdev_async_io));
caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL);
spin_lock_init(&caio->lock);
iocb->private = caio;
caio->iocb = iocb;
caio->write = true;
caio->cancel = false;
caio->req_cnt = count;
for (i = 0; i < count; i++) {
memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb));
caio->cb[i].buf = io[i].iov_base;
caio->cb[i].len = io[i].iov_len;
caio->cb[i].ep_addr = (u64)pos;
caio->cb[i].write = true;
caio->cb[i].private = caio;
caio->cb[i].io_done = async_io_handler;
rv = check_transfer_align(engine, caio->cb[i].buf, caio->cb[i].len, pos, 1);
if (rv) {
pr_info("Invalid transfer alignment detected\n");
kmem_cache_free(cdev_cache, caio);
return rv;
}
rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true);
if (rv < 0) {
return rv;
}
rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, engine->channel, caio->cb[i].write, caio->cb[i].ep_addr, &caio->cb[i].sgt,
0, sgdma_timeout * 1000);
}
if (engine->cmplthp)
xdma_kthread_wakeup(engine->cmplthp);
return -EIOCBQUEUED;
}
static ssize_t cdev_aio_read(struct kiocb *iocb, const struct iovec *io,
unsigned long count, loff_t pos)
{
struct xdma_cdev *xcdev = (struct xdma_cdev *)iocb->ki_filp->private_data;
struct cdev_async_io *caio;
struct xdma_engine *engine;
struct xdma_dev *xdev;
int rv;
unsigned long i;
if (!xcdev) {
pr_info("file 0x%p, xcdev NULL, %llu, pos %llu, W %d.\n",
iocb->ki_filp, (u64)count, (u64)pos, 1);
return -EINVAL;
}
engine = xcdev->engine;
xdev = xcdev->xdev;
if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
rv = xdma_cyclic_transfer_setup(engine);
if (rv < 0 && rv != -EBUSY)
if (engine->dir != DMA_FROM_DEVICE) {
pr_err("r/w mismatch. READ, dir %d.\n",
engine->dir);
return -EINVAL;
}
caio = kmem_cache_alloc(cdev_cache, GFP_KERNEL);
memset(caio, 0, sizeof(struct cdev_async_io));
caio->cb = kzalloc(count * (sizeof(struct xdma_io_cb)), GFP_KERNEL);
spin_lock_init(&caio->lock);
iocb->private = caio;
caio->iocb = iocb;
caio->write = false;
caio->cancel = false;
caio->req_cnt = count;
for (i = 0; i < count; i++) {
memset(&(caio->cb[i]), 0, sizeof(struct xdma_io_cb));
caio->cb[i].buf = io[i].iov_base;
caio->cb[i].len = io[i].iov_len;
caio->cb[i].ep_addr = (u64)pos;
caio->cb[i].write = false;
caio->cb[i].private = caio;
caio->cb[i].io_done = async_io_handler;
rv = check_transfer_align(engine, caio->cb[i].buf, caio->cb[i].len, pos, 1);
if (rv) {
pr_info("Invalid transfer alignment detected\n");
kmem_cache_free(cdev_cache, caio);
return rv;
/* 600 sec. timeout */
return xdma_engine_read_cyclic(engine, buf, count, 600000);
}
return char_sgdma_read_write(file, (char *)buf, count, pos, 0);
rv = char_sgdma_map_user_buf_to_sgl(&caio->cb[i], true);
if (rv < 0) {
return rv;
}
rv = xdma_xfer_submit_nowait((void *)&caio->cb[i], xdev, engine->channel, caio->cb[i].write, caio->cb[i].ep_addr, &caio->cb[i].sgt,
0, sgdma_timeout * 1000);
}
if (engine->cmplthp)
xdma_kthread_wakeup(engine->cmplthp);
return -EIOCBQUEUED;
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
static ssize_t cdev_write_iter(struct kiocb *iocb, struct iov_iter *io)
{
return cdev_aio_write(iocb, io->iov, io->nr_segs, io->iov_offset);
}
static ssize_t cdev_read_iter(struct kiocb *iocb, struct iov_iter *io)
{
return cdev_aio_read(iocb, io->iov, io->nr_segs, io->iov_offset);
}
#endif
static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
{
int rv;
struct xdma_dev *xdev;
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
xdev = engine->xdev;
BUG_ON(!xdev);
if (!xdev) {
pr_err("Invalid xdev\n");
return -EINVAL;
}
/* performance measurement already running on this engine? */
if (engine->xdma_perf) {
......@@ -345,7 +595,7 @@ static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
return -ENOMEM;
rv = copy_from_user(engine->xdma_perf,
(struct xdma_performance_ioctl *)arg,
(struct xdma_performance_ioctl __user *)arg,
sizeof(struct xdma_performance_ioctl));
if (rv < 0) {
......@@ -361,14 +611,15 @@ static int ioctl_do_perf_start(struct xdma_engine *engine, unsigned long arg)
enable_perf(engine);
dbg_perf("transfer_size = %d\n", engine->xdma_perf->transfer_size);
/* initialize wait queue */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0)
#if KERNEL_VERSION(4, 6, 0) <= LINUX_VERSION_CODE
init_swait_queue_head(&engine->xdma_perf_wq);
#else
init_waitqueue_head(&engine->xdma_perf_wq);
#endif
xdma_performance_submit(xdev, engine);
return 0;
rv = xdma_performance_submit(xdev, engine);
if (rv < 0)
pr_err("Failed to submit dma performance\n");
return rv;
}
static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
......@@ -376,6 +627,11 @@ static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
struct xdma_transfer *transfer = NULL;
int rv;
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
dbg_perf("IOCTL_XDMA_PERF_STOP\n");
/* no performance measurement running on this engine? */
......@@ -386,23 +642,22 @@ static int ioctl_do_perf_stop(struct xdma_engine *engine, unsigned long arg)
/* stop measurement */
transfer = engine_cyclic_stop(engine);
if (!transfer) {
pr_err("Failed to stop cyclic transfer\n");
return -EINVAL;
}
dbg_perf("Waiting for measurement to stop\n");
if (engine->xdma_perf) {
get_perf_stats(engine);
rv = copy_to_user((void __user *)arg, engine->xdma_perf,
sizeof(struct xdma_performance_ioctl));
if (rv) {
dbg_perf("Error copying result to user\n");
return -EINVAL;
return rv;
}
if (transfer)
kfree(transfer);
} else {
dbg_perf("engine->xdma_perf == NULL?\n");
}
kfree(engine->xdma_perf);
engine->xdma_perf = NULL;
......@@ -414,7 +669,10 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg)
{
int rc;
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
dbg_perf("IOCTL_XDMA_PERF_GET\n");
......@@ -425,7 +683,7 @@ static int ioctl_do_perf_get(struct xdma_engine *engine, unsigned long arg)
sizeof(struct xdma_performance_ioctl));
if (rc) {
dbg_perf("Error copying result to user\n");
return -EINVAL;
return rc;
}
} else {
dbg_perf("engine->xdma_perf == NULL?\n");
......@@ -445,7 +703,10 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg)
int rv;
unsigned long src;
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
src = !!engine->non_incr_addr;
dbg_perf("IOCTL_XDMA_ADDRMODE_GET\n");
......@@ -456,7 +717,10 @@ static int ioctl_do_addrmode_get(struct xdma_engine *engine, unsigned long arg)
static int ioctl_do_align_get(struct xdma_engine *engine, unsigned long arg)
{
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return -EINVAL;
}
dbg_perf("IOCTL_XDMA_ALIGN_GET\n");
return put_user(engine->addr_align, (int __user *)arg);
......@@ -519,7 +783,6 @@ static int char_sgdma_open(struct inode *inode, struct file *file)
if (engine->streaming && engine->dir == DMA_FROM_DEVICE) {
if (engine->device_open == 1)
return -EBUSY;
else
engine->device_open = 1;
}
......@@ -551,7 +814,17 @@ static const struct file_operations sgdma_fops = {
.open = char_sgdma_open,
.release = char_sgdma_close,
.write = char_sgdma_write,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
.write_iter = cdev_write_iter,
#else
.aio_write = cdev_aio_write,
#endif
.read = char_sgdma_read,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 16, 0)
.read_iter = cdev_read_iter,
#else
.aio_read = cdev_aio_read,
#endif
.unlocked_ioctl = char_sgdma_ioctl,
.llseek = char_sgdma_llseek,
};
......
XDMA/linux-kernel/xdma/cdev_sgdma.h
View file @ a0df2803
......@@ -46,8 +46,7 @@
* _IOC_SIZE(nr) returns size
*/
struct xdma_performance_ioctl
{
struct xdma_performance_ioctl {
/* IOCTL_XDMA_IOCTL_Vx */
uint32_t version;
uint32_t transfer_size;
......
XDMA/linux-kernel/xdma/cdev_xvc.c
View file @ a0df2803
......@@ -32,30 +32,30 @@
#ifdef __REG_DEBUG__
/* SECTION: Function definitions */
inline void __write_register(const char *fn, u32 value, void *base,
inline void __write_register(const char *fn, u32 value, void __iomem *base,
unsigned int off)
{
pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
iowrite32(value, base + off);
}
inline u32 __read_register(const char *fn, void *base, unsigned int off)
inline u32 __read_register(const char *fn, void __iomem *base, unsigned int off)
{
u32 v = ioread32(base + off);
pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
return v;
}
#define write_register(v,base,off) __write_register(__func__, v, base, off)
#define read_register(base,off) __read_register(__func__, base, off)
#define write_register(v, base, off) __write_register(__func__, v, base, off)
#define read_register(base, off) __read_register(__func__, base, off)
#else
#define write_register(v,base,off) iowrite32(v, (base) + (off))
#define read_register(base,off) ioread32((base) + (off))
#define write_register(v, base, off) iowrite32(v, (base) + (off))
#define read_register(base, off) ioread32((base) + (off))
#endif /* #ifdef __REG_DEBUG__ */
static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
static int xvc_shift_bits(void __iomem *base, u32 tms_bits, u32 tdi_bits,
u32 *tdo_bits)
{
u32 control;
......@@ -109,6 +109,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
rv = xcdev_check(__func__, xcdev, 0);
if (rv < 0)
return rv;
xdev = xcdev->xdev;
if (cmd != XDMA_IOCXVC) {
......@@ -135,7 +136,7 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
total_bits = xvc_obj.length;
total_bytes = (total_bits + 7) >> 3;
buffer = (char *)kmalloc(total_bytes * 3, GFP_KERNEL);
buffer = (unsigned char *)kmalloc(total_bytes * 3, GFP_KERNEL);
if (!buffer) {
pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n",
3 * total_bytes, opcode, total_bits, total_bytes);
......@@ -146,12 +147,16 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
tdi_buf = tms_buf + total_bytes;
tdo_buf = tdi_buf + total_bytes;
rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes);
rv = copy_from_user((void *)tms_buf,
(const char __user *)xvc_obj.tms_buf,
total_bytes);
if (rv) {
pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes);
goto cleanup;
}
rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes);
rv = copy_from_user((void *)tdi_buf,
(const char __user *)xvc_obj.tdi_buf,
total_bytes);
if (rv) {
pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes);
goto cleanup;
......@@ -162,7 +167,8 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
iobase = xdev->bar[xcdev->bar] + xcdev->base;
/* set length register to 32 initially if more than one
* word-transaction is to be done */
* word-transaction is to be done
*/
if (total_bits >= 32)
write_register(0x20, iobase, XVC_BAR_LENGTH_REG);
......@@ -186,33 +192,33 @@ static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
/* Shift data out and copy to output buffer */
rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store);
if (rv < 0)
goto cleanup;
break;
memcpy(tdo_buf + bytes, &tdo_store, shift_bytes);
}
if (rv < 0)
goto unlock;
/* if testing bar access swap tdi and tdo bufferes to "loopback" */
if (opcode == 0x2) {
char *tmp = tdo_buf;
unsigned char *tmp = tdo_buf;
tdo_buf = tdi_buf;
tdi_buf = tmp;
}
rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes);
if (rv) {
rv = copy_to_user(xvc_obj.tdo_buf, (const void *)tdo_buf, total_bytes);
if (rv)
pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes);
rv = -EFAULT;
goto cleanup;
}
cleanup:
if (buffer)
kfree(buffer);
unlock:
mmiowb();
spin_unlock(&xcdev->lock);
cleanup:
kfree(buffer);
return rv;
}
......
XDMA/linux-kernel/xdma/cdev_xvc.h
View file @ a0df2803
......@@ -33,9 +33,9 @@
struct xvc_ioc {
unsigned int opcode;
unsigned int length;
unsigned char *tms_buf;
unsigned char *tdi_buf;
unsigned char *tdo_buf;
const char __user *tms_buf;
const char __user *tdi_buf;
void __user *tdo_buf;
};
#define XDMA_IOCXVC _IOWR(XVC_MAGIC, 1, struct xvc_ioc)
......
XDMA/linux-kernel/xdma/version.h
View file @ a0df2803
......@@ -20,9 +20,9 @@
#ifndef __XDMA_VERSION_H__
#define __XDMA_VERSION_H__
#define DRV_MOD_MAJOR 2018
#define DRV_MOD_MINOR 3
#define DRV_MOD_PATCHLEVEL 50
#define DRV_MOD_MAJOR 2019
#define DRV_MOD_MINOR 2
#define DRV_MOD_PATCHLEVEL 51
#define DRV_MODULE_VERSION \
__stringify(DRV_MOD_MAJOR) "." \
......
XDMA/linux-kernel/xdma/xdma_cdev.c
View file @ a0df2803
......@@ -21,7 +21,9 @@
#include "xdma_cdev.h"
struct class *g_xdma_class;
static struct class *g_xdma_class;
struct kmem_cache *cdev_cache;
enum cdev_type {
CHAR_USER,
......@@ -75,16 +77,18 @@ static inline int xpdev_flag_test(struct xdma_pci_dev *xpdev,
}
#ifdef __XDMA_SYSFS__
ssize_t show_device_numbers(struct device *dev, struct device_attribute *attr,
ssize_t xdma_dev_instance_show(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct xdma_pci_dev *xpdev = (struct xdma_pci_dev *)dev_get_drvdata(dev);
struct xdma_pci_dev *xpdev =
(struct xdma_pci_dev *)dev_get_drvdata(dev);
return snprintf(buf, PAGE_SIZE, "%d\t%d\n",
xpdev->major, xpdev->xdev->idx);
}
static DEVICE_ATTR(xdma_dev_instance, S_IRUGO, show_device_numbers, NULL);
static DEVICE_ATTR_RO(xdma_dev_instance);
#endif
static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type)
......@@ -98,7 +102,10 @@ static int config_kobject(struct xdma_cdev *xcdev, enum cdev_type type)
case CHAR_XDMA_C2H:
case CHAR_BYPASS_H2C:
case CHAR_BYPASS_C2H:
BUG_ON(!engine);
if (!engine) {
pr_err("Invalid DMA engine\n");
return rv;
}
rv = kobject_set_name(&xcdev->cdev.kobj, devnode_names[type],
xdev->idx, engine->channel);
break;
......@@ -142,6 +149,7 @@ int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine)
if (check_engine) {
struct xdma_engine *engine = xcdev->engine;
if (!engine || engine->magic != MAGIC_ENGINE) {
pr_info("%s, engine 0x%p, magic 0x%lx.\n", fname,
engine, engine ? engine->magic : 0xFFFFFFFF);
......@@ -158,7 +166,11 @@ int char_open(struct inode *inode, struct file *file)
/* pointer to containing structure of the character device inode */
xcdev = container_of(inode->i_cdev, struct xdma_cdev, cdev);
BUG_ON(xcdev->magic != MAGIC_CHAR);
if (xcdev->magic != MAGIC_CHAR) {
pr_err("xcdev 0x%p inode 0x%lx magic mismatch 0x%lx\n",
xcdev, inode->i_ino, xcdev->magic);
return -EINVAL;
}
/* create a reference to our char device in the opened file */
file->private_data = xcdev;
......@@ -173,13 +185,30 @@ int char_close(struct inode *inode, struct file *file)
struct xdma_dev *xdev;
struct xdma_cdev *xcdev = (struct xdma_cdev *)file->private_data;
BUG_ON(!xcdev);
BUG_ON(xcdev->magic != MAGIC_CHAR);
if (!xcdev) {
pr_err("char device with inode 0x%lx xcdev NULL\n",
inode->i_ino);
return -EINVAL;
}
if (xcdev->magic != MAGIC_CHAR) {
pr_err("xcdev 0x%p magic mismatch 0x%lx\n",
xcdev, xcdev->magic);
return -EINVAL;
}
/* fetch device specific data stored earlier during open */
xdev = xcdev->xdev;
BUG_ON(!xdev);
BUG_ON(xdev->magic != MAGIC_DEVICE);
if (!xdev) {
pr_err("char device with inode 0x%lx xdev NULL\n",
inode->i_ino);
return -EINVAL;
}
if (xdev->magic != MAGIC_DEVICE) {
pr_err("xdev 0x%p magic mismatch 0x%lx\n", xdev, xdev->magic);
return -EINVAL;
}
return 0;
}
......@@ -218,15 +247,27 @@ static int destroy_xcdev(struct xdma_cdev *cdev)
{
if (!cdev) {
pr_warn("cdev NULL.\n");
return 0;
return -EINVAL;
}
if (cdev->magic != MAGIC_CHAR) {
pr_warn("cdev 0x%p magic mismatch 0x%lx\n", cdev, cdev->magic);
return 0;
return -EINVAL;
}
if (!cdev->xdev) {
pr_err("xdev NULL\n");
return -EINVAL;
}
if (!g_xdma_class) {
pr_err("g_xdma_class NULL\n");
return -EINVAL;
}
if (!cdev->sys_device) {
pr_err("cdev sys_device NULL\n");
return -EINVAL;
}
BUG_ON(!cdev->xdev);
BUG_ON(!g_xdma_class);
BUG_ON(!cdev->sys_device);
if (cdev->sys_device)
device_destroy(g_xdma_class, cdev->cdevno);
......@@ -343,52 +384,85 @@ unregister_region:
void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev)
{
int i;
int i = 0;
int rv;
#ifdef __XDMA_SYSFS__
device_remove_file(&xpdev->pdev->dev, &dev_attr_xdma_dev_instance);
#endif
if (xpdev_flag_test(xpdev, XDF_CDEV_SG)) {
/* iterate over channels */
for (i = 0; i < xpdev->h2c_channel_max; i++)
for (i = 0; i < xpdev->h2c_channel_max; i++) {
/* remove SG DMA character device */
destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]);
for (i = 0; i < xpdev->c2h_channel_max; i++)
destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]);
rv = destroy_xcdev(&xpdev->sgdma_h2c_cdev[i]);
if (rv < 0)
pr_err("Failed to destroy h2c xcdev %d error :0x%x\n",
i, rv);
}
for (i = 0; i < xpdev->c2h_channel_max; i++) {
rv = destroy_xcdev(&xpdev->sgdma_c2h_cdev[i]);
if (rv < 0)
pr_err("Failed to destroy c2h xcdev %d error 0x%x\n",
i, rv);
}
}
if (xpdev_flag_test(xpdev, XDF_CDEV_EVENT)) {
for (i = 0; i < xpdev->user_max; i++)
destroy_xcdev(&xpdev->events_cdev[i]);
for (i = 0; i < xpdev->user_max; i++) {
rv = destroy_xcdev(&xpdev->events_cdev[i]);
if (rv < 0)
pr_err("Failed to destroy cdev event %d error 0x%x\n",
i, rv);
}
}
/* remove control character device */
if (xpdev_flag_test(xpdev, XDF_CDEV_CTRL)) {
destroy_xcdev(&xpdev->ctrl_cdev);
rv = destroy_xcdev(&xpdev->ctrl_cdev);
if (rv < 0)
pr_err("Failed to destroy cdev ctrl event %d error 0x%x\n",
i, rv);
}
/* remove user character device */
if (xpdev_flag_test(xpdev, XDF_CDEV_USER)) {
destroy_xcdev(&xpdev->user_cdev);
rv = destroy_xcdev(&xpdev->user_cdev);
if (rv < 0)
pr_err("Failed to destroy user cdev %d error 0x%x\n",
i, rv);
}
if (xpdev_flag_test(xpdev, XDF_CDEV_XVC)) {
destroy_xcdev(&xpdev->xvc_cdev);
rv = destroy_xcdev(&xpdev->xvc_cdev);
if (rv < 0)
pr_err("Failed to destroy xvc cdev %d error 0x%x\n",
i, rv);
}
if (xpdev_flag_test(xpdev, XDF_CDEV_BYPASS)) {
/* iterate over channels */
for (i = 0; i < xpdev->h2c_channel_max; i++)
for (i = 0; i < xpdev->h2c_channel_max; i++) {
/* remove DMA Bypass character device */
destroy_xcdev(&xpdev->bypass_h2c_cdev[i]);
for (i = 0; i < xpdev->c2h_channel_max; i++)
destroy_xcdev(&xpdev->bypass_c2h_cdev[i]);
destroy_xcdev(&xpdev->bypass_cdev_base);
rv = destroy_xcdev(&xpdev->bypass_h2c_cdev[i]);
if (rv < 0)
pr_err("Failed to destroy bypass h2c cdev %d error 0x%x\n",
i, rv);
}
for (i = 0; i < xpdev->c2h_channel_max; i++) {
rv = destroy_xcdev(&xpdev->bypass_c2h_cdev[i]);
if (rv < 0)
pr_err("Failed to destroy bypass c2h %d error 0x%x\n",
i, rv);
}
rv = destroy_xcdev(&xpdev->bypass_cdev_base);
if (rv < 0)
pr_err("Failed to destroy base cdev\n");
}
if (xpdev->major)
unregister_chrdev_region(MKDEV(xpdev->major, XDMA_MINOR_BASE), XDMA_MINOR_COUNT);
unregister_chrdev_region(
MKDEV(xpdev->major, XDMA_MINOR_BASE),
XDMA_MINOR_COUNT);
}
int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
......@@ -450,7 +524,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
/* ??? Bypass */
/* Initialize Bypass Character Device */
if (xdev->bypass_bar_idx > 0){
if (xdev->bypass_bar_idx > 0) {
for (i = 0; i < xpdev->h2c_channel_max; i++) {
engine = &xdev->engine_h2c[i];
......@@ -515,7 +589,7 @@ int xpdev_create_interfaces(struct xdma_pci_dev *xpdev)
rv = device_create_file(&xpdev->pdev->dev,
&dev_attr_xdma_dev_instance);
if (rv) {
pr_err("Failed to create device file \n");
pr_err("Failed to create device file\n");
goto fail;
}
#endif
......@@ -536,11 +610,29 @@ int xdma_cdev_init(void)
return -1;
}
/* using kmem_cache_create to enable sequential cleanup */
cdev_cache = kmem_cache_create("cdev_cache",
sizeof(struct cdev_async_io),
0,
SLAB_HWCACHE_ALIGN,
NULL);
if (!cdev_cache) {
pr_info("memory allocation for cdev_cache failed. OOM\n");
return -ENOMEM;
}
xdma_threads_create(8);
return 0;
}
void xdma_cdev_cleanup(void)
{
if (cdev_cache)
kmem_cache_destroy(cdev_cache);
if (g_xdma_class)
class_destroy(g_xdma_class);
xdma_threads_destroy();
}
XDMA/linux-kernel/xdma/xdma_cdev.h
View file @ a0df2803
......@@ -35,13 +35,13 @@ int xdma_cdev_init(void);
int char_open(struct inode *inode, struct file *file);
int char_close(struct inode *inode, struct file *file);
int xcdev_check(const char *, struct xdma_cdev *, bool);
int xcdev_check(const char *fname, struct xdma_cdev *xcdev, bool check_engine);
void cdev_ctrl_init(struct xdma_cdev *xcdev);
void cdev_xvc_init(struct xdma_cdev *xcdev);
void cdev_event_init(struct xdma_cdev *xcdev);
void cdev_sgdma_init(struct xdma_cdev *xcdev);
void cdev_bypass_init(struct xdma_cdev *xcdev);
long char_ctrl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
void xpdev_destroy_interfaces(struct xdma_pci_dev *xpdev);
int xpdev_create_interfaces(struct xdma_pci_dev *xpdev);
......
XDMA/linux-kernel/xdma/xdma_mod.c
View file @ a0df2803
......@@ -40,10 +40,10 @@ static char version[] =
MODULE_AUTHOR("Xilinx, Inc.");
MODULE_DESCRIPTION(DRV_MODULE_DESC);
MODULE_VERSION(DRV_MODULE_VERSION);
MODULE_LICENSE("GPL");
MODULE_LICENSE("Dual BSD/GPL");
/* SECTION: Module global variables */
static int xpdev_cnt = 0;
static int xpdev_cnt;
static const struct pci_device_id pci_ids[] = {
{ PCI_DEVICE(0x10ee, 0x903f), },
......@@ -154,14 +154,28 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
hndl = xdma_device_open(DRV_MODULE_NAME, pdev, &xpdev->user_max,
&xpdev->h2c_channel_max, &xpdev->c2h_channel_max);
if (!hndl){
if (!hndl) {
rv = -EINVAL;
goto err_out;
}
BUG_ON(xpdev->user_max > MAX_USER_IRQ);
BUG_ON(xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX);
BUG_ON(xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX);
if (xpdev->user_max > MAX_USER_IRQ) {
pr_err("Maximum users limit reached\n");
rv = -EINVAL;
goto err_out;
}
if (xpdev->h2c_channel_max > XDMA_CHANNEL_NUM_MAX) {
pr_err("Maximun H2C channel limit reached\n");
rv = -EINVAL;
goto err_out;
}
if (xpdev->c2h_channel_max > XDMA_CHANNEL_NUM_MAX) {
pr_err("Maximun C2H channel limit reached\n");
rv = -EINVAL;
goto err_out;
}
if (!xpdev->h2c_channel_max && !xpdev->c2h_channel_max)
pr_warn("NO engine found!\n");
......@@ -181,7 +195,12 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
rv = -EINVAL;
goto err_out;
}
BUG_ON(hndl != xdev );
if (hndl != xdev) {
pr_err("xdev handle mismatch\n");
rv = -EINVAL;
goto err_out;
}
pr_info("%s xdma%d, pdev 0x%p, xdev 0x%p, 0x%p, usr %d, ch %d,%d.\n",
dev_name(&pdev->dev), xdev->idx, pdev, xpdev, xdev,
......@@ -270,7 +289,7 @@ static void xdma_error_resume(struct pci_dev *pdev)
pci_cleanup_aer_uncorrect_error_status(pdev);
}
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
static void xdma_reset_prepare(struct pci_dev *pdev)
{
struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
......@@ -287,7 +306,7 @@ static void xdma_reset_done(struct pci_dev *pdev)
xdma_device_online(pdev, xpdev->xdev);
}
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
static void xdma_reset_notify(struct pci_dev *pdev, bool prepare)
{
struct xdma_pci_dev *xpdev = dev_get_drvdata(&pdev->dev);
......@@ -305,10 +324,10 @@ static const struct pci_error_handlers xdma_err_handler = {
.error_detected = xdma_error_detected,
.slot_reset = xdma_slot_reset,
.resume = xdma_error_resume,
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,13,0)
#if KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE
.reset_prepare = xdma_reset_prepare,
.reset_done = xdma_reset_done,
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3,16,0)
#elif KERNEL_VERSION(3, 16, 0) <= LINUX_VERSION_CODE
.reset_notify = xdma_reset_notify,
#endif
};
......@@ -324,8 +343,6 @@ static struct pci_driver pci_driver = {
static int __init xdma_mod_init(void)
{
int rv;
extern unsigned int desc_blen_max;
extern unsigned int sgdma_timeout;
pr_info("%s", version);
......
XDMA/linux-kernel/xdma/xdma_mod.h
View file @ a0df2803
......@@ -44,14 +44,19 @@
#include <linux/splice.h>
#include <linux/version.h>
#include <linux/uio.h>
#include <linux/spinlock_types.h>
#include "libxdma.h"
#include "xdma_thread.h"
#define MAGIC_ENGINE 0xEEEEEEEEUL
#define MAGIC_DEVICE 0xDDDDDDDDUL
#define MAGIC_CHAR 0xCCCCCCCCUL
#define MAGIC_BITSTREAM 0xBBBBBBBBUL
extern unsigned int desc_blen_max;
extern unsigned int sgdma_timeout;
struct xdma_cdev {
unsigned long magic; /* structure ID for sanity checks */
struct xdma_pci_dev *xpdev;
......@@ -94,12 +99,19 @@ struct xdma_pci_dev {
void *data;
};
struct xdma_io_cb {
void __user *buf;
size_t len;
unsigned int pages_nr;
struct sg_table sgt;
struct page **pages;
struct cdev_async_io {
struct kiocb *iocb;
struct xdma_io_cb* cb;
bool write;
bool cancel;
int cmpl_cnt;
int req_cnt;
spinlock_t lock;
struct work_struct wrk_itm;
struct cdev_async_io *next;
ssize_t res;
ssize_t res2;
int err_cnt;
};
#endif /* ifndef __XDMA_MODULE_H__ */
XDMA/linux-kernel/xdma/xdma_thread.c 0 → 100644
View file @ a0df2803
/*
* This file is part of the Xilinx DMA IP Core driver for Linux
*
* Copyright (c) 2017-present, Xilinx, Inc.
* All rights reserved.
*
* This source code is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* The full GNU General Public License is included in this distribution in
* the file called "COPYING".
*/
#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
#include "xdma_thread.h"
#include <linux/kernel.h>
#include <linux/slab.h>
/* ********************* global variables *********************************** */
static struct xdma_kthread *cs_threads;
static unsigned int thread_cnt;
/* ********************* static function definitions ************************ */
static int xdma_thread_cmpl_status_pend(struct list_head *work_item)
{
struct xdma_engine *engine = list_entry(work_item, struct xdma_engine,
cmplthp_list);
int pend = 0;
unsigned long flags;
spin_lock_irqsave(&engine->lock, flags);
pend = !list_empty(&engine->transfer_list);
spin_unlock_irqrestore(&engine->lock, flags);
return pend;
}
static int xdma_thread_cmpl_status_proc(struct list_head *work_item)
{
struct xdma_engine *engine;
struct xdma_transfer * transfer;
engine = list_entry(work_item, struct xdma_engine, cmplthp_list);
transfer = list_entry(engine->transfer_list.next, struct xdma_transfer,
entry);
engine_service_poll(engine, transfer->desc_num);
return 0;
}
static inline int xthread_work_pending(struct xdma_kthread *thp)
{
struct list_head *work_item, *next;
/* any work items assigned to this thread? */
if (list_empty(&thp->work_list))
return 0;
/* any work item has pending work to do? */
list_for_each_safe(work_item, next, &thp->work_list) {
if (thp->fpending && thp->fpending(work_item))
return 1;
}
return 0;
}
static inline void xthread_reschedule(struct xdma_kthread *thp)
{
if (thp->timeout) {
pr_debug_thread("%s rescheduling for %u seconds",
thp->name, thp->timeout);
wait_event_interruptible_timeout(thp->waitq, thp->schedule,
msecs_to_jiffies(thp->timeout));
} else {
pr_debug_thread("%s rescheduling", thp->name);
wait_event_interruptible(thp->waitq, thp->schedule);
}
}
static int xthread_main(void *data)
{
struct xdma_kthread *thp = (struct xdma_kthread *)data;
pr_debug_thread("%s UP.\n", thp->name);
disallow_signal(SIGPIPE);
if (thp->finit)
thp->finit(thp);
while (!kthread_should_stop()) {
struct list_head *work_item, *next;
pr_debug_thread("%s interruptible\n", thp->name);
/* any work to do? */
lock_thread(thp);
if (!xthread_work_pending(thp)) {
unlock_thread(thp);
xthread_reschedule(thp);
lock_thread(thp);
}
thp->schedule = 0;
if (thp->work_cnt) {
pr_debug_thread("%s processing %u work items\n",
thp->name, thp->work_cnt);
/* do work */
list_for_each_safe(work_item, next, &thp->work_list) {
thp->fproc(work_item);
}
}
unlock_thread(thp);
schedule();
}
pr_debug_thread("%s, work done.\n", thp->name);
if (thp->fdone)
thp->fdone(thp);
pr_debug_thread("%s, exit.\n", thp->name);
return 0;
}
int xdma_kthread_start(struct xdma_kthread *thp, char *name, int id)
{
int len;
if (thp->task) {
pr_warn("kthread %s task already running?\n", thp->name);
return -EINVAL;
}
len = snprintf(thp->name, sizeof(thp->name), "%s%d", name, id);
if (len < 0)
return -EINVAL;
thp->id = id;
spin_lock_init(&thp->lock);
INIT_LIST_HEAD(&thp->work_list);
init_waitqueue_head(&thp->waitq);
thp->task = kthread_create_on_node(xthread_main, (void *)thp,
cpu_to_node(thp->cpu), "%s", thp->name);
if (IS_ERR(thp->task)) {
pr_err("kthread %s, create task failed: 0x%lx\n",
thp->name, (unsigned long)IS_ERR(thp->task));
thp->task = NULL;
return -EFAULT;
}
kthread_bind(thp->task, thp->cpu);
pr_debug_thread("kthread 0x%p, %s, cpu %u, task 0x%p.\n",
thp, thp->name, thp->cpu, thp->task);
wake_up_process(thp->task);
return 0;
}
int xdma_kthread_stop(struct xdma_kthread *thp)
{
int rv;
if (!thp->task) {
pr_debug_thread("kthread %s, already stopped.\n", thp->name);
return 0;
}
thp->schedule = 1;
rv = kthread_stop(thp->task);
if (rv < 0) {
pr_warn("kthread %s, stop err %d.\n", thp->name, rv);
return rv;
}
pr_debug_thread("kthread %s, 0x%p, stopped.\n", thp->name, thp->task);
thp->task = NULL;
return 0;
}
void xdma_thread_remove_work(struct xdma_engine *engine)
{
struct xdma_kthread *cmpl_thread;
unsigned long flags;
spin_lock_irqsave(&engine->lock, flags);
cmpl_thread = engine->cmplthp;
engine->cmplthp = NULL;
// pr_debug("%s removing from thread %s, %u.\n",
// descq->conf.name, cmpl_thread ? cmpl_thread->name : "?",
// cpu_idx);
spin_unlock_irqrestore(&engine->lock, flags);
#if 0
if (cpu_idx < cpu_count) {
spin_lock(&qcnt_lock);
per_cpu_qcnt[cpu_idx]--;
spin_unlock(&qcnt_lock);
}
#endif
if (cmpl_thread) {
lock_thread(cmpl_thread);
list_del(&engine->cmplthp_list);
cmpl_thread->work_cnt--;
unlock_thread(cmpl_thread);
}
}
void xdma_thread_add_work(struct xdma_engine *engine)
{
struct xdma_kthread *thp = cs_threads;
unsigned int v = 0;
int i, idx = thread_cnt;
unsigned long flags;
/* Polled mode only */
for (i = 0; i < thread_cnt; i++, thp++) {
lock_thread(thp);
if (idx == thread_cnt) {
v = thp->work_cnt;
idx = i;
} else if (!thp->work_cnt) {
idx = i;
unlock_thread(thp);
break;
} else if (thp->work_cnt < v)
idx = i;
unlock_thread(thp);
}
thp = cs_threads + idx;
lock_thread(thp);
list_add_tail(&engine->cmplthp_list, &thp->work_list);
engine->intr_work_cpu = idx;
thp->work_cnt++;
unlock_thread(thp);
pr_info("%s 0x%p assigned to cmpl status thread %s,%u.\n",
engine->name, engine, thp->name, thp->work_cnt);
spin_lock_irqsave(&engine->lock, flags);
engine->cmplthp = thp;
spin_unlock_irqrestore(&engine->lock, flags);
}
int xdma_threads_create(unsigned int num_threads)
{
struct xdma_kthread *thp;
int i;
int rv;
if (thread_cnt) {
pr_warn("threads already created!");
return 0;
}
pr_info("xdma_threads_create\n");
thread_cnt = num_threads;
cs_threads = kzalloc(thread_cnt * sizeof(struct xdma_kthread),
GFP_KERNEL);
if (!cs_threads)
return -ENOMEM;
/* N dma writeback monitoring threads */
thp = cs_threads;
for (i = 0; i < thread_cnt; i++, thp++) {
thp->cpu = i;
thp->timeout = 0;
thp->fproc = xdma_thread_cmpl_status_proc;
thp->fpending = xdma_thread_cmpl_status_pend;
rv = xdma_kthread_start(thp, "cmpl_status_th", i);
if (rv < 0)
goto cleanup_threads;
}
return 0;
cleanup_threads:
kfree(cs_threads);
cs_threads = NULL;
thread_cnt = 0;
return rv;
}
void xdma_threads_destroy(void)
{
int i;
struct xdma_kthread *thp;
if (!thread_cnt)
return;
/* N dma writeback monitoring threads */
thp = cs_threads;
for (i = 0; i < thread_cnt; i++, thp++)
if (thp->fproc)
xdma_kthread_stop(thp);
kfree(cs_threads);
cs_threads = NULL;
thread_cnt = 0;
}
XDMA/linux-kernel/xdma/xdma_thread.h 0 → 100644
View file @ a0df2803
/*
* This file is part of the Xilinx DMA IP Core driver for Linux
*
* Copyright (c) 2017-present, Xilinx, Inc.
* All rights reserved.
*
* This source code is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* The full GNU General Public License is included in this distribution in
* the file called "COPYING".
*/
#ifndef __XDMA_KTHREAD_H__
#define __XDMA_KTHREAD_H__
/**
* @file
* @brief This file contains the declarations for xdma kernel threads
*
*/
#include <linux/version.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/cpuset.h>
#include <linux/signal.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/uaccess.h>
#include <linux/errno.h>
#include "libxdma.h"
#ifdef DEBUG_THREADS
#define lock_thread(thp) \
do { \
pr_debug("locking thp %s ...\n", (thp)->name); \
spin_lock(&(thp)->lock); \
} while (0)
#define unlock_thread(thp) \
do { \
pr_debug("unlock thp %s ...\n", (thp)->name); \
spin_unlock(&(thp)->lock); \
} while (0)
#define xdma_kthread_wakeup(thp) \
do { \
pr_info("signaling thp %s ...\n", (thp)->name); \
wake_up_process((thp)->task); \
} while (0)
#define pr_debug_thread(fmt, ...) pr_info(fmt, __VA_ARGS__)
#else
/** lock thread macro */
#define lock_thread(thp) spin_lock(&(thp)->lock)
/** un lock thread macro */
#define unlock_thread(thp) spin_unlock(&(thp)->lock)
#define xdma_kthread_wakeup(thp) \
do { \
thp->schedule = 1; \
wake_up_interruptible(&thp->waitq); \
} while (0)
/** pr_debug_thread */
#define pr_debug_thread(fmt, ...)
#endif
/**
* @struct - xdma_kthread
* @brief xdma thread book keeping parameters
*/
struct xdma_kthread {
/** thread lock*/
spinlock_t lock;
/** name of the thread */
char name[16];
/** cpu number for which the thread associated with */
unsigned short cpu;
/** thread id */
unsigned short id;
/** thread sleep timeout value */
unsigned int timeout;
/** flags for thread */
unsigned long flag;
/** thread wait queue */
wait_queue_head_t waitq;
/* flag to indicate scheduling of thread */
unsigned int schedule;
/** kernel task structure associated with thread*/
struct task_struct *task;
/** thread work list count */
unsigned int work_cnt;
/** thread work list count */
struct list_head work_list;
/** thread initialization handler */
int (*finit)(struct xdma_kthread *);
/** thread pending handler */
int (*fpending)(struct list_head *);
/** thread peocessing handler */
int (*fproc)(struct list_head *);
/** thread done handler */
int (*fdone)(struct xdma_kthread *);
};
/*****************************************************************************/
/**
* xdma_threads_create() - create xdma threads
*********/
int xdma_threads_create(unsigned int num_threads);
/*****************************************************************************/
/**
* xdma_threads_destroy() - destroy all the xdma threads created
* during system initialization
*
* @return none
*****************************************************************************/
void xdma_threads_destroy(void);
/*****************************************************************************/
/**
* xdma_thread_remove_work() - handler to remove the attached work thread
*
* @param[in] engine: pointer to xdma_engine
*
* @return none
*****************************************************************************/
void xdma_thread_remove_work(struct xdma_engine *engine);
/*****************************************************************************/
/**
* xdma_thread_add_work() - handler to add a work thread
*
* @param[in] engine: pointer to xdma_engine
*
* @return none
*****************************************************************************/
void xdma_thread_add_work(struct xdma_engine *engine);
#endif /* #ifndef __XDMA_KTHREAD_H__ */
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment