So After a first attempt to write and read from a PCI with DMA device in the probe function, I tried to write a binary file to the DMA then read it using an ioctl(), based on the edu device supplied by QEMU and this driver.
I defined my ioctls as follow:
#define IOCTL_LOAD_MODEL _IOW('q', 1, struct user_data*)
#define IOCTL_READ_MODEL _IOR('q', 2, struct user_data*)
static long ioctl_load_model(struct file *file, unsigned int cmd, unsigned long arg)
{
struct test *test = (struct test*)file->private_data;
int return_code = 0;
int buffer_length = 0;
dma_addr_t dma_phyisical_address;
void* allocated_vaddress;
int rv;
pr_info("Welcome to my ioctl\n");
struct user_data *user_data= (struct user_data*)arg;
pr_info("dma status:%d\n",test->dma_status);
if(user_data==NULL){
pr_info("Error user buffer is empty");
return EFAULT;
}
if(user_data->buffer == NULL || user_data->buffer_length == 0){
pr_info("Error: data structure is empty\n");
return EFAULT;
}
pr_info("buffer length:%ld\n",user_data->buffer_length);
if(test->dma_status == DMA_BUSY){
pr_info("Error: could not allocate buffer for the dma\n");
return EINVAL;
}
pr_info("DMA is available\n");
if(user_data->buffer_length > MAX_BUFFER_LENGTH){
pr_info("Error: currentrly we do not handle big buffers\n");
return EINVAL;
}
buffer_length = user_data->buffer_length;
/* first treat only up to one page sized buufers */
if(buffer_length > MAX_BUFFER_LENGTH){
buffer_length = MAX_BUFFER_LENGTH;
}
allocated_vaddress = dma_alloc_coherent(&(pdev->dev),
buffer_length,
&dma_phyisical_address,
GFP_ATOMIC);
pr_info("checking that memory was allocated properly\n");
if(allocated_vaddress == NULL){
pr_info("Error: could not allocate buffer for the dma\n");
return EINVAL;
}
test->allocated_buffer = allocated_vaddress;
pr_info("allocated data to the buffer:%p\n",dma_phyisical_address);
pr_info("test info:test->allocated_buffer on initialization %d\n", (*((u32*)test->allocated_buffer)));
switch (cmd) {
case IOCTL_LOAD_MODEL:
pr_info("load model\n");
/* copy data from userspace to internal buffer now define the ioctl*/
copy_from_user(user_data->buffer, test->allocated_buffer, buffer_length);
pr_info("copied data from user");
iowrite32((u32)dma_phyisical_address, mmio + IO_DMA_SRC);
iowrite32(DMA_BASE, mmio + IO_DMA_DST);
iowrite32(buffer_length, mmio + IO_DMA_CNT);
iowrite32(DMA_CMD | DMA_IRQ, mmio + IO_DMA_CMD);
test->dma_status = DMA_BUSY;
rv = wait_event_interruptible_timeout(test->rw_queue, test->dma_status==DMA_FREE, msecs_to_jiffies(1000));
if(rv < 0){
pr_info("Error:interrupt was not handled properly, data was not copied\n");
goto l_release_buffer;
}
break;
case IOCTL_READ_MODEL:
pr_info("read model\n");
iowrite32(DMA_BASE, mmio + IO_DMA_SRC);
iowrite32((u32)dma_phyisical_address, mmio + IO_DMA_DST);
iowrite32(buffer_length, mmio + IO_DMA_CNT);
iowrite32(DMA_CMD | DMA_FROM_DEV | DMA_IRQ, mmio + IO_DMA_CMD);
test->dma_status = DMA_BUSY;
wait_event_interruptible_timeout(test->rw_queue, test->dma_status!=DMA_BUSY, msecs_to_jiffies(10));
copy_to_user(user_data->buffer, test->allocated_buffer, buffer_length);
break;
default:
pr_info("Error: unknown command\n");
goto l_release_buffer;
}
dma_free_coherent(&(pdev->dev), user_data->buffer_length, test->allocated_buffer, dma_phyisical_address);
return 0;
l_release_buffer:
dma_free_coherent(&(pdev->dev), user_data->buffer_length, test->allocated_buffer, dma_phyisical_address);
return EFAULT;
}
and also defined a small user code to check out the ioctl:
//const size_t TEST_DATA_SIZE = 8;
//const uint8_t test_data[TEST_DATA_SIZE]= {0x86,0x44,0x55,0x11,0x10,0x78,0x96, 0xFE};
struct user_data{
uint8_t *buffer;
size_t buffer_length;
};
#define IOCTL_LOAD_MODEL _IOW('q', 1, struct user_data*)
#define IOCTL_READ_MODEL _IOR('q', 2, struct user_data*)
int main(){
FILE *binary_file;
struct stat binary_file_stat;
size_t file_size;
struct user_data data_to_send;
struct user_data data_to_receive;
int ioctl_load_model_rc;
int ioctl_read_model_rc;
int exit_code = 0;
int device_fd = open("/dev/lkmc_pci",O_RDWR);
if(device_fd < 0){
printf("could not open device: %s\n", strerror(errno));
exit_code = 1;
goto l_exit;
}
binary_file = fopen("main.bin","rb");
if(NULL == binary_file){
printf("could not open binary file:%s\n",strerror(errno));
goto l_close_device;
}
fseek(binary_file, 0 , SEEK_END);
file_size = ftell(binary_file);
fseek(binary_file, 0 , SEEK_SET);// needed for next read from beginning of file
printf("file size is:%ld\n", file_size);
data_to_send.buffer = (uint8_t*) malloc(file_size);
if(data_to_send.buffer == NULL){
printf("Error: could not allocate memory for data to send\n");
exit_code = 1;
goto l_close_device;
}
data_to_send.buffer_length = file_size;
int read_bytes = fread(data_to_send.buffer,1,data_to_send.buffer_length,binary_file);
if (read_bytes <=0 || read_bytes != data_to_send.buffer_length){
printf("Error:read %d/%ld bytes",read_bytes,data_to_send.buffer_length);
exit_code = 1;
goto l_release_write;
}
printf("read %d bytes\n",read_bytes);
ioctl_load_model_rc = ioctl(device_fd,IOCTL_LOAD_MODEL, &data_to_send);
if(ioctl_load_model_rc != 0){
printf("Error: ioctl to write to device failed");
exit_code = 1;
goto l_release_write;
}
data_to_receive.buffer = (uint8_t*)malloc(file_size);
if(data_to_receive.buffer == NULL){
printf("Error: could not allocate memory for data to receive\n");
exit_code = 1;
goto l_release_read;
}
data_to_receive.buffer_length = file_size;
ioctl_read_model_rc = ioctl(device_fd, IOCTL_READ_MODEL, &data_to_receive);
if(ioctl_read_model_rc != 0){
printf("Error: could not read data from the device\n");
goto l_release_read;
}
for(int i=0; i <data_to_receive.buffer_length; i++){
if(data_to_receive.buffer[i] != data_to_send.buffer[i]){
printf("Error:diff in the %d-th place\n",i);
exit_code = i + 1;
goto l_release_read;
}
}
if(exit_code == 0){
printf("successfull copy\n");
}
l_release_read:
free(data_to_receive.buffer);
l_release_write:
free(data_to_send.buffer);
l_close_device:
close(device_fd);
l_exit:
return exit_code;
return 0;
}
and of course treatment in the irq_handler:
if (test->major == major) {
irq_status = ioread32(mmio + IO_IRQ_STATUS);
pr_info("irq_handler irq = %d dev = %d irq_status = %llx\n",
irq, devi, (unsigned long long)irq_status);
/* Must do this ACK, or else the interrupts just keeps firing. */
iowrite32(irq_status, mmio + IO_IRQ_ACK);
test->dma_status = DMA_FREE;
wake_up_interruptible(&(test->rw_queue));
However, when I try to write to the DMA the ioctl I get this error:
general protection fault, probably for non-canonical address 0xff13eb6400029cc0: 0000 [#1] SMP NOPTI
when looking at call trace it doesn't make any sense:
[ 43.512780] Call Trace:
[ 43.512780] _raw_spin_lock_irqsave+0x1d/0x30
[ 43.512780] prepare_to_wait_event+0x16/0xf0
[ 43.512780] ioctl_load_model+0x313/0x490 [qemu_edu]
[ 43.512780] ? wait_woken+0x80/0x80
[ 43.512780] ksys_ioctl+0x8d/0xb0
[ 43.512780] __x64_sys_ioctl+0x11/0x20
[ 43.512780] do_syscall_64+0x3e/0x70
[ 43.512780] entry_SYSCALL_64_after_hwframe+0x44/0xa9
[ 43.512780] RIP: 0033:0x44cc97
[ 43.512780] Code: 4f 98 02 00 85 c0 78 df 48 83 c4 08 48 89 d8 5b 5d c3 90 48 89 e8 48 f7 d8 48 39 c3 0f 92 c0 eb b4 66 90 b8 10 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48
What is going wrong?
edit: here are probe intialization code:
static int pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
{
struct test * test= (struct test*)NULL;
int rv;
dev_info(&(dev->dev), "pci_probe\n");
dev_t dev_number;
test = (struct test*)(kmalloc(sizeof(struct test),GFP_KERNEL));
if(test == NULL){
pr_info("Could not allocate data structruce for driver, need to ");
return 1;
}
major = register_chrdev(0, CDEV_NAME, &fops);
pdev = dev;
if (pci_enable_device(dev) < 0) {
dev_err(&(dev->dev), "pci_enable_device\n");
goto error;
}
if (pci_request_region(dev, BAR, "myregion0")) {
dev_err(&(dev->dev), "pci_request_region\n");
goto error;
}
mmio = pci_iomap(dev, BAR, pci_resource_len(dev, BAR));
test->major = major;
if (request_irq(dev->irq, irq_handler, IRQF_SHARED, "pci_irq_handler0", (void *)test) < 0) {
dev_err(&(dev->dev), "request_irq\n");
goto error;
}
test->dma_status = DMA_FREE;
init_waitqueue_head(&queue);
test->rw_queue = &queue;
/* Optional sanity checks. The PCI is ready now, all of this could also be called from fops. */
{
unsigned i;
u8 val;
/* Check that we are using MEM instead of IO.
*
* In QEMU, the type is defiened by either:
*
* - PCI_BASE_ADDRESS_SPACE_IO
* - PCI_BASE_ADDRESS_SPACE_MEMORY
*/
if ((pci_resource_flags(dev, BAR) & IORESOURCE_MEM) != IORESOURCE_MEM) {
dev_err(&(dev->dev), "pci_resource_flags\n");
goto error;
}
/* 1Mb, as defined by the "1 << 20" in QEMU's memory_region_init_io. Same as pci_resource_len. */
resource_size_t start = pci_resource_start(dev, BAR);
resource_size_t end = pci_resource_end(dev, BAR);
pr_info("start %llx, end %llx, length %llx\n", (unsigned long long)start, (unsigned long long)end, (unsigned long)(end + 1 - start));
/* The PCI standardized 64 bytes of the configuration space, see LDD3. */
for (i = 0; i < 64u; ++i) {
pci_read_config_byte(dev, i, &val);
pr_info("config %x %x\n", i, val);
}
pr_info("dev->irq %x\n", dev->irq);
/* Initial value of the IO memory. */
for (i = 0; i < 0x28; i += 4) {
pr_info("io %x %x\n", i, ioread32((void*)(mmio + i)));
}
pci_set_master(dev);
}
return 0;
unregister_region:
unregister_chrdev_region(test->cdev_number, 1);
return rv;
error:
return 1;
}
test data structure is:
struct test {
int major;
void* allocated_buffer;
struct cdev cdev; /* character device embedded struct */
dev_t cdev_number;/* major:minor*/
size_t buffer_length;
dma_status_t dma_status;
wait_queue_head_t* rw_queue;
};
struct user_data{
uint8_t *buffer;
size_t buffer_length;
};