Add the linux device driver and netfpga files

This adds the device driver and netfpga files from the master thesis
(in Dutch) "Implementatie van de Recursive Internet Architecture op
een FPGA platform" by Alexander D'hoore.

1 files changed, 940 insertions, 0 deletions

diff --git a/linux/raptor.c b/linux/raptor.c
new file mode 100644
index 0000000..cb28c44
--- /dev/null
+++ b/linux/raptor.c
@@ -0,0 +1,940 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2017
+ *
+ * Raptor device driver
+ *
+ *    Alexander D'hoore <[email protected]>
+ *    Dimitri Staessens <[email protected]>
+ *    Sander Vrijders   <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/wait.h>
+#include <linux/time.h>
+#include <linux/string.h>
+#include <linux/cdev.h>
+#include <linux/kthread.h>
+#include <linux/uaccess.h>
+
+#include "phy_conf.h"
+
+MODULE_LICENSE("GPL");
+
+#define RAPTOR_VENDOR_ID 0x10EE
+#define RAPTOR_DEVICE_ID 0xAD02
+#define RAPTOR_BAR 0
+
+#define BAR_MDIO 30
+
+#define CMD_STATUS 0
+#define CMD_SEND 1
+#define CMD_RECV 2
+#define CMD_MDIO 3
+
+#define IOCTL_SEND 0xAD420000
+#define IOCTL_RECV 0xAD430000
+#define IOCTL_SEND_DONE 0xAD440000
+#define IOCTL_RECV_DONE 0xAD450000
+#define IOCTL_RECV_NEED 0xAD460000
+#define IOCTL_DEBUG 0xAD470000
+
+#define BUFF_COUNT 16
+
+#define BUFF_EMPTY 0
+#define BUFF_ADDR 1
+#define BUFF_DATA 2
+#define BUFF_FULL 3
+
+#define PACKET_COUNT 1000
+
+struct packet;
+
+struct packet {
+        uint64_t u_addr;
+        uint16_t length;
+        struct page* page;
+        dma_addr_t dma_addr;
+        struct packet* next;
+};
+
+struct queue {
+        struct packet* first;
+        struct packet* last;
+        uint64_t count;
+};
+
+struct raptor {
+        dev_t base;
+        struct class * class;
+
+        dev_t dev;
+        struct device * device;
+        struct cdev cdev;
+
+        uint64_t *bar;
+        struct pci_dev *pdev;
+
+        bool interrupt;
+
+        struct queue send_queue;
+        struct queue recv_queue;
+        struct queue send_done_queue;
+        struct queue recv_done_queue;
+        struct queue send_free_queue;
+        struct queue recv_free_queue;
+
+        struct packet* send_packets;
+        struct packet* recv_packets;
+
+        uint64_t recv_need;
+
+        spinlock_t lock;
+        wait_queue_head_t wait;
+
+        struct task_struct* kthread;
+
+        uint64_t int_count;
+        uint64_t send_busy;
+        uint64_t recv_busy;
+};
+
+struct raptor raptor;
+
+// -------------------------------------------
+
+static void queue_push(struct queue* queue, struct packet* packet)
+{
+        spin_lock_irq(&raptor.lock);
+
+        if (queue->count == 0) {
+                queue->first = packet;
+                queue->last = packet;
+        }
+        else {
+                queue->last->next = packet;
+                queue->last = packet;
+        }
+        queue->count++;
+
+        spin_unlock_irq(&raptor.lock);
+        wake_up(&raptor.wait);
+}
+
+static struct packet* queue_pop(struct queue* queue, bool block)
+{
+        struct packet* packet;
+
+        spin_lock_irq(&raptor.lock);
+
+        if (block) {
+                if (wait_event_interruptible_lock_irq_timeout(
+                                raptor.wait, queue->count > 0, raptor.lock, HZ) <= 0) {
+
+                        spin_unlock_irq(&raptor.lock);
+                        return NULL;
+                }
+        }
+        else {
+                if (queue->count == 0) {
+                        spin_unlock_irq(&raptor.lock);
+                        return NULL;
+                }
+        }
+
+        packet = queue->first;
+        queue->first = packet->next;
+        queue->count--;
+
+        spin_unlock_irq(&raptor.lock);
+        wake_up(&raptor.wait);
+        return packet;
+}
+
+// -------------------------------------------
+
+uint64_t raptor_read(uint64_t offset)
+{
+        return raptor.bar[offset];
+}
+
+void raptor_write(uint64_t offset, uint64_t data)
+{
+        raptor.bar[offset] = data;
+}
+
+uint64_t endian64(uint64_t data)
+{
+        uint64_t byte0 = (data & 0x00000000000000FF) << 7*8;
+        uint64_t byte1 = (data & 0x000000000000FF00) << 5*8;
+        uint64_t byte2 = (data & 0x0000000000FF0000) << 3*8;
+        uint64_t byte3 = (data & 0x00000000FF000000) << 1*8;
+        uint64_t byte4 = (data & 0x000000FF00000000) >> 1*8;
+        uint64_t byte5 = (data & 0x0000FF0000000000) >> 3*8;
+        uint64_t byte6 = (data & 0x00FF000000000000) >> 5*8;
+        uint64_t byte7 = (data & 0xFF00000000000000) >> 7*8;
+
+        return byte0 | byte1 | byte2 | byte3 | byte4 | byte5 | byte6 | byte7;
+}
+
+static irqreturn_t raptor_interrupt(int irq, void *dev_id)
+{
+        spin_lock(&raptor.lock);
+        raptor.interrupt = true;
+        spin_unlock(&raptor.lock);
+        wake_up(&raptor.wait);
+
+        return IRQ_HANDLED;
+}
+
+void mdio_raw(unsigned operation, unsigned phy_addr, unsigned dev_addr, unsigned data)
+{
+        unsigned command = (operation << 26) | (phy_addr << 21) | (dev_addr << 16) | (data & 0xFFFF);
+
+        raptor_write(CMD_MDIO, endian64((uint64_t)command));
+        msleep(1);
+}
+
+void mdio_write(unsigned phy_addr, unsigned dev_addr, unsigned data_addr, unsigned data)
+{
+        mdio_raw(0, phy_addr, dev_addr, data_addr);
+        mdio_raw(1, phy_addr, dev_addr, data);
+}
+
+unsigned mdio_read(unsigned phy_addr, unsigned dev_addr, unsigned data_addr)
+{
+        mdio_raw(0, phy_addr, dev_addr, data_addr);
+        mdio_raw(3, phy_addr, dev_addr, 0);
+
+        return (unsigned)endian64(raptor_read(BAR_MDIO));
+}
+
+int mdio_i2c_read(uint32_t phy_addr, uint16_t dev_addr, uint16_t word_addr, uint16_t * data)
+{
+        uint16_t stat;
+        int i;
+
+        mdio_write(phy_addr, 1, AEL_I2C_CTRL, (dev_addr << 8) | (1 << 8) | word_addr);
+
+        for (i = 0; i < 20; i++) {
+                msleep(2);
+                stat = mdio_read(phy_addr, 1, AEL_I2C_STAT);
+
+                if ((stat & 3) == 1){
+                        stat = mdio_read(phy_addr, 1, AEL_I2C_DATA);
+
+                        *data = stat >> 8;
+                        return 0;
+                }
+        }
+        return -1;
+}
+
+void mdio_initialize(uint32_t phy_addr, int mode)
+{
+        int size, i;
+
+        // Step 1
+        size = sizeof(reset) / sizeof(uint16_t);
+
+        for(i = 0; i < size; i += 2) {
+                mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, reset[i], reset[i+1]);
+        }
+
+        msleep(5);
+
+        // Step 2
+        if (mode == MODE_SR) {
+                size = sizeof(sr_edc) / sizeof(uint16_t);
+
+                for(i = 0; i < size; i += 2) {
+                        mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, sr_edc[i], sr_edc[i+1]);
+                }
+        }
+        else if (mode == MODE_TWINAX) {
+                size = sizeof(twinax_edc) / sizeof(uint16_t);
+
+                for(i = 0; i < size; i += 2) {
+                        mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, twinax_edc[i], twinax_edc[i+1]);
+                }
+        }
+
+        // Step 3
+        size = sizeof(regs1) / sizeof(uint16_t);
+        for(i = 0; i < size; i+=2) {
+                mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, regs1[i], regs1[i+1]);
+        }
+
+        msleep(5);
+}
+
+int phy_configuration(void)
+{
+        int port, dev;
+        uint16_t value = 0;
+        char port_mode;
+
+        // check if we need initialization
+        value = mdio_read(2, PMA_MDIO_DEVICE_ADDRESS, AEL_MICRO_CONTROLLER_CTL_ADDRESS);
+
+        //printk(KERN_INFO "raptor: 0xc04a: %04x\n",value);
+        if (value & 0x8000) {   // uC held in reset
+                printk(KERN_INFO "raptor: Programming the AEL2005 PHY chips...\n");
+
+                for (port = 0; port < 4; port++) {
+                        if (port == 0) dev = 2;
+                        else if (port == 1) dev = 1;
+                        else if (port == 2) dev = 0;
+                        else dev = 3;
+
+                        value = 0;
+
+                        // check if we have a 10GBASE-SR cable
+                        mdio_i2c_read(dev, MODULE_DEV_ADDR, 0x3, &value);
+
+                        if((value >> 4) == 1) {
+                                port_mode = MODE_SR;
+                        }
+                        else {
+                                port_mode = MODE_TWINAX;
+                        }
+
+                        printk(KERN_INFO "raptor: Programming PHY %d...\n", port);
+
+                        mdio_initialize(dev, port_mode);
+                }
+
+                return 0;
+        }
+
+        return -1;
+}
+
+static void read_debug(void)
+{
+        uint64_t status;
+        uint64_t address;
+        uint64_t data;
+        uint64_t cmd;
+        uint64_t buff;
+        uint64_t length;
+        uint64_t cmpl_count;
+        uint64_t cmpl_data;
+        uint64_t cmpl_tag;
+        uint64_t cmpl_state;
+        uint64_t write_count;
+        uint64_t write_length;
+        uint64_t write_addr;
+        uint64_t write_data;
+        uint64_t accept_count;
+        uint64_t done_count;
+
+        status = endian64(raptor_read(0));
+        address = endian64(raptor_read(1));
+        data = endian64(raptor_read(2));
+        cmd = endian64(raptor_read(3));
+        buff = endian64(raptor_read(4));
+        length = endian64(raptor_read(5));
+        cmpl_count = endian64(raptor_read(10));
+        cmpl_data = endian64(raptor_read(11));
+        cmpl_tag = endian64(raptor_read(12));
+        cmpl_state = endian64(raptor_read(13));
+
+        write_count = endian64(raptor_read(20));
+        write_length = endian64(raptor_read(21));
+        write_addr = endian64(raptor_read(22));
+        write_data = endian64(raptor_read(23));
+        accept_count = endian64(raptor_read(24));
+        done_count = endian64(raptor_read(25));
+
+        printk(KERN_INFO "raptor: read_debug:\n");
+        printk(KERN_INFO "raptor:     status = 0x%llx.\n", status);
+        printk(KERN_INFO "raptor:     address = 0x%llx.\n", address);
+        printk(KERN_INFO "raptor:     data = 0x%llx.\n", data);
+        printk(KERN_INFO "raptor:     cmd = 0x%llx.\n", cmd);
+        printk(KERN_INFO "raptor:     buff = 0x%llx.\n", buff);
+        printk(KERN_INFO "raptor:     length = %llu.\n", length);
+        printk(KERN_INFO "raptor:     cmpl_count = %llu.\n", cmpl_count);
+        printk(KERN_INFO "raptor:     cmpl_data = 0x%llx.\n", cmpl_data);
+        printk(KERN_INFO "raptor:     cmpl_tag = 0x%llx.\n", cmpl_tag);
+        printk(KERN_INFO "raptor:     cmpl_state = 0x%llx.\n", cmpl_state);
+        printk(KERN_INFO "raptor:     write_count = %llu.\n", write_count);
+        printk(KERN_INFO "raptor:     write_length = %llu.\n", write_length);
+        printk(KERN_INFO "raptor:     write_addr = 0x%llx.\n", write_addr);
+        printk(KERN_INFO "raptor:     write_data = 0x%llx.\n", write_data);
+        printk(KERN_INFO "raptor:     accept_count = %llu.\n", accept_count);
+        printk(KERN_INFO "raptor:     done_count = %llu.\n", done_count);
+}
+
+#define get_send(status, i) ((status >> (i * 4 + 0)) & 0x3)
+#define get_recv(status, i) ((status >> (i * 4 + 2)) & 0x3)
+
+struct packet* send_packets[BUFF_COUNT];
+struct packet* recv_packets[BUFF_COUNT];
+uint8_t send_buff[BUFF_COUNT];
+uint8_t recv_buff[BUFF_COUNT];
+
+static int raptor_kthread(void* data)
+{
+        bool interrupt;
+        uint64_t status = 0;
+        uint8_t send_new;
+        uint8_t recv_new;
+        uint8_t send_old;
+        uint8_t recv_old;
+        uint64_t i;
+        bool send_full = false;
+        bool recv_full = false;
+        struct packet* packet;
+        struct device* dev = &raptor.pdev->dev;
+        uint64_t offset;
+        uint64_t rest;
+
+        printk(KERN_INFO "raptor_kthread: started\n");
+
+        memset(send_packets, 0, sizeof(send_packets));
+        memset(recv_packets, 0, sizeof(recv_packets));
+
+        raptor.interrupt = true;
+
+        while (true) {
+                spin_lock_irq(&raptor.lock);
+
+                if (wait_event_interruptible_lock_irq(
+                                raptor.wait,
+                                kthread_should_stop() ||
+                                raptor.interrupt ||
+                                (! send_full && raptor.send_queue.count > 0) ||
+                                (! recv_full && raptor.recv_queue.count > 0),
+                                raptor.lock) != 0) {
+
+                        printk(KERN_ERR "raptor_kthread: wait_event failed\n");
+                        spin_unlock_irq(&raptor.lock);
+                        break;
+                }
+
+                interrupt = raptor.interrupt;
+                raptor.interrupt = false;
+                spin_unlock_irq(&raptor.lock);
+
+                if (kthread_should_stop())
+                        break;
+
+                if (interrupt) {
+                        // read status
+                        // update done_queues
+                        // incr recv_need
+                        status = endian64(raptor_read(0));
+
+                        raptor.int_count++;
+
+                        for (i = 0; i < BUFF_COUNT; i++) {
+                                send_new = get_send(status, i);
+                                recv_new = get_recv(status, i);
+                                send_old = send_buff[i];
+                                recv_old = recv_buff[i];
+
+                                if (send_new != BUFF_EMPTY)
+                                        raptor.send_busy++;
+                                if (recv_new != BUFF_EMPTY)
+                                        raptor.recv_busy++;
+
+                                if ((send_old == BUFF_ADDR || send_old == BUFF_DATA) &&
+                                    (send_new == BUFF_FULL || send_new == BUFF_EMPTY) &&
+                                    send_packets[i] != NULL) {
+
+                                        packet = send_packets[i];
+                                        offset = packet->u_addr & (~PAGE_MASK);
+                                        rest = PAGE_SIZE - offset;
+
+                                        dma_unmap_page(dev, packet->dma_addr, rest, DMA_TO_DEVICE);
+                                        queue_push(&raptor.send_done_queue, packet);
+                                        send_packets[i] = NULL;
+                                }
+                                if ((recv_old == BUFF_ADDR || recv_old == BUFF_DATA) &&
+                                    (recv_new == BUFF_FULL || recv_new == BUFF_EMPTY) &&
+                                    recv_packets[i] != NULL) {
+
+                                        packet = recv_packets[i];
+                                        offset = packet->u_addr & (~PAGE_MASK);
+                                        rest = PAGE_SIZE - offset;
+
+                                        dma_unmap_page(dev, packet->dma_addr, rest, DMA_FROM_DEVICE);
+                                        queue_push(&raptor.recv_done_queue, packet);
+                                        recv_packets[i] = NULL;
+                                }
+                                if (recv_old != BUFF_FULL && recv_new == BUFF_FULL) {
+                                        spin_lock_irq(&raptor.lock);
+                                        raptor.recv_need++;
+                                        spin_unlock_irq(&raptor.lock);
+                                        wake_up(&raptor.wait);
+                                }
+
+                                send_buff[i] = send_new;
+                                recv_buff[i] = recv_new;
+                        }
+                }
+
+                // handle send_queue
+                send_full = true;
+
+                for (i = 0; i < BUFF_COUNT; i++) {
+                        if (send_buff[i] == BUFF_EMPTY && send_packets[i] == NULL) {
+
+                                packet = queue_pop(&raptor.send_queue, false);
+
+                                if (packet == NULL) {
+                                        send_full = false;
+                                        break;
+                                }
+
+                                // map page to dma_addr
+                                // write that to hardware
+
+                                offset = packet->u_addr & (~PAGE_MASK);
+                                rest = PAGE_SIZE - offset;
+
+                                packet->dma_addr = dma_map_page(dev, packet->page, offset, rest, DMA_TO_DEVICE);
+
+                                raptor_write(CMD_SEND | (i << 2) | (packet->length << 7), endian64(packet->dma_addr));
+
+                                send_buff[i] = BUFF_ADDR;
+                                send_packets[i] = packet;
+                        }
+                }
+
+                // handle recv_queue
+                recv_full = true;
+
+                for (i = 0; i < BUFF_COUNT; i++) {
+                        if (recv_buff[i] == BUFF_FULL && recv_packets[i] == NULL) {
+
+                                packet = queue_pop(&raptor.recv_queue, false);
+
+                                if (packet == NULL) {
+                                        recv_full = false;
+                                        break;
+                                }
+
+                                // map page to dma_addr
+                                // write that to hardware
+
+                                offset = packet->u_addr & (~PAGE_MASK);
+                                rest = PAGE_SIZE - offset;
+
+                                packet->dma_addr = dma_map_page(dev, packet->page, offset, rest, DMA_FROM_DEVICE);
+
+                                raptor_write(CMD_RECV | (i << 2), endian64(packet->dma_addr));
+
+                                recv_buff[i] = BUFF_ADDR;
+                                recv_packets[i] = packet;
+                        }
+                }
+        }
+
+        printk(KERN_INFO "raptor_kthread: stopped\n");
+        return 0;
+}
+
+static int pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+        if (pci_enable_device(pdev) != 0)
+                goto error_no_cleanup;
+
+        if (pci_request_region(pdev, RAPTOR_BAR, "raptor") != 0)
+                goto error_disable_device;
+
+        if ((raptor.bar = pci_ioremap_bar(pdev, RAPTOR_BAR)) == NULL)
+                goto error_release_region;
+
+        if (pci_enable_msi(pdev) < 0)
+                goto error_iounmap;
+
+        if (request_irq(pdev->irq, raptor_interrupt, 0, "raptor", NULL) < 0)
+                goto error_disable_msi;
+
+        if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0)
+                goto error_free_irq;
+
+        if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0)
+                goto error_free_irq;
+
+        pci_set_master(pdev);
+        raptor.pdev = pdev;
+
+        phy_configuration();
+
+        read_debug();
+
+        raptor.kthread = kthread_run(raptor_kthread, NULL, "raptor_kthread");
+
+        if (IS_ERR(raptor.kthread))
+                goto error_clear_master;
+
+        printk(KERN_INFO "raptor: pci_probe succes\n");
+
+        return 0;
+
+error_clear_master:
+        pci_clear_master(pdev);
+error_free_irq:
+        free_irq(pdev->irq, NULL);
+error_disable_msi:
+        pci_disable_msi(pdev);
+error_iounmap:
+        iounmap(raptor.bar);
+error_release_region:
+        pci_release_region(pdev, 0);
+error_disable_device:
+        pci_disable_device(pdev);
+error_no_cleanup:
+        raptor.bar = NULL;
+        raptor.pdev = NULL;
+        printk(KERN_ALERT "raptor: pci_probe FAILED\n");
+        return -1;
+}
+
+static void pci_remove(struct pci_dev *pdev)
+{
+        printk(KERN_INFO "raptor: pci_remove started...\n");
+
+        (void)kthread_stop(raptor.kthread);
+
+        pci_clear_master(pdev);
+        free_irq(pdev->irq, NULL);
+        pci_disable_msi(pdev);
+        iounmap(raptor.bar);
+        pci_release_region(pdev, 0);
+        pci_disable_device(pdev);
+        raptor.bar = NULL;
+        raptor.pdev = NULL;
+
+        printk(KERN_INFO "raptor: pci_remove done\n");
+}
+
+static pci_ers_result_t pci_error_detected(struct pci_dev *pdev, enum pci_channel_state state)
+{
+        printk(KERN_ALERT "raptor: pcie error %d\n", state);
+
+        return PCI_ERS_RESULT_RECOVERED;
+}
+
+static int raptor_send(uint64_t count, uint64_t* u_addrs)
+{
+        uint64_t i;
+        uint64_t u_addr;
+        uint16_t bytes;
+        uint16_t length;
+        struct page* page;
+        struct packet* packet;
+
+        for (i = 0; i < count; i++) {
+                if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) {
+                        printk(KERN_ERR "raptor_send: copy_from_user u_addr failed\n");
+                        return i;
+                }
+
+                if (copy_from_user(&bytes, (void*)u_addr, sizeof(bytes)) != 0) {
+                        printk(KERN_ERR "raptor_send: copy_from_user bytes failed\n");
+                        return i;
+                }
+
+                if (get_user_pages_fast(u_addr, 1, false, &page) != 1) {
+                        printk(KERN_ERR "raptor_send: get_user_pages_fast failed\n");
+                        return i;
+                }
+
+                packet = queue_pop(&raptor.send_free_queue, true);
+
+                if (packet == NULL) {
+                        printk(KERN_ERR "raptor_send: queue_pop send_free_queue failed\n");
+                        put_page(page);
+                        return i;
+                }
+
+                length = (bytes >> 3);
+
+                if (bytes & 0x7)
+                        length += 1;
+
+                packet->u_addr = u_addr;
+                packet->length = length;
+                packet->page = page;
+
+                queue_push(&raptor.send_queue, packet);
+        }
+        return count;
+}
+
+static int raptor_recv(uint64_t count, uint64_t* u_addrs)
+{
+        uint64_t i;
+        uint64_t u_addr;
+        struct page* page;
+        struct packet* packet;
+
+        for (i = 0; i < count; i++) {
+                if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) {
+                        printk(KERN_ERR "raptor_recv: copy_from_user u_addr failed\n");
+                        return i;
+                }
+
+                if (get_user_pages_fast(u_addr, 1, true, &page) != 1) {
+                        printk(KERN_ERR "raptor_recv: get_user_pages_fast failed\n");
+                        return i;
+                }
+
+                packet = queue_pop(&raptor.recv_free_queue, true);
+
+                if (packet == NULL) {
+                        printk(KERN_ERR "raptor_recv: queue_pop recv_free_queue failed\n");
+                        put_page(page);
+                        return i;
+                }
+
+                packet->u_addr = u_addr;
+                packet->page = page;
+
+                queue_push(&raptor.recv_queue, packet);
+        }
+        return count;
+}
+
+static int raptor_send_done(uint64_t count, uint64_t* u_addrs)
+{
+        uint64_t i;
+        struct packet* packet;
+
+        for (i = 0; i < count; i++) {
+                packet = queue_pop(&raptor.send_done_queue, i == 0);
+
+                if (packet == NULL)
+                        return i;
+
+                put_page(packet->page);
+
+                if (copy_to_user(&u_addrs[i], &packet->u_addr, sizeof(packet->u_addr)) != 0) {
+                        printk(KERN_ERR "raptor_send_done: copy_to_user u_addr failed\n");
+                        return i;
+                }
+
+                queue_push(&raptor.send_free_queue, packet);
+        }
+        return count;
+}
+
+static int raptor_recv_done(uint64_t count, uint64_t* u_addrs)
+{
+        uint64_t i;
+        struct packet* packet;
+
+        for (i = 0; i < count; i++) {
+                packet = queue_pop(&raptor.recv_done_queue, i == 0);
+
+                if (packet == NULL)
+                        return i;
+
+                put_page(packet->page);
+
+                if (copy_to_user(&u_addrs[i], &packet->u_addr, sizeof(packet->u_addr)) != 0) {
+                        printk(KERN_ERR "raptor_recv_done: copy_to_user u_addr failed\n");
+                        return i;
+                }
+
+                queue_push(&raptor.recv_free_queue, packet);
+        }
+        return count;
+}
+
+static int raptor_recv_need(uint64_t count)
+{
+        uint64_t ret;
+
+        spin_lock_irq(&raptor.lock);
+
+        if (wait_event_interruptible_lock_irq_timeout(
+                        raptor.wait, raptor.recv_need > 0, raptor.lock, HZ) <= 0) {
+
+                spin_unlock_irq(&raptor.lock);
+                return -1;
+        }
+
+        if (raptor.recv_need > count) {
+                ret = count;
+                raptor.recv_need -= count;
+        }
+        else {
+                ret = raptor.recv_need;
+                raptor.recv_need = 0;
+        }
+
+        spin_unlock_irq(&raptor.lock);
+        return ret;
+}
+
+static int raptor_debug(uint64_t* u_array)
+{
+        if (copy_to_user(&u_array[0], &raptor.int_count, sizeof(uint64_t)) ||
+            copy_to_user(&u_array[1], &raptor.send_busy, sizeof(uint64_t)) ||
+            copy_to_user(&u_array[2], &raptor.recv_busy, sizeof(uint64_t))) {
+
+                printk(KERN_ERR "raptor_debug: copy_to_user failed\n");
+                return -1;
+        }
+
+        raptor.int_count = 0;
+        raptor.send_busy = 0;
+        raptor.recv_busy = 0;
+
+        return 0;
+}
+
+
+static long raptor_ioctl(struct file * file, unsigned int cmd, unsigned long arg)
+{
+        uint64_t type = cmd & 0xFFFF0000;
+        uint64_t count = cmd & 0x0000FFFF;
+
+        if (raptor.bar == NULL) {
+                printk(KERN_ERR "raptor: no pcie device\n");
+                return -1;
+        }
+
+        switch (type) {
+                case IOCTL_SEND:
+                        return raptor_send(count, (uint64_t*)arg);
+                case IOCTL_RECV:
+                        return raptor_recv(count, (uint64_t*)arg);
+                case IOCTL_SEND_DONE:
+                        return raptor_send_done(count, (uint64_t*)arg);
+                case IOCTL_RECV_DONE:
+                        return raptor_recv_done(count, (uint64_t*)arg);
+                case IOCTL_RECV_NEED:
+                        return raptor_recv_need(count);
+                case IOCTL_DEBUG:
+                        return raptor_debug((uint64_t*)arg);
+                default:
+                        printk(KERN_ERR "raptor: unknown ioctl\n");
+                        return -1;
+        }
+}
+
+static struct pci_device_id pci_id_table[] = {
+        {PCI_DEVICE(RAPTOR_VENDOR_ID, RAPTOR_DEVICE_ID)},
+        {0}
+};
+MODULE_DEVICE_TABLE(pci, pci_id_table);
+
+static struct pci_error_handlers pci_err_handlers = {
+        .error_detected = pci_error_detected
+};
+
+static struct pci_driver pci_driver = {
+        .name = "raptor",
+        .id_table = pci_id_table,
+        .probe = pci_probe,
+        .remove = pci_remove,
+        .err_handler = &pci_err_handlers
+};
+
+static struct file_operations raptor_fops = {
+        .unlocked_ioctl = raptor_ioctl,
+};
+
+int init_module(void)
+{
+        int i;
+
+        printk(KERN_INFO "raptor: inserting module...\n");
+
+        memset(&raptor, 0, sizeof(raptor));
+
+        spin_lock_init(&raptor.lock);
+        init_waitqueue_head(&raptor.wait);
+
+        raptor.send_packets = kmalloc(PACKET_COUNT * sizeof(struct packet), GFP_KERNEL);
+        raptor.recv_packets = kmalloc(PACKET_COUNT * sizeof(struct packet), GFP_KERNEL);
+
+        if (raptor.send_packets == NULL || raptor.recv_packets == NULL) {
+                printk(KERN_ERR "raptor: failed to allocate packets\n");
+                return -1;
+        }
+
+        for (i = 0; i < PACKET_COUNT; i++) {
+                queue_push(&raptor.send_free_queue, &raptor.send_packets[i]);
+                queue_push(&raptor.recv_free_queue, &raptor.recv_packets[i]);
+        }
+
+        if (alloc_chrdev_region(&raptor.base, 0, 1, "raptor") < 0) {
+                printk(KERN_ERR "raptor: alloc_chrdev_region failed\n");
+                return -1;
+        }
+
+        raptor.class = class_create(THIS_MODULE, "raptor");
+
+        if (IS_ERR(raptor.class)) {
+                printk(KERN_ERR "raptor: class_create failed\n");
+                return -1;
+        }
+
+        raptor.dev = MKDEV(MAJOR(raptor.base), 0);
+
+        raptor.device = device_create(raptor.class, NULL, raptor.dev, NULL, "raptor");
+
+        if (IS_ERR(raptor.device)) {
+                printk(KERN_ERR "raptor: device_create failed\n");
+                return -1;
+        }
+
+        cdev_init(&raptor.cdev, &raptor_fops);
+
+        if (cdev_add(&raptor.cdev, raptor.dev, 1) < 0) {
+                printk(KERN_ERR "raptor: cdev_add failed\n");
+                return -1;
+        }
+
+        if (pci_register_driver(&pci_driver) != 0) {
+                printk(KERN_ALERT "raptor: pci driver NOT loaded\n");
+                return -1;
+        }
+
+        printk(KERN_INFO "raptor: module inserted\n");
+
+        return 0;
+}
+
+void cleanup_module(void)
+{
+        printk(KERN_INFO "raptor: removing module...\n");
+
+        pci_unregister_driver(&pci_driver);
+
+        cdev_del(&raptor.cdev);
+        device_destroy(raptor.class, raptor.dev);
+
+        class_destroy(raptor.class);
+        unregister_chrdev_region(raptor.base, 1);
+
+        kfree(raptor.send_packets);
+        kfree(raptor.recv_packets);
+
+        memset(&raptor, 0, sizeof(raptor));
+
+        printk(KERN_INFO "raptor: module removed\n");
+}