/*
 * Copyright (C) Mar 2012 Mellanox Technologies Ltd. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *  Version: $Id: $
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/dma-mapping.h>
#include <asm/io.h>
#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 18)
//"linux/uaccess.h" was added in 2.6.18
#include <linux/uaccess.h>
#else
#include <asm/uaccess.h>
#endif
#include <linux/pci.h>
#include <linux/mm.h>
#include <linux/delay.h>
#include <linux/dmapool.h>

#include "driver_common.h"
#include "mst_pciconf.h"
#define INIT PCICONF_INIT
#define STOP PCICONF_STOP
#define READ4 PCICONF_READ4
#define READ4_NEW PCICONF_READ4_NEW
#define WRITE4 PCICONF_WRITE4
#define WRITE4_NEW PCICONF_WRITE4_NEW
#define MODIFY PCICONF_MODIFY
#define READ4_BUFFER PCICONF_READ4_BUFFER
#define READ4_BUFFER_EX PCICONF_READ4_BUFFER_EX
#define WRITE4_BUFFER PCICONF_WRITE4_BUFFER
#define MST_PARAMS PCICONF_MST_PARAMS
#define MST_META_DATA PCICONF_MST_META_DATA
#define GET_DMA_PAGES PCICONF_GET_DMA_PAGES
#define RELEASE_DMA_PAGES PCICONF_RELEASE_DMA_PAGES
#define READ_DWORD_FROM_CONFIG_SPACE PCICONF_READ_DWORD_FROM_CONFIG_SPACE

/* Versions */
#define MST_HDR_VERSION 1

/* Command: MST_META_DATA versions*/
#define MST_META_DATA_VERSION_MAJOR 1
#define MST_META_DATA_VERSION_MINOR 0
#define MST_API_VERSION_MAJOR 1
#define MST_API_VERSION_MINOR 0

static char* name = "mst_pciconf";

#ifndef DMA_BIT_MASK
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL << (n)) - 1))
#endif

#if defined(CONFIG_COMPAT) && CONFIG_COMPAT && !(defined(HAVE_COMPAT_IOCTL) && HAVE_COMPAT_IOCTL)
static unsigned int ioctl_cmds[] = {INIT, READ4, WRITE4, MODIFY, READ4_OLD, WRITE4_OLD, MODIFY_OLD};
#endif

/* Allow minor numbers 0-255 */
#define MAXMINOR 256
#define BUFFER_SIZE 256
#define MLNX_VENDOR_SPECIFIC_CAP_ID 0x9
#define CRSPACE_DOMAIN 0x2
#define AS_ICMD 0x3
#define AS_CR_SPACE 0x2
#define AS_SEMAPHORE 0xa

#define DMA_MBOX_SIZE (1 << PAGE_SHIFT)

MODULE_AUTHOR("Adrian Chiris - Mellanox Technologies LTD");
MODULE_DESCRIPTION("Mellanox configuration registers access driver (pci conf)");

struct dma_props_internal
{
    void* mem_pa;
    dma_addr_t dma_map;
    unsigned int mem_size;
};

#ifdef RETPOLINE_MLNX
MODULE_INFO(retpoline, "Y");
#endif

struct dma_page
{
    struct page** page_list;
    dma_addr_t dma_addr[PCICONF_MAX_PAGES_SIZE];
};

struct dev_data
{
    struct pci_dev* pci_dev;
    int addr_reg;
    int data_reg;
    int wo_addr;
    volatile int valid; // dotanb-cr: use pci_dev as valid
    struct semaphore sem;
    char buf[BUFFER_SIZE];
    int bufused;
    /* Vendor specific capability address */
    int vendor_specific_cap;
    /* status mask on VSEC supported capabilities*/
    u_int32_t vsec_cap_mask;
    /* multipurpose DMA properity */
    struct dma_props_internal dma_props[MST_DMA_END];
    // Allocated pages for the user space.
    struct dma_page dma_page;
};

static struct dev_data devices[MAXMINOR]; // dotanb-cr: use dynamic alloc

typedef enum
{
    S_ICMD = 0x1,
    S_CRSPACE = 0x2,
    S_ALL_ICMD = 0x3,
    S_NODNIC_INIT_SEG = 0x4,
    S_EXPANSION_ROM = 0x5,
    S_ND_CRSPACE = 0x6,
    S_SCAN_CRSPACE = 0x7,
    S_GLOBAL_SEMAPHORE = 0xa,
    S_MAC = 0xf,
    S_END
} Space;

typedef enum
{
    VC_INITIALIZED = 0x0,
    VC_ICMD_SPACE_SUPPORTED = 0x1,
    VC_CRSPACE_SPACE_SUPPORTED = 0x2,
    VC_ALL_ICMD_SPACE_SUPPORTED = 0x3,
    VC_NODNIC_INIT_SEG_SPACE_SUPPORTED = 0x4,
    VC_EXPANSION_ROM_SPACE_SUPPORTED = 0x5,
    VC_ND_CRSPACE_SPACE_SUPPORTED = 0x6,
    VC_SCAN_CRSPACE_SPACE_SUPPORTED = 0x7,
    VC_GLOBAL_SEMAPHORE_SPACE_SUPPORTED = 0x8,
    VC_MAC_SPACE_SUPPORTED = 0x9,
} VSCCap;

/*
 * Read/Write CR-Space (Old Cap)
 */
static int write4_old(struct dev_data* dev, unsigned int offset, unsigned int data)
{
    int ret;

    if (dev->wo_addr)
    {
        /*
         * Write operation with new WO GW
         * 1. Write data
         * 2. Write address
         */

        // printk("WRITE DATA: %#x, TO %#x\n", data, dev->data_reg);
        ret = pci_write_config_dword(dev->pci_dev, dev->data_reg, data);
        if (ret)
            return ret;
        // printk("WRITE ADDR: %#x, to %#x\n", offset, dev->addr_reg);
        ret = pci_write_config_dword(dev->pci_dev, dev->addr_reg, offset);
    }
    else
    {
        ret = pci_write_config_dword(dev->pci_dev, dev->addr_reg, offset);
        if (ret)
            return ret;

        ret = pci_write_config_dword(dev->pci_dev, dev->data_reg, data);
    }
    return ret;
}

static int read4_old(struct dev_data* dev, unsigned int offset, unsigned int* data)
{
    int ret;
    unsigned int newOffset = offset;
    if (dev->wo_addr)
    {
        /*
         * Read operation, Address LSB should be 1
         */
        newOffset = offset | 0x1;
    }
    // printk("WRITE ADDR: %#x, to %#x\n", newOffset, dev->addr_reg);
    ret = pci_write_config_dword(dev->pci_dev, dev->addr_reg, newOffset);
    if (ret)
        return ret;

    ret = pci_read_config_dword(dev->pci_dev, dev->data_reg, data);
    // printk("READ DATA: %#x, from %#x\n", *data, dev->data_reg);
    return ret;
}

/*
 * Read/Write Address-Domain (New Cap)
 */

// BIT Slicing macros
#define ONES32(size) ((size) ? (0xffffffff >> (32 - (size))) : 0)
#define MASK32(offset, size) (ONES32(size) << (offset))

#define EXTRACT_C(source, offset, size) ((((unsigned)(source)) >> (offset)) & ONES32(size))
#define EXTRACT(src, start, len) (((len) == 32) ? (src) : EXTRACT_C(src, start, len))

#define MERGE_C(rsrc1, rsrc2, start, len) \
    ((((rsrc2) << (start)) & (MASK32((start), (len)))) | ((rsrc1) & (~MASK32((start), (len)))))
#define MERGE(rsrc1, rsrc2, start, len) (((len) == 32) ? (rsrc2) : MERGE_C(rsrc1, rsrc2, start, len))

/* PCI address space related enum*/
enum
{
    PCI_CAP_PTR = 0x34,
    PCI_HDR_SIZE = 0x40,
    PCI_EXT_SPACE_ADDR = 0xff,

    PCI_CTRL_OFFSET = 0x4, // for space / semaphore / auto-increment bit
    PCI_COUNTER_OFFSET = 0x8,
    PCI_SEMAPHORE_OFFSET = 0xc,
    PCI_ADDR_OFFSET = 0x10,
    PCI_DATA_OFFSET = 0x14,

    PCI_FLAG_BIT_OFFS = 31,

    PCI_SPACE_BIT_OFFS = 0,
    PCI_SPACE_BIT_LEN = 16,

    PCI_STATUS_BIT_OFFS = 29,
    PCI_STATUS_BIT_LEN = 3,
};

/* Mellanox vendor specific enum */
enum
{
    CAP_ID = 0x9,
    IFC_MAX_RETRIES = 0x10000,
    SEM_MAX_RETRIES = 0x1000
};

/* PCI operation enum(read or write)*/
enum
{
    READ_OP = 0,
    WRITE_OP = 1,
};

#define VSEC_MIN_SUPPORT_KERN(dev)                                                                                   \
    (((dev)->vsec_cap_mask & (1 << VC_INITIALIZED)) && ((dev)->vsec_cap_mask & (1 << VC_CRSPACE_SPACE_SUPPORTED)) && \
     ((dev)->vsec_cap_mask & (1 << VC_ALL_ICMD_SPACE_SUPPORTED)) &&                                                  \
     ((dev)->vsec_cap_mask & (1 << VC_GLOBAL_SEMAPHORE_SPACE_SUPPORTED)))

// VSEC supported macro
#define VSEC_FULLY_SUPPORTED(dev) (((dev)->vendor_specific_cap) && (VSEC_MIN_SUPPORT_KERN(dev)))

#define VSEC_ASSUMED_SUPPORTED(dev) \
    (((dev)->vendor_specific_cap) && (VSEC_MIN_SUPPORT_KERN(dev) || (!((dev)->vsec_cap_mask & (1 << VC_INITIALIZED)))))

static int _vendor_specific_sem(struct dev_data* dev, int state)
{
    u32 lock_val;
    u32 counter = 0;
    int retries = 0;
    int ret;
    if (!state)
    { // unlock
        ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_SEMAPHORE_OFFSET, 0);
        if (ret)
            return ret;
    }
    else
    { // lock
        do
        {
            if (retries > SEM_MAX_RETRIES)
            {
                return -1;
            }
            // read semaphore untill 0x0
            ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_SEMAPHORE_OFFSET, &lock_val);
            if (ret)
                return ret;

            if (lock_val)
            { // semaphore is taken
                retries++;
                msleep(1); // wait for current op to end
                continue;
            }
            // read ticket
            ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_COUNTER_OFFSET, &counter);
            if (ret)
                return ret;
            // write ticket to semaphore dword
            ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_SEMAPHORE_OFFSET, counter);
            if (ret)
                return ret;
            // read back semaphore make sure ticket == semaphore else repeat
            ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_SEMAPHORE_OFFSET, &lock_val);
            if (ret)
                return ret;
            retries++;
        } while (counter != lock_val);
    }
    return 0;
}

static int _wait_on_flag(struct dev_data* dev, u8 expected_val)
{
    int retries = 0;
    int ret;
    u32 flag;
    do
    {
        if (retries > IFC_MAX_RETRIES)
        {
            return -1;
        }

        ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_ADDR_OFFSET, &flag);
        if (ret)
            return ret;

        flag = EXTRACT(flag, PCI_FLAG_BIT_OFFS, 1);
        retries++;
        if ((retries & 0xf) == 0)
        { // dont sleep always
          // usleep_range(1,5);
        }
    } while (flag != expected_val);
    return 0;
}

static int _space_to_cap_offset(u16 space)
{
    switch (space)
    {
        case S_ICMD:
            return VC_ICMD_SPACE_SUPPORTED;
        case S_CRSPACE:
            return VC_CRSPACE_SPACE_SUPPORTED;
        case S_ALL_ICMD:
            return VC_ALL_ICMD_SPACE_SUPPORTED;
        case S_NODNIC_INIT_SEG:
            return VC_NODNIC_INIT_SEG_SPACE_SUPPORTED;
        case S_EXPANSION_ROM:
            return VC_EXPANSION_ROM_SPACE_SUPPORTED;
        case S_ND_CRSPACE:
            return VC_ND_CRSPACE_SPACE_SUPPORTED;
        case S_SCAN_CRSPACE:
            return VC_SCAN_CRSPACE_SPACE_SUPPORTED;
        case S_GLOBAL_SEMAPHORE:
            return VC_GLOBAL_SEMAPHORE_SPACE_SUPPORTED;
        case S_MAC:
            return VC_MAC_SPACE_SUPPORTED;
        default:
            return 0;
    }
}

static int _set_addr_space(struct dev_data* dev, u16 space)
{
    // read modify write
    u32 val;
    int ret;
    ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_CTRL_OFFSET, &val);
    if (ret)
        return ret;
    val = MERGE(val, space, PCI_SPACE_BIT_OFFS, PCI_SPACE_BIT_LEN);
    ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_CTRL_OFFSET, val);
    if (ret)
        return ret;
    // read status and make sure space is supported
    ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_CTRL_OFFSET, &val);
    if (ret)
        return ret;

    if (EXTRACT(val, PCI_STATUS_BIT_OFFS, PCI_STATUS_BIT_LEN) == 0)
    {
        // printk("[MST]: CRSPACE %d is not supported !\n", space);
        return -1;
    }
    // printk("[MST]: CRSPACE %d is supported !\n", space);
    return 0;
}

static int _pciconf_rw(struct dev_data* dev, unsigned int offset, u32* data, int rw)
{
    int ret = 0;
    u32 address = offset;

    // last 2 bits must be zero as we only allow 30 bits addresses
    if (EXTRACT(address, 30, 2))
    {
        return -1;
    }

    address = MERGE(address, (rw ? 1 : 0), PCI_FLAG_BIT_OFFS, 1);
    if (rw == WRITE_OP)
    {
        // write data
        ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_DATA_OFFSET, *data);
        if (ret)
            return ret;
        // write address
        ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_ADDR_OFFSET, address);
        if (ret)
            return ret;
        // wait on flag
        ret = _wait_on_flag(dev, 0);
    }
    else
    {
        // write address
        ret = pci_write_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_ADDR_OFFSET, address);
        if (ret)
            return ret;
        // wait on flag
        ret = _wait_on_flag(dev, 1);
        // read data
        ret = pci_read_config_dword(dev->pci_dev, dev->vendor_specific_cap + PCI_DATA_OFFSET, data);
        if (ret)
            return ret;
    }
    return ret;
}

static int _send_pci_cmd_int(struct dev_data* dev, int space, unsigned int offset, u32* data, int rw)
{
    int ret = 0;

    // take semaphore
    ret = _vendor_specific_sem(dev, 1);
    if (ret)
    {
        return ret;
    }
    // set address space
    ret = _set_addr_space(dev, space);
    if (ret)
    {
        goto cleanup;
    }
    // read/write the data
    ret = _pciconf_rw(dev, offset, data, rw);
cleanup:
    // clear semaphore
    _vendor_specific_sem(dev, 0);
    return ret;
}

static int _block_op(struct dev_data* dev, int space, unsigned int offset, int size, u32* data, int rw)
{
    int i;
    int ret = 0;
    int wrote_or_read = size;
    if (size % 4)
    {
        return -1;
    }
    // lock semaphore and set address space
    ret = _vendor_specific_sem(dev, 1);
    if (ret)
    {
        return -1;
    }
    // set address space
    ret = _set_addr_space(dev, space);
    if (ret)
    {
        wrote_or_read = -1;
        goto cleanup;
    }

    for (i = 0; i < size; i += 4)
    {
        if (_pciconf_rw(dev, offset + i, &(data[(i >> 2)]), rw))
        {
            wrote_or_read = i;
            goto cleanup;
        }
    }
cleanup:
    _vendor_specific_sem(dev, 0);
    return wrote_or_read;
}

static int write4_new(struct dev_data* dev, int addresss_domain, unsigned int offset, unsigned int data)
{
    int ret;

    ret = _send_pci_cmd_int(dev, addresss_domain, offset, &data, WRITE_OP);
    if (ret)
    {
        return -1;
    }
    return 0;
}

static int read4_new(struct dev_data* dev, int address_space, unsigned int offset, unsigned int* data)
{
    int ret;

    ret = _send_pci_cmd_int(dev, address_space, offset, data, READ_OP);
    if (ret)
    {
        return -1;
    }
    return 0;
}

static int write4_block_new(struct dev_data* dev, int address_space, unsigned int offset, int size, u32* data)
{
    return _block_op(dev, address_space, offset, size, data, WRITE_OP);
}

static int read4_block_new(struct dev_data* dev, int address_space, unsigned int offset, int size, u32* data)
{
    return _block_op(dev, address_space, offset, size, data, READ_OP);
}

// Turning on the space capability bit in vsec_cap_mask iff
// space capability supported
static int get_space_support_status(struct dev_data* dev, u_int16_t space)
{
    int status = _set_addr_space(dev, space) == 0 ? 1 : 0;
    dev->vsec_cap_mask |= (status << _space_to_cap_offset(space));
    return status;
}

static int get_vsec_cap_mask(struct dev_data* dev)
{
    int ret;
    // printk("[MST] Checking if the Vendor CAP %d supports the SPACES in devices\n", vend_cap);
    if (!dev->vendor_specific_cap)
    {
        return 0;
    }
    if (dev->vsec_cap_mask & (1 << VC_INITIALIZED))
    {
        return 0;
    }
    // take semaphore
    ret = _vendor_specific_sem(dev, 1);
    if (ret)
    {
        // printk("[MST] Failed to lock semaphore\n");
        return 1;
    }

    get_space_support_status(dev, S_ICMD);
    get_space_support_status(dev, S_CRSPACE);
    get_space_support_status(dev, S_ALL_ICMD);
    get_space_support_status(dev, S_NODNIC_INIT_SEG);
    get_space_support_status(dev, S_EXPANSION_ROM);
    get_space_support_status(dev, S_ND_CRSPACE);
    get_space_support_status(dev, S_SCAN_CRSPACE);
    get_space_support_status(dev, S_GLOBAL_SEMAPHORE);
    get_space_support_status(dev, S_MAC);
    dev->vsec_cap_mask |= (1 << VC_INITIALIZED);
    // clear semaphore
    _vendor_specific_sem(dev, 0);
    return 0;
}

/*
 * End of Capabilities section
 */
#define WO_REG_ADDR_DATA 0xbadacce5
#define DEVID_OFFSET 0xf0014
int is_wo_gw(struct pci_dev* pcidev, unsigned addr_reg)
{
    int ret;
    unsigned int data = 0;
    ret = pci_write_config_dword(pcidev, addr_reg, DEVID_OFFSET);
    if (ret)
    {
        return 0;
    }
    ret = pci_read_config_dword(pcidev, addr_reg, &data);
    if (ret)
    {
        return 0;
    }
    if (data == WO_REG_ADDR_DATA)
    {
        return 1;
    }
    return 0;
}

static void init_dma(struct dev_data* dev)
{
    /*
        if((pci_set_dma_mask (dev->pci_dev, DMA_BIT_MASK(64)) >= 0) ){

            dev->dma_props[MST_DMA_ICMD].mem_pa = kmalloc (DMA_MBOX_SIZE, GFP_DMA | GFP_KERNEL);
            if (!dev->dma_props[MST_DMA_ICMD].mem_pa) {
                return;
            }
            dev->dma_props[MST_DMA_ICMD].mem_size = DMA_MBOX_SIZE;
            memset(dev->dma_props[MST_DMA_ICMD].mem_pa, 0, DMA_MBOX_SIZE);
            dev->dma_props[MST_DMA_ICMD].dma_map = pci_map_single(dev->pci_dev, dev->dma_props[MST_DMA_ICMD].mem_pa,
       DMA_MBOX_SIZE, DMA_BIDIRECTIONAL); if (!dev->dma_props[MST_DMA_ICMD].dma_map) {
                kfree(dev->dma_props[MST_DMA_ICMD].mem_pa);
                dev->dma_props[MST_DMA_ICMD].mem_pa = 0;
            }

        } else {
        }
    */
}

static void close_dma(struct dev_data* dev)
{
    int i = 0;
    for (i = 0; i < MST_DMA_END; i++)
    {
        if (dev->dma_props[i].mem_pa)
        {
            dma_unmap_page(&dev->pci_dev->dev, dev->dma_props[i].dma_map, DMA_MBOX_SIZE, DMA_BIDIRECTIONAL);
            kfree(dev->dma_props[i].mem_pa);
            dev->dma_props[i].mem_pa = NULL;
        }
    }
}

static int page_pin(struct dev_data* dev, struct page_info_st* page_info)
{
    unsigned long page_pointer_start = page_info->page_pointer_start;
    unsigned int page_amount = page_info->page_amount;
    unsigned int pages_size = page_amount * PAGE_SIZE;
    unsigned long end_of_buffer = page_pointer_start + pages_size;
    unsigned int gup_flags = FOLL_WRITE;
    int page_mapped_counter = 0;
    int page_counter = 0;
    int total_pinned = 0;

    // If the combination of the addr and size requested for this memory
    // region causes an integer overflow, return error.
    if (((end_of_buffer) < page_pointer_start) || PAGE_ALIGN(end_of_buffer) < (end_of_buffer) || page_amount < 1)
    {
        return -EINVAL;
    }

    // Check if we allow locking memory.
    if (!can_do_mlock())
    {
        return -EPERM;
    }

    // Allocate the page list.
    dev->dma_page.page_list = kcalloc(page_amount, sizeof(struct page*), GFP_KERNEL);
    if (!dev->dma_page.page_list)
    {
        return -ENOMEM;
    }

    // Go over the user memory buffer and pin user pages in memory.
    while (total_pinned < page_amount)
    {
        // Save the current number of pages to pin
        int num_pages = page_amount - total_pinned;

        // Save the current pointer to the right offset.
        uint64_t current_ptr = page_pointer_start + (total_pinned * PAGE_SIZE);

        // Save the current page.
        struct page** current_pages = dev->dma_page.page_list + total_pinned;

        // Attempt to pin user pages in memory.
        // Returns number of pages pinned - this may be fewer than the number requested
        //   or -errno in case of error.
        int pinned_pages = get_user_pages_fast(current_ptr, num_pages, gup_flags, current_pages);
        if (pinned_pages < 1)
        {
            kfree(dev->dma_page.page_list);
            return -EFAULT;
        }

        // When the parameter 'inter_iommu' is on, we need to set up
        // a mapping on a pages in order to access the physical address
        while (page_mapped_counter < pinned_pages)
        {
            int current_page = total_pinned + page_mapped_counter;

            // Get the dma address.
            dev->dma_page.dma_addr[current_page] =
              dma_map_page(&dev->pci_dev->dev, current_pages[current_page], 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
            // Do we get a valid dma address ?
            if (dma_mapping_error(&dev->pci_dev->dev, dev->dma_page.dma_addr[current_page]))
            {
                printk(KERN_ERR "Failed to get DMA addresses\n");
                return -EINVAL;
            }

            page_info->page_address_array[current_page].dma_address = dev->dma_page.dma_addr[current_page];

            page_mapped_counter++;
        }

        // Advance the memory that already pinned.
        total_pinned += pinned_pages;
    }

    // There is a page that not pinned in the kernel space ?
    if (total_pinned != page_amount)
    {
        return -EFAULT;
    }

    // Print the pages to the dmesg.
    for (page_counter = 0; page_counter < page_amount; page_counter++)
    {
        printk(KERN_INFO "Page address structure number: %d, device: %04x:%02x:%02x.%0x\n", page_counter,
               pci_domain_nr(dev->pci_dev->bus), dev->pci_dev->bus->number, PCI_SLOT(dev->pci_dev->devfn),
               PCI_FUNC(dev->pci_dev->devfn));
    }

    return 0;
}

static int page_unpin(struct dev_data* dev, struct page_info_st* page_info)
{
    int page_counter;

    // Check if the page list is allocated.
    if (!dev || !dev->dma_page.page_list)
    {
        return -EINVAL;
    }

    // Deallocate the pages.
    for (page_counter = 0; page_counter < page_info->page_amount; page_counter++)
    {
        // DMA activity is finished.
        dma_unmap_page(&dev->pci_dev->dev, dev->dma_page.dma_addr[page_counter], PAGE_SIZE, DMA_BIDIRECTIONAL);

        // Release the page list.
        set_page_dirty(dev->dma_page.page_list[page_counter]);
        put_page(dev->dma_page.page_list[page_counter]);
        dev->dma_page.page_list[page_counter] = NULL;
        dev->dma_page.dma_addr[page_counter] = 0;

        printk(KERN_INFO "Page structure number: %d was released. device:%04x:%02x:%02x.%0x\n", page_counter,
               pci_domain_nr(dev->pci_dev->bus), dev->pci_dev->bus->number, PCI_SLOT(dev->pci_dev->devfn),
               PCI_FUNC(dev->pci_dev->devfn));
    }

    // All the pages are clean.
    dev->dma_page.page_list = NULL;

    return 0;
}

static int read_dword_from_config_space(struct dev_data* dev, struct read_dword_from_config_space* read_from_cspace)
{
    int ret = 0;

    // take semaphore
    ret = _vendor_specific_sem(dev, 1);
    if (ret)
    {
        return ret;
    }

    // Read dword from config space
    ret = pci_read_config_dword(dev->pci_dev, read_from_cspace->offset, &read_from_cspace->data);
    if (ret)
    {
        goto cleanup;
    }

cleanup:
    // clear semaphore
    _vendor_specific_sem(dev, 0);
    return ret;
}

static int ioctl(struct inode* inode, struct file* file, unsigned int opcode, unsigned long udata_l)
{
    void* udata = (void*)udata_l;
    int minor = MINOR(inode->i_rdev);
    struct dev_data* dev = &devices[minor];
    int ret = 0;
    unsigned int d;

    /* By convention, any user gets read access
     * and is allowed to use the device.
     * Commands with no direction are administration
     * commands, and you need write permission
     * for this */

    if (_IOC_DIR(opcode) == _IOC_NONE)
    {
        if (!(file->f_mode & FMODE_WRITE))
            return -EPERM;
    }
    else
    {
        if (!(file->f_mode & FMODE_READ))
            return -EPERM;
    }

    if (down_interruptible(&devices[minor].sem))
    {
        return -ERESTARTSYS;
    }

    switch (opcode)
    {
        case INIT:
        {
            struct mst_pciconf_init_st initd;
            struct pci_bus* bus = NULL;
            /* Now have to init the new device */

            if (copy_from_user(&initd, udata, sizeof(initd)))
            {
                ret = -EFAULT;
                goto fin;
            }

            dev->pci_dev = NULL;
            if (initd.domain == -1)
            {
                initd.domain = 0;
            }
            bus = pci_find_bus(initd.domain, initd.bus);

            if (!bus)
            {
                ret = -ENXIO;
                goto fin;
            }

            dev->pci_dev = pci_get_slot(bus, initd.devfn);

            if (!dev->pci_dev)
            {
                ret = -ENXIO;
                goto fin;
            }
            dev->valid = 1;
            dev->bufused = 0;
            /* Old Cap for CR-Access*/
            dev->addr_reg = initd.addr_reg;
            dev->data_reg = initd.data_reg;
            /*
             * Check LiveFish GW mode
             */
            dev->wo_addr = is_wo_gw(dev->pci_dev, initd.addr_reg);
            //          printk("%04x:%02x:%02x.%0x - ADDR REG WO: %d\n", pci_domain_nr(dev->pci_dev->bus),
            //                  dev->pci_dev->bus->number, PCI_SLOT(dev->pci_dev->devfn),
            //                  PCI_FUNC(dev->pci_dev->devfn), dev->wo_addr);
            /* New Cap for CR-Access*/
            dev->vendor_specific_cap = pci_find_capability(dev->pci_dev, MLNX_VENDOR_SPECIFIC_CAP_ID);
            dev->vsec_cap_mask = 0;
            init_dma(dev);
            goto fin;
        }

        case STOP:
            close_dma(dev);
            if (dev->valid)
            {
                pci_dev_put(dev->pci_dev);
            }
            dev->valid = 0;
            goto fin;

        case WRITE4_NEW:
        {
            struct mst_write4_new_st write4d;
            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&write4d, udata, sizeof(write4d)))
            {
                ret = -EFAULT;
                goto fin;
            }
            if (get_vsec_cap_mask(dev))
            {
                ret = -EBUSY;
                goto fin;
            }
            if (VSEC_FULLY_SUPPORTED(dev))
            {
                ret = write4_new(dev, write4d.address_space, write4d.offset, write4d.data);
            }
            else
            {
                ret = write4_old(dev, write4d.offset, write4d.data);
            }
            goto fin;
        }
        case WRITE4:
        {
            struct mst_write4_st write4d;
            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&write4d, udata, sizeof(write4d)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = write4_old(dev, write4d.offset, write4d.data);

            goto fin;
        }

        case READ4_NEW:
        {
            struct mst_read4_new_st read4d;
            struct mst_read4_new_st* r_udata = (struct mst_read4_new_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&read4d, udata, sizeof(read4d)))
            {
                ret = -EFAULT;
                goto fin;
            }

            if (get_vsec_cap_mask(dev))
            {
                ret = -EBUSY;
                goto fin;
            }
            if (VSEC_FULLY_SUPPORTED(dev))
            {
                ret = read4_new(dev, read4d.address_space, read4d.offset, &d);
            }
            else
            {
                ret = read4_old(dev, read4d.offset, &d);
            }
            if (ret)
                goto fin;
            ret = put_user(d, &(r_udata->data)) ? -EFAULT : 0;
            goto fin;
        }
        case READ4:
        {
            struct mst_read4_st read4d;
            struct mst_read4_st* r_udata = (struct mst_read4_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&read4d, udata, sizeof(read4d)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = read4_old(dev, read4d.offset, &d);
            if (ret)
                goto fin;
            ret = put_user(d, &(r_udata->data)) ? -EFAULT : 0;
            goto fin;
        }
        case MODIFY:
        {
            struct mst_modify_st modifyd;
            struct mst_modify_st* m_udata = (struct mst_modify_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }
            if (copy_from_user(&modifyd, udata, sizeof(modifyd)))
            {
                ret = -EFAULT;
                goto fin;
            }

            if (get_vsec_cap_mask(dev))
            {
                ret = -EBUSY;
                goto fin;
            }

            // Read
            if (VSEC_FULLY_SUPPORTED(dev))
            {
                ret = read4_new(dev, modifyd.address_space, modifyd.offset, &d);
            }
            else
            {
                ret = read4_old(dev, modifyd.offset, &d);
            }
            if (ret)
                goto fin;
            // Modify
            d = (d & ~modifyd.mask) | (modifyd.data & modifyd.mask);
            // Write
            if (VSEC_FULLY_SUPPORTED(dev))
            {
                ret = write4_new(dev, modifyd.address_space, modifyd.offset, d);
            }
            else
            {
                ret = write4_old(dev, modifyd.offset, d);
            }
            if (ret)
                goto fin;

            ret = put_user(d, &(m_udata->old_data)) ? -EFAULT : 0;
            goto fin;
        }
        case READ4_BUFFER_EX:
        case READ4_BUFFER:
        {
            struct mst_read4_buffer_st read4_buf;
            struct mst_read4_buffer_st* rb_udata = (struct mst_read4_buffer_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (get_vsec_cap_mask(dev))
            {
                ret = -EBUSY;
                goto fin;
            }

            if (!VSEC_MIN_SUPPORT_KERN(dev))
            {
                ret = -ENOSYS;
                goto fin;
            }

            if (copy_from_user(&read4_buf, udata, sizeof(read4_buf)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = read4_block_new(dev, read4_buf.address_space, read4_buf.offset, read4_buf.size, read4_buf.data);
            if (ret != read4_buf.size)
                goto fin;

            ret = copy_to_user(rb_udata, &read4_buf, sizeof(read4_buf)) ? -EFAULT : read4_buf.size;
            goto fin;
        }
        case WRITE4_BUFFER:
        {
            struct mst_write4_buffer_st write4_buf;
            struct mst_write4_buffer_st* wb_udata = (struct mst_write4_buffer_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (get_vsec_cap_mask(dev))
            {
                ret = -EBUSY;
                goto fin;
            }

            if (!VSEC_MIN_SUPPORT_KERN(dev))
            {
                ret = -ENOSYS;
                goto fin;
            }

            if (copy_from_user(&write4_buf, udata, sizeof(write4_buf)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = write4_block_new(dev, write4_buf.address_space, write4_buf.offset, write4_buf.size, write4_buf.data);
            if (ret != write4_buf.size)
                goto fin;

            ret = copy_to_user(wb_udata, &write4_buf, sizeof(write4_buf)) ? -EFAULT : write4_buf.size;
            goto fin;
        }
        case MST_META_DATA:
        {
            struct mst_meta_data meta_data;
            struct mst_hdr hdr;
            memset(&meta_data, 0, sizeof(meta_data));
            // check hdr
            if (copy_from_user(&hdr, udata, sizeof(struct mst_hdr)))
            {
                ret = -EFAULT;
                goto fin;
            }

            if (hdr.payload_version_major != MST_META_DATA_VERSION_MAJOR || hdr.payload_len < sizeof(meta_data.data))
            {
                ret = -EINVAL;
                goto fin;
            }
            // fill meta_data hdr
            meta_data.hdr.hdr_version = MST_HDR_VERSION;
            meta_data.hdr.hdr_len = sizeof(meta_data.hdr);
            meta_data.hdr.payload_len = sizeof(meta_data.data);
            meta_data.hdr.payload_version_major = MST_META_DATA_VERSION_MAJOR;
            meta_data.hdr.payload_version_minor = MST_META_DATA_VERSION_MINOR;
            // fill payload
            meta_data.data.api_version_major = MST_API_VERSION_MAJOR;
            meta_data.data.api_version_minor = MST_API_VERSION_MINOR;

            ret = copy_to_user(udata, &meta_data, sizeof(meta_data));
            goto fin;
        }
        case MST_PARAMS:
        {
            struct mst_params_st params;
            struct mst_params_st* params_udata = (struct mst_params_st*)udata;
            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }
            // best effort : try to get space spport status if we fail assume we got vsec support.
            get_vsec_cap_mask(dev);

            params.bus = dev->pci_dev->bus->number;
            params.bar = 0;
            params.domain = pci_domain_nr(dev->pci_dev->bus);
            params.func = PCI_FUNC(dev->pci_dev->devfn);
            params.slot = PCI_SLOT(dev->pci_dev->devfn);
            params.device = dev->pci_dev->device;
            params.vendor = dev->pci_dev->vendor;
            params.subsystem_device = dev->pci_dev->subsystem_device;
            params.subsystem_vendor = dev->pci_dev->subsystem_vendor;
            if (VSEC_ASSUMED_SUPPORTED(dev))
            { // assume supported if uninitialized (since semaphore is locked)
                params.vendor_specific_cap = dev->vendor_specific_cap;
                params.vsec_cap_mask = dev->vsec_cap_mask;
            }
            else
            {
                params.vendor_specific_cap = 0;
            }
            params.multifunction = dev->pci_dev->multifunction;
            ret = copy_to_user(params_udata, &params, sizeof(params));
            goto fin;
        }
        case PCICONF_VPD_READ4:
#if (LINUX_VERSION_CODE >= 0x02061D)
        {
            struct mst_vpd_read4_st vpd_read4;
            struct mst_vpd_read4_st* r_udata = (struct mst_vpd_read4_st*)udata;
            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&vpd_read4, udata, sizeof(vpd_read4)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = pci_read_vpd(dev->pci_dev, vpd_read4.offset, 4, &d);
            if (ret < 1)
            {
                ret = -EFAULT;
                goto fin;
            }
            ret = put_user(d, &(r_udata->data)) ? -EFAULT : 0;
            goto fin;
        }
#endif
        case PCICONF_VPD_WRITE4:
#if (LINUX_VERSION_CODE >= 0x02061D)
        {
            struct mst_vpd_write4_st vpd_write4;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            if (copy_from_user(&vpd_write4, udata, sizeof(vpd_write4)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = pci_write_vpd(dev->pci_dev, vpd_write4.offset, 4, &vpd_write4.data);
            goto fin;
        }
#else
        {
            ret = -ENOTTY;
            goto fin;
        }
#endif
        case PCICONF_DMA_PROPS:
        {
            struct mst_dma_props_st dma_props;
            struct mst_dma_props_st* dma_props_udata = (struct mst_dma_props_st*)udata;
            int i;
            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }

            for (i = 0; i < MST_DMA_END; i++)
            {
                dma_props.dma_props[i].dma_pa = (unsigned long long int)dev->dma_props[i].dma_map;
                dma_props.dma_props[i].mem_size = dev->dma_props[i].mem_size;
            }
            ret = copy_to_user(dma_props_udata, &dma_props, sizeof(dma_props));
            goto fin;
        }
        case PCICONF_MEM_ACCESS:
        {
            struct mst_mem_access_st mem_access;
            struct mst_mem_access_st* mm_udata = (struct mst_mem_access_st*)udata;

            if (!dev->valid)
            {
                ret = -ENOTTY;
                goto fin;
            }
            if (copy_from_user(&mem_access, udata, sizeof(mem_access)))
            {
                ret = -EFAULT;
                goto fin;
            }
            if (mem_access.mem_type >= MST_DMA_END)
            {
                ret = -EFAULT;
                goto fin;
            }
            if (!dev->dma_props[mem_access.mem_type].mem_pa)
            {
                ret = -EINVAL;
                goto fin;
            }
            if (mem_access.size + mem_access.offset > dev->dma_props[mem_access.mem_type].mem_size)
            {
                ret = -EINVAL;
                goto fin;
            }
            dma_sync_single_for_cpu(&dev->pci_dev->dev, dev->dma_props[mem_access.mem_type].dma_map,
                                    dev->dma_props[mem_access.mem_type].mem_size, DMA_BIDIRECTIONAL);
            if (mem_access._rw == 0)
            {
                /* READ */
                memcpy(mem_access.data, dev->dma_props[mem_access.mem_type].mem_pa + mem_access.offset,
                       mem_access.size);
            }
            else
            {
                memcpy(dev->dma_props[mem_access.mem_type].mem_pa + mem_access.offset, mem_access.data,
                       mem_access.size);
            }
            dma_sync_single_for_device(&dev->pci_dev->dev, dev->dma_props[mem_access.mem_type].dma_map,
                                       dev->dma_props[mem_access.mem_type].mem_size, DMA_BIDIRECTIONAL);
            ret = copy_to_user(mm_udata, &mem_access, sizeof(mem_access));
            break;
        }
        case GET_DMA_PAGES:
        case RELEASE_DMA_PAGES:
        {
            struct page_info_st page_info;

            // Device validation.
            if (!dev->valid || !dev->pci_dev)
            {
                ret = -ENOTTY;
                goto fin;
            }

            // Copy the page info structure from the user space.
            if (copy_from_user(&page_info, udata, sizeof(struct page_info_st)))
            {
                ret = -EFAULT;
                goto fin;
            }

            if (opcode == GET_DMA_PAGES)
            {
                ret = page_pin(dev, &page_info);
                if (ret)
                {
                    goto fin;
                }

                // Return the dma address to the user.
                if (copy_to_user(udata, &page_info, sizeof(struct page_info_st)) != 0)
                {
                    ret = -EFAULT;
                    goto fin;
                }
            }
            else
            {
                ret = page_unpin(dev, &page_info);
            }

            break;
        }
        case READ_DWORD_FROM_CONFIG_SPACE:
        {
            struct read_dword_from_config_space read_from_cspace;

            // Device validation.
            if (!dev->valid || !dev->pci_dev)
            {
                ret = -ENOTTY;
                goto fin;
            }

            // Copy the page info structure from the user space.
            if (copy_from_user(&read_from_cspace, udata, sizeof(struct read_dword_from_config_space)))
            {
                ret = -EFAULT;
                goto fin;
            }

            ret = read_dword_from_config_space(dev, &read_from_cspace);
            if (ret)
            {
                goto fin;
            }
            // Return the physical address to the user.
            if (copy_to_user(udata, &read_from_cspace, sizeof(struct read_dword_from_config_space)) != 0)
            {
                ret = -EFAULT;
                goto fin;
            }

            break;
        }
        default:
            ret = -ENOTTY;
            goto fin;
    }

fin:
    up(&dev->sem);
    return ret;
}

#include "driver_common.c"
