/*******************************************************************************
    Copyright (c) 2013 NVIDIA Corporation

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to
    deal in the Software without restriction, including without limitation the
    rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
    sell copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

        The above copyright notice and this permission notice shall be
        included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.

*******************************************************************************/

#include "uvmtypes.h"
#include "uvm_common.h"
#include "uvm_lite.h"
#include "uvm_events.h"
#include "uvm_lite_events.h"

//
// uvmlite_gpu_event_(start|stop)_device() deal with events from the RM to the
// UVM-lite driver.
//
NV_STATUS uvmlite_gpu_event_start_device(UvmGpuUuid *gpuUuidStruct)
{
    UVM_DBG_PRINT_UUID("Start", gpuUuidStruct);

    if (uvmlite_enable_gpu_uuid(gpuUuidStruct) != NV_OK)
        return NV_ERR_GENERIC;
    return NV_OK;
}

NV_STATUS uvmlite_gpu_event_stop_device(UvmGpuUuid *gpuUuidStruct)
{

    //
    // TODO: implement
    //
    // Grab the global lock
    // Grab the per process lock
    // Clean up all the migration resources for this UUID
    // and other relevant data structures
    // Also:
    // Other related fixes that need to be added
    // need to take this global lock when UVM is being rmmodded
    // The copy/mig loop needs to now follow this
    // 1) Take per process lock
    // 2) Check if migration resources exist(channel is valid)
    // 3) migrate pages()
    // 4) Unlock per process lock
    // 5) is_schedule()
    //

    UVM_DBG_PRINT_UUID("Stop", gpuUuidStruct);
    if (uvmlite_disable_gpu_uuid(gpuUuidStruct) != NV_OK)
        return NV_ERR_GENERIC;

    umvlite_destroy_per_process_gpu_resources(gpuUuidStruct);
    return NV_OK;
}

//
// The following functions deal with events from the UVM-lite driver
// to usermode clients.
//

void uvm_init_event_listener_list(UvmProcessRecord *pCurrentProcessRecord)
{
    unsigned i;

    init_rwsem(&pCurrentProcessRecord->eventListenerListLock);

    for (i = 0; i < UvmEventNumTypes; i++)
    {
        INIT_LIST_HEAD(&pCurrentProcessRecord->eventListenerLists[i]);
    }

    init_waitqueue_head(&pCurrentProcessRecord->wait_queue);
}

//
// This function must be called with a write lock on
// pSessionInfo->eventQueueInfoListLock
//
NV_STATUS
uvm_create_event_queue(UvmSessionInfo *pSessionInfo,
                       UvmEventQueueInfo **ppEventQueueInfo,
                       NvLength queueSize,
                       unsigned int notificationCount,
                       UvmEventTimeStampType timeStampType)
{
    NV_STATUS rmStatus = NV_OK;
    UvmEventQueueInfo *pEventQueueInfo;
    const unsigned entrySize = sizeof(UvmEventEntry);
    NvLength i, numQueuePages = PAGE_ALIGN(queueSize * entrySize) >> PAGE_SHIFT;

    *ppEventQueueInfo = vmalloc(sizeof(UvmEventQueueInfo));
    if (*ppEventQueueInfo == NULL)
    {
        UVM_ERR_PRINT("failed to allocate memory for UvmEventQueueInfo\n");
        rmStatus = NV_ERR_NO_MEMORY;
        goto done;
    }

    pEventQueueInfo = *ppEventQueueInfo;
    pEventQueueInfo->enabledEventsBitmask = 0;
    pEventQueueInfo->notificationCount = notificationCount;
    pEventQueueInfo->numQueuePages = numQueuePages;

    // Alocate memory for the page that will be mapped RO into the client
    pEventQueueInfo->pUserRODataPage = alloc_page(NV_UVM_GFP_FLAGS |
                                                  GFP_HIGHUSER);

    if (pEventQueueInfo->pUserRODataPage == NULL)
    {
        UVM_ERR_PRINT("failed to allocate page for pUserRODataPage\n");
        rmStatus = NV_ERR_NO_MEMORY;
        goto done;
    }

    pEventQueueInfo->pUserROData = kmap(pEventQueueInfo->pUserRODataPage);
    if (pEventQueueInfo->pUserROData == NULL)
    {
        UVM_ERR_PRINT("failed to map page for pUserROData\n");
        rmStatus = NV_ERR_INSUFFICIENT_RESOURCES;
        goto done;
    }

    memset(pEventQueueInfo->pUserROData, 0, PAGE_SIZE);

    NV_ATOMIC64_SET(pEventQueueInfo->pUserROData->writeIndex, 0);
    pEventQueueInfo->pUserROData->maxEventCapacity = queueSize;

    // Alocate memory for the page that will be mapped RW into the client
    pEventQueueInfo->pUserRWDataPage = alloc_page(NV_UVM_GFP_FLAGS |
                                                  GFP_HIGHUSER);

    if (pEventQueueInfo->pUserRODataPage == NULL)
    {
        UVM_ERR_PRINT("failed to allocate page for pUserRWDataPage\n");
        rmStatus = NV_ERR_NO_MEMORY;
        goto done;
    }

    pEventQueueInfo->pUserRWData = kmap(pEventQueueInfo->pUserRWDataPage);
    if (pEventQueueInfo->pUserRWData == NULL)
    {
        UVM_ERR_PRINT("failed to map page for pUserRWData\n");
        rmStatus = NV_ERR_INSUFFICIENT_RESOURCES;
        goto done;
    }

    memset(pEventQueueInfo->pUserRWData, 0, PAGE_SIZE);

    NV_ATOMIC64_SET(pEventQueueInfo->pUserRWData->readIndex, 0);
    NV_ATOMIC64_SET(pEventQueueInfo->pUserRWData->writeIndex, 0);

    // Alocate memory for the list of event queue buffer pages
    pEventQueueInfo->ppBufferPageList = vmalloc(numQueuePages *
                                                         sizeof(struct page *));
    if (pEventQueueInfo->ppBufferPageList == NULL)
    {
        UVM_ERR_PRINT("failed to allocate page for ppBufferPageList\n");
        rmStatus = NV_ERR_NO_MEMORY;
        goto done;
    }

    // Allocate the page descriptor table for the event queue buffer
    for (i = 0; i < numQueuePages; i++)
    {
        pEventQueueInfo->ppBufferPageList[i] =
            alloc_page(NV_UVM_GFP_FLAGS | GFP_HIGHUSER);
        if (pEventQueueInfo->ppBufferPageList[i] == NULL)
        {
            UVM_ERR_PRINT("failed to allocate page for ppBufferPageList\n");
            rmStatus = NV_ERR_NO_MEMORY;
            goto done;
        }
    }

    //
    // Map this buffer into kernel VA space
    // TODO: Double-check these flags
    //
    pEventQueueInfo->pBuffer = vmap(pEventQueueInfo->ppBufferPageList,
                                    numQueuePages,
                                    VM_READ | VM_WRITE,
                                    PAGE_KERNEL);
    if (pEventQueueInfo->pBuffer == NULL)
    {
        UVM_ERR_PRINT("failed to map pBuffer\n");
        rmStatus = NV_ERR_NO_MEMORY;
        goto done;
    }

    // Initialize the lock that will protect the event queue buffer
    init_rwsem(&pEventQueueInfo->eventQueueBufferLock);

    // Add the new UvmEventQueueInfo to pSessionInfo's list.
    list_add_tail(&pEventQueueInfo->eventQueueInfoListNode,
                  &pSessionInfo->eventQueueInfoList);

    //
    // Initialize hooks with which this UvmEventQueueInfo connects to
    // the Event Listener Lists of the debuggee.
    //
    for (i = 0; i < UvmEventNumTypes; i++)
    {
        INIT_LIST_HEAD(&pEventQueueInfo->eventListenerListNode[i]);
    }

    pEventQueueInfo->index = pSessionInfo->nextEventQueueInfoIndex++;

done:
    if (rmStatus != NV_OK)
    {
        uvm_remove_event_queue(pSessionInfo, *ppEventQueueInfo);
        *ppEventQueueInfo = NULL;
    }

    return rmStatus;
}

//
// This function must be called with a lock on
// pSessionInfo->eventQueueInfoListLock
//
NV_STATUS
uvm_get_event_queue(UvmSessionInfo *pSessionInfo,
                    UvmEventQueueInfo **ppEventQueueInfo,
                    NvUPtr eventQueueHandle)
{
    struct list_head *ptr;
    UvmEventQueueInfo *entry; 

    list_for_each(ptr, &pSessionInfo->eventQueueInfoList)
    {
        entry = list_entry(ptr, UvmEventQueueInfo, eventQueueInfoListNode);
        if (eventQueueHandle == entry->index)
        {
            *ppEventQueueInfo = entry;
            return NV_OK;
        }
    }

    // Not found in the list
    *ppEventQueueInfo = NULL;
    return NV_ERR_INVALID_ARGUMENT;
}

//
// Free as much of the event queue as was allocated
// This function must be called with a read lock on
// g_uvmDriverPrivateTableLock and a write lock on
// pSessionInfo->eventQueueInfoListLock.
//
void
uvm_remove_event_queue
(
    UvmSessionInfo *pSessionInfo,
    UvmEventQueueInfo *pEventQueueInfo
)
{
    NV_STATUS rmStatus;
    NvLength i, numQueuePages;
    UvmEventType eventType = 0;

    if (!pEventQueueInfo || !pEventQueueInfo->pUserROData)
    {
        goto done;
    }

    // Remove all enabled events
    while (pEventQueueInfo->enabledEventsBitmask != 0)
    {
        if (((1 << eventType) & pEventQueueInfo->enabledEventsBitmask) != 0)
        {
            rmStatus = uvm_disable_event(pEventQueueInfo,
                                         eventType,
                                         pSessionInfo->pidTarget);
            if (rmStatus != NV_OK)
                goto done;
        }

        eventType++;
    }

    numQueuePages = pEventQueueInfo->numQueuePages;

    kunmap(pEventQueueInfo->pUserRODataPage);
    __free_page(pEventQueueInfo->pUserRODataPage);

    if (!pEventQueueInfo->pUserRWData)
    {
        goto done;
    }

    kunmap(pEventQueueInfo->pUserRWDataPage);
    __free_page(pEventQueueInfo->pUserRWDataPage);

    vunmap(pEventQueueInfo->pBuffer);

    // Free the pages in the page descriptor table
    for (i = 0; i < numQueuePages; i++)
    {
        __free_page(pEventQueueInfo->ppBufferPageList[i]);
    }

    vfree(pEventQueueInfo->ppBufferPageList);

    // Remove the UvmEventQueueInfo from pSessionInfo's list.
    list_del_init(&pEventQueueInfo->eventQueueInfoListNode);

    vfree(pEventQueueInfo);

done:

    return;
}

// This function must be called with a write lock on mmap_sem.
NV_STATUS
uvm_map_event_queue(UvmEventQueueInfo *pEventQueueInfo,
                    NvP64 userRODataAddr,
                    NvP64 userRWDataAddr,
                    NvP64 *pReadIndexAddr,
                    NvP64 *pWriteIndexAddr,
                    NvP64 *pQueueBufferAddr,
                    struct vm_area_struct *roVma,
                    struct vm_area_struct *rwVma,
                    struct file *filp)
{
    NV_STATUS rmStatus;
    struct page *pPage;
    NvLength i, numQueuePages = pEventQueueInfo->numQueuePages;
    NvP64 userAddress;

    // Map the RW page into userspace
    pPage = pEventQueueInfo->pUserRWDataPage,
    rmStatus = uvm_map_page(rwVma, pPage, (NvUPtr)userRWDataAddr);
    if (rmStatus != NV_OK)
    {
        goto done;
    }

    // Map the RO page into userspace
    pPage = pEventQueueInfo->pUserRODataPage;
    rmStatus = uvm_map_page(roVma, pPage, (NvUPtr)userRODataAddr);
    if (rmStatus != NV_OK)
    {
        goto done;
    }

    // Map the event queue buffer into userspace
    userAddress = userRODataAddr + PAGE_SIZE;
    for (i = 0; i < numQueuePages; i++)
    {
        pPage = pEventQueueInfo->ppBufferPageList[i];
        rmStatus = uvm_map_page(roVma, pPage, (NvUPtr)userAddress);
        if (rmStatus != NV_OK)
        {
            goto done;
        }

        userAddress += PAGE_SIZE;
    }

    *pReadIndexAddr = userRWDataAddr + offsetof(UvmEventQueueInfoUserRWData,
                                                readIndex);
    *pWriteIndexAddr = userRWDataAddr + offsetof(UvmEventQueueInfoUserRWData,
                                                 writeIndex);
    *pQueueBufferAddr = userRODataAddr + PAGE_SIZE;

done:
    return rmStatus;
}

//
// This function must be called with a read lock on
// g_uvmDriverPrivateTableLock and a write lock on
// pSessionInfo->eventQueueInfoListLock.
//
NV_STATUS
uvm_enable_event
(
    UvmEventQueueInfo *pEventQueueInfo,
    UvmEventType eventType,
    unsigned pidTarget
)
{
    NV_STATUS rmStatus;
    UvmProcessRecord *pTargetProcessRecord;

    // Nothing to do if the event is already enabled.
    if ((pEventQueueInfo->enabledEventsBitmask & (1 << eventType)) != 0)
        return NV_OK;

    rmStatus = uvmlite_get_process_record(pidTarget, &pTargetProcessRecord);

    // uvmlite_get_process_record() may fail if the target process has exited.
    if (rmStatus == NV_OK)
    {
        // Add this UvmEventQueueInfo structure to the debuggee's
        // eventListenerList
        down_write(&pTargetProcessRecord->eventListenerListLock);

        list_add_tail(&pEventQueueInfo->eventListenerListNode[eventType],
                      &pTargetProcessRecord->eventListenerLists[eventType]);

        up_write(&pTargetProcessRecord->eventListenerListLock);

        // Increment the enabled events count
        atomic_inc(&pTargetProcessRecord->enabledEventsCount);
    }

    // Finally, mark the event as enabled
    pEventQueueInfo->enabledEventsBitmask |= (1 << eventType);

    return NV_OK;
}

//
// This function must be called with a read lock on
// g_uvmDriverPrivateTableLock and a write lock on
// pSessionInfo->eventQueueInfoListLock.
//
NV_STATUS
uvm_disable_event
(
    UvmEventQueueInfo *pEventQueueInfo,
    UvmEventType eventType,
    unsigned pidTarget
)
{
    NV_STATUS rmStatus;
    UvmProcessRecord *pTargetProcessRecord;

    // Nothing to do if the event is already disabled.
    if ((pEventQueueInfo->enabledEventsBitmask & (1 << eventType)) == 0)
        return NV_OK;

    rmStatus = uvmlite_get_process_record(pidTarget, &pTargetProcessRecord);

    // uvmlite_get_process_record() may fail if the target process has exited.
    if (rmStatus == NV_OK)
    {
        // Remove this UvmEventQueueInfo structure from the debuggee's
        // eventListenerList
        down_write(&pTargetProcessRecord->eventListenerListLock);

        list_del_init(&pEventQueueInfo->eventListenerListNode[eventType]);

        up_write(&pTargetProcessRecord->eventListenerListLock);

        // Decrement the enabled events count
        atomic_dec(&pTargetProcessRecord->enabledEventsCount);
    }

    // Finally, mark the event as disabled 
    pEventQueueInfo->enabledEventsBitmask &= ~(1 << eventType);

    return NV_OK;
}

// Locking: The caller must hold a read lock on DriverPrivate.uvmPrivLock.
NV_STATUS
uvm_record_memory_violation_event
(
    UvmProcessRecord *pProcessRecord,
    NvU8 accessType,
    NvU64 address,
    NvU64 timeStamp,
    NvU32 pid,
    NvU32 threadId
)
{
    NV_STATUS rmStatus = NV_OK;
    struct list_head *ptr;
    UvmEventQueueInfo *entry;
    struct list_head *memoryViolationEventsList;
    UvmEventMemoryViolationInfo *pViolationInfo;
    NvU64 queueWriteIndex, queueSize;

    UVM_DBG_PRINT_RL("begin\n");

    down_read(&pProcessRecord->eventListenerListLock);

    memoryViolationEventsList =
        &pProcessRecord->eventListenerLists[UvmEventTypeMemoryViolation];

    if (list_empty(memoryViolationEventsList))
    {
        up_read(&pProcessRecord->eventListenerListLock);
        return rmStatus;
    }

    list_for_each(ptr, memoryViolationEventsList)
    {
        entry = list_entry(ptr,
                           UvmEventQueueInfo,
                           eventListenerListNode[UvmEventTypeMemoryViolation]);

        down_write(&entry->eventQueueBufferLock);

        queueSize = entry->pUserROData->maxEventCapacity;
        (void)NV_DIV64(NV_ATOMIC64_READ(entry->pUserROData->writeIndex),
                       queueSize,
                       &queueWriteIndex);

        pViolationInfo =
            (UvmEventMemoryViolationInfo*)((UvmEventEntry*)entry->pBuffer +
                                                              queueWriteIndex);

        pViolationInfo->eventType = UvmEventTypeMemoryViolation;
        pViolationInfo->accessType = accessType;
        pViolationInfo->address = address;
        pViolationInfo->timeStamp = timeStamp;
        pViolationInfo->pid = pid;
        pViolationInfo->threadId = threadId;

        // Make prior writes visible to all CPUs
        smp_wmb();

        //
        // Atomically increment both the kernel and the user's copy of
        // the write counter
        //
        NV_ATOMIC64_INC(entry->pUserROData->writeIndex);
        NV_ATOMIC64_INC(entry->pUserRWData->writeIndex);

        // Signal the wait queue if enough events are available
        if (NV_ATOMIC64_READ(entry->pUserROData->writeIndex) -
            NV_ATOMIC64_READ(entry->pUserRWData->readIndex) >
            entry->notificationCount)
            wake_up_interruptible_all(&pProcessRecord->wait_queue);

        up_write(&entry->eventQueueBufferLock);
    }

    up_read(&pProcessRecord->eventListenerListLock);

    UVM_DBG_PRINT_RL("end\n");

    return rmStatus;
}

// Locking: The caller must hold a read lock on DriverPrivate.uvmPrivLock.
NV_STATUS
uvm_record_migration_event
(
    UvmProcessRecord *pProcessRecord,
    NvU8 direction,
    NvU8 srcIndex,
    NvU8 dstIndex,
    NvU64 address,
    NvU64 migratedBytes,
    NvU64 beginTimeStamp,
    NvU64 endTimeStamp,
    NvU64 streamId
)
{
    NV_STATUS rmStatus = NV_OK;
    struct list_head *ptr;
    UvmEventQueueInfo *entry; 
    struct list_head *migrationEventsList;
    UvmEventMigrationInfo *pMigInfo;
    NvU64 queueWriteIndex, queueSize;

    UVM_DBG_PRINT_RL("begin\n");

    down_read(&pProcessRecord->eventListenerListLock);

    migrationEventsList =
        &pProcessRecord->eventListenerLists[UvmEventTypeMigration];

    if (list_empty(migrationEventsList))
    {
        up_read(&pProcessRecord->eventListenerListLock);
        return rmStatus;
    }

    list_for_each(ptr, migrationEventsList)
    {
        entry = list_entry(ptr,
                           UvmEventQueueInfo,
                           eventListenerListNode[UvmEventTypeMigration]);

        down_write(&entry->eventQueueBufferLock);

        queueSize = entry->pUserROData->maxEventCapacity;
        (void)NV_DIV64(NV_ATOMIC64_READ(entry->pUserROData->writeIndex),
                       queueSize,
                       &queueWriteIndex);

        pMigInfo = (UvmEventMigrationInfo*)((UvmEventEntry*)entry->pBuffer +
                                                              queueWriteIndex);

        pMigInfo->eventType = UvmEventTypeMigration;
        pMigInfo->direction = direction;
        pMigInfo->srcIndex = srcIndex;
        pMigInfo->dstIndex = dstIndex;
        pMigInfo->address = address;
        pMigInfo->migratedBytes = migratedBytes;
        pMigInfo->beginTimeStamp = beginTimeStamp;
        pMigInfo->endTimeStamp = endTimeStamp;
        pMigInfo->streamId = streamId;

        // Make prior writes visible to all CPUs
        smp_wmb();

        //
        // Atomically increment both the kernel and the user's copy of
        // the write counter
        //
        NV_ATOMIC64_INC(entry->pUserROData->writeIndex);
        NV_ATOMIC64_INC(entry->pUserRWData->writeIndex);

        // Signal the wait queue if enough events are available
        if (NV_ATOMIC64_READ(entry->pUserROData->writeIndex) -
            NV_ATOMIC64_READ(entry->pUserRWData->readIndex) >
            entry->notificationCount)
            wake_up_interruptible_all(&pProcessRecord->wait_queue);

        up_write(&entry->eventQueueBufferLock);
    }

    up_read(&pProcessRecord->eventListenerListLock);

    UVM_DBG_PRINT_RL("end\n");

    return rmStatus;
}

NvBool
uvm_is_event_enabled(UvmProcessRecord *pProcessRecord, UvmEventType eventType)
{
    NvBool bEnabled = NV_TRUE;

    down_read(&pProcessRecord->eventListenerListLock);

    bEnabled = !(list_empty(&pProcessRecord->eventListenerLists[eventType]));

    up_read(&pProcessRecord->eventListenerListLock);

    return bEnabled;
}

// Locking: The caller must hold a read lock on DriverPrivate.uvmPrivLock.
NvBool
uvm_any_event_notifications_pending(UvmProcessRecord *pProcessRecord)
{
    unsigned i;
    struct list_head *eventList, *ptr;
    UvmEventQueueInfo *entry; 

    UVM_DBG_PRINT_RL("begin\n");
    down_read(&pProcessRecord->eventListenerListLock);

    for (i = 0; i < UvmEventNumTypes; i++)
    {
        eventList = &pProcessRecord->eventListenerLists[i];

        list_for_each(ptr, eventList)
        {
            entry = list_entry(ptr, UvmEventQueueInfo, eventListenerListNode[i]);

            if (NV_ATOMIC64_READ(entry->pUserROData->writeIndex) -
                NV_ATOMIC64_READ(entry->pUserRWData->readIndex) >
                entry->notificationCount)
            {
                up_read(&pProcessRecord->eventListenerListLock);
                UVM_DBG_PRINT_RL("notification pending\n");
                return NV_TRUE;
            }
        }
    }

    up_read(&pProcessRecord->eventListenerListLock);
    UVM_DBG_PRINT_RL("no notification pending\n");

    return NV_FALSE;
}
