request_transform.h File Reference

#include "ftl_config.h"
#include "nvme/nvme.h"

Include dependency graph for request_transform.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	_ROW_ADDR_DEPENDENCY_ENTRY
	The dependency info of this physical block. More...
struct	_ROW_ADDR_DEPENDENCY_TABLE
	The row address dependency table for all the user blocks. More...

Macros
#define	NVME_COMMAND_AUTO_COMPLETION_OFF   0
#define	NVME_COMMAND_AUTO_COMPLETION_ON   1
#define	ROW_ADDR_DEPENDENCY_CHECK_OPT_SELECT   0
#define	ROW_ADDR_DEPENDENCY_CHECK_OPT_RELEASE   1
#define	BUF_DEPENDENCY_REPORT_BLOCKED   0
#define	BUF_DEPENDENCY_REPORT_PASS   1
#define	ROW_ADDR_DEPENDENCY_REPORT_BLOCKED   0
#define	ROW_ADDR_DEPENDENCY_REPORT_PASS   1
#define	ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_DONE   0
#define	ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_SYNC   1
#define	ROW_ADDR_DEP_ENTRY(iCh, iWay, iBlk)   (&rowAddrDependencyTablePtr->block[(iCh)][(iWay)][(iBlk)])

Typedefs
typedef struct _ROW_ADDR_DEPENDENCY_ENTRY	ROW_ADDR_DEPENDENCY_ENTRY
	The dependency info of this physical block. More...
typedef struct _ROW_ADDR_DEPENDENCY_ENTRY *	P_ROW_ADDR_DEPENDENCY_ENTRY
typedef struct _ROW_ADDR_DEPENDENCY_TABLE	ROW_ADDR_DEPENDENCY_TABLE
	The row address dependency table for all the user blocks. More...
typedef struct _ROW_ADDR_DEPENDENCY_TABLE *	P_ROW_ADDR_DEPENDENCY_TABLE

Functions
void	InitDependencyTable ()
void	ReqTransNvmeToSlice (unsigned int cmdSlotTag, unsigned int startLba, unsigned int nlb, unsigned int cmdCode)
	Split NVMe command into slice requests. More...
void	ReqTransSliceToLowLevel ()
	Data Buffer Manager. Handle all the pending slice requests. More...
void	IssueNvmeDmaReq (unsigned int reqSlotTag)
	Allocate data buffer for the specified DMA request and inform the controller. More...
void	CheckDoneNvmeDmaReq ()
void	SelectLowLevelReqQ (unsigned int reqSlotTag)
	Dispatch given NVMe/NAND request to corresponding request queue. More...
void	ReleaseBlockedByBufDepReq (unsigned int reqSlotTag)
	Pop the specified request from the buffer dependency queue. More...
void	ReleaseBlockedByRowAddrDepReq (unsigned int chNo, unsigned int wayNo)
	Update the row address dependency of all the requests on the specified die. More...

Variables
P_ROW_ADDR_DEPENDENCY_TABLE	rowAddrDependencyTablePtr

Macro Definition Documentation

BUF_DEPENDENCY_REPORT_BLOCKED

#define BUF_DEPENDENCY_REPORT_BLOCKED   0

Definition at line 58 of file request_transform.h.

BUF_DEPENDENCY_REPORT_PASS

#define BUF_DEPENDENCY_REPORT_PASS   1

Definition at line 59 of file request_transform.h.

NVME_COMMAND_AUTO_COMPLETION_OFF

#define NVME_COMMAND_AUTO_COMPLETION_OFF   0

Definition at line 52 of file request_transform.h.

NVME_COMMAND_AUTO_COMPLETION_ON

#define NVME_COMMAND_AUTO_COMPLETION_ON   1

Definition at line 53 of file request_transform.h.

ROW_ADDR_DEP_ENTRY

#define ROW_ADDR_DEP_ENTRY	(		iCh,
			iWay,
			iBlk
	)		(&rowAddrDependencyTablePtr->block[(iCh)][(iWay)][(iBlk)])

Definition at line 112 of file request_transform.h.

ROW_ADDR_DEPENDENCY_CHECK_OPT_RELEASE

#define ROW_ADDR_DEPENDENCY_CHECK_OPT_RELEASE   1

Definition at line 56 of file request_transform.h.

ROW_ADDR_DEPENDENCY_CHECK_OPT_SELECT

#define ROW_ADDR_DEPENDENCY_CHECK_OPT_SELECT   0

Definition at line 55 of file request_transform.h.

ROW_ADDR_DEPENDENCY_REPORT_BLOCKED

#define ROW_ADDR_DEPENDENCY_REPORT_BLOCKED   0

Definition at line 61 of file request_transform.h.

ROW_ADDR_DEPENDENCY_REPORT_PASS

#define ROW_ADDR_DEPENDENCY_REPORT_PASS   1

Definition at line 62 of file request_transform.h.

ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_DONE

#define ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_DONE   0

Definition at line 64 of file request_transform.h.

ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_SYNC

#define ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_SYNC   1

Definition at line 65 of file request_transform.h.

Typedef Documentation

P_ROW_ADDR_DEPENDENCY_ENTRY

typedef struct _ROW_ADDR_DEPENDENCY_ENTRY * P_ROW_ADDR_DEPENDENCY_ENTRY

P_ROW_ADDR_DEPENDENCY_TABLE

typedef struct _ROW_ADDR_DEPENDENCY_TABLE * P_ROW_ADDR_DEPENDENCY_TABLE

ROW_ADDR_DEPENDENCY_ENTRY

typedef struct _ROW_ADDR_DEPENDENCY_ENTRY ROW_ADDR_DEPENDENCY_ENTRY

The dependency info of this physical block.

To ensure the integrity of the data will not affected by the scheduler, the fw should maintain some information for each flash block to avoid situations such as the erase request was being reorderd before the read requests.

Note

important for scheduling

See also

CheckRowAddrDep(), UpdateRowAddrDepTableForBufBlockedReq().

ROW_ADDR_DEPENDENCY_TABLE

typedef struct _ROW_ADDR_DEPENDENCY_TABLE ROW_ADDR_DEPENDENCY_TABLE

The row address dependency table for all the user blocks.

See also

ROW_ADDR_DEPENDENCY_ENTRY.

Function Documentation

CheckDoneNvmeDmaReq()

void CheckDoneNvmeDmaReq

(

)

Definition at line 918 of file request_transform.c.

{
    unsigned int reqSlotTag, prevReq;
    unsigned int rxDone, txDone;
 
    reqSlotTag = nvmeDmaReqQ.tailReq;
    rxDone     = 0;
    txDone     = 0;
 
    while (reqSlotTag != REQ_SLOT_TAG_NONE)
    {
        prevReq = reqPoolPtr->reqPool[reqSlotTag].prevReq;
 
        if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_RxDMA)
        {
            if (!rxDone)
                rxDone = check_auto_rx_dma_partial_done(reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.reqTail,
                                                        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.overFlowCnt);
 
            if (rxDone)
                SelectiveGetFromNvmeDmaReqQ(reqSlotTag);
        }
        else
        {
            if (!txDone)
                txDone = check_auto_tx_dma_partial_done(reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.reqTail,
                                                        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.overFlowCnt);
 
            if (txDone)
                SelectiveGetFromNvmeDmaReqQ(reqSlotTag);
        }
 
        reqSlotTag = prevReq;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

InitDependencyTable()

void InitDependencyTable

(

)

Definition at line 58 of file request_transform.c.

{
    unsigned int blockNo, wayNo, chNo;
    rowAddrDependencyTablePtr = (P_ROW_ADDR_DEPENDENCY_TABLE)ROW_ADDR_DEPENDENCY_TABLE_ADDR;
 
    for (blockNo = 0; blockNo < MAIN_BLOCKS_PER_DIE; blockNo++)
    {
        for (wayNo = 0; wayNo < USER_WAYS; wayNo++)
        {
            for (chNo = 0; chNo < USER_CHANNELS; chNo++)
            {
                rowAddrDependencyTablePtr->block[chNo][wayNo][blockNo].permittedProgPage   = 0;
                rowAddrDependencyTablePtr->block[chNo][wayNo][blockNo].blockedReadReqCnt   = 0;
                rowAddrDependencyTablePtr->block[chNo][wayNo][blockNo].blockedEraseReqFlag = 0;
            }
        }
    }
}

Here is the caller graph for this function:

IssueNvmeDmaReq()

void IssueNvmeDmaReq

(

unsigned int

reqSlotTag

)

Allocate data buffer for the specified DMA request and inform the controller.

This function is used for issuing a new DMA request, the DMA procedure can be split into 2 steps:

1. Prepare a buffer based on the member dataBufFormat of the specified DMA request

2. Inform NVMe controller

   For a DMA request, it might want to rx/tx a data whose size is larger than 4K which is the NVMe block size, so the firmware need to inform the NVMe controller for each NVMe block.

   The tail reg of the DMA queue will be updated during the set_auto_rx_dma() and set_auto_tx_dma(), so we need to update the nvmeDmaInfo.reqTail after issuing the DMA request.

Warning

For a DMA request, the buffer address generated by GenerateDataBufAddr() is chosen based on the REQ_OPT_DATA_BUF_ENTRY, however, since the size of a data entry is BYTES_PER_DATA_REGION_OF_SLICE (default 4), will the data buffer used by the DMA request overlap with other requests' data buffer if the numOfNvmeBlock of the DMA request is larger than NVME_BLOCKS_PER_PAGE?

Parameters

reqSlotTag

the request pool index of the given request.

Definition at line 878 of file request_transform.c.

{
    unsigned int devAddr, dmaIndex, numOfNvmeBlock;
 
    dmaIndex       = reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.startIndex;
    devAddr        = GenerateDataBufAddr(reqSlotTag);
    numOfNvmeBlock = 0;
 
    if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_RxDMA)
    {
        while (numOfNvmeBlock < reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock)
        {
            set_auto_rx_dma(reqPoolPtr->reqPool[reqSlotTag].nvmeCmdSlotTag, dmaIndex, devAddr,
                            NVME_COMMAND_AUTO_COMPLETION_ON);
 
            numOfNvmeBlock++;
            dmaIndex++;
            devAddr += BYTES_PER_NVME_BLOCK;
        }
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.reqTail     = g_hostDmaStatus.fifoTail.autoDmaRx;
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.overFlowCnt = g_hostDmaAssistStatus.autoDmaRxOverFlowCnt;
    }
    else if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_TxDMA)
    {
        while (numOfNvmeBlock < reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock)
        {
            set_auto_tx_dma(reqPoolPtr->reqPool[reqSlotTag].nvmeCmdSlotTag, dmaIndex, devAddr,
                            NVME_COMMAND_AUTO_COMPLETION_ON);
 
            numOfNvmeBlock++;
            dmaIndex++;
            devAddr += BYTES_PER_NVME_BLOCK;
        }
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.reqTail     = g_hostDmaStatus.fifoTail.autoDmaTx;
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.overFlowCnt = g_hostDmaAssistStatus.autoDmaTxOverFlowCnt;
    }
    else
        assert(!"[WARNING] Not supported reqCode [WARNING]");
}

Here is the call graph for this function:

Here is the caller graph for this function:

ReleaseBlockedByBufDepReq()

void ReleaseBlockedByBufDepReq

(

unsigned int

reqSlotTag

)

Pop the specified request from the buffer dependency queue.

In the current implementation, this function is only called after the specified request entry is moved to the free request queue, which means that the previous request has released the data buffer entry it occupied. Therefore, we now need to update the relevant information about the data buffer dependency.

Warning

Only the NAND requests with VSA can use this function.

Since the struct DATA_BUF_ENTRY maintains only the tail of blocked requests, the specified request should be the head of blocked requests to ensure that the request order is not messed up.

See also

UpdateDataBufEntryInfoBlockingReq(), CheckBufDep(), SSD_REQ_FORMAT.

Parameters

reqSlotTag

The request entry index of the given request

Definition at line 729 of file request_transform.c.

{
    unsigned int targetReqSlotTag, dieNo, chNo, wayNo, rowAddrDepCheckReport;
 
    // split the blocking request queue into 2 parts at `reqSlotTag`
    targetReqSlotTag = REQ_SLOT_TAG_NONE;
    if (reqPoolPtr->reqPool[reqSlotTag].nextBlockingReq != REQ_SLOT_TAG_NONE)
    {
        targetReqSlotTag                                      = reqPoolPtr->reqPool[reqSlotTag].nextBlockingReq;
        reqPoolPtr->reqPool[targetReqSlotTag].prevBlockingReq = REQ_SLOT_TAG_NONE;
        reqPoolPtr->reqPool[reqSlotTag].nextBlockingReq       = REQ_SLOT_TAG_NONE;
    }
 
    // reset blocking request queue if it is the last request blocked by the buffer dependency
    if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.dataBufFormat == REQ_OPT_DATA_BUF_ENTRY)
    {
        if (dataBufMapPtr->dataBuf[reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry].blockingReqTail ==
            reqSlotTag)
            dataBufMapPtr->dataBuf[reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry].blockingReqTail =
                REQ_SLOT_TAG_NONE;
    }
    else if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.dataBufFormat == REQ_OPT_DATA_BUF_TEMP_ENTRY)
    {
        if (tempDataBufMapPtr->tempDataBuf[reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry].blockingReqTail ==
            reqSlotTag)
            tempDataBufMapPtr->tempDataBuf[reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry].blockingReqTail =
                REQ_SLOT_TAG_NONE;
    }
 
    /*
     * the specified request is released, so if its next request is only blocked by data
     * buffer, it can be released now.
     */
    if ((targetReqSlotTag != REQ_SLOT_TAG_NONE) &&
        (reqPoolPtr->reqPool[targetReqSlotTag].reqQueueType == REQ_QUEUE_TYPE_BLOCKED_BY_BUF_DEP))
    {
        SelectiveGetFromBlockedByBufDepReqQ(targetReqSlotTag);
 
        if (reqPoolPtr->reqPool[targetReqSlotTag].reqType == REQ_TYPE_NVME_DMA)
        {
            IssueNvmeDmaReq(targetReqSlotTag);
            PutToNvmeDmaReqQ(targetReqSlotTag);
        }
        else if (reqPoolPtr->reqPool[targetReqSlotTag].reqType == REQ_TYPE_NAND)
        {
            if (reqPoolPtr->reqPool[targetReqSlotTag].reqOpt.nandAddr == REQ_OPT_NAND_ADDR_VSA)
            {
                dieNo = Vsa2VdieTranslation(reqPoolPtr->reqPool[targetReqSlotTag].nandInfo.virtualSliceAddr);
                chNo  = Vdie2PchTranslation(dieNo);
                wayNo = Vdie2PwayTranslation(dieNo);
            }
            else
                assert(!"[WARNING] Not supported reqOpt-nandAddress [WARNING]");
 
            // check the row address dependency if needed
            if (reqPoolPtr->reqPool[targetReqSlotTag].reqOpt.rowAddrDependencyCheck ==
                REQ_OPT_ROW_ADDR_DEPENDENCY_CHECK)
            {
                rowAddrDepCheckReport = CheckRowAddrDep(targetReqSlotTag, ROW_ADDR_DEPENDENCY_CHECK_OPT_RELEASE);
 
                if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_PASS)
                    PutToNandReqQ(targetReqSlotTag, chNo, wayNo);
                else if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_BLOCKED)
                    PutToBlockedByRowAddrDepReqQ(targetReqSlotTag, chNo, wayNo);
                else
                    assert(!"[WARNING] Not supported report [WARNING]");
            }
            else if (reqPoolPtr->reqPool[targetReqSlotTag].reqOpt.rowAddrDependencyCheck ==
                     REQ_OPT_ROW_ADDR_DEPENDENCY_NONE)
                PutToNandReqQ(targetReqSlotTag, chNo, wayNo);
            else
                assert(!"[WARNING] Not supported reqOpt [WARNING]");
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

ReleaseBlockedByRowAddrDepReq()

void ReleaseBlockedByRowAddrDepReq	(	unsigned int	chNo,
		unsigned int	wayNo
	)

Update the row address dependency of all the requests on the specified die.

Traverse the blockedByRowAddrDepReqQ of the specified die, and then recheck the row address dependency for all the requests on that die. When a request is found that it can pass the dependency check, it will be dispatched (move to the NAND request queue).

By updating the row address dependency info, some requests on the target die may be released.

See also

CheckRowAddrDep().

Parameters

chNo	The channel number of the specified die.
wayNo	The way number of the specified die.

Definition at line 820 of file request_transform.c.

{
    unsigned int reqSlotTag, nextReq, rowAddrDepCheckReport;
 
    reqSlotTag = blockedByRowAddrDepReqQ[chNo][wayNo].headReq;
 
    while (reqSlotTag != REQ_SLOT_TAG_NONE)
    {
        nextReq = reqPoolPtr->reqPool[reqSlotTag].nextReq;
 
        if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.rowAddrDependencyCheck == REQ_OPT_ROW_ADDR_DEPENDENCY_CHECK)
        {
            rowAddrDepCheckReport = CheckRowAddrDep(reqSlotTag, ROW_ADDR_DEPENDENCY_CHECK_OPT_RELEASE);
 
            if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_PASS)
            {
                SelectiveGetFromBlockedByRowAddrDepReqQ(reqSlotTag, chNo, wayNo);
                PutToNandReqQ(reqSlotTag, chNo, wayNo);
            }
            else if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_BLOCKED)
            {
                // pass, go to while loop
            }
            else
                assert(!"[WARNING] Not supported report [WARNING]");
        }
        else
            assert(!"[WARNING] Not supported reqOpt [WARNING]");
 
        reqSlotTag = nextReq;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

ReqTransNvmeToSlice()

void ReqTransNvmeToSlice	(	unsigned int	cmdSlotTag,
		unsigned int	startLba,
		unsigned int	nlb,
		unsigned int	cmdCode
	)

Split NVMe command into slice requests.

Note

The unit of the given startLba and nlb is NVMe block, not NAND block.

To get the starting LSA of this NVMe command, we need to divide the given startLba by NVME_BLOCKS_PER_SLICE which indicates that how many NVMe blocks can be merged into a slice request.

To get the number of NAND blocks needed by this NVMe command, we should first align the starting NVMe block address startLba to slice 0, then convert the ending NVMe block address (startLba % NVME_BLOCKS_PER_SLICE + requestedNvmeBlock) to LSA, then the result indicates the number of slice requests needed by this NVMe command.

Note

Accroding to the NVMe spec, NLB is a 0's based value, so we should increase the requestedNvmeBlock by 1 to get the real number of NVMe blocks to be read/written by this NVMe command.

Now the address translation part is finished and we can start to split the NVMe command into slice requests. The splitting process can be separated into 3 steps:

1. Fill the remaining NVMe blocks in first slice request (head)

   Since the startLba may not perfectly align to the first NVMe block of first slice command, we should access the trailing N NVMe blocks in the first slice request, where N is the number of misaligned NVMe blocks in the first slice requests.

2. Generate slice requests for the aligned NVMe blocks (body)

   General case. The number of the NVMe blocks to be filled by these slice requests is exactly NVME_BLOCKS_PER_SLICE. So here just simply use a loop to generate same slice requests.

3. Generate slice request for the remaining NVMe blocks (tail)

   Similar to the first step, but here we need to access the first K NVMe blocks in the last slice request, where K is the number of remaining NVMe blocks in this slice request.

   Todo:

   generalize the three steps

Parameters

cmdSlotTag

Todo:

//TODO

Parameters

startLba	address of the first logical NVMe block to read/write.
nlb	number of logical NVMe blocks to read/write.
cmdCode	opcode of the given NVMe command.

Definition at line 123 of file request_transform.c.

{
    unsigned int reqSlotTag, requestedNvmeBlock, tempNumOfNvmeBlock, transCounter, tempLsa, loop, nvmeBlockOffset,
        nvmeDmaStartIndex, reqCode;
 
    requestedNvmeBlock = nlb + 1;
    transCounter       = 0;
    nvmeDmaStartIndex  = 0;
    tempLsa            = startLba / NVME_BLOCKS_PER_SLICE;
    loop               = ((startLba % NVME_BLOCKS_PER_SLICE) + requestedNvmeBlock) / NVME_BLOCKS_PER_SLICE;
 
    // translate the opcode for NVMe command into that for slice requests.
    if (cmdCode == IO_NVM_WRITE)
        reqCode = REQ_CODE_WRITE;
    else if (cmdCode == IO_NVM_READ)
        reqCode = REQ_CODE_READ;
    else
        assert(!"[WARNING] Not supported command code [WARNING]");
 
    // first transform
    nvmeBlockOffset = (startLba % NVME_BLOCKS_PER_SLICE);
    if (loop)
        tempNumOfNvmeBlock = NVME_BLOCKS_PER_SLICE - nvmeBlockOffset;
    else
        tempNumOfNvmeBlock = requestedNvmeBlock;
 
    reqSlotTag = GetFromFreeReqQ();
 
    reqPoolPtr->reqPool[reqSlotTag].reqType                     = REQ_TYPE_SLICE;
    reqPoolPtr->reqPool[reqSlotTag].reqCode                     = reqCode;
    reqPoolPtr->reqPool[reqSlotTag].nvmeCmdSlotTag              = cmdSlotTag;
    reqPoolPtr->reqPool[reqSlotTag].logicalSliceAddr            = tempLsa;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.startIndex      = nvmeDmaStartIndex;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.nvmeBlockOffset = nvmeBlockOffset;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock  = tempNumOfNvmeBlock;
 
    PutToSliceReqQ(reqSlotTag);
 
    tempLsa++;
    transCounter++;
    nvmeDmaStartIndex += tempNumOfNvmeBlock;
 
    // transform continue
    while (transCounter < loop)
    {
        nvmeBlockOffset    = 0;
        tempNumOfNvmeBlock = NVME_BLOCKS_PER_SLICE;
 
        reqSlotTag = GetFromFreeReqQ();
 
        reqPoolPtr->reqPool[reqSlotTag].reqType                     = REQ_TYPE_SLICE;
        reqPoolPtr->reqPool[reqSlotTag].reqCode                     = reqCode;
        reqPoolPtr->reqPool[reqSlotTag].nvmeCmdSlotTag              = cmdSlotTag;
        reqPoolPtr->reqPool[reqSlotTag].logicalSliceAddr            = tempLsa;
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.startIndex      = nvmeDmaStartIndex;
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.nvmeBlockOffset = nvmeBlockOffset;
        reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock  = tempNumOfNvmeBlock;
 
        PutToSliceReqQ(reqSlotTag);
 
        tempLsa++;
        transCounter++;
        nvmeDmaStartIndex += tempNumOfNvmeBlock;
    }
 
    // last transform
    nvmeBlockOffset    = 0;
    tempNumOfNvmeBlock = (startLba + requestedNvmeBlock) % NVME_BLOCKS_PER_SLICE;
    if ((tempNumOfNvmeBlock == 0) || (loop == 0))
        return;
 
    reqSlotTag = GetFromFreeReqQ();
 
    reqPoolPtr->reqPool[reqSlotTag].reqType                     = REQ_TYPE_SLICE;
    reqPoolPtr->reqPool[reqSlotTag].reqCode                     = reqCode;
    reqPoolPtr->reqPool[reqSlotTag].nvmeCmdSlotTag              = cmdSlotTag;
    reqPoolPtr->reqPool[reqSlotTag].logicalSliceAddr            = tempLsa;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.startIndex      = nvmeDmaStartIndex;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.nvmeBlockOffset = nvmeBlockOffset;
    reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock  = tempNumOfNvmeBlock;
 
    PutToSliceReqQ(reqSlotTag);
}

Here is the call graph for this function:

Here is the caller graph for this function:

ReqTransSliceToLowLevel()

void ReqTransSliceToLowLevel

(

)

Data Buffer Manager. Handle all the pending slice requests.

This function will repeat the following steps until all the pending slice requests are consumed:

1. Select a slice request from the slice request queue sliceReqQ.

2. Allocate a data buffer entry for the request and generate flash requests if needed.

   Warning

   Why no need to modify logicalSliceAddr and generate flash request when buffer hit? data cache hit??

3. Generate NVMe transfer/receive request for read/write request.

   Warning

   Why mark the data buffer dirty for write request?

4. Dispatch the transfer/receive request by calling SelectLowLevelReqQ().

Note

This function is currently only called after handle_nvme_io_cmd() during the process of handling NVMe I/O commands in nvme_main.c.

Definition at line 323 of file request_transform.c.

{
    unsigned int reqSlotTag, dataBufEntry;
 
    // consume all pending slice requests in slice request queue
    while (sliceReqQ.headReq != REQ_SLOT_TAG_NONE)
    {
        // get the request pool entry index of the slice request
        reqSlotTag = GetFromSliceReqQ();
        if (reqSlotTag == REQ_SLOT_TAG_FAIL)
            return;
 
        /*
         * In current implementation, the data buffer to be used is determined on the
         * `logicalSliceAddr` of this request, so the data buffer may already be allocated
         * before and so we can simply reuse that data buffer.
         *
         * If the data buffer not exists, we must allocate a data buffer entry by calling
         * `AllocateDataBuf()` and initialize the newly created data buffer.
         */
        dataBufEntry = CheckDataBufHit(reqSlotTag);
        if (dataBufEntry != DATA_BUF_FAIL)
        {
            // data buffer hit
            reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry = dataBufEntry;
        }
        else
        {
            // data buffer miss, allocate a new buffer entry
            dataBufEntry                                      = AllocateDataBuf();
            reqPoolPtr->reqPool[reqSlotTag].dataBufInfo.entry = dataBufEntry;
 
            // initialize the newly allocated data buffer entry for this request
            EvictDataBufEntry(reqSlotTag);
            dataBufMapPtr->dataBuf[dataBufEntry].logicalSliceAddr =
                reqPoolPtr->reqPool[reqSlotTag].logicalSliceAddr;
            PutToDataBufHashList(dataBufEntry);
 
            /*
             * The allocated buffer will be used to store the data to be sent to host, or
             * received from the host. So before transfering the data to host, we need to
             * call the function `DataReadFromNand()` to read the desired data to buffer.
             */
            if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_READ)
                DataReadFromNand(reqSlotTag);
            else if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_WRITE)
                // in case of not overwriting a whole page, read current page content for migration
                if (reqPoolPtr->reqPool[reqSlotTag].nvmeDmaInfo.numOfNvmeBlock != NVME_BLOCKS_PER_SLICE)
                    // for read modify write
                    DataReadFromNand(reqSlotTag);
        }
 
        // generate NVMe request by replacing the slice request entry directly
        if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_WRITE)
        {
            dataBufMapPtr->dataBuf[dataBufEntry].dirty = DATA_BUF_DIRTY;
            reqPoolPtr->reqPool[reqSlotTag].reqCode    = REQ_CODE_RxDMA;
        }
        else if (reqPoolPtr->reqPool[reqSlotTag].reqCode == REQ_CODE_READ)
            reqPoolPtr->reqPool[reqSlotTag].reqCode = REQ_CODE_TxDMA;
        else
            assert(!"[WARNING] Not supported reqCode. [WARNING]");
 
        reqPoolPtr->reqPool[reqSlotTag].reqType              = REQ_TYPE_NVME_DMA;
        reqPoolPtr->reqPool[reqSlotTag].reqOpt.dataBufFormat = REQ_OPT_DATA_BUF_ENTRY;
 
        UpdateDataBufEntryInfoBlockingReq(dataBufEntry, reqSlotTag);
        SelectLowLevelReqQ(reqSlotTag);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

SelectLowLevelReqQ()

void SelectLowLevelReqQ

(

unsigned int

reqSlotTag

)

Dispatch given NVMe/NAND request to corresponding request queue.

This function is in charge of issuing the given NVMe/NAND request. But before issuing the request, we should first make sure that this request is safe to be issued.

We first need to check whether this request is blocked by any other request that uses the same data buffer (check UpdateDataBufEntryInfoBlockingReq() for details).

• If the request is not blocked by the blocking request queue, we can start issuing the request now, but NVMe/NAND request have different process:
  • For a NVNe DMA request (Tx from/to data buffer to/from host), we can just issue the request and wait for completion.

  • For a NAND request, we must do something before issuing the request:

    However, for NAND requests, since there may be some dependency problems between the requests (e.g., ERASE cannot be simply reordered before READ), we must check this kind of dependency problems (called "row address dependency" here) before dispatching the NAND requests by using the function CheckRowAddrDep().

    Once it was confirmed to have no row address dependency problem on this request, the request can then be dispatched; otherwise, the request should be blocked and inserted to the row address dependency queue.

    Note

    In current implementation, the address format of the request to be check must be VSA. So, for requests that using the physical address, the check option should be set to REQ_OPT_ROW_ADDR_DEPENDENCY_CHECK.

• If the request is blocked by data buffer dependency

  The fw will try to recheck the data buffer dependency problem and release the request if possible, by calling UpdateRowAddrDepTableForBufBlockedReq(), which is similar to CheckRowAddrDep().

See also

CheckRowAddrDep(), UpdateDataBufEntryInfoBlockingReq(), NAND_INFO.

Parameters

reqSlotTag

the request pool index of the given request.

Definition at line 634 of file request_transform.c.

{
    unsigned int dieNo, chNo, wayNo, bufDepCheckReport, rowAddrDepCheckReport, rowAddrDepTableUpdateReport;
 
    bufDepCheckReport = CheckBufDep(reqSlotTag);
 
    if (bufDepCheckReport == BUF_DEPENDENCY_REPORT_PASS)
    {
        if (reqPoolPtr->reqPool[reqSlotTag].reqType == REQ_TYPE_NVME_DMA)
        {
            IssueNvmeDmaReq(reqSlotTag);
            PutToNvmeDmaReqQ(reqSlotTag);
        }
        else if (reqPoolPtr->reqPool[reqSlotTag].reqType == REQ_TYPE_NAND)
        {
            // get physical organization info from VSA
            if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.nandAddr == REQ_OPT_NAND_ADDR_VSA)
            {
                dieNo = Vsa2VdieTranslation(reqPoolPtr->reqPool[reqSlotTag].nandInfo.virtualSliceAddr);
                chNo  = Vdie2PchTranslation(dieNo);
                wayNo = Vdie2PwayTranslation(dieNo);
            }
            // if the physical organization info is already specified, use it without translating
            else if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.nandAddr == REQ_OPT_NAND_ADDR_PHY_ORG)
            {
                chNo  = reqPoolPtr->reqPool[reqSlotTag].nandInfo.physicalCh;
                wayNo = reqPoolPtr->reqPool[reqSlotTag].nandInfo.physicalWay;
            }
            else
                assert(!"[WARNING] Not supported reqOpt-nandAddress [WARNING]");
 
            // check row address dependency problem before dispatching
            if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.rowAddrDependencyCheck == REQ_OPT_ROW_ADDR_DEPENDENCY_CHECK)
            {
                // NOTE: VSA translation in `CheckRowAddrDep()` could be skipped
                rowAddrDepCheckReport = CheckRowAddrDep(reqSlotTag, ROW_ADDR_DEPENDENCY_CHECK_OPT_SELECT);
 
                if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_PASS)
                    PutToNandReqQ(reqSlotTag, chNo, wayNo);
                else if (rowAddrDepCheckReport == ROW_ADDR_DEPENDENCY_REPORT_BLOCKED)
                    PutToBlockedByRowAddrDepReqQ(reqSlotTag, chNo, wayNo);
                else
                    assert(!"[WARNING] Not supported report [WARNING]");
            }
            else if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.rowAddrDependencyCheck ==
                     REQ_OPT_ROW_ADDR_DEPENDENCY_NONE)
                PutToNandReqQ(reqSlotTag, chNo, wayNo);
            else
                assert(!"[WARNING] Not supported reqOpt [WARNING]");
        }
        else
            assert(!"[WARNING] Not supported reqType [WARNING]");
    }
    else if (bufDepCheckReport == BUF_DEPENDENCY_REPORT_BLOCKED)
    {
        if (reqPoolPtr->reqPool[reqSlotTag].reqType == REQ_TYPE_NAND)
            if (reqPoolPtr->reqPool[reqSlotTag].reqOpt.rowAddrDependencyCheck == REQ_OPT_ROW_ADDR_DEPENDENCY_CHECK)
            {
                // update row addr dep info and insert to `blockedByRowAddrDepReqQ` if needed
                rowAddrDepTableUpdateReport = UpdateRowAddrDepTableForBufBlockedReq(reqSlotTag);
 
                if (rowAddrDepTableUpdateReport == ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_DONE)
                {
                    // no row addr dep problem, so put to blockedByBufDepReqQ
                }
                else if (rowAddrDepTableUpdateReport == ROW_ADDR_DEPENDENCY_TABLE_UPDATE_REPORT_SYNC)
                    return;
                else
                    assert(!"[WARNING] Not supported report [WARNING]");
            }
 
        PutToBlockedByBufDepReqQ(reqSlotTag);
    }
    else
        assert(!"[WARNING] Not supported report [WARNING]");
}

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

rowAddrDependencyTablePtr

P_ROW_ADDR_DEPENDENCY_TABLE rowAddrDependencyTablePtr

extern

Definition at line 56 of file request_transform.c.