--- /dev/null
+/*++
+
+Copyright (c) Microsoft Corporation
+
+Module Name:
+
+ FxDmaTransaction.cpp
+
+Abstract:
+
+ WDF DMA Transaction Object
+
+Environment:
+
+ Kernel mode only.
+
+Notes:
+
+
+Revision History:
+
+--*/
+
+#include "FxDmaPCH.hpp"
+
+extern "C" {
+#include "FxDmaTransaction.tmh"
+}
+
+FxDmaTransactionBase::FxDmaTransactionBase(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in USHORT ObjectSize,
+ __in USHORT ExtraSize,
+ __in FxDmaEnabler *DmaEnabler
+ ) :
+ FxNonPagedObject(
+ FX_TYPE_DMA_TRANSACTION,
+ ExtraSize == 0 ? ObjectSize : COMPUTE_OBJECT_SIZE(ObjectSize, ExtraSize),
+ FxDriverGlobals)
+{
+ m_DmaEnabler = DmaEnabler;
+ m_EncodedRequest = NULL;
+ m_MaxFragmentLength = 0;
+ m_DmaDirection = WdfDmaDirectionReadFromDevice;
+ m_DmaAcquiredContext = NULL;
+ m_CurrentFragmentMdl = NULL;
+ m_CurrentFragmentOffset = 0;
+ m_StartOffset = NULL;
+ m_StartMdl = NULL;
+ m_Remaining = 0;
+ m_CurrentFragmentLength = 0;
+ m_TransactionLength = 0;
+ m_Transferred = 0;
+ m_Flags = 0;
+
+ m_DmaAcquiredFunction.Method.ProgramDma = NULL;
+
+ m_State = FxDmaTransactionStateCreated;
+
+ if (ExtraSize == 0) {
+ m_TransferContext = NULL;
+ } else {
+ m_TransferContext = WDF_PTR_ADD_OFFSET_TYPE(
+ this,
+ COMPUTE_RAW_OBJECT_SIZE(ObjectSize),
+ PVOID
+ );
+ }
+
+ MarkDisposeOverride(ObjectDoNotLock);
+}
+
+BOOLEAN
+FxDmaTransactionBase::Dispose(
+ VOID
+ )
+{
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ //
+ // Must not be in transfer state.
+ //
+ if (m_State == FxDmaTransactionStateTransfer) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", GetHandle(), m_State);
+
+ if (pFxDriverGlobals->IsVerificationEnabled(1, 9, OkForDownLevel)) {
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+ }
+
+ m_State = FxDmaTransactionStateDeleted;
+
+ //
+ // Release resources for this Dma Transaction.
+ //
+ ReleaseResources(TRUE);
+
+ if (m_EncodedRequest != NULL) {
+ ClearRequest();
+ }
+
+ return TRUE;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaTransactionBase::Initialize(
+ __in PFN_WDF_PROGRAM_DMA ProgramDmaFunction,
+ __in WDF_DMA_DIRECTION DmaDirection,
+ __in PMDL Mdl,
+ __in size_t Offset,
+ __in ULONG Length
+ )
+{
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p", GetHandle());
+ //
+ // Must be in Reserve, Created or Released state.
+ //
+ if (m_State != FxDmaTransactionStateCreated &&
+ m_State != FxDmaTransactionStateReserved &&
+ m_State != FxDmaTransactionStateReleased) {
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", GetHandle(), m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ if (DmaDirection == WdfDmaDirectionReadFromDevice) {
+ m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription();
+ } else {
+ m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription();
+ }
+
+ //
+ // Initialize the DmaTransaction object
+ //
+
+ m_MaxFragmentLength = m_AdapterInfo->MaximumFragmentLength;
+ m_DmaDirection = DmaDirection;
+ m_StartMdl = Mdl;
+ m_StartOffset = Offset;
+ m_CurrentFragmentMdl = Mdl;
+ m_CurrentFragmentOffset = Offset;
+ m_Remaining = Length;
+ m_TransactionLength = Length;
+ m_DmaAcquiredFunction.Method.ProgramDma = ProgramDmaFunction;
+
+ //
+ // If needed, initialize the transfer context.
+ //
+
+ if (m_DmaEnabler->UsesDmaV3()) {
+ m_DmaEnabler->InitializeTransferContext(GetTransferContext(),
+ m_DmaDirection);
+ }
+
+ status = InitializeResources();
+ if (NT_SUCCESS(status)) {
+ m_State = FxDmaTransactionStateInitialized;
+ } else {
+ ReleaseForReuse(FALSE);
+ }
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p, %!STATUS!",
+ GetHandle(), status);
+
+ return status;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaTransactionBase::Execute(
+ __in PVOID Context
+ )
+{
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ //
+ // Must be in Initialized state.
+ //
+ if (m_State != FxDmaTransactionStateInitialized) {
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", GetHandle(), m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ //
+ // If this was initialized with a request, then reference the
+ // request now.
+ //
+ if (m_EncodedRequest != NULL) {
+ ReferenceRequest();
+ }
+
+ //
+ // Set state to Transfer.
+ // This is necessary because the Execute path complete
+ // all the way to DmaCompleted before returning to this point.
+ //
+ m_State = FxDmaTransactionStateTransfer;
+
+ //
+ // Save the caller's context
+ //
+ m_DmaAcquiredContext = Context;
+
+ ASSERT(m_Transferred == 0);
+ ASSERT(m_CurrentFragmentLength == 0);
+
+ status = StartTransfer();
+ if (!NT_SUCCESS(status)) {
+ m_State = FxDmaTransactionStateTransferFailed;
+ m_DmaAcquiredContext = NULL;
+
+ if (m_EncodedRequest != NULL) {
+ ReleaseButRetainRequest();
+ }
+ }
+
+ //
+ // StartTransfer results in a call to the EvtProgramDma routine
+ // where driver could complete and delete the object. So
+ // don't touch the object beyond this point.
+ //
+
+ return status;
+}
+
+BOOLEAN
+FxDmaTransactionBase::DmaCompleted(
+ __in size_t TransferredLength,
+ __out NTSTATUS * ReturnStatus,
+ __in FxDmaCompletionType CompletionType
+ )
+{
+ BOOLEAN hasTransitioned;
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+ WDFDMATRANSACTION dmaTransaction;
+
+ //
+ // In the case of partial completion, we will start a new transfer
+ // from with in this function by calling StageTransfer. After that
+ // call, we lose ownership of the object. Since we need the handle
+ // for tracing purposes, we will save the value in a local variable and
+ // use that.
+ //
+ dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p, length %d",
+ dmaTransaction, (ULONG)TransferredLength);
+ }
+
+ //
+ // Must be in Transfer state.
+ //
+ if (m_State != FxDmaTransactionStateTransfer) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", dmaTransaction, m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) dmaTransaction, // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ if (TransferredLength > m_CurrentFragmentLength) {
+ status = STATUS_INVALID_PARAMETER;
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p Transfered Length %I64d can't be more "
+ "than the length asked to transfer %I64d "
+ "%!STATUS!", dmaTransaction, TransferredLength,
+ m_CurrentFragmentLength, status);
+ FxVerifierDbgBreakPoint(pFxDriverGlobals);
+ goto End;
+ }
+
+
+ if (CompletionType == FxDmaCompletionTypePartial ||
+ CompletionType == FxDmaCompletionTypeAbort) {
+ //
+ // Tally this DMA tranferred byte count into the accumulator.
+ //
+ m_Transferred += TransferredLength;
+
+ //
+ // Adjust the remaining length to account for the partial transfer.
+ //
+ m_Remaining += (m_CurrentFragmentLength - TransferredLength);
+
+ //
+ // Update CurrentDmaLength to reflect actual transfer because
+ // we need to FlushAdapterBuffers based on this value in
+ // TransferCompleted for packet based transfer.
+ //
+ m_CurrentFragmentLength = TransferredLength;
+
+ } else {
+ //
+ // Tally this DMA tranferred byte count into the accumulator.
+ //
+ m_Transferred += m_CurrentFragmentLength;
+ }
+
+ ASSERT(m_Transferred <= m_TransactionLength);
+
+ //
+ // Inform the derived object that transfer is completed so it
+ // can release resources specific to last transfer.
+ //
+ status = TransferCompleted();
+ if (!NT_SUCCESS(status)) {
+ goto End;
+ }
+
+ //
+ // If remaining DmaTransaction length is zero or if the driver wants
+ // this to be the last transfer then free the map registers and
+ // change the state to completed.
+ //
+ if (m_Remaining == 0 || CompletionType == FxDmaCompletionTypeAbort) {
+ status = STATUS_SUCCESS;
+ goto End;
+ }
+
+ //
+ // Stage the next packet for this DmaTransaction...
+ //
+ status = StageTransfer();
+
+ if (NT_SUCCESS(status)) {
+ //
+ // StageTransfer results in a call to the EvtProgramDma routine
+ // where driver could complete and delete the object. So
+ // don't touch the object beyond this point.
+ //
+ status = STATUS_MORE_PROCESSING_REQUIRED;
+ }
+ else {
+ //
+ // The error will be returned to the caller of
+ // WdfDmaTransactionDmaComplete*()
+ //
+ }
+
+End:
+
+ if (status != STATUS_MORE_PROCESSING_REQUIRED) {
+ //
+ // Failed or succeeded. Either way free
+ // map registers and release the device.
+ //
+ if (NT_SUCCESS(status)) {
+ m_State = FxDmaTransactionStateTransferCompleted;
+ } else {
+ m_State = FxDmaTransactionStateTransferFailed;
+ }
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "WDFDMATRANSACTION %p completed with status %!STATUS! - "
+ "releasing DMA resources",
+ GetHandle(),
+ status);
+ }
+
+ ReleaseResources(FALSE);
+
+ if (m_EncodedRequest != NULL) {
+ ReleaseButRetainRequest();
+ }
+
+ m_CurrentFragmentLength = 0;
+
+ hasTransitioned = TRUE;
+ } else {
+ hasTransitioned = FALSE;
+ }
+
+ *ReturnStatus = status;
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p "
+ "Transitioned(%!BOOLEAN!)",
+ dmaTransaction, hasTransitioned);
+ }
+
+ return hasTransitioned;
+}
+
+VOID
+FxDmaTransactionBase::ReleaseForReuse(
+ __in BOOLEAN ForceRelease
+ )
+{
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ if (ForceRelease == FALSE)
+ {
+ if (m_State == FxDmaTransactionStateReleased) {
+
+ //
+ // Double release is probably due to cancel during early in transaction
+ // initialization. DC2 on very slow machines shows this behavior.
+ // The double release case is rare and benign.
+ //
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_WARNING, TRACINGDMA,
+ "WDFDMATRANSACTION %p is already released, "
+ "%!STATUS!", GetHandle(), STATUS_SUCCESS);
+
+ return; // already released.
+ }
+
+ //
+ // Must not be in transfer state.
+ //
+ if (m_State == FxDmaTransactionStateTransfer) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid (release transaction)", GetHandle(), m_State);
+
+ if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) {
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+ }
+ }
+
+ m_State = FxDmaTransactionStateReleased;
+
+ ReleaseResources(ForceRelease);
+
+ //
+ // Except DMA enabler field and adapter info everything else should be
+ // cleared. Adapter info is cleared by ReleaseResources above.
+ //
+ m_DmaAcquiredContext = NULL;
+
+ if (m_EncodedRequest != NULL) {
+ ClearRequest();
+ }
+
+ m_StartMdl = NULL;
+ m_CurrentFragmentMdl = NULL;
+ m_StartOffset = 0;
+ m_CurrentFragmentOffset = 0;
+ m_CurrentFragmentLength = 0;
+ m_Transferred = 0;
+ m_Remaining = 0;
+ m_MaxFragmentLength = 0;
+ m_TransactionLength = 0;
+ m_Flags = 0;
+
+ m_DmaAcquiredFunction.Method.ProgramDma = NULL;
+
+}
+
+VOID
+FxDmaTransactionBase::SetImmediateExecution(
+ __in BOOLEAN Value
+ )
+{
+ if (m_State != FxDmaTransactionStateCreated &&
+ m_State != FxDmaTransactionStateInitialized &&
+ m_State != FxDmaTransactionStateReleased) {
+ DoTraceLevelMessage(
+ GetDriverGlobals(), TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Must set immediate execution flag for WDFDMATRANSACTION "
+ "%p before calling AllocateResources or Execute (current "
+ "state is %!FxDmaTransactionState!)",
+ GetHandle(),
+ m_State
+ );
+ FxVerifierDbgBreakPoint(GetDriverGlobals());
+ }
+
+ if (Value) {
+ m_Flags |= DMA_SYNCHRONOUS_CALLBACK;
+ }
+ else {
+ m_Flags &= ~DMA_SYNCHRONOUS_CALLBACK;
+ }
+}
+
+BOOLEAN
+FxDmaTransactionBase::CancelResourceAllocation(
+ VOID
+ )
+{
+ if ((m_State == FxDmaTransactionStateCreated) ||
+ (m_State == FxDmaTransactionStateReleased) ||
+ (m_State == FxDmaTransactionStateDeleted)) {
+
+ DoTraceLevelMessage(
+ GetDriverGlobals(),
+ TRACE_LEVEL_ERROR,
+ TRACINGDMA,
+ "WDFDMATRANSACTION %p cannot be cancelled in state "
+ "%!FxDmaTransactionState!",
+ GetHandle(),
+ m_State
+ );
+
+ FxVerifierBugCheck(GetDriverGlobals(),
+ WDF_DMA_FATAL_ERROR,
+ (ULONG_PTR) GetObjectHandle(),
+ (ULONG_PTR) m_State);
+ // unreachable code
+ }
+
+ PDMA_OPERATIONS dmaOperations =
+ m_AdapterInfo->AdapterObject->DmaOperations;
+
+ BOOLEAN result;
+
+ result = dmaOperations->CancelAdapterChannel(
+ m_AdapterInfo->AdapterObject,
+ m_DmaEnabler->m_FDO,
+ GetTransferContext()
+ );
+
+ if (result) {
+ m_State = FxDmaTransactionStateTransferFailed;
+
+ if (m_EncodedRequest != NULL) {
+ ReleaseButRetainRequest();
+ }
+ }
+
+ return result;
+}
+
+VOID
+FxDmaTransactionBase::_ComputeNextTransferAddress(
+ __in PMDL CurrentMdl,
+ __in size_t CurrentOffset,
+ __in ULONG Transferred,
+ __deref_out PMDL *NextMdl,
+ __out size_t *NextOffset
+ )
+/*++
+
+Routine Description:
+
+ This function computes the next mdl and offset given the current MDL,
+ offset and bytes transfered.
+
+Arguments:
+
+ CurrentMdl - Mdl where the transfer currently took place.
+
+ CurrentVa - Current virtual address in the buffer
+
+ Transfered - Bytes transfered or to be transfered
+
+ NextMdl - Mdl where the next transfer will take place
+
+ NextVA - Offset within NextMdl where the transfer will start
+
+--*/
+{
+ size_t transfered, mdlSize;
+ PMDL mdl;
+
+ mdlSize = MmGetMdlByteCount(CurrentMdl) - CurrentOffset;
+
+ if (Transferred < mdlSize) {
+ //
+ // We are still in the first MDL
+ //
+ *NextMdl = CurrentMdl;
+ *NextOffset = CurrentOffset + Transferred;
+ return;
+ }
+
+ //
+ // We have transfered the content of the first MDL.
+ // Move to the next one.
+ //
+ transfered = Transferred - mdlSize;
+ mdl = CurrentMdl->Next;
+ ASSERT(mdl != NULL);
+
+ while (transfered >= MmGetMdlByteCount(mdl)) {
+ //
+ // We have transfered the content of this MDL.
+ // Move to the next one.
+ //
+ transfered -= MmGetMdlByteCount(mdl);
+ mdl = mdl->Next;
+ ASSERT(mdl != NULL);
+ }
+
+ //
+ // This is the mdl where the last transfer occured.
+ //
+ *NextMdl = mdl;
+ *NextOffset = transfered;
+
+ return;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaTransactionBase::_CalculateRequiredMapRegisters(
+ __in PMDL Mdl,
+ __in size_t CurrentOffset,
+ __in ULONG Length,
+ __in ULONG AvailableMapRegisters,
+ __out_opt PULONG PossibleTransferLength,
+ __out PULONG MapRegistersRequired
+ )
+/*++
+
+Routine Description:
+
+ Used on Windows 2000 to compute number of map registered required
+ for this transfer. This is derived from HalCalculateScatterGatherListSize.
+
+Arguments:
+
+ Mdl - Pointer to the MDL that describes the pages of memory that are being
+ read or written.
+
+ CurrentVa - Current virtual address in the buffer described by the MDL
+ that the transfer is being done to or from.
+
+ Length - Supplies the length of the transfer.
+
+ AvailableMapRegisters - Map registers available to do the transfer
+
+ PossibleTransferLength - Length that can transfered for the
+
+ MapRegistersRequired - Map registers required to the entire transfer
+
+Return Value:
+
+ NTSTATUS
+
+Notes:
+
+--*/
+{
+ PMDL tempMdl;
+ ULONG requiredMapRegisters;
+ ULONG transferLength;
+ ULONG mdlLength;
+ ULONG pageOffset;
+ ULONG possTransferLength;
+
+ //
+ // Calculate the number of required map registers.
+ //
+ tempMdl = Mdl;
+ transferLength = (ULONG) MmGetMdlByteCount(tempMdl) - (ULONG) CurrentOffset;
+ mdlLength = transferLength;
+
+ pageOffset = BYTE_OFFSET(GetStartVaFromOffset(tempMdl, CurrentOffset));
+ requiredMapRegisters = 0;
+ possTransferLength = 0;
+
+ //
+ // The virtual address should fit in the first MDL.
+ //
+
+ ASSERT(CurrentOffset <= tempMdl->ByteCount);
+
+ //
+ // Loop through chained MDLs, accumulating the required
+ // number of map registers.
+ //
+
+ while (transferLength < Length && tempMdl->Next != NULL) {
+
+ //
+ // With pageOffset and length, calculate number of pages spanned by
+ // the buffer.
+ //
+ requiredMapRegisters += (pageOffset + mdlLength + PAGE_SIZE - 1) >>
+ PAGE_SHIFT;
+
+ if (requiredMapRegisters <= AvailableMapRegisters) {
+ possTransferLength = transferLength;
+ }
+
+ tempMdl = tempMdl->Next;
+ pageOffset = tempMdl->ByteOffset;
+ mdlLength = tempMdl->ByteCount;
+ transferLength += mdlLength;
+ }
+
+ if ((transferLength + PAGE_SIZE) < (Length + pageOffset )) {
+ ASSERT(transferLength >= Length);
+ return STATUS_BUFFER_TOO_SMALL;
+ }
+
+ //
+ // Calculate the last number of map registers based on the requested
+ // length not the length of the last MDL.
+ //
+
+ ASSERT( transferLength <= mdlLength + Length );
+
+ requiredMapRegisters += (pageOffset + Length + mdlLength - transferLength +
+ PAGE_SIZE - 1) >> PAGE_SHIFT;
+
+ if (requiredMapRegisters <= AvailableMapRegisters) {
+ possTransferLength += (Length + mdlLength - transferLength);
+ }
+
+ if (PossibleTransferLength != NULL) {
+ *PossibleTransferLength = possTransferLength;
+ }
+
+ ASSERT(*PossibleTransferLength);
+
+ *MapRegistersRequired = requiredMapRegisters;
+
+ return STATUS_SUCCESS;
+}
+
+// ----------------------------------------------------------------------------
+// ------------------- Scatter/Gather DMA Section -----------------------------
+// ----------------------------------------------------------------------------
+
+FxDmaScatterGatherTransaction::FxDmaScatterGatherTransaction(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in USHORT ExtraSize,
+ __in FxDmaEnabler *DmaEnabler
+ ) :
+ FxDmaTransactionBase(FxDriverGlobals,
+ sizeof(FxDmaScatterGatherTransaction),
+ ExtraSize,
+ DmaEnabler)
+{
+ m_LookasideBuffer = NULL;
+ m_SGList = NULL;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaScatterGatherTransaction::_Create(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in PWDF_OBJECT_ATTRIBUTES Attributes,
+ __in FxDmaEnabler* DmaEnabler,
+ __out WDFDMATRANSACTION* Transaction
+ )
+{
+ FxDmaScatterGatherTransaction* pTransaction;
+ WDFOBJECT hTransaction;
+ NTSTATUS status;
+
+ pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
+ FxDmaScatterGatherTransaction(FxDriverGlobals,
+ DmaEnabler->GetTransferContextSize(),
+ DmaEnabler);
+
+ if (pTransaction == NULL) {
+ status = STATUS_INSUFFICIENT_RESOURCES;
+ DoTraceLevelMessage(
+ FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
+ return status;
+ }
+
+ //
+ // Commit and apply the attributes
+ //
+ status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
+
+ if (NT_SUCCESS(status) && DmaEnabler->m_IsSGListAllocated) {
+
+ //
+ // Allocate buffer for SGList from lookaside list.
+ //
+ pTransaction->m_LookasideBuffer = (SCATTER_GATHER_LIST *)
+ FxAllocateFromNPagedLookasideList(
+ &DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside
+ );
+
+ if (pTransaction->m_LookasideBuffer == NULL) {
+ status = STATUS_INSUFFICIENT_RESOURCES;
+ DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Unable to allocate memory for SG List, "
+ "WDFDMATRANSACTION %p, %!STATUS! ",
+ pTransaction->GetHandle(), status);
+ }
+ else {
+ //
+ // Take a reference on the enabler to ensure that it remains valid
+ // if the transaction's disposal is deferred.
+ //
+ DmaEnabler->ADDREF(pTransaction);
+ }
+ }
+
+ if (NT_SUCCESS(status)) {
+ *Transaction = (WDFDMATRANSACTION)hTransaction;
+ }
+ else {
+ //
+ // This will properly clean up the target's state and free it
+ //
+ pTransaction->DeleteFromFailedCreate();
+ }
+
+ return status;
+}
+
+BOOLEAN
+FxDmaScatterGatherTransaction::Dispose(
+ VOID
+ )
+{
+ BOOLEAN ret;
+
+ ret = __super::Dispose();
+
+ //
+ // Free Lookaside Buffer which held SGList
+ //
+ if (m_LookasideBuffer != NULL) {
+
+ FxFreeToNPagedLookasideList(
+ &m_DmaEnabler->m_SGList.ScatterGatherProfile.Lookaside,
+ m_LookasideBuffer
+ );
+ m_LookasideBuffer = NULL;
+ m_DmaEnabler->RELEASE(this);
+ }
+
+ return ret;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaScatterGatherTransaction::InitializeResources(
+ VOID
+ )
+{
+ NTSTATUS status;
+ PMDL nextMdl;
+ size_t nextOffset;
+ ULONG mapRegistersRequired;
+ size_t remLength, transferLength, transferred, possibleLength=0;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ status = STATUS_SUCCESS;
+
+ //
+ // If the caller has specified a limit on the number of scatter-gather
+ // elements each transfer can support then make sure it's within the
+ // limit by breaking up the whole transfer into m_MaxFragmentLength and
+ // computing the number of map-registers required for each fragment.
+ // This check may not be valid if the driver starts to do partial
+ // transfers. So driver that do partial transfer with sg-element limit
+ // should be capable of handling STATUS_WDF_TOO_FRAGMENTED failures during
+ // dma execution.
+ //
+ remLength = m_TransactionLength;
+ transferred = 0;
+ nextMdl = m_StartMdl;
+ nextOffset = m_StartOffset;
+ transferLength = 0;
+
+ while (remLength != 0) {
+
+ _ComputeNextTransferAddress(nextMdl,
+ nextOffset,
+ (ULONG) transferLength,
+ &nextMdl,
+ &nextOffset);
+
+ transferLength = FxSizeTMin(remLength, m_MaxFragmentLength);
+
+ status = _CalculateRequiredMapRegisters(nextMdl,
+ nextOffset,
+ (ULONG) transferLength,
+ m_AdapterInfo->NumberOfMapRegisters,
+ (PULONG) &possibleLength,
+ &mapRegistersRequired
+ );
+
+ if (!NT_SUCCESS(status)) {
+ DoTraceLevelMessage(
+ pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "CalculateScatterGatherList failed for "
+ "WDFDMATRANSACTION %p, %!STATUS!", GetHandle(), status);
+ FxVerifierDbgBreakPoint(pFxDriverGlobals);
+ return status;
+ }
+
+ if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) {
+ status = STATUS_WDF_TOO_FRAGMENTED;
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) "
+ "than the limit (%I64d) specified by the driver, %!STATUS! ",
+ GetHandle(), nextMdl, mapRegistersRequired,
+ m_DmaEnabler->m_MaxSGElements, status);
+ return status;
+ }
+
+ transferred += transferLength;
+ remLength -= transferLength;
+ }
+
+ return status;
+}
+
+VOID
+FxDmaScatterGatherTransaction::ReleaseResources(
+ __in BOOLEAN /* ForceRelease */
+ )
+{
+ if (m_SGList != NULL) {
+ PutScatterGatherList(m_SGList);
+ m_SGList = NULL;
+ }
+ m_AdapterInfo = NULL;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaScatterGatherTransaction::StartTransfer(
+ VOID
+ )
+{
+ ASSERT(m_CurrentFragmentMdl == m_StartMdl);
+ ASSERT(m_CurrentFragmentOffset == m_StartOffset);
+ ASSERT(m_CurrentFragmentLength == 0);
+ ASSERT(m_Transferred == 0);
+
+ return StageTransfer();
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaScatterGatherTransaction::StageTransfer(
+ VOID
+ )
+{
+ NTSTATUS status;
+ ULONG mapRegistersRequired;
+ WDFDMATRANSACTION dmaTransaction;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ //
+ // Use an invalid value to make the function fail if the var is not
+ // updated correctly below.
+ //
+ mapRegistersRequired = 0xFFFFFFFF;
+
+ //
+ // Client driver could complete and delete the object in
+ // EvtProgramDmaFunction. So, save the handle because we need it
+ // for tracing after we invoke the callback.
+ //
+ dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p ", GetHandle());
+ }
+
+ //
+ // Given the first MDL and the bytes transfered, find the next MDL
+ // and byteoffset within that MDL.
+ //
+ _ComputeNextTransferAddress(m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset,
+ (ULONG) m_CurrentFragmentLength,
+ &m_CurrentFragmentMdl,
+ &m_CurrentFragmentOffset);
+
+ //
+ // Get the next possible transfer size.
+ //
+ m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
+
+ //
+ // Fix m_CurrentFragmentLength to meet the map registers limit. This is done
+ // in case the MDL is a chained MDL for an highly fragmented buffer.
+ //
+ status = _CalculateRequiredMapRegisters(m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset,
+ (ULONG) m_CurrentFragmentLength,
+ m_AdapterInfo->NumberOfMapRegisters,
+ (PULONG)&m_CurrentFragmentLength,
+ &mapRegistersRequired);
+ //
+ // We have already validated the entire transfer during initialize
+ // to see each transfer meets the sglimit. So this call shouldn't fail.
+ // But, if the driver does partial transfer and changes the fragment
+ // boundaries then it's possible for the sg-elements to vary. So, check
+ // one more time to see if we are within the bounds before building
+ // the sglist and calling into the driver.
+ //
+ ASSERT(NT_SUCCESS(status));
+
+ if (mapRegistersRequired > m_DmaEnabler->m_MaxSGElements) {
+ status = STATUS_WDF_TOO_FRAGMENTED;
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p for MDL %p is more fragmented (%d) "
+ "than the limit (%I64d) specified by the driver, %!STATUS! ",
+ dmaTransaction, m_CurrentFragmentMdl, mapRegistersRequired,
+ m_DmaEnabler->m_MaxSGElements, status);
+ return status;
+ }
+
+
+ m_Remaining -= m_CurrentFragmentLength;
+
+ if (m_DmaEnabler->m_IsSGListAllocated) {
+
+ ASSERT(m_LookasideBuffer != NULL);
+ status = BuildScatterGatherList(m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset,
+ (ULONG) m_CurrentFragmentLength,
+#pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.")
+ _AdapterListControl,
+ this,
+ m_LookasideBuffer,
+ (ULONG) m_AdapterInfo->PreallocatedSGListSize);
+
+ } else {
+
+ status = GetScatterGatherList(m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset,
+ (ULONG) m_CurrentFragmentLength,
+#pragma prefast(suppress: __WARNING_CLASS_MISMATCH_NONE, "This warning requires a wrapper class for the DRIVER_LIST_CONTROL type.")
+ _AdapterListControl,
+ this);
+ }
+
+ if (!NT_SUCCESS(status)) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Build or GetScatterGatherList failed for "
+ "WDFDMATRANSACTION %p, %!STATUS!",
+ dmaTransaction, status);
+ //
+ // Readjust remaining bytes transfered.
+ //
+ m_Remaining += m_CurrentFragmentLength;
+ return status;
+ }
+
+ //
+ // Before GetScatterGatherList returns, _AdapterListControl can get called
+ // on another thread and the driver could delete the transaction object.
+ // So don't touch the object after this point.
+ //
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p, "
+ "%!STATUS!", dmaTransaction, status);
+ }
+
+ return status;
+}
+
+
+VOID
+FxDmaScatterGatherTransaction::_AdapterListControl(
+ __in PDEVICE_OBJECT DeviceObject,
+ __in PIRP Irp, // UNUSED
+ __in PSCATTER_GATHER_LIST SgList,
+ __in PVOID Context
+ )
+{
+ PFX_DRIVER_GLOBALS pFxDriverGlobals;
+ WDFDMATRANSACTION dmaTransaction;
+ FxDmaScatterGatherTransaction * pDmaTransaction;
+
+ UNREFERENCED_PARAMETER(Irp);
+ UNREFERENCED_PARAMETER(DeviceObject);
+
+ pDmaTransaction = (FxDmaScatterGatherTransaction*) Context;
+ pFxDriverGlobals = pDmaTransaction->GetDriverGlobals();
+ dmaTransaction = pDmaTransaction->GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p",
+ dmaTransaction);
+ }
+
+ ASSERT(pDmaTransaction != NULL);
+ ASSERT(pDmaTransaction->m_DmaAcquiredFunction.Method.ProgramDma != NULL);
+
+ ASSERT(SgList->NumberOfElements <= pDmaTransaction->m_DmaEnabler->GetMaxSGElements());
+
+ pDmaTransaction->m_SGList = SgList;
+
+ //
+ // We ignore the return value. The pattern we want the driver to follow is
+ // that if it fails to program DMA transfer, it should call DmaCompletedFinal
+ // to abort the transfer.
+ //
+ (VOID) pDmaTransaction->m_DmaAcquiredFunction.InvokeProgramDma(
+ dmaTransaction,
+ pDmaTransaction->m_DmaEnabler->m_DeviceBase->GetHandle(),
+ pDmaTransaction->m_DmaAcquiredContext,
+ pDmaTransaction->m_DmaDirection,
+ SgList);
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p",
+ dmaTransaction);
+ }
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaScatterGatherTransaction::TransferCompleted(
+ VOID
+ )
+{
+ //
+ // All we have to do is release the scatter-gather list.
+ //
+ if (m_SGList != NULL) {
+
+ PutScatterGatherList(m_SGList);
+ m_SGList = NULL;
+ }
+
+ return STATUS_SUCCESS;
+}
+
+
+// ----------------------------------------------------------------------------
+// ------------------- PACKET DMA SECTION -------------------------------------
+// ----------------------------------------------------------------------------
+
+FxDmaPacketTransaction::FxDmaPacketTransaction(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in USHORT ObjectSize,
+ __in USHORT ExtraSize,
+ __in FxDmaEnabler *DmaEnabler
+ ) :
+ FxDmaTransactionBase(FxDriverGlobals, ObjectSize, ExtraSize, DmaEnabler)
+{
+ m_MapRegistersNeeded = 0;
+ m_MapRegisterBase = NULL;
+ m_MapRegisterBaseSet = FALSE;
+ m_DeviceAddressOffset = 0;
+ m_MapRegistersReserved = 0;
+
+ m_IsCancelled = FALSE;
+
+ m_TransferState.CurrentStagingThread = NULL;
+ m_TransferState.RerunStaging = FALSE;
+ m_TransferState.RerunCompletion = FALSE;
+ m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::_Create(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in PWDF_OBJECT_ATTRIBUTES Attributes,
+ __in FxDmaEnabler* DmaEnabler,
+ __out WDFDMATRANSACTION* Transaction
+ )
+{
+ FxDmaPacketTransaction* pTransaction;
+ WDFOBJECT hTransaction;
+ NTSTATUS status;
+
+ pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
+ FxDmaPacketTransaction(FxDriverGlobals,
+ sizeof(FxDmaPacketTransaction),
+ DmaEnabler->GetTransferContextSize(),
+ DmaEnabler);
+
+ if (pTransaction == NULL) {
+ status = STATUS_INSUFFICIENT_RESOURCES;
+ DoTraceLevelMessage(
+ FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
+ return status;
+ }
+
+ //
+ // Commit and apply the attributes
+ //
+ status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
+ if (NT_SUCCESS(status)) {
+ *Transaction = (WDFDMATRANSACTION)hTransaction;
+ }
+ else {
+ //
+ // This will properly clean up the target's state and free it
+ //
+ pTransaction->DeleteFromFailedCreate();
+ }
+
+ return status;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::InitializeResources(
+ VOID
+ )
+{
+ KIRQL oldIrql;
+ m_DeviceAddressOffset = 0;
+ LockTransferState(&oldIrql);
+ m_IsCancelled = FALSE;
+ UnlockTransferState(oldIrql);
+ return STATUS_SUCCESS;
+}
+
+VOID
+FxDmaPacketTransaction::ReleaseResources(
+ __in BOOLEAN ForceRelease
+ )
+{
+ //
+ // If the map register base hasn't been assigned, then just
+ // skip this.
+ //
+
+ if (IsMapRegisterBaseSet() == FALSE) {
+ return;
+ }
+
+ //
+ // Map registers are reserved. Unless the caller is forcing
+ // us to free them, just return. Otherwise updated the
+ // number of map registers that FreeMapRegistersAndAdapter
+ // is going to look at.
+ //
+ if ((m_MapRegistersReserved > 0) && (ForceRelease == FALSE))
+ {
+ return;
+ }
+
+ //
+ // Free the map registers and release the device.
+ //
+ FreeMapRegistersAndAdapter();
+
+ ClearMapRegisterBase();
+
+ ReleaseDevice();
+
+ m_AdapterInfo = NULL;
+ m_MapRegistersReserved = 0;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::ReserveAdapter(
+ __in ULONG NumberOfMapRegisters,
+ __in WDF_DMA_DIRECTION DmaDirection,
+ __in PFN_WDF_RESERVE_DMA Callback,
+ __in_opt PVOID Context
+ )
+{
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+ WDFDMATRANSACTION dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p", dmaTransaction);
+ }
+
+ //
+ // If caller doesn't supply a map register count then we get the count
+ // out of the transaction. So the transaction must be initialized.
+ //
+ // Otherwise the transaction can't be executing.
+ //
+ if (NumberOfMapRegisters == 0) {
+ if (m_State != FxDmaTransactionStateInitialized) {
+ status = STATUS_INVALID_PARAMETER;
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "RequiredMapRegisters cannot be 0 because "
+ "WDFDMATRANSACTION %p is not initialized ("
+ "state is %!FxDmaTransactionState!) - %!STATUS!",
+ GetHandle(),
+ m_State,
+ status);
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+ }
+ else if (m_State != FxDmaTransactionStateCreated &&
+ m_State != FxDmaTransactionStateInitialized &&
+ m_State != FxDmaTransactionStateReleased) {
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", dmaTransaction, m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ //
+ // Must not already have reserved map registers
+ //
+ if (m_MapRegistersReserved != 0)
+ {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p already has allocated map registers.",
+ dmaTransaction);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ //
+ // Get the adapter
+ //
+ if (DmaDirection == WdfDmaDirectionReadFromDevice) {
+ m_AdapterInfo = m_DmaEnabler->GetReadDmaDescription();
+ } else {
+ m_AdapterInfo = m_DmaEnabler->GetWriteDmaDescription();
+ }
+
+ //
+ // Save the number of map registers being reserved.
+ //
+ if (NumberOfMapRegisters != 0) {
+
+ //
+ // Use the number the caller passed us
+ //
+ m_MapRegistersReserved = NumberOfMapRegisters;
+ }
+ else if (m_DmaEnabler->IsBusMaster() == FALSE) {
+
+ //
+ // For system DMA use all the map registers we have
+ //
+ m_MapRegistersReserved = m_AdapterInfo->NumberOfMapRegisters;
+
+ } else {
+
+ //
+ // Compute the number of map registers required based on
+ // the MDL and length passed in
+ //
+ status = _CalculateRequiredMapRegisters(
+ m_StartMdl,
+ m_StartOffset,
+ (ULONG) m_TransactionLength,
+ m_AdapterInfo->NumberOfMapRegisters,
+ NULL,
+ &m_MapRegistersReserved
+ );
+
+ if (!NT_SUCCESS(status)) {
+ ReleaseForReuse(TRUE);
+ goto End;
+ }
+ }
+
+ //
+ // Initialize the DmaTransaction object with enough data to
+ // trick StartTransfer into allocating the adapter channel for us.
+ //
+ m_DmaDirection = DmaDirection;
+ m_StartMdl = NULL;
+ m_StartOffset = 0;
+ m_CurrentFragmentMdl = NULL;
+ m_CurrentFragmentOffset = 0;
+ m_Remaining = 0;
+ m_TransactionLength = 0;
+
+ //
+ // Save the callback and context
+ //
+ m_DmaAcquiredFunction.Method.ReserveDma = Callback;
+ m_DmaAcquiredContext = Context;
+
+ //
+ // If needed, initialize the transfer context.
+ //
+ if (m_DmaEnabler->UsesDmaV3()) {
+ m_DmaEnabler->InitializeTransferContext(GetTransferContext(),
+ m_DmaDirection);
+ }
+
+ status = InitializeResources();
+ if (NT_SUCCESS(status)) {
+ //
+ // Set the state to reserved so _AdapterControl knows which
+ // callback to invoke.
+ //
+ m_State = FxDmaTransactionStateReserved;
+ } else {
+ ReleaseForReuse(TRUE);
+ goto End;
+ }
+
+ //
+ // Start the adapter channel allocation through StartTransfer
+ //
+ status = StartTransfer();
+
+End:
+ if (!NT_SUCCESS(status)) {
+ m_State = FxDmaTransactionStateTransferFailed;
+ m_DmaAcquiredFunction.Method.ReserveDma = NULL;
+ m_DmaAcquiredContext = NULL;
+ m_MapRegistersReserved = 0;
+
+ if (m_EncodedRequest != NULL) {
+ ReleaseButRetainRequest();
+ }
+ }
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p, %!STATUS!",
+ dmaTransaction, status);
+ }
+
+ return status;
+}
+
+VOID
+FxDmaPacketTransaction::ReleaseAdapter(
+ VOID
+ )
+{
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+ WDFDMATRANSACTION dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p", dmaTransaction);
+ }
+
+ //
+ // Must not be in invalid, created, transfer or deleted state
+ //
+ if (m_State == FxDmaTransactionStateInvalid ||
+ m_State == FxDmaTransactionStateCreated ||
+ m_State == FxDmaTransactionStateTransfer ||
+ m_State == FxDmaTransactionStateDeleted) {
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", dmaTransaction, m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ //
+ // The caller wants to free the reserved map registers, so force their
+ // release.
+ //
+ ReleaseForReuse(TRUE);
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p",
+ dmaTransaction);
+ }
+
+ return;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::StartTransfer(
+ VOID
+ )
+{
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+ WDFDMATRANSACTION dmaTransaction;
+
+ //
+ // Client driver could complete and delete the object in
+ // EvtProgramDmaFunction. So, save the handle because we need it
+ // for tracing after we invoke the callback.
+ //
+ dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p",
+ dmaTransaction);
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Starting WDFDMATRANSACTION %p - MDL %#p, "
+ "offset %I64x, length %I64x",
+ dmaTransaction,
+ m_StartMdl,
+ m_StartOffset,
+ m_TransactionLength);
+ }
+
+ //
+ // Reference the device when using DMA v2. For DMA v3 we can support
+ // concurrent attempts to allocate the channel.
+ //
+ status = AcquireDevice();
+ if (!NT_SUCCESS(status)) {
+
+ NT_ASSERTMSG("AcquireDevice should never fail when DMAv3 is in use",
+ m_DmaEnabler->UsesDmaV3());
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Only one transaction can be queued "
+ "at one time on a packet based WDFDMAENABLER %p "
+ "%!STATUS!", m_DmaEnabler->GetHandle(),
+ status);
+ FxVerifierDbgBreakPoint(pFxDriverGlobals);
+ return status;
+ }
+
+ //
+ // Calculate the initial DMA transfer length.
+ //
+ m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
+
+ m_CurrentFragmentOffset = m_StartOffset;
+
+ if (m_State == FxDmaTransactionStateReserved) {
+ //
+ // Caller is simply reserving the DMA adapter for later use. Ask for
+ // as many map registers as the driver requested.
+ //
+ m_MapRegistersNeeded = m_MapRegistersReserved;
+
+ ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters);
+
+ status = AllocateAdapterChannel(FALSE);
+
+ }
+ else {
+
+ if (m_DmaEnabler->IsBusMaster() == FALSE) {
+
+ //
+ // Use as many map registers as we were granted.
+ //
+ m_MapRegistersNeeded = m_AdapterInfo->NumberOfMapRegisters;
+ } else {
+
+ //
+ // If the transfer is the size of the transaction then use the offset
+ // to determine the number of map registers needed. If it's smaller
+ // then use the worst-case offset to make sure we ask for enough MR's
+ // to account for a bigger offset in one of the later transfers.
+ //
+ // Example:
+ // Transaction is 8 KB and is page aligned
+ // if max transfer is >= 8KB then this will be one transfer and only
+ // requires two map registers. Even if the driver completes a partial
+ // transfer and we have to do the rest in a second transfer it will
+ // fit within two map registers becuase the overall transaction does
+ // (and a partial transfer can't take more map registers than the
+ // whole transaction would).
+ //
+ // If max transfer is 2KB then this nominally requires 4 2KB transfers.
+ // In this case however, a partial completion of one of those transfers
+ // would leave us attempting a second 2KB transfer starting on an
+ // unaligned address. For example, we might transfer 2KB, then 1KB
+ // then 2KB. Even though the first transfer was page aligned, the
+ // 3rd transfer isn't and could cross a page boundary, requiring two
+ // map registers rather than one.
+ //
+ // To account for this second case, ignore the actual MDL offset and
+ // instead compute the maximum number of map registers than an N byte
+ // transfer could take (with worst-case alignment).
+ //
+ //
+ m_MapRegistersNeeded =
+ (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES(
+ ((m_CurrentFragmentLength == m_Remaining) ?
+ GetStartVaFromOffset(m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset) :
+ (PVOID)(ULONG_PTR) (PAGE_SIZE -1)),
+ m_CurrentFragmentLength
+ );
+
+
+ ASSERT(m_MapRegistersNeeded <= m_AdapterInfo->NumberOfMapRegisters);
+ }
+
+ //
+ // NOTE: the number of map registers needed for this transfer may
+ // exceed the number that we've reserved. StageTransfer will
+ // take care of fragmenting the transaction accordingly.
+ //
+ status = AllocateAdapterChannel(m_MapRegistersReserved > 0);
+ }
+
+ if (!NT_SUCCESS(status)) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "AllocateAdapterChannel failed for "
+ "WDFDMATRANSACTION %p, %!STATUS!",
+ dmaTransaction, status);
+ ReleaseDevice();
+ }
+
+ //
+ // Before AllocateAdapterChannel returns, _AdapterControl can get called
+ // on another thread and the driver could delete the transaction object.
+ // So don't touch the object after this point.
+ //
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p, "
+ "%!STATUS!", dmaTransaction, status);
+ }
+
+ return status;
+}
+
+IO_ALLOCATION_ACTION
+FxDmaPacketTransaction::_AdapterControl(
+ __in PDEVICE_OBJECT DeviceObject,
+ __in PIRP Irp,
+ __in PVOID MapRegisterBase,
+ __in PVOID Context
+ )
+{
+ FxDmaPacketTransaction * pDmaTransaction;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals;
+ IO_ALLOCATION_ACTION action;
+ NTSTATUS status;
+
+ UNREFERENCED_PARAMETER(Irp);
+ UNREFERENCED_PARAMETER(DeviceObject);
+
+ pDmaTransaction = (FxDmaPacketTransaction*) Context;
+ ASSERT(pDmaTransaction);
+
+ pFxDriverGlobals = pDmaTransaction->GetDriverGlobals();
+
+ //
+ // Cache the return value while we can still touch the transaction
+ //
+ action = pDmaTransaction->GetAdapterControlReturnValue();
+
+ //
+ // Save the MapRegister base, unless it was previously set
+ // during a reserve.
+ //
+ if (pDmaTransaction->IsMapRegisterBaseSet() == FALSE) {
+ pDmaTransaction->SetMapRegisterBase(MapRegisterBase);
+ }
+ else {
+ NT_ASSERTMSG("Caller was expected to use existing map register base",
+ MapRegisterBase == pDmaTransaction->m_MapRegisterBase);
+ }
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Map registers for WDFDMATRANSACTION %p allocated "
+ "(base %p)",
+ pDmaTransaction->GetHandle(),
+ MapRegisterBase);
+ }
+
+ //
+ // NOTE: KMDF used to call KeFlushIoBuffers() here to "ensure the
+ // data buffers were flushed." However KeFlushIoBuffers did
+ // nothing on x86 & amd64 (which are cache coherent WRT DMA)
+ // and calling FlushAdapterBuffers() does any necessary
+ // flushing anyway. Plus on non-cache-coherent architectures
+ // (such as ARM) the flush operation has to be cache-line aligned
+ // to avoid cache line tearing. So the flush is not necessary
+ // and has been removed.
+
+ //
+ // Check the state of the transaction. If it's reserve then call the
+ // reserve callback and return. Otherwise stage the first fragment.
+ //
+ if (pDmaTransaction->m_State == FxDmaTransactionStateReserved)
+ {
+ FxDmaTransactionProgramOrReserveDma callback;
+
+ //
+ // Save off and clear the callback before calling it.
+ //
+ callback = pDmaTransaction->m_DmaAcquiredFunction;
+ pDmaTransaction->m_DmaAcquiredFunction.Clear();
+
+ ASSERTMSG("Mismatch between map register counts",
+ (pDmaTransaction->m_MapRegistersReserved ==
+ pDmaTransaction->m_MapRegistersNeeded));
+
+ //
+ // Invoke the callback. Note that from here the driver may initialize
+ // and execute the transaction.
+ //
+ callback.InvokeReserveDma(
+ pDmaTransaction->GetHandle(),
+ pDmaTransaction->m_DmaAcquiredContext
+ );
+ }
+ else {
+
+ //
+ // Stage next fragment
+ //
+ status = pDmaTransaction->StageTransfer();
+
+ if (!NT_SUCCESS(status)) {
+
+ DMA_COMPLETION_STATUS dmaStatus =
+ (status == STATUS_CANCELLED ? DmaCancelled : DmaError);
+ FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) pDmaTransaction;
+
+ //
+ // Map transfer failed. There will be no DMA completion callback
+ // and no call to EvtProgramDma. And we have no way to hand this
+ // status back directly to the driver. Fake a DMA completion with
+ // the appropriate status.
+ //
+ // This should only happen for system DMA (and there most likely
+ // only during cancelation, though we leave the possibility that
+ // the DMA extension may fail the transfer)
+ //
+ ASSERTMSG("Unexpected failure of StageTransfer for packet based "
+ "DMA",
+ (pDmaTransaction->GetDmaEnabler()->IsBusMaster() == false));
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Invoking DmaCompleted callback %p (context %p) "
+ "for WDFDMATRANSACTION %p (status %x) "
+ "due to staging failure (%!STATUS!)",
+ systemTransaction->m_TransferCompleteFunction.Method,
+ systemTransaction->m_TransferCompleteContext,
+ pDmaTransaction->GetHandle(),
+ dmaStatus,
+ status);
+ }
+
+ pDmaTransaction->CallEvtDmaCompleted(
+ status == STATUS_CANCELLED ? DmaCancelled : DmaError
+ );
+ }
+ }
+
+ //
+ // Indicate that MapRegs are to be kept
+ //
+ return action;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::StageTransfer(
+ VOID
+ )
+{
+ PSCATTER_GATHER_LIST sgList;
+
+ PFX_DRIVER_GLOBALS pFxDriverGlobals;
+ UCHAR_MEMORY_ALIGNED sgListBuffer[sizeof(SCATTER_GATHER_LIST)
+ + sizeof(SCATTER_GATHER_ELEMENT)];
+ WDFDMATRANSACTION dmaTransaction;
+
+ KIRQL oldIrql;
+ BOOLEAN stagingNeeded;
+
+ NTSTATUS status = STATUS_SUCCESS;
+
+ //
+ // Client driver could complete and delete the object in
+ // EvtProgramDmaFunction. So, save the handle because we need it
+ // for tracing after we invoke the callback.
+ //
+ pFxDriverGlobals = GetDriverGlobals();
+ dmaTransaction = GetHandle();
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Enter WDFDMATRANSACTION %p ", dmaTransaction);
+ }
+
+ //
+ // For packet base DMA, current and startMDL will always be
+ // same. For V2 DMA we don't support MDL chains. For V3 DMA
+ // we use the HAL's support for MDL chains and don't walk through
+ // the MDL chain on our own.
+ //
+ ASSERT(m_CurrentFragmentMdl == m_StartMdl);
+
+ LockTransferState(&oldIrql);
+
+ if (m_TransferState.CurrentStagingThread != NULL) {
+
+ //
+ // Staging in progress. Indicate that another staging will
+ // be needed.
+ //
+ m_TransferState.RerunStaging = TRUE;
+
+ stagingNeeded = FALSE;
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(
+ pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Staging next fragment of WDFDMATRANSACTION %p "
+ "deferred",
+ dmaTransaction
+ );
+ }
+ }
+ else {
+ //
+ // Staging isn't in progress anyplace else. Indicate that it's
+ // running now so that any parallel attempt is blocked.
+ //
+ m_TransferState.CurrentStagingThread = KeGetCurrentThread();
+
+ ASSERTMSG("The thread which was staging didn't clear "
+ "RerunStaging",
+ (m_TransferState.RerunStaging == FALSE));
+
+ stagingNeeded = TRUE;
+ }
+
+ UnlockTransferState(oldIrql);
+
+ //
+ // Take a reference on the transaction so that we can safely
+ // manipulate the transfer state even after it's destroyed.
+ //
+ AddRef(&pFxDriverGlobals);
+
+ //
+ // Loop for as long as staging is required
+ //
+ while (stagingNeeded) {
+
+ //
+ // Calculate length for this packet.
+ //
+ m_CurrentFragmentLength = FxSizeTMin(m_Remaining, m_MaxFragmentLength);
+
+ //
+ // Calculate address for this packet.
+ //
+ m_CurrentFragmentOffset = m_StartOffset + m_Transferred;
+
+ //
+ // Adjust the fragment length for the number of reserved map registers.
+ //
+ if ((m_MapRegistersReserved > 0) &&
+ (m_MapRegistersNeeded > m_MapRegistersReserved))
+ {
+ size_t currentOffset = m_CurrentFragmentOffset;
+ size_t currentPageOffset;
+ PMDL mdl;
+
+ for (mdl = m_CurrentFragmentMdl; mdl != NULL; mdl = mdl->Next)
+ {
+ //
+ // For packet/system transfers of chained MDLs, m_CurrentFragmentMdl
+ // is never adjusted, and m_CurrentFragmentOFfset is the offset
+ // into the entire chain.
+ //
+ // Locate the MDL which contains the current fragment.
+ //
+ ULONG mdlBytes = MmGetMdlByteCount(mdl);
+ if (mdlBytes >= currentOffset)
+ {
+ //
+ // This MDL is larger than the remaining offset, so it
+ // contains the start address.
+ //
+ break;
+ }
+
+ currentOffset -= mdlBytes;
+ }
+
+ ASSERT(mdl != NULL);
+
+ //
+ // Compute page offset from current MDL's initial page offset
+ // and the offset into that MDL
+ //
+
+ currentPageOffset = BYTE_OFFSET(MmGetMdlByteOffset(mdl) +
+ currentOffset);
+
+ //
+ // Compute the maximum number of bytes we can transfer with
+ // the number of map registers we have reserved, taking into
+ // account the offset of the first page.
+ //
+ size_t l = ((PAGE_SIZE * (m_MapRegistersReserved - 1)) +
+ (PAGE_SIZE - currentPageOffset));
+
+ m_CurrentFragmentLength = FxSizeTMin(m_CurrentFragmentLength, l);
+ }
+
+ m_Remaining -= m_CurrentFragmentLength;
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Initiating %s transfer for WDFDMATRANSACTION %p. "
+ "Mdl %p, Offset %I64x, Length %I64x",
+ m_Transferred == 0 ? "first" : "next",
+ GetHandle(),
+ m_CurrentFragmentMdl,
+ m_Transferred,
+ m_CurrentFragmentLength);
+ }
+
+ //
+ // Check for a pending cancellation. This can happen if the cancel
+ // occurred between DMA completion and FlushAdapterBuffers -
+ // FlushAdapterBuffers will clear the canceled bit in the transfer
+ // context (TC), which would allow MapTransfer to succeed.
+ //
+ // An unprotected check of IsCancelled here is safe. A concurrent
+ // cancel at this point would mark the TC cancelled such that
+ // MapTransfer will fail.
+ //
+ if (m_IsCancelled == TRUE) {
+ status = STATUS_CANCELLED;
+ goto End;
+ }
+
+ //
+ // Profile specific work before mapping the transfer. if this
+ // fails consider 'this' invalid.
+ //
+ if (PreMapTransfer() == FALSE) {
+ status = STATUS_SUCCESS;
+ goto End;
+ }
+
+ //
+ // Map this packet for transfer.
+ //
+
+ //
+ // For packet based DMA we use a single entry on-stack SGL. This
+ // allows us to map multiple packet-based requests concurrently and
+ // we know packet base DMA only requires a single SGL
+ //
+ // NOTE: It turns out the HAL doesn't handle chained MDLs in packet
+ // mode correctly. It makes each MDL continguous, but returns
+ // a list of SG elements - one for each MDL. That's scatter
+ // gather DMA, not packet DMA.
+ //
+ // So it's actually very important in Win8 that we only use a
+ // single entry SGL when calling MapTransferEx. This ensures
+ // we only map the first MDL in the chain and thus get a
+ // contiguous buffer
+ //
+ // For system DMA we use the SystemSGList stored in the DMA enabler
+ // We can use a shared one in that case because we won't be mapping
+ // multiple system DMA requests concurrently (HAL doesn't allow it)
+ //
+ FxDmaEnabler* enabler = GetDmaEnabler();
+ size_t sgListSize;
+
+ if (enabler->IsBusMaster()) {
+ sgList = (PSCATTER_GATHER_LIST)sgListBuffer;
+ sgListSize = sizeof(sgListBuffer);
+ } else {
+ sgList = enabler->m_SGList.SystemProfile.List;
+ sgListSize = enabler->m_SGListSize;
+ }
+
+ ULONG mappedBytes;
+
+ status = MapTransfer(sgList,
+ (ULONG) sgListSize,
+ GetTransferCompletionRoutine(),
+ this,
+ &mappedBytes);
+
+ NT_ASSERTMSG("Unexpected failure of MapTransfer",
+ ((NT_SUCCESS(status) == TRUE) ||
+ (status == STATUS_CANCELLED)));
+
+ if (NT_SUCCESS(status)) {
+
+ NT_ASSERTMSG("unexpected number of mapped bytes",
+ ((mappedBytes > 0) &&
+ (mappedBytes <= m_CurrentFragmentLength)));
+
+ //
+ // Adjust the remaining byte count if the HAL mapped less data than we
+ // requested.
+ //
+ if (mappedBytes < m_CurrentFragmentLength) {
+ m_Remaining += m_CurrentFragmentLength - mappedBytes;
+ m_CurrentFragmentLength = mappedBytes;
+ }
+
+ //
+ // Do client PFN_WDF_PROGRAM_DMA callback.
+ //
+ if (m_DmaAcquiredFunction.Method.ProgramDma != NULL) {
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Invoking ProgramDma callback %p (context %p) for "
+ "WDFDMATRANSACTION %p.",
+ m_DmaAcquiredFunction.Method.ProgramDma,
+ m_DmaAcquiredContext,
+ GetHandle()
+ );
+ }
+
+ //
+ // Call program DMA
+ //
+ (VOID) m_DmaAcquiredFunction.InvokeProgramDma(
+ GetHandle(),
+ m_DmaEnabler->m_DeviceBase->GetHandle(),
+ m_DmaAcquiredContext,
+ m_DmaDirection,
+ sgList
+ );
+ }
+ }
+
+End:
+ //
+ // Process any pending completion or nested staging.
+ //
+ {
+ LockTransferState(&oldIrql);
+
+ //
+ // While staging we could either have deferred a call to the
+ // completion routine or deferred another call to stage the
+ // next fragment. We should not ever have to do both - this
+ // would imply that the driver didn't wait for its DMA completion
+ // routine to run when calling TransferComplete*.
+ //
+ ASSERTMSG("driver called TransferComplete with pending DMA "
+ "completion callback",
+ !((m_TransferState.RerunCompletion == TRUE) &&
+ (m_TransferState.RerunStaging == TRUE)));
+
+ //
+ // Check for pending completion. save the status away and clear it
+ // before dropping the lock.
+ //
+ if (m_TransferState.RerunCompletion == TRUE) {
+ DMA_COMPLETION_STATUS completionStatus;
+ FxDmaSystemTransaction* systemTransaction = (FxDmaSystemTransaction*) this;
+
+ //
+ // Save the completion status for when we drop the lock.
+ //
+ completionStatus = m_TransferState.CompletionStatus;
+
+ ASSERTMSG("completion needed, but status was not set or was "
+ "already cleared",
+ completionStatus != UNDEFINED_DMA_COMPLETION_STATUS);
+
+ ASSERTMSG("completion needed, but mapping failed so there shouldn't "
+ "be any parallel work going on",
+ NT_SUCCESS(status));
+
+ //
+ // Clear the completion needed state.
+ //
+ m_TransferState.RerunCompletion = FALSE;
+ m_TransferState.CompletionStatus = UNDEFINED_DMA_COMPLETION_STATUS;
+
+ //
+ // Drop the lock, call the completion routine, then take the
+ // lock again.
+ //
+ UnlockTransferState(oldIrql);
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Invoking DmaCompleted callback %p (context %p) "
+ "for WDFDMATRANSACTION %p (status %x) "
+ "after deferral",
+ systemTransaction->m_TransferCompleteFunction.Method,
+ systemTransaction->m_TransferCompleteContext,
+ GetHandle(),
+ completionStatus);
+ }
+ CallEvtDmaCompleted(completionStatus);
+ LockTransferState(&oldIrql);
+
+ //
+ // Staging is blocked, which means we aren't starting up the
+ // next transfer. Therefore we cannot have a queued completion.
+ //
+ ASSERTMSG("RerunCompletion should not be set on an unstaged "
+ "transaction",
+ m_TransferState.RerunCompletion == FALSE);
+ }
+
+ //
+ // Capture whether another staging is needed. If none is needed
+ // then we can clear staging in progress.
+ //
+ if (m_TransferState.RerunStaging == TRUE) {
+ stagingNeeded = TRUE;
+ m_TransferState.RerunStaging = FALSE;
+ }
+ else {
+ m_TransferState.CurrentStagingThread = NULL;
+ stagingNeeded = FALSE;
+ }
+
+ UnlockTransferState(oldIrql);
+ }
+
+#if DBG
+ if (!NT_SUCCESS(status)) {
+ ASSERTMSG("MapTransfer returned an error - there should not be any "
+ "deferred work.",
+ (stagingNeeded == FALSE));
+ }
+#endif
+
+ } // while(stagingNeeded)
+
+ Release(&pFxDriverGlobals);
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Exit WDFDMATRANSACTION %p, "
+ "%!STATUS!", dmaTransaction,
+ status);
+ }
+
+ return status;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaPacketTransaction::TransferCompleted(
+ VOID
+ )
+{
+ NTSTATUS status;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ //
+ // Flush the buffers
+ //
+ status = FlushAdapterBuffers();
+
+ if (!NT_SUCCESS(status)) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "FlushAdapterBuffers on WDFDMATRANSACTION %p "
+ "failed, %!STATUS!",
+ GetHandle(), status);
+ FxVerifierDbgBreakPoint(pFxDriverGlobals);
+ }
+
+ return status;
+}
+
+// ----------------------------------------------------------------------------
+// ------------------- SYSTEM DMA SECTION -------------------------------------
+// ----------------------------------------------------------------------------
+
+FxDmaSystemTransaction::FxDmaSystemTransaction(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in USHORT ExtraSize,
+ __in FxDmaEnabler *DmaEnabler
+ ) :
+ FxDmaPacketTransaction(FxDriverGlobals, sizeof(FxDmaSystemTransaction), ExtraSize, DmaEnabler)
+{
+ return;
+}
+
+_Must_inspect_result_
+NTSTATUS
+FxDmaSystemTransaction::_Create(
+ __in PFX_DRIVER_GLOBALS FxDriverGlobals,
+ __in PWDF_OBJECT_ATTRIBUTES Attributes,
+ __in FxDmaEnabler* DmaEnabler,
+ __out WDFDMATRANSACTION* Transaction
+ )
+{
+ FxDmaPacketTransaction* pTransaction;
+ WDFOBJECT hTransaction;
+ NTSTATUS status;
+
+ pTransaction = new (FxDriverGlobals, Attributes, DmaEnabler->GetTransferContextSize())
+ FxDmaSystemTransaction(FxDriverGlobals, DmaEnabler->GetTransferContextSize(), DmaEnabler);
+
+ if (pTransaction == NULL) {
+ status = STATUS_INSUFFICIENT_RESOURCES;
+ DoTraceLevelMessage(
+ FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "Could not allocate memory for WDFTRANSACTION, %!STATUS!", status);
+ return status;
+ }
+
+ //
+ // Commit and apply the attributes
+ //
+ status = pTransaction->Commit(Attributes, &hTransaction, DmaEnabler);
+ if (NT_SUCCESS(status)) {
+ *Transaction = (WDFDMATRANSACTION)hTransaction;
+ }
+ else {
+ //
+ // This will properly clean up the target's state and free it
+ //
+ pTransaction->DeleteFromFailedCreate();
+ }
+
+ return status;
+}
+
+BOOLEAN
+FxDmaSystemTransaction::PreMapTransfer(
+ VOID
+ )
+{
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+ BOOLEAN result = TRUE;
+
+ if (m_ConfigureChannelFunction.Method != NULL) {
+ //
+ // Invoke the callback. If it returns false then the driver has
+ // completed the transaction in the callback and we must abort
+ // processing.
+ //
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Invoking ConfigureChannel callback %p (context "
+ "%p) for WDFDMATRANSACTION %p.",
+ m_ConfigureChannelFunction.Method,
+ m_ConfigureChannelContext,
+ GetHandle());
+ }
+
+ result = m_ConfigureChannelFunction.Invoke(
+ GetHandle(),
+ m_DmaEnabler->m_DeviceBase->GetHandle(),
+ m_ConfigureChannelContext,
+ m_CurrentFragmentMdl,
+ m_CurrentFragmentOffset,
+ m_CurrentFragmentLength
+ );
+ }
+
+ return result;
+}
+
+
+PDMA_COMPLETION_ROUTINE
+FxDmaSystemTransaction::GetTransferCompletionRoutine(
+ VOID
+ )
+{
+ if (m_TransferCompleteFunction.Method == NULL) {
+ return NULL;
+ }
+ else {
+ return _SystemDmaCompletion;
+ }
+}
+
+VOID
+FxDmaSystemTransaction::CallEvtDmaCompleted(
+ __in DMA_COMPLETION_STATUS Status
+ )
+{
+ //
+ // Call the TransferComplete callback to indicate that the
+ // transfer was aborted.
+ //
+ m_TransferCompleteFunction.Invoke(
+ GetHandle(),
+ m_DmaEnabler->m_Device->GetHandle(),
+ m_TransferCompleteContext,
+ m_DmaDirection,
+ Status
+ );
+}
+
+VOID
+FxDmaTransactionBase::GetTransferInfo(
+ __out_opt ULONG *MapRegisterCount,
+ __out_opt ULONG *ScatterGatherElementCount
+ )
+{
+ if (m_State != FxDmaTransactionStateInitialized) {
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p state %!FxDmaTransactionState! "
+ "is invalid", GetHandle(), m_State);
+
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // specific type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+
+ DMA_TRANSFER_INFO info = {0};
+
+
+
+
+
+
+ if (m_DmaEnabler->UsesDmaV3()) {
+
+ //
+ // Ask the HAL for information about the MDL and how many resources
+ // it will require to transfer.
+ //
+ m_AdapterInfo->AdapterObject->DmaOperations->GetDmaTransferInfo(
+ m_AdapterInfo->AdapterObject,
+ m_StartMdl,
+ m_StartOffset,
+ (ULONG) this->m_TransactionLength,
+ this->m_DmaDirection == WDF_DMA_DIRECTION::WdfDmaDirectionWriteToDevice,
+ &info
+ );
+ } else {
+ size_t offset = m_StartOffset;
+ size_t length = m_TransactionLength;
+
+ //
+ // Walk through the MDL chain and make a worst-case computation of
+ // the number of scatter gather entries and map registers the
+ // transaction would require.
+ //
+ for(PMDL mdl = m_StartMdl;
+ mdl != NULL && length != 0;
+ mdl = mdl->Next) {
+
+ size_t byteCount = MmGetMdlByteCount(mdl);
+ if (byteCount <= offset) {
+ offset -= byteCount;
+ } else {
+ ULONG_PTR startVa = (ULONG_PTR) MmGetMdlVirtualAddress(mdl);
+
+ startVa += offset;
+ byteCount -= offset;
+
+ info.V1.MapRegisterCount +=
+ (ULONG) ADDRESS_AND_SIZE_TO_SPAN_PAGES(
+ startVa,
+ min(byteCount, length)
+ );
+
+ length -= min(byteCount, length);
+ }
+ }
+
+ info.V1.ScatterGatherElementCount = info.V1.MapRegisterCount;
+ }
+
+ if (ARGUMENT_PRESENT(MapRegisterCount)) {
+ *MapRegisterCount = info.V1.MapRegisterCount;
+ }
+
+ if (ARGUMENT_PRESENT(ScatterGatherElementCount)) {
+ *ScatterGatherElementCount = info.V1.ScatterGatherElementCount;
+ }
+
+ return;
+}
+
+VOID
+FxDmaSystemTransaction::_SystemDmaCompletion(
+ __in PDMA_ADAPTER /* DmaAdapter */,
+ __in PDEVICE_OBJECT /* DeviceObject */,
+ __in PVOID CompletionContext,
+ __in DMA_COMPLETION_STATUS Status
+ )
+{
+ FxDmaSystemTransaction* transaction = (FxDmaSystemTransaction*) CompletionContext;
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = transaction->GetDriverGlobals();
+ KIRQL oldIrql;
+ BOOLEAN completionDeferred;
+
+ //
+ // Lock the transfer state so that a staging or cancelling thread
+ // cannot change it.
+ //
+ transaction->LockTransferState(&oldIrql);
+
+ ASSERTMSG("Completion state was already set",
+ (transaction->m_TransferState.CompletionStatus ==
+ UNDEFINED_DMA_COMPLETION_STATUS));
+ ASSERTMSG("Deferred completion is already pending",
+ (transaction->m_TransferState.RerunCompletion == FALSE));
+
+ //
+ // If a staging is in progress then defer the completion.
+ //
+ if (transaction->m_TransferState.CurrentStagingThread != NULL) {
+ transaction->m_TransferState.CompletionStatus = Status;
+ transaction->m_TransferState.RerunCompletion = TRUE;
+ completionDeferred = TRUE;
+ }
+ else {
+ completionDeferred = FALSE;
+ }
+
+ transaction->UnlockTransferState(oldIrql);
+
+ //
+ // Process the old state.
+ //
+ if (completionDeferred == TRUE) {
+ //
+ // The staging thread has not moved past EvtProgramDma. The staging thread
+ // will detect the state change and call the completion routine.
+ //
+ // Nothing to do in this case.
+ //
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Deferring DmaCompleted callback for WDFDMATRANSACTION %p"
+ "(status %x)",
+ transaction->GetHandle(),
+ Status);
+ }
+ }
+ else {
+ //
+ // Completion occurred while the transfer was running or
+ // being cancelled. Call the completion routine.
+ //
+ // Note: a cancel when in programming state leaves the
+ // state as programming. that we're not in programming
+ // means we don't need to worry about racing with
+ // EvtProgramDma.
+ //
+
+ if (pFxDriverGlobals->FxVerifierOn) {
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGDMA,
+ "Invoking DmaCompleted callback %p (context %p) "
+ "for WDFDMATRANSACTION %p (status %x)",
+ transaction->m_TransferCompleteFunction.Method,
+ transaction->m_TransferCompleteContext,
+ transaction->GetHandle(),
+ Status);
+ }
+ transaction->CallEvtDmaCompleted(Status);
+ }
+}
+
+VOID
+FxDmaSystemTransaction::StopTransfer(
+ VOID
+ )
+{
+ //
+ // Mark the transfer cancelled so we have a record of it even if
+ // a racing call to FlushAdapterBuffers clears the TC.
+ //
+ m_IsCancelled = TRUE;
+
+ //
+ // Cancel the system DMA transfer. This arranges for one of two things
+ // to happen:
+ // * the next call to MapTransfer will fail
+ // * the DMA completion routine will run
+ //
+ if (CancelMappedTransfer() == FALSE) {
+
+ //
+ // The cancel failed. Someone has already stopped this transfer.
+ // That's illegal.
+ //
+ PFX_DRIVER_GLOBALS pFxDriverGlobals = GetDriverGlobals();
+
+ DoTraceLevelMessage(pFxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDMA,
+ "WDFDMATRANSACTION %p has already been stopped",
+ GetHandle());
+
+ if (pFxDriverGlobals->IsVerificationEnabled(1, 11, OkForDownLevel)) {
+ FxVerifierBugCheck(pFxDriverGlobals, // globals
+ WDF_DMA_FATAL_ERROR, // type
+ (ULONG_PTR) GetObjectHandle(), // parm 2
+ (ULONG_PTR) m_State); // parm 3
+ }
+ }
+}
projects / reactos.git / blobdiff