3 * COPYRIGHT: See COPYING in the top level directory
4 * PROJECT: ReactOS kernel
5 * FILE: ntoskrnl/hal/x86/dma.c
6 * PURPOSE: DMA functions
7 * PROGRAMMERS: David Welch (welch@mcmail.com)
8 * Filip Navara (navaraf@reactos.com)
14 * @page DMA Implementation Notes
20 * Abstract encapsulation of physically contiguous buffer that resides
21 * in memory accessible by both the DMA device / controller and the system.
22 * The map registers are allocated and distributed on demand and are
25 * The actual use of map registers is to allow transfers from/to buffer
26 * located in physical memory at address inaccessible by the DMA device /
27 * controller directly. For such transfers the map register buffers
28 * are used as intermediate data storage.
32 * A container for map registers (typically corresponding to one physical
33 * bus connection type). There can be master adapters for 24-bit address
34 * ranges, 32-bit address ranges, etc. Every time a new DMA adapter is
35 * created it's associated with a corresponding master adapter that
36 * is used for any map register allocation requests.
38 * - Bus-master / Slave DMA
40 * Slave DMA is term used for DMA transfers done by the system (E)ISA
41 * controller as opposed to transfers mastered by the device itself
44 * For slave DMA special care is taken to actually access the system
45 * controller and handle the transfers. The relevant code is in
46 * HalpDmaInitializeEisaAdapter, HalReadDmaCounter, IoFlushAdapterBuffers
51 * - Allocation of map registers
53 * Initial set of map registers is allocated on the system start to
54 * ensure that low memory won't get filled up later. Additional map
55 * registers are allocated as needed by HalpGrowMapBuffers. This
56 * routine is called on two places:
58 * - HalGetAdapter, since we're at PASSIVE_LEVEL and it's known that
59 * more map registers will probably be needed.
60 * - IoAllocateAdapterChannel (indirectly using HalpGrowMapBufferWorker
61 * since we're at DISPATCH_LEVEL and call HalpGrowMapBuffers directly)
62 * when no more map registers are free.
64 * Note that even if no more map registers can be allocated it's not
65 * the end of the world. The adapters waiting for free map registers
66 * are queued in the master adapter's queue and once one driver hands
67 * back it's map registers (using IoFreeMapRegisters or indirectly using
68 * the execution routine callback in IoAllocateAdapterChannel) the
69 * queue gets processed and the map registers are reassigned.
72 /* INCLUDES *****************************************************************/
78 static KEVENT HalpDmaLock
;
79 static LIST_ENTRY HalpDmaAdapterList
;
80 static PADAPTER_OBJECT HalpEisaAdapter
[8];
81 static BOOLEAN HalpEisaDma
;
82 static PADAPTER_OBJECT HalpMasterAdapter
;
84 static const ULONG_PTR HalpEisaPortPage
[8] = {
85 FIELD_OFFSET(DMA_PAGE
, Channel0
),
86 FIELD_OFFSET(DMA_PAGE
, Channel1
),
87 FIELD_OFFSET(DMA_PAGE
, Channel2
),
88 FIELD_OFFSET(DMA_PAGE
, Channel3
),
90 FIELD_OFFSET(DMA_PAGE
, Channel5
),
91 FIELD_OFFSET(DMA_PAGE
, Channel6
),
92 FIELD_OFFSET(DMA_PAGE
, Channel7
)
95 static DMA_OPERATIONS HalpDmaOperations
= {
96 sizeof(DMA_OPERATIONS
),
97 (PPUT_DMA_ADAPTER
)HalPutDmaAdapter
,
98 (PALLOCATE_COMMON_BUFFER
)HalAllocateCommonBuffer
,
99 (PFREE_COMMON_BUFFER
)HalFreeCommonBuffer
,
100 NULL
, /* Initialized in HalpInitDma() */
101 NULL
, /* Initialized in HalpInitDma() */
102 NULL
, /* Initialized in HalpInitDma() */
103 NULL
, /* Initialized in HalpInitDma() */
104 NULL
, /* Initialized in HalpInitDma() */
105 (PGET_DMA_ALIGNMENT
)HalpDmaGetDmaAlignment
,
106 (PREAD_DMA_COUNTER
)HalReadDmaCounter
,
107 /* FIXME: Implement the S/G funtions. */
108 NULL
/*(PGET_SCATTER_GATHER_LIST)HalGetScatterGatherList*/,
109 NULL
/*(PPUT_SCATTER_GATHER_LIST)HalPutScatterGatherList*/,
110 NULL
/*(PCALCULATE_SCATTER_GATHER_LIST_SIZE)HalCalculateScatterGatherListSize*/,
111 NULL
/*(PBUILD_SCATTER_GATHER_LIST)HalBuildScatterGatherList*/,
112 NULL
/*(PBUILD_MDL_FROM_SCATTER_GATHER_LIST)HalBuildMdlFromScatterGatherList*/
115 #define MAX_MAP_REGISTERS 64
117 #define TAG_DMA ' AMD'
119 /* FUNCTIONS *****************************************************************/
125 * Initialize the DMA Operation table
127 HalpDmaOperations
.AllocateAdapterChannel
= (PALLOCATE_ADAPTER_CHANNEL
)IoAllocateAdapterChannel
;
128 HalpDmaOperations
.FlushAdapterBuffers
= (PFLUSH_ADAPTER_BUFFERS
)IoFlushAdapterBuffers
;
129 HalpDmaOperations
.FreeAdapterChannel
= (PFREE_ADAPTER_CHANNEL
)IoFreeAdapterChannel
;
130 HalpDmaOperations
.FreeMapRegisters
= (PFREE_MAP_REGISTERS
)IoFreeMapRegisters
;
131 HalpDmaOperations
.MapTransfer
= (PMAP_TRANSFER
)IoMapTransfer
;
134 * Check if Extended DMA is available. We're just going to do a random
138 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
.Channel2
), 0x2A);
139 if (READ_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
.Channel2
)) == 0x2A)
143 * Intialize all the global variables and allocate master adapter with
147 InitializeListHead(&HalpDmaAdapterList
);
148 KeInitializeEvent(&HalpDmaLock
, NotificationEvent
, TRUE
);
150 HalpMasterAdapter
= HalpDmaAllocateMasterAdapter();
153 * Setup the HalDispatchTable callback for creating PnP DMA adapters. It's
154 * used by IoGetDmaAdapter in the kernel.
157 HalGetDmaAdapter
= HalpGetDmaAdapter
;
161 * @name HalpGetAdapterMaximumPhysicalAddress
163 * Get the maximum physical address acceptable by the device represented
164 * by the passed DMA adapter.
167 PHYSICAL_ADDRESS NTAPI
168 HalpGetAdapterMaximumPhysicalAddress(
169 IN PADAPTER_OBJECT AdapterObject
)
171 PHYSICAL_ADDRESS HighestAddress
;
173 if (AdapterObject
->MasterDevice
)
175 if (AdapterObject
->Dma64BitAddresses
)
177 HighestAddress
.QuadPart
= 0xFFFFFFFFFFFFFFFFULL
;
178 return HighestAddress
;
180 else if (AdapterObject
->Dma32BitAddresses
)
182 HighestAddress
.QuadPart
= 0xFFFFFFFF;
183 return HighestAddress
;
187 HighestAddress
.QuadPart
= 0xFFFFFF;
188 return HighestAddress
;
192 * @name HalpGrowMapBuffers
194 * Allocate initial, or additional, map buffers for DMA master adapter.
196 * @param MasterAdapter
197 * DMA master adapter to allocate buffers for.
198 * @param SizeOfMapBuffers
199 * Size of the map buffers to allocate (not including the size
200 * already allocated).
205 IN PADAPTER_OBJECT AdapterObject
,
206 IN ULONG SizeOfMapBuffers
)
208 PVOID VirtualAddress
;
209 PHYSICAL_ADDRESS PhysicalAddress
;
210 PHYSICAL_ADDRESS HighestAcceptableAddress
;
211 PHYSICAL_ADDRESS LowestAcceptableAddress
;
212 PHYSICAL_ADDRESS BoundryAddressMultiple
;
214 ULONG MapRegisterCount
;
216 /* FIXME: Check if enough map register slots are available. */
218 MapRegisterCount
= BYTES_TO_PAGES(SizeOfMapBuffers
);
221 * Allocate memory for the new map registers. For 32-bit adapters we use
222 * two passes in order not to waste scare resource (low memory).
225 HighestAcceptableAddress
=
226 HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
227 LowestAcceptableAddress
.HighPart
= 0;
228 LowestAcceptableAddress
.LowPart
=
229 HighestAcceptableAddress
.LowPart
== 0xFFFFFFFF ? 0x1000000 : 0;
230 BoundryAddressMultiple
.QuadPart
= 0;
232 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(
233 MapRegisterCount
<< PAGE_SHIFT
, LowestAcceptableAddress
,
234 HighestAcceptableAddress
, BoundryAddressMultiple
, MmNonCached
);
236 if (VirtualAddress
== NULL
&& LowestAcceptableAddress
.LowPart
!= 0)
238 LowestAcceptableAddress
.LowPart
= 0;
239 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(
240 MapRegisterCount
<< PAGE_SHIFT
, LowestAcceptableAddress
,
241 HighestAcceptableAddress
, BoundryAddressMultiple
, MmNonCached
);
244 if (VirtualAddress
== NULL
)
247 PhysicalAddress
= MmGetPhysicalAddress(VirtualAddress
);
250 * All the following must be done with the master adapter lock held
251 * to prevent corruption.
254 KeAcquireSpinLock(&AdapterObject
->SpinLock
, &OldIrql
);
257 * Setup map register entries for the buffer allocated. Each entry has
258 * a virtual and physical address and corresponds to PAGE_SIZE large
262 if (MapRegisterCount
> 0)
264 PROS_MAP_REGISTER_ENTRY CurrentEntry
, PreviousEntry
;
266 CurrentEntry
= AdapterObject
->MapRegisterBase
+
267 AdapterObject
->NumberOfMapRegisters
;
271 * Leave one entry free for every non-contiguous memory region
272 * in the map register bitmap. This ensures that we can search
273 * using RtlFindClearBits for contiguous map register regions.
275 * Also for non-EISA DMA leave one free entry for every 64Kb
276 * break, because the DMA controller can handle only coniguous
280 if (CurrentEntry
!= AdapterObject
->MapRegisterBase
)
282 PreviousEntry
= CurrentEntry
- 1;
283 if (PreviousEntry
->PhysicalAddress
.LowPart
+ PAGE_SIZE
==
284 PhysicalAddress
.LowPart
)
288 if ((PreviousEntry
->PhysicalAddress
.LowPart
^
289 PhysicalAddress
.LowPart
) & 0xFFFF0000)
292 AdapterObject
->NumberOfMapRegisters
++;
299 AdapterObject
->NumberOfMapRegisters
++;
303 RtlClearBit(AdapterObject
->MapRegisters
,
304 CurrentEntry
- AdapterObject
->MapRegisterBase
);
305 CurrentEntry
->VirtualAddress
= VirtualAddress
;
306 CurrentEntry
->PhysicalAddress
= PhysicalAddress
;
308 PhysicalAddress
.LowPart
+= PAGE_SIZE
;
309 VirtualAddress
= (PVOID
)((ULONG_PTR
)VirtualAddress
+ PAGE_SIZE
);
312 AdapterObject
->NumberOfMapRegisters
++;
315 while (MapRegisterCount
!= 0);
318 KeReleaseSpinLock(&AdapterObject
->SpinLock
, OldIrql
);
324 * @name HalpDmaAllocateMasterAdapter
326 * Helper routine to allocate and initialize master adapter object and it's
327 * associated map register buffers.
332 PADAPTER_OBJECT NTAPI
333 HalpDmaAllocateMasterAdapter(VOID
)
335 PADAPTER_OBJECT MasterAdapter
;
336 ULONG Size
, SizeOfBitmap
;
338 SizeOfBitmap
= MAX_MAP_REGISTERS
;
339 Size
= sizeof(ADAPTER_OBJECT
);
340 Size
+= sizeof(RTL_BITMAP
);
341 Size
+= (SizeOfBitmap
+ 7) >> 3;
343 MasterAdapter
= ExAllocatePoolWithTag(NonPagedPool
, Size
, TAG_DMA
);
344 if (MasterAdapter
== NULL
)
347 RtlZeroMemory(MasterAdapter
, Size
);
349 KeInitializeSpinLock(&MasterAdapter
->SpinLock
);
350 InitializeListHead(&MasterAdapter
->AdapterQueue
);
352 MasterAdapter
->MapRegisters
= (PVOID
)(MasterAdapter
+ 1);
354 MasterAdapter
->MapRegisters
,
355 (PULONG
)(MasterAdapter
->MapRegisters
+ 1),
357 RtlSetAllBits(MasterAdapter
->MapRegisters
);
358 MasterAdapter
->NumberOfMapRegisters
= 0;
359 MasterAdapter
->CommittedMapRegisters
= 0;
361 MasterAdapter
->MapRegisterBase
= ExAllocatePoolWithTag(
363 SizeOfBitmap
* sizeof(ROS_MAP_REGISTER_ENTRY
),
365 if (MasterAdapter
->MapRegisterBase
== NULL
)
367 ExFreePool(MasterAdapter
);
371 RtlZeroMemory(MasterAdapter
->MapRegisterBase
,
372 SizeOfBitmap
* sizeof(ROS_MAP_REGISTER_ENTRY
));
373 if (!HalpGrowMapBuffers(MasterAdapter
, 0x10000))
375 ExFreePool(MasterAdapter
);
379 return MasterAdapter
;
383 * @name HalpDmaAllocateChildAdapter
385 * Helper routine of HalGetAdapter. Allocate child adapter object and
386 * fill out some basic fields.
391 PADAPTER_OBJECT NTAPI
392 HalpDmaAllocateChildAdapter(
393 ULONG NumberOfMapRegisters
,
394 PDEVICE_DESCRIPTION DeviceDescription
)
396 PADAPTER_OBJECT AdapterObject
;
397 OBJECT_ATTRIBUTES ObjectAttributes
;
401 InitializeObjectAttributes(
404 OBJ_KERNEL_HANDLE
| OBJ_PERMANENT
,
408 Status
= ObCreateObject(
414 sizeof(ADAPTER_OBJECT
),
417 (PVOID
)&AdapterObject
);
418 if (!NT_SUCCESS(Status
))
421 Status
= ObReferenceObjectByPointer(
423 FILE_READ_DATA
| FILE_WRITE_DATA
,
426 if (!NT_SUCCESS(Status
))
429 RtlZeroMemory(AdapterObject
, sizeof(ADAPTER_OBJECT
));
431 Status
= ObInsertObject(
434 FILE_READ_DATA
| FILE_WRITE_DATA
,
438 if (!NT_SUCCESS(Status
))
443 AdapterObject
->DmaHeader
.Version
= (USHORT
)DeviceDescription
->Version
;
444 AdapterObject
->DmaHeader
.Size
= sizeof(ADAPTER_OBJECT
);
445 AdapterObject
->DmaHeader
.DmaOperations
= &HalpDmaOperations
;
446 AdapterObject
->MapRegistersPerChannel
= 1;
447 AdapterObject
->Dma32BitAddresses
= DeviceDescription
->Dma32BitAddresses
;
448 AdapterObject
->ChannelNumber
= 0xFF;
449 AdapterObject
->MasterAdapter
= HalpMasterAdapter
;
450 KeInitializeDeviceQueue(&AdapterObject
->ChannelWaitQueue
);
452 return AdapterObject
;
456 * @name HalpDmaInitializeEisaAdapter
458 * Setup DMA modes and extended modes for (E)ISA DMA adapter object.
462 HalpDmaInitializeEisaAdapter(
463 PADAPTER_OBJECT AdapterObject
,
464 PDEVICE_DESCRIPTION DeviceDescription
)
467 DMA_MODE DmaMode
= {{0 }};
468 DMA_EXTENDED_MODE ExtendedMode
= {{ 0 }};
471 Controller
= (DeviceDescription
->DmaChannel
& 4) ? 2 : 1;
474 AdapterBaseVa
= (PVOID
)FIELD_OFFSET(EISA_CONTROL
, DmaController1
);
476 AdapterBaseVa
= (PVOID
)FIELD_OFFSET(EISA_CONTROL
, DmaController2
);
478 AdapterObject
->AdapterNumber
= Controller
;
479 AdapterObject
->ChannelNumber
= (UCHAR
)(DeviceDescription
->DmaChannel
& 3);
480 AdapterObject
->PagePort
= (PUCHAR
)HalpEisaPortPage
[DeviceDescription
->DmaChannel
];
481 AdapterObject
->Width16Bits
= FALSE
;
482 AdapterObject
->AdapterBaseVa
= AdapterBaseVa
;
486 ExtendedMode
.ChannelNumber
= AdapterObject
->ChannelNumber
;
488 switch (DeviceDescription
->DmaSpeed
)
490 case Compatible
: ExtendedMode
.TimingMode
= COMPATIBLE_TIMING
; break;
491 case TypeA
: ExtendedMode
.TimingMode
= TYPE_A_TIMING
; break;
492 case TypeB
: ExtendedMode
.TimingMode
= TYPE_B_TIMING
; break;
493 case TypeC
: ExtendedMode
.TimingMode
= BURST_TIMING
; break;
498 switch (DeviceDescription
->DmaWidth
)
500 case Width8Bits
: ExtendedMode
.TransferSize
= B_8BITS
; break;
501 case Width16Bits
: ExtendedMode
.TransferSize
= B_16BITS
; break;
502 case Width32Bits
: ExtendedMode
.TransferSize
= B_32BITS
; break;
508 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaExtendedMode1
),
511 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaExtendedMode2
),
517 * Validate setup for non-busmaster DMA adapter. Secondary controller
518 * supports only 16-bit transfers and main controller supports only
519 * 8-bit transfers. Anything else is invalid.
522 if (!DeviceDescription
->Master
)
524 if (Controller
== 2 && DeviceDescription
->DmaWidth
== Width16Bits
)
525 AdapterObject
->Width16Bits
= TRUE
;
526 else if (Controller
!= 1 || DeviceDescription
->DmaWidth
!= Width8Bits
)
531 DmaMode
.Channel
= AdapterObject
->ChannelNumber
;
532 DmaMode
.AutoInitialize
= DeviceDescription
->AutoInitialize
;
535 * Set the DMA request mode.
537 * For (E)ISA bus master devices just unmask (enable) the DMA channel
538 * and set it to cascade mode. Otherwise just select the right one
539 * bases on the passed device description.
542 if (DeviceDescription
->Master
)
544 DmaMode
.RequestMode
= CASCADE_REQUEST_MODE
;
547 /* Set the Request Data */
548 WRITE_PORT_UCHAR(&((PDMA1_CONTROL
)AdapterBaseVa
)->Mode
,
550 /* Unmask DMA Channel */
551 WRITE_PORT_UCHAR(&((PDMA1_CONTROL
)AdapterBaseVa
)->SingleMask
,
552 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
554 /* Set the Request Data */
555 WRITE_PORT_UCHAR(&((PDMA2_CONTROL
)AdapterBaseVa
)->Mode
,
557 /* Unmask DMA Channel */
558 WRITE_PORT_UCHAR(&((PDMA2_CONTROL
)AdapterBaseVa
)->SingleMask
,
559 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
564 if (DeviceDescription
->DemandMode
)
565 DmaMode
.RequestMode
= DEMAND_REQUEST_MODE
;
567 DmaMode
.RequestMode
= SINGLE_REQUEST_MODE
;
570 AdapterObject
->AdapterMode
= DmaMode
;
576 * @name HalGetAdapter
578 * Allocate an adapter object for DMA device.
580 * @param DeviceDescription
581 * Structure describing the attributes of the device.
582 * @param NumberOfMapRegisters
583 * On return filled with the maximum number of map registers the
584 * device driver can allocate for DMA transfer operations.
586 * @return The DMA adapter on success, NULL otherwise.
591 PADAPTER_OBJECT NTAPI
593 PDEVICE_DESCRIPTION DeviceDescription
,
594 PULONG NumberOfMapRegisters
)
596 PADAPTER_OBJECT AdapterObject
= NULL
;
597 PADAPTER_OBJECT MasterAdapter
;
602 /* Validate parameters in device description */
603 if (DeviceDescription
->Version
> DEVICE_DESCRIPTION_VERSION2
)
607 * See if we're going to use ISA/EISA DMA adapter. These adapters are
608 * special since they're reused.
610 * Also note that we check for channel number since there are only 8 DMA
611 * channels on ISA, so any request above this requires new adapter.
614 if (DeviceDescription
->InterfaceType
== Isa
|| !DeviceDescription
->Master
)
616 if (DeviceDescription
->InterfaceType
== Isa
&&
617 DeviceDescription
->DmaChannel
>= 8)
628 * Disallow creating adapter for ISA/EISA DMA channel 4 since it's used
629 * for cascading the controllers and it's not available for software use.
632 if (EisaAdapter
&& DeviceDescription
->DmaChannel
== 4)
636 * Calculate the number of map registers.
638 * - For EISA and PCI scatter/gather no map registers are needed.
639 * - For ISA slave scatter/gather one map register is needed.
640 * - For all other cases the number of map registers depends on
641 * DeviceDescription->MaximumLength.
644 MaximumLength
= DeviceDescription
->MaximumLength
& MAXLONG
;
645 if (DeviceDescription
->ScatterGather
&&
646 (DeviceDescription
->InterfaceType
== Eisa
||
647 DeviceDescription
->InterfaceType
== PCIBus
))
651 else if (DeviceDescription
->ScatterGather
&&
652 !DeviceDescription
->Master
)
659 * In the equation below the additional map register added by
660 * the "+1" accounts for the case when a transfer does not start
661 * at a page-aligned address.
663 MapRegisters
= BYTES_TO_PAGES(MaximumLength
) + 1;
664 if (MapRegisters
> 16)
669 * Acquire the DMA lock that is used to protect adapter lists and
670 * EISA adapter array.
673 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
,
677 * Now we must get ahold of the adapter object. For first eight ISA/EISA
678 * channels there are static adapter objects that are reused and updated
679 * on succesive HalGetAdapter calls. In other cases a new adapter object
680 * is always created and it's to the DMA adapter list (HalpDmaAdapterList).
685 AdapterObject
= HalpEisaAdapter
[DeviceDescription
->DmaChannel
];
686 if (AdapterObject
!= NULL
)
688 if (AdapterObject
->NeedsMapRegisters
&&
689 MapRegisters
> AdapterObject
->MapRegistersPerChannel
)
690 AdapterObject
->MapRegistersPerChannel
= MapRegisters
;
694 if (AdapterObject
== NULL
)
696 AdapterObject
= HalpDmaAllocateChildAdapter(
697 MapRegisters
, DeviceDescription
);
698 if (AdapterObject
== NULL
)
700 KeSetEvent(&HalpDmaLock
, 0, 0);
706 HalpEisaAdapter
[DeviceDescription
->DmaChannel
] = AdapterObject
;
709 if (MapRegisters
> 0)
711 AdapterObject
->NeedsMapRegisters
= TRUE
;
712 MasterAdapter
= HalpMasterAdapter
;
713 AdapterObject
->MapRegistersPerChannel
= MapRegisters
;
716 * FIXME: Verify that the following makes sense. Actually
717 * MasterAdapter->NumberOfMapRegisters contains even the number
718 * of gaps, so this will not work correctly all the time. It
719 * doesn't matter much since it's only optimization to avoid
720 * queuing work items in HalAllocateAdapterChannel.
723 MasterAdapter
->CommittedMapRegisters
+= MapRegisters
;
724 if (MasterAdapter
->CommittedMapRegisters
> MasterAdapter
->NumberOfMapRegisters
)
725 HalpGrowMapBuffers(MasterAdapter
, 0x10000);
729 AdapterObject
->NeedsMapRegisters
= FALSE
;
730 if (DeviceDescription
->Master
)
731 AdapterObject
->MapRegistersPerChannel
= BYTES_TO_PAGES(MaximumLength
) + 1;
733 AdapterObject
->MapRegistersPerChannel
= 1;
738 InsertTailList(&HalpDmaAdapterList
, &AdapterObject
->AdapterList
);
741 * Release the DMA lock. HalpDmaAdapterList and HalpEisaAdapter will
742 * no longer be touched, so we don't need it.
745 KeSetEvent(&HalpDmaLock
, 0, 0);
748 * Setup the values in the adapter object that are common for all
752 if (DeviceDescription
->Version
>= DEVICE_DESCRIPTION_VERSION1
)
753 AdapterObject
->IgnoreCount
= DeviceDescription
->IgnoreCount
;
755 AdapterObject
->IgnoreCount
= 0;
757 AdapterObject
->Dma32BitAddresses
= DeviceDescription
->Dma32BitAddresses
;
758 AdapterObject
->Dma64BitAddresses
= DeviceDescription
->Dma64BitAddresses
;
759 AdapterObject
->ScatterGather
= DeviceDescription
->ScatterGather
;
760 AdapterObject
->MasterDevice
= DeviceDescription
->Master
;
761 *NumberOfMapRegisters
= AdapterObject
->MapRegistersPerChannel
;
764 * For non-(E)ISA adapters we have already done all the work. On the
765 * other hand for (E)ISA adapters we must still setup the DMA modes
766 * and prepare the controller.
771 if (!HalpDmaInitializeEisaAdapter(AdapterObject
, DeviceDescription
))
773 ObDereferenceObject(AdapterObject
);
778 return AdapterObject
;
782 * @name HalpGetDmaAdapter
784 * Internal routine to allocate PnP DMA adapter object. It's exported through
785 * HalDispatchTable and used by IoGetDmaAdapter.
793 IN PDEVICE_DESCRIPTION DeviceDescription
,
794 OUT PULONG NumberOfMapRegisters
)
796 return &HalGetAdapter(DeviceDescription
, NumberOfMapRegisters
)->DmaHeader
;
800 * @name HalPutDmaAdapter
802 * Internal routine to free DMA adapter and resources for reuse. It's exported
803 * using the DMA_OPERATIONS interface by HalGetAdapter.
810 PADAPTER_OBJECT AdapterObject
)
812 if (AdapterObject
->ChannelNumber
== 0xFF)
814 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
,
816 RemoveEntryList(&AdapterObject
->AdapterList
);
817 KeSetEvent(&HalpDmaLock
, 0, 0);
820 ObDereferenceObject(AdapterObject
);
824 * @name HalAllocateCommonBuffer
826 * Allocates memory that is visible to both the processor(s) and the DMA
829 * @param AdapterObject
830 * Adapter object representing the bus master or system dma controller.
832 * Number of bytes to allocate.
833 * @param LogicalAddress
834 * Logical address the driver can use to access the buffer.
835 * @param CacheEnabled
836 * Specifies if the memory can be cached.
838 * @return The base virtual address of the memory allocated or NULL on failure.
841 * On real NT x86 systems the CacheEnabled parameter is ignored, we honour
842 * it. If it proves to cause problems change it.
844 * @see HalFreeCommonBuffer
850 HalAllocateCommonBuffer(
851 PADAPTER_OBJECT AdapterObject
,
853 PPHYSICAL_ADDRESS LogicalAddress
,
854 BOOLEAN CacheEnabled
)
856 PHYSICAL_ADDRESS LowestAcceptableAddress
;
857 PHYSICAL_ADDRESS HighestAcceptableAddress
;
858 PHYSICAL_ADDRESS BoundryAddressMultiple
;
859 PVOID VirtualAddress
;
861 LowestAcceptableAddress
.QuadPart
= 0;
862 HighestAcceptableAddress
=
863 HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
864 BoundryAddressMultiple
.QuadPart
= 0;
867 * For bus-master DMA devices the buffer mustn't cross 4Gb boundary. For
868 * slave DMA devices the 64Kb boundary mustn't be crossed since the
869 * controller wouldn't be able to handle it.
872 if (AdapterObject
->MasterDevice
)
873 BoundryAddressMultiple
.HighPart
= 1;
875 BoundryAddressMultiple
.LowPart
= 0x10000;
877 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(
878 Length
, LowestAcceptableAddress
, HighestAcceptableAddress
,
879 BoundryAddressMultiple
, CacheEnabled
? MmCached
: MmNonCached
);
880 if (VirtualAddress
== NULL
)
883 *LogicalAddress
= MmGetPhysicalAddress(VirtualAddress
);
885 return VirtualAddress
;
889 * @name HalFreeCommonBuffer
891 * Free common buffer allocated with HalAllocateCommonBuffer.
893 * @see HalAllocateCommonBuffer
900 PADAPTER_OBJECT AdapterObject
,
902 PHYSICAL_ADDRESS LogicalAddress
,
903 PVOID VirtualAddress
,
904 BOOLEAN CacheEnabled
)
906 MmFreeContiguousMemorySpecifyCache(VirtualAddress
,
908 CacheEnabled
? MmCached
: MmNonCached
);
912 * @name HalpDmaGetDmaAlignment
914 * Internal routine to return the DMA alignment requirement. It's exported
915 * using the DMA_OPERATIONS interface by HalGetAdapter.
921 HalpDmaGetDmaAlignment(
922 PADAPTER_OBJECT AdapterObject
)
928 * @name HalReadDmaCounter
930 * Read DMA operation progress counter.
937 PADAPTER_OBJECT AdapterObject
)
940 ULONG Count
, OldCount
;
942 ASSERT(!AdapterObject
->MasterDevice
);
945 * Acquire the master adapter lock since we're going to mess with the
946 * system DMA controller registers and we really don't want anyone
947 * to do the same at the same time.
950 KeAcquireSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, &OldIrql
);
952 /* Send the request to the specific controller. */
953 if (AdapterObject
->AdapterNumber
== 1)
955 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
962 WRITE_PORT_UCHAR(&DmaControl1
->ClearBytePointer
, 0);
964 Count
= READ_PORT_UCHAR(&DmaControl1
->DmaAddressCount
965 [AdapterObject
->ChannelNumber
].DmaBaseCount
);
966 Count
|= READ_PORT_UCHAR(&DmaControl1
->DmaAddressCount
967 [AdapterObject
->ChannelNumber
].DmaBaseCount
) << 8;
969 while (0xffff00 & (OldCount
^ Count
));
973 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
980 WRITE_PORT_UCHAR(&DmaControl2
->ClearBytePointer
, 0);
982 Count
= READ_PORT_UCHAR(&DmaControl2
->DmaAddressCount
983 [AdapterObject
->ChannelNumber
].DmaBaseCount
);
984 Count
|= READ_PORT_UCHAR(&DmaControl2
->DmaAddressCount
985 [AdapterObject
->ChannelNumber
].DmaBaseCount
) << 8;
987 while (0xffff00 & (OldCount
^ Count
));
990 KeReleaseSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, OldIrql
);
994 if (AdapterObject
->Width16Bits
)
1001 * @name HalpGrowMapBufferWorker
1003 * Helper routine of HalAllocateAdapterChannel for allocating map registers
1004 * at PASSIVE_LEVEL in work item.
1008 HalpGrowMapBufferWorker(PVOID DeferredContext
)
1010 PGROW_WORK_ITEM WorkItem
= (PGROW_WORK_ITEM
)DeferredContext
;
1015 * Try to allocate new map registers for the adapter.
1017 * NOTE: The NT implementation actually tries to allocate more map
1018 * registers than needed as an optimization.
1021 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
,
1023 Succeeded
= HalpGrowMapBuffers(WorkItem
->AdapterObject
->MasterAdapter
,
1024 WorkItem
->NumberOfMapRegisters
);
1025 KeSetEvent(&HalpDmaLock
, 0, 0);
1030 * Flush the adapter queue now that new map registers are ready. The
1031 * easiest way to do that is to call IoFreeMapRegisters to not free
1032 * any registers. Note that we use the magic (PVOID)2 map register
1033 * base to bypass the parameter checking.
1036 OldIrql
= KfRaiseIrql(DISPATCH_LEVEL
);
1037 IoFreeMapRegisters(WorkItem
->AdapterObject
, (PVOID
)2, 0);
1038 KfLowerIrql(OldIrql
);
1041 ExFreePool(WorkItem
);
1045 * @name HalAllocateAdapterChannel
1047 * Setup map registers for an adapter object.
1049 * @param AdapterObject
1050 * Pointer to an ADAPTER_OBJECT to set up.
1051 * @param WaitContextBlock
1052 * Context block to be used with ExecutionRoutine.
1053 * @param NumberOfMapRegisters
1054 * Number of map registers requested.
1055 * @param ExecutionRoutine
1056 * Callback to call when map registers are allocated.
1059 * If not enough map registers can be allocated then
1060 * STATUS_INSUFFICIENT_RESOURCES is returned. If the function
1061 * succeeds or the callback is queued for later delivering then
1062 * STATUS_SUCCESS is returned.
1064 * @see IoFreeAdapterChannel
1070 HalAllocateAdapterChannel(
1071 PADAPTER_OBJECT AdapterObject
,
1072 PWAIT_CONTEXT_BLOCK WaitContextBlock
,
1073 ULONG NumberOfMapRegisters
,
1074 PDRIVER_CONTROL ExecutionRoutine
)
1076 PADAPTER_OBJECT MasterAdapter
;
1077 PGROW_WORK_ITEM WorkItem
;
1078 ULONG Index
= MAXULONG
;
1082 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1084 /* Set up the wait context block in case we can't run right away. */
1085 WaitContextBlock
->DeviceRoutine
= ExecutionRoutine
;
1086 WaitContextBlock
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1088 /* Returns true if queued, else returns false and sets the queue to busy */
1089 if (KeInsertDeviceQueue(&AdapterObject
->ChannelWaitQueue
, &WaitContextBlock
->WaitQueueEntry
))
1090 return STATUS_SUCCESS
;
1092 MasterAdapter
= AdapterObject
->MasterAdapter
;
1094 AdapterObject
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1095 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1097 if (NumberOfMapRegisters
&& AdapterObject
->NeedsMapRegisters
)
1099 if (NumberOfMapRegisters
> AdapterObject
->MapRegistersPerChannel
)
1101 AdapterObject
->NumberOfMapRegisters
= 0;
1102 IoFreeAdapterChannel(AdapterObject
);
1103 return STATUS_INSUFFICIENT_RESOURCES
;
1107 * Get the map registers. This is partly complicated by the fact
1108 * that new map registers can only be allocated at PASSIVE_LEVEL
1109 * and we're currently at DISPATCH_LEVEL. The following code has
1112 * - If there is no adapter queued for map register allocation,
1113 * try to see if enough contiguous map registers are present.
1114 * In case they're we can just get them and proceed further.
1116 * - If some adapter is already present in the queue we must
1117 * respect the order of adapters asking for map registers and
1118 * so the fast case described above can't take place.
1119 * This case is also entered if not enough coniguous map
1120 * registers are present.
1122 * A work queue item is allocated and queued, the adapter is
1123 * also queued into the master adapter queue. The worker
1124 * routine does the job of allocating the map registers at
1125 * PASSIVE_LEVEL and calling the ExecutionRoutine.
1128 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1130 if (IsListEmpty(&MasterAdapter
->AdapterQueue
))
1132 Index
= RtlFindClearBitsAndSet(
1133 MasterAdapter
->MapRegisters
, NumberOfMapRegisters
, 0);
1134 if (Index
!= MAXULONG
)
1136 AdapterObject
->MapRegisterBase
=
1137 MasterAdapter
->MapRegisterBase
+ Index
;
1138 if (!AdapterObject
->ScatterGather
)
1140 AdapterObject
->MapRegisterBase
=
1141 (PROS_MAP_REGISTER_ENTRY
)(
1142 (ULONG_PTR
)AdapterObject
->MapRegisterBase
|
1148 if (Index
== MAXULONG
)
1150 WorkItem
= ExAllocatePoolWithTag(
1151 NonPagedPool
, sizeof(GROW_WORK_ITEM
), TAG_DMA
);
1152 if (WorkItem
== NULL
)
1154 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1155 AdapterObject
->NumberOfMapRegisters
= 0;
1156 IoFreeAdapterChannel(AdapterObject
);
1157 return STATUS_INSUFFICIENT_RESOURCES
;
1160 InsertTailList(&MasterAdapter
->AdapterQueue
, &AdapterObject
->AdapterQueue
);
1162 ExInitializeWorkItem(
1163 &WorkItem
->WorkQueueItem
, HalpGrowMapBufferWorker
, WorkItem
);
1164 WorkItem
->AdapterObject
= AdapterObject
;
1165 WorkItem
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1167 ExQueueWorkItem(&WorkItem
->WorkQueueItem
, DelayedWorkQueue
);
1169 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1171 return STATUS_SUCCESS
;
1174 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1178 AdapterObject
->MapRegisterBase
= NULL
;
1179 AdapterObject
->NumberOfMapRegisters
= 0;
1182 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1184 Result
= ExecutionRoutine(
1185 WaitContextBlock
->DeviceObject
, WaitContextBlock
->CurrentIrp
,
1186 AdapterObject
->MapRegisterBase
, WaitContextBlock
->DeviceContext
);
1189 * Possible return values:
1192 * Don't free any resources, the ADAPTER_OBJECT is still in use and
1193 * the caller will call IoFreeAdapterChannel later.
1195 * - DeallocateObject
1196 * Deallocate the map registers and release the ADAPTER_OBJECT, so
1197 * someone else can use it.
1199 * - DeallocateObjectKeepRegisters
1200 * Release the ADAPTER_OBJECT, but hang on to the map registers. The
1201 * client will later call IoFreeMapRegisters.
1204 * IoFreeAdapterChannel runs the queue, so it must be called unless
1205 * the adapter object is not to be freed.
1208 if (Result
== DeallocateObject
)
1210 IoFreeAdapterChannel(AdapterObject
);
1212 else if (Result
== DeallocateObjectKeepRegisters
)
1214 AdapterObject
->NumberOfMapRegisters
= 0;
1215 IoFreeAdapterChannel(AdapterObject
);
1218 return STATUS_SUCCESS
;
1222 * @name IoFreeAdapterChannel
1224 * Free DMA resources allocated by IoAllocateAdapterChannel.
1226 * @param AdapterObject
1227 * Adapter object with resources to free.
1230 * This function releases map registers registers assigned to the DMA
1231 * adapter. After releasing the adapter, it checks the adapter's queue
1232 * and runs each queued device object in series until the queue is
1233 * empty. This is the only way the device queue is emptied.
1235 * @see IoAllocateAdapterChannel
1241 IoFreeAdapterChannel(
1242 PADAPTER_OBJECT AdapterObject
)
1244 PADAPTER_OBJECT MasterAdapter
;
1245 PKDEVICE_QUEUE_ENTRY DeviceQueueEntry
;
1246 PWAIT_CONTEXT_BLOCK WaitContextBlock
;
1247 ULONG Index
= MAXULONG
;
1251 MasterAdapter
= AdapterObject
->MasterAdapter
;
1256 * To keep map registers, call here with AdapterObject->
1257 * NumberOfMapRegisters set to zero. This trick is used in
1258 * HalAllocateAdapterChannel for example.
1260 if (AdapterObject
->NumberOfMapRegisters
)
1264 AdapterObject
->MapRegisterBase
,
1265 AdapterObject
->NumberOfMapRegisters
);
1268 DeviceQueueEntry
= KeRemoveDeviceQueue(&AdapterObject
->ChannelWaitQueue
);
1269 if (DeviceQueueEntry
== NULL
)
1274 WaitContextBlock
= CONTAINING_RECORD(
1279 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1280 AdapterObject
->NumberOfMapRegisters
= WaitContextBlock
->NumberOfMapRegisters
;
1282 if (WaitContextBlock
->NumberOfMapRegisters
&&
1283 AdapterObject
->MasterAdapter
)
1285 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1287 if (IsListEmpty(&MasterAdapter
->AdapterQueue
))
1289 Index
= RtlFindClearBitsAndSet(
1290 MasterAdapter
->MapRegisters
,
1291 WaitContextBlock
->NumberOfMapRegisters
, 0);
1292 if (Index
!= MAXULONG
)
1294 AdapterObject
->MapRegisterBase
=
1295 MasterAdapter
->MapRegisterBase
+ Index
;
1296 if (!AdapterObject
->ScatterGather
)
1298 AdapterObject
->MapRegisterBase
=
1299 (PROS_MAP_REGISTER_ENTRY
)(
1300 (ULONG_PTR
)AdapterObject
->MapRegisterBase
|
1306 if (Index
== MAXULONG
)
1308 InsertTailList(&MasterAdapter
->AdapterQueue
, &AdapterObject
->AdapterQueue
);
1309 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1313 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1317 AdapterObject
->MapRegisterBase
= NULL
;
1318 AdapterObject
->NumberOfMapRegisters
= 0;
1321 /* Call the adapter control routine. */
1322 Result
= ((PDRIVER_CONTROL
)WaitContextBlock
->DeviceRoutine
)(
1323 WaitContextBlock
->DeviceObject
, WaitContextBlock
->CurrentIrp
,
1324 AdapterObject
->MapRegisterBase
, WaitContextBlock
->DeviceContext
);
1330 * We're done until the caller manually calls IoFreeAdapterChannel
1331 * or IoFreeMapRegisters.
1335 case DeallocateObjectKeepRegisters
:
1337 * Hide the map registers so they aren't deallocated next time
1340 AdapterObject
->NumberOfMapRegisters
= 0;
1350 * @name IoFreeMapRegisters
1352 * Free map registers reserved by the system for a DMA.
1354 * @param AdapterObject
1355 * DMA adapter to free map registers on.
1356 * @param MapRegisterBase
1357 * Handle to map registers to free.
1358 * @param NumberOfRegisters
1359 * Number of map registers to be freed.
1366 IN PADAPTER_OBJECT AdapterObject
,
1367 IN PVOID MapRegisterBase
,
1368 IN ULONG NumberOfMapRegisters
)
1370 PADAPTER_OBJECT MasterAdapter
= AdapterObject
->MasterAdapter
;
1371 PLIST_ENTRY ListEntry
;
1376 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1378 if (MasterAdapter
== NULL
|| MapRegisterBase
== NULL
)
1381 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1383 if (NumberOfMapRegisters
!= 0)
1385 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1387 RealMapRegisterBase
=
1388 (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1389 RtlClearBits(MasterAdapter
->MapRegisters
,
1390 RealMapRegisterBase
- MasterAdapter
->MapRegisterBase
,
1391 NumberOfMapRegisters
);
1395 * Now that we freed few map registers it's time to look at the master
1396 * adapter queue and see if there is someone waiting for map registers.
1399 while (!IsListEmpty(&MasterAdapter
->AdapterQueue
))
1401 ListEntry
= RemoveHeadList(&MasterAdapter
->AdapterQueue
);
1402 AdapterObject
= CONTAINING_RECORD(
1403 ListEntry
, struct _ADAPTER_OBJECT
, AdapterQueue
);
1405 Index
= RtlFindClearBitsAndSet(
1406 MasterAdapter
->MapRegisters
,
1407 AdapterObject
->NumberOfMapRegisters
,
1408 MasterAdapter
->NumberOfMapRegisters
);
1409 if (Index
== MAXULONG
)
1411 InsertHeadList(&MasterAdapter
->AdapterQueue
, ListEntry
);
1415 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1417 AdapterObject
->MapRegisterBase
=
1418 MasterAdapter
->MapRegisterBase
+ Index
;
1419 if (!AdapterObject
->ScatterGather
)
1421 AdapterObject
->MapRegisterBase
=
1422 (PROS_MAP_REGISTER_ENTRY
)(
1423 (ULONG_PTR
)AdapterObject
->MapRegisterBase
|
1427 Result
= ((PDRIVER_CONTROL
)AdapterObject
->CurrentWcb
->DeviceRoutine
)(
1428 AdapterObject
->CurrentWcb
->DeviceObject
,
1429 AdapterObject
->CurrentWcb
->CurrentIrp
,
1430 AdapterObject
->MapRegisterBase
,
1431 AdapterObject
->CurrentWcb
->DeviceContext
);
1435 case DeallocateObjectKeepRegisters
:
1436 AdapterObject
->NumberOfMapRegisters
= 0;
1439 case DeallocateObject
:
1440 if (AdapterObject
->NumberOfMapRegisters
)
1442 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1443 RtlClearBits(MasterAdapter
->MapRegisters
,
1444 AdapterObject
->MapRegisterBase
-
1445 MasterAdapter
->MapRegisterBase
,
1446 AdapterObject
->NumberOfMapRegisters
);
1447 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1449 IoFreeAdapterChannel(AdapterObject
);
1456 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1459 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1463 * @name HalpCopyBufferMap
1465 * Helper function for copying data from/to map register buffers.
1467 * @see IoFlushAdapterBuffers, IoMapTransfer
1473 PROS_MAP_REGISTER_ENTRY MapRegisterBase
,
1476 BOOLEAN WriteToDevice
)
1478 ULONG CurrentLength
;
1479 ULONG_PTR CurrentAddress
;
1481 PVOID VirtualAddress
;
1483 VirtualAddress
= MmGetSystemAddressForMdlSafe(Mdl
, HighPagePriority
);
1484 if (VirtualAddress
== NULL
)
1487 * NOTE: On real NT a mechanism with reserved pages is implemented
1488 * to handle this case in a slow, but graceful non-fatal way.
1490 KeBugCheckEx(HAL_MEMORY_ALLOCATION
, PAGE_SIZE
, 0, (ULONG_PTR
)__FILE__
, 0);
1493 CurrentAddress
= (ULONG_PTR
)VirtualAddress
+
1494 (ULONG_PTR
)CurrentVa
-
1495 (ULONG_PTR
)MmGetMdlVirtualAddress(Mdl
);
1499 ByteOffset
= BYTE_OFFSET(CurrentAddress
);
1500 CurrentLength
= PAGE_SIZE
- ByteOffset
;
1501 if (CurrentLength
> Length
)
1502 CurrentLength
= Length
;
1507 (PVOID
)((ULONG_PTR
)MapRegisterBase
->VirtualAddress
+ ByteOffset
),
1508 (PVOID
)CurrentAddress
,
1514 (PVOID
)CurrentAddress
,
1515 (PVOID
)((ULONG_PTR
)MapRegisterBase
->VirtualAddress
+ ByteOffset
),
1519 Length
-= CurrentLength
;
1520 CurrentAddress
+= CurrentLength
;
1526 * @name IoFlushAdapterBuffers
1528 * Flush any data remaining in the DMA controller's memory into the host
1531 * @param AdapterObject
1532 * The adapter object to flush.
1534 * Original MDL to flush data into.
1535 * @param MapRegisterBase
1536 * Map register base that was just used by IoMapTransfer, etc.
1538 * Offset into Mdl to be flushed into, same as was passed to
1541 * Length of the buffer to be flushed into.
1542 * @param WriteToDevice
1543 * TRUE if it's a write, FALSE if it's a read.
1545 * @return TRUE in all cases.
1548 * This copies data from the map register-backed buffer to the user's
1549 * target buffer. Data are not in the user buffer until this function
1551 * For slave DMA transfers the controller channel is masked effectively
1552 * stopping the current transfer.
1558 IoFlushAdapterBuffers(
1559 PADAPTER_OBJECT AdapterObject
,
1561 PVOID MapRegisterBase
,
1564 BOOLEAN WriteToDevice
)
1566 BOOLEAN SlaveDma
= FALSE
;
1567 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1568 PHYSICAL_ADDRESS HighestAcceptableAddress
;
1569 PHYSICAL_ADDRESS PhysicalAddress
;
1570 PPFN_NUMBER MdlPagesPtr
;
1573 ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL
);
1574 ASSERT(AdapterObject
);
1576 if (!AdapterObject
->MasterDevice
)
1578 /* Mask out (disable) the DMA channel. */
1579 if (AdapterObject
->AdapterNumber
== 1)
1581 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
1582 WRITE_PORT_UCHAR(&DmaControl1
->SingleMask
,
1583 AdapterObject
->ChannelNumber
| DMA_SETMASK
);
1587 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
1588 WRITE_PORT_UCHAR(&DmaControl2
->SingleMask
,
1589 AdapterObject
->ChannelNumber
| DMA_SETMASK
);
1594 /* This can happen if the device supports hardware scatter/gather. */
1595 if (MapRegisterBase
== NULL
)
1598 RealMapRegisterBase
=
1599 (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1603 if ((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
)
1605 if (RealMapRegisterBase
->Counter
!= MAXULONG
)
1607 if (SlaveDma
&& !AdapterObject
->IgnoreCount
)
1608 Length
-= HalReadDmaCounter(AdapterObject
);
1610 HalpCopyBufferMap(Mdl
, RealMapRegisterBase
, CurrentVa
, Length
, FALSE
);
1614 MdlPagesPtr
= MmGetMdlPfnArray(Mdl
);
1615 MdlPagesPtr
+= ((ULONG_PTR
)CurrentVa
- (ULONG_PTR
)Mdl
->StartVa
) >> PAGE_SHIFT
;
1617 PhysicalAddress
.QuadPart
= *MdlPagesPtr
<< PAGE_SHIFT
;
1618 PhysicalAddress
.QuadPart
+= BYTE_OFFSET(CurrentVa
);
1620 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
1621 if (PhysicalAddress
.QuadPart
+ Length
>
1622 HighestAcceptableAddress
.QuadPart
)
1624 HalpCopyBufferMap(Mdl
, RealMapRegisterBase
, CurrentVa
, Length
, FALSE
);
1629 RealMapRegisterBase
->Counter
= 0;
1635 * @name IoMapTransfer
1637 * Map a DMA for transfer and do the DMA if it's a slave.
1639 * @param AdapterObject
1640 * Adapter object to do the DMA on. Bus-master may pass NULL.
1642 * Locked-down user buffer to DMA in to or out of.
1643 * @param MapRegisterBase
1644 * Handle to map registers to use for this dma.
1646 * Index into Mdl to transfer into/out of.
1648 * Length of transfer. Number of bytes actually transferred on
1650 * @param WriteToDevice
1651 * TRUE if it's an output DMA, FALSE otherwise.
1654 * A logical address that can be used to program a DMA controller, it's
1655 * not meaningful for slave DMA device.
1658 * This function does a copyover to contiguous memory <16MB represented
1659 * by the map registers if needed. If the buffer described by MDL can be
1660 * used as is no copyover is done.
1661 * If it's a slave transfer, this function actually performs it.
1666 PHYSICAL_ADDRESS NTAPI
1668 IN PADAPTER_OBJECT AdapterObject
,
1670 IN PVOID MapRegisterBase
,
1672 IN OUT PULONG Length
,
1673 IN BOOLEAN WriteToDevice
)
1675 PPFN_NUMBER MdlPagesPtr
;
1676 PFN_NUMBER MdlPage1
, MdlPage2
;
1678 ULONG TransferOffset
;
1679 ULONG TransferLength
;
1680 BOOLEAN UseMapRegisters
;
1681 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1682 PHYSICAL_ADDRESS PhysicalAddress
;
1683 PHYSICAL_ADDRESS HighestAcceptableAddress
;
1685 DMA_MODE AdapterMode
;
1689 * Precalculate some values that are used in all cases.
1691 * ByteOffset is offset inside the page at which the transfer starts.
1692 * MdlPagesPtr is pointer inside the MDL page chain at the page where the
1694 * PhysicalAddress is physical address corresponding to the transfer
1695 * start page and offset.
1696 * TransferLength is the inital length of the transfer, which is reminder
1697 * of the first page. The actual value is calculated below.
1699 * Note that all the variables can change during the processing which
1700 * takes place below. These are just initial values.
1703 ByteOffset
= BYTE_OFFSET(CurrentVa
);
1705 MdlPagesPtr
= MmGetMdlPfnArray(Mdl
);
1706 MdlPagesPtr
+= ((ULONG_PTR
)CurrentVa
- (ULONG_PTR
)Mdl
->StartVa
) >> PAGE_SHIFT
;
1708 PhysicalAddress
.QuadPart
= *MdlPagesPtr
<< PAGE_SHIFT
;
1709 PhysicalAddress
.QuadPart
+= ByteOffset
;
1711 TransferLength
= PAGE_SIZE
- ByteOffset
;
1714 * Special case for bus master adapters with S/G support. We can directly
1715 * use the buffer specified by the MDL, so not much work has to be done.
1717 * Just return the passed VA's corresponding physical address and update
1718 * length to the number of physically contiguous bytes found. Also
1719 * pages crossing the 4Gb boundary aren't considered physically contiguous.
1722 if (MapRegisterBase
== NULL
)
1724 while (TransferLength
< *Length
)
1726 MdlPage1
= *MdlPagesPtr
;
1727 MdlPage2
= *(MdlPagesPtr
+ 1);
1728 if (MdlPage1
+ 1 != MdlPage2
)
1730 if ((MdlPage1
^ MdlPage2
) & ~0xFFFFF)
1732 TransferLength
+= PAGE_SIZE
;
1736 if (TransferLength
< *Length
)
1737 *Length
= TransferLength
;
1739 return PhysicalAddress
;
1743 * The code below applies to slave DMA adapters and bus master adapters
1744 * without hardward S/G support.
1747 RealMapRegisterBase
=
1748 (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1751 * Try to calculate the size of the transfer. We can only transfer
1752 * pages that are physically contiguous and that don't cross the
1753 * 64Kb boundary (this limitation applies only for ISA controllers).
1756 while (TransferLength
< *Length
)
1758 MdlPage1
= *MdlPagesPtr
;
1759 MdlPage2
= *(MdlPagesPtr
+ 1);
1760 if (MdlPage1
+ 1 != MdlPage2
)
1762 if (!HalpEisaDma
&& ((MdlPage1
^ MdlPage2
) & ~0xF))
1764 TransferLength
+= PAGE_SIZE
;
1768 if (TransferLength
> *Length
)
1769 TransferLength
= *Length
;
1772 * If we're about to simulate software S/G and not all the pages are
1773 * physically contiguous then we must use the map registers to store
1774 * the data and allow the whole transfer to proceed at once.
1777 if ((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
&&
1778 TransferLength
< *Length
)
1780 UseMapRegisters
= TRUE
;
1781 PhysicalAddress
= RealMapRegisterBase
->PhysicalAddress
;
1782 PhysicalAddress
.QuadPart
+= ByteOffset
;
1783 TransferLength
= *Length
;
1784 RealMapRegisterBase
->Counter
= MAXULONG
;
1790 * This is ordinary DMA transfer, so just update the progress
1791 * counters. These are used by IoFlushAdapterBuffers to track
1792 * the transfer progress.
1795 UseMapRegisters
= FALSE
;
1796 Counter
= RealMapRegisterBase
->Counter
;
1797 RealMapRegisterBase
->Counter
+= BYTES_TO_PAGES(ByteOffset
+ TransferLength
);
1800 * Check if the buffer doesn't exceed the highest physical address
1801 * limit of the device. In that case we must use the map registers to
1805 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
1806 if (PhysicalAddress
.QuadPart
+ TransferLength
>
1807 HighestAcceptableAddress
.QuadPart
)
1809 UseMapRegisters
= TRUE
;
1810 PhysicalAddress
= RealMapRegisterBase
[Counter
].PhysicalAddress
;
1811 PhysicalAddress
.QuadPart
+= ByteOffset
;
1812 if ((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
)
1814 RealMapRegisterBase
->Counter
= MAXULONG
;
1821 * If we decided to use the map registers (see above) and we're about
1822 * to transfer data to the device then copy the buffers into the map
1826 if (UseMapRegisters
&& WriteToDevice
)
1828 HalpCopyBufferMap(Mdl
, RealMapRegisterBase
+ Counter
,
1829 CurrentVa
, TransferLength
, WriteToDevice
);
1833 * Return the length of transfer that actually takes place.
1836 *Length
= TransferLength
;
1839 * If we're doing slave (system) DMA then program the (E)ISA controller
1840 * to actually start the transfer.
1843 if (AdapterObject
!= NULL
&& !AdapterObject
->MasterDevice
)
1845 AdapterMode
= AdapterObject
->AdapterMode
;
1849 AdapterMode
.TransferType
= WRITE_TRANSFER
;
1853 AdapterMode
.TransferType
= READ_TRANSFER
;
1854 if (AdapterObject
->IgnoreCount
)
1856 RtlZeroMemory((PUCHAR
)RealMapRegisterBase
[Counter
].VirtualAddress
+
1857 ByteOffset
, TransferLength
);
1861 TransferOffset
= PhysicalAddress
.LowPart
& 0xFFFF;
1862 if (AdapterObject
->Width16Bits
)
1864 TransferLength
>>= 1;
1865 TransferOffset
>>= 1;
1868 KeAcquireSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, &OldIrql
);
1870 if (AdapterObject
->AdapterNumber
== 1)
1872 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
1874 /* Reset Register */
1875 WRITE_PORT_UCHAR(&DmaControl1
->ClearBytePointer
, 0);
1877 WRITE_PORT_UCHAR(&DmaControl1
->Mode
, AdapterMode
.Byte
);
1878 /* Set the Offset Register */
1879 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1880 (UCHAR
)(TransferOffset
));
1881 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1882 (UCHAR
)(TransferOffset
>> 8));
1883 /* Set the Page Register */
1884 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+
1885 FIELD_OFFSET(EISA_CONTROL
, DmaController1Pages
),
1886 (UCHAR
)(PhysicalAddress
.LowPart
>> 16));
1889 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+
1890 FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
),
1893 /* Set the Length */
1894 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1895 (UCHAR
)(TransferLength
- 1));
1896 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1897 (UCHAR
)((TransferLength
- 1) >> 8));
1898 /* Unmask the Channel */
1899 WRITE_PORT_UCHAR(&DmaControl1
->SingleMask
,
1900 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
1904 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
1906 /* Reset Register */
1907 WRITE_PORT_UCHAR(&DmaControl2
->ClearBytePointer
, 0);
1909 WRITE_PORT_UCHAR(&DmaControl2
->Mode
, AdapterMode
.Byte
);
1910 /* Set the Offset Register */
1911 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1912 (UCHAR
)(TransferOffset
));
1913 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1914 (UCHAR
)(TransferOffset
>> 8));
1915 /* Set the Page Register */
1916 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+
1917 FIELD_OFFSET(EISA_CONTROL
, DmaController1Pages
),
1918 (UCHAR
)(PhysicalAddress
.u
.LowPart
>> 16));
1921 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+
1922 FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
),
1925 /* Set the Length */
1926 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1927 (UCHAR
)(TransferLength
- 1));
1928 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1929 (UCHAR
)((TransferLength
- 1) >> 8));
1930 /* Unmask the Channel */
1931 WRITE_PORT_UCHAR(&DmaControl2
->SingleMask
,
1932 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
1935 KeReleaseSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, OldIrql
);
1939 * Return physical address of the buffer with data that is used for the
1940 * transfer. It can either point inside the Mdl that was passed by the
1941 * caller or into the map registers if the Mdl buffer can't be used
1945 return PhysicalAddress
;
1949 * @name HalFlushCommonBuffer
1955 HalFlushCommonBuffer(IN PADAPTER_OBJECT AdapterObject
,
1957 IN PHYSICAL_ADDRESS LogicalAddress
,
1958 IN PVOID VirtualAddress
)
1960 /* Function always returns true */
1969 HalAllocateCrashDumpRegisters(IN PADAPTER_OBJECT AdapterObject
,
1970 IN OUT PULONG NumberOfMapRegisters
)
1972 PADAPTER_OBJECT MasterAdapter
= AdapterObject
->MasterAdapter
;
1973 ULONG MapRegisterNumber
;
1975 /* Check if it needs map registers */
1976 if (AdapterObject
->NeedsMapRegisters
)
1978 /* Check if we have enough */
1979 if (*NumberOfMapRegisters
> AdapterObject
->MapRegistersPerChannel
)
1981 /* We don't, fail */
1982 AdapterObject
->NumberOfMapRegisters
= 0;
1986 /* Try to find free map registers */
1987 MapRegisterNumber
= RtlFindClearBitsAndSet(MasterAdapter
->MapRegisters
,
1988 *NumberOfMapRegisters
,
1991 /* Check if nothing was found */
1992 if (MapRegisterNumber
== MAXULONG
)
1994 /* No free registers found, so use the base registers */
1995 RtlSetBits(MasterAdapter
->MapRegisters
,
1997 *NumberOfMapRegisters
);
1998 MapRegisterNumber
= 0;
2001 /* Calculate the new base */
2002 AdapterObject
->MapRegisterBase
=
2003 (PROS_MAP_REGISTER_ENTRY
)(MasterAdapter
->MapRegisterBase
+
2006 /* Check if scatter gather isn't supported */
2007 if (!AdapterObject
->ScatterGather
)
2010 AdapterObject
->MapRegisterBase
=
2011 (PROS_MAP_REGISTER_ENTRY
)
2012 ((ULONG_PTR
)AdapterObject
->MapRegisterBase
| MAP_BASE_SW_SG
);
2017 AdapterObject
->MapRegisterBase
= NULL
;
2018 AdapterObject
->NumberOfMapRegisters
= 0;
2021 /* Return the base */
2022 return AdapterObject
->MapRegisterBase
;