3 * COPYRIGHT: See COPYING in the top level directory
4 * PROJECT: ReactOS kernel
5 * FILE: ntoskrnl/hal/x86/dma.c
6 * PURPOSE: DMA functions
7 * PROGRAMMERS: David Welch (welch@mcmail.com)
8 * Filip Navara (navaraf@reactos.com)
14 * @page DMA Implementation Notes
20 * Abstract encapsulation of physically contiguous buffer that resides
21 * in memory accessible by both the DMA device / controller and the system.
22 * The map registers are allocated and distributed on demand and are
25 * The actual use of map registers is to allow transfers from/to buffer
26 * located in physical memory at address inaccessible by the DMA device /
27 * controller directly. For such transfers the map register buffers
28 * are used as intermediate data storage.
32 * A container for map registers (typically corresponding to one physical
33 * bus connection type). There can be master adapters for 24-bit address
34 * ranges, 32-bit address ranges, etc. Every time a new DMA adapter is
35 * created it's associated with a corresponding master adapter that
36 * is used for any map register allocation requests.
38 * - Bus-master / Slave DMA
40 * Slave DMA is term used for DMA transfers done by the system (E)ISA
41 * controller as opposed to transfers mastered by the device itself
44 * For slave DMA special care is taken to actually access the system
45 * controller and handle the transfers. The relevant code is in
46 * HalpDmaInitializeEisaAdapter, HalReadDmaCounter, IoFlushAdapterBuffers
51 * - Allocation of map registers
53 * Initial set of map registers is allocated on the system start to
54 * ensure that low memory won't get filled up later. Additional map
55 * registers are allocated as needed by HalpGrowMapBuffers. This
56 * routine is called on two places:
58 * - HalGetAdapter, since we're at PASSIVE_LEVEL and it's known that
59 * more map registers will probably be needed.
60 * - IoAllocateAdapterChannel (indirectly using HalpGrowMapBufferWorker
61 * since we're at DISPATCH_LEVEL and call HalpGrowMapBuffers directly)
62 * when no more map registers are free.
64 * Note that even if no more map registers can be allocated it's not
65 * the end of the world. The adapters waiting for free map registers
66 * are queued in the master adapter's queue and once one driver hands
67 * back it's map registers (using IoFreeMapRegisters or indirectly using
68 * the execution routine callback in IoAllocateAdapterChannel) the
69 * queue gets processed and the map registers are reassigned.
72 /* INCLUDES *****************************************************************/
79 static KEVENT HalpDmaLock
;
80 static LIST_ENTRY HalpDmaAdapterList
;
81 static PADAPTER_OBJECT HalpEisaAdapter
[8];
83 static BOOLEAN HalpEisaDma
;
85 static PADAPTER_OBJECT HalpMasterAdapter
;
88 static const ULONG_PTR HalpEisaPortPage
[8] = {
89 FIELD_OFFSET(DMA_PAGE
, Channel0
),
90 FIELD_OFFSET(DMA_PAGE
, Channel1
),
91 FIELD_OFFSET(DMA_PAGE
, Channel2
),
92 FIELD_OFFSET(DMA_PAGE
, Channel3
),
94 FIELD_OFFSET(DMA_PAGE
, Channel5
),
95 FIELD_OFFSET(DMA_PAGE
, Channel6
),
96 FIELD_OFFSET(DMA_PAGE
, Channel7
)
100 static DMA_OPERATIONS HalpDmaOperations
= {
101 sizeof(DMA_OPERATIONS
),
102 (PPUT_DMA_ADAPTER
)HalPutDmaAdapter
,
103 (PALLOCATE_COMMON_BUFFER
)HalAllocateCommonBuffer
,
104 (PFREE_COMMON_BUFFER
)HalFreeCommonBuffer
,
105 NULL
, /* Initialized in HalpInitDma() */
106 NULL
, /* Initialized in HalpInitDma() */
107 NULL
, /* Initialized in HalpInitDma() */
108 NULL
, /* Initialized in HalpInitDma() */
109 NULL
, /* Initialized in HalpInitDma() */
110 (PGET_DMA_ALIGNMENT
)HalpDmaGetDmaAlignment
,
111 (PREAD_DMA_COUNTER
)HalReadDmaCounter
,
112 /* FIXME: Implement the S/G funtions. */
113 NULL
/*(PGET_SCATTER_GATHER_LIST)HalGetScatterGatherList*/,
114 NULL
/*(PPUT_SCATTER_GATHER_LIST)HalPutScatterGatherList*/,
115 NULL
/*(PCALCULATE_SCATTER_GATHER_LIST_SIZE)HalCalculateScatterGatherListSize*/,
116 NULL
/*(PBUILD_SCATTER_GATHER_LIST)HalBuildScatterGatherList*/,
117 NULL
/*(PBUILD_MDL_FROM_SCATTER_GATHER_LIST)HalBuildMdlFromScatterGatherList*/
121 #define MAX_MAP_REGISTERS 64
123 #define TAG_DMA ' AMD'
125 /* FUNCTIONS *****************************************************************/
133 * Initialize the DMA Operation table
135 HalpDmaOperations
.AllocateAdapterChannel
= (PALLOCATE_ADAPTER_CHANNEL
)IoAllocateAdapterChannel
;
136 HalpDmaOperations
.FlushAdapterBuffers
= (PFLUSH_ADAPTER_BUFFERS
)IoFlushAdapterBuffers
;
137 HalpDmaOperations
.FreeAdapterChannel
= (PFREE_ADAPTER_CHANNEL
)IoFreeAdapterChannel
;
138 HalpDmaOperations
.FreeMapRegisters
= (PFREE_MAP_REGISTERS
)IoFreeMapRegisters
;
139 HalpDmaOperations
.MapTransfer
= (PMAP_TRANSFER
)IoMapTransfer
;
142 * Check if Extended DMA is available. We're just going to do a random
145 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
.Channel2
), 0x2A);
146 if (READ_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
.Channel2
)) == 0x2A)
152 * Intialize all the global variables and allocate master adapter with
155 InitializeListHead(&HalpDmaAdapterList
);
156 KeInitializeEvent(&HalpDmaLock
, NotificationEvent
, TRUE
);
157 HalpMasterAdapter
= HalpDmaAllocateMasterAdapter();
160 * Setup the HalDispatchTable callback for creating PnP DMA adapters. It's
161 * used by IoGetDmaAdapter in the kernel.
163 HalGetDmaAdapter
= HalpGetDmaAdapter
;
168 * @name HalpGetAdapterMaximumPhysicalAddress
170 * Get the maximum physical address acceptable by the device represented
171 * by the passed DMA adapter.
175 HalpGetAdapterMaximumPhysicalAddress(IN PADAPTER_OBJECT AdapterObject
)
177 PHYSICAL_ADDRESS HighestAddress
;
179 if (AdapterObject
->MasterDevice
)
181 if (AdapterObject
->Dma64BitAddresses
)
183 HighestAddress
.QuadPart
= 0xFFFFFFFFFFFFFFFFULL
;
184 return HighestAddress
;
186 else if (AdapterObject
->Dma32BitAddresses
)
188 HighestAddress
.QuadPart
= 0xFFFFFFFF;
189 return HighestAddress
;
193 HighestAddress
.QuadPart
= 0xFFFFFF;
194 return HighestAddress
;
199 * @name HalpGrowMapBuffers
201 * Allocate initial, or additional, map buffers for DMA master adapter.
203 * @param MasterAdapter
204 * DMA master adapter to allocate buffers for.
205 * @param SizeOfMapBuffers
206 * Size of the map buffers to allocate (not including the size
207 * already allocated).
211 HalpGrowMapBuffers(IN PADAPTER_OBJECT AdapterObject
,
212 IN ULONG SizeOfMapBuffers
)
214 PVOID VirtualAddress
;
215 PHYSICAL_ADDRESS PhysicalAddress
;
216 PHYSICAL_ADDRESS HighestAcceptableAddress
;
217 PHYSICAL_ADDRESS LowestAcceptableAddress
;
218 PHYSICAL_ADDRESS BoundryAddressMultiple
;
220 ULONG MapRegisterCount
;
222 /* FIXME: Check if enough map register slots are available. */
223 MapRegisterCount
= BYTES_TO_PAGES(SizeOfMapBuffers
);
226 * Allocate memory for the new map registers. For 32-bit adapters we use
227 * two passes in order not to waste scare resource (low memory).
229 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
230 LowestAcceptableAddress
.HighPart
= 0;
231 LowestAcceptableAddress
.LowPart
= HighestAcceptableAddress
.LowPart
== 0xFFFFFFFF ? 0x1000000 : 0;
232 BoundryAddressMultiple
.QuadPart
= 0;
234 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(MapRegisterCount
<< PAGE_SHIFT
,
235 LowestAcceptableAddress
,
236 HighestAcceptableAddress
,
237 BoundryAddressMultiple
,
239 if (!(VirtualAddress
) && (LowestAcceptableAddress
.LowPart
))
241 LowestAcceptableAddress
.LowPart
= 0;
242 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(MapRegisterCount
<< PAGE_SHIFT
,
243 LowestAcceptableAddress
,
244 HighestAcceptableAddress
,
245 BoundryAddressMultiple
,
249 if (!VirtualAddress
) return FALSE
;
251 PhysicalAddress
= MmGetPhysicalAddress(VirtualAddress
);
254 * All the following must be done with the master adapter lock held
255 * to prevent corruption.
257 KeAcquireSpinLock(&AdapterObject
->SpinLock
, &OldIrql
);
260 * Setup map register entries for the buffer allocated. Each entry has
261 * a virtual and physical address and corresponds to PAGE_SIZE large
264 if (MapRegisterCount
> 0)
266 PROS_MAP_REGISTER_ENTRY CurrentEntry
, PreviousEntry
;
268 CurrentEntry
= AdapterObject
->MapRegisterBase
+ AdapterObject
->NumberOfMapRegisters
;
272 * Leave one entry free for every non-contiguous memory region
273 * in the map register bitmap. This ensures that we can search
274 * using RtlFindClearBits for contiguous map register regions.
276 * Also for non-EISA DMA leave one free entry for every 64Kb
277 * break, because the DMA controller can handle only coniguous
280 if (CurrentEntry
!= AdapterObject
->MapRegisterBase
)
282 PreviousEntry
= CurrentEntry
- 1;
283 if ((PreviousEntry
->PhysicalAddress
.LowPart
+ PAGE_SIZE
) == PhysicalAddress
.LowPart
)
287 if ((PreviousEntry
->PhysicalAddress
.LowPart
^ PhysicalAddress
.LowPart
) & 0xFFFF0000)
290 AdapterObject
->NumberOfMapRegisters
++;
297 AdapterObject
->NumberOfMapRegisters
++;
301 RtlClearBit(AdapterObject
->MapRegisters
,
302 CurrentEntry
- AdapterObject
->MapRegisterBase
);
303 CurrentEntry
->VirtualAddress
= VirtualAddress
;
304 CurrentEntry
->PhysicalAddress
= PhysicalAddress
;
306 PhysicalAddress
.LowPart
+= PAGE_SIZE
;
307 VirtualAddress
= (PVOID
)((ULONG_PTR
)VirtualAddress
+ PAGE_SIZE
);
310 AdapterObject
->NumberOfMapRegisters
++;
312 } while (MapRegisterCount
);
315 KeReleaseSpinLock(&AdapterObject
->SpinLock
, OldIrql
);
321 * @name HalpDmaAllocateMasterAdapter
323 * Helper routine to allocate and initialize master adapter object and it's
324 * associated map register buffers.
330 HalpDmaAllocateMasterAdapter(VOID
)
332 PADAPTER_OBJECT MasterAdapter
;
333 ULONG Size
, SizeOfBitmap
;
335 SizeOfBitmap
= MAX_MAP_REGISTERS
;
336 Size
= sizeof(ADAPTER_OBJECT
);
337 Size
+= sizeof(RTL_BITMAP
);
338 Size
+= (SizeOfBitmap
+ 7) >> 3;
340 MasterAdapter
= ExAllocatePoolWithTag(NonPagedPool
, Size
, TAG_DMA
);
341 if (!MasterAdapter
) return NULL
;
343 RtlZeroMemory(MasterAdapter
, Size
);
345 KeInitializeSpinLock(&MasterAdapter
->SpinLock
);
346 InitializeListHead(&MasterAdapter
->AdapterQueue
);
348 MasterAdapter
->MapRegisters
= (PVOID
)(MasterAdapter
+ 1);
349 RtlInitializeBitMap(MasterAdapter
->MapRegisters
,
350 (PULONG
)(MasterAdapter
->MapRegisters
+ 1),
352 RtlSetAllBits(MasterAdapter
->MapRegisters
);
353 MasterAdapter
->NumberOfMapRegisters
= 0;
354 MasterAdapter
->CommittedMapRegisters
= 0;
356 MasterAdapter
->MapRegisterBase
= ExAllocatePoolWithTag(NonPagedPool
,
358 sizeof(ROS_MAP_REGISTER_ENTRY
),
360 if (!MasterAdapter
->MapRegisterBase
)
362 ExFreePool(MasterAdapter
);
366 RtlZeroMemory(MasterAdapter
->MapRegisterBase
,
367 SizeOfBitmap
* sizeof(ROS_MAP_REGISTER_ENTRY
));
368 if (!HalpGrowMapBuffers(MasterAdapter
, 0x10000))
370 ExFreePool(MasterAdapter
);
374 return MasterAdapter
;
378 * @name HalpDmaAllocateChildAdapter
380 * Helper routine of HalGetAdapter. Allocate child adapter object and
381 * fill out some basic fields.
387 HalpDmaAllocateChildAdapter(IN ULONG NumberOfMapRegisters
,
388 IN PDEVICE_DESCRIPTION DeviceDescription
)
390 PADAPTER_OBJECT AdapterObject
;
391 OBJECT_ATTRIBUTES ObjectAttributes
;
395 InitializeObjectAttributes(&ObjectAttributes
,
397 OBJ_KERNEL_HANDLE
| OBJ_PERMANENT
,
401 Status
= ObCreateObject(KernelMode
,
406 sizeof(ADAPTER_OBJECT
),
409 (PVOID
)&AdapterObject
);
410 if (!NT_SUCCESS(Status
)) return NULL
;
412 Status
= ObReferenceObjectByPointer(AdapterObject
,
413 FILE_READ_DATA
| FILE_WRITE_DATA
,
416 if (!NT_SUCCESS(Status
)) return NULL
;
418 RtlZeroMemory(AdapterObject
, sizeof(ADAPTER_OBJECT
));
420 Status
= ObInsertObject(AdapterObject
,
422 FILE_READ_DATA
| FILE_WRITE_DATA
,
426 if (!NT_SUCCESS(Status
)) return NULL
;
430 AdapterObject
->DmaHeader
.Version
= (USHORT
)DeviceDescription
->Version
;
431 AdapterObject
->DmaHeader
.Size
= sizeof(ADAPTER_OBJECT
);
432 AdapterObject
->DmaHeader
.DmaOperations
= &HalpDmaOperations
;
433 AdapterObject
->MapRegistersPerChannel
= 1;
434 AdapterObject
->Dma32BitAddresses
= DeviceDescription
->Dma32BitAddresses
;
435 AdapterObject
->ChannelNumber
= 0xFF;
436 AdapterObject
->MasterAdapter
= HalpMasterAdapter
;
437 KeInitializeDeviceQueue(&AdapterObject
->ChannelWaitQueue
);
439 return AdapterObject
;
444 * @name HalpDmaInitializeEisaAdapter
446 * Setup DMA modes and extended modes for (E)ISA DMA adapter object.
450 HalpDmaInitializeEisaAdapter(IN PADAPTER_OBJECT AdapterObject
,
451 IN PDEVICE_DESCRIPTION DeviceDescription
)
454 DMA_MODE DmaMode
= {{0 }};
455 DMA_EXTENDED_MODE ExtendedMode
= {{ 0 }};
458 Controller
= (DeviceDescription
->DmaChannel
& 4) ? 2 : 1;
462 AdapterBaseVa
= (PVOID
)FIELD_OFFSET(EISA_CONTROL
, DmaController1
);
466 AdapterBaseVa
= (PVOID
)FIELD_OFFSET(EISA_CONTROL
, DmaController2
);
469 AdapterObject
->AdapterNumber
= Controller
;
470 AdapterObject
->ChannelNumber
= (UCHAR
)(DeviceDescription
->DmaChannel
& 3);
471 AdapterObject
->PagePort
= (PUCHAR
)HalpEisaPortPage
[DeviceDescription
->DmaChannel
];
472 AdapterObject
->Width16Bits
= FALSE
;
473 AdapterObject
->AdapterBaseVa
= AdapterBaseVa
;
477 ExtendedMode
.ChannelNumber
= AdapterObject
->ChannelNumber
;
479 switch (DeviceDescription
->DmaSpeed
)
481 case Compatible
: ExtendedMode
.TimingMode
= COMPATIBLE_TIMING
; break;
482 case TypeA
: ExtendedMode
.TimingMode
= TYPE_A_TIMING
; break;
483 case TypeB
: ExtendedMode
.TimingMode
= TYPE_B_TIMING
; break;
484 case TypeC
: ExtendedMode
.TimingMode
= BURST_TIMING
; break;
489 switch (DeviceDescription
->DmaWidth
)
491 case Width8Bits
: ExtendedMode
.TransferSize
= B_8BITS
; break;
492 case Width16Bits
: ExtendedMode
.TransferSize
= B_16BITS
; break;
493 case Width32Bits
: ExtendedMode
.TransferSize
= B_32BITS
; break;
500 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaExtendedMode1
),
505 WRITE_PORT_UCHAR((PUCHAR
)FIELD_OFFSET(EISA_CONTROL
, DmaExtendedMode2
),
512 * Validate setup for non-busmaster DMA adapter. Secondary controller
513 * supports only 16-bit transfers and main controller supports only
514 * 8-bit transfers. Anything else is invalid.
516 if (!DeviceDescription
->Master
)
518 if ((Controller
== 2) && (DeviceDescription
->DmaWidth
== Width16Bits
))
520 AdapterObject
->Width16Bits
= TRUE
;
522 else if ((Controller
!= 1) || (DeviceDescription
->DmaWidth
!= Width8Bits
))
529 DmaMode
.Channel
= AdapterObject
->ChannelNumber
;
530 DmaMode
.AutoInitialize
= DeviceDescription
->AutoInitialize
;
533 * Set the DMA request mode.
535 * For (E)ISA bus master devices just unmask (enable) the DMA channel
536 * and set it to cascade mode. Otherwise just select the right one
537 * bases on the passed device description.
539 if (DeviceDescription
->Master
)
541 DmaMode
.RequestMode
= CASCADE_REQUEST_MODE
;
544 /* Set the Request Data */
545 WRITE_PORT_UCHAR(&((PDMA1_CONTROL
)AdapterBaseVa
)->Mode
, DmaMode
.Byte
);
547 /* Unmask DMA Channel */
548 WRITE_PORT_UCHAR(&((PDMA1_CONTROL
)AdapterBaseVa
)->SingleMask
,
549 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
553 /* Set the Request Data */
554 WRITE_PORT_UCHAR(&((PDMA2_CONTROL
)AdapterBaseVa
)->Mode
, DmaMode
.Byte
);
556 /* Unmask DMA Channel */
557 WRITE_PORT_UCHAR(&((PDMA2_CONTROL
)AdapterBaseVa
)->SingleMask
,
558 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
563 if (DeviceDescription
->DemandMode
)
565 DmaMode
.RequestMode
= DEMAND_REQUEST_MODE
;
569 DmaMode
.RequestMode
= SINGLE_REQUEST_MODE
;
573 AdapterObject
->AdapterMode
= DmaMode
;
580 * @name HalGetAdapter
582 * Allocate an adapter object for DMA device.
584 * @param DeviceDescription
585 * Structure describing the attributes of the device.
586 * @param NumberOfMapRegisters
587 * On return filled with the maximum number of map registers the
588 * device driver can allocate for DMA transfer operations.
590 * @return The DMA adapter on success, NULL otherwise.
596 HalGetAdapter(IN PDEVICE_DESCRIPTION DeviceDescription
,
597 OUT PULONG NumberOfMapRegisters
)
599 PADAPTER_OBJECT AdapterObject
= NULL
;
600 PADAPTER_OBJECT MasterAdapter
;
605 /* Validate parameters in device description */
606 if (DeviceDescription
->Version
> DEVICE_DESCRIPTION_VERSION2
) return NULL
;
609 * See if we're going to use ISA/EISA DMA adapter. These adapters are
610 * special since they're reused.
612 * Also note that we check for channel number since there are only 8 DMA
613 * channels on ISA, so any request above this requires new adapter.
615 if ((DeviceDescription
->InterfaceType
== Isa
) || !(DeviceDescription
->Master
))
617 if ((DeviceDescription
->InterfaceType
== Isa
) && (DeviceDescription
->DmaChannel
>= 8))
632 * Disallow creating adapter for ISA/EISA DMA channel 4 since it's used
633 * for cascading the controllers and it's not available for software use.
635 if ((EisaAdapter
) && (DeviceDescription
->DmaChannel
== 4)) return NULL
;
638 * Calculate the number of map registers.
640 * - For EISA and PCI scatter/gather no map registers are needed.
641 * - For ISA slave scatter/gather one map register is needed.
642 * - For all other cases the number of map registers depends on
643 * DeviceDescription->MaximumLength.
645 MaximumLength
= DeviceDescription
->MaximumLength
& MAXLONG
;
646 if ((DeviceDescription
->ScatterGather
) &&
647 ((DeviceDescription
->InterfaceType
== Eisa
) ||
648 (DeviceDescription
->InterfaceType
== PCIBus
)))
652 else if ((DeviceDescription
->ScatterGather
) && !(DeviceDescription
->Master
))
659 * In the equation below the additional map register added by
660 * the "+1" accounts for the case when a transfer does not start
661 * at a page-aligned address.
663 MapRegisters
= BYTES_TO_PAGES(MaximumLength
) + 1;
664 if (MapRegisters
> 16) MapRegisters
= 16;
668 * Acquire the DMA lock that is used to protect adapter lists and
669 * EISA adapter array.
671 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
, FALSE
, NULL
);
674 * Now we must get ahold of the adapter object. For first eight ISA/EISA
675 * channels there are static adapter objects that are reused and updated
676 * on succesive HalGetAdapter calls. In other cases a new adapter object
677 * is always created and it's to the DMA adapter list (HalpDmaAdapterList).
681 AdapterObject
= HalpEisaAdapter
[DeviceDescription
->DmaChannel
];
684 if ((AdapterObject
->NeedsMapRegisters
) &&
685 (MapRegisters
> AdapterObject
->MapRegistersPerChannel
))
687 AdapterObject
->MapRegistersPerChannel
= MapRegisters
;
692 if (AdapterObject
== NULL
)
694 AdapterObject
= HalpDmaAllocateChildAdapter(MapRegisters
, DeviceDescription
);
695 if (AdapterObject
== NULL
)
697 KeSetEvent(&HalpDmaLock
, 0, 0);
703 HalpEisaAdapter
[DeviceDescription
->DmaChannel
] = AdapterObject
;
706 if (MapRegisters
> 0)
708 AdapterObject
->NeedsMapRegisters
= TRUE
;
709 MasterAdapter
= HalpMasterAdapter
;
710 AdapterObject
->MapRegistersPerChannel
= MapRegisters
;
713 * FIXME: Verify that the following makes sense. Actually
714 * MasterAdapter->NumberOfMapRegisters contains even the number
715 * of gaps, so this will not work correctly all the time. It
716 * doesn't matter much since it's only optimization to avoid
717 * queuing work items in HalAllocateAdapterChannel.
719 MasterAdapter
->CommittedMapRegisters
+= MapRegisters
;
720 if (MasterAdapter
->CommittedMapRegisters
> MasterAdapter
->NumberOfMapRegisters
)
722 HalpGrowMapBuffers(MasterAdapter
, 0x10000);
727 AdapterObject
->NeedsMapRegisters
= FALSE
;
728 if (DeviceDescription
->Master
)
730 AdapterObject
->MapRegistersPerChannel
= BYTES_TO_PAGES(MaximumLength
) + 1;
734 AdapterObject
->MapRegistersPerChannel
= 1;
739 if (!EisaAdapter
) InsertTailList(&HalpDmaAdapterList
, &AdapterObject
->AdapterList
);
742 * Release the DMA lock. HalpDmaAdapterList and HalpEisaAdapter will
743 * no longer be touched, so we don't need it.
745 KeSetEvent(&HalpDmaLock
, 0, 0);
748 * Setup the values in the adapter object that are common for all
751 if (DeviceDescription
->Version
>= DEVICE_DESCRIPTION_VERSION1
)
753 AdapterObject
->IgnoreCount
= DeviceDescription
->IgnoreCount
;
757 AdapterObject
->IgnoreCount
= 0;
760 AdapterObject
->Dma32BitAddresses
= DeviceDescription
->Dma32BitAddresses
;
761 AdapterObject
->Dma64BitAddresses
= DeviceDescription
->Dma64BitAddresses
;
762 AdapterObject
->ScatterGather
= DeviceDescription
->ScatterGather
;
763 AdapterObject
->MasterDevice
= DeviceDescription
->Master
;
764 *NumberOfMapRegisters
= AdapterObject
->MapRegistersPerChannel
;
767 * For non-(E)ISA adapters we have already done all the work. On the
768 * other hand for (E)ISA adapters we must still setup the DMA modes
769 * and prepare the controller.
773 if (!HalpDmaInitializeEisaAdapter(AdapterObject
, DeviceDescription
))
775 ObDereferenceObject(AdapterObject
);
780 return AdapterObject
;
784 * @name HalpGetDmaAdapter
786 * Internal routine to allocate PnP DMA adapter object. It's exported through
787 * HalDispatchTable and used by IoGetDmaAdapter.
793 HalpGetDmaAdapter(IN PVOID Context
,
794 IN PDEVICE_DESCRIPTION DeviceDescription
,
795 OUT PULONG NumberOfMapRegisters
)
797 return &HalGetAdapter(DeviceDescription
, NumberOfMapRegisters
)->DmaHeader
;
801 * @name HalPutDmaAdapter
803 * Internal routine to free DMA adapter and resources for reuse. It's exported
804 * using the DMA_OPERATIONS interface by HalGetAdapter.
810 HalPutDmaAdapter(IN PADAPTER_OBJECT AdapterObject
)
812 if (AdapterObject
->ChannelNumber
== 0xFF)
814 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
, FALSE
, NULL
);
815 RemoveEntryList(&AdapterObject
->AdapterList
);
816 KeSetEvent(&HalpDmaLock
, 0, 0);
819 ObDereferenceObject(AdapterObject
);
823 * @name HalAllocateCommonBuffer
825 * Allocates memory that is visible to both the processor(s) and the DMA
828 * @param AdapterObject
829 * Adapter object representing the bus master or system dma controller.
831 * Number of bytes to allocate.
832 * @param LogicalAddress
833 * Logical address the driver can use to access the buffer.
834 * @param CacheEnabled
835 * Specifies if the memory can be cached.
837 * @return The base virtual address of the memory allocated or NULL on failure.
840 * On real NT x86 systems the CacheEnabled parameter is ignored, we honour
841 * it. If it proves to cause problems change it.
843 * @see HalFreeCommonBuffer
849 HalAllocateCommonBuffer(IN PADAPTER_OBJECT AdapterObject
,
851 IN PPHYSICAL_ADDRESS LogicalAddress
,
852 IN BOOLEAN CacheEnabled
)
854 PHYSICAL_ADDRESS LowestAcceptableAddress
;
855 PHYSICAL_ADDRESS HighestAcceptableAddress
;
856 PHYSICAL_ADDRESS BoundryAddressMultiple
;
857 PVOID VirtualAddress
;
859 LowestAcceptableAddress
.QuadPart
= 0;
860 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
861 BoundryAddressMultiple
.QuadPart
= 0;
864 * For bus-master DMA devices the buffer mustn't cross 4Gb boundary. For
865 * slave DMA devices the 64Kb boundary mustn't be crossed since the
866 * controller wouldn't be able to handle it.
868 if (AdapterObject
->MasterDevice
)
870 BoundryAddressMultiple
.HighPart
= 1;
874 BoundryAddressMultiple
.LowPart
= 0x10000;
877 VirtualAddress
= MmAllocateContiguousMemorySpecifyCache(Length
,
878 LowestAcceptableAddress
,
879 HighestAcceptableAddress
,
880 BoundryAddressMultiple
,
881 CacheEnabled
? MmCached
:
883 if (VirtualAddress
== NULL
) return NULL
;
885 *LogicalAddress
= MmGetPhysicalAddress(VirtualAddress
);
887 return VirtualAddress
;
891 * @name HalFreeCommonBuffer
893 * Free common buffer allocated with HalAllocateCommonBuffer.
895 * @see HalAllocateCommonBuffer
901 HalFreeCommonBuffer(IN PADAPTER_OBJECT AdapterObject
,
903 IN PHYSICAL_ADDRESS LogicalAddress
,
904 IN PVOID VirtualAddress
,
905 IN BOOLEAN CacheEnabled
)
907 MmFreeContiguousMemorySpecifyCache(VirtualAddress
,
909 CacheEnabled
? MmCached
: MmNonCached
);
914 * @name HalpDmaGetDmaAlignment
916 * Internal routine to return the DMA alignment requirement. It's exported
917 * using the DMA_OPERATIONS interface by HalGetAdapter.
923 HalpDmaGetDmaAlignment(IN PADAPTER_OBJECT AdapterObject
)
929 * @name HalReadDmaCounter
931 * Read DMA operation progress counter.
937 HalReadDmaCounter(IN PADAPTER_OBJECT AdapterObject
)
940 ULONG Count
, OldCount
;
942 ASSERT(!AdapterObject
->MasterDevice
);
945 * Acquire the master adapter lock since we're going to mess with the
946 * system DMA controller registers and we really don't want anyone
947 * to do the same at the same time.
949 KeAcquireSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, &OldIrql
);
951 /* Send the request to the specific controller. */
952 if (AdapterObject
->AdapterNumber
== 1)
954 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
962 WRITE_PORT_UCHAR(&DmaControl1
->ClearBytePointer
, 0);
965 Count
= READ_PORT_UCHAR(&DmaControl1
->DmaAddressCount
966 [AdapterObject
->ChannelNumber
].DmaBaseCount
);
967 Count
|= READ_PORT_UCHAR(&DmaControl1
->DmaAddressCount
968 [AdapterObject
->ChannelNumber
].DmaBaseCount
) << 8;
969 } while (0xffff00 & (OldCount
^ Count
));
973 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
981 WRITE_PORT_UCHAR(&DmaControl2
->ClearBytePointer
, 0);
984 Count
= READ_PORT_UCHAR(&DmaControl2
->DmaAddressCount
985 [AdapterObject
->ChannelNumber
].DmaBaseCount
);
986 Count
|= READ_PORT_UCHAR(&DmaControl2
->DmaAddressCount
987 [AdapterObject
->ChannelNumber
].DmaBaseCount
) << 8;
988 } while (0xffff00 & (OldCount
^ Count
));
991 KeReleaseSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, OldIrql
);
995 if (AdapterObject
->Width16Bits
) Count
*= 2;
1002 * @name HalpGrowMapBufferWorker
1004 * Helper routine of HalAllocateAdapterChannel for allocating map registers
1005 * at PASSIVE_LEVEL in work item.
1009 HalpGrowMapBufferWorker(IN PVOID DeferredContext
)
1011 PGROW_WORK_ITEM WorkItem
= (PGROW_WORK_ITEM
)DeferredContext
;
1016 * Try to allocate new map registers for the adapter.
1018 * NOTE: The NT implementation actually tries to allocate more map
1019 * registers than needed as an optimization.
1021 KeWaitForSingleObject(&HalpDmaLock
, Executive
, KernelMode
, FALSE
, NULL
);
1022 Succeeded
= HalpGrowMapBuffers(WorkItem
->AdapterObject
->MasterAdapter
,
1023 WorkItem
->NumberOfMapRegisters
);
1024 KeSetEvent(&HalpDmaLock
, 0, 0);
1029 * Flush the adapter queue now that new map registers are ready. The
1030 * easiest way to do that is to call IoFreeMapRegisters to not free
1031 * any registers. Note that we use the magic (PVOID)2 map register
1032 * base to bypass the parameter checking.
1034 OldIrql
= KfRaiseIrql(DISPATCH_LEVEL
);
1035 IoFreeMapRegisters(WorkItem
->AdapterObject
, (PVOID
)2, 0);
1036 KfLowerIrql(OldIrql
);
1039 ExFreePool(WorkItem
);
1043 * @name HalAllocateAdapterChannel
1045 * Setup map registers for an adapter object.
1047 * @param AdapterObject
1048 * Pointer to an ADAPTER_OBJECT to set up.
1049 * @param WaitContextBlock
1050 * Context block to be used with ExecutionRoutine.
1051 * @param NumberOfMapRegisters
1052 * Number of map registers requested.
1053 * @param ExecutionRoutine
1054 * Callback to call when map registers are allocated.
1057 * If not enough map registers can be allocated then
1058 * STATUS_INSUFFICIENT_RESOURCES is returned. If the function
1059 * succeeds or the callback is queued for later delivering then
1060 * STATUS_SUCCESS is returned.
1062 * @see IoFreeAdapterChannel
1068 HalAllocateAdapterChannel(IN PADAPTER_OBJECT AdapterObject
,
1069 IN PWAIT_CONTEXT_BLOCK WaitContextBlock
,
1070 IN ULONG NumberOfMapRegisters
,
1071 IN PDRIVER_CONTROL ExecutionRoutine
)
1073 PADAPTER_OBJECT MasterAdapter
;
1074 PGROW_WORK_ITEM WorkItem
;
1075 ULONG Index
= MAXULONG
;
1079 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1081 /* Set up the wait context block in case we can't run right away. */
1082 WaitContextBlock
->DeviceRoutine
= ExecutionRoutine
;
1083 WaitContextBlock
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1085 /* Returns true if queued, else returns false and sets the queue to busy */
1086 if (KeInsertDeviceQueue(&AdapterObject
->ChannelWaitQueue
,
1087 &WaitContextBlock
->WaitQueueEntry
))
1089 return STATUS_SUCCESS
;
1092 MasterAdapter
= AdapterObject
->MasterAdapter
;
1094 AdapterObject
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1095 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1097 if ((NumberOfMapRegisters
) && (AdapterObject
->NeedsMapRegisters
))
1099 if (NumberOfMapRegisters
> AdapterObject
->MapRegistersPerChannel
)
1101 AdapterObject
->NumberOfMapRegisters
= 0;
1102 IoFreeAdapterChannel(AdapterObject
);
1103 return STATUS_INSUFFICIENT_RESOURCES
;
1107 * Get the map registers. This is partly complicated by the fact
1108 * that new map registers can only be allocated at PASSIVE_LEVEL
1109 * and we're currently at DISPATCH_LEVEL. The following code has
1112 * - If there is no adapter queued for map register allocation,
1113 * try to see if enough contiguous map registers are present.
1114 * In case they're we can just get them and proceed further.
1116 * - If some adapter is already present in the queue we must
1117 * respect the order of adapters asking for map registers and
1118 * so the fast case described above can't take place.
1119 * This case is also entered if not enough coniguous map
1120 * registers are present.
1122 * A work queue item is allocated and queued, the adapter is
1123 * also queued into the master adapter queue. The worker
1124 * routine does the job of allocating the map registers at
1125 * PASSIVE_LEVEL and calling the ExecutionRoutine.
1128 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1130 if (IsListEmpty(&MasterAdapter
->AdapterQueue
))
1132 Index
= RtlFindClearBitsAndSet(MasterAdapter
->MapRegisters
, NumberOfMapRegisters
, 0);
1133 if (Index
!= MAXULONG
)
1135 AdapterObject
->MapRegisterBase
= MasterAdapter
->MapRegisterBase
+ Index
;
1136 if (!AdapterObject
->ScatterGather
)
1138 AdapterObject
->MapRegisterBase
= (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)AdapterObject
->MapRegisterBase
| MAP_BASE_SW_SG
);
1143 if (Index
== MAXULONG
)
1145 WorkItem
= ExAllocatePoolWithTag(NonPagedPool
,
1146 sizeof(GROW_WORK_ITEM
),
1150 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1151 AdapterObject
->NumberOfMapRegisters
= 0;
1152 IoFreeAdapterChannel(AdapterObject
);
1153 return STATUS_INSUFFICIENT_RESOURCES
;
1156 InsertTailList(&MasterAdapter
->AdapterQueue
, &AdapterObject
->AdapterQueue
);
1158 ExInitializeWorkItem(&WorkItem
->WorkQueueItem
, HalpGrowMapBufferWorker
, WorkItem
);
1159 WorkItem
->AdapterObject
= AdapterObject
;
1160 WorkItem
->NumberOfMapRegisters
= NumberOfMapRegisters
;
1162 ExQueueWorkItem(&WorkItem
->WorkQueueItem
, DelayedWorkQueue
);
1164 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1166 return STATUS_SUCCESS
;
1169 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1173 AdapterObject
->MapRegisterBase
= NULL
;
1174 AdapterObject
->NumberOfMapRegisters
= 0;
1177 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1179 Result
= ExecutionRoutine(WaitContextBlock
->DeviceObject
,
1180 WaitContextBlock
->CurrentIrp
,
1181 AdapterObject
->MapRegisterBase
,
1182 WaitContextBlock
->DeviceContext
);
1185 * Possible return values:
1188 * Don't free any resources, the ADAPTER_OBJECT is still in use and
1189 * the caller will call IoFreeAdapterChannel later.
1191 * - DeallocateObject
1192 * Deallocate the map registers and release the ADAPTER_OBJECT, so
1193 * someone else can use it.
1195 * - DeallocateObjectKeepRegisters
1196 * Release the ADAPTER_OBJECT, but hang on to the map registers. The
1197 * client will later call IoFreeMapRegisters.
1200 * IoFreeAdapterChannel runs the queue, so it must be called unless
1201 * the adapter object is not to be freed.
1203 if (Result
== DeallocateObject
)
1205 IoFreeAdapterChannel(AdapterObject
);
1207 else if (Result
== DeallocateObjectKeepRegisters
)
1209 AdapterObject
->NumberOfMapRegisters
= 0;
1210 IoFreeAdapterChannel(AdapterObject
);
1213 return STATUS_SUCCESS
;
1217 * @name IoFreeAdapterChannel
1219 * Free DMA resources allocated by IoAllocateAdapterChannel.
1221 * @param AdapterObject
1222 * Adapter object with resources to free.
1225 * This function releases map registers registers assigned to the DMA
1226 * adapter. After releasing the adapter, it checks the adapter's queue
1227 * and runs each queued device object in series until the queue is
1228 * empty. This is the only way the device queue is emptied.
1230 * @see IoAllocateAdapterChannel
1236 IoFreeAdapterChannel(IN PADAPTER_OBJECT AdapterObject
)
1238 PADAPTER_OBJECT MasterAdapter
;
1239 PKDEVICE_QUEUE_ENTRY DeviceQueueEntry
;
1240 PWAIT_CONTEXT_BLOCK WaitContextBlock
;
1241 ULONG Index
= MAXULONG
;
1245 MasterAdapter
= AdapterObject
->MasterAdapter
;
1250 * To keep map registers, call here with AdapterObject->
1251 * NumberOfMapRegisters set to zero. This trick is used in
1252 * HalAllocateAdapterChannel for example.
1254 if (AdapterObject
->NumberOfMapRegisters
)
1256 IoFreeMapRegisters(AdapterObject
,
1257 AdapterObject
->MapRegisterBase
,
1258 AdapterObject
->NumberOfMapRegisters
);
1261 DeviceQueueEntry
= KeRemoveDeviceQueue(&AdapterObject
->ChannelWaitQueue
);
1262 if (!DeviceQueueEntry
) break;
1264 WaitContextBlock
= CONTAINING_RECORD(DeviceQueueEntry
,
1268 AdapterObject
->CurrentWcb
= WaitContextBlock
;
1269 AdapterObject
->NumberOfMapRegisters
= WaitContextBlock
->NumberOfMapRegisters
;
1271 if ((WaitContextBlock
->NumberOfMapRegisters
) && (AdapterObject
->MasterAdapter
))
1273 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1275 if (IsListEmpty(&MasterAdapter
->AdapterQueue
))
1277 Index
= RtlFindClearBitsAndSet(MasterAdapter
->MapRegisters
,
1278 WaitContextBlock
->NumberOfMapRegisters
,
1280 if (Index
!= MAXULONG
)
1282 AdapterObject
->MapRegisterBase
= MasterAdapter
->MapRegisterBase
+ Index
;
1283 if (!AdapterObject
->ScatterGather
)
1285 AdapterObject
->MapRegisterBase
=(PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)AdapterObject
->MapRegisterBase
| MAP_BASE_SW_SG
);
1290 if (Index
== MAXULONG
)
1292 InsertTailList(&MasterAdapter
->AdapterQueue
, &AdapterObject
->AdapterQueue
);
1293 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1297 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1301 AdapterObject
->MapRegisterBase
= NULL
;
1302 AdapterObject
->NumberOfMapRegisters
= 0;
1305 /* Call the adapter control routine. */
1306 Result
= ((PDRIVER_CONTROL
)WaitContextBlock
->DeviceRoutine
)(WaitContextBlock
->DeviceObject
,
1307 WaitContextBlock
->CurrentIrp
,
1308 AdapterObject
->MapRegisterBase
,
1309 WaitContextBlock
->DeviceContext
);
1314 * We're done until the caller manually calls IoFreeAdapterChannel
1315 * or IoFreeMapRegisters.
1319 case DeallocateObjectKeepRegisters
:
1321 * Hide the map registers so they aren't deallocated next time
1324 AdapterObject
->NumberOfMapRegisters
= 0;
1334 * @name IoFreeMapRegisters
1336 * Free map registers reserved by the system for a DMA.
1338 * @param AdapterObject
1339 * DMA adapter to free map registers on.
1340 * @param MapRegisterBase
1341 * Handle to map registers to free.
1342 * @param NumberOfRegisters
1343 * Number of map registers to be freed.
1349 IoFreeMapRegisters(IN PADAPTER_OBJECT AdapterObject
,
1350 IN PVOID MapRegisterBase
,
1351 IN ULONG NumberOfMapRegisters
)
1353 PADAPTER_OBJECT MasterAdapter
= AdapterObject
->MasterAdapter
;
1354 PLIST_ENTRY ListEntry
;
1359 ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL
);
1361 if (!(MasterAdapter
) || !(MapRegisterBase
)) return;
1363 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1365 if (NumberOfMapRegisters
!= 0)
1367 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1369 RealMapRegisterBase
= (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1370 RtlClearBits(MasterAdapter
->MapRegisters
,
1371 RealMapRegisterBase
- MasterAdapter
->MapRegisterBase
,
1372 NumberOfMapRegisters
);
1376 * Now that we freed few map registers it's time to look at the master
1377 * adapter queue and see if there is someone waiting for map registers.
1379 while (!IsListEmpty(&MasterAdapter
->AdapterQueue
))
1381 ListEntry
= RemoveHeadList(&MasterAdapter
->AdapterQueue
);
1382 AdapterObject
= CONTAINING_RECORD(ListEntry
, struct _ADAPTER_OBJECT
, AdapterQueue
);
1384 Index
= RtlFindClearBitsAndSet(MasterAdapter
->MapRegisters
,
1385 AdapterObject
->NumberOfMapRegisters
,
1386 MasterAdapter
->NumberOfMapRegisters
);
1387 if (Index
== MAXULONG
)
1389 InsertHeadList(&MasterAdapter
->AdapterQueue
, ListEntry
);
1393 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1395 AdapterObject
->MapRegisterBase
= MasterAdapter
->MapRegisterBase
+ Index
;
1396 if (!AdapterObject
->ScatterGather
)
1398 AdapterObject
->MapRegisterBase
=
1399 (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)AdapterObject
->MapRegisterBase
| MAP_BASE_SW_SG
);
1402 Result
= ((PDRIVER_CONTROL
)AdapterObject
->CurrentWcb
->DeviceRoutine
)(AdapterObject
->CurrentWcb
->DeviceObject
,
1403 AdapterObject
->CurrentWcb
->CurrentIrp
,
1404 AdapterObject
->MapRegisterBase
,
1405 AdapterObject
->CurrentWcb
->DeviceContext
);
1408 case DeallocateObjectKeepRegisters
:
1409 AdapterObject
->NumberOfMapRegisters
= 0;
1412 case DeallocateObject
:
1413 if (AdapterObject
->NumberOfMapRegisters
)
1415 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1416 RtlClearBits(MasterAdapter
->MapRegisters
,
1417 AdapterObject
->MapRegisterBase
-
1418 MasterAdapter
->MapRegisterBase
,
1419 AdapterObject
->NumberOfMapRegisters
);
1420 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1423 IoFreeAdapterChannel(AdapterObject
);
1430 KeAcquireSpinLock(&MasterAdapter
->SpinLock
, &OldIrql
);
1433 KeReleaseSpinLock(&MasterAdapter
->SpinLock
, OldIrql
);
1437 * @name HalpCopyBufferMap
1439 * Helper function for copying data from/to map register buffers.
1441 * @see IoFlushAdapterBuffers, IoMapTransfer
1445 HalpCopyBufferMap(IN PMDL Mdl
,
1446 IN PROS_MAP_REGISTER_ENTRY MapRegisterBase
,
1449 IN BOOLEAN WriteToDevice
)
1451 ULONG CurrentLength
;
1452 ULONG_PTR CurrentAddress
;
1454 PVOID VirtualAddress
;
1456 VirtualAddress
= MmGetSystemAddressForMdlSafe(Mdl
, HighPagePriority
);
1457 if (!VirtualAddress
)
1460 * NOTE: On real NT a mechanism with reserved pages is implemented
1461 * to handle this case in a slow, but graceful non-fatal way.
1463 KeBugCheckEx(HAL_MEMORY_ALLOCATION
, PAGE_SIZE
, 0, (ULONG_PTR
)__FILE__
, 0);
1466 CurrentAddress
= (ULONG_PTR
)VirtualAddress
+
1467 (ULONG_PTR
)CurrentVa
-
1468 (ULONG_PTR
)MmGetMdlVirtualAddress(Mdl
);
1472 ByteOffset
= BYTE_OFFSET(CurrentAddress
);
1473 CurrentLength
= PAGE_SIZE
- ByteOffset
;
1474 if (CurrentLength
> Length
) CurrentLength
= Length
;
1478 RtlCopyMemory((PVOID
)((ULONG_PTR
)MapRegisterBase
->VirtualAddress
+ ByteOffset
),
1479 (PVOID
)CurrentAddress
,
1484 RtlCopyMemory((PVOID
)CurrentAddress
,
1485 (PVOID
)((ULONG_PTR
)MapRegisterBase
->VirtualAddress
+ ByteOffset
),
1489 Length
-= CurrentLength
;
1490 CurrentAddress
+= CurrentLength
;
1496 * @name IoFlushAdapterBuffers
1498 * Flush any data remaining in the DMA controller's memory into the host
1501 * @param AdapterObject
1502 * The adapter object to flush.
1504 * Original MDL to flush data into.
1505 * @param MapRegisterBase
1506 * Map register base that was just used by IoMapTransfer, etc.
1508 * Offset into Mdl to be flushed into, same as was passed to
1511 * Length of the buffer to be flushed into.
1512 * @param WriteToDevice
1513 * TRUE if it's a write, FALSE if it's a read.
1515 * @return TRUE in all cases.
1518 * This copies data from the map register-backed buffer to the user's
1519 * target buffer. Data are not in the user buffer until this function
1521 * For slave DMA transfers the controller channel is masked effectively
1522 * stopping the current transfer.
1528 IoFlushAdapterBuffers(IN PADAPTER_OBJECT AdapterObject
,
1530 IN PVOID MapRegisterBase
,
1533 IN BOOLEAN WriteToDevice
)
1535 BOOLEAN SlaveDma
= FALSE
;
1536 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1537 PHYSICAL_ADDRESS HighestAcceptableAddress
;
1538 PHYSICAL_ADDRESS PhysicalAddress
;
1539 PPFN_NUMBER MdlPagesPtr
;
1542 ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL
);
1543 ASSERT(AdapterObject
);
1545 if (!AdapterObject
->MasterDevice
)
1547 /* Mask out (disable) the DMA channel. */
1548 if (AdapterObject
->AdapterNumber
== 1)
1550 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
1551 WRITE_PORT_UCHAR(&DmaControl1
->SingleMask
,
1552 AdapterObject
->ChannelNumber
| DMA_SETMASK
);
1556 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
1557 WRITE_PORT_UCHAR(&DmaControl2
->SingleMask
,
1558 AdapterObject
->ChannelNumber
| DMA_SETMASK
);
1563 /* This can happen if the device supports hardware scatter/gather. */
1564 if (MapRegisterBase
== NULL
) return TRUE
;
1566 RealMapRegisterBase
= (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1570 if ((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
)
1572 if (RealMapRegisterBase
->Counter
!= MAXULONG
)
1574 if ((SlaveDma
) && !(AdapterObject
->IgnoreCount
))
1576 Length
-= HalReadDmaCounter(AdapterObject
);
1579 HalpCopyBufferMap(Mdl
,
1580 RealMapRegisterBase
,
1587 MdlPagesPtr
= MmGetMdlPfnArray(Mdl
);
1588 MdlPagesPtr
+= ((ULONG_PTR
)CurrentVa
- (ULONG_PTR
)Mdl
->StartVa
) >> PAGE_SHIFT
;
1590 PhysicalAddress
.QuadPart
= *MdlPagesPtr
<< PAGE_SHIFT
;
1591 PhysicalAddress
.QuadPart
+= BYTE_OFFSET(CurrentVa
);
1593 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
1594 if ((PhysicalAddress
.QuadPart
+ Length
) > HighestAcceptableAddress
.QuadPart
)
1596 HalpCopyBufferMap(Mdl
,
1597 RealMapRegisterBase
,
1605 RealMapRegisterBase
->Counter
= 0;
1611 * @name IoMapTransfer
1613 * Map a DMA for transfer and do the DMA if it's a slave.
1615 * @param AdapterObject
1616 * Adapter object to do the DMA on. Bus-master may pass NULL.
1618 * Locked-down user buffer to DMA in to or out of.
1619 * @param MapRegisterBase
1620 * Handle to map registers to use for this dma.
1622 * Index into Mdl to transfer into/out of.
1624 * Length of transfer. Number of bytes actually transferred on
1626 * @param WriteToDevice
1627 * TRUE if it's an output DMA, FALSE otherwise.
1630 * A logical address that can be used to program a DMA controller, it's
1631 * not meaningful for slave DMA device.
1634 * This function does a copyover to contiguous memory <16MB represented
1635 * by the map registers if needed. If the buffer described by MDL can be
1636 * used as is no copyover is done.
1637 * If it's a slave transfer, this function actually performs it.
1643 IoMapTransfer(IN PADAPTER_OBJECT AdapterObject
,
1645 IN PVOID MapRegisterBase
,
1647 IN OUT PULONG Length
,
1648 IN BOOLEAN WriteToDevice
)
1650 PPFN_NUMBER MdlPagesPtr
;
1651 PFN_NUMBER MdlPage1
, MdlPage2
;
1653 ULONG TransferOffset
;
1654 ULONG TransferLength
;
1655 BOOLEAN UseMapRegisters
;
1656 PROS_MAP_REGISTER_ENTRY RealMapRegisterBase
;
1657 PHYSICAL_ADDRESS PhysicalAddress
;
1658 PHYSICAL_ADDRESS HighestAcceptableAddress
;
1660 DMA_MODE AdapterMode
;
1664 * Precalculate some values that are used in all cases.
1666 * ByteOffset is offset inside the page at which the transfer starts.
1667 * MdlPagesPtr is pointer inside the MDL page chain at the page where the
1669 * PhysicalAddress is physical address corresponding to the transfer
1670 * start page and offset.
1671 * TransferLength is the inital length of the transfer, which is reminder
1672 * of the first page. The actual value is calculated below.
1674 * Note that all the variables can change during the processing which
1675 * takes place below. These are just initial values.
1677 ByteOffset
= BYTE_OFFSET(CurrentVa
);
1679 MdlPagesPtr
= MmGetMdlPfnArray(Mdl
);
1680 MdlPagesPtr
+= ((ULONG_PTR
)CurrentVa
- (ULONG_PTR
)Mdl
->StartVa
) >> PAGE_SHIFT
;
1682 PhysicalAddress
.QuadPart
= *MdlPagesPtr
<< PAGE_SHIFT
;
1683 PhysicalAddress
.QuadPart
+= ByteOffset
;
1685 TransferLength
= PAGE_SIZE
- ByteOffset
;
1688 * Special case for bus master adapters with S/G support. We can directly
1689 * use the buffer specified by the MDL, so not much work has to be done.
1691 * Just return the passed VA's corresponding physical address and update
1692 * length to the number of physically contiguous bytes found. Also
1693 * pages crossing the 4Gb boundary aren't considered physically contiguous.
1695 if (MapRegisterBase
== NULL
)
1697 while (TransferLength
< *Length
)
1699 MdlPage1
= *MdlPagesPtr
;
1700 MdlPage2
= *(MdlPagesPtr
+ 1);
1701 if (MdlPage1
+ 1 != MdlPage2
) break;
1702 if ((MdlPage1
^ MdlPage2
) & ~0xFFFFF) break;
1703 TransferLength
+= PAGE_SIZE
;
1707 if (TransferLength
< *Length
) *Length
= TransferLength
;
1709 return PhysicalAddress
;
1713 * The code below applies to slave DMA adapters and bus master adapters
1714 * without hardward S/G support.
1716 RealMapRegisterBase
= (PROS_MAP_REGISTER_ENTRY
)((ULONG_PTR
)MapRegisterBase
& ~MAP_BASE_SW_SG
);
1719 * Try to calculate the size of the transfer. We can only transfer
1720 * pages that are physically contiguous and that don't cross the
1721 * 64Kb boundary (this limitation applies only for ISA controllers).
1723 while (TransferLength
< *Length
)
1725 MdlPage1
= *MdlPagesPtr
;
1726 MdlPage2
= *(MdlPagesPtr
+ 1);
1727 if (MdlPage1
+ 1 != MdlPage2
) break;
1728 if (!HalpEisaDma
&& ((MdlPage1
^ MdlPage2
) & ~0xF)) break;
1729 TransferLength
+= PAGE_SIZE
;
1733 if (TransferLength
> *Length
) TransferLength
= *Length
;
1736 * If we're about to simulate software S/G and not all the pages are
1737 * physically contiguous then we must use the map registers to store
1738 * the data and allow the whole transfer to proceed at once.
1740 if (((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
) && (TransferLength
< *Length
))
1742 UseMapRegisters
= TRUE
;
1743 PhysicalAddress
= RealMapRegisterBase
->PhysicalAddress
;
1744 PhysicalAddress
.QuadPart
+= ByteOffset
;
1745 TransferLength
= *Length
;
1746 RealMapRegisterBase
->Counter
= MAXULONG
;
1752 * This is ordinary DMA transfer, so just update the progress
1753 * counters. These are used by IoFlushAdapterBuffers to track
1754 * the transfer progress.
1756 UseMapRegisters
= FALSE
;
1757 Counter
= RealMapRegisterBase
->Counter
;
1758 RealMapRegisterBase
->Counter
+= BYTES_TO_PAGES(ByteOffset
+ TransferLength
);
1761 * Check if the buffer doesn't exceed the highest physical address
1762 * limit of the device. In that case we must use the map registers to
1765 HighestAcceptableAddress
= HalpGetAdapterMaximumPhysicalAddress(AdapterObject
);
1766 if ((PhysicalAddress
.QuadPart
+ TransferLength
) > HighestAcceptableAddress
.QuadPart
)
1768 UseMapRegisters
= TRUE
;
1769 PhysicalAddress
= RealMapRegisterBase
[Counter
].PhysicalAddress
;
1770 PhysicalAddress
.QuadPart
+= ByteOffset
;
1771 if ((ULONG_PTR
)MapRegisterBase
& MAP_BASE_SW_SG
)
1773 RealMapRegisterBase
->Counter
= MAXULONG
;
1780 * If we decided to use the map registers (see above) and we're about
1781 * to transfer data to the device then copy the buffers into the map
1784 if ((UseMapRegisters
) && (WriteToDevice
))
1786 HalpCopyBufferMap(Mdl
,
1787 RealMapRegisterBase
+ Counter
,
1794 * Return the length of transfer that actually takes place.
1796 *Length
= TransferLength
;
1799 * If we're doing slave (system) DMA then program the (E)ISA controller
1800 * to actually start the transfer.
1802 if ((AdapterObject
) && !(AdapterObject
->MasterDevice
))
1804 AdapterMode
= AdapterObject
->AdapterMode
;
1808 AdapterMode
.TransferType
= WRITE_TRANSFER
;
1812 AdapterMode
.TransferType
= READ_TRANSFER
;
1813 if (AdapterObject
->IgnoreCount
)
1815 RtlZeroMemory((PUCHAR
)RealMapRegisterBase
[Counter
].VirtualAddress
+ ByteOffset
,
1820 TransferOffset
= PhysicalAddress
.LowPart
& 0xFFFF;
1821 if (AdapterObject
->Width16Bits
)
1823 TransferLength
>>= 1;
1824 TransferOffset
>>= 1;
1827 KeAcquireSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, &OldIrql
);
1829 if (AdapterObject
->AdapterNumber
== 1)
1831 PDMA1_CONTROL DmaControl1
= AdapterObject
->AdapterBaseVa
;
1833 /* Reset Register */
1834 WRITE_PORT_UCHAR(&DmaControl1
->ClearBytePointer
, 0);
1837 WRITE_PORT_UCHAR(&DmaControl1
->Mode
, AdapterMode
.Byte
);
1839 /* Set the Offset Register */
1840 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1841 (UCHAR
)(TransferOffset
));
1842 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1843 (UCHAR
)(TransferOffset
>> 8));
1845 /* Set the Page Register */
1846 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+ FIELD_OFFSET(EISA_CONTROL
, DmaController1Pages
),
1847 (UCHAR
)(PhysicalAddress
.LowPart
>> 16));
1850 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+ FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
),
1854 /* Set the Length */
1855 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1856 (UCHAR
)(TransferLength
- 1));
1857 WRITE_PORT_UCHAR(&DmaControl1
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1858 (UCHAR
)((TransferLength
- 1) >> 8));
1860 /* Unmask the Channel */
1861 WRITE_PORT_UCHAR(&DmaControl1
->SingleMask
, AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
1865 PDMA2_CONTROL DmaControl2
= AdapterObject
->AdapterBaseVa
;
1867 /* Reset Register */
1868 WRITE_PORT_UCHAR(&DmaControl2
->ClearBytePointer
, 0);
1871 WRITE_PORT_UCHAR(&DmaControl2
->Mode
, AdapterMode
.Byte
);
1873 /* Set the Offset Register */
1874 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1875 (UCHAR
)(TransferOffset
));
1876 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseAddress
,
1877 (UCHAR
)(TransferOffset
>> 8));
1879 /* Set the Page Register */
1880 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+ FIELD_OFFSET(EISA_CONTROL
, DmaController1Pages
),
1881 (UCHAR
)(PhysicalAddress
.u
.LowPart
>> 16));
1884 WRITE_PORT_UCHAR(AdapterObject
->PagePort
+ FIELD_OFFSET(EISA_CONTROL
, DmaController2Pages
),
1888 /* Set the Length */
1889 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1890 (UCHAR
)(TransferLength
- 1));
1891 WRITE_PORT_UCHAR(&DmaControl2
->DmaAddressCount
[AdapterObject
->ChannelNumber
].DmaBaseCount
,
1892 (UCHAR
)((TransferLength
- 1) >> 8));
1894 /* Unmask the Channel */
1895 WRITE_PORT_UCHAR(&DmaControl2
->SingleMask
,
1896 AdapterObject
->ChannelNumber
| DMA_CLEARMASK
);
1899 KeReleaseSpinLock(&AdapterObject
->MasterAdapter
->SpinLock
, OldIrql
);
1903 * Return physical address of the buffer with data that is used for the
1904 * transfer. It can either point inside the Mdl that was passed by the
1905 * caller or into the map registers if the Mdl buffer can't be used
1908 return PhysicalAddress
;
1913 * @name HalFlushCommonBuffer
1919 HalFlushCommonBuffer(IN PADAPTER_OBJECT AdapterObject
,
1921 IN PHYSICAL_ADDRESS LogicalAddress
,
1922 IN PVOID VirtualAddress
)
1924 /* Function always returns true */
1933 HalAllocateCrashDumpRegisters(IN PADAPTER_OBJECT AdapterObject
,
1934 IN OUT PULONG NumberOfMapRegisters
)
1936 PADAPTER_OBJECT MasterAdapter
= AdapterObject
->MasterAdapter
;
1937 ULONG MapRegisterNumber
;
1939 /* Check if it needs map registers */
1940 if (AdapterObject
->NeedsMapRegisters
)
1942 /* Check if we have enough */
1943 if (*NumberOfMapRegisters
> AdapterObject
->MapRegistersPerChannel
)
1945 /* We don't, fail */
1946 AdapterObject
->NumberOfMapRegisters
= 0;
1950 /* Try to find free map registers */
1951 MapRegisterNumber
= RtlFindClearBitsAndSet(MasterAdapter
->MapRegisters
,
1952 *NumberOfMapRegisters
,
1955 /* Check if nothing was found */
1956 if (MapRegisterNumber
== MAXULONG
)
1958 /* No free registers found, so use the base registers */
1959 RtlSetBits(MasterAdapter
->MapRegisters
,
1961 *NumberOfMapRegisters
);
1962 MapRegisterNumber
= 0;
1965 /* Calculate the new base */
1966 AdapterObject
->MapRegisterBase
=
1967 (PROS_MAP_REGISTER_ENTRY
)(MasterAdapter
->MapRegisterBase
+
1970 /* Check if scatter gather isn't supported */
1971 if (!AdapterObject
->ScatterGather
)
1974 AdapterObject
->MapRegisterBase
=
1975 (PROS_MAP_REGISTER_ENTRY
)
1976 ((ULONG_PTR
)AdapterObject
->MapRegisterBase
| MAP_BASE_SW_SG
);
1981 AdapterObject
->MapRegisterBase
= NULL
;
1982 AdapterObject
->NumberOfMapRegisters
= 0;
1985 /* Return the base */
1986 return AdapterObject
->MapRegisterBase
;