hal/halx86/generic/dma.c

   1 /*
   2  *
   3  * COPYRIGHT:       See COPYING in the top level directory
   4  * PROJECT:         ReactOS kernel
   5  * FILE:            ntoskrnl/hal/x86/dma.c
   6  * PURPOSE:         DMA functions
   7  * PROGRAMMERS:     David Welch (welch@mcmail.com)
   8  *                  Filip Navara (navaraf@reactos.com)
   9  * UPDATE HISTORY:
  10  *                  Created 22/05/98
  11  */
  12
  13 /**
  14  * @page DMA Implementation Notes
  15  *
  16  * Concepts:
  17  *
  18  * - Map register
  19  *
  20  *   Abstract encapsulation of physically contiguous buffer that resides
  21  *   in memory accessible by both the DMA device / controller and the system.
  22  *   The map registers are allocated and distributed on demand and are
  23  *   scarce resource.
  24  *
  25  *   The actual use of map registers is to allow transfers from/to buffer
  26  *   located in physical memory at address inaccessible by the DMA device /
  27  *   controller directly. For such transfers the map register buffers
  28  *   are used as intermediate data storage.
  29  *
  30  * - Master adapter
  31  *
  32  *   A container for map registers (typically corresponding to one physical
  33  *   bus connection type). There can be master adapters for 24-bit address
  34  *   ranges, 32-bit address ranges, etc. Every time a new DMA adapter is
  35  *   created it's associated with a corresponding master adapter that
  36  *   is used for any map register allocation requests.
  37  *
  38  * - Bus-master / Slave DMA
  39  *
  40  *   Slave DMA is term used for DMA transfers done by the system (E)ISA
  41  *   controller as opposed to transfers mastered by the device itself
  42  *   (hence the name).
  43  *
  44  *   For slave DMA special care is taken to actually access the system
  45  *   controller and handle the transfers. The relevant code is in
  46  *   HalpDmaInitializeEisaAdapter, HalReadDmaCounter, IoFlushAdapterBuffers
  47  *   and IoMapTransfer.
  48  *
  49  * Implementation:
  50  *
  51  * - Allocation of map registers
  52  *
  53  *   Initial set of map registers is allocated on the system start to
  54  *   ensure that low memory won't get filled up later. Additional map
  55  *   registers are allocated as needed by HalpGrowMapBuffers. This
  56  *   routine is called on two places:
  57  *
  58  *   - HalGetAdapter, since we're at PASSIVE_LEVEL and it's known that
  59  *     more map registers will probably be needed.
  60  *   - IoAllocateAdapterChannel (indirectly using HalpGrowMapBufferWorker
  61  *     since we're at DISPATCH_LEVEL and call HalpGrowMapBuffers directly)
  62  *     when no more map registers are free.
  63  *
  64  *   Note that even if no more map registers can be allocated it's not
  65  *   the end of the world. The adapters waiting for free map registers
  66  *   are queued in the master adapter's queue and once one driver hands
  67  *   back it's map registers (using IoFreeMapRegisters or indirectly using
  68  *   the execution routine callback in IoAllocateAdapterChannel) the
  69  *   queue gets processed and the map registers are reassigned.
  70  */
  71
  72 /* INCLUDES *****************************************************************/
  73
  74 #include <hal.h>
  75 #include <suppress.h>
  76
  77 #define NDEBUG
  78 #include <debug.h>
  79
  80 #define MAX_SG_ELEMENTS 0x10
  81
  82 #ifndef _MINIHAL_
  83 static KEVENT HalpDmaLock;
  84 static LIST_ENTRY HalpDmaAdapterList;
  85 static PADAPTER_OBJECT HalpEisaAdapter[8];
  86 #endif
  87 static BOOLEAN HalpEisaDma;
  88 #ifndef _MINIHAL_
  89 static PADAPTER_OBJECT HalpMasterAdapter;
  90 #endif
  91
  92 static const ULONG_PTR HalpEisaPortPage[8] = {
  93    FIELD_OFFSET(DMA_PAGE, Channel0),
  94    FIELD_OFFSET(DMA_PAGE, Channel1),
  95    FIELD_OFFSET(DMA_PAGE, Channel2),
  96    FIELD_OFFSET(DMA_PAGE, Channel3),
  97    0,
  98    FIELD_OFFSET(DMA_PAGE, Channel5),
  99    FIELD_OFFSET(DMA_PAGE, Channel6),
 100    FIELD_OFFSET(DMA_PAGE, Channel7)
 101 };
 102
 103 #ifndef _MINIHAL_
 104 static DMA_OPERATIONS HalpDmaOperations = {
 105    sizeof(DMA_OPERATIONS),
 106    (PPUT_DMA_ADAPTER)HalPutDmaAdapter,
 107    (PALLOCATE_COMMON_BUFFER)HalAllocateCommonBuffer,
 108    (PFREE_COMMON_BUFFER)HalFreeCommonBuffer,
 109    NULL, /* Initialized in HalpInitDma() */
 110    NULL, /* Initialized in HalpInitDma() */
 111    NULL, /* Initialized in HalpInitDma() */
 112    NULL, /* Initialized in HalpInitDma() */
 113    NULL, /* Initialized in HalpInitDma() */
 114    (PGET_DMA_ALIGNMENT)HalpDmaGetDmaAlignment,
 115    (PREAD_DMA_COUNTER)HalReadDmaCounter,
 116    /* FIXME: Implement the S/G funtions. */
 117    (PGET_SCATTER_GATHER_LIST)HalGetScatterGatherList,
 118    (PPUT_SCATTER_GATHER_LIST)HalPutScatterGatherList,
 119    NULL /*(PCALCULATE_SCATTER_GATHER_LIST_SIZE)HalCalculateScatterGatherListSize*/,
 120    NULL /*(PBUILD_SCATTER_GATHER_LIST)HalBuildScatterGatherList*/,
 121    NULL /*(PBUILD_MDL_FROM_SCATTER_GATHER_LIST)HalBuildMdlFromScatterGatherList*/
 122 };
 123 #endif
 124
 125 #define MAX_MAP_REGISTERS 64
 126
 127 #define TAG_DMA ' AMD'
 128
 129 /* FUNCTIONS *****************************************************************/
 130
 131 #ifndef _MINIHAL_
 132 VOID
 133 INIT_FUNCTION
 134 HalpInitDma(VOID)
 135 {
 136     /*
 137      * Initialize the DMA Operation table
 138      */
 139     HalpDmaOperations.AllocateAdapterChannel = (PALLOCATE_ADAPTER_CHANNEL)IoAllocateAdapterChannel;
 140     HalpDmaOperations.FlushAdapterBuffers = (PFLUSH_ADAPTER_BUFFERS)IoFlushAdapterBuffers;
 141     HalpDmaOperations.FreeAdapterChannel = (PFREE_ADAPTER_CHANNEL)IoFreeAdapterChannel;
 142     HalpDmaOperations.FreeMapRegisters = (PFREE_MAP_REGISTERS)IoFreeMapRegisters;
 143     HalpDmaOperations.MapTransfer = (PMAP_TRANSFER)IoMapTransfer;
 144
 145     /*
 146      * Check if Extended DMA is available. We're just going to do a random
 147      * read and write.
 148      */
 149     WRITE_PORT_UCHAR((PUCHAR)FIELD_OFFSET(EISA_CONTROL, DmaController2Pages.Channel2), 0x2A);
 150     if (READ_PORT_UCHAR((PUCHAR)FIELD_OFFSET(EISA_CONTROL, DmaController2Pages.Channel2)) == 0x2A)
 151     {
 152         HalpEisaDma = TRUE;
 153     }
 154
 155     /*
 156      * Intialize all the global variables and allocate master adapter with
 157      * first map buffers.
 158      */
 159     InitializeListHead(&HalpDmaAdapterList);
 160     KeInitializeEvent(&HalpDmaLock, NotificationEvent, TRUE);
 161     HalpMasterAdapter = HalpDmaAllocateMasterAdapter();
 162
 163     /*
 164      * Setup the HalDispatchTable callback for creating PnP DMA adapters. It's
 165      * used by IoGetDmaAdapter in the kernel.
 166      */
 167     HalGetDmaAdapter = HalpGetDmaAdapter;
 168 }
 169 #endif
 170
 171 /**
 172  * @name HalpGetAdapterMaximumPhysicalAddress
 173  *
 174  * Get the maximum physical address acceptable by the device represented
 175  * by the passed DMA adapter.
 176  */
 177 PHYSICAL_ADDRESS
 178 NTAPI
 179 HalpGetAdapterMaximumPhysicalAddress(IN PADAPTER_OBJECT AdapterObject)
 180 {
 181     PHYSICAL_ADDRESS HighestAddress;
 182
 183     if (AdapterObject->MasterDevice)
 184     {
 185         if (AdapterObject->Dma64BitAddresses)
 186         {
 187             HighestAddress.QuadPart = 0xFFFFFFFFFFFFFFFFULL;
 188             return HighestAddress;
 189         }
 190         else if (AdapterObject->Dma32BitAddresses)
 191         {
 192             HighestAddress.QuadPart = 0xFFFFFFFF;
 193             return HighestAddress;
 194         }
 195     }
 196
 197     HighestAddress.QuadPart = 0xFFFFFF;
 198     return HighestAddress;
 199 }
 200
 201 #ifndef _MINIHAL_
 202 /**
 203  * @name HalpGrowMapBuffers
 204  *
 205  * Allocate initial, or additional, map buffers for DMA master adapter.
 206  *
 207  * @param MasterAdapter
 208  *        DMA master adapter to allocate buffers for.
 209  * @param SizeOfMapBuffers
 210  *        Size of the map buffers to allocate (not including the size
 211  *        already allocated).
 212  */
 213 BOOLEAN
 214 NTAPI
 215 HalpGrowMapBuffers(IN PADAPTER_OBJECT AdapterObject,
 216                   IN ULONG SizeOfMapBuffers)
 217 {
 218     PVOID VirtualAddress;
 219     PHYSICAL_ADDRESS PhysicalAddress;
 220     PHYSICAL_ADDRESS HighestAcceptableAddress;
 221     PHYSICAL_ADDRESS LowestAcceptableAddress;
 222     PHYSICAL_ADDRESS BoundryAddressMultiple;
 223     KIRQL OldIrql;
 224     ULONG MapRegisterCount;
 225
 226     /* Check if enough map register slots are available. */
 227     MapRegisterCount = BYTES_TO_PAGES(SizeOfMapBuffers);
 228     if (MapRegisterCount + AdapterObject->NumberOfMapRegisters > MAX_MAP_REGISTERS)
 229     {
 230         DPRINT("No more map register slots available! (Current: %d | Requested: %d | Limit: %d)\n",
 231                AdapterObject->NumberOfMapRegisters,
 232                MapRegisterCount,
 233                MAX_MAP_REGISTERS);
 234         return FALSE;
 235     }
 236
 237     /*
 238      * Allocate memory for the new map registers. For 32-bit adapters we use
 239      * two passes in order not to waste scare resource (low memory).
 240      */
 241     HighestAcceptableAddress = HalpGetAdapterMaximumPhysicalAddress(AdapterObject);
 242     LowestAcceptableAddress.HighPart = 0;
 243     LowestAcceptableAddress.LowPart = HighestAcceptableAddress.LowPart == 0xFFFFFFFF ? 0x1000000 : 0;
 244     BoundryAddressMultiple.QuadPart = 0;
 245
 246     VirtualAddress = MmAllocateContiguousMemorySpecifyCache(MapRegisterCount << PAGE_SHIFT,
 247                                                             LowestAcceptableAddress,
 248                                                             HighestAcceptableAddress,
 249                                                             BoundryAddressMultiple,
 250                                                             MmNonCached);
 251     if (!(VirtualAddress) && (LowestAcceptableAddress.LowPart))
 252     {
 253         LowestAcceptableAddress.LowPart = 0;
 254         VirtualAddress = MmAllocateContiguousMemorySpecifyCache(MapRegisterCount << PAGE_SHIFT,
 255                                                                 LowestAcceptableAddress,
 256                                                                 HighestAcceptableAddress,
 257                                                                 BoundryAddressMultiple,
 258                                                                 MmNonCached);
 259     }
 260
 261     if (!VirtualAddress) return FALSE;
 262
 263     PhysicalAddress = MmGetPhysicalAddress(VirtualAddress);
 264
 265     /*
 266      * All the following must be done with the master adapter lock held
 267      * to prevent corruption.
 268      */
 269     KeAcquireSpinLock(&AdapterObject->SpinLock, &OldIrql);
 270
 271     /*
 272      * Setup map register entries for the buffer allocated. Each entry has
 273      * a virtual and physical address and corresponds to PAGE_SIZE large
 274      * buffer.
 275      */
 276     if (MapRegisterCount > 0)
 277     {
 278         PROS_MAP_REGISTER_ENTRY CurrentEntry, PreviousEntry;
 279
 280         CurrentEntry = AdapterObject->MapRegisterBase + AdapterObject->NumberOfMapRegisters;
 281         do
 282         {
 283             /*
 284              * Leave one entry free for every non-contiguous memory region
 285              * in the map register bitmap. This ensures that we can search
 286              * using RtlFindClearBits for contiguous map register regions.
 287              *
 288              * Also for non-EISA DMA leave one free entry for every 64Kb
 289              * break, because the DMA controller can handle only coniguous
 290              * 64Kb regions.
 291              */
 292             if (CurrentEntry != AdapterObject->MapRegisterBase)
 293             {
 294                 PreviousEntry = CurrentEntry - 1;
 295                 if ((PreviousEntry->PhysicalAddress.LowPart + PAGE_SIZE) == PhysicalAddress.LowPart)
 296                 {
 297                     if (!HalpEisaDma)
 298                     {
 299                         if ((PreviousEntry->PhysicalAddress.LowPart ^ PhysicalAddress.LowPart) & 0xFFFF0000)
 300                         {
 301                             CurrentEntry++;
 302                             AdapterObject->NumberOfMapRegisters++;
 303                         }
 304                     }
 305                 }
 306                 else
 307                 {
 308                     CurrentEntry++;
 309                     AdapterObject->NumberOfMapRegisters++;
 310                 }
 311             }
 312
 313             RtlClearBit(AdapterObject->MapRegisters,
 314                         (ULONG)(CurrentEntry - AdapterObject->MapRegisterBase));
 315             CurrentEntry->VirtualAddress = VirtualAddress;
 316             CurrentEntry->PhysicalAddress = PhysicalAddress;
 317
 318             PhysicalAddress.LowPart += PAGE_SIZE;
 319             VirtualAddress = (PVOID)((ULONG_PTR)VirtualAddress + PAGE_SIZE);
 320
 321             CurrentEntry++;
 322             AdapterObject->NumberOfMapRegisters++;
 323             MapRegisterCount--;
 324         } while (MapRegisterCount);
 325     }
 326
 327     KeReleaseSpinLock(&AdapterObject->SpinLock, OldIrql);
 328
 329     return TRUE;
 330 }
 331
 332 /**
 333  * @name HalpDmaAllocateMasterAdapter
 334  *
 335  * Helper routine to allocate and initialize master adapter object and it's
 336  * associated map register buffers.
 337  *
 338  * @see HalpInitDma
 339  */
 340 PADAPTER_OBJECT
 341 NTAPI
 342 HalpDmaAllocateMasterAdapter(VOID)
 343 {
 344     PADAPTER_OBJECT MasterAdapter;
 345     ULONG Size, SizeOfBitmap;
 346
 347     SizeOfBitmap = MAX_MAP_REGISTERS;
 348     Size = sizeof(ADAPTER_OBJECT);
 349     Size += sizeof(RTL_BITMAP);
 350     Size += (SizeOfBitmap + 7) >> 3;
 351
 352     MasterAdapter = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_DMA);
 353     if (!MasterAdapter) return NULL;
 354
 355     RtlZeroMemory(MasterAdapter, Size);
 356
 357     KeInitializeSpinLock(&MasterAdapter->SpinLock);
 358     InitializeListHead(&MasterAdapter->AdapterQueue);
 359
 360     MasterAdapter->MapRegisters = (PVOID)(MasterAdapter + 1);
 361     RtlInitializeBitMap(MasterAdapter->MapRegisters,
 362                         (PULONG)(MasterAdapter->MapRegisters + 1),
 363                         SizeOfBitmap);
 364     RtlSetAllBits(MasterAdapter->MapRegisters);
 365     MasterAdapter->NumberOfMapRegisters = 0;
 366     MasterAdapter->CommittedMapRegisters = 0;
 367
 368     MasterAdapter->MapRegisterBase = ExAllocatePoolWithTag(NonPagedPool,
 369                                                            SizeOfBitmap *
 370                                                            sizeof(ROS_MAP_REGISTER_ENTRY),
 371                                                            TAG_DMA);
 372     if (!MasterAdapter->MapRegisterBase)
 373     {
 374         ExFreePool(MasterAdapter);
 375         return NULL;
 376     }
 377
 378     RtlZeroMemory(MasterAdapter->MapRegisterBase,
 379                   SizeOfBitmap * sizeof(ROS_MAP_REGISTER_ENTRY));
 380     if (!HalpGrowMapBuffers(MasterAdapter, 0x10000))
 381     {
 382         ExFreePool(MasterAdapter);
 383         return NULL;
 384     }
 385
 386     return MasterAdapter;
 387 }
 388
 389 /**
 390  * @name HalpDmaAllocateChildAdapter
 391  *
 392  * Helper routine of HalGetAdapter. Allocate child adapter object and
 393  * fill out some basic fields.
 394  *
 395  * @see HalGetAdapter
 396  */
 397 PADAPTER_OBJECT
 398 NTAPI
 399 HalpDmaAllocateChildAdapter(IN ULONG NumberOfMapRegisters,
 400                             IN PDEVICE_DESCRIPTION DeviceDescription)
 401 {
 402     PADAPTER_OBJECT AdapterObject;
 403     OBJECT_ATTRIBUTES ObjectAttributes;
 404     NTSTATUS Status;
 405     HANDLE Handle;
 406
 407     InitializeObjectAttributes(&ObjectAttributes,
 408                                NULL,
 409                                OBJ_KERNEL_HANDLE | OBJ_PERMANENT,
 410                                NULL,
 411                                NULL);
 412
 413     Status = ObCreateObject(KernelMode,
 414                             IoAdapterObjectType,
 415                             &ObjectAttributes,
 416                             KernelMode,
 417                             NULL,
 418                             sizeof(ADAPTER_OBJECT),
 419                             0,
 420                             0,
 421                             (PVOID)&AdapterObject);
 422     if (!NT_SUCCESS(Status)) return NULL;
 423
 424     Status = ObReferenceObjectByPointer(AdapterObject,
 425                                         FILE_READ_DATA | FILE_WRITE_DATA,
 426                                         IoAdapterObjectType,
 427                                         KernelMode);
 428     if (!NT_SUCCESS(Status)) return NULL;
 429
 430     RtlZeroMemory(AdapterObject, sizeof(ADAPTER_OBJECT));
 431
 432     Status = ObInsertObject(AdapterObject,
 433                             NULL,
 434                             FILE_READ_DATA | FILE_WRITE_DATA,
 435                             0,
 436                             NULL,
 437                             &Handle);
 438     if (!NT_SUCCESS(Status)) return NULL;
 439
 440     ZwClose(Handle);
 441
 442     AdapterObject->DmaHeader.Version = (USHORT)DeviceDescription->Version;
 443     AdapterObject->DmaHeader.Size = sizeof(ADAPTER_OBJECT);
 444     AdapterObject->DmaHeader.DmaOperations = &HalpDmaOperations;
 445     AdapterObject->MapRegistersPerChannel = 1;
 446     AdapterObject->Dma32BitAddresses = DeviceDescription->Dma32BitAddresses;
 447     AdapterObject->ChannelNumber = 0xFF;
 448     AdapterObject->MasterAdapter = HalpMasterAdapter;
 449     KeInitializeDeviceQueue(&AdapterObject->ChannelWaitQueue);
 450
 451     return AdapterObject;
 452 }
 453 #endif
 454
 455 /**
 456  * @name HalpDmaInitializeEisaAdapter
 457  *
 458  * Setup DMA modes and extended modes for (E)ISA DMA adapter object.
 459  */
 460 BOOLEAN
 461 NTAPI
 462 HalpDmaInitializeEisaAdapter(IN PADAPTER_OBJECT AdapterObject,
 463                              IN PDEVICE_DESCRIPTION DeviceDescription)
 464 {
 465     UCHAR Controller;
 466     DMA_MODE DmaMode = {{0 }};
 467     DMA_EXTENDED_MODE ExtendedMode = {{ 0 }};
 468     PVOID AdapterBaseVa;
 469
 470     Controller = (DeviceDescription->DmaChannel & 4) ? 2 : 1;
 471
 472     if (Controller == 1)
 473     {
 474         AdapterBaseVa = (PVOID)FIELD_OFFSET(EISA_CONTROL, DmaController1);
 475     }
 476     else
 477     {
 478         AdapterBaseVa = (PVOID)FIELD_OFFSET(EISA_CONTROL, DmaController2);
 479     }
 480
 481     AdapterObject->AdapterNumber = Controller;
 482     AdapterObject->ChannelNumber = (UCHAR)(DeviceDescription->DmaChannel & 3);
 483     AdapterObject->PagePort = (PUCHAR)HalpEisaPortPage[DeviceDescription->DmaChannel];
 484     AdapterObject->Width16Bits = FALSE;
 485     AdapterObject->AdapterBaseVa = AdapterBaseVa;
 486
 487     if (HalpEisaDma)
 488     {
 489         ExtendedMode.ChannelNumber = AdapterObject->ChannelNumber;
 490
 491         switch (DeviceDescription->DmaSpeed)
 492         {
 493             case Compatible: ExtendedMode.TimingMode = COMPATIBLE_TIMING; break;
 494             case TypeA: ExtendedMode.TimingMode = TYPE_A_TIMING; break;
 495             case TypeB: ExtendedMode.TimingMode = TYPE_B_TIMING; break;
 496             case TypeC: ExtendedMode.TimingMode = BURST_TIMING; break;
 497             default:
 498                 return FALSE;
 499         }
 500
 501         switch (DeviceDescription->DmaWidth)
 502         {
 503             case Width8Bits: ExtendedMode.TransferSize = B_8BITS; break;
 504             case Width16Bits: ExtendedMode.TransferSize = B_16BITS; break;
 505             case Width32Bits: ExtendedMode.TransferSize = B_32BITS; break;
 506             default:
 507                 return FALSE;
 508         }
 509
 510         if (Controller == 1)
 511         {
 512             WRITE_PORT_UCHAR((PUCHAR)FIELD_OFFSET(EISA_CONTROL, DmaExtendedMode1),
 513                             ExtendedMode.Byte);
 514         }
 515         else
 516         {
 517             WRITE_PORT_UCHAR((PUCHAR)FIELD_OFFSET(EISA_CONTROL, DmaExtendedMode2),
 518                             ExtendedMode.Byte);
 519         }
 520     }
 521     else
 522     {
 523         /*
 524          * Validate setup for non-busmaster DMA adapter. Secondary controller
 525          * supports only 16-bit transfers and main controller supports only
 526          * 8-bit transfers. Anything else is invalid.
 527          */
 528         if (!DeviceDescription->Master)
 529         {
 530             if ((Controller == 2) && (DeviceDescription->DmaWidth == Width16Bits))
 531             {
 532                 AdapterObject->Width16Bits = TRUE;
 533             }
 534             else if ((Controller != 1) || (DeviceDescription->DmaWidth != Width8Bits))
 535             {
 536                 return FALSE;
 537             }
 538         }
 539     }
 540
 541     DmaMode.Channel = AdapterObject->ChannelNumber;
 542     DmaMode.AutoInitialize = DeviceDescription->AutoInitialize;
 543
 544     /*
 545      * Set the DMA request mode.
 546      *
 547      * For (E)ISA bus master devices just unmask (enable) the DMA channel
 548      * and set it to cascade mode. Otherwise just select the right one
 549      * bases on the passed device description.
 550      */
 551     if (DeviceDescription->Master)
 552     {
 553         DmaMode.RequestMode = CASCADE_REQUEST_MODE;
 554         if (Controller == 1)
 555         {
 556             /* Set the Request Data */
 557             _PRAGMA_WARNING_SUPPRESS(__WARNING_DEREF_NULL_PTR)
 558             WRITE_PORT_UCHAR(&((PDMA1_CONTROL)AdapterBaseVa)->Mode, DmaMode.Byte);
 559
 560             /* Unmask DMA Channel */
 561             WRITE_PORT_UCHAR(&((PDMA1_CONTROL)AdapterBaseVa)->SingleMask,
 562                              AdapterObject->ChannelNumber | DMA_CLEARMASK);
 563         }
 564         else
 565         {
 566             /* Set the Request Data */
 567             WRITE_PORT_UCHAR(&((PDMA2_CONTROL)AdapterBaseVa)->Mode, DmaMode.Byte);
 568
 569             /* Unmask DMA Channel */
 570             WRITE_PORT_UCHAR(&((PDMA2_CONTROL)AdapterBaseVa)->SingleMask,
 571                              AdapterObject->ChannelNumber | DMA_CLEARMASK);
 572         }
 573     }
 574     else
 575     {
 576         if (DeviceDescription->DemandMode)
 577         {
 578             DmaMode.RequestMode = DEMAND_REQUEST_MODE;
 579         }
 580         else
 581         {
 582             DmaMode.RequestMode = SINGLE_REQUEST_MODE;
 583         }
 584     }
 585
 586     AdapterObject->AdapterMode = DmaMode;
 587
 588     return TRUE;
 589 }
 590
 591 #ifndef _MINIHAL_
 592 /**
 593  * @name HalGetAdapter
 594  *
 595  * Allocate an adapter object for DMA device.
 596  *
 597  * @param DeviceDescription
 598  *        Structure describing the attributes of the device.
 599  * @param NumberOfMapRegisters
 600  *        On return filled with the maximum number of map registers the
 601  *        device driver can allocate for DMA transfer operations.
 602  *
 603  * @return The DMA adapter on success, NULL otherwise.
 604  *
 605  * @implemented
 606  */
 607 PADAPTER_OBJECT
 608 NTAPI
 609 HalGetAdapter(IN PDEVICE_DESCRIPTION DeviceDescription,
 610               OUT PULONG NumberOfMapRegisters)
 611 {
 612     PADAPTER_OBJECT AdapterObject = NULL;
 613     BOOLEAN EisaAdapter;
 614     ULONG MapRegisters;
 615     ULONG MaximumLength;
 616
 617     /* Validate parameters in device description */
 618     if (DeviceDescription->Version > DEVICE_DESCRIPTION_VERSION2) return NULL;
 619
 620     /*
 621      * See if we're going to use ISA/EISA DMA adapter. These adapters are
 622      * special since they're reused.
 623      *
 624      * Also note that we check for channel number since there are only 8 DMA
 625      * channels on ISA, so any request above this requires new adapter.
 626      */
 627     if (((DeviceDescription->InterfaceType == Eisa) ||
 628          (DeviceDescription->InterfaceType == Isa)) || !(DeviceDescription->Master))
 629     {
 630         if (((DeviceDescription->InterfaceType == Isa) ||
 631              (DeviceDescription->InterfaceType == Eisa)) &&
 632             (DeviceDescription->DmaChannel >= 8))
 633         {
 634             EisaAdapter = FALSE;
 635         }
 636         else
 637         {
 638             EisaAdapter = TRUE;
 639         }
 640     }
 641     else
 642     {
 643         EisaAdapter = FALSE;
 644     }
 645
 646     /*
 647      * Disallow creating adapter for ISA/EISA DMA channel 4 since it's used
 648      * for cascading the controllers and it's not available for software use.
 649      */
 650     if ((EisaAdapter) && (DeviceDescription->DmaChannel == 4)) return NULL;
 651
 652     /*
 653      * Calculate the number of map registers.
 654      *
 655      * - For EISA and PCI scatter/gather no map registers are needed.
 656      * - For ISA slave scatter/gather one map register is needed.
 657      * - For all other cases the number of map registers depends on
 658      *   DeviceDescription->MaximumLength.
 659      */
 660     MaximumLength = DeviceDescription->MaximumLength & MAXLONG;
 661     if ((DeviceDescription->ScatterGather) &&
 662         ((DeviceDescription->InterfaceType == Eisa) ||
 663          (DeviceDescription->InterfaceType == PCIBus)))
 664     {
 665         MapRegisters = 0;
 666     }
 667     else if ((DeviceDescription->ScatterGather) && !(DeviceDescription->Master))
 668     {
 669         MapRegisters = 1;
 670     }
 671     else
 672     {
 673         /*
 674          * In the equation below the additional map register added by
 675          * the "+1" accounts for the case when a transfer does not start
 676          * at a page-aligned address.
 677          */
 678         MapRegisters = BYTES_TO_PAGES(MaximumLength) + 1;
 679         if (MapRegisters > 16) MapRegisters = 16;
 680     }
 681
 682     /*
 683      * Acquire the DMA lock that is used to protect adapter lists and
 684      * EISA adapter array.
 685      */
 686     KeWaitForSingleObject(&HalpDmaLock, Executive, KernelMode, FALSE, NULL);
 687
 688     /*
 689      * Now we must get ahold of the adapter object. For first eight ISA/EISA
 690      * channels there are static adapter objects that are reused and updated
 691      * on succesive HalGetAdapter calls. In other cases a new adapter object
 692      * is always created and it's to the DMA adapter list (HalpDmaAdapterList).
 693      */
 694     if (EisaAdapter)
 695     {
 696         AdapterObject = HalpEisaAdapter[DeviceDescription->DmaChannel];
 697         if (AdapterObject)
 698         {
 699             if ((AdapterObject->NeedsMapRegisters) &&
 700                 (MapRegisters > AdapterObject->MapRegistersPerChannel))
 701             {
 702                 AdapterObject->MapRegistersPerChannel = MapRegisters;
 703             }
 704         }
 705     }
 706
 707     if (AdapterObject == NULL)
 708     {
 709         AdapterObject = HalpDmaAllocateChildAdapter(MapRegisters, DeviceDescription);
 710         if (AdapterObject == NULL)
 711         {
 712             KeSetEvent(&HalpDmaLock, 0, 0);
 713             return NULL;
 714         }
 715
 716         if (EisaAdapter)
 717         {
 718             HalpEisaAdapter[DeviceDescription->DmaChannel] = AdapterObject;
 719         }
 720
 721         if (MapRegisters > 0)
 722         {
 723             AdapterObject->NeedsMapRegisters = TRUE;
 724             AdapterObject->MapRegistersPerChannel = MapRegisters;
 725         }
 726         else
 727         {
 728             AdapterObject->NeedsMapRegisters = FALSE;
 729             if (DeviceDescription->Master)
 730             {
 731                 AdapterObject->MapRegistersPerChannel = BYTES_TO_PAGES(MaximumLength) + 1;
 732             }
 733             else
 734             {
 735                 AdapterObject->MapRegistersPerChannel = 1;
 736             }
 737         }
 738     }
 739
 740     if (!EisaAdapter) InsertTailList(&HalpDmaAdapterList, &AdapterObject->AdapterList);
 741
 742     /*
 743      * Release the DMA lock. HalpDmaAdapterList and HalpEisaAdapter will
 744      * no longer be touched, so we don't need it.
 745      */
 746     KeSetEvent(&HalpDmaLock, 0, 0);
 747
 748     /*
 749      * Setup the values in the adapter object that are common for all
 750      * types of buses.
 751      */
 752     if (DeviceDescription->Version >= DEVICE_DESCRIPTION_VERSION1)
 753     {
 754         AdapterObject->IgnoreCount = DeviceDescription->IgnoreCount;
 755     }
 756     else
 757     {
 758         AdapterObject->IgnoreCount = 0;
 759     }
 760
 761     AdapterObject->Dma32BitAddresses = DeviceDescription->Dma32BitAddresses;
 762     AdapterObject->Dma64BitAddresses = DeviceDescription->Dma64BitAddresses;
 763     AdapterObject->ScatterGather = DeviceDescription->ScatterGather;
 764     AdapterObject->MasterDevice = DeviceDescription->Master;
 765     *NumberOfMapRegisters = AdapterObject->MapRegistersPerChannel;
 766
 767     /*
 768      * For non-(E)ISA adapters we have already done all the work. On the
 769      * other hand for (E)ISA adapters we must still setup the DMA modes
 770      * and prepare the controller.
 771      */
 772     if (EisaAdapter)
 773     {
 774         if (!HalpDmaInitializeEisaAdapter(AdapterObject, DeviceDescription))
 775         {
 776             ObDereferenceObject(AdapterObject);
 777             return NULL;
 778         }
 779     }
 780
 781     return AdapterObject;
 782 }
 783
 784 /**
 785  * @name HalpGetDmaAdapter
 786  *
 787  * Internal routine to allocate PnP DMA adapter object. It's exported through
 788  * HalDispatchTable and used by IoGetDmaAdapter.
 789  *
 790  * @see HalGetAdapter
 791  */
 792 PDMA_ADAPTER
 793 NTAPI
 794 HalpGetDmaAdapter(IN PVOID Context,
 795                   IN PDEVICE_DESCRIPTION DeviceDescription,
 796                   OUT PULONG NumberOfMapRegisters)
 797 {
 798     return &HalGetAdapter(DeviceDescription, NumberOfMapRegisters)->DmaHeader;
 799 }
 800
 801 /**
 802  * @name HalPutDmaAdapter
 803  *
 804  * Internal routine to free DMA adapter and resources for reuse. It's exported
 805  * using the DMA_OPERATIONS interface by HalGetAdapter.
 806  *
 807  * @see HalGetAdapter
 808  */
 809 VOID
 810 NTAPI
 811 HalPutDmaAdapter(IN PADAPTER_OBJECT AdapterObject)
 812 {
 813     if (AdapterObject->ChannelNumber == 0xFF)
 814     {
 815         KeWaitForSingleObject(&HalpDmaLock, Executive, KernelMode, FALSE, NULL);
 816         RemoveEntryList(&AdapterObject->AdapterList);
 817         KeSetEvent(&HalpDmaLock, 0, 0);
 818     }
 819
 820     ObDereferenceObject(AdapterObject);
 821 }
 822
 823 /**
 824  * @name HalAllocateCommonBuffer
 825  *
 826  * Allocates memory that is visible to both the processor(s) and the DMA
 827  * device.
 828  *
 829  * @param AdapterObject
 830  *        Adapter object representing the bus master or system dma controller.
 831  * @param Length
 832  *        Number of bytes to allocate.
 833  * @param LogicalAddress
 834  *        Logical address the driver can use to access the buffer.
 835  * @param CacheEnabled
 836  *        Specifies if the memory can be cached.
 837  *
 838  * @return The base virtual address of the memory allocated or NULL on failure.
 839  *
 840  * @remarks
 841  *    On real NT x86 systems the CacheEnabled parameter is ignored, we honour
 842  *    it. If it proves to cause problems change it.
 843  *
 844  * @see HalFreeCommonBuffer
 845  *
 846  * @implemented
 847  */
 848 PVOID
 849 NTAPI
 850 HalAllocateCommonBuffer(IN PADAPTER_OBJECT AdapterObject,
 851                         IN ULONG Length,
 852                         IN PPHYSICAL_ADDRESS LogicalAddress,
 853                         IN BOOLEAN CacheEnabled)
 854 {
 855     PHYSICAL_ADDRESS LowestAcceptableAddress;
 856     PHYSICAL_ADDRESS HighestAcceptableAddress;
 857     PHYSICAL_ADDRESS BoundryAddressMultiple;
 858     PVOID VirtualAddress;
 859
 860     LowestAcceptableAddress.QuadPart = 0;
 861     HighestAcceptableAddress = HalpGetAdapterMaximumPhysicalAddress(AdapterObject);
 862     BoundryAddressMultiple.QuadPart = 0;
 863
 864     /*
 865      * For bus-master DMA devices the buffer mustn't cross 4Gb boundary. For
 866      * slave DMA devices the 64Kb boundary mustn't be crossed since the
 867      * controller wouldn't be able to handle it.
 868      */
 869     if (AdapterObject->MasterDevice)
 870     {
 871         BoundryAddressMultiple.HighPart = 1;
 872     }
 873     else
 874     {
 875         BoundryAddressMultiple.LowPart = 0x10000;
 876     }
 877
 878     VirtualAddress = MmAllocateContiguousMemorySpecifyCache(Length,
 879                                                             LowestAcceptableAddress,
 880                                                             HighestAcceptableAddress,
 881                                                             BoundryAddressMultiple,
 882                                                             CacheEnabled ? MmCached :
 883                                                             MmNonCached);
 884     if (VirtualAddress == NULL) return NULL;
 885
 886     *LogicalAddress = MmGetPhysicalAddress(VirtualAddress);
 887
 888     return VirtualAddress;
 889 }
 890
 891 /**
 892  * @name HalFreeCommonBuffer
 893  *
 894  * Free common buffer allocated with HalAllocateCommonBuffer.
 895  *
 896  * @see HalAllocateCommonBuffer
 897  *
 898  * @implemented
 899  */
 900 VOID
 901 NTAPI
 902 HalFreeCommonBuffer(IN PADAPTER_OBJECT AdapterObject,
 903                     IN ULONG Length,
 904                     IN PHYSICAL_ADDRESS LogicalAddress,
 905                     IN PVOID VirtualAddress,
 906                     IN BOOLEAN CacheEnabled)
 907 {
 908     MmFreeContiguousMemorySpecifyCache(VirtualAddress,
 909                                        Length,
 910                                        CacheEnabled ? MmCached : MmNonCached);
 911 }
 912
 913 typedef struct _SCATTER_GATHER_CONTEXT {
 914         PADAPTER_OBJECT AdapterObject;
 915         PMDL Mdl;
 916         PUCHAR CurrentVa;
 917         ULONG Length;
 918         PDRIVER_LIST_CONTROL AdapterListControlRoutine;
 919         PVOID AdapterListControlContext, MapRegisterBase;
 920         ULONG MapRegisterCount;
 921         BOOLEAN WriteToDevice;
 922 } SCATTER_GATHER_CONTEXT, *PSCATTER_GATHER_CONTEXT;
 923
 924
 925 IO_ALLOCATION_ACTION
 926 NTAPI
 927 HalpScatterGatherAdapterControl(IN PDEVICE_OBJECT DeviceObject,
 928                                 IN PIRP Irp,
 929                                                                 IN PVOID MapRegisterBase,
 930                                                                 IN PVOID Context)
 931 {
 932         PSCATTER_GATHER_CONTEXT AdapterControlContext = Context;
 933         PADAPTER_OBJECT AdapterObject = AdapterControlContext->AdapterObject;
 934         PSCATTER_GATHER_LIST ScatterGatherList;
 935         SCATTER_GATHER_ELEMENT TempElements[MAX_SG_ELEMENTS];
 936         ULONG ElementCount = 0, RemainingLength = AdapterControlContext->Length;
 937         PUCHAR CurrentVa = AdapterControlContext->CurrentVa;
 938
 939         /* Store the map register base for later in HalPutScatterGatherList */
 940         AdapterControlContext->MapRegisterBase = MapRegisterBase;
 941
 942         while (RemainingLength > 0 && ElementCount < MAX_SG_ELEMENTS)
 943         {
 944             TempElements[ElementCount].Length = RemainingLength;
 945                 TempElements[ElementCount].Reserved = 0;
 946             TempElements[ElementCount].Address = IoMapTransfer(AdapterObject,
 947                                                                    AdapterControlContext->Mdl,
 948                                                                                                                    MapRegisterBase,
 949                                                                                                                    CurrentVa + (AdapterControlContext->Length - RemainingLength),
 950                                                                                                                    &TempElements[ElementCount].Length,
 951                                                                                                                    AdapterControlContext->WriteToDevice);
 952                 if (TempElements[ElementCount].Length == 0)
 953                         break;
 954
 955                 DPRINT("Allocated one S/G element: 0x%I64u with length: 0x%x\n",
 956                         TempElements[ElementCount].Address.QuadPart,
 957                                 TempElements[ElementCount].Length);
 958
 959                 ASSERT(TempElements[ElementCount].Length <= RemainingLength);
 960                 RemainingLength -= TempElements[ElementCount].Length;
 961                 ElementCount++;
 962         }
 963
 964         if (RemainingLength > 0)
 965         {
 966                 DPRINT1("Scatter/gather list construction failed!\n");
 967                 return DeallocateObject;
 968         }
 969
 970         ScatterGatherList = ExAllocatePoolWithTag(NonPagedPool,
 971                                                   sizeof(SCATTER_GATHER_LIST) + sizeof(SCATTER_GATHER_ELEMENT) * ElementCount,
 972                                                                                           TAG_DMA);
 973         ASSERT(ScatterGatherList);
 974
 975         ScatterGatherList->NumberOfElements = ElementCount;
 976         ScatterGatherList->Reserved = (ULONG_PTR)AdapterControlContext;
 977         RtlCopyMemory(ScatterGatherList->Elements,
 978                       TempElements,
 979                                   sizeof(SCATTER_GATHER_ELEMENT) * ElementCount);
 980
 981         DPRINT("Initiating S/G DMA with %d element(s)\n", ElementCount);
 982
 983         AdapterControlContext->AdapterListControlRoutine(DeviceObject,
 984                                                          Irp,
 985                                                                                                          ScatterGatherList,
 986                                                                                                          AdapterControlContext->AdapterListControlContext);
 987
 988         return DeallocateObjectKeepRegisters;
 989 }
 990
 991 /**
 992  * @name HalGetScatterGatherList
 993  *
 994  * Creates a scatter-gather list to be using in scatter/gather DMA
 995  *
 996  * @param AdapterObject
 997  *        Adapter object representing the bus master or system dma controller.
 998  * @param DeviceObject
 999  *        The device target for DMA.
1000  * @param Mdl
1001  *        The MDL that describes the buffer to be mapped.
1002  * @param CurrentVa
1003  *        The current VA in the buffer to be mapped for transfer.
1004  * @param Length
1005  *        Specifies the length of data in bytes to be mapped.
1006  * @param ExecutionRoutine
1007  *        A caller supplied AdapterListControl routine to be called when DMA is available.
1008  * @param Context
1009  *        Context passed to the AdapterListControl routine.
1010  * @param WriteToDevice
1011  *        Indicates direction of DMA operation.
1012  *
1013  * @return The status of the operation.
1014  *
1015  * @see HalPutScatterGatherList
1016  *
1017  * @implemented
1018  */
1019  NTSTATUS
1020  NTAPI
1021  HalGetScatterGatherList(IN PADAPTER_OBJECT AdapterObject,
1022                          IN PDEVICE_OBJECT DeviceObject,
1023                                                  IN PMDL Mdl,
1024                                                  IN PVOID CurrentVa,
1025                                                  IN ULONG Length,
1026                                                  IN PDRIVER_LIST_CONTROL ExecutionRoutine,
1027                                                  IN PVOID Context,
1028                                                  IN BOOLEAN WriteToDevice)
1029 {
1030         PSCATTER_GATHER_CONTEXT AdapterControlContext;
1031
1032         AdapterControlContext = ExAllocatePoolWithTag(NonPagedPool, sizeof(SCATTER_GATHER_CONTEXT), TAG_DMA);
1033         if (!AdapterControlContext) return STATUS_INSUFFICIENT_RESOURCES;
1034
1035         AdapterControlContext->AdapterObject = AdapterObject;
1036         AdapterControlContext->Mdl = Mdl;
1037         AdapterControlContext->CurrentVa = CurrentVa;
1038         AdapterControlContext->Length = Length;
1039         AdapterControlContext->MapRegisterCount = PAGE_ROUND_UP(Length) >> PAGE_SHIFT;
1040         AdapterControlContext->AdapterListControlRoutine = ExecutionRoutine;
1041         AdapterControlContext->AdapterListControlContext = Context;
1042         AdapterControlContext->WriteToDevice = WriteToDevice;
1043
1044         return IoAllocateAdapterChannel(AdapterObject,
1045                                         DeviceObject,
1046                                                                         AdapterControlContext->MapRegisterCount,
1047                                                                         HalpScatterGatherAdapterControl,
1048                                                                         AdapterControlContext);
1049 }
1050
1051 /**
1052  * @name HalPutScatterGatherList
1053  *
1054  * Frees a scatter-gather list allocated from HalGetScatterGatherList
1055  *
1056  * @param AdapterObject
1057  *        Adapter object representing the bus master or system dma controller.
1058  * @param ScatterGather
1059  *        The scatter/gather list to be freed.
1060  * @param WriteToDevice
1061  *        Indicates direction of DMA operation.
1062  *
1063  * @return None
1064  *
1065  * @see HalGetScatterGatherList
1066  *
1067  * @implemented
1068  */
1069  VOID
1070  NTAPI
1071  HalPutScatterGatherList(IN PADAPTER_OBJECT AdapterObject,
1072                          IN PSCATTER_GATHER_LIST ScatterGather,
1073                                                  IN BOOLEAN WriteToDevice)
1074 {
1075     PSCATTER_GATHER_CONTEXT AdapterControlContext = (PSCATTER_GATHER_CONTEXT)ScatterGather->Reserved;
1076         ULONG i;
1077
1078         for (i = 0; i < ScatterGather->NumberOfElements; i++)
1079         {
1080              IoFlushAdapterBuffers(AdapterObject,
1081                                        AdapterControlContext->Mdl,
1082                                                            AdapterControlContext->MapRegisterBase,
1083                                                            AdapterControlContext->CurrentVa,
1084                                                            ScatterGather->Elements[i].Length,
1085                                                            AdapterControlContext->WriteToDevice);
1086                  AdapterControlContext->CurrentVa += ScatterGather->Elements[i].Length;
1087         }
1088
1089         IoFreeMapRegisters(AdapterObject,
1090                            AdapterControlContext->MapRegisterBase,
1091                                            AdapterControlContext->MapRegisterCount);
1092
1093         DPRINT("S/G DMA has finished!\n");
1094
1095         ExFreePoolWithTag(AdapterControlContext, TAG_DMA);
1096         ExFreePoolWithTag(ScatterGather, TAG_DMA);
1097 }
1098 #endif
1099
1100 /**
1101  * @name HalpDmaGetDmaAlignment
1102  *
1103  * Internal routine to return the DMA alignment requirement. It's exported
1104  * using the DMA_OPERATIONS interface by HalGetAdapter.
1105  *
1106  * @see HalGetAdapter
1107  */
1108 ULONG
1109 NTAPI
1110 HalpDmaGetDmaAlignment(IN PADAPTER_OBJECT AdapterObject)
1111 {
1112     return 1;
1113 }
1114
1115 /*
1116  * @name HalReadDmaCounter
1117  *
1118  * Read DMA operation progress counter.
1119  *
1120  * @implemented
1121  */
1122 ULONG
1123 NTAPI
1124 HalReadDmaCounter(IN PADAPTER_OBJECT AdapterObject)
1125 {
1126     KIRQL OldIrql;
1127     ULONG Count, OldCount;
1128
1129     ASSERT(!AdapterObject->MasterDevice);
1130
1131     /*
1132      * Acquire the master adapter lock since we're going to mess with the
1133      * system DMA controller registers and we really don't want anyone
1134      * to do the same at the same time.
1135      */
1136     KeAcquireSpinLock(&AdapterObject->MasterAdapter->SpinLock, &OldIrql);
1137
1138     /* Send the request to the specific controller. */
1139     if (AdapterObject->AdapterNumber == 1)
1140     {
1141         PDMA1_CONTROL DmaControl1 = AdapterObject->AdapterBaseVa;
1142
1143         Count = 0xffff00;
1144         do
1145         {
1146             OldCount = Count;
1147
1148             /* Send Reset */
1149             WRITE_PORT_UCHAR(&DmaControl1->ClearBytePointer, 0);
1150
1151             /* Read Count */
1152             Count = READ_PORT_UCHAR(&DmaControl1->DmaAddressCount
1153                                     [AdapterObject->ChannelNumber].DmaBaseCount);
1154             Count |= READ_PORT_UCHAR(&DmaControl1->DmaAddressCount
1155                                      [AdapterObject->ChannelNumber].DmaBaseCount) << 8;
1156         } while (0xffff00 & (OldCount ^ Count));
1157     }
1158     else
1159     {
1160         PDMA2_CONTROL DmaControl2 = AdapterObject->AdapterBaseVa;
1161
1162         Count = 0xffff00;
1163         do
1164         {
1165             OldCount = Count;
1166
1167             /* Send Reset */
1168             WRITE_PORT_UCHAR(&DmaControl2->ClearBytePointer, 0);
1169
1170             /* Read Count */
1171             Count = READ_PORT_UCHAR(&DmaControl2->DmaAddressCount
1172                                     [AdapterObject->ChannelNumber].DmaBaseCount);
1173             Count |= READ_PORT_UCHAR(&DmaControl2->DmaAddressCount
1174                                      [AdapterObject->ChannelNumber].DmaBaseCount) << 8;
1175         } while (0xffff00 & (OldCount ^ Count));
1176     }
1177
1178     KeReleaseSpinLock(&AdapterObject->MasterAdapter->SpinLock, OldIrql);
1179
1180     Count++;
1181     Count &= 0xffff;
1182     if (AdapterObject->Width16Bits) Count *= 2;
1183
1184     return Count;
1185 }
1186
1187 #ifndef _MINIHAL_
1188 /**
1189  * @name HalpGrowMapBufferWorker
1190  *
1191  * Helper routine of HalAllocateAdapterChannel for allocating map registers
1192  * at PASSIVE_LEVEL in work item.
1193  */
1194 VOID
1195 NTAPI
1196 HalpGrowMapBufferWorker(IN PVOID DeferredContext)
1197 {
1198     PGROW_WORK_ITEM WorkItem = (PGROW_WORK_ITEM)DeferredContext;
1199     KIRQL OldIrql;
1200     BOOLEAN Succeeded;
1201
1202     /*
1203      * Try to allocate new map registers for the adapter.
1204      *
1205      * NOTE: The NT implementation actually tries to allocate more map
1206      * registers than needed as an optimization.
1207      */
1208     KeWaitForSingleObject(&HalpDmaLock, Executive, KernelMode, FALSE, NULL);
1209     Succeeded = HalpGrowMapBuffers(WorkItem->AdapterObject->MasterAdapter,
1210                                    WorkItem->NumberOfMapRegisters << PAGE_SHIFT);
1211     KeSetEvent(&HalpDmaLock, 0, 0);
1212
1213     if (Succeeded)
1214     {
1215         /*
1216          * Flush the adapter queue now that new map registers are ready. The
1217          * easiest way to do that is to call IoFreeMapRegisters to not free
1218          * any registers. Note that we use the magic (PVOID)2 map register
1219          * base to bypass the parameter checking.
1220          */
1221         OldIrql = KfRaiseIrql(DISPATCH_LEVEL);
1222         IoFreeMapRegisters(WorkItem->AdapterObject, (PVOID)2, 0);
1223         KfLowerIrql(OldIrql);
1224     }
1225
1226     ExFreePool(WorkItem);
1227 }
1228
1229 /**
1230  * @name HalAllocateAdapterChannel
1231  *
1232  * Setup map registers for an adapter object.
1233  *
1234  * @param AdapterObject
1235  *        Pointer to an ADAPTER_OBJECT to set up.
1236  * @param WaitContextBlock
1237  *        Context block to be used with ExecutionRoutine.
1238  * @param NumberOfMapRegisters
1239  *        Number of map registers requested.
1240  * @param ExecutionRoutine
1241  *        Callback to call when map registers are allocated.
1242  *
1243  * @return
1244  *    If not enough map registers can be allocated then
1245  *    STATUS_INSUFFICIENT_RESOURCES is returned. If the function
1246  *    succeeds or the callback is queued for later delivering then
1247  *    STATUS_SUCCESS is returned.
1248  *
1249  * @see IoFreeAdapterChannel
1250  *
1251  * @implemented
1252  */
1253 NTSTATUS
1254 NTAPI
1255 HalAllocateAdapterChannel(IN PADAPTER_OBJECT AdapterObject,
1256                           IN PWAIT_CONTEXT_BLOCK WaitContextBlock,
1257                           IN ULONG NumberOfMapRegisters,
1258                           IN PDRIVER_CONTROL ExecutionRoutine)
1259 {
1260     PADAPTER_OBJECT MasterAdapter;
1261     PGROW_WORK_ITEM WorkItem;
1262     ULONG Index = MAXULONG;
1263     ULONG Result;
1264     KIRQL OldIrql;
1265
1266     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1267
1268     /* Set up the wait context block in case we can't run right away. */
1269     WaitContextBlock->DeviceRoutine = ExecutionRoutine;
1270     WaitContextBlock->NumberOfMapRegisters = NumberOfMapRegisters;
1271
1272     /* Returns true if queued, else returns false and sets the queue to busy */
1273     if (KeInsertDeviceQueue(&AdapterObject->ChannelWaitQueue,
1274                             &WaitContextBlock->WaitQueueEntry))
1275     {
1276         return STATUS_SUCCESS;
1277     }
1278
1279     MasterAdapter = AdapterObject->MasterAdapter;
1280
1281     AdapterObject->NumberOfMapRegisters = NumberOfMapRegisters;
1282     AdapterObject->CurrentWcb = WaitContextBlock;
1283
1284     if ((NumberOfMapRegisters) && (AdapterObject->NeedsMapRegisters))
1285     {
1286         if (NumberOfMapRegisters > AdapterObject->MapRegistersPerChannel)
1287         {
1288             AdapterObject->NumberOfMapRegisters = 0;
1289             IoFreeAdapterChannel(AdapterObject);
1290             return STATUS_INSUFFICIENT_RESOURCES;
1291         }
1292
1293         /*
1294          * Get the map registers. This is partly complicated by the fact
1295          * that new map registers can only be allocated at PASSIVE_LEVEL
1296          * and we're currently at DISPATCH_LEVEL. The following code has
1297          * two code paths:
1298          *
1299          * - If there is no adapter queued for map register allocation,
1300          *   try to see if enough contiguous map registers are present.
1301          *   In case they're we can just get them and proceed further.
1302          *
1303          * - If some adapter is already present in the queue we must
1304          *   respect the order of adapters asking for map registers and
1305          *   so the fast case described above can't take place.
1306          *   This case is also entered if not enough coniguous map
1307          *   registers are present.
1308          *
1309          *   A work queue item is allocated and queued, the adapter is
1310          *   also queued into the master adapter queue. The worker
1311          *   routine does the job of allocating the map registers at
1312          *   PASSIVE_LEVEL and calling the ExecutionRoutine.
1313          */
1314
1315         KeAcquireSpinLock(&MasterAdapter->SpinLock, &OldIrql);
1316
1317         if (IsListEmpty(&MasterAdapter->AdapterQueue))
1318         {
1319             Index = RtlFindClearBitsAndSet(MasterAdapter->MapRegisters, NumberOfMapRegisters, 0);
1320             if (Index != MAXULONG)
1321             {
1322                 AdapterObject->MapRegisterBase = MasterAdapter->MapRegisterBase + Index;
1323                 if (!AdapterObject->ScatterGather)
1324                 {
1325                     AdapterObject->MapRegisterBase = (PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)AdapterObject->MapRegisterBase | MAP_BASE_SW_SG);
1326                 }
1327             }
1328         }
1329
1330         if (Index == MAXULONG)
1331         {
1332             InsertTailList(&MasterAdapter->AdapterQueue, &AdapterObject->AdapterQueue);
1333
1334             WorkItem = ExAllocatePoolWithTag(NonPagedPool,
1335                                              sizeof(GROW_WORK_ITEM),
1336                                              TAG_DMA);
1337             if (WorkItem)
1338             {
1339                 ExInitializeWorkItem(&WorkItem->WorkQueueItem, HalpGrowMapBufferWorker, WorkItem);
1340                 WorkItem->AdapterObject = AdapterObject;
1341                 WorkItem->NumberOfMapRegisters = NumberOfMapRegisters;
1342
1343                 ExQueueWorkItem(&WorkItem->WorkQueueItem, DelayedWorkQueue);
1344             }
1345
1346             KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1347
1348             return STATUS_SUCCESS;
1349         }
1350
1351         KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1352     }
1353     else
1354     {
1355         AdapterObject->MapRegisterBase = NULL;
1356         AdapterObject->NumberOfMapRegisters = 0;
1357     }
1358
1359     AdapterObject->CurrentWcb = WaitContextBlock;
1360
1361     Result = ExecutionRoutine(WaitContextBlock->DeviceObject,
1362                               WaitContextBlock->CurrentIrp,
1363                               AdapterObject->MapRegisterBase,
1364                               WaitContextBlock->DeviceContext);
1365
1366     /*
1367      * Possible return values:
1368      *
1369      * - KeepObject
1370      *   Don't free any resources, the ADAPTER_OBJECT is still in use and
1371      *   the caller will call IoFreeAdapterChannel later.
1372      *
1373      * - DeallocateObject
1374      *   Deallocate the map registers and release the ADAPTER_OBJECT, so
1375      *   someone else can use it.
1376      *
1377      * - DeallocateObjectKeepRegisters
1378      *   Release the ADAPTER_OBJECT, but hang on to the map registers. The
1379      *   client will later call IoFreeMapRegisters.
1380      *
1381      * NOTE:
1382      * IoFreeAdapterChannel runs the queue, so it must be called unless
1383      * the adapter object is not to be freed.
1384      */
1385     if (Result == DeallocateObject)
1386     {
1387         IoFreeAdapterChannel(AdapterObject);
1388     }
1389     else if (Result == DeallocateObjectKeepRegisters)
1390     {
1391         AdapterObject->NumberOfMapRegisters = 0;
1392         IoFreeAdapterChannel(AdapterObject);
1393     }
1394
1395     return STATUS_SUCCESS;
1396 }
1397
1398 /**
1399  * @name IoFreeAdapterChannel
1400  *
1401  * Free DMA resources allocated by IoAllocateAdapterChannel.
1402  *
1403  * @param AdapterObject
1404  *        Adapter object with resources to free.
1405  *
1406  * @remarks
1407  *    This function releases map registers registers assigned to the DMA
1408  *    adapter. After releasing the adapter, it checks the adapter's queue
1409  *    and runs each queued device object in series until the queue is
1410  *    empty. This is the only way the device queue is emptied.
1411  *
1412  * @see IoAllocateAdapterChannel
1413  *
1414  * @implemented
1415  */
1416 VOID
1417 NTAPI
1418 IoFreeAdapterChannel(IN PADAPTER_OBJECT AdapterObject)
1419 {
1420     PADAPTER_OBJECT MasterAdapter;
1421     PKDEVICE_QUEUE_ENTRY DeviceQueueEntry;
1422     PWAIT_CONTEXT_BLOCK WaitContextBlock;
1423     ULONG Index = MAXULONG;
1424     ULONG Result;
1425     KIRQL OldIrql;
1426
1427     MasterAdapter = AdapterObject->MasterAdapter;
1428
1429     for (;;)
1430     {
1431         /*
1432          * To keep map registers, call here with AdapterObject->
1433          * NumberOfMapRegisters set to zero. This trick is used in
1434          * HalAllocateAdapterChannel for example.
1435          */
1436         if (AdapterObject->NumberOfMapRegisters)
1437         {
1438             IoFreeMapRegisters(AdapterObject,
1439                                AdapterObject->MapRegisterBase,
1440                                AdapterObject->NumberOfMapRegisters);
1441         }
1442
1443         DeviceQueueEntry = KeRemoveDeviceQueue(&AdapterObject->ChannelWaitQueue);
1444         if (!DeviceQueueEntry) break;
1445
1446         WaitContextBlock = CONTAINING_RECORD(DeviceQueueEntry,
1447                                              WAIT_CONTEXT_BLOCK,
1448                                              WaitQueueEntry);
1449
1450         AdapterObject->CurrentWcb = WaitContextBlock;
1451         AdapterObject->NumberOfMapRegisters = WaitContextBlock->NumberOfMapRegisters;
1452
1453         if ((WaitContextBlock->NumberOfMapRegisters) && (AdapterObject->MasterAdapter))
1454         {
1455             KeAcquireSpinLock(&MasterAdapter->SpinLock, &OldIrql);
1456
1457             if (IsListEmpty(&MasterAdapter->AdapterQueue))
1458             {
1459                 Index = RtlFindClearBitsAndSet(MasterAdapter->MapRegisters,
1460                                                WaitContextBlock->NumberOfMapRegisters,
1461                                                0);
1462                 if (Index != MAXULONG)
1463                 {
1464                     AdapterObject->MapRegisterBase = MasterAdapter->MapRegisterBase + Index;
1465                     if (!AdapterObject->ScatterGather)
1466                     {
1467                         AdapterObject->MapRegisterBase =(PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)AdapterObject->MapRegisterBase | MAP_BASE_SW_SG);
1468                     }
1469                 }
1470             }
1471
1472             if (Index == MAXULONG)
1473             {
1474                 InsertTailList(&MasterAdapter->AdapterQueue, &AdapterObject->AdapterQueue);
1475                 KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1476                 break;
1477             }
1478
1479             KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1480         }
1481         else
1482         {
1483             AdapterObject->MapRegisterBase = NULL;
1484             AdapterObject->NumberOfMapRegisters = 0;
1485         }
1486
1487         /* Call the adapter control routine. */
1488         Result = ((PDRIVER_CONTROL)WaitContextBlock->DeviceRoutine)(WaitContextBlock->DeviceObject,
1489                                                                     WaitContextBlock->CurrentIrp,
1490                                                                     AdapterObject->MapRegisterBase,
1491                                                                     WaitContextBlock->DeviceContext);
1492         switch (Result)
1493         {
1494             case KeepObject:
1495                 /*
1496                  * We're done until the caller manually calls IoFreeAdapterChannel
1497                  * or IoFreeMapRegisters.
1498                  */
1499                 return;
1500
1501             case DeallocateObjectKeepRegisters:
1502                 /*
1503                  * Hide the map registers so they aren't deallocated next time
1504                  * around.
1505                  */
1506                 AdapterObject->NumberOfMapRegisters = 0;
1507                 break;
1508
1509             default:
1510                 break;
1511         }
1512     }
1513 }
1514
1515 /**
1516  * @name IoFreeMapRegisters
1517  *
1518  * Free map registers reserved by the system for a DMA.
1519  *
1520  * @param AdapterObject
1521  *        DMA adapter to free map registers on.
1522  * @param MapRegisterBase
1523  *        Handle to map registers to free.
1524  * @param NumberOfRegisters
1525  *        Number of map registers to be freed.
1526  *
1527  * @implemented
1528  */
1529 VOID
1530 NTAPI
1531 IoFreeMapRegisters(IN PADAPTER_OBJECT AdapterObject,
1532                    IN PVOID MapRegisterBase,
1533                    IN ULONG NumberOfMapRegisters)
1534 {
1535     PADAPTER_OBJECT MasterAdapter = AdapterObject->MasterAdapter;
1536     PLIST_ENTRY ListEntry;
1537     KIRQL OldIrql;
1538     ULONG Index;
1539     ULONG Result;
1540
1541     ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
1542
1543     if (!(MasterAdapter) || !(MapRegisterBase)) return;
1544
1545     KeAcquireSpinLock(&MasterAdapter->SpinLock, &OldIrql);
1546
1547     if (NumberOfMapRegisters != 0)
1548     {
1549         PROS_MAP_REGISTER_ENTRY RealMapRegisterBase;
1550
1551         RealMapRegisterBase = (PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)MapRegisterBase & ~MAP_BASE_SW_SG);
1552         RtlClearBits(MasterAdapter->MapRegisters,
1553                      (ULONG)(RealMapRegisterBase - MasterAdapter->MapRegisterBase),
1554                      NumberOfMapRegisters);
1555     }
1556
1557     /*
1558      * Now that we freed few map registers it's time to look at the master
1559      * adapter queue and see if there is someone waiting for map registers.
1560      */
1561     while (!IsListEmpty(&MasterAdapter->AdapterQueue))
1562     {
1563         ListEntry = RemoveHeadList(&MasterAdapter->AdapterQueue);
1564         AdapterObject = CONTAINING_RECORD(ListEntry, struct _ADAPTER_OBJECT, AdapterQueue);
1565
1566         Index = RtlFindClearBitsAndSet(MasterAdapter->MapRegisters,
1567                                        AdapterObject->NumberOfMapRegisters,
1568                                        0);
1569         if (Index == MAXULONG)
1570         {
1571             InsertHeadList(&MasterAdapter->AdapterQueue, ListEntry);
1572             break;
1573         }
1574
1575         KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1576
1577         AdapterObject->MapRegisterBase = MasterAdapter->MapRegisterBase + Index;
1578         if (!AdapterObject->ScatterGather)
1579         {
1580             AdapterObject->MapRegisterBase =
1581                 (PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)AdapterObject->MapRegisterBase | MAP_BASE_SW_SG);
1582         }
1583
1584         Result = ((PDRIVER_CONTROL)AdapterObject->CurrentWcb->DeviceRoutine)(AdapterObject->CurrentWcb->DeviceObject,
1585                                                                              AdapterObject->CurrentWcb->CurrentIrp,
1586                                                                              AdapterObject->MapRegisterBase,
1587                                                                              AdapterObject->CurrentWcb->DeviceContext);
1588         switch (Result)
1589         {
1590             case DeallocateObjectKeepRegisters:
1591                 AdapterObject->NumberOfMapRegisters = 0;
1592                 /* fall through */
1593
1594             case DeallocateObject:
1595                 if (AdapterObject->NumberOfMapRegisters)
1596                 {
1597                     KeAcquireSpinLock(&MasterAdapter->SpinLock, &OldIrql);
1598                     RtlClearBits(MasterAdapter->MapRegisters,
1599                                  (ULONG)(AdapterObject->MapRegisterBase -
1600                                          MasterAdapter->MapRegisterBase),
1601                                  AdapterObject->NumberOfMapRegisters);
1602                     KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1603                 }
1604
1605                 IoFreeAdapterChannel(AdapterObject);
1606                 break;
1607
1608             default:
1609                 break;
1610         }
1611
1612         KeAcquireSpinLock(&MasterAdapter->SpinLock, &OldIrql);
1613     }
1614
1615     KeReleaseSpinLock(&MasterAdapter->SpinLock, OldIrql);
1616 }
1617
1618 /**
1619  * @name HalpCopyBufferMap
1620  *
1621  * Helper function for copying data from/to map register buffers.
1622  *
1623  * @see IoFlushAdapterBuffers, IoMapTransfer
1624  */
1625 VOID
1626 NTAPI
1627 HalpCopyBufferMap(IN PMDL Mdl,
1628                   IN PROS_MAP_REGISTER_ENTRY MapRegisterBase,
1629                   IN PVOID CurrentVa,
1630                   IN ULONG Length,
1631                   IN BOOLEAN WriteToDevice)
1632 {
1633     ULONG CurrentLength;
1634     ULONG_PTR CurrentAddress;
1635     ULONG ByteOffset;
1636     PVOID VirtualAddress;
1637
1638     VirtualAddress = MmGetSystemAddressForMdlSafe(Mdl, HighPagePriority);
1639     if (!VirtualAddress)
1640     {
1641         /*
1642          * NOTE: On real NT a mechanism with reserved pages is implemented
1643          * to handle this case in a slow, but graceful non-fatal way.
1644          */
1645          KeBugCheckEx(HAL_MEMORY_ALLOCATION, PAGE_SIZE, 0, (ULONG_PTR)__FILE__, 0);
1646     }
1647
1648     CurrentAddress = (ULONG_PTR)VirtualAddress +
1649                      (ULONG_PTR)CurrentVa -
1650                      (ULONG_PTR)MmGetMdlVirtualAddress(Mdl);
1651
1652     while (Length > 0)
1653     {
1654         ByteOffset = BYTE_OFFSET(CurrentAddress);
1655         CurrentLength = PAGE_SIZE - ByteOffset;
1656         if (CurrentLength > Length) CurrentLength = Length;
1657
1658         if (WriteToDevice)
1659         {
1660             RtlCopyMemory((PVOID)((ULONG_PTR)MapRegisterBase->VirtualAddress + ByteOffset),
1661                           (PVOID)CurrentAddress,
1662                           CurrentLength);
1663         }
1664         else
1665         {
1666             RtlCopyMemory((PVOID)CurrentAddress,
1667                           (PVOID)((ULONG_PTR)MapRegisterBase->VirtualAddress + ByteOffset),
1668                           CurrentLength);
1669         }
1670
1671         Length -= CurrentLength;
1672         CurrentAddress += CurrentLength;
1673         MapRegisterBase++;
1674     }
1675 }
1676
1677 /**
1678  * @name IoFlushAdapterBuffers
1679  *
1680  * Flush any data remaining in the DMA controller's memory into the host
1681  * memory.
1682  *
1683  * @param AdapterObject
1684  *        The adapter object to flush.
1685  * @param Mdl
1686  *        Original MDL to flush data into.
1687  * @param MapRegisterBase
1688  *        Map register base that was just used by IoMapTransfer, etc.
1689  * @param CurrentVa
1690  *        Offset into Mdl to be flushed into, same as was passed to
1691  *        IoMapTransfer.
1692  * @param Length
1693  *        Length of the buffer to be flushed into.
1694  * @param WriteToDevice
1695  *        TRUE if it's a write, FALSE if it's a read.
1696  *
1697  * @return TRUE in all cases.
1698  *
1699  * @remarks
1700  *    This copies data from the map register-backed buffer to the user's
1701  *    target buffer. Data are not in the user buffer until this function
1702  *    is called.
1703  *    For slave DMA transfers the controller channel is masked effectively
1704  *    stopping the current transfer.
1705  *
1706  * @unimplemented.
1707  */
1708 BOOLEAN
1709 NTAPI
1710 IoFlushAdapterBuffers(IN PADAPTER_OBJECT AdapterObject,
1711                       IN PMDL Mdl,
1712                       IN PVOID MapRegisterBase,
1713                       IN PVOID CurrentVa,
1714                       IN ULONG Length,
1715                       IN BOOLEAN WriteToDevice)
1716 {
1717     BOOLEAN SlaveDma = FALSE;
1718     PROS_MAP_REGISTER_ENTRY RealMapRegisterBase;
1719     PHYSICAL_ADDRESS HighestAcceptableAddress;
1720     PHYSICAL_ADDRESS PhysicalAddress;
1721     PPFN_NUMBER MdlPagesPtr;
1722
1723     /* Sanity checks */
1724     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
1725     ASSERT(AdapterObject);
1726
1727     if (!AdapterObject->MasterDevice)
1728     {
1729         /* Mask out (disable) the DMA channel. */
1730         if (AdapterObject->AdapterNumber == 1)
1731         {
1732             PDMA1_CONTROL DmaControl1 = AdapterObject->AdapterBaseVa;
1733             WRITE_PORT_UCHAR(&DmaControl1->SingleMask,
1734                              AdapterObject->ChannelNumber | DMA_SETMASK);
1735         }
1736         else
1737         {
1738             PDMA2_CONTROL DmaControl2 = AdapterObject->AdapterBaseVa;
1739             WRITE_PORT_UCHAR(&DmaControl2->SingleMask,
1740                              AdapterObject->ChannelNumber | DMA_SETMASK);
1741         }
1742         SlaveDma = TRUE;
1743     }
1744
1745     /* This can happen if the device supports hardware scatter/gather. */
1746     if (MapRegisterBase == NULL) return TRUE;
1747
1748     RealMapRegisterBase = (PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)MapRegisterBase & ~MAP_BASE_SW_SG);
1749
1750     if (!WriteToDevice)
1751     {
1752         if ((ULONG_PTR)MapRegisterBase & MAP_BASE_SW_SG)
1753         {
1754             if (RealMapRegisterBase->Counter != MAXULONG)
1755             {
1756                 if ((SlaveDma) && !(AdapterObject->IgnoreCount))
1757                 {
1758                     Length -= HalReadDmaCounter(AdapterObject);
1759                 }
1760             }
1761             HalpCopyBufferMap(Mdl,
1762                               RealMapRegisterBase,
1763                               CurrentVa,
1764                               Length,
1765                               FALSE);
1766         }
1767         else
1768         {
1769             MdlPagesPtr = MmGetMdlPfnArray(Mdl);
1770             MdlPagesPtr += ((ULONG_PTR)CurrentVa - (ULONG_PTR)Mdl->StartVa) >> PAGE_SHIFT;
1771
1772             PhysicalAddress.QuadPart = *MdlPagesPtr << PAGE_SHIFT;
1773             PhysicalAddress.QuadPart += BYTE_OFFSET(CurrentVa);
1774
1775             HighestAcceptableAddress = HalpGetAdapterMaximumPhysicalAddress(AdapterObject);
1776             if ((PhysicalAddress.QuadPart + Length) > HighestAcceptableAddress.QuadPart)
1777             {
1778                 HalpCopyBufferMap(Mdl,
1779                                   RealMapRegisterBase,
1780                                   CurrentVa,
1781                                   Length,
1782                                   FALSE);
1783             }
1784         }
1785     }
1786
1787     RealMapRegisterBase->Counter = 0;
1788
1789     return TRUE;
1790 }
1791
1792 /**
1793  * @name IoMapTransfer
1794  *
1795  * Map a DMA for transfer and do the DMA if it's a slave.
1796  *
1797  * @param AdapterObject
1798  *        Adapter object to do the DMA on. Bus-master may pass NULL.
1799  * @param Mdl
1800  *        Locked-down user buffer to DMA in to or out of.
1801  * @param MapRegisterBase
1802  *        Handle to map registers to use for this dma.
1803  * @param CurrentVa
1804  *        Index into Mdl to transfer into/out of.
1805  * @param Length
1806  *        Length of transfer. Number of bytes actually transferred on
1807  *        output.
1808  * @param WriteToDevice
1809  *        TRUE if it's an output DMA, FALSE otherwise.
1810  *
1811  * @return
1812  *    A logical address that can be used to program a DMA controller, it's
1813  *    not meaningful for slave DMA device.
1814  *
1815  * @remarks
1816  *    This function does a copyover to contiguous memory <16MB represented
1817  *    by the map registers if needed. If the buffer described by MDL can be
1818  *    used as is no copyover is done.
1819  *    If it's a slave transfer, this function actually performs it.
1820  *
1821  * @implemented
1822  */
1823 PHYSICAL_ADDRESS
1824 NTAPI
1825 IoMapTransfer(IN PADAPTER_OBJECT AdapterObject,
1826               IN PMDL Mdl,
1827               IN PVOID MapRegisterBase,
1828               IN PVOID CurrentVa,
1829               IN OUT PULONG Length,
1830               IN BOOLEAN WriteToDevice)
1831 {
1832     PPFN_NUMBER MdlPagesPtr;
1833     PFN_NUMBER MdlPage1, MdlPage2;
1834     ULONG ByteOffset;
1835     ULONG TransferOffset;
1836     ULONG TransferLength;
1837     BOOLEAN UseMapRegisters;
1838     PROS_MAP_REGISTER_ENTRY RealMapRegisterBase;
1839     PHYSICAL_ADDRESS PhysicalAddress;
1840     PHYSICAL_ADDRESS HighestAcceptableAddress;
1841     ULONG Counter;
1842     DMA_MODE AdapterMode;
1843     KIRQL OldIrql;
1844
1845     /*
1846      * Precalculate some values that are used in all cases.
1847      *
1848      * ByteOffset is offset inside the page at which the transfer starts.
1849      * MdlPagesPtr is pointer inside the MDL page chain at the page where the
1850      *             transfer start.
1851      * PhysicalAddress is physical address corresponding to the transfer
1852      *                 start page and offset.
1853      * TransferLength is the initial length of the transfer, which is reminder
1854      *                of the first page. The actual value is calculated below.
1855      *
1856      * Note that all the variables can change during the processing which
1857      * takes place below. These are just initial values.
1858      */
1859     ByteOffset = BYTE_OFFSET(CurrentVa);
1860
1861     MdlPagesPtr = MmGetMdlPfnArray(Mdl);
1862     MdlPagesPtr += ((ULONG_PTR)CurrentVa - (ULONG_PTR)Mdl->StartVa) >> PAGE_SHIFT;
1863
1864     PhysicalAddress.QuadPart = *MdlPagesPtr << PAGE_SHIFT;
1865     PhysicalAddress.QuadPart += ByteOffset;
1866
1867     TransferLength = PAGE_SIZE - ByteOffset;
1868
1869     /*
1870      * Special case for bus master adapters with S/G support. We can directly
1871      * use the buffer specified by the MDL, so not much work has to be done.
1872      *
1873      * Just return the passed VA's corresponding physical address and update
1874      * length to the number of physically contiguous bytes found. Also
1875      * pages crossing the 4Gb boundary aren't considered physically contiguous.
1876      */
1877     if (MapRegisterBase == NULL)
1878     {
1879         while (TransferLength < *Length)
1880         {
1881             MdlPage1 = *MdlPagesPtr;
1882             MdlPage2 = *(MdlPagesPtr + 1);
1883             if (MdlPage1 + 1 != MdlPage2) break;
1884             if ((MdlPage1 ^ MdlPage2) & ~0xFFFFF) break;
1885             TransferLength += PAGE_SIZE;
1886             MdlPagesPtr++;
1887         }
1888
1889         if (TransferLength < *Length) *Length = TransferLength;
1890
1891         return PhysicalAddress;
1892     }
1893
1894     /*
1895      * The code below applies to slave DMA adapters and bus master adapters
1896      * without hardward S/G support.
1897      */
1898     RealMapRegisterBase = (PROS_MAP_REGISTER_ENTRY)((ULONG_PTR)MapRegisterBase & ~MAP_BASE_SW_SG);
1899
1900     /*
1901      * Try to calculate the size of the transfer. We can only transfer
1902      * pages that are physically contiguous and that don't cross the
1903      * 64Kb boundary (this limitation applies only for ISA controllers).
1904      */
1905     while (TransferLength < *Length)
1906     {
1907         MdlPage1 = *MdlPagesPtr;
1908         MdlPage2 = *(MdlPagesPtr + 1);
1909         if (MdlPage1 + 1 != MdlPage2) break;
1910         if (!HalpEisaDma && ((MdlPage1 ^ MdlPage2) & ~0xF)) break;
1911         TransferLength += PAGE_SIZE;
1912         MdlPagesPtr++;
1913     }
1914
1915     if (TransferLength > *Length) TransferLength = *Length;
1916
1917     /*
1918      * If we're about to simulate software S/G and not all the pages are
1919      * physically contiguous then we must use the map registers to store
1920      * the data and allow the whole transfer to proceed at once.
1921      */
1922     if (((ULONG_PTR)MapRegisterBase & MAP_BASE_SW_SG) && (TransferLength < *Length))
1923     {
1924         UseMapRegisters = TRUE;
1925         PhysicalAddress = RealMapRegisterBase->PhysicalAddress;
1926         PhysicalAddress.QuadPart += ByteOffset;
1927         TransferLength = *Length;
1928         RealMapRegisterBase->Counter = MAXULONG;
1929         Counter = 0;
1930     }
1931     else
1932     {
1933         /*
1934          * This is ordinary DMA transfer, so just update the progress
1935          * counters. These are used by IoFlushAdapterBuffers to track
1936          * the transfer progress.
1937          */
1938         UseMapRegisters = FALSE;
1939         Counter = RealMapRegisterBase->Counter;
1940         RealMapRegisterBase->Counter += BYTES_TO_PAGES(ByteOffset + TransferLength);
1941
1942         /*
1943          * Check if the buffer doesn't exceed the highest physical address
1944          * limit of the device. In that case we must use the map registers to
1945          * store the data.
1946          */
1947         HighestAcceptableAddress = HalpGetAdapterMaximumPhysicalAddress(AdapterObject);
1948         if ((PhysicalAddress.QuadPart + TransferLength) > HighestAcceptableAddress.QuadPart)
1949         {
1950             UseMapRegisters = TRUE;
1951             PhysicalAddress = RealMapRegisterBase[Counter].PhysicalAddress;
1952             PhysicalAddress.QuadPart += ByteOffset;
1953             if ((ULONG_PTR)MapRegisterBase & MAP_BASE_SW_SG)
1954             {
1955                 RealMapRegisterBase->Counter = MAXULONG;
1956                 Counter = 0;
1957             }
1958         }
1959     }
1960
1961     /*
1962      * If we decided to use the map registers (see above) and we're about
1963      * to transfer data to the device then copy the buffers into the map
1964      * register memory.
1965      */
1966     if ((UseMapRegisters) && (WriteToDevice))
1967     {
1968         HalpCopyBufferMap(Mdl,
1969                           RealMapRegisterBase + Counter,
1970                           CurrentVa,
1971                           TransferLength,
1972                           WriteToDevice);
1973     }
1974
1975     /*
1976      * Return the length of transfer that actually takes place.
1977      */
1978     *Length = TransferLength;
1979
1980     /*
1981      * If we're doing slave (system) DMA then program the (E)ISA controller
1982      * to actually start the transfer.
1983      */
1984     if ((AdapterObject) && !(AdapterObject->MasterDevice))
1985     {
1986         AdapterMode = AdapterObject->AdapterMode;
1987
1988         if (WriteToDevice)
1989         {
1990             AdapterMode.TransferType = WRITE_TRANSFER;
1991         }
1992         else
1993         {
1994             AdapterMode.TransferType = READ_TRANSFER;
1995             if (AdapterObject->IgnoreCount)
1996             {
1997                 RtlZeroMemory((PUCHAR)RealMapRegisterBase[Counter].VirtualAddress + ByteOffset,
1998                               TransferLength);
1999             }
2000         }
2001
2002         TransferOffset = PhysicalAddress.LowPart & 0xFFFF;
2003         if (AdapterObject->Width16Bits)
2004         {
2005             TransferLength >>= 1;
2006             TransferOffset >>= 1;
2007         }
2008
2009         KeAcquireSpinLock(&AdapterObject->MasterAdapter->SpinLock, &OldIrql);
2010
2011         if (AdapterObject->AdapterNumber == 1)
2012         {
2013             PDMA1_CONTROL DmaControl1 = AdapterObject->AdapterBaseVa;
2014
2015             /* Reset Register */
2016             WRITE_PORT_UCHAR(&DmaControl1->ClearBytePointer, 0);
2017
2018             /* Set the Mode */
2019             WRITE_PORT_UCHAR(&DmaControl1->Mode, AdapterMode.Byte);
2020
2021             /* Set the Offset Register */
2022             WRITE_PORT_UCHAR(&DmaControl1->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseAddress,
2023                              (UCHAR)(TransferOffset));
2024             WRITE_PORT_UCHAR(&DmaControl1->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseAddress,
2025                              (UCHAR)(TransferOffset >> 8));
2026
2027             /* Set the Page Register */
2028             WRITE_PORT_UCHAR(AdapterObject->PagePort + FIELD_OFFSET(EISA_CONTROL, DmaController1Pages),
2029                              (UCHAR)(PhysicalAddress.LowPart >> 16));
2030             if (HalpEisaDma)
2031             {
2032                 WRITE_PORT_UCHAR(AdapterObject->PagePort + FIELD_OFFSET(EISA_CONTROL, DmaController2Pages),
2033                                  0);
2034             }
2035
2036             /* Set the Length */
2037             WRITE_PORT_UCHAR(&DmaControl1->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseCount,
2038                              (UCHAR)(TransferLength - 1));
2039             WRITE_PORT_UCHAR(&DmaControl1->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseCount,
2040                              (UCHAR)((TransferLength - 1) >> 8));
2041
2042             /* Unmask the Channel */
2043             WRITE_PORT_UCHAR(&DmaControl1->SingleMask, AdapterObject->ChannelNumber | DMA_CLEARMASK);
2044         }
2045         else
2046         {
2047             PDMA2_CONTROL DmaControl2 = AdapterObject->AdapterBaseVa;
2048
2049             /* Reset Register */
2050             WRITE_PORT_UCHAR(&DmaControl2->ClearBytePointer, 0);
2051
2052             /* Set the Mode */
2053             WRITE_PORT_UCHAR(&DmaControl2->Mode, AdapterMode.Byte);
2054
2055             /* Set the Offset Register */
2056             WRITE_PORT_UCHAR(&DmaControl2->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseAddress,
2057                              (UCHAR)(TransferOffset));
2058             WRITE_PORT_UCHAR(&DmaControl2->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseAddress,
2059                              (UCHAR)(TransferOffset >> 8));
2060
2061             /* Set the Page Register */
2062             WRITE_PORT_UCHAR(AdapterObject->PagePort + FIELD_OFFSET(EISA_CONTROL, DmaController1Pages),
2063                              (UCHAR)(PhysicalAddress.u.LowPart >> 16));
2064             if (HalpEisaDma)
2065             {
2066                 WRITE_PORT_UCHAR(AdapterObject->PagePort + FIELD_OFFSET(EISA_CONTROL, DmaController2Pages),
2067                                  0);
2068             }
2069
2070             /* Set the Length */
2071             WRITE_PORT_UCHAR(&DmaControl2->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseCount,
2072                              (UCHAR)(TransferLength - 1));
2073             WRITE_PORT_UCHAR(&DmaControl2->DmaAddressCount[AdapterObject->ChannelNumber].DmaBaseCount,
2074                              (UCHAR)((TransferLength - 1) >> 8));
2075
2076             /* Unmask the Channel */
2077             WRITE_PORT_UCHAR(&DmaControl2->SingleMask,
2078                              AdapterObject->ChannelNumber | DMA_CLEARMASK);
2079         }
2080
2081         KeReleaseSpinLock(&AdapterObject->MasterAdapter->SpinLock, OldIrql);
2082     }
2083
2084     /*
2085      * Return physical address of the buffer with data that is used for the
2086      * transfer. It can either point inside the Mdl that was passed by the
2087      * caller or into the map registers if the Mdl buffer can't be used
2088      * directly.
2089      */
2090      return PhysicalAddress;
2091 }
2092 #endif
2093
2094 /**
2095  * @name HalFlushCommonBuffer
2096  *
2097  * @implemented
2098  */
2099 BOOLEAN
2100 NTAPI
2101 HalFlushCommonBuffer(IN PADAPTER_OBJECT AdapterObject,
2102                      IN ULONG Length,
2103                      IN PHYSICAL_ADDRESS LogicalAddress,
2104                      IN PVOID VirtualAddress)
2105 {
2106     /* Function always returns true */
2107     return TRUE;
2108 }
2109
2110 /*
2111  * @implemented
2112  */
2113 PVOID
2114 NTAPI
2115 HalAllocateCrashDumpRegisters(IN PADAPTER_OBJECT AdapterObject,
2116                               IN OUT PULONG NumberOfMapRegisters)
2117 {
2118     PADAPTER_OBJECT MasterAdapter = AdapterObject->MasterAdapter;
2119     ULONG MapRegisterNumber;
2120
2121     /* Check if it needs map registers */
2122     if (AdapterObject->NeedsMapRegisters)
2123     {
2124         /* Check if we have enough */
2125         if (*NumberOfMapRegisters > AdapterObject->MapRegistersPerChannel)
2126         {
2127             /* We don't, fail */
2128             AdapterObject->NumberOfMapRegisters = 0;
2129             return NULL;
2130         }
2131
2132         /* Try to find free map registers */
2133         MapRegisterNumber = RtlFindClearBitsAndSet(MasterAdapter->MapRegisters,
2134                                                    *NumberOfMapRegisters,
2135                                                    0);
2136
2137         /* Check if nothing was found */
2138         if (MapRegisterNumber == MAXULONG)
2139         {
2140             /* No free registers found, so use the base registers */
2141             RtlSetBits(MasterAdapter->MapRegisters,
2142                        0,
2143                        *NumberOfMapRegisters);
2144             MapRegisterNumber = 0;
2145         }
2146
2147         /* Calculate the new base */
2148         AdapterObject->MapRegisterBase =
2149             (PROS_MAP_REGISTER_ENTRY)(MasterAdapter->MapRegisterBase +
2150                                       MapRegisterNumber);
2151
2152         /* Check if scatter gather isn't supported */
2153         if (!AdapterObject->ScatterGather)
2154         {
2155             /* Set the flag */
2156             AdapterObject->MapRegisterBase =
2157                 (PROS_MAP_REGISTER_ENTRY)
2158                 ((ULONG_PTR)AdapterObject->MapRegisterBase | MAP_BASE_SW_SG);
2159         }
2160     }
2161     else
2162     {
2163         AdapterObject->MapRegisterBase = NULL;
2164         AdapterObject->NumberOfMapRegisters = 0;
2165     }
2166
2167     /* Return the base */
2168     return AdapterObject->MapRegisterBase;
2169 }
2170
2171 /* EOF */