drivers/storage/classpnp/xferpkt.c

   1 /*++
   2
   3 Copyright (C) Microsoft Corporation, 1991 - 1999
   4
   5 Module Name:
   6
   7     xferpkt.c
   8
   9 Abstract:
  10
  11     Packet routines for CLASSPNP
  12
  13 Environment:
  14
  15     kernel mode only
  16
  17 Notes:
  18
  19
  20 Revision History:
  21
  22 --*/
  23
  24 #include "classp.h"
  25 #include "debug.h"
  26
  27 #ifdef ALLOC_PRAGMA
  28     #pragma alloc_text(PAGE, InitializeTransferPackets)
  29     #pragma alloc_text(PAGE, DestroyAllTransferPackets)
  30     #pragma alloc_text(PAGE, SetupEjectionTransferPacket)
  31     #pragma alloc_text(PAGE, SetupModeSenseTransferPacket)
  32 #endif
  33
  34
  35 ULONG MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
  36 ULONG MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
  37
  38
  39 /*
  40  *  InitializeTransferPackets
  41  *
  42  *      Allocate/initialize TRANSFER_PACKETs and related resources.
  43  */
  44 NTSTATUS NTAPI InitializeTransferPackets(PDEVICE_OBJECT Fdo)
  45 {
  46     PCOMMON_DEVICE_EXTENSION commonExt = Fdo->DeviceExtension;
  47     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
  48     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
  49     PSTORAGE_ADAPTER_DESCRIPTOR adapterDesc = commonExt->PartitionZeroExtension->AdapterDescriptor;
  50     ULONG hwMaxPages;
  51     NTSTATUS status = STATUS_SUCCESS;
  52
  53     PAGED_CODE();
  54
  55     /*
  56      *  Precompute the maximum transfer length
  57      */
  58     ASSERT(adapterDesc->MaximumTransferLength);
  59     ASSERT(adapterDesc->MaximumPhysicalPages);
  60     hwMaxPages = adapterDesc->MaximumPhysicalPages ? adapterDesc->MaximumPhysicalPages-1 : 0;
  61
  62 #if defined(_AMD64_SIMULATOR_)
  63
  64     //
  65     // The simulator appears to have a problem with large transfers.
  66     //
  67
  68     if (hwMaxPages > 4) {
  69         hwMaxPages = 4;
  70     }
  71
  72 #endif
  73
  74     fdoData->HwMaxXferLen = MIN(adapterDesc->MaximumTransferLength, hwMaxPages << PAGE_SHIFT);
  75     fdoData->HwMaxXferLen = MAX(fdoData->HwMaxXferLen, PAGE_SIZE);
  76
  77     fdoData->NumTotalTransferPackets = 0;
  78     fdoData->NumFreeTransferPackets = 0;
  79     InitializeSListHead(&fdoData->FreeTransferPacketsList);
  80     InitializeListHead(&fdoData->AllTransferPacketsList);
  81     InitializeListHead(&fdoData->DeferredClientIrpList);
  82
  83     /*
  84      *  Set the packet threshold numbers based on the Windows SKU.
  85      */
  86     if (ExVerifySuite(Personal)){
  87         // this is Windows Personal
  88         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
  89         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
  90     }
  91     else if (ExVerifySuite(Enterprise) || ExVerifySuite(DataCenter)){
  92         // this is Advanced Server or Datacenter
  93         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Enterprise;
  94         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Enterprise;
  95     }
  96     else if (ExVerifySuite(TerminalServer)){
  97         // this is standard Server or Pro with terminal server
  98         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Server;
  99         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Server;
 100     }
 101     else {
 102         // this is Professional without terminal server
 103         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
 104         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
 105     }
 106
 107     while (fdoData->NumFreeTransferPackets < MIN_INITIAL_TRANSFER_PACKETS){
 108         PTRANSFER_PACKET pkt = NewTransferPacket(Fdo);
 109         if (pkt){
 110             InterlockedIncrement(&fdoData->NumTotalTransferPackets);
 111             EnqueueFreeTransferPacket(Fdo, pkt);
 112         }
 113         else {
 114             status = STATUS_INSUFFICIENT_RESOURCES;
 115             break;
 116         }
 117     }
 118     fdoData->DbgPeakNumTransferPackets = fdoData->NumTotalTransferPackets;
 119
 120     /*
 121      *  Pre-initialize our SCSI_REQUEST_BLOCK template with all
 122      *  the constant fields.  This will save a little time for each xfer.
 123      *  NOTE: a CdbLength field of 10 may not always be appropriate
 124      */
 125     RtlZeroMemory(&fdoData->SrbTemplate, sizeof(SCSI_REQUEST_BLOCK));
 126     fdoData->SrbTemplate.Length = sizeof(SCSI_REQUEST_BLOCK);
 127     fdoData->SrbTemplate.Function = SRB_FUNCTION_EXECUTE_SCSI;
 128     fdoData->SrbTemplate.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 129     fdoData->SrbTemplate.SenseInfoBufferLength = sizeof(SENSE_DATA);
 130     fdoData->SrbTemplate.CdbLength = 10;
 131
 132     return status;
 133 }
 134
 135 VOID NTAPI DestroyAllTransferPackets(PDEVICE_OBJECT Fdo)
 136 {
 137     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 138     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 139     TRANSFER_PACKET *pkt;
 140
 141     PAGED_CODE();
 142
 143     ASSERT(IsListEmpty(&fdoData->DeferredClientIrpList));
 144
 145     while (pkt = DequeueFreeTransferPacket(Fdo, FALSE)){
 146         DestroyTransferPacket(pkt);
 147         InterlockedDecrement(&fdoData->NumTotalTransferPackets);
 148     }
 149
 150     ASSERT(fdoData->NumTotalTransferPackets == 0);
 151 }
 152
 153 PTRANSFER_PACKET NTAPI NewTransferPacket(PDEVICE_OBJECT Fdo)
 154 {
 155     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 156     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 157     PTRANSFER_PACKET newPkt;
 158
 159     newPkt = ExAllocatePoolWithTag(NonPagedPool, sizeof(TRANSFER_PACKET), 'pnPC');
 160     if (newPkt){
 161         RtlZeroMemory(newPkt, sizeof(TRANSFER_PACKET)); // just to be sure
 162
 163         /*
 164          *  Allocate resources for the packet.
 165          */
 166         newPkt->Irp = IoAllocateIrp(Fdo->StackSize, FALSE);
 167         if (newPkt->Irp){
 168             KIRQL oldIrql;
 169
 170             newPkt->Fdo = Fdo;
 171
 172             /*
 173              *  Enqueue the packet in our static AllTransferPacketsList
 174              *  (just so we can find it during debugging if its stuck somewhere).
 175              */
 176             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 177             InsertTailList(&fdoData->AllTransferPacketsList, &newPkt->AllPktsListEntry);
 178             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 179         }
 180         else {
 181             ExFreePool(newPkt);
 182             newPkt = NULL;
 183         }
 184     }
 185
 186     return newPkt;
 187 }
 188
 189 /*
 190  *  DestroyTransferPacket
 191  *
 192  */
 193 VOID NTAPI DestroyTransferPacket(PTRANSFER_PACKET Pkt)
 194 {
 195     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 196     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 197     KIRQL oldIrql;
 198
 199     ASSERT(!Pkt->SlistEntry.Next);
 200     ASSERT(!Pkt->OriginalIrp);
 201
 202     KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 203
 204     /*
 205      *  Delete the packet from our all-packets queue.
 206      */
 207     ASSERT(!IsListEmpty(&Pkt->AllPktsListEntry));
 208     ASSERT(!IsListEmpty(&fdoData->AllTransferPacketsList));
 209     RemoveEntryList(&Pkt->AllPktsListEntry);
 210     InitializeListHead(&Pkt->AllPktsListEntry);
 211
 212     KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 213
 214     IoFreeIrp(Pkt->Irp);
 215     ExFreePool(Pkt);
 216 }
 217
 218 VOID NTAPI EnqueueFreeTransferPacket(PDEVICE_OBJECT Fdo, PTRANSFER_PACKET Pkt)
 219 {
 220     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 221     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 222     KIRQL oldIrql;
 223     ULONG newNumPkts;
 224
 225     ASSERT(!Pkt->SlistEntry.Next);
 226
 227     InterlockedPushEntrySList(&fdoData->FreeTransferPacketsList, &Pkt->SlistEntry);
 228     newNumPkts = InterlockedIncrement(&fdoData->NumFreeTransferPackets);
 229     ASSERT(newNumPkts <= fdoData->NumTotalTransferPackets);
 230
 231     /*
 232      *  If the total number of packets is larger than MinWorkingSetTransferPackets,
 233      *  that means that we've been in stress.  If all those packets are now
 234      *  free, then we are now out of stress and can free the extra packets.
 235      *  Free down to MaxWorkingSetTransferPackets immediately, and
 236      *  down to MinWorkingSetTransferPackets lazily (one at a time).
 237      */
 238     if (fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets){
 239
 240         /*
 241          *  1.  Immediately snap down to our UPPER threshold.
 242          */
 243         if (fdoData->NumTotalTransferPackets > MaxWorkingSetTransferPackets){
 244             SINGLE_LIST_ENTRY pktList;
 245             PSINGLE_LIST_ENTRY slistEntry;
 246             PTRANSFER_PACKET pktToDelete;
 247
 248             DBGTRACE(ClassDebugTrace, ("Exiting stress, block freeing (%d-%d) packets.", fdoData->NumTotalTransferPackets, MaxWorkingSetTransferPackets));
 249
 250             /*
 251              *  Check the counter again with lock held.  This eliminates a race condition
 252              *  while still allowing us to not grab the spinlock in the common codepath.
 253              *
 254              *  Note that the spinlock does not synchronize with threads dequeuing free
 255              *  packets to send (DequeueFreeTransferPacket does that with a lightweight
 256              *  interlocked exchange); the spinlock prevents multiple threads in this function
 257              *  from deciding to free too many extra packets at once.
 258              */
 259             SimpleInitSlistHdr(&pktList);
 260             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 261             while ((fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets) &&
 262                    (fdoData->NumTotalTransferPackets > MaxWorkingSetTransferPackets)){
 263
 264                 pktToDelete = DequeueFreeTransferPacket(Fdo, FALSE);
 265                 if (pktToDelete){
 266                     SimplePushSlist(&pktList, &pktToDelete->SlistEntry);
 267                     InterlockedDecrement(&fdoData->NumTotalTransferPackets);
 268                 }
 269                 else {
 270                     DBGTRACE(ClassDebugTrace, ("Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (1).", MaxWorkingSetTransferPackets, Fdo, fdoData->NumTotalTransferPackets));
 271                     break;
 272                 }
 273             }
 274             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 275
 276             while (slistEntry = SimplePopSlist(&pktList)){
 277                 pktToDelete = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
 278                 DestroyTransferPacket(pktToDelete);
 279             }
 280
 281         }
 282
 283         /*
 284          *  2.  Lazily work down to our LOWER threshold (by only freeing one packet at a time).
 285          */
 286         if (fdoData->NumTotalTransferPackets > MinWorkingSetTransferPackets){
 287             /*
 288              *  Check the counter again with lock held.  This eliminates a race condition
 289              *  while still allowing us to not grab the spinlock in the common codepath.
 290              *
 291              *  Note that the spinlock does not synchronize with threads dequeuing free
 292              *  packets to send (DequeueFreeTransferPacket does that with a lightweight
 293              *  interlocked exchange); the spinlock prevents multiple threads in this function
 294              *  from deciding to free too many extra packets at once.
 295              */
 296             PTRANSFER_PACKET pktToDelete = NULL;
 297
 298             DBGTRACE(ClassDebugTrace, ("Exiting stress, lazily freeing one of %d/%d packets.", fdoData->NumTotalTransferPackets, MinWorkingSetTransferPackets));
 299
 300             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 301             if ((fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets) &&
 302                 (fdoData->NumTotalTransferPackets > MinWorkingSetTransferPackets)){
 303
 304                 pktToDelete = DequeueFreeTransferPacket(Fdo, FALSE);
 305                 if (pktToDelete){
 306                     InterlockedDecrement(&fdoData->NumTotalTransferPackets);
 307                 }
 308                 else {
 309                     DBGTRACE(ClassDebugTrace, ("Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (2).", MinWorkingSetTransferPackets, Fdo, fdoData->NumTotalTransferPackets));
 310                 }
 311             }
 312             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 313
 314             if (pktToDelete){
 315                 DestroyTransferPacket(pktToDelete);
 316             }
 317         }
 318
 319     }
 320
 321 }
 322
 323 PTRANSFER_PACKET NTAPI DequeueFreeTransferPacket(PDEVICE_OBJECT Fdo, BOOLEAN AllocIfNeeded)
 324 {
 325     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 326     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 327     PTRANSFER_PACKET pkt;
 328     PSINGLE_LIST_ENTRY slistEntry;
 329     //KIRQL oldIrql;
 330
 331     slistEntry = InterlockedPopEntrySList(&fdoData->FreeTransferPacketsList);
 332     if (slistEntry){
 333         slistEntry->Next = NULL;
 334         pkt = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
 335         ASSERT(fdoData->NumFreeTransferPackets > 0);
 336         InterlockedDecrement(&fdoData->NumFreeTransferPackets);
 337     }
 338     else {
 339         if (AllocIfNeeded){
 340             /*
 341              *  We are in stress and have run out of lookaside packets.
 342              *  In order to service the current transfer,
 343              *  allocate an extra packet.
 344              *  We will free it lazily when we are out of stress.
 345              */
 346             pkt = NewTransferPacket(Fdo);
 347             if (pkt){
 348                 InterlockedIncrement(&fdoData->NumTotalTransferPackets);
 349                 fdoData->DbgPeakNumTransferPackets = max(fdoData->DbgPeakNumTransferPackets, fdoData->NumTotalTransferPackets);
 350             }
 351             else {
 352                 DBGWARN(("DequeueFreeTransferPacket: packet allocation failed"));
 353             }
 354         }
 355         else {
 356             pkt = NULL;
 357         }
 358     }
 359
 360     return pkt;
 361 }
 362
 363 /*
 364  *  SetupReadWriteTransferPacket
 365  *
 366  *        This function is called once to set up the first attempt to send a packet.
 367  *        It is not called before a retry, as SRB fields may be modified for the retry.
 368  *
 369  *      Set up the Srb of the TRANSFER_PACKET for the transfer.
 370  *        The Irp is set up in SubmitTransferPacket because it must be reset
 371  *        for each packet submission.
 372  */
 373 VOID NTAPI SetupReadWriteTransferPacket(PTRANSFER_PACKET Pkt,
 374                                         PVOID Buf,
 375                                         ULONG Len,
 376                                         LARGE_INTEGER DiskLocation,
 377                                         PIRP OriginalIrp)
 378 {
 379     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 380     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 381     PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(OriginalIrp);
 382     UCHAR majorFunc = origCurSp->MajorFunction;
 383     ULONG logicalBlockAddr;
 384     ULONG numTransferBlocks;
 385     PCDB pCdb;
 386
 387     logicalBlockAddr = (ULONG)Int64ShrlMod32(DiskLocation.QuadPart, fdoExt->SectorShift);
 388     numTransferBlocks = Len >> fdoExt->SectorShift;
 389
 390     /*
 391      *  Slap the constant SRB fields in from our pre-initialized template.
 392      *  We'll then only have to fill in the unique fields for this transfer.
 393      *  Tell lower drivers to sort the SRBs by the logical block address
 394      *  so that disk seeks are minimized.
 395      */
 396     Pkt->Srb = fdoData->SrbTemplate;    // copies _contents_ of SRB blocks
 397     Pkt->Srb.DataBuffer = Buf;
 398     Pkt->Srb.DataTransferLength = Len;
 399     Pkt->Srb.QueueSortKey = logicalBlockAddr;
 400     Pkt->Srb.OriginalRequest = Pkt->Irp;
 401     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 402     Pkt->Srb.TimeOutValue = (Len/0x10000) + ((Len%0x10000) ? 1 : 0);
 403     Pkt->Srb.TimeOutValue *= fdoExt->TimeOutValue;
 404
 405     /*
 406      *  Arrange values in CDB in big-endian format.
 407      */
 408     pCdb = (PCDB)Pkt->Srb.Cdb;
 409     pCdb->CDB10.LogicalBlockByte0 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte3;
 410     pCdb->CDB10.LogicalBlockByte1 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte2;
 411     pCdb->CDB10.LogicalBlockByte2 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte1;
 412     pCdb->CDB10.LogicalBlockByte3 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte0;
 413     pCdb->CDB10.TransferBlocksMsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte1;
 414     pCdb->CDB10.TransferBlocksLsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte0;
 415     pCdb->CDB10.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ : SCSIOP_WRITE;
 416
 417     /*
 418      *  Set SRB and IRP flags
 419      */
 420     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 421     if (TEST_FLAG(OriginalIrp->Flags, IRP_PAGING_IO) ||
 422         TEST_FLAG(OriginalIrp->Flags, IRP_SYNCHRONOUS_PAGING_IO)){
 423         SET_FLAG(Pkt->Srb.SrbFlags, SRB_CLASS_FLAGS_PAGING);
 424     }
 425     SET_FLAG(Pkt->Srb.SrbFlags, (majorFunc==IRP_MJ_READ) ? SRB_FLAGS_DATA_IN : SRB_FLAGS_DATA_OUT);
 426
 427     /*
 428      *  Allow caching only if this is not a write-through request.
 429      *  If write-through and caching is enabled on the device, force
 430      *  media access.
 431      */
 432     if (TEST_FLAG(origCurSp->Flags, SL_WRITE_THROUGH)){
 433         if (TEST_FLAG(fdoExt->DeviceFlags, DEV_WRITE_CACHE)){
 434             pCdb->CDB10.ForceUnitAccess = TRUE;
 435         }
 436     }
 437     else {
 438         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_ADAPTER_CACHE_ENABLE);
 439     }
 440
 441     /*
 442      *  Remember the buf and len in the SRB because miniports
 443      *  can overwrite SRB.DataTransferLength and we may need it again
 444      *  for the retry.
 445      */
 446     Pkt->BufPtrCopy = Buf;
 447     Pkt->BufLenCopy = Len;
 448     Pkt->TargetLocationCopy = DiskLocation;
 449
 450     Pkt->OriginalIrp = OriginalIrp;
 451     Pkt->NumRetries = MAXIMUM_RETRIES;
 452     Pkt->SyncEventPtr = NULL;
 453     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = TRUE;
 454 }
 455
 456 /*
 457  *  SubmitTransferPacket
 458  *
 459  *        Set up the IRP for the TRANSFER_PACKET submission and send it down.
 460  */
 461 VOID NTAPI SubmitTransferPacket(PTRANSFER_PACKET Pkt)
 462 {
 463     PCOMMON_DEVICE_EXTENSION commonExtension = Pkt->Fdo->DeviceExtension;
 464     PDEVICE_OBJECT nextDevObj = commonExtension->LowerDeviceObject;
 465     PIO_STACK_LOCATION nextSp = IoGetNextIrpStackLocation(Pkt->Irp);
 466
 467     ASSERT(Pkt->Irp->CurrentLocation == Pkt->Irp->StackCount+1);
 468
 469     /*
 470      *  Attach the SRB to the IRP.
 471      *  The reused IRP's stack location has to be rewritten for each retry
 472      *  call because IoCompleteRequest clears the stack locations.
 473      */
 474     IoReuseIrp(Pkt->Irp, STATUS_NOT_SUPPORTED);
 475     nextSp->MajorFunction = IRP_MJ_SCSI;
 476     nextSp->Parameters.Scsi.Srb = &Pkt->Srb;
 477     Pkt->Srb.ScsiStatus = Pkt->Srb.SrbStatus = 0;
 478     if (Pkt->CompleteOriginalIrpWhenLastPacketCompletes){
 479         /*
 480          *  Only dereference the "original IRP"'s stack location
 481          *  if its a real client irp (as opposed to a static irp
 482          *  we're using just for result status for one of the non-IO scsi commands).
 483          *
 484          *  For read/write, propagate the storage-specific IRP stack location flags
 485          *  (e.g. SL_OVERRIDE_VERIFY_VOLUME, SL_WRITE_THROUGH).
 486          */
 487         PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(Pkt->OriginalIrp);
 488         nextSp->Flags = origCurSp->Flags;
 489     }
 490
 491     /*
 492      *  Write MDL address to new IRP. In the port driver the SRB DataBuffer
 493      *  field is used as the actual buffer pointer within the MDL,
 494      *  so the same MDL can be used for each partial transfer.
 495      *  This saves having to build a new MDL for each partial transfer.
 496      */
 497     Pkt->Irp->MdlAddress = Pkt->OriginalIrp->MdlAddress;
 498
 499     IoSetCompletionRoutine(Pkt->Irp, TransferPktComplete, Pkt, TRUE, TRUE, TRUE);
 500     IoCallDriver(nextDevObj, Pkt->Irp);
 501 }
 502
 503 NTSTATUS NTAPI TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
 504 {
 505     PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
 506     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
 507     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 508     PIO_STACK_LOCATION origCurrentSp = IoGetCurrentIrpStackLocation(pkt->OriginalIrp);
 509     BOOLEAN packetDone = FALSE;
 510
 511     /*
 512      *  Put all the assertions and spew in here so we don't have to look at them.
 513      */
 514     DBGCHECKRETURNEDPKT(pkt);
 515
 516     if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS){
 517
 518         fdoData->LoggedTURFailureSinceLastIO = FALSE;
 519
 520         /*
 521          *  The port driver should not have allocated a sense buffer
 522          *  if the SRB succeeded.
 523          */
 524         ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
 525
 526         /*
 527          *  Add this packet's transferred length to the original IRP's.
 528          */
 529         InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
 530                               (LONG)pkt->Srb.DataTransferLength);
 531
 532         if (pkt->InLowMemRetry){
 533             packetDone = StepLowMemRetry(pkt);
 534         }
 535         else {
 536             packetDone = TRUE;
 537         }
 538
 539     }
 540     else {
 541         /*
 542          *  The packet failed.  We may retry it if possible.
 543          */
 544         BOOLEAN shouldRetry;
 545
 546         /*
 547          *  Make sure IRP status matches SRB error status (since we propagate it).
 548          */
 549         if (NT_SUCCESS(Irp->IoStatus.Status)){
 550             Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
 551         }
 552
 553         /*
 554          *  Interpret the SRB error (to a meaningful IRP status)
 555          *  and determine if we should retry this packet.
 556          *  This call looks at the returned SENSE info to figure out what to do.
 557          */
 558         shouldRetry = InterpretTransferPacketError(pkt);
 559
 560         /*
 561          *  Sometimes the port driver can allocates a new 'sense' buffer
 562          *  to report transfer errors, e.g. when the default sense buffer
 563          *  is too small.  If so, it is up to us to free it.
 564          *  Now that we're done interpreting the sense info, free it if appropriate.
 565          */
 566         if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
 567             DBGTRACE(ClassDebugSenseInfo, ("Freeing port-allocated sense buffer for pkt %ph.", pkt));
 568             FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
 569             pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
 570             pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 571         }
 572
 573         /*
 574          *  If the SRB queue is locked-up, release it.
 575          *  Do this after calling the error handler.
 576          */
 577         if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
 578             ClassReleaseQueue(pkt->Fdo);
 579         }
 580
 581         if (shouldRetry && (pkt->NumRetries > 0)){
 582             packetDone = RetryTransferPacket(pkt);
 583         }
 584         else {
 585             packetDone = TRUE;
 586         }
 587
 588     }
 589
 590     /*
 591      *  If the packet is completed, put it back in the free list.
 592      *  If it is the last packet servicing the original request, complete the original irp.
 593      */
 594     if (packetDone){
 595         LONG numPacketsRemaining;
 596         PIRP deferredIrp;
 597         PDEVICE_OBJECT Fdo = pkt->Fdo;
 598         UCHAR uniqueAddr;
 599
 600         /*
 601          *  In case a remove is pending, bump the lock count so we don't get freed
 602          *  right after we complete the original irp.
 603          */
 604         ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
 605
 606         /*
 607          *  The original IRP should get an error code
 608          *  if any one of the packets failed.
 609          */
 610         if (!NT_SUCCESS(Irp->IoStatus.Status)){
 611             pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
 612
 613             /*
 614              *  If the original I/O originated in user space (i.e. it is thread-queued),
 615              *  and the error is user-correctable (e.g. media is missing, for removable media),
 616              *  alert the user.
 617              *  Since this is only one of possibly several packets completing for the original IRP,
 618              *  we may do this more than once for a single request.  That's ok; this allows
 619              *  us to test each returned status with IoIsErrorUserInduced().
 620              */
 621             if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
 622                 pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
 623                 pkt->OriginalIrp->Tail.Overlay.Thread){
 624
 625                 IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, pkt->Fdo);
 626             }
 627         }
 628
 629         /*
 630          *  We use a field in the original IRP to count
 631          *  down the transfer pieces as they complete.
 632          */
 633         numPacketsRemaining = InterlockedDecrement(
 634             (PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
 635
 636         if (numPacketsRemaining > 0){
 637             /*
 638              *  More transfer pieces remain for the original request.
 639              *  Wait for them to complete before completing the original irp.
 640              */
 641         }
 642         else {
 643
 644             /*
 645              *  All the transfer pieces are done.
 646              *  Complete the original irp if appropriate.
 647              */
 648             ASSERT(numPacketsRemaining == 0);
 649             if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
 650                 if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
 651                     ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == origCurrentSp->Parameters.Read.Length);
 652                     ClasspPerfIncrementSuccessfulIo(fdoExt);
 653                 }
 654                 ClassReleaseRemoveLock(pkt->Fdo, pkt->OriginalIrp);
 655
 656                 ClassCompleteRequest(pkt->Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
 657
 658                 /*
 659                  *  We may have been called by one of the class drivers (e.g. cdrom)
 660                  *  via the legacy API ClassSplitRequest.
 661                  *  This is the only case for which the packet engine is called for an FDO
 662                  *  with a StartIo routine; in that case, we have to call IoStartNextPacket
 663                  *  now that the original irp has been completed.
 664                  */
 665                 if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
 666                     if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
 667                         DBGTRAP(("SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (?)"));
 668                     }
 669                     else {
 670                         KIRQL oldIrql;
 671                         KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
 672                         IoStartNextPacket(pkt->Fdo, FALSE);
 673                         KeLowerIrql(oldIrql);
 674                     }
 675                 }
 676             }
 677         }
 678
 679         /*
 680          *  If the packet was synchronous, write the final
 681          *  result back to the issuer's status buffer and
 682          *  signal his event.
 683          */
 684         if (pkt->SyncEventPtr){
 685             KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
 686             pkt->SyncEventPtr = NULL;
 687         }
 688
 689         /*
 690          *  Free the completed packet.
 691          */
 692         pkt->OriginalIrp = NULL;
 693         pkt->InLowMemRetry = FALSE;
 694         EnqueueFreeTransferPacket(pkt->Fdo, pkt);
 695
 696         /*
 697          *  Now that we have freed some resources,
 698          *  try again to send one of the previously deferred irps.
 699          */
 700         deferredIrp = DequeueDeferredClientIrp(fdoData);
 701         if (deferredIrp){
 702             DBGWARN(("... retrying deferred irp %xh.", deferredIrp));
 703             ServiceTransferRequest(pkt->Fdo, deferredIrp);
 704         }
 705
 706         ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
 707     }
 708
 709     return STATUS_MORE_PROCESSING_REQUIRED;
 710 }
 711
 712 /*
 713  *  SetupEjectionTransferPacket
 714  *
 715  *      Set up a transferPacket for a synchronous Ejection Control transfer.
 716  */
 717 VOID NTAPI SetupEjectionTransferPacket(   TRANSFER_PACKET *Pkt,
 718                                         BOOLEAN PreventMediaRemoval,
 719                                         PKEVENT SyncEventPtr,
 720                                         PIRP OriginalIrp)
 721 {
 722     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 723     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 724     PCDB pCdb;
 725
 726     PAGED_CODE();
 727
 728     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 729
 730     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 731     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 732     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 733     Pkt->Srb.CdbLength = 6;
 734     Pkt->Srb.OriginalRequest = Pkt->Irp;
 735     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 736     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 737     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 738
 739     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 740     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 741     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 742
 743     pCdb = (PCDB)Pkt->Srb.Cdb;
 744     pCdb->MEDIA_REMOVAL.OperationCode = SCSIOP_MEDIUM_REMOVAL;
 745     pCdb->MEDIA_REMOVAL.Prevent = PreventMediaRemoval;
 746
 747     Pkt->BufPtrCopy = NULL;
 748     Pkt->BufLenCopy = 0;
 749
 750     Pkt->OriginalIrp = OriginalIrp;
 751     Pkt->NumRetries = NUM_LOCKMEDIAREMOVAL_RETRIES;
 752     Pkt->SyncEventPtr = SyncEventPtr;
 753     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 754 }
 755
 756 /*
 757  *  SetupModeSenseTransferPacket
 758  *
 759  *      Set up a transferPacket for a synchronous Mode Sense transfer.
 760  */
 761 VOID NTAPI SetupModeSenseTransferPacket(TRANSFER_PACKET *Pkt,
 762                                         PKEVENT SyncEventPtr,
 763                                         PVOID ModeSenseBuffer,
 764                                         UCHAR ModeSenseBufferLen,
 765                                         UCHAR PageMode,
 766                                         PIRP OriginalIrp)
 767 {
 768     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 769     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 770     PCDB pCdb;
 771
 772     PAGED_CODE();
 773
 774     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 775
 776     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 777     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 778     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 779     Pkt->Srb.CdbLength = 6;
 780     Pkt->Srb.OriginalRequest = Pkt->Irp;
 781     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 782     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 783     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 784     Pkt->Srb.DataBuffer = ModeSenseBuffer;
 785     Pkt->Srb.DataTransferLength = ModeSenseBufferLen;
 786
 787     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 788     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DATA_IN);
 789     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 790     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 791
 792     pCdb = (PCDB)Pkt->Srb.Cdb;
 793     pCdb->MODE_SENSE.OperationCode = SCSIOP_MODE_SENSE;
 794     pCdb->MODE_SENSE.PageCode = PageMode;
 795     pCdb->MODE_SENSE.AllocationLength = (UCHAR)ModeSenseBufferLen;
 796
 797     Pkt->BufPtrCopy = ModeSenseBuffer;
 798     Pkt->BufLenCopy = ModeSenseBufferLen;
 799
 800     Pkt->OriginalIrp = OriginalIrp;
 801     Pkt->NumRetries = NUM_MODESENSE_RETRIES;
 802     Pkt->SyncEventPtr = SyncEventPtr;
 803     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 804 }
 805
 806 /*
 807  *  SetupDriveCapacityTransferPacket
 808  *
 809  *      Set up a transferPacket for a synchronous Drive Capacity transfer.
 810  */
 811 VOID NTAPI SetupDriveCapacityTransferPacket(TRANSFER_PACKET *Pkt,
 812                                             PVOID ReadCapacityBuffer,
 813                                             ULONG ReadCapacityBufferLen,
 814                                             PKEVENT SyncEventPtr,
 815                                             PIRP OriginalIrp)
 816 {
 817     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 818     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 819     PCDB pCdb;
 820
 821     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 822
 823     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 824     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 825     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 826     Pkt->Srb.CdbLength = 10;
 827     Pkt->Srb.OriginalRequest = Pkt->Irp;
 828     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 829     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 830     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 831     Pkt->Srb.DataBuffer = ReadCapacityBuffer;
 832     Pkt->Srb.DataTransferLength = ReadCapacityBufferLen;
 833
 834     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 835     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DATA_IN);
 836     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 837     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 838
 839     pCdb = (PCDB)Pkt->Srb.Cdb;
 840     pCdb->CDB10.OperationCode = SCSIOP_READ_CAPACITY;
 841
 842     Pkt->BufPtrCopy = ReadCapacityBuffer;
 843     Pkt->BufLenCopy = ReadCapacityBufferLen;
 844
 845     Pkt->OriginalIrp = OriginalIrp;
 846     Pkt->NumRetries = NUM_DRIVECAPACITY_RETRIES;
 847     Pkt->SyncEventPtr = SyncEventPtr;
 848     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 849 }
 850
 851 #if 0
 852     /*
 853      *  SetupSendStartUnitTransferPacket
 854      *
 855      *      Set up a transferPacket for a synchronous Send Start Unit transfer.
 856      */
 857     VOID SetupSendStartUnitTransferPacket(   TRANSFER_PACKET *Pkt,
 858                                                     PIRP OriginalIrp)
 859     {
 860         PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 861         PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 862         PCDB pCdb;
 863
 864         PAGED_CODE();
 865
 866         RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 867
 868         /*
 869          *  Initialize the SRB.
 870          *  Use a very long timeout value to give the drive time to spin up.
 871          */
 872         Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 873         Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 874         Pkt->Srb.TimeOutValue = START_UNIT_TIMEOUT;
 875         Pkt->Srb.CdbLength = 6;
 876         Pkt->Srb.OriginalRequest = Pkt->Irp;
 877         Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 878         Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 879         Pkt->Srb.Lun = 0;
 880
 881         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_DATA_TRANSFER);
 882         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_AUTOSENSE);
 883         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 884
 885         pCdb = (PCDB)Pkt->Srb.Cdb;
 886         pCdb->START_STOP.OperationCode = SCSIOP_START_STOP_UNIT;
 887         pCdb->START_STOP.Start = 1;
 888         pCdb->START_STOP.Immediate = 0;
 889         pCdb->START_STOP.LogicalUnitNumber = 0;
 890
 891         Pkt->OriginalIrp = OriginalIrp;
 892         Pkt->NumRetries = 0;
 893         Pkt->SyncEventPtr = NULL;
 894         Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 895     }
 896 #endif