drivers/storage/classpnp/xferpkt.c

   1 /*++
   2
   3 Copyright (C) Microsoft Corporation, 1991 - 1999
   4
   5 Module Name:
   6
   7     xferpkt.c
   8
   9 Abstract:
  10
  11     Packet routines for CLASSPNP
  12
  13 Environment:
  14
  15     kernel mode only
  16
  17 Notes:
  18
  19
  20 Revision History:
  21
  22 --*/
  23
  24 #include "classp.h"
  25
  26 #ifdef ALLOC_PRAGMA
  27     #pragma alloc_text(PAGE, InitializeTransferPackets)
  28     #pragma alloc_text(PAGE, DestroyAllTransferPackets)
  29     #pragma alloc_text(PAGE, SetupEjectionTransferPacket)
  30     #pragma alloc_text(PAGE, SetupModeSenseTransferPacket)
  31 #endif
  32
  33
  34 ULONG MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
  35 ULONG MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
  36
  37
  38 /*
  39  *  InitializeTransferPackets
  40  *
  41  *      Allocate/initialize TRANSFER_PACKETs and related resources.
  42  */
  43 NTSTATUS NTAPI InitializeTransferPackets(PDEVICE_OBJECT Fdo)
  44 {
  45     PCOMMON_DEVICE_EXTENSION commonExt = Fdo->DeviceExtension;
  46     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
  47     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
  48     PSTORAGE_ADAPTER_DESCRIPTOR adapterDesc = commonExt->PartitionZeroExtension->AdapterDescriptor;
  49     ULONG hwMaxPages;
  50     NTSTATUS status = STATUS_SUCCESS;
  51
  52     PAGED_CODE();
  53
  54     /*
  55      *  Precompute the maximum transfer length
  56      */
  57     ASSERT(adapterDesc->MaximumTransferLength);
  58     ASSERT(adapterDesc->MaximumPhysicalPages);
  59     hwMaxPages = adapterDesc->MaximumPhysicalPages ? adapterDesc->MaximumPhysicalPages-1 : 0;
  60
  61 #if defined(_AMD64_SIMULATOR_)
  62
  63     //
  64     // The simulator appears to have a problem with large transfers.
  65     //
  66
  67     if (hwMaxPages > 4) {
  68         hwMaxPages = 4;
  69     }
  70
  71 #endif
  72
  73     fdoData->HwMaxXferLen = MIN(adapterDesc->MaximumTransferLength, hwMaxPages << PAGE_SHIFT);
  74     fdoData->HwMaxXferLen = MAX(fdoData->HwMaxXferLen, PAGE_SIZE);
  75
  76     fdoData->NumTotalTransferPackets = 0;
  77     fdoData->NumFreeTransferPackets = 0;
  78     InitializeSListHead(&fdoData->FreeTransferPacketsList);
  79     InitializeListHead(&fdoData->AllTransferPacketsList);
  80     InitializeListHead(&fdoData->DeferredClientIrpList);
  81
  82     /*
  83      *  Set the packet threshold numbers based on the Windows SKU.
  84      */
  85     if (ExVerifySuite(Personal)){
  86         // this is Windows Personal
  87         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
  88         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
  89     }
  90     else if (ExVerifySuite(Enterprise) || ExVerifySuite(DataCenter)){
  91         // this is Advanced Server or Datacenter
  92         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Enterprise;
  93         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Enterprise;
  94     }
  95     else if (ExVerifySuite(TerminalServer)){
  96         // this is standard Server or Pro with terminal server
  97         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Server;
  98         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Server;
  99     }
 100     else {
 101         // this is Professional without terminal server
 102         MinWorkingSetTransferPackets = MIN_WORKINGSET_TRANSFER_PACKETS_Consumer;
 103         MaxWorkingSetTransferPackets = MAX_WORKINGSET_TRANSFER_PACKETS_Consumer;
 104     }
 105
 106     while (fdoData->NumFreeTransferPackets < MIN_INITIAL_TRANSFER_PACKETS){
 107         PTRANSFER_PACKET pkt = NewTransferPacket(Fdo);
 108         if (pkt){
 109             InterlockedIncrement((PLONG)&fdoData->NumTotalTransferPackets);
 110             EnqueueFreeTransferPacket(Fdo, pkt);
 111         }
 112         else {
 113             status = STATUS_INSUFFICIENT_RESOURCES;
 114             break;
 115         }
 116     }
 117     fdoData->DbgPeakNumTransferPackets = fdoData->NumTotalTransferPackets;
 118
 119     /*
 120      *  Pre-initialize our SCSI_REQUEST_BLOCK template with all
 121      *  the constant fields.  This will save a little time for each xfer.
 122      *  NOTE: a CdbLength field of 10 may not always be appropriate
 123      */
 124     RtlZeroMemory(&fdoData->SrbTemplate, sizeof(SCSI_REQUEST_BLOCK));
 125     fdoData->SrbTemplate.Length = sizeof(SCSI_REQUEST_BLOCK);
 126     fdoData->SrbTemplate.Function = SRB_FUNCTION_EXECUTE_SCSI;
 127     fdoData->SrbTemplate.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 128     fdoData->SrbTemplate.SenseInfoBufferLength = sizeof(SENSE_DATA);
 129     fdoData->SrbTemplate.CdbLength = 10;
 130
 131     return status;
 132 }
 133
 134 VOID NTAPI DestroyAllTransferPackets(PDEVICE_OBJECT Fdo)
 135 {
 136     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 137     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 138     TRANSFER_PACKET *pkt;
 139
 140     PAGED_CODE();
 141
 142     ASSERT(IsListEmpty(&fdoData->DeferredClientIrpList));
 143
 144     while ((pkt = DequeueFreeTransferPacket(Fdo, FALSE))){
 145         DestroyTransferPacket(pkt);
 146         InterlockedDecrement((PLONG)&fdoData->NumTotalTransferPackets);
 147     }
 148
 149     ASSERT(fdoData->NumTotalTransferPackets == 0);
 150 }
 151
 152 PTRANSFER_PACKET NTAPI NewTransferPacket(PDEVICE_OBJECT Fdo)
 153 {
 154     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 155     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 156     PTRANSFER_PACKET newPkt;
 157
 158     newPkt = ExAllocatePoolWithTag(NonPagedPool, sizeof(TRANSFER_PACKET), 'pnPC');
 159     if (newPkt){
 160         RtlZeroMemory(newPkt, sizeof(TRANSFER_PACKET)); // just to be sure
 161
 162         /*
 163          *  Allocate resources for the packet.
 164          */
 165         newPkt->Irp = IoAllocateIrp(Fdo->StackSize, FALSE);
 166         if (newPkt->Irp){
 167             KIRQL oldIrql;
 168
 169             newPkt->Fdo = Fdo;
 170
 171             /*
 172              *  Enqueue the packet in our static AllTransferPacketsList
 173              *  (just so we can find it during debugging if its stuck somewhere).
 174              */
 175             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 176             InsertTailList(&fdoData->AllTransferPacketsList, &newPkt->AllPktsListEntry);
 177             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 178         }
 179         else {
 180             ExFreePool(newPkt);
 181             newPkt = NULL;
 182         }
 183     }
 184
 185     return newPkt;
 186 }
 187
 188 /*
 189  *  DestroyTransferPacket
 190  *
 191  */
 192 VOID NTAPI DestroyTransferPacket(PTRANSFER_PACKET Pkt)
 193 {
 194     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 195     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 196     KIRQL oldIrql;
 197
 198     ASSERT(!Pkt->SlistEntry.Next);
 199     ASSERT(!Pkt->OriginalIrp);
 200
 201     KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 202
 203     /*
 204      *  Delete the packet from our all-packets queue.
 205      */
 206     ASSERT(!IsListEmpty(&Pkt->AllPktsListEntry));
 207     ASSERT(!IsListEmpty(&fdoData->AllTransferPacketsList));
 208     RemoveEntryList(&Pkt->AllPktsListEntry);
 209     InitializeListHead(&Pkt->AllPktsListEntry);
 210
 211     KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 212
 213     IoFreeIrp(Pkt->Irp);
 214     ExFreePool(Pkt);
 215 }
 216
 217 VOID NTAPI EnqueueFreeTransferPacket(PDEVICE_OBJECT Fdo, PTRANSFER_PACKET Pkt)
 218 {
 219     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 220     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 221     KIRQL oldIrql;
 222     ULONG newNumPkts;
 223
 224     ASSERT(!Pkt->SlistEntry.Next);
 225
 226     InterlockedPushEntrySList(&fdoData->FreeTransferPacketsList, &Pkt->SlistEntry);
 227     newNumPkts = InterlockedIncrement((PLONG)&fdoData->NumFreeTransferPackets);
 228     ASSERT(newNumPkts <= fdoData->NumTotalTransferPackets);
 229
 230     /*
 231      *  If the total number of packets is larger than MinWorkingSetTransferPackets,
 232      *  that means that we've been in stress.  If all those packets are now
 233      *  free, then we are now out of stress and can free the extra packets.
 234      *  Free down to MaxWorkingSetTransferPackets immediately, and
 235      *  down to MinWorkingSetTransferPackets lazily (one at a time).
 236      */
 237     if (fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets){
 238
 239         /*
 240          *  1.  Immediately snap down to our UPPER threshold.
 241          */
 242         if (fdoData->NumTotalTransferPackets > MaxWorkingSetTransferPackets){
 243             SINGLE_LIST_ENTRY pktList;
 244             PSINGLE_LIST_ENTRY slistEntry;
 245             PTRANSFER_PACKET pktToDelete;
 246
 247             DBGTRACE(ClassDebugTrace, ("Exiting stress, block freeing (%d-%d) packets.", fdoData->NumTotalTransferPackets, MaxWorkingSetTransferPackets));
 248
 249             /*
 250              *  Check the counter again with lock held.  This eliminates a race condition
 251              *  while still allowing us to not grab the spinlock in the common codepath.
 252              *
 253              *  Note that the spinlock does not synchronize with threads dequeuing free
 254              *  packets to send (DequeueFreeTransferPacket does that with a lightweight
 255              *  interlocked exchange); the spinlock prevents multiple threads in this function
 256              *  from deciding to free too many extra packets at once.
 257              */
 258             SimpleInitSlistHdr(&pktList);
 259             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 260             while ((fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets) &&
 261                    (fdoData->NumTotalTransferPackets > MaxWorkingSetTransferPackets)){
 262
 263                 pktToDelete = DequeueFreeTransferPacket(Fdo, FALSE);
 264                 if (pktToDelete){
 265                     SimplePushSlist(&pktList, &pktToDelete->SlistEntry);
 266                     InterlockedDecrement((PLONG)&fdoData->NumTotalTransferPackets);
 267                 }
 268                 else {
 269                     DBGTRACE(ClassDebugTrace, ("Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (1).", MaxWorkingSetTransferPackets, Fdo, fdoData->NumTotalTransferPackets));
 270                     break;
 271                 }
 272             }
 273             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 274
 275             while ((slistEntry = SimplePopSlist(&pktList))){
 276                 pktToDelete = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
 277                 DestroyTransferPacket(pktToDelete);
 278             }
 279
 280         }
 281
 282         /*
 283          *  2.  Lazily work down to our LOWER threshold (by only freeing one packet at a time).
 284          */
 285         if (fdoData->NumTotalTransferPackets > MinWorkingSetTransferPackets){
 286             /*
 287              *  Check the counter again with lock held.  This eliminates a race condition
 288              *  while still allowing us to not grab the spinlock in the common codepath.
 289              *
 290              *  Note that the spinlock does not synchronize with threads dequeuing free
 291              *  packets to send (DequeueFreeTransferPacket does that with a lightweight
 292              *  interlocked exchange); the spinlock prevents multiple threads in this function
 293              *  from deciding to free too many extra packets at once.
 294              */
 295             PTRANSFER_PACKET pktToDelete = NULL;
 296
 297             DBGTRACE(ClassDebugTrace, ("Exiting stress, lazily freeing one of %d/%d packets.", fdoData->NumTotalTransferPackets, MinWorkingSetTransferPackets));
 298
 299             KeAcquireSpinLock(&fdoData->SpinLock, &oldIrql);
 300             if ((fdoData->NumFreeTransferPackets >= fdoData->NumTotalTransferPackets) &&
 301                 (fdoData->NumTotalTransferPackets > MinWorkingSetTransferPackets)){
 302
 303                 pktToDelete = DequeueFreeTransferPacket(Fdo, FALSE);
 304                 if (pktToDelete){
 305                     InterlockedDecrement((PLONG)&fdoData->NumTotalTransferPackets);
 306                 }
 307                 else {
 308                     DBGTRACE(ClassDebugTrace, ("Extremely unlikely condition (non-fatal): %d packets dequeued at once for Fdo %p. NumTotalTransferPackets=%d (2).", MinWorkingSetTransferPackets, Fdo, fdoData->NumTotalTransferPackets));
 309                 }
 310             }
 311             KeReleaseSpinLock(&fdoData->SpinLock, oldIrql);
 312
 313             if (pktToDelete){
 314                 DestroyTransferPacket(pktToDelete);
 315             }
 316         }
 317
 318     }
 319
 320 }
 321
 322 PTRANSFER_PACKET NTAPI DequeueFreeTransferPacket(PDEVICE_OBJECT Fdo, BOOLEAN AllocIfNeeded)
 323 {
 324     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Fdo->DeviceExtension;
 325     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 326     PTRANSFER_PACKET pkt;
 327     PSINGLE_LIST_ENTRY slistEntry;
 328     //KIRQL oldIrql;
 329
 330     slistEntry = InterlockedPopEntrySList(&fdoData->FreeTransferPacketsList);
 331     if (slistEntry){
 332         slistEntry->Next = NULL;
 333         pkt = CONTAINING_RECORD(slistEntry, TRANSFER_PACKET, SlistEntry);
 334         ASSERT(fdoData->NumFreeTransferPackets > 0);
 335         InterlockedDecrement((PLONG)&fdoData->NumFreeTransferPackets);
 336     }
 337     else {
 338         if (AllocIfNeeded){
 339             /*
 340              *  We are in stress and have run out of lookaside packets.
 341              *  In order to service the current transfer,
 342              *  allocate an extra packet.
 343              *  We will free it lazily when we are out of stress.
 344              */
 345             pkt = NewTransferPacket(Fdo);
 346             if (pkt){
 347                 InterlockedIncrement((PLONG)&fdoData->NumTotalTransferPackets);
 348                 fdoData->DbgPeakNumTransferPackets = max(fdoData->DbgPeakNumTransferPackets, fdoData->NumTotalTransferPackets);
 349             }
 350             else {
 351                 DBGWARN(("DequeueFreeTransferPacket: packet allocation failed"));
 352             }
 353         }
 354         else {
 355             pkt = NULL;
 356         }
 357     }
 358
 359     return pkt;
 360 }
 361
 362 /*
 363  *  SetupReadWriteTransferPacket
 364  *
 365  *        This function is called once to set up the first attempt to send a packet.
 366  *        It is not called before a retry, as SRB fields may be modified for the retry.
 367  *
 368  *      Set up the Srb of the TRANSFER_PACKET for the transfer.
 369  *        The Irp is set up in SubmitTransferPacket because it must be reset
 370  *        for each packet submission.
 371  */
 372 VOID NTAPI SetupReadWriteTransferPacket(PTRANSFER_PACKET Pkt,
 373                                         PVOID Buf,
 374                                         ULONG Len,
 375                                         LARGE_INTEGER DiskLocation,
 376                                         PIRP OriginalIrp)
 377 {
 378     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 379     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 380     PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(OriginalIrp);
 381     UCHAR majorFunc = origCurSp->MajorFunction;
 382     ULONG logicalBlockAddr;
 383     ULONG numTransferBlocks;
 384     PCDB pCdb;
 385
 386     logicalBlockAddr = (ULONG)Int64ShrlMod32(DiskLocation.QuadPart, fdoExt->SectorShift);
 387     numTransferBlocks = Len >> fdoExt->SectorShift;
 388
 389     /*
 390      *  Slap the constant SRB fields in from our pre-initialized template.
 391      *  We'll then only have to fill in the unique fields for this transfer.
 392      *  Tell lower drivers to sort the SRBs by the logical block address
 393      *  so that disk seeks are minimized.
 394      */
 395     Pkt->Srb = fdoData->SrbTemplate;    // copies _contents_ of SRB blocks
 396     Pkt->Srb.DataBuffer = Buf;
 397     Pkt->Srb.DataTransferLength = Len;
 398     Pkt->Srb.QueueSortKey = logicalBlockAddr;
 399     Pkt->Srb.OriginalRequest = Pkt->Irp;
 400     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 401     Pkt->Srb.TimeOutValue = (Len/0x10000) + ((Len%0x10000) ? 1 : 0);
 402     Pkt->Srb.TimeOutValue *= fdoExt->TimeOutValue;
 403
 404     /*
 405      *  Arrange values in CDB in big-endian format.
 406      */
 407     pCdb = (PCDB)Pkt->Srb.Cdb;
 408     pCdb->CDB10.LogicalBlockByte0 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte3;
 409     pCdb->CDB10.LogicalBlockByte1 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte2;
 410     pCdb->CDB10.LogicalBlockByte2 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte1;
 411     pCdb->CDB10.LogicalBlockByte3 = ((PFOUR_BYTE)&logicalBlockAddr)->Byte0;
 412     pCdb->CDB10.TransferBlocksMsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte1;
 413     pCdb->CDB10.TransferBlocksLsb = ((PFOUR_BYTE)&numTransferBlocks)->Byte0;
 414     pCdb->CDB10.OperationCode = (majorFunc==IRP_MJ_READ) ? SCSIOP_READ : SCSIOP_WRITE;
 415
 416     /*
 417      *  Set SRB and IRP flags
 418      */
 419     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 420     if (TEST_FLAG(OriginalIrp->Flags, IRP_PAGING_IO) ||
 421         TEST_FLAG(OriginalIrp->Flags, IRP_SYNCHRONOUS_PAGING_IO)){
 422         SET_FLAG(Pkt->Srb.SrbFlags, SRB_CLASS_FLAGS_PAGING);
 423     }
 424     SET_FLAG(Pkt->Srb.SrbFlags, (majorFunc==IRP_MJ_READ) ? SRB_FLAGS_DATA_IN : SRB_FLAGS_DATA_OUT);
 425
 426     /*
 427      *  Allow caching only if this is not a write-through request.
 428      *  If write-through and caching is enabled on the device, force
 429      *  media access.
 430      */
 431     if (TEST_FLAG(origCurSp->Flags, SL_WRITE_THROUGH)){
 432         if (TEST_FLAG(fdoExt->DeviceFlags, DEV_WRITE_CACHE)){
 433             pCdb->CDB10.ForceUnitAccess = TRUE;
 434         }
 435     }
 436     else {
 437         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_ADAPTER_CACHE_ENABLE);
 438     }
 439
 440     /*
 441      *  Remember the buf and len in the SRB because miniports
 442      *  can overwrite SRB.DataTransferLength and we may need it again
 443      *  for the retry.
 444      */
 445     Pkt->BufPtrCopy = Buf;
 446     Pkt->BufLenCopy = Len;
 447     Pkt->TargetLocationCopy = DiskLocation;
 448
 449     Pkt->OriginalIrp = OriginalIrp;
 450     Pkt->NumRetries = MAXIMUM_RETRIES;
 451     Pkt->SyncEventPtr = NULL;
 452     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = TRUE;
 453 }
 454
 455 /*
 456  *  SubmitTransferPacket
 457  *
 458  *        Set up the IRP for the TRANSFER_PACKET submission and send it down.
 459  */
 460 VOID NTAPI SubmitTransferPacket(PTRANSFER_PACKET Pkt)
 461 {
 462     PCOMMON_DEVICE_EXTENSION commonExtension = Pkt->Fdo->DeviceExtension;
 463     PDEVICE_OBJECT nextDevObj = commonExtension->LowerDeviceObject;
 464     PIO_STACK_LOCATION nextSp = IoGetNextIrpStackLocation(Pkt->Irp);
 465
 466     ASSERT(Pkt->Irp->CurrentLocation == Pkt->Irp->StackCount+1);
 467
 468     /*
 469      *  Attach the SRB to the IRP.
 470      *  The reused IRP's stack location has to be rewritten for each retry
 471      *  call because IoCompleteRequest clears the stack locations.
 472      */
 473     IoReuseIrp(Pkt->Irp, STATUS_NOT_SUPPORTED);
 474     nextSp->MajorFunction = IRP_MJ_SCSI;
 475     nextSp->Parameters.Scsi.Srb = &Pkt->Srb;
 476     Pkt->Srb.ScsiStatus = Pkt->Srb.SrbStatus = 0;
 477     if (Pkt->CompleteOriginalIrpWhenLastPacketCompletes){
 478         /*
 479          *  Only dereference the "original IRP"'s stack location
 480          *  if its a real client irp (as opposed to a static irp
 481          *  we're using just for result status for one of the non-IO scsi commands).
 482          *
 483          *  For read/write, propagate the storage-specific IRP stack location flags
 484          *  (e.g. SL_OVERRIDE_VERIFY_VOLUME, SL_WRITE_THROUGH).
 485          */
 486         PIO_STACK_LOCATION origCurSp = IoGetCurrentIrpStackLocation(Pkt->OriginalIrp);
 487         nextSp->Flags = origCurSp->Flags;
 488     }
 489
 490     /*
 491      *  Write MDL address to new IRP. In the port driver the SRB DataBuffer
 492      *  field is used as the actual buffer pointer within the MDL,
 493      *  so the same MDL can be used for each partial transfer.
 494      *  This saves having to build a new MDL for each partial transfer.
 495      */
 496     Pkt->Irp->MdlAddress = Pkt->OriginalIrp->MdlAddress;
 497
 498     IoSetCompletionRoutine(Pkt->Irp, TransferPktComplete, Pkt, TRUE, TRUE, TRUE);
 499     IoCallDriver(nextDevObj, Pkt->Irp);
 500 }
 501
 502 NTSTATUS NTAPI TransferPktComplete(IN PDEVICE_OBJECT NullFdo, IN PIRP Irp, IN PVOID Context)
 503 {
 504     PTRANSFER_PACKET pkt = (PTRANSFER_PACKET)Context;
 505     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = pkt->Fdo->DeviceExtension;
 506     PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 507     PIO_STACK_LOCATION origCurrentSp = IoGetCurrentIrpStackLocation(pkt->OriginalIrp);
 508     BOOLEAN packetDone = FALSE;
 509
 510     /*
 511      *  Put all the assertions and spew in here so we don't have to look at them.
 512      */
 513     DBGCHECKRETURNEDPKT(pkt);
 514
 515     if (SRB_STATUS(pkt->Srb.SrbStatus) == SRB_STATUS_SUCCESS){
 516
 517         fdoData->LoggedTURFailureSinceLastIO = FALSE;
 518
 519         /*
 520          *  The port driver should not have allocated a sense buffer
 521          *  if the SRB succeeded.
 522          */
 523         ASSERT(!PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb));
 524
 525         /*
 526          *  Add this packet's transferred length to the original IRP's.
 527          */
 528         InterlockedExchangeAdd((PLONG)&pkt->OriginalIrp->IoStatus.Information,
 529                               (LONG)pkt->Srb.DataTransferLength);
 530
 531         if (pkt->InLowMemRetry){
 532             packetDone = StepLowMemRetry(pkt);
 533         }
 534         else {
 535             packetDone = TRUE;
 536         }
 537
 538     }
 539     else {
 540         /*
 541          *  The packet failed.  We may retry it if possible.
 542          */
 543         BOOLEAN shouldRetry;
 544
 545         /*
 546          *  Make sure IRP status matches SRB error status (since we propagate it).
 547          */
 548         if (NT_SUCCESS(Irp->IoStatus.Status)){
 549             Irp->IoStatus.Status = STATUS_UNSUCCESSFUL;
 550         }
 551
 552         /*
 553          *  Interpret the SRB error (to a meaningful IRP status)
 554          *  and determine if we should retry this packet.
 555          *  This call looks at the returned SENSE info to figure out what to do.
 556          */
 557         shouldRetry = InterpretTransferPacketError(pkt);
 558
 559         /*
 560          *  Sometimes the port driver can allocates a new 'sense' buffer
 561          *  to report transfer errors, e.g. when the default sense buffer
 562          *  is too small.  If so, it is up to us to free it.
 563          *  Now that we're done interpreting the sense info, free it if appropriate.
 564          */
 565         if (PORT_ALLOCATED_SENSE(fdoExt, &pkt->Srb)) {
 566             DBGTRACE(ClassDebugSenseInfo, ("Freeing port-allocated sense buffer for pkt %ph.", pkt));
 567             FREE_PORT_ALLOCATED_SENSE_BUFFER(fdoExt, &pkt->Srb);
 568             pkt->Srb.SenseInfoBuffer = &pkt->SrbErrorSenseData;
 569             pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 570         }
 571
 572         /*
 573          *  If the SRB queue is locked-up, release it.
 574          *  Do this after calling the error handler.
 575          */
 576         if (pkt->Srb.SrbStatus & SRB_STATUS_QUEUE_FROZEN){
 577             ClassReleaseQueue(pkt->Fdo);
 578         }
 579
 580         if (shouldRetry && (pkt->NumRetries > 0)){
 581             packetDone = RetryTransferPacket(pkt);
 582         }
 583         else {
 584             packetDone = TRUE;
 585         }
 586
 587     }
 588
 589     /*
 590      *  If the packet is completed, put it back in the free list.
 591      *  If it is the last packet servicing the original request, complete the original irp.
 592      */
 593     if (packetDone){
 594         LONG numPacketsRemaining;
 595         PIRP deferredIrp;
 596         PDEVICE_OBJECT Fdo = pkt->Fdo;
 597         UCHAR uniqueAddr;
 598
 599         /*
 600          *  In case a remove is pending, bump the lock count so we don't get freed
 601          *  right after we complete the original irp.
 602          */
 603         ClassAcquireRemoveLock(Fdo, (PIRP)&uniqueAddr);
 604
 605         /*
 606          *  The original IRP should get an error code
 607          *  if any one of the packets failed.
 608          */
 609         if (!NT_SUCCESS(Irp->IoStatus.Status)){
 610             pkt->OriginalIrp->IoStatus.Status = Irp->IoStatus.Status;
 611
 612             /*
 613              *  If the original I/O originated in user space (i.e. it is thread-queued),
 614              *  and the error is user-correctable (e.g. media is missing, for removable media),
 615              *  alert the user.
 616              *  Since this is only one of possibly several packets completing for the original IRP,
 617              *  we may do this more than once for a single request.  That's ok; this allows
 618              *  us to test each returned status with IoIsErrorUserInduced().
 619              */
 620             if (IoIsErrorUserInduced(Irp->IoStatus.Status) &&
 621                 pkt->CompleteOriginalIrpWhenLastPacketCompletes &&
 622                 pkt->OriginalIrp->Tail.Overlay.Thread){
 623
 624                 IoSetHardErrorOrVerifyDevice(pkt->OriginalIrp, pkt->Fdo);
 625             }
 626         }
 627
 628         /*
 629          *  We use a field in the original IRP to count
 630          *  down the transfer pieces as they complete.
 631          */
 632         numPacketsRemaining = InterlockedDecrement(
 633             (PLONG)&pkt->OriginalIrp->Tail.Overlay.DriverContext[0]);
 634
 635         if (numPacketsRemaining > 0){
 636             /*
 637              *  More transfer pieces remain for the original request.
 638              *  Wait for them to complete before completing the original irp.
 639              */
 640         }
 641         else {
 642
 643             /*
 644              *  All the transfer pieces are done.
 645              *  Complete the original irp if appropriate.
 646              */
 647             ASSERT(numPacketsRemaining == 0);
 648             if (pkt->CompleteOriginalIrpWhenLastPacketCompletes){
 649                 if (NT_SUCCESS(pkt->OriginalIrp->IoStatus.Status)){
 650                     ASSERT((ULONG)pkt->OriginalIrp->IoStatus.Information == origCurrentSp->Parameters.Read.Length);
 651                     ClasspPerfIncrementSuccessfulIo(fdoExt);
 652                 }
 653                 ClassReleaseRemoveLock(pkt->Fdo, pkt->OriginalIrp);
 654
 655                 ClassCompleteRequest(pkt->Fdo, pkt->OriginalIrp, IO_DISK_INCREMENT);
 656
 657                 /*
 658                  *  We may have been called by one of the class drivers (e.g. cdrom)
 659                  *  via the legacy API ClassSplitRequest.
 660                  *  This is the only case for which the packet engine is called for an FDO
 661                  *  with a StartIo routine; in that case, we have to call IoStartNextPacket
 662                  *  now that the original irp has been completed.
 663                  */
 664                 if (fdoExt->CommonExtension.DriverExtension->InitData.ClassStartIo) {
 665                     if (TEST_FLAG(pkt->Srb.SrbFlags, SRB_FLAGS_DONT_START_NEXT_PACKET)){
 666                         DBGTRAP(("SRB_FLAGS_DONT_START_NEXT_PACKET should never be set here (?)"));
 667                     }
 668                     else {
 669                         KIRQL oldIrql;
 670                         KeRaiseIrql(DISPATCH_LEVEL, &oldIrql);
 671                         IoStartNextPacket(pkt->Fdo, FALSE);
 672                         KeLowerIrql(oldIrql);
 673                     }
 674                 }
 675             }
 676         }
 677
 678         /*
 679          *  If the packet was synchronous, write the final
 680          *  result back to the issuer's status buffer and
 681          *  signal his event.
 682          */
 683         if (pkt->SyncEventPtr){
 684             KeSetEvent(pkt->SyncEventPtr, 0, FALSE);
 685             pkt->SyncEventPtr = NULL;
 686         }
 687
 688         /*
 689          *  Free the completed packet.
 690          */
 691         pkt->OriginalIrp = NULL;
 692         pkt->InLowMemRetry = FALSE;
 693         EnqueueFreeTransferPacket(pkt->Fdo, pkt);
 694
 695         /*
 696          *  Now that we have freed some resources,
 697          *  try again to send one of the previously deferred irps.
 698          */
 699         deferredIrp = DequeueDeferredClientIrp(fdoData);
 700         if (deferredIrp){
 701             DBGWARN(("... retrying deferred irp %xh.", deferredIrp));
 702             ServiceTransferRequest(pkt->Fdo, deferredIrp);
 703         }
 704
 705         ClassReleaseRemoveLock(Fdo, (PIRP)&uniqueAddr);
 706     }
 707
 708     return STATUS_MORE_PROCESSING_REQUIRED;
 709 }
 710
 711 /*
 712  *  SetupEjectionTransferPacket
 713  *
 714  *      Set up a transferPacket for a synchronous Ejection Control transfer.
 715  */
 716 VOID NTAPI SetupEjectionTransferPacket(   TRANSFER_PACKET *Pkt,
 717                                         BOOLEAN PreventMediaRemoval,
 718                                         PKEVENT SyncEventPtr,
 719                                         PIRP OriginalIrp)
 720 {
 721     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 722     PCDB pCdb;
 723
 724     PAGED_CODE();
 725
 726     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 727
 728     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 729     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 730     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 731     Pkt->Srb.CdbLength = 6;
 732     Pkt->Srb.OriginalRequest = Pkt->Irp;
 733     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 734     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 735     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 736
 737     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 738     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 739     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 740
 741     pCdb = (PCDB)Pkt->Srb.Cdb;
 742     pCdb->MEDIA_REMOVAL.OperationCode = SCSIOP_MEDIUM_REMOVAL;
 743     pCdb->MEDIA_REMOVAL.Prevent = PreventMediaRemoval;
 744
 745     Pkt->BufPtrCopy = NULL;
 746     Pkt->BufLenCopy = 0;
 747
 748     Pkt->OriginalIrp = OriginalIrp;
 749     Pkt->NumRetries = NUM_LOCKMEDIAREMOVAL_RETRIES;
 750     Pkt->SyncEventPtr = SyncEventPtr;
 751     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 752 }
 753
 754 /*
 755  *  SetupModeSenseTransferPacket
 756  *
 757  *      Set up a transferPacket for a synchronous Mode Sense transfer.
 758  */
 759 VOID NTAPI SetupModeSenseTransferPacket(TRANSFER_PACKET *Pkt,
 760                                         PKEVENT SyncEventPtr,
 761                                         PVOID ModeSenseBuffer,
 762                                         UCHAR ModeSenseBufferLen,
 763                                         UCHAR PageMode,
 764                                         PIRP OriginalIrp)
 765 {
 766     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 767     PCDB pCdb;
 768
 769     PAGED_CODE();
 770
 771     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 772
 773     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 774     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 775     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 776     Pkt->Srb.CdbLength = 6;
 777     Pkt->Srb.OriginalRequest = Pkt->Irp;
 778     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 779     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 780     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 781     Pkt->Srb.DataBuffer = ModeSenseBuffer;
 782     Pkt->Srb.DataTransferLength = ModeSenseBufferLen;
 783
 784     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 785     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DATA_IN);
 786     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 787     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 788
 789     pCdb = (PCDB)Pkt->Srb.Cdb;
 790     pCdb->MODE_SENSE.OperationCode = SCSIOP_MODE_SENSE;
 791     pCdb->MODE_SENSE.PageCode = PageMode;
 792     pCdb->MODE_SENSE.AllocationLength = (UCHAR)ModeSenseBufferLen;
 793
 794     Pkt->BufPtrCopy = ModeSenseBuffer;
 795     Pkt->BufLenCopy = ModeSenseBufferLen;
 796
 797     Pkt->OriginalIrp = OriginalIrp;
 798     Pkt->NumRetries = NUM_MODESENSE_RETRIES;
 799     Pkt->SyncEventPtr = SyncEventPtr;
 800     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 801 }
 802
 803 /*
 804  *  SetupDriveCapacityTransferPacket
 805  *
 806  *      Set up a transferPacket for a synchronous Drive Capacity transfer.
 807  */
 808 VOID NTAPI SetupDriveCapacityTransferPacket(TRANSFER_PACKET *Pkt,
 809                                             PVOID ReadCapacityBuffer,
 810                                             ULONG ReadCapacityBufferLen,
 811                                             PKEVENT SyncEventPtr,
 812                                             PIRP OriginalIrp)
 813 {
 814     PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 815     PCDB pCdb;
 816
 817     RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 818
 819     Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 820     Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 821     Pkt->Srb.QueueAction = SRB_SIMPLE_TAG_REQUEST;
 822     Pkt->Srb.CdbLength = 10;
 823     Pkt->Srb.OriginalRequest = Pkt->Irp;
 824     Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 825     Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 826     Pkt->Srb.TimeOutValue = fdoExt->TimeOutValue;
 827     Pkt->Srb.DataBuffer = ReadCapacityBuffer;
 828     Pkt->Srb.DataTransferLength = ReadCapacityBufferLen;
 829
 830     Pkt->Srb.SrbFlags = fdoExt->SrbFlags;
 831     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DATA_IN);
 832     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 833     SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_QUEUE_FREEZE);
 834
 835     pCdb = (PCDB)Pkt->Srb.Cdb;
 836     pCdb->CDB10.OperationCode = SCSIOP_READ_CAPACITY;
 837
 838     Pkt->BufPtrCopy = ReadCapacityBuffer;
 839     Pkt->BufLenCopy = ReadCapacityBufferLen;
 840
 841     Pkt->OriginalIrp = OriginalIrp;
 842     Pkt->NumRetries = NUM_DRIVECAPACITY_RETRIES;
 843     Pkt->SyncEventPtr = SyncEventPtr;
 844     Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 845 }
 846
 847 #if 0
 848     /*
 849      *  SetupSendStartUnitTransferPacket
 850      *
 851      *      Set up a transferPacket for a synchronous Send Start Unit transfer.
 852      */
 853     VOID SetupSendStartUnitTransferPacket(   TRANSFER_PACKET *Pkt,
 854                                                     PIRP OriginalIrp)
 855     {
 856         PFUNCTIONAL_DEVICE_EXTENSION fdoExt = Pkt->Fdo->DeviceExtension;
 857         PCLASS_PRIVATE_FDO_DATA fdoData = fdoExt->PrivateFdoData;
 858         PCDB pCdb;
 859
 860         PAGED_CODE();
 861
 862         RtlZeroMemory(&Pkt->Srb, sizeof(SCSI_REQUEST_BLOCK));
 863
 864         /*
 865          *  Initialize the SRB.
 866          *  Use a very long timeout value to give the drive time to spin up.
 867          */
 868         Pkt->Srb.Length = sizeof(SCSI_REQUEST_BLOCK);
 869         Pkt->Srb.Function = SRB_FUNCTION_EXECUTE_SCSI;
 870         Pkt->Srb.TimeOutValue = START_UNIT_TIMEOUT;
 871         Pkt->Srb.CdbLength = 6;
 872         Pkt->Srb.OriginalRequest = Pkt->Irp;
 873         Pkt->Srb.SenseInfoBuffer = &Pkt->SrbErrorSenseData;
 874         Pkt->Srb.SenseInfoBufferLength = sizeof(SENSE_DATA);
 875         Pkt->Srb.Lun = 0;
 876
 877         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_NO_DATA_TRANSFER);
 878         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_AUTOSENSE);
 879         SET_FLAG(Pkt->Srb.SrbFlags, SRB_FLAGS_DISABLE_SYNCH_TRANSFER);
 880
 881         pCdb = (PCDB)Pkt->Srb.Cdb;
 882         pCdb->START_STOP.OperationCode = SCSIOP_START_STOP_UNIT;
 883         pCdb->START_STOP.Start = 1;
 884         pCdb->START_STOP.Immediate = 0;
 885         pCdb->START_STOP.LogicalUnitNumber = 0;
 886
 887         Pkt->OriginalIrp = OriginalIrp;
 888         Pkt->NumRetries = 0;
 889         Pkt->SyncEventPtr = NULL;
 890         Pkt->CompleteOriginalIrpWhenLastPacketCompletes = FALSE;
 891     }
 892 #endif