ntoskrnl/io/iomgr/irp.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/io/iomgr/irp.c
   5  * PURPOSE:         IRP Handling Functions
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Gunnar Dalsnes
   8  *                  Filip Navara (navaraf@reactos.org)
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 PIRP IopDeadIrp;
  18
  19 /* PRIVATE FUNCTIONS  ********************************************************/
  20
  21 VOID
  22 NTAPI
  23 IopFreeIrpKernelApc(IN PKAPC Apc,
  24                     IN PKNORMAL_ROUTINE *NormalRoutine,
  25                     IN PVOID *NormalContext,
  26                     IN PVOID *SystemArgument1,
  27                     IN PVOID *SystemArgument2)
  28 {
  29     /* Free the IRP */
  30     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  31 }
  32
  33 VOID
  34 NTAPI
  35 IopAbortIrpKernelApc(IN PKAPC Apc)
  36 {
  37     /* Free the IRP */
  38     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  39 }
  40
  41 NTSTATUS
  42 NTAPI
  43 IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
  44                     IN PKEVENT EventObject OPTIONAL,
  45                     IN PVOID Buffer OPTIONAL)
  46 {
  47     PAGED_CODE();
  48
  49     /* Dereference the event */
  50     if (EventObject) ObDereferenceObject(EventObject);
  51
  52     /* Free a buffer, if any */
  53     if (Buffer) ExFreePool(Buffer);
  54
  55     /* If this was a file opened for synch I/O, then unlock it */
  56     if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
  57
  58     /* Now dereference it and return */
  59     ObDereferenceObject(FileObject);
  60     return STATUS_INSUFFICIENT_RESOURCES;
  61 }
  62
  63 VOID
  64 NTAPI
  65 IopAbortInterruptedIrp(IN PKEVENT EventObject,
  66                        IN PIRP Irp)
  67 {
  68     KIRQL OldIrql;
  69     BOOLEAN CancelResult;
  70     LARGE_INTEGER Wait;
  71     PAGED_CODE();
  72
  73     /* Raise IRQL to APC */
  74     KeRaiseIrql(APC_LEVEL, &OldIrql);
  75
  76     /* Check if nobody completed it yet */
  77     if (!KeReadStateEvent(EventObject))
  78     {
  79         /* First, cancel it */
  80         CancelResult = IoCancelIrp(Irp);
  81         KeLowerIrql(OldIrql);
  82
  83         /* Check if we cancelled it */
  84         if (CancelResult)
  85         {
  86             /* Wait for the IRP to be cancelled */
  87             Wait.QuadPart = -100000;
  88             while (!KeReadStateEvent(EventObject))
  89             {
  90                 /* Delay indefintely */
  91                 KeDelayExecutionThread(KernelMode, FALSE, &Wait);
  92             }
  93         }
  94         else
  95         {
  96             /* No cancellation done, so wait for the I/O system to kill it */
  97             KeWaitForSingleObject(EventObject,
  98                                   Executive,
  99                                   KernelMode,
 100                                   FALSE,
 101                                   NULL);
 102         }
 103     }
 104     else
 105     {
 106         /* We got preempted, so give up */
 107         KeLowerIrql(OldIrql);
 108     }
 109 }
 110
 111 VOID
 112 NTAPI
 113 IopDisassociateThreadIrp(VOID)
 114 {
 115     KIRQL OldIrql, LockIrql;
 116     PETHREAD IrpThread;
 117     PLIST_ENTRY IrpEntry;
 118     PIO_ERROR_LOG_PACKET ErrorLogEntry;
 119     PDEVICE_OBJECT DeviceObject = NULL;
 120     PIO_STACK_LOCATION IoStackLocation;
 121
 122     /* First, raise to APC to protect IrpList */
 123     KeRaiseIrql(APC_LEVEL, &OldIrql);
 124
 125     /* Get the Thread and check the list */
 126     IrpThread = PsGetCurrentThread();
 127     if (IsListEmpty(&IrpThread->IrpList))
 128     {
 129         /* It got completed now, so quit */
 130         KeLowerIrql(OldIrql);
 131         return;
 132     }
 133
 134     /* Ensure no one will come disturb */
 135     LockIrql = KeAcquireQueuedSpinLock(LockQueueIoCompletionLock);
 136
 137     /* Get the misbehaving IRP */
 138     IrpEntry = IrpThread->IrpList.Flink;
 139     IopDeadIrp = CONTAINING_RECORD(IrpEntry, IRP, ThreadListEntry);
 140     IOTRACE(IO_IRP_DEBUG,
 141             "%s - Deassociating IRP %p for %p\n",
 142             __FUNCTION__,
 143             IopDeadIrp,
 144             IrpThread);
 145
 146     /* Don't cancel the IRP if it's already been completed far */
 147     if (IopDeadIrp->CurrentLocation == (IopDeadIrp->StackCount + 2))
 148     {
 149         /* Return */
 150         KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 151         KeLowerIrql(OldIrql);
 152         return;
 153     }
 154
 155     /* Disown the IRP! */
 156     IopDeadIrp->Tail.Overlay.Thread = NULL;
 157     RemoveHeadList(&IrpThread->IrpList);
 158     InitializeListHead(&IopDeadIrp->ThreadListEntry);
 159
 160     /* Get the stack location and check if it's valid */
 161     IoStackLocation = IoGetCurrentIrpStackLocation(IopDeadIrp);
 162     if (IopDeadIrp->CurrentLocation <= IopDeadIrp->StackCount)
 163     {
 164         /* Get the device object */
 165         DeviceObject = IoStackLocation->DeviceObject;
 166     }
 167
 168     KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 169     /* Lower IRQL now, since we have the pointers we need */
 170     KeLowerIrql(OldIrql);
 171
 172     /* Check if we can send an Error Log Entry*/
 173     if (DeviceObject)
 174     {
 175         /* Allocate an entry */
 176         ErrorLogEntry = IoAllocateErrorLogEntry(DeviceObject,
 177                                                 sizeof(IO_ERROR_LOG_PACKET));
 178         if (ErrorLogEntry)
 179         {
 180             /* Write the entry */
 181             ErrorLogEntry->ErrorCode = IO_DRIVER_CANCEL_TIMEOUT;
 182             IoWriteErrorLogEntry(ErrorLogEntry);
 183         }
 184     }
 185 }
 186
 187 VOID
 188 NTAPI
 189 IopCleanupIrp(IN PIRP Irp,
 190               IN PFILE_OBJECT FileObject)
 191 {
 192     PMDL Mdl;
 193     IOTRACE(IO_IRP_DEBUG,
 194             "%s - Cleaning IRP %p for %p\n",
 195             __FUNCTION__,
 196             Irp,
 197             FileObject);
 198
 199     /* Check if there's an MDL */
 200     while ((Mdl = Irp->MdlAddress))
 201     {
 202         /* Clear all of them */
 203         Irp->MdlAddress = Mdl->Next;
 204         IoFreeMdl(Mdl);
 205     }
 206
 207     /* Check if the IRP has system buffer */
 208     if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 209     {
 210         /* Free the buffer */
 211         ExFreePoolWithTag(Irp->AssociatedIrp.SystemBuffer, TAG_SYS_BUF);
 212     }
 213
 214     /* Check if this IRP has a user event, a file object, and is async */
 215     if ((Irp->UserEvent) &&
 216         !(Irp->Flags & IRP_SYNCHRONOUS_API) &&
 217         (FileObject))
 218     {
 219         /* Dereference the User Event */
 220         ObDereferenceObject(Irp->UserEvent);
 221     }
 222
 223     /* Check if we have a file object and this isn't a create operation */
 224     if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 225     {
 226         /* Dereference the file object */
 227         ObDereferenceObject(FileObject);
 228     }
 229
 230     /* Free the IRP */
 231     IoFreeIrp(Irp);
 232 }
 233
 234 VOID
 235 NTAPI
 236 IopCompleteRequest(IN PKAPC Apc,
 237                    IN PKNORMAL_ROUTINE* NormalRoutine,
 238                    IN PVOID* NormalContext,
 239                    IN PVOID* SystemArgument1,
 240                    IN PVOID* SystemArgument2)
 241 {
 242     PFILE_OBJECT FileObject;
 243     PIRP Irp;
 244     PMDL Mdl, NextMdl;
 245     PVOID Port = NULL, Key = NULL;
 246     BOOLEAN SignaledCreateRequest = FALSE;
 247
 248     /* Get data from the APC */
 249     FileObject = (PFILE_OBJECT)*SystemArgument1;
 250     Irp = CONTAINING_RECORD(Apc, IRP, Tail.Apc);
 251     IOTRACE(IO_IRP_DEBUG,
 252             "%s - Completing IRP %p for %p\n",
 253             __FUNCTION__,
 254             Irp,
 255             FileObject);
 256
 257     /* Sanity check */
 258     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
 259
 260     /* Check if we have a file object */
 261     if (*SystemArgument2)
 262     {
 263         /* Check if we're reparsing */
 264         if ((Irp->IoStatus.Status == STATUS_REPARSE) &&
 265             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT))
 266         {
 267             PREPARSE_DATA_BUFFER ReparseData;
 268
 269             ReparseData = (PREPARSE_DATA_BUFFER)*SystemArgument2;
 270
 271             ASSERT(ReparseData->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT);
 272             ASSERT(ReparseData->ReparseDataLength < MAXIMUM_REPARSE_DATA_BUFFER_SIZE);
 273             ASSERT(ReparseData->Reserved < MAXIMUM_REPARSE_DATA_BUFFER_SIZE);
 274
 275             IopDoNameTransmogrify(Irp, FileObject, ReparseData);
 276         }
 277     }
 278
 279     /* Handle Buffered case first */
 280     if (Irp->Flags & IRP_BUFFERED_IO)
 281     {
 282         /* Check if we have an input buffer and if we succeeded */
 283         if ((Irp->Flags & IRP_INPUT_OPERATION) &&
 284             (Irp->IoStatus.Status != STATUS_VERIFY_REQUIRED) &&
 285             !(NT_ERROR(Irp->IoStatus.Status)))
 286         {
 287             /* Copy the buffer back to the user */
 288             RtlCopyMemory(Irp->UserBuffer,
 289                           Irp->AssociatedIrp.SystemBuffer,
 290                           Irp->IoStatus.Information);
 291         }
 292
 293         /* Also check if we should de-allocate it */
 294         if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 295         {
 296             /* Deallocate it */
 297             ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 298         }
 299     }
 300
 301     /* Now we got rid of these two... */
 302     Irp->Flags &= ~(IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER);
 303
 304     /* Check if there's an MDL */
 305     for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
 306     {
 307         /* Free it */
 308         NextMdl = Mdl->Next;
 309         IoFreeMdl(Mdl);
 310     }
 311
 312     /* No MDLs left */
 313     Irp->MdlAddress = NULL;
 314
 315     /*
 316      * Check if either the request was completed without any errors
 317      * (but warnings are OK!), or if it was completed with an error, but
 318      * did return from a pending I/O Operation and is not synchronous.
 319      */
 320     if (!(NT_ERROR(Irp->IoStatus.Status)) ||
 321          (NT_ERROR(Irp->IoStatus.Status) &&
 322           (Irp->PendingReturned) &&
 323           !(IsIrpSynchronous(Irp, FileObject))))
 324     {
 325         /* Get any information we need from the FO before we kill it */
 326         if ((FileObject) && (FileObject->CompletionContext))
 327         {
 328             /* Save Completion Data */
 329             Port = FileObject->CompletionContext->Port;
 330             Key = FileObject->CompletionContext->Key;
 331         }
 332
 333         /* Check for UserIos */
 334         if (Irp->UserIosb != NULL)
 335         {
 336             /* Use SEH to make sure we don't write somewhere invalid */
 337             _SEH2_TRY
 338             {
 339                 /*  Save the IOSB Information */
 340                 *Irp->UserIosb = Irp->IoStatus;
 341             }
 342             _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 343             {
 344                 /* Ignore any error */
 345             }
 346             _SEH2_END;
 347         }
 348
 349         /* Check if we have an event or a file object */
 350         if (Irp->UserEvent)
 351         {
 352             /* At the very least, this is a PKEVENT, so signal it always */
 353             KeSetEvent(Irp->UserEvent, 0, FALSE);
 354
 355             /* Check if we also have a File Object */
 356             if (FileObject)
 357             {
 358                 /* Check if this is an Asynch API */
 359                 if (!(Irp->Flags & IRP_SYNCHRONOUS_API))
 360                 {
 361                     /* Dereference the event */
 362                     ObDereferenceObject(Irp->UserEvent);
 363                 }
 364
 365                 /*
 366                  * Now, if this is a Synch I/O File Object, then this event is
 367                  * NOT an actual Executive Event, so we won't dereference it,
 368                  * and instead, we will signal the File Object
 369                  */
 370                 if ((FileObject->Flags & FO_SYNCHRONOUS_IO) &&
 371                     !(Irp->Flags & IRP_OB_QUERY_NAME))
 372                 {
 373                     /* Signal the file object and set the status */
 374                     KeSetEvent(&FileObject->Event, 0, FALSE);
 375                     FileObject->FinalStatus = Irp->IoStatus.Status;
 376                 }
 377
 378                 /*
 379                  * This could also be a create operation, in which case we want
 380                  * to make sure there's no APC fired.
 381                  */
 382                 if (Irp->Flags & IRP_CREATE_OPERATION)
 383                 {
 384                     /* Clear the APC Routine and remember this */
 385                     Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 386                     SignaledCreateRequest = TRUE;
 387                 }
 388             }
 389         }
 390         else if (FileObject)
 391         {
 392             /* Signal the file object and set the status */
 393             KeSetEvent(&FileObject->Event, 0, FALSE);
 394             FileObject->FinalStatus = Irp->IoStatus.Status;
 395
 396             /*
 397             * This could also be a create operation, in which case we want
 398             * to make sure there's no APC fired.
 399             */
 400             if (Irp->Flags & IRP_CREATE_OPERATION)
 401             {
 402                 /* Clear the APC Routine and remember this */
 403                 Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 404                 SignaledCreateRequest = TRUE;
 405             }
 406         }
 407
 408         /* Update transfer count for everything but create operation */
 409         if (!(Irp->Flags & IRP_CREATE_OPERATION))
 410         {
 411             if (Irp->Flags & IRP_WRITE_OPERATION)
 412             {
 413                 /* Update write transfer count */
 414                 IopUpdateTransferCount(IopWriteTransfer,
 415                                        (ULONG)Irp->IoStatus.Information);
 416             }
 417             else if (Irp->Flags & IRP_READ_OPERATION)
 418             {
 419                 /* Update read transfer count */
 420                 IopUpdateTransferCount(IopReadTransfer,
 421                                        (ULONG)Irp->IoStatus.Information);
 422             }
 423             else
 424             {
 425                 /* Update other transfer count */
 426                 IopUpdateTransferCount(IopOtherTransfer,
 427                                        (ULONG)Irp->IoStatus.Information);
 428             }
 429         }
 430
 431         /* Now that we've signaled the events, de-associate the IRP */
 432         IopUnQueueIrpFromThread(Irp);
 433
 434         /* Now check if a User APC Routine was requested */
 435         if (Irp->Overlay.AsynchronousParameters.UserApcRoutine)
 436         {
 437             /* Initialize it */
 438             KeInitializeApc(&Irp->Tail.Apc,
 439                             KeGetCurrentThread(),
 440                             CurrentApcEnvironment,
 441                             IopFreeIrpKernelApc,
 442                             IopAbortIrpKernelApc,
 443                             (PKNORMAL_ROUTINE)Irp->
 444                             Overlay.AsynchronousParameters.UserApcRoutine,
 445                             Irp->RequestorMode,
 446                             Irp->
 447                             Overlay.AsynchronousParameters.UserApcContext);
 448
 449             /* Queue it */
 450             KeInsertQueueApc(&Irp->Tail.Apc, Irp->UserIosb, NULL, 2);
 451         }
 452         else if ((Port) &&
 453                  (Irp->Overlay.AsynchronousParameters.UserApcContext))
 454         {
 455             /* We have an I/O Completion setup... create the special Overlay */
 456             Irp->Tail.CompletionKey = Key;
 457             Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp;
 458             KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry);
 459         }
 460         else
 461         {
 462             /* Free the IRP since we don't need it anymore */
 463             IoFreeIrp(Irp);
 464         }
 465
 466         /* Check if we have a file object that wasn't part of a create */
 467         if ((FileObject) && !(SignaledCreateRequest))
 468         {
 469             /* Dereference it, since it's not needed anymore either */
 470             ObDereferenceObjectDeferDelete(FileObject);
 471         }
 472     }
 473     else
 474     {
 475         /*
 476          * Either we didn't return from the request, or we did return but this
 477          * request was synchronous.
 478          */
 479         if ((Irp->PendingReturned) && (FileObject))
 480         {
 481             /* So we did return with a synch operation, was it the IRP? */
 482             if (Irp->Flags & IRP_SYNCHRONOUS_API)
 483             {
 484                 /* Yes, this IRP was synchronous, so return the I/O Status */
 485                 *Irp->UserIosb = Irp->IoStatus;
 486
 487                 /* Now check if the user gave an event */
 488                 if (Irp->UserEvent)
 489                 {
 490                     /* Signal it */
 491                     KeSetEvent(Irp->UserEvent, 0, FALSE);
 492                 }
 493                 else
 494                 {
 495                     /* No event was given, so signal the FO instead */
 496                     KeSetEvent(&FileObject->Event, 0, FALSE);
 497                 }
 498             }
 499             else
 500             {
 501                 /*
 502                  * It's not the IRP that was synchronous, it was the FO
 503                  * that was opened this way. Signal its event.
 504                  */
 505                 FileObject->FinalStatus = Irp->IoStatus.Status;
 506                 KeSetEvent(&FileObject->Event, 0, FALSE);
 507             }
 508         }
 509
 510         /* Now that we got here, we do this for incomplete I/Os as well */
 511         if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 512         {
 513             /* Dereference the File Object unless this was a create */
 514             ObDereferenceObjectDeferDelete(FileObject);
 515         }
 516
 517         /*
 518          * Check if this was an Executive Event (remember that we know this
 519          * by checking if the IRP is synchronous)
 520          */
 521         if ((Irp->UserEvent) &&
 522             (FileObject) &&
 523             !(Irp->Flags & IRP_SYNCHRONOUS_API))
 524         {
 525             /* This isn't a PKEVENT, so dereference it */
 526             ObDereferenceObject(Irp->UserEvent);
 527         }
 528
 529         /* Now that we've signaled the events, de-associate the IRP */
 530         IopUnQueueIrpFromThread(Irp);
 531
 532         /* Free the IRP as well */
 533         IoFreeIrp(Irp);
 534     }
 535 }
 536
 537 /* FUNCTIONS *****************************************************************/
 538
 539 /*
 540  * @implemented
 541  */
 542 PIRP
 543 NTAPI
 544 IoAllocateIrp(IN CCHAR StackSize,
 545               IN BOOLEAN ChargeQuota)
 546 {
 547     PIRP Irp = NULL;
 548     USHORT Size = IoSizeOfIrp(StackSize);
 549     PKPRCB Prcb;
 550     UCHAR Flags = 0;
 551     PNPAGED_LOOKASIDE_LIST List = NULL;
 552     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
 553
 554     /* Set Charge Quota Flag */
 555     if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
 556
 557     /* FIXME: Implement Lookaside Floats */
 558
 559     /* Figure out which Lookaside List to use */
 560     if ((StackSize <= 8) && (ChargeQuota == FALSE))
 561     {
 562         /* Set Fixed Size Flag */
 563         Flags = IRP_ALLOCATED_FIXED_SIZE;
 564
 565         /* See if we should use big list */
 566         if (StackSize != 1)
 567         {
 568             Size = IoSizeOfIrp(8);
 569             ListType = LookasideLargeIrpList;
 570         }
 571
 572         /* Get the PRCB */
 573         Prcb = KeGetCurrentPrcb();
 574
 575         /* Get the P List First */
 576         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
 577
 578         /* Attempt allocation */
 579         List->L.TotalAllocates++;
 580         Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 581
 582         /* Check if the P List failed */
 583         if (!Irp)
 584         {
 585             /* Let the balancer know */
 586             List->L.AllocateMisses++;
 587
 588             /* Try the L List */
 589             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
 590             List->L.TotalAllocates++;
 591             Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 592         }
 593     }
 594
 595     /* Check if we have to use the pool */
 596     if (!Irp)
 597     {
 598         /* Did we try lookaside and fail? */
 599         if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
 600
 601         /* Check if we should charge quota */
 602         if (ChargeQuota)
 603         {
 604             Irp = ExAllocatePoolWithQuotaTag(NonPagedPool | POOL_QUOTA_FAIL_INSTEAD_OF_RAISE,
 605                                              Size,
 606                                              TAG_IRP);
 607         }
 608         else
 609         {
 610             /* Allocate the IRP with no quota charge */
 611             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 612         }
 613
 614         /* Make sure it was sucessful */
 615         if (!Irp) return NULL;
 616     }
 617     else
 618     {
 619         /* In this case there is no charge quota */
 620         Flags &= ~IRP_QUOTA_CHARGED;
 621     }
 622
 623     /* Now Initialize it */
 624     IoInitializeIrp(Irp, Size, StackSize);
 625
 626     /* Set the Allocation Flags */
 627     Irp->AllocationFlags = Flags;
 628
 629     /* Return it */
 630     IOTRACE(IO_IRP_DEBUG,
 631             "%s - Allocated IRP %p with allocation flags %lx\n",
 632             __FUNCTION__,
 633             Irp,
 634             Flags);
 635     return Irp;
 636 }
 637
 638 /*
 639  * @implemented
 640  */
 641 PIRP
 642 NTAPI
 643 IopAllocateIrpMustSucceed(IN CCHAR StackSize)
 644 {
 645     LONG i;
 646     PIRP Irp;
 647     LARGE_INTEGER Sleep;
 648
 649     /* Try to get an IRP */
 650     Irp = IoAllocateIrp(StackSize, FALSE);
 651     if (Irp)
 652         return Irp;
 653
 654     /* If we fail, start looping till we may get one */
 655     i = LONG_MAX;
 656     do {
 657         i--;
 658
 659         /* First, sleep for 10ms */
 660         Sleep.QuadPart = -10 * 1000 * 10;;
 661         KeDelayExecutionThread(KernelMode, FALSE, &Sleep);
 662
 663         /* Then, retry allocation */
 664         Irp = IoAllocateIrp(StackSize, FALSE);
 665         if (Irp)
 666             return Irp;
 667     } while (i > 0);
 668
 669     return Irp;
 670 }
 671
 672 /*
 673  * @implemented
 674  */
 675 PIRP
 676 NTAPI
 677 IoBuildAsynchronousFsdRequest(IN ULONG MajorFunction,
 678                               IN PDEVICE_OBJECT DeviceObject,
 679                               IN PVOID Buffer,
 680                               IN ULONG Length,
 681                               IN PLARGE_INTEGER StartingOffset,
 682                               IN PIO_STATUS_BLOCK IoStatusBlock)
 683 {
 684     PIRP Irp;
 685     PIO_STACK_LOCATION StackPtr;
 686
 687     /* Allocate IRP */
 688     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 689     if (!Irp) return NULL;
 690
 691     /* Get the Stack */
 692     StackPtr = IoGetNextIrpStackLocation(Irp);
 693
 694     /* Write the Major function and then deal with it */
 695     StackPtr->MajorFunction = (UCHAR)MajorFunction;
 696
 697     /* Do not handle the following here */
 698     if ((MajorFunction != IRP_MJ_FLUSH_BUFFERS) &&
 699         (MajorFunction != IRP_MJ_SHUTDOWN) &&
 700         (MajorFunction != IRP_MJ_PNP) &&
 701         (MajorFunction != IRP_MJ_POWER))
 702     {
 703         /* Check if this is Buffered IO */
 704         if (DeviceObject->Flags & DO_BUFFERED_IO)
 705         {
 706             /* Allocate the System Buffer */
 707             Irp->AssociatedIrp.SystemBuffer =
 708                 ExAllocatePoolWithTag(NonPagedPool, Length, TAG_SYS_BUF);
 709             if (!Irp->AssociatedIrp.SystemBuffer)
 710             {
 711                 /* Free the IRP and fail */
 712                 IoFreeIrp(Irp);
 713                 return NULL;
 714             }
 715
 716             /* Set flags */
 717             Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 718
 719             /* Handle special IRP_MJ_WRITE Case */
 720             if (MajorFunction == IRP_MJ_WRITE)
 721             {
 722                 /* Copy the buffer data */
 723                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, Buffer, Length);
 724             }
 725             else
 726             {
 727                 /* Set the Input Operation flag and set this as a User Buffer */
 728                 Irp->Flags |= IRP_INPUT_OPERATION;
 729                 Irp->UserBuffer = Buffer;
 730             }
 731         }
 732         else if (DeviceObject->Flags & DO_DIRECT_IO)
 733         {
 734             /* Use an MDL for Direct I/O */
 735             Irp->MdlAddress = IoAllocateMdl(Buffer,
 736                                             Length,
 737                                             FALSE,
 738                                             FALSE,
 739                                             NULL);
 740             if (!Irp->MdlAddress)
 741             {
 742                 /* Free the IRP and fail */
 743                 IoFreeIrp(Irp);
 744                 return NULL;
 745             }
 746
 747                         /* Probe and Lock */
 748                         _SEH2_TRY
 749                         {
 750                                 /* Do the probe */
 751                                 MmProbeAndLockPages(Irp->MdlAddress,
 752                                                                         KernelMode,
 753                                                                         MajorFunction == IRP_MJ_READ ?
 754                                                                         IoWriteAccess : IoReadAccess);
 755                         }
 756                         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 757                         {
 758                                 /* Free the IRP and its MDL */
 759                                 IoFreeMdl(Irp->MdlAddress);
 760                                 IoFreeIrp(Irp);
 761
 762                 /* Fail */
 763                                 _SEH2_YIELD(return NULL);
 764                         }
 765                         _SEH2_END;
 766         }
 767         else
 768         {
 769             /* Neither, use the buffer */
 770             Irp->UserBuffer = Buffer;
 771         }
 772
 773         /* Check if this is a read */
 774         if (MajorFunction == IRP_MJ_READ)
 775         {
 776             /* Set the parameters for a read */
 777             StackPtr->Parameters.Read.Length = Length;
 778             StackPtr->Parameters.Read.ByteOffset = *StartingOffset;
 779         }
 780         else if (MajorFunction == IRP_MJ_WRITE)
 781         {
 782             /* Otherwise, set write parameters */
 783             StackPtr->Parameters.Write.Length = Length;
 784             StackPtr->Parameters.Write.ByteOffset = *StartingOffset;
 785         }
 786     }
 787
 788     /* Set the Current Thread and IOSB */
 789     Irp->UserIosb = IoStatusBlock;
 790     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 791
 792     /* Return the IRP */
 793     IOTRACE(IO_IRP_DEBUG,
 794             "%s - Built IRP %p with Major, Buffer, DO %lx %p %p\n",
 795             __FUNCTION__,
 796             Irp,
 797             MajorFunction,
 798             Buffer,
 799             DeviceObject);
 800     return Irp;
 801 }
 802
 803 /*
 804  * @implemented
 805  */
 806 PIRP
 807 NTAPI
 808 IoBuildDeviceIoControlRequest(IN ULONG IoControlCode,
 809                               IN PDEVICE_OBJECT DeviceObject,
 810                               IN PVOID InputBuffer,
 811                               IN ULONG InputBufferLength,
 812                               IN PVOID OutputBuffer,
 813                               IN ULONG OutputBufferLength,
 814                               IN BOOLEAN InternalDeviceIoControl,
 815                               IN PKEVENT Event,
 816                               IN PIO_STATUS_BLOCK IoStatusBlock)
 817 {
 818     PIRP Irp;
 819     PIO_STACK_LOCATION StackPtr;
 820     ULONG BufferLength;
 821
 822     /* Allocate IRP */
 823     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 824     if (!Irp) return NULL;
 825
 826     /* Get the Stack */
 827     StackPtr = IoGetNextIrpStackLocation(Irp);
 828
 829     /* Set the DevCtl Type */
 830     StackPtr->MajorFunction = InternalDeviceIoControl ?
 831                               IRP_MJ_INTERNAL_DEVICE_CONTROL :
 832                               IRP_MJ_DEVICE_CONTROL;
 833
 834     /* Set the IOCTL Data */
 835     StackPtr->Parameters.DeviceIoControl.IoControlCode = IoControlCode;
 836     StackPtr->Parameters.DeviceIoControl.InputBufferLength = InputBufferLength;
 837     StackPtr->Parameters.DeviceIoControl.OutputBufferLength =
 838         OutputBufferLength;
 839
 840     /* Handle the Methods */
 841     switch (IO_METHOD_FROM_CTL_CODE(IoControlCode))
 842     {
 843         /* Buffered I/O */
 844         case METHOD_BUFFERED:
 845
 846             /* Select the right Buffer Length */
 847             BufferLength = InputBufferLength > OutputBufferLength ?
 848                            InputBufferLength : OutputBufferLength;
 849
 850             /* Make sure there is one */
 851             if (BufferLength)
 852             {
 853                 /* Allocate the System Buffer */
 854                 Irp->AssociatedIrp.SystemBuffer =
 855                     ExAllocatePoolWithTag(NonPagedPool,
 856                                           BufferLength,
 857                                           TAG_SYS_BUF);
 858                 if (!Irp->AssociatedIrp.SystemBuffer)
 859                 {
 860                     /* Free the IRP and fail */
 861                     IoFreeIrp(Irp);
 862                     return NULL;
 863                 }
 864
 865                 /* Check if we got a buffer */
 866                 if (InputBuffer)
 867                 {
 868                     /* Copy into the System Buffer */
 869                     RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 870                                   InputBuffer,
 871                                   InputBufferLength);
 872                 }
 873
 874                 /* Write the flags */
 875                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 876                 if (OutputBuffer) Irp->Flags |= IRP_INPUT_OPERATION;
 877
 878                 /* Save the Buffer */
 879                 Irp->UserBuffer = OutputBuffer;
 880             }
 881             else
 882             {
 883                 /* Clear the Flags and Buffer */
 884                 Irp->Flags = 0;
 885                 Irp->UserBuffer = NULL;
 886             }
 887             break;
 888
 889         /* Direct I/O */
 890         case METHOD_IN_DIRECT:
 891         case METHOD_OUT_DIRECT:
 892
 893             /* Check if we got an input buffer */
 894             if (InputBuffer)
 895             {
 896                 /* Allocate the System Buffer */
 897                 Irp->AssociatedIrp.SystemBuffer =
 898                     ExAllocatePoolWithTag(NonPagedPool,
 899                                           InputBufferLength,
 900                                           TAG_SYS_BUF);
 901                 if (!Irp->AssociatedIrp.SystemBuffer)
 902                 {
 903                     /* Free the IRP and fail */
 904                     IoFreeIrp(Irp);
 905                     return NULL;
 906                 }
 907
 908                 /* Copy into the System Buffer */
 909                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 910                               InputBuffer,
 911                               InputBufferLength);
 912
 913                 /* Write the flags */
 914                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 915             }
 916             else
 917             {
 918                 /* Clear the flags */
 919                 Irp->Flags = 0;
 920             }
 921
 922             /* Check if we got an output buffer */
 923             if (OutputBuffer)
 924             {
 925                 /* Allocate the System Buffer */
 926                 Irp->MdlAddress = IoAllocateMdl(OutputBuffer,
 927                                                 OutputBufferLength,
 928                                                 FALSE,
 929                                                 FALSE,
 930                                                 Irp);
 931                 if (!Irp->MdlAddress)
 932                 {
 933                     /* Free the IRP and fail */
 934                     IoFreeIrp(Irp);
 935                     return NULL;
 936                 }
 937
 938                 /* Probe and Lock */
 939                 _SEH2_TRY
 940                 {
 941                     /* Do the probe */
 942                     MmProbeAndLockPages(Irp->MdlAddress,
 943                                         KernelMode,
 944                                         IO_METHOD_FROM_CTL_CODE(IoControlCode) ==
 945                                         METHOD_IN_DIRECT ?
 946                                         IoReadAccess : IoWriteAccess);
 947                 }
 948                 _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 949                 {
 950                     /* Free the MDL */
 951                     IoFreeMdl(Irp->MdlAddress);
 952
 953                     /* Free the input buffer and IRP */
 954                     if (InputBuffer) ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 955                     IoFreeIrp(Irp);
 956
 957                     /* Fail */
 958                     _SEH2_YIELD(return NULL);
 959                 }
 960                 _SEH2_END;
 961             }
 962             break;
 963
 964         case METHOD_NEITHER:
 965
 966             /* Just save the Buffer */
 967             Irp->UserBuffer = OutputBuffer;
 968             StackPtr->Parameters.DeviceIoControl.Type3InputBuffer = InputBuffer;
 969     }
 970
 971     /* Now write the Event and IoSB */
 972     Irp->UserIosb = IoStatusBlock;
 973     Irp->UserEvent = Event;
 974
 975     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 976     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 977     IoQueueThreadIrp(Irp);
 978
 979     /* Return the IRP */
 980     IOTRACE(IO_IRP_DEBUG,
 981             "%s - Built IRP %p with IOCTL, Buffers, DO %lx %p %p %p\n",
 982             __FUNCTION__,
 983             Irp,
 984             IoControlCode,
 985             InputBuffer,
 986             OutputBuffer,
 987             DeviceObject);
 988     return Irp;
 989 }
 990
 991 /*
 992  * @implemented
 993  */
 994 PIRP
 995 NTAPI
 996 IoBuildSynchronousFsdRequest(IN ULONG MajorFunction,
 997                              IN PDEVICE_OBJECT DeviceObject,
 998                              IN PVOID Buffer,
 999                              IN ULONG Length,
1000                              IN PLARGE_INTEGER StartingOffset,
1001                              IN PKEVENT Event,
1002                              IN PIO_STATUS_BLOCK IoStatusBlock)
1003 {
1004     PIRP Irp;
1005
1006     /* Do the big work to set up the IRP */
1007     Irp = IoBuildAsynchronousFsdRequest(MajorFunction,
1008                                         DeviceObject,
1009                                         Buffer,
1010                                         Length,
1011                                         StartingOffset,
1012                                         IoStatusBlock );
1013     if (!Irp) return NULL;
1014
1015     /* Set the Event which makes it Syncronous */
1016     Irp->UserEvent = Event;
1017
1018     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
1019     IoQueueThreadIrp(Irp);
1020     return Irp;
1021 }
1022
1023 /*
1024  * @implemented
1025  */
1026 BOOLEAN
1027 NTAPI
1028 IoCancelIrp(IN PIRP Irp)
1029 {
1030     KIRQL OldIrql;
1031     PDRIVER_CANCEL CancelRoutine;
1032     IOTRACE(IO_IRP_DEBUG,
1033             "%s - Canceling IRP %p\n",
1034             __FUNCTION__,
1035             Irp);
1036     ASSERT(Irp->Type == IO_TYPE_IRP);
1037
1038     /* Acquire the cancel lock and cancel the IRP */
1039     IoAcquireCancelSpinLock(&OldIrql);
1040     Irp->Cancel = TRUE;
1041
1042     /* Clear the cancel routine and get the old one */
1043     CancelRoutine = (PVOID)IoSetCancelRoutine(Irp, NULL);
1044     if (CancelRoutine)
1045     {
1046         /* We had a routine, make sure the IRP isn't completed */
1047         if (Irp->CurrentLocation > (Irp->StackCount + 1))
1048         {
1049             /* It is, bugcheck */
1050             KeBugCheckEx(CANCEL_STATE_IN_COMPLETED_IRP,
1051                          (ULONG_PTR)Irp,
1052                          (ULONG_PTR)CancelRoutine,
1053                          0,
1054                          0);
1055         }
1056
1057         /* Set the cancel IRQL And call the routine */
1058         Irp->CancelIrql = OldIrql;
1059         CancelRoutine(IoGetCurrentIrpStackLocation(Irp)->DeviceObject, Irp);
1060         return TRUE;
1061     }
1062
1063     /* Otherwise, release the cancel lock and fail */
1064     IoReleaseCancelSpinLock(OldIrql);
1065     return FALSE;
1066 }
1067
1068 /*
1069  * @implemented
1070  */
1071 VOID
1072 NTAPI
1073 IoCancelThreadIo(IN PETHREAD Thread)
1074 {
1075     KIRQL OldIrql;
1076     ULONG Retries = 3000;
1077     LARGE_INTEGER Interval;
1078     PLIST_ENTRY ListHead, NextEntry;
1079     PIRP Irp;
1080     PAGED_CODE();
1081
1082     /* Windows isn't using given thread, but using current. */
1083     Thread = PsGetCurrentThread();
1084
1085     IOTRACE(IO_IRP_DEBUG,
1086             "%s - Canceling IRPs for Thread %p\n",
1087             __FUNCTION__,
1088             Thread);
1089
1090     /* Raise to APC to protect the IrpList */
1091     KeRaiseIrql(APC_LEVEL, &OldIrql);
1092
1093     /* Start by cancelling all the IRPs in the current thread queue. */
1094     ListHead = &Thread->IrpList;
1095     NextEntry = ListHead->Flink;
1096     while (ListHead != NextEntry)
1097     {
1098         /* Get the IRP */
1099         Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
1100
1101         /* Cancel it */
1102         IoCancelIrp(Irp);
1103
1104         /* Move to the next entry */
1105         NextEntry = NextEntry->Flink;
1106     }
1107
1108      /* Wait 100 milliseconds */
1109     Interval.QuadPart = -1000000;
1110
1111     /* Wait till all the IRPs are completed or cancelled. */
1112     while (!IsListEmpty(&Thread->IrpList))
1113     {
1114         /* Now we can lower */
1115         KeLowerIrql(OldIrql);
1116
1117         /* Wait a short while and then look if all our IRPs were completed. */
1118         KeDelayExecutionThread(KernelMode, FALSE, &Interval);
1119
1120         /*
1121          * Don't stay here forever if some broken driver doesn't complete
1122          * the IRP.
1123          */
1124         if (!(Retries--))
1125         {
1126             /* Print out a message and remove the IRP */
1127             DPRINT1("Broken driver did not complete!\n");
1128             IopDisassociateThreadIrp();
1129         }
1130
1131         /* Raise the IRQL Again */
1132         KeRaiseIrql(APC_LEVEL, &OldIrql);
1133     }
1134
1135     /* We're done, lower the IRQL */
1136     KeLowerIrql(OldIrql);
1137 }
1138
1139 /*
1140  * @implemented
1141  */
1142 #undef IoCallDriver
1143 NTSTATUS
1144 NTAPI
1145 IoCallDriver(IN PDEVICE_OBJECT DeviceObject,
1146              IN PIRP Irp)
1147 {
1148     /* Call fastcall */
1149     return IofCallDriver(DeviceObject, Irp);
1150 }
1151
1152 #define IoCallDriver IofCallDriver
1153
1154 /*
1155  * @implemented
1156  */
1157 #undef IoCompleteRequest
1158 VOID
1159 NTAPI
1160 IoCompleteRequest(IN PIRP Irp,
1161                   IN CCHAR PriorityBoost)
1162 {
1163     /* Call the fastcall */
1164     IofCompleteRequest(Irp, PriorityBoost);
1165 }
1166
1167 #define IoCompleteRequest IofCompleteRequest
1168
1169 /*
1170  * @implemented
1171  */
1172 VOID
1173 NTAPI
1174 IoEnqueueIrp(IN PIRP Irp)
1175 {
1176     /* This is the same as calling IoQueueThreadIrp */
1177     IoQueueThreadIrp(Irp);
1178 }
1179
1180 /*
1181  * @implemented
1182  */
1183 NTSTATUS
1184 FASTCALL
1185 IofCallDriver(IN PDEVICE_OBJECT DeviceObject,
1186               IN PIRP Irp)
1187 {
1188     PDRIVER_OBJECT DriverObject;
1189     PIO_STACK_LOCATION StackPtr;
1190
1191     /* Make sure this is a valid IRP */
1192     ASSERT(Irp->Type == IO_TYPE_IRP);
1193
1194     /* Get the Driver Object */
1195     DriverObject = DeviceObject->DriverObject;
1196
1197     /* Decrease the current location and check if */
1198     Irp->CurrentLocation--;
1199     if (Irp->CurrentLocation <= 0)
1200     {
1201         /* This IRP ran out of stack, bugcheck */
1202         KeBugCheckEx(NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR)Irp, 0, 0, 0);
1203     }
1204
1205     /* Now update the stack location */
1206     StackPtr = IoGetNextIrpStackLocation(Irp);
1207     Irp->Tail.Overlay.CurrentStackLocation = StackPtr;
1208
1209     /* Get the Device Object */
1210     StackPtr->DeviceObject = DeviceObject;
1211
1212     /* Call it */
1213     return DriverObject->MajorFunction[StackPtr->MajorFunction](DeviceObject,
1214                                                                 Irp);
1215 }
1216
1217 FORCEINLINE
1218 VOID
1219 IopClearStackLocation(IN PIO_STACK_LOCATION IoStackLocation)
1220 {
1221     IoStackLocation->MinorFunction = 0;
1222     IoStackLocation->Flags = 0;
1223     IoStackLocation->Control &= SL_ERROR_RETURNED;
1224     IoStackLocation->Parameters.Others.Argument1 = 0;
1225     IoStackLocation->Parameters.Others.Argument2 = 0;
1226     IoStackLocation->Parameters.Others.Argument3 = 0;
1227     IoStackLocation->FileObject = NULL;
1228 }
1229
1230 /*
1231  * @implemented
1232  */
1233 VOID
1234 FASTCALL
1235 IofCompleteRequest(IN PIRP Irp,
1236                    IN CCHAR PriorityBoost)
1237 {
1238     PIO_STACK_LOCATION StackPtr, LastStackPtr;
1239     PDEVICE_OBJECT DeviceObject;
1240     PFILE_OBJECT FileObject;
1241     PETHREAD Thread;
1242     NTSTATUS Status;
1243     PMDL Mdl, NextMdl;
1244     ULONG MasterCount;
1245     PIRP MasterIrp;
1246     ULONG Flags;
1247     NTSTATUS ErrorCode = STATUS_SUCCESS;
1248     PREPARSE_DATA_BUFFER DataBuffer;
1249     IOTRACE(IO_IRP_DEBUG,
1250             "%s - Completing IRP %p\n",
1251             __FUNCTION__,
1252             Irp);
1253
1254     /* Make sure this IRP isn't getting completed twice or is invalid */
1255     if ((Irp->CurrentLocation) > (Irp->StackCount + 1))
1256     {
1257         /* Bugcheck */
1258         KeBugCheckEx(MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR)Irp, 0, 0, 0);
1259     }
1260
1261     /* Some sanity checks */
1262     ASSERT(Irp->Type == IO_TYPE_IRP);
1263     ASSERT(!Irp->CancelRoutine);
1264     ASSERT(Irp->IoStatus.Status != STATUS_PENDING);
1265     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
1266
1267     /* Get the last stack */
1268     LastStackPtr = (PIO_STACK_LOCATION)(Irp + 1);
1269     if (LastStackPtr->Control & SL_ERROR_RETURNED)
1270     {
1271         /* Get the error code */
1272         ErrorCode = PtrToUlong(LastStackPtr->Parameters.Others.Argument4);
1273     }
1274
1275     /*
1276      * Start the loop with the current stack and point the IRP to the next stack
1277      * and then keep incrementing the stack as we loop through. The IRP should
1278      * always point to the next stack location w.r.t the one currently being
1279      * analyzed, so completion routine code will see the appropriate value.
1280      * Because of this, we must loop until the current stack location is +1 of
1281      * the stack count, because when StackPtr is at the end, CurrentLocation is +1.
1282      */
1283     for (StackPtr = IoGetCurrentIrpStackLocation(Irp),
1284          Irp->CurrentLocation++,
1285          Irp->Tail.Overlay.CurrentStackLocation++;
1286          Irp->CurrentLocation <= (Irp->StackCount + 1);
1287          StackPtr++,
1288          Irp->CurrentLocation++,
1289          Irp->Tail.Overlay.CurrentStackLocation++)
1290     {
1291         /* Set Pending Returned */
1292         Irp->PendingReturned = StackPtr->Control & SL_PENDING_RETURNED;
1293
1294         /* Check if we failed */
1295         if (!NT_SUCCESS(Irp->IoStatus.Status))
1296         {
1297             /* Check if it was changed by a completion routine */
1298             if (Irp->IoStatus.Status != ErrorCode)
1299             {
1300                 /* Update the error for the current stack */
1301                 ErrorCode = Irp->IoStatus.Status;
1302                 StackPtr->Control |= SL_ERROR_RETURNED;
1303                 LastStackPtr->Parameters.Others.Argument4 = UlongToPtr(ErrorCode);
1304                 LastStackPtr->Control |= SL_ERROR_RETURNED;
1305             }
1306         }
1307
1308         /* Check if there is a Completion Routine to Call */
1309         if ((NT_SUCCESS(Irp->IoStatus.Status) &&
1310              (StackPtr->Control & SL_INVOKE_ON_SUCCESS)) ||
1311             (!NT_SUCCESS(Irp->IoStatus.Status) &&
1312              (StackPtr->Control & SL_INVOKE_ON_ERROR)) ||
1313             (Irp->Cancel &&
1314              (StackPtr->Control & SL_INVOKE_ON_CANCEL)))
1315         {
1316             /* Clear the stack location */
1317             IopClearStackLocation(StackPtr);
1318
1319             /* Check for highest-level device completion routines */
1320             if (Irp->CurrentLocation == (Irp->StackCount + 1))
1321             {
1322                 /* Clear the DO, since the current stack location is invalid */
1323                 DeviceObject = NULL;
1324             }
1325             else
1326             {
1327                 /* Otherwise, return the real one */
1328                 DeviceObject = IoGetCurrentIrpStackLocation(Irp)->DeviceObject;
1329             }
1330
1331             /* Call the completion routine */
1332             Status = StackPtr->CompletionRoutine(DeviceObject,
1333                                                  Irp,
1334                                                  StackPtr->Context);
1335
1336             /* Don't touch the Packet in this case, since it might be gone! */
1337             if (Status == STATUS_MORE_PROCESSING_REQUIRED) return;
1338         }
1339         else
1340         {
1341             /* Otherwise, check if this is a completed IRP */
1342             if ((Irp->CurrentLocation <= Irp->StackCount) &&
1343                 (Irp->PendingReturned))
1344             {
1345                 /* Mark it as pending */
1346                 IoMarkIrpPending(Irp);
1347             }
1348
1349             /* Clear the stack location */
1350             IopClearStackLocation(StackPtr);
1351         }
1352     }
1353
1354     /* Check if the IRP is an associated IRP */
1355     if (Irp->Flags & IRP_ASSOCIATED_IRP)
1356     {
1357         /* Get the master IRP and count */
1358         MasterIrp = Irp->AssociatedIrp.MasterIrp;
1359         MasterCount = InterlockedDecrement(&MasterIrp->AssociatedIrp.IrpCount);
1360
1361         /* Free the MDLs */
1362         for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
1363         {
1364             /* Go to the next one */
1365             NextMdl = Mdl->Next;
1366             IoFreeMdl(Mdl);
1367         }
1368
1369         /* Free the IRP itself */
1370         IoFreeIrp(Irp);
1371
1372         /* Complete the Master IRP */
1373         if (!MasterCount) IofCompleteRequest(MasterIrp, PriorityBoost);
1374         return;
1375     }
1376
1377     /* Check whether we have to reparse */
1378     if (Irp->IoStatus.Status == STATUS_REPARSE)
1379     {
1380         if (Irp->IoStatus.Information > IO_REMOUNT)
1381         {
1382             /* If that's a reparse tag we understand, save the buffer from deletion */
1383             if (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT)
1384             {
1385                 ASSERT(Irp->Tail.Overlay.AuxiliaryBuffer != NULL);
1386                 DataBuffer = (PREPARSE_DATA_BUFFER)Irp->Tail.Overlay.AuxiliaryBuffer;
1387                 Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1388             }
1389             else
1390             {
1391                 Irp->IoStatus.Status = STATUS_IO_REPARSE_TAG_NOT_HANDLED;
1392             }
1393         }
1394     }
1395
1396     /* Check if we have an auxiliary buffer */
1397     if (Irp->Tail.Overlay.AuxiliaryBuffer)
1398     {
1399         /* Free it */
1400         ExFreePool(Irp->Tail.Overlay.AuxiliaryBuffer);
1401         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1402     }
1403
1404     /* Check if this is a Paging I/O or Close Operation */
1405     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION))
1406     {
1407         /* Handle a Close Operation or Sync Paging I/O */
1408         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION))
1409         {
1410             /* Set the I/O Status and Signal the Event */
1411             Flags = Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_PAGING_IO);
1412             *Irp->UserIosb = Irp->IoStatus;
1413             KeSetEvent(Irp->UserEvent, PriorityBoost, FALSE);
1414
1415             /* Free the IRP for a Paging I/O Only, Close is handled by us */
1416             if (Flags) IoFreeIrp(Irp);
1417         }
1418         else
1419         {
1420 #if 0
1421             /* Page 166 */
1422             KeInitializeApc(&Irp->Tail.Apc
1423                             &Irp->Tail.Overlay.Thread->Tcb,
1424                             Irp->ApcEnvironment,
1425                             IopCompletePageWrite,
1426                             NULL,
1427                             NULL,
1428                             KernelMode,
1429                             NULL);
1430             KeInsertQueueApc(&Irp->Tail.Apc,
1431                              NULL,
1432                              NULL,
1433                              PriorityBoost);
1434 #else
1435             /* Not implemented yet. */
1436             UNIMPLEMENTED_DBGBREAK("Not supported!\n");
1437 #endif
1438         }
1439
1440         /* Get out of here */
1441         return;
1442     }
1443
1444     /* Unlock MDL Pages, page 167. */
1445     Mdl = Irp->MdlAddress;
1446     while (Mdl)
1447     {
1448         MmUnlockPages(Mdl);
1449         Mdl = Mdl->Next;
1450     }
1451
1452     /* Check if we should exit because of a Deferred I/O (page 168) */
1453     if ((Irp->Flags & IRP_DEFER_IO_COMPLETION) && !(Irp->PendingReturned))
1454     {
1455         /* Restore the saved reparse buffer for the caller */
1456         if (Irp->IoStatus.Status == STATUS_REPARSE &&
1457             Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT)
1458         {
1459             Irp->Tail.Overlay.AuxiliaryBuffer = (PCHAR)DataBuffer;
1460         }
1461
1462         /*
1463          * Return without queuing the completion APC, since the caller will
1464          * take care of doing its own optimized completion at PASSIVE_LEVEL.
1465          */
1466         return;
1467     }
1468
1469     /* Get the thread and file object */
1470     Thread = Irp->Tail.Overlay.Thread;
1471     FileObject = Irp->Tail.Overlay.OriginalFileObject;
1472
1473     /* Make sure the IRP isn't canceled */
1474     if (!Irp->Cancel)
1475     {
1476         /* Initialize the APC */
1477         KeInitializeApc(&Irp->Tail.Apc,
1478                         &Thread->Tcb,
1479                         Irp->ApcEnvironment,
1480                         IopCompleteRequest,
1481                         NULL,
1482                         NULL,
1483                         KernelMode,
1484                         NULL);
1485
1486         /* Queue it */
1487         KeInsertQueueApc(&Irp->Tail.Apc,
1488                          FileObject,
1489                          DataBuffer,
1490                          PriorityBoost);
1491     }
1492     else
1493     {
1494         /* The IRP just got canceled... does a thread still own it? */
1495         if (Thread)
1496         {
1497             /* Yes! There is still hope! Initialize the APC */
1498             KeInitializeApc(&Irp->Tail.Apc,
1499                             &Thread->Tcb,
1500                             Irp->ApcEnvironment,
1501                             IopCompleteRequest,
1502                             NULL,
1503                             NULL,
1504                             KernelMode,
1505                             NULL);
1506
1507             /* Queue it */
1508             KeInsertQueueApc(&Irp->Tail.Apc,
1509                              FileObject,
1510                              DataBuffer,
1511                              PriorityBoost);
1512         }
1513         else
1514         {
1515             /* Nothing left for us to do, kill it */
1516             ASSERT(Irp->Cancel);
1517             IopCleanupIrp(Irp, FileObject);
1518         }
1519     }
1520 }
1521
1522 NTSTATUS
1523 NTAPI
1524 IopSynchronousCompletion(IN PDEVICE_OBJECT DeviceObject,
1525                          IN PIRP Irp,
1526                          IN PVOID Context)
1527 {
1528     if (Irp->PendingReturned)
1529         KeSetEvent((PKEVENT)Context, IO_NO_INCREMENT, FALSE);
1530     return STATUS_MORE_PROCESSING_REQUIRED;
1531 }
1532
1533 /*
1534  * @implemented
1535  */
1536 BOOLEAN
1537 NTAPI
1538 IoForwardIrpSynchronously(IN PDEVICE_OBJECT DeviceObject,
1539                           IN PIRP Irp)
1540 {
1541     KEVENT Event;
1542     NTSTATUS Status;
1543
1544     /* Check if next stack location is available */
1545     if (Irp->CurrentLocation < Irp->StackCount)
1546     {
1547         /* No more stack location */
1548         return FALSE;
1549     }
1550
1551     /* Initialize event */
1552     KeInitializeEvent(&Event, NotificationEvent, FALSE);
1553
1554     /* Copy stack location for next driver */
1555     IoCopyCurrentIrpStackLocationToNext(Irp);
1556
1557     /* Set a completion routine, which will signal the event */
1558     IoSetCompletionRoutine(Irp, IopSynchronousCompletion, &Event, TRUE, TRUE, TRUE);
1559
1560     /* Call next driver */
1561     Status = IoCallDriver(DeviceObject, Irp);
1562
1563     /* Check if irp is pending */
1564     if (Status == STATUS_PENDING)
1565     {
1566         /* Yes, wait for its completion */
1567         KeWaitForSingleObject(&Event, Suspended, KernelMode, FALSE, NULL);
1568     }
1569
1570     /* Return success */
1571     return TRUE;
1572 }
1573
1574 /*
1575  * @implemented
1576  */
1577 VOID
1578 NTAPI
1579 IoFreeIrp(IN PIRP Irp)
1580 {
1581     PNPAGED_LOOKASIDE_LIST List;
1582     PP_NPAGED_LOOKASIDE_NUMBER ListType =  LookasideSmallIrpList;
1583     PKPRCB Prcb;
1584     IOTRACE(IO_IRP_DEBUG,
1585             "%s - Freeing IRPs %p\n",
1586             __FUNCTION__,
1587             Irp);
1588
1589     /* Make sure the Thread IRP list is empty and that it OK to free it */
1590     ASSERT(Irp->Type == IO_TYPE_IRP);
1591     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1592     ASSERT(Irp->CurrentLocation >= Irp->StackCount);
1593
1594     /* If this was a pool alloc, free it with the pool */
1595     if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
1596     {
1597         /* Free it */
1598         ExFreePoolWithTag(Irp, TAG_IRP);
1599     }
1600     else
1601     {
1602         /* Check if this was a Big IRP */
1603         if (Irp->StackCount != 1) ListType = LookasideLargeIrpList;
1604
1605         /* Get the PRCB */
1606         Prcb = KeGetCurrentPrcb();
1607
1608         /* Use the P List */
1609         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
1610         List->L.TotalFrees++;
1611
1612         /* Check if the Free was within the Depth or not */
1613         if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1614         {
1615             /* Let the balancer know */
1616             List->L.FreeMisses++;
1617
1618             /* Use the L List */
1619             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
1620             List->L.TotalFrees++;
1621
1622             /* Check if the Free was within the Depth or not */
1623             if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1624             {
1625                 /* All lists failed, use the pool */
1626                 List->L.FreeMisses++;
1627                 ExFreePoolWithTag(Irp, TAG_IRP);
1628                 Irp = NULL;
1629             }
1630         }
1631
1632         /* The free was within the Depth */
1633         if (Irp)
1634         {
1635            InterlockedPushEntrySList(&List->L.ListHead,
1636                                      (PSLIST_ENTRY)Irp);
1637         }
1638     }
1639 }
1640
1641 /*
1642  * @implemented
1643  */
1644 IO_PAGING_PRIORITY
1645 FASTCALL
1646 IoGetPagingIoPriority(IN PIRP Irp)
1647 {
1648     IO_PAGING_PRIORITY Priority;
1649     ULONG Flags;
1650
1651     /* Get the flags */
1652     Flags = Irp->Flags;
1653
1654     /* Check what priority it has */
1655     if (Flags & IRP_CLASS_CACHE_OPERATION)
1656     {
1657         /* High priority */
1658         Priority = IoPagingPriorityHigh;
1659     }
1660     else if (Flags & IRP_PAGING_IO)
1661     {
1662         /* Normal priority */
1663         Priority = IoPagingPriorityNormal;
1664     }
1665     else
1666     {
1667         /* Invalid -- not a paging IRP */
1668         Priority = IoPagingPriorityInvalid;
1669     }
1670
1671     /* Return the priority */
1672     return Priority;
1673 }
1674
1675 /*
1676  * @implemented
1677  */
1678 PEPROCESS
1679 NTAPI
1680 IoGetRequestorProcess(IN PIRP Irp)
1681 {
1682     /* Return the requestor process */
1683     if (Irp->Tail.Overlay.Thread)
1684     {
1685         if (Irp->ApcEnvironment == OriginalApcEnvironment)
1686         {
1687             return Irp->Tail.Overlay.Thread->ThreadsProcess;
1688         }
1689         else if (Irp->ApcEnvironment == AttachedApcEnvironment)
1690         {
1691             return (PEPROCESS)Irp->Tail.Overlay.Thread->Tcb.ApcState.Process;
1692         }
1693     }
1694
1695     return NULL;
1696 }
1697
1698 /*
1699  * @implemented
1700  */
1701 ULONG
1702 NTAPI
1703 IoGetRequestorProcessId(IN PIRP Irp)
1704 {
1705     PEPROCESS Process;
1706
1707     /* Return the requestor process' id */
1708     Process = IoGetRequestorProcess(Irp);
1709     if (Process) return PtrToUlong(Process->UniqueProcessId);
1710
1711     return 0;
1712 }
1713
1714 /*
1715  * @implemented
1716  */
1717 NTSTATUS
1718 NTAPI
1719 IoGetRequestorSessionId(IN PIRP Irp,
1720                         OUT PULONG pSessionId)
1721 {
1722     PEPROCESS Process;
1723
1724     /* Return the session */
1725     if (Irp->Tail.Overlay.Thread)
1726     {
1727         Process = Irp->Tail.Overlay.Thread->ThreadsProcess;
1728         *pSessionId = MmGetSessionId(Process);
1729         return STATUS_SUCCESS;
1730     }
1731
1732     *pSessionId = (ULONG)-1;
1733     return STATUS_UNSUCCESSFUL;
1734 }
1735
1736 /*
1737  * @implemented
1738  */
1739 PIRP
1740 NTAPI
1741 IoGetTopLevelIrp(VOID)
1742 {
1743     /* Return the IRP */
1744     return (PIRP)PsGetCurrentThread()->TopLevelIrp;
1745 }
1746
1747 /*
1748  * @implemented
1749  */
1750 VOID
1751 NTAPI
1752 IoInitializeIrp(IN PIRP Irp,
1753                 IN USHORT PacketSize,
1754                 IN CCHAR StackSize)
1755 {
1756     /* Clear it */
1757     IOTRACE(IO_IRP_DEBUG,
1758             "%s - Initializing IRP %p\n",
1759             __FUNCTION__,
1760             Irp);
1761     RtlZeroMemory(Irp, PacketSize);
1762
1763     /* Set the Header and other data */
1764     Irp->Type = IO_TYPE_IRP;
1765     Irp->Size = PacketSize;
1766     Irp->StackCount = StackSize;
1767     Irp->CurrentLocation = StackSize + 1;
1768     Irp->ApcEnvironment =  KeGetCurrentThread()->ApcStateIndex;
1769     Irp->Tail.Overlay.CurrentStackLocation = (PIO_STACK_LOCATION)(Irp + 1) + StackSize;
1770
1771     /* Initialize the Thread List */
1772     InitializeListHead(&Irp->ThreadListEntry);
1773 }
1774
1775 /*
1776  * @implemented
1777  */
1778 BOOLEAN
1779 NTAPI
1780 IoIsOperationSynchronous(IN PIRP Irp)
1781 {
1782     BOOLEAN SynchIO;
1783     BOOLEAN ForceAsync;
1784
1785     /* If the IRP requests synchronous paging I/O, if the file object was opened
1786        for synchronous I/O, if the IRP_SYNCHRONOUS_API flag is set in the IRP
1787        the operation is synchronous */
1788     SynchIO = (IoGetCurrentIrpStackLocation(Irp)->FileObject->Flags & FO_SYNCHRONOUS_IO) ||
1789               (Irp->Flags & IRP_SYNCHRONOUS_API) || (Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1790
1791     /* If the IRP requests asynchronous paging I/O, the operation is asynchronous,
1792        even if one of the above conditions is true */
1793     ForceAsync = (Irp->Flags & IRP_PAGING_IO) && !(Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1794
1795     /* Check the flags */
1796     if (SynchIO && !ForceAsync)
1797     {
1798         /* Synch API or Paging I/O is OK, as is Sync File I/O */
1799         return TRUE;
1800     }
1801
1802     /* Otherwise, it is an asynchronous operation. */
1803     return FALSE;
1804 }
1805
1806 /*
1807  * @unimplemented
1808  */
1809 BOOLEAN
1810 NTAPI
1811 IoIsValidNameGraftingBuffer(IN PIRP Irp,
1812                             IN PREPARSE_DATA_BUFFER ReparseBuffer)
1813 {
1814     UNIMPLEMENTED;
1815     return FALSE;
1816 }
1817
1818 /*
1819  * @implemented
1820  */
1821 PIRP
1822 NTAPI
1823 IoMakeAssociatedIrp(IN PIRP Irp,
1824                     IN CCHAR StackSize)
1825 {
1826     PIRP AssocIrp;
1827     IOTRACE(IO_IRP_DEBUG,
1828             "%s - Associating IRP %p\n",
1829             __FUNCTION__,
1830             Irp);
1831
1832    /* Allocate the IRP */
1833    AssocIrp = IoAllocateIrp(StackSize, FALSE);
1834    if (!AssocIrp) return NULL;
1835
1836    /* Set the Flags */
1837    AssocIrp->Flags |= IRP_ASSOCIATED_IRP;
1838
1839    /* Set the Thread */
1840    AssocIrp->Tail.Overlay.Thread = Irp->Tail.Overlay.Thread;
1841
1842    /* Associate them */
1843    AssocIrp->AssociatedIrp.MasterIrp = Irp;
1844    return AssocIrp;
1845 }
1846
1847 /*
1848  * @implemented
1849  */
1850 VOID
1851 NTAPI
1852 IoQueueThreadIrp(IN PIRP Irp)
1853 {
1854     IOTRACE(IO_IRP_DEBUG,
1855             "%s - Queueing IRP %p\n",
1856             __FUNCTION__,
1857             Irp);
1858
1859     /* Use our inlined routine */
1860     IopQueueIrpToThread(Irp);
1861 }
1862
1863 /*
1864  * @implemented
1865  * Reference: Chris Cant's "Writing WDM Device Drivers"
1866  */
1867 VOID
1868 NTAPI
1869 IoReuseIrp(IN OUT PIRP Irp,
1870            IN NTSTATUS Status)
1871 {
1872     UCHAR AllocationFlags;
1873     IOTRACE(IO_IRP_DEBUG,
1874             "%s - Reusing IRP %p\n",
1875             __FUNCTION__,
1876             Irp);
1877
1878     /* Make sure it's OK to reuse it */
1879     ASSERT(!Irp->CancelRoutine);
1880     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1881
1882     /* Get the old flags */
1883     AllocationFlags = Irp->AllocationFlags;
1884
1885     /* Reinitialize the IRP */
1886     IoInitializeIrp(Irp, Irp->Size, Irp->StackCount);
1887
1888     /* Duplicate the data */
1889     Irp->IoStatus.Status = Status;
1890     Irp->AllocationFlags = AllocationFlags;
1891 }
1892
1893 /*
1894  * @implemented
1895  */
1896 VOID
1897 NTAPI
1898 IoSetTopLevelIrp(IN PIRP Irp)
1899 {
1900     /* Set the IRP */
1901     PsGetCurrentThread()->TopLevelIrp = (ULONG_PTR)Irp;
1902 }
1903
1904 #if defined (_WIN64)
1905 BOOLEAN
1906 NTAPI
1907 IoIs32bitProcess(
1908     IN PIRP Irp OPTIONAL)
1909 {
1910     UNIMPLEMENTED;
1911     return FALSE;
1912 }
1913 #endif