reactos/ntoskrnl/io/iomgr/irp.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/io/iomgr/irp.c
   5  * PURPOSE:         IRP Handling Functions
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Gunnar Dalsnes
   8  *                  Filip Navara (navaraf@reactos.org)
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 PIRP IopDeadIrp;
  18
  19 /* PRIVATE FUNCTIONS  ********************************************************/
  20
  21 VOID
  22 NTAPI
  23 IopFreeIrpKernelApc(IN PKAPC Apc,
  24                     IN PKNORMAL_ROUTINE *NormalRoutine,
  25                     IN PVOID *NormalContext,
  26                     IN PVOID *SystemArgument1,
  27                     IN PVOID *SystemArgument2)
  28 {
  29     /* Free the IRP */
  30     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  31 }
  32
  33 VOID
  34 NTAPI
  35 IopAbortIrpKernelApc(IN PKAPC Apc)
  36 {
  37     /* Free the IRP */
  38     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  39 }
  40
  41 NTSTATUS
  42 NTAPI
  43 IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
  44                     IN PKEVENT EventObject OPTIONAL,
  45                     IN PVOID Buffer OPTIONAL)
  46 {
  47     PAGED_CODE();
  48
  49     /* Dereference the event */
  50     if (EventObject) ObDereferenceObject(EventObject);
  51
  52     /* Free a buffer, if any */
  53     if (Buffer) ExFreePool(Buffer);
  54
  55     /* If this was a file opened for synch I/O, then unlock it */
  56     if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
  57
  58     /* Now dereference it and return */
  59     ObDereferenceObject(FileObject);
  60     return STATUS_INSUFFICIENT_RESOURCES;
  61 }
  62
  63 VOID
  64 NTAPI
  65 IopAbortInterruptedIrp(IN PKEVENT EventObject,
  66                        IN PIRP Irp)
  67 {
  68     KIRQL OldIrql;
  69     BOOLEAN CancelResult;
  70     LARGE_INTEGER Wait;
  71     PAGED_CODE();
  72
  73     /* Raise IRQL to APC */
  74     KeRaiseIrql(APC_LEVEL, &OldIrql);
  75
  76     /* Check if nobody completed it yet */
  77     if (!KeReadStateEvent(EventObject))
  78     {
  79         /* First, cancel it */
  80         CancelResult = IoCancelIrp(Irp);
  81         KeLowerIrql(OldIrql);
  82
  83         /* Check if we cancelled it */
  84         if (CancelResult)
  85         {
  86             /* Wait for the IRP to be cancelled */
  87             Wait.QuadPart = -100000;
  88             while (!KeReadStateEvent(EventObject))
  89             {
  90                 /* Delay indefintely */
  91                 KeDelayExecutionThread(KernelMode, FALSE, &Wait);
  92             }
  93         }
  94         else
  95         {
  96             /* No cancellation done, so wait for the I/O system to kill it */
  97             KeWaitForSingleObject(EventObject,
  98                                   Executive,
  99                                   KernelMode,
 100                                   FALSE,
 101                                   NULL);
 102         }
 103     }
 104     else
 105     {
 106         /* We got preempted, so give up */
 107         KeLowerIrql(OldIrql);
 108     }
 109 }
 110
 111 VOID
 112 NTAPI
 113 IopDisassociateThreadIrp(VOID)
 114 {
 115     KIRQL OldIrql, LockIrql;
 116     PETHREAD IrpThread;
 117     PLIST_ENTRY IrpEntry;
 118     PIO_ERROR_LOG_PACKET ErrorLogEntry;
 119     PDEVICE_OBJECT DeviceObject = NULL;
 120     PIO_STACK_LOCATION IoStackLocation;
 121
 122     /* First, raise to APC to protect IrpList */
 123     KeRaiseIrql(APC_LEVEL, &OldIrql);
 124
 125     /* Get the Thread and check the list */
 126     IrpThread = PsGetCurrentThread();
 127     if (IsListEmpty(&IrpThread->IrpList))
 128     {
 129         /* It got completed now, so quit */
 130         KeLowerIrql(OldIrql);
 131         return;
 132     }
 133
 134     /* Ensure no one will come disturb */
 135     LockIrql = KeAcquireQueuedSpinLock(LockQueueIoCompletionLock);
 136
 137     /* Get the misbehaving IRP */
 138     IrpEntry = IrpThread->IrpList.Flink;
 139     IopDeadIrp = CONTAINING_RECORD(IrpEntry, IRP, ThreadListEntry);
 140     IOTRACE(IO_IRP_DEBUG,
 141             "%s - Deassociating IRP %p for %p\n",
 142             __FUNCTION__,
 143             IopDeadIrp,
 144             IrpThread);
 145
 146     /* Don't cancel the IRP if it's already been completed far */
 147     if (IopDeadIrp->CurrentLocation == (IopDeadIrp->StackCount + 2))
 148     {
 149         /* Return */
 150         KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 151         KeLowerIrql(OldIrql);
 152         return;
 153     }
 154
 155     /* Disown the IRP! */
 156     IopDeadIrp->Tail.Overlay.Thread = NULL;
 157     RemoveHeadList(&IrpThread->IrpList);
 158     InitializeListHead(&IopDeadIrp->ThreadListEntry);
 159
 160     /* Get the stack location and check if it's valid */
 161     IoStackLocation = IoGetCurrentIrpStackLocation(IopDeadIrp);
 162     if (IopDeadIrp->CurrentLocation <= IopDeadIrp->StackCount)
 163     {
 164         /* Get the device object */
 165         DeviceObject = IoStackLocation->DeviceObject;
 166     }
 167
 168     KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 169     /* Lower IRQL now, since we have the pointers we need */
 170     KeLowerIrql(OldIrql);
 171
 172     /* Check if we can send an Error Log Entry*/
 173     if (DeviceObject)
 174     {
 175         /* Allocate an entry */
 176         ErrorLogEntry = IoAllocateErrorLogEntry(DeviceObject,
 177                                                 sizeof(IO_ERROR_LOG_PACKET));
 178         if (ErrorLogEntry)
 179         {
 180             /* Write the entry */
 181             ErrorLogEntry->ErrorCode = IO_DRIVER_CANCEL_TIMEOUT;
 182             IoWriteErrorLogEntry(ErrorLogEntry);
 183         }
 184     }
 185 }
 186
 187 VOID
 188 NTAPI
 189 IopCleanupIrp(IN PIRP Irp,
 190               IN PFILE_OBJECT FileObject)
 191 {
 192     PMDL Mdl;
 193     IOTRACE(IO_IRP_DEBUG,
 194             "%s - Cleaning IRP %p for %p\n",
 195             __FUNCTION__,
 196             Irp,
 197             FileObject);
 198
 199     /* Check if there's an MDL */
 200     while ((Mdl = Irp->MdlAddress))
 201     {
 202         /* Clear all of them */
 203         Irp->MdlAddress = Mdl->Next;
 204         IoFreeMdl(Mdl);
 205     }
 206
 207     /* Check if the IRP has system buffer */
 208     if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 209     {
 210         /* Free the buffer */
 211         ExFreePoolWithTag(Irp->AssociatedIrp.SystemBuffer, TAG_SYS_BUF);
 212     }
 213
 214     /* Check if this IRP has a user event, a file object, and is async */
 215     if ((Irp->UserEvent) &&
 216         !(Irp->Flags & IRP_SYNCHRONOUS_API) &&
 217         (FileObject))
 218     {
 219         /* Dereference the User Event */
 220         ObDereferenceObject(Irp->UserEvent);
 221     }
 222
 223     /* Check if we have a file object and this isn't a create operation */
 224     if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 225     {
 226         /* Dereference the file object */
 227         ObDereferenceObject(FileObject);
 228     }
 229
 230     /* Free the IRP */
 231     IoFreeIrp(Irp);
 232 }
 233
 234 VOID
 235 NTAPI
 236 IopCompleteRequest(IN PKAPC Apc,
 237                    IN PKNORMAL_ROUTINE* NormalRoutine,
 238                    IN PVOID* NormalContext,
 239                    IN PVOID* SystemArgument1,
 240                    IN PVOID* SystemArgument2)
 241 {
 242     PFILE_OBJECT FileObject;
 243     PIRP Irp;
 244     PMDL Mdl, NextMdl;
 245     PVOID Port = NULL, Key = NULL;
 246     BOOLEAN SignaledCreateRequest = FALSE;
 247
 248     /* Get data from the APC */
 249     FileObject = (PFILE_OBJECT)*SystemArgument1;
 250     Irp = CONTAINING_RECORD(Apc, IRP, Tail.Apc);
 251     IOTRACE(IO_IRP_DEBUG,
 252             "%s - Completing IRP %p for %p\n",
 253             __FUNCTION__,
 254             Irp,
 255             FileObject);
 256
 257     /* Sanity check */
 258     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
 259
 260     /* Check if we have a file object */
 261     if (*SystemArgument2)
 262     {
 263         /* Check if we're reparsing */
 264         if ((Irp->IoStatus.Status == STATUS_REPARSE) &&
 265             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT))
 266         {
 267             PREPARSE_DATA_BUFFER ReparseData;
 268
 269             ReparseData = (PREPARSE_DATA_BUFFER)*SystemArgument2;
 270
 271             ASSERT(ReparseData->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT);
 272             ASSERT(ReparseData->ReparseDataLength < MAXIMUM_REPARSE_DATA_BUFFER_SIZE);
 273             ASSERT(ReparseData->Reserved < MAXIMUM_REPARSE_DATA_BUFFER_SIZE);
 274
 275             IopDoNameTransmogrify(Irp, FileObject, ReparseData);
 276         }
 277     }
 278
 279     /* Handle Buffered case first */
 280     if (Irp->Flags & IRP_BUFFERED_IO)
 281     {
 282         /* Check if we have an input buffer and if we succeeded */
 283         if ((Irp->Flags & IRP_INPUT_OPERATION) &&
 284             (Irp->IoStatus.Status != STATUS_VERIFY_REQUIRED) &&
 285             !(NT_ERROR(Irp->IoStatus.Status)))
 286         {
 287             _SEH2_TRY
 288             {
 289                 /* Copy the buffer back to the user */
 290                 RtlCopyMemory(Irp->UserBuffer,
 291                               Irp->AssociatedIrp.SystemBuffer,
 292                               Irp->IoStatus.Information);
 293             }
 294             _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 295             {
 296                 /* Fail the IRP */
 297                 Irp->IoStatus.Status = _SEH2_GetExceptionCode();
 298             }
 299             _SEH2_END;
 300         }
 301
 302         /* Also check if we should de-allocate it */
 303         if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 304         {
 305             /* Deallocate it */
 306             ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 307         }
 308     }
 309
 310     /* Now we got rid of these two... */
 311     Irp->Flags &= ~(IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER);
 312
 313     /* Check if there's an MDL */
 314     for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
 315     {
 316         /* Free it */
 317         NextMdl = Mdl->Next;
 318         IoFreeMdl(Mdl);
 319     }
 320
 321     /* No MDLs left */
 322     Irp->MdlAddress = NULL;
 323
 324     /*
 325      * Check if either the request was completed without any errors
 326      * (but warnings are OK!), or if it was completed with an error, but
 327      * did return from a pending I/O Operation and is not synchronous.
 328      */
 329     if (!NT_ERROR(Irp->IoStatus.Status) ||
 330         (Irp->PendingReturned &&
 331          !IsIrpSynchronous(Irp, FileObject)))
 332     {
 333         /* Get any information we need from the FO before we kill it */
 334         if ((FileObject) && (FileObject->CompletionContext))
 335         {
 336             /* Save Completion Data */
 337             Port = FileObject->CompletionContext->Port;
 338             Key = FileObject->CompletionContext->Key;
 339         }
 340
 341         /* Check for UserIos */
 342         if (Irp->UserIosb != NULL)
 343         {
 344             /* Use SEH to make sure we don't write somewhere invalid */
 345             _SEH2_TRY
 346             {
 347                 /*  Save the IOSB Information */
 348                 *Irp->UserIosb = Irp->IoStatus;
 349             }
 350             _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 351             {
 352                 /* Ignore any error */
 353             }
 354             _SEH2_END;
 355         }
 356
 357         /* Check if we have an event or a file object */
 358         if (Irp->UserEvent)
 359         {
 360             /* At the very least, this is a PKEVENT, so signal it always */
 361             KeSetEvent(Irp->UserEvent, 0, FALSE);
 362
 363             /* Check if we also have a File Object */
 364             if (FileObject)
 365             {
 366                 /* Check if this is an Asynch API */
 367                 if (!(Irp->Flags & IRP_SYNCHRONOUS_API))
 368                 {
 369                     /* Dereference the event */
 370                     ObDereferenceObject(Irp->UserEvent);
 371                 }
 372
 373                 /*
 374                  * Now, if this is a Synch I/O File Object, then this event is
 375                  * NOT an actual Executive Event, so we won't dereference it,
 376                  * and instead, we will signal the File Object
 377                  */
 378                 if ((FileObject->Flags & FO_SYNCHRONOUS_IO) &&
 379                     !(Irp->Flags & IRP_OB_QUERY_NAME))
 380                 {
 381                     /* Signal the file object and set the status */
 382                     KeSetEvent(&FileObject->Event, 0, FALSE);
 383                     FileObject->FinalStatus = Irp->IoStatus.Status;
 384                 }
 385
 386                 /*
 387                  * This could also be a create operation, in which case we want
 388                  * to make sure there's no APC fired.
 389                  */
 390                 if (Irp->Flags & IRP_CREATE_OPERATION)
 391                 {
 392                     /* Clear the APC Routine and remember this */
 393                     Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 394                     SignaledCreateRequest = TRUE;
 395                 }
 396             }
 397         }
 398         else if (FileObject)
 399         {
 400             /* Signal the file object and set the status */
 401             KeSetEvent(&FileObject->Event, 0, FALSE);
 402             FileObject->FinalStatus = Irp->IoStatus.Status;
 403
 404             /*
 405             * This could also be a create operation, in which case we want
 406             * to make sure there's no APC fired.
 407             */
 408             if (Irp->Flags & IRP_CREATE_OPERATION)
 409             {
 410                 /* Clear the APC Routine and remember this */
 411                 Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 412                 SignaledCreateRequest = TRUE;
 413             }
 414         }
 415
 416         /* Update transfer count for everything but create operation */
 417         if (!(Irp->Flags & IRP_CREATE_OPERATION))
 418         {
 419             if (Irp->Flags & IRP_WRITE_OPERATION)
 420             {
 421                 /* Update write transfer count */
 422                 IopUpdateTransferCount(IopWriteTransfer,
 423                                        (ULONG)Irp->IoStatus.Information);
 424             }
 425             else if (Irp->Flags & IRP_READ_OPERATION)
 426             {
 427                 /* Update read transfer count */
 428                 IopUpdateTransferCount(IopReadTransfer,
 429                                        (ULONG)Irp->IoStatus.Information);
 430             }
 431             else
 432             {
 433                 /* Update other transfer count */
 434                 IopUpdateTransferCount(IopOtherTransfer,
 435                                        (ULONG)Irp->IoStatus.Information);
 436             }
 437         }
 438
 439         /* Now that we've signaled the events, de-associate the IRP */
 440         IopUnQueueIrpFromThread(Irp);
 441
 442         /* Now check if a User APC Routine was requested */
 443         if (Irp->Overlay.AsynchronousParameters.UserApcRoutine)
 444         {
 445             /* Initialize it */
 446             KeInitializeApc(&Irp->Tail.Apc,
 447                             KeGetCurrentThread(),
 448                             CurrentApcEnvironment,
 449                             IopFreeIrpKernelApc,
 450                             IopAbortIrpKernelApc,
 451                             (PKNORMAL_ROUTINE)Irp->
 452                             Overlay.AsynchronousParameters.UserApcRoutine,
 453                             Irp->RequestorMode,
 454                             Irp->
 455                             Overlay.AsynchronousParameters.UserApcContext);
 456
 457             /* Queue it */
 458             KeInsertQueueApc(&Irp->Tail.Apc, Irp->UserIosb, NULL, 2);
 459         }
 460         else if ((Port) &&
 461                  (Irp->Overlay.AsynchronousParameters.UserApcContext))
 462         {
 463             /* We have an I/O Completion setup... create the special Overlay */
 464             Irp->Tail.CompletionKey = Key;
 465             Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp;
 466             KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry);
 467         }
 468         else
 469         {
 470             /* Free the IRP since we don't need it anymore */
 471             IoFreeIrp(Irp);
 472         }
 473
 474         /* Check if we have a file object that wasn't part of a create */
 475         if ((FileObject) && !(SignaledCreateRequest))
 476         {
 477             /* Dereference it, since it's not needed anymore either */
 478             ObDereferenceObjectDeferDelete(FileObject);
 479         }
 480     }
 481     else
 482     {
 483         /*
 484          * Either we didn't return from the request, or we did return but this
 485          * request was synchronous.
 486          */
 487         if ((Irp->PendingReturned) && (FileObject))
 488         {
 489             /* So we did return with a synch operation, was it the IRP? */
 490             if (Irp->Flags & IRP_SYNCHRONOUS_API)
 491             {
 492                 /* Yes, this IRP was synchronous, so return the I/O Status */
 493                 *Irp->UserIosb = Irp->IoStatus;
 494
 495                 /* Now check if the user gave an event */
 496                 if (Irp->UserEvent)
 497                 {
 498                     /* Signal it */
 499                     KeSetEvent(Irp->UserEvent, 0, FALSE);
 500                 }
 501                 else
 502                 {
 503                     /* No event was given, so signal the FO instead */
 504                     KeSetEvent(&FileObject->Event, 0, FALSE);
 505                 }
 506             }
 507             else
 508             {
 509                 /*
 510                  * It's not the IRP that was synchronous, it was the FO
 511                  * that was opened this way. Signal its event.
 512                  */
 513                 FileObject->FinalStatus = Irp->IoStatus.Status;
 514                 KeSetEvent(&FileObject->Event, 0, FALSE);
 515             }
 516         }
 517
 518         /* Now that we got here, we do this for incomplete I/Os as well */
 519         if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 520         {
 521             /* Dereference the File Object unless this was a create */
 522             ObDereferenceObjectDeferDelete(FileObject);
 523         }
 524
 525         /*
 526          * Check if this was an Executive Event (remember that we know this
 527          * by checking if the IRP is synchronous)
 528          */
 529         if ((Irp->UserEvent) &&
 530             (FileObject) &&
 531             !(Irp->Flags & IRP_SYNCHRONOUS_API))
 532         {
 533             /* This isn't a PKEVENT, so dereference it */
 534             ObDereferenceObject(Irp->UserEvent);
 535         }
 536
 537         /* Now that we've signaled the events, de-associate the IRP */
 538         IopUnQueueIrpFromThread(Irp);
 539
 540         /* Free the IRP as well */
 541         IoFreeIrp(Irp);
 542     }
 543 }
 544
 545 /* FUNCTIONS *****************************************************************/
 546
 547 /*
 548  * @implemented
 549  */
 550 PIRP
 551 NTAPI
 552 IoAllocateIrp(IN CCHAR StackSize,
 553               IN BOOLEAN ChargeQuota)
 554 {
 555     PIRP Irp = NULL;
 556     USHORT Size = IoSizeOfIrp(StackSize);
 557     PKPRCB Prcb;
 558     UCHAR Flags = 0;
 559     PNPAGED_LOOKASIDE_LIST List = NULL;
 560     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
 561
 562     /* Set Charge Quota Flag */
 563     if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
 564
 565     /* Get the PRCB */
 566     Prcb = KeGetCurrentPrcb();
 567
 568     /* Figure out which Lookaside List to use */
 569     if ((StackSize <= 8) && (ChargeQuota == FALSE || Prcb->LookasideIrpFloat > 0))
 570     {
 571         /* Set Fixed Size Flag */
 572         Flags |= IRP_ALLOCATED_FIXED_SIZE;
 573
 574         /* See if we should use big list */
 575         if (StackSize != 1)
 576         {
 577             Size = IoSizeOfIrp(8);
 578             ListType = LookasideLargeIrpList;
 579         }
 580
 581         /* Get the P List First */
 582         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
 583
 584         /* Attempt allocation */
 585         List->L.TotalAllocates++;
 586         Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 587
 588         /* Check if the P List failed */
 589         if (!Irp)
 590         {
 591             /* Let the balancer know */
 592             List->L.AllocateMisses++;
 593
 594             /* Try the L List */
 595             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
 596             List->L.TotalAllocates++;
 597             Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 598         }
 599     }
 600
 601     /* Check if we have to use the pool */
 602     if (!Irp)
 603     {
 604         /* Did we try lookaside and fail? */
 605         if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
 606
 607         /* Check if we should charge quota */
 608         if (ChargeQuota)
 609         {
 610             Irp = ExAllocatePoolWithQuotaTag(NonPagedPool | POOL_QUOTA_FAIL_INSTEAD_OF_RAISE,
 611                                              Size,
 612                                              TAG_IRP);
 613         }
 614         else
 615         {
 616             /* Allocate the IRP with no quota charge */
 617             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 618         }
 619
 620         /* Make sure it was sucessful */
 621         if (!Irp) return NULL;
 622     }
 623     else if (Flags & IRP_QUOTA_CHARGED)
 624     {
 625         /* Decrement lookaside float */
 626         InterlockedDecrement(&Prcb->LookasideIrpFloat);
 627         Flags |= IRP_LOOKASIDE_ALLOCATION;
 628
 629         /* In this case there is no charge quota */
 630         Flags &= ~IRP_QUOTA_CHARGED;
 631     }
 632
 633     /* Now Initialize it */
 634     IoInitializeIrp(Irp, Size, StackSize);
 635
 636     /* Set the Allocation Flags */
 637     Irp->AllocationFlags = Flags;
 638
 639     /* Return it */
 640     IOTRACE(IO_IRP_DEBUG,
 641             "%s - Allocated IRP %p with allocation flags %lx\n",
 642             __FUNCTION__,
 643             Irp,
 644             Flags);
 645     return Irp;
 646 }
 647
 648 /*
 649  * @implemented
 650  */
 651 PIRP
 652 NTAPI
 653 IopAllocateIrpMustSucceed(IN CCHAR StackSize)
 654 {
 655     LONG i;
 656     PIRP Irp;
 657     LARGE_INTEGER Sleep;
 658
 659     /* Try to get an IRP */
 660     Irp = IoAllocateIrp(StackSize, FALSE);
 661     if (Irp)
 662         return Irp;
 663
 664     /* If we fail, start looping till we may get one */
 665     i = LONG_MAX;
 666     do {
 667         i--;
 668
 669         /* First, sleep for 10ms */
 670         Sleep.QuadPart = -10 * 1000 * 10;
 671         KeDelayExecutionThread(KernelMode, FALSE, &Sleep);
 672
 673         /* Then, retry allocation */
 674         Irp = IoAllocateIrp(StackSize, FALSE);
 675         if (Irp)
 676             return Irp;
 677     } while (i > 0);
 678
 679     return Irp;
 680 }
 681
 682 /*
 683  * @implemented
 684  */
 685 PIRP
 686 NTAPI
 687 IoBuildAsynchronousFsdRequest(IN ULONG MajorFunction,
 688                               IN PDEVICE_OBJECT DeviceObject,
 689                               IN PVOID Buffer,
 690                               IN ULONG Length,
 691                               IN PLARGE_INTEGER StartingOffset,
 692                               IN PIO_STATUS_BLOCK IoStatusBlock)
 693 {
 694     PIRP Irp;
 695     PIO_STACK_LOCATION StackPtr;
 696
 697     /* Allocate IRP */
 698     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 699     if (!Irp) return NULL;
 700
 701     /* Get the Stack */
 702     StackPtr = IoGetNextIrpStackLocation(Irp);
 703
 704     /* Write the Major function and then deal with it */
 705     StackPtr->MajorFunction = (UCHAR)MajorFunction;
 706
 707     /* Do not handle the following here */
 708     if ((MajorFunction != IRP_MJ_FLUSH_BUFFERS) &&
 709         (MajorFunction != IRP_MJ_SHUTDOWN) &&
 710         (MajorFunction != IRP_MJ_PNP) &&
 711         (MajorFunction != IRP_MJ_POWER))
 712     {
 713         /* Check if this is Buffered IO */
 714         if (DeviceObject->Flags & DO_BUFFERED_IO)
 715         {
 716             /* Allocate the System Buffer */
 717             Irp->AssociatedIrp.SystemBuffer =
 718                 ExAllocatePoolWithTag(NonPagedPool, Length, TAG_SYS_BUF);
 719             if (!Irp->AssociatedIrp.SystemBuffer)
 720             {
 721                 /* Free the IRP and fail */
 722                 IoFreeIrp(Irp);
 723                 return NULL;
 724             }
 725
 726             /* Set flags */
 727             Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 728
 729             /* Handle special IRP_MJ_WRITE Case */
 730             if (MajorFunction == IRP_MJ_WRITE)
 731             {
 732                 /* Copy the buffer data */
 733                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, Buffer, Length);
 734             }
 735             else
 736             {
 737                 /* Set the Input Operation flag and set this as a User Buffer */
 738                 Irp->Flags |= IRP_INPUT_OPERATION;
 739                 Irp->UserBuffer = Buffer;
 740             }
 741         }
 742         else if (DeviceObject->Flags & DO_DIRECT_IO)
 743         {
 744             /* Use an MDL for Direct I/O */
 745             Irp->MdlAddress = IoAllocateMdl(Buffer,
 746                                             Length,
 747                                             FALSE,
 748                                             FALSE,
 749                                             NULL);
 750             if (!Irp->MdlAddress)
 751             {
 752                 /* Free the IRP and fail */
 753                 IoFreeIrp(Irp);
 754                 return NULL;
 755             }
 756
 757                         /* Probe and Lock */
 758                         _SEH2_TRY
 759                         {
 760                                 /* Do the probe */
 761                                 MmProbeAndLockPages(Irp->MdlAddress,
 762                                                                         KernelMode,
 763                                                                         MajorFunction == IRP_MJ_READ ?
 764                                                                         IoWriteAccess : IoReadAccess);
 765                         }
 766                         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 767                         {
 768                                 /* Free the IRP and its MDL */
 769                                 IoFreeMdl(Irp->MdlAddress);
 770                                 IoFreeIrp(Irp);
 771
 772                 /* Fail */
 773                                 _SEH2_YIELD(return NULL);
 774                         }
 775                         _SEH2_END;
 776         }
 777         else
 778         {
 779             /* Neither, use the buffer */
 780             Irp->UserBuffer = Buffer;
 781         }
 782
 783         /* Check if this is a read */
 784         if (MajorFunction == IRP_MJ_READ)
 785         {
 786             /* Set the parameters for a read */
 787             StackPtr->Parameters.Read.Length = Length;
 788             StackPtr->Parameters.Read.ByteOffset = *StartingOffset;
 789         }
 790         else if (MajorFunction == IRP_MJ_WRITE)
 791         {
 792             /* Otherwise, set write parameters */
 793             StackPtr->Parameters.Write.Length = Length;
 794             StackPtr->Parameters.Write.ByteOffset = *StartingOffset;
 795         }
 796     }
 797
 798     /* Set the Current Thread and IOSB */
 799     Irp->UserIosb = IoStatusBlock;
 800     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 801
 802     /* Return the IRP */
 803     IOTRACE(IO_IRP_DEBUG,
 804             "%s - Built IRP %p with Major, Buffer, DO %lx %p %p\n",
 805             __FUNCTION__,
 806             Irp,
 807             MajorFunction,
 808             Buffer,
 809             DeviceObject);
 810     return Irp;
 811 }
 812
 813 /*
 814  * @implemented
 815  */
 816 PIRP
 817 NTAPI
 818 IoBuildDeviceIoControlRequest(IN ULONG IoControlCode,
 819                               IN PDEVICE_OBJECT DeviceObject,
 820                               IN PVOID InputBuffer,
 821                               IN ULONG InputBufferLength,
 822                               IN PVOID OutputBuffer,
 823                               IN ULONG OutputBufferLength,
 824                               IN BOOLEAN InternalDeviceIoControl,
 825                               IN PKEVENT Event,
 826                               IN PIO_STATUS_BLOCK IoStatusBlock)
 827 {
 828     PIRP Irp;
 829     PIO_STACK_LOCATION StackPtr;
 830     ULONG BufferLength;
 831
 832     /* Allocate IRP */
 833     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 834     if (!Irp) return NULL;
 835
 836     /* Get the Stack */
 837     StackPtr = IoGetNextIrpStackLocation(Irp);
 838
 839     /* Set the DevCtl Type */
 840     StackPtr->MajorFunction = InternalDeviceIoControl ?
 841                               IRP_MJ_INTERNAL_DEVICE_CONTROL :
 842                               IRP_MJ_DEVICE_CONTROL;
 843
 844     /* Set the IOCTL Data */
 845     StackPtr->Parameters.DeviceIoControl.IoControlCode = IoControlCode;
 846     StackPtr->Parameters.DeviceIoControl.InputBufferLength = InputBufferLength;
 847     StackPtr->Parameters.DeviceIoControl.OutputBufferLength =
 848         OutputBufferLength;
 849
 850     /* Handle the Methods */
 851     switch (IO_METHOD_FROM_CTL_CODE(IoControlCode))
 852     {
 853         /* Buffered I/O */
 854         case METHOD_BUFFERED:
 855
 856             /* Select the right Buffer Length */
 857             BufferLength = InputBufferLength > OutputBufferLength ?
 858                            InputBufferLength : OutputBufferLength;
 859
 860             /* Make sure there is one */
 861             if (BufferLength)
 862             {
 863                 /* Allocate the System Buffer */
 864                 Irp->AssociatedIrp.SystemBuffer =
 865                     ExAllocatePoolWithTag(NonPagedPool,
 866                                           BufferLength,
 867                                           TAG_SYS_BUF);
 868                 if (!Irp->AssociatedIrp.SystemBuffer)
 869                 {
 870                     /* Free the IRP and fail */
 871                     IoFreeIrp(Irp);
 872                     return NULL;
 873                 }
 874
 875                 /* Check if we got a buffer */
 876                 if (InputBuffer)
 877                 {
 878                     /* Copy into the System Buffer */
 879                     RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 880                                   InputBuffer,
 881                                   InputBufferLength);
 882                 }
 883
 884                 /* Write the flags */
 885                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 886                 if (OutputBuffer) Irp->Flags |= IRP_INPUT_OPERATION;
 887
 888                 /* Save the Buffer */
 889                 Irp->UserBuffer = OutputBuffer;
 890             }
 891             else
 892             {
 893                 /* Clear the Flags and Buffer */
 894                 Irp->Flags = 0;
 895                 Irp->UserBuffer = NULL;
 896             }
 897             break;
 898
 899         /* Direct I/O */
 900         case METHOD_IN_DIRECT:
 901         case METHOD_OUT_DIRECT:
 902
 903             /* Check if we got an input buffer */
 904             if (InputBuffer)
 905             {
 906                 /* Allocate the System Buffer */
 907                 Irp->AssociatedIrp.SystemBuffer =
 908                     ExAllocatePoolWithTag(NonPagedPool,
 909                                           InputBufferLength,
 910                                           TAG_SYS_BUF);
 911                 if (!Irp->AssociatedIrp.SystemBuffer)
 912                 {
 913                     /* Free the IRP and fail */
 914                     IoFreeIrp(Irp);
 915                     return NULL;
 916                 }
 917
 918                 /* Copy into the System Buffer */
 919                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 920                               InputBuffer,
 921                               InputBufferLength);
 922
 923                 /* Write the flags */
 924                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 925             }
 926             else
 927             {
 928                 /* Clear the flags */
 929                 Irp->Flags = 0;
 930             }
 931
 932             /* Check if we got an output buffer */
 933             if (OutputBuffer)
 934             {
 935                 /* Allocate the System Buffer */
 936                 Irp->MdlAddress = IoAllocateMdl(OutputBuffer,
 937                                                 OutputBufferLength,
 938                                                 FALSE,
 939                                                 FALSE,
 940                                                 Irp);
 941                 if (!Irp->MdlAddress)
 942                 {
 943                     /* Free the IRP and fail */
 944                     IoFreeIrp(Irp);
 945                     return NULL;
 946                 }
 947
 948                 /* Probe and Lock */
 949                 _SEH2_TRY
 950                 {
 951                     /* Do the probe */
 952                     MmProbeAndLockPages(Irp->MdlAddress,
 953                                         KernelMode,
 954                                         IO_METHOD_FROM_CTL_CODE(IoControlCode) ==
 955                                         METHOD_IN_DIRECT ?
 956                                         IoReadAccess : IoWriteAccess);
 957                 }
 958                 _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 959                 {
 960                     /* Free the MDL */
 961                     IoFreeMdl(Irp->MdlAddress);
 962
 963                     /* Free the input buffer and IRP */
 964                     if (InputBuffer) ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 965                     IoFreeIrp(Irp);
 966
 967                     /* Fail */
 968                     _SEH2_YIELD(return NULL);
 969                 }
 970                 _SEH2_END;
 971             }
 972             break;
 973
 974         case METHOD_NEITHER:
 975
 976             /* Just save the Buffer */
 977             Irp->UserBuffer = OutputBuffer;
 978             StackPtr->Parameters.DeviceIoControl.Type3InputBuffer = InputBuffer;
 979     }
 980
 981     /* Now write the Event and IoSB */
 982     Irp->UserIosb = IoStatusBlock;
 983     Irp->UserEvent = Event;
 984
 985     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 986     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 987     IoQueueThreadIrp(Irp);
 988
 989     /* Return the IRP */
 990     IOTRACE(IO_IRP_DEBUG,
 991             "%s - Built IRP %p with IOCTL, Buffers, DO %lx %p %p %p\n",
 992             __FUNCTION__,
 993             Irp,
 994             IoControlCode,
 995             InputBuffer,
 996             OutputBuffer,
 997             DeviceObject);
 998     return Irp;
 999 }
1000
1001 /*
1002  * @implemented
1003  */
1004 PIRP
1005 NTAPI
1006 IoBuildSynchronousFsdRequest(IN ULONG MajorFunction,
1007                              IN PDEVICE_OBJECT DeviceObject,
1008                              IN PVOID Buffer,
1009                              IN ULONG Length,
1010                              IN PLARGE_INTEGER StartingOffset,
1011                              IN PKEVENT Event,
1012                              IN PIO_STATUS_BLOCK IoStatusBlock)
1013 {
1014     PIRP Irp;
1015
1016     /* Do the big work to set up the IRP */
1017     Irp = IoBuildAsynchronousFsdRequest(MajorFunction,
1018                                         DeviceObject,
1019                                         Buffer,
1020                                         Length,
1021                                         StartingOffset,
1022                                         IoStatusBlock );
1023     if (!Irp) return NULL;
1024
1025     /* Set the Event which makes it Syncronous */
1026     Irp->UserEvent = Event;
1027
1028     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
1029     IoQueueThreadIrp(Irp);
1030     return Irp;
1031 }
1032
1033 /*
1034  * @implemented
1035  */
1036 BOOLEAN
1037 NTAPI
1038 IoCancelIrp(IN PIRP Irp)
1039 {
1040     KIRQL OldIrql;
1041     PDRIVER_CANCEL CancelRoutine;
1042     IOTRACE(IO_IRP_DEBUG,
1043             "%s - Canceling IRP %p\n",
1044             __FUNCTION__,
1045             Irp);
1046     ASSERT(Irp->Type == IO_TYPE_IRP);
1047
1048     /* Acquire the cancel lock and cancel the IRP */
1049     IoAcquireCancelSpinLock(&OldIrql);
1050     Irp->Cancel = TRUE;
1051
1052     /* Clear the cancel routine and get the old one */
1053     CancelRoutine = (PVOID)IoSetCancelRoutine(Irp, NULL);
1054     if (CancelRoutine)
1055     {
1056         /* We had a routine, make sure the IRP isn't completed */
1057         if (Irp->CurrentLocation > (Irp->StackCount + 1))
1058         {
1059             /* It is, bugcheck */
1060             KeBugCheckEx(CANCEL_STATE_IN_COMPLETED_IRP,
1061                          (ULONG_PTR)Irp,
1062                          (ULONG_PTR)CancelRoutine,
1063                          0,
1064                          0);
1065         }
1066
1067         /* Set the cancel IRQL And call the routine */
1068         Irp->CancelIrql = OldIrql;
1069         CancelRoutine(IoGetCurrentIrpStackLocation(Irp)->DeviceObject, Irp);
1070         return TRUE;
1071     }
1072
1073     /* Otherwise, release the cancel lock and fail */
1074     IoReleaseCancelSpinLock(OldIrql);
1075     return FALSE;
1076 }
1077
1078 /*
1079  * @implemented
1080  */
1081 VOID
1082 NTAPI
1083 IoCancelThreadIo(IN PETHREAD Thread)
1084 {
1085     KIRQL OldIrql;
1086     ULONG Retries = 3000;
1087     LARGE_INTEGER Interval;
1088     PLIST_ENTRY ListHead, NextEntry;
1089     PIRP Irp;
1090     PAGED_CODE();
1091
1092     /* Windows isn't using given thread, but using current. */
1093     Thread = PsGetCurrentThread();
1094
1095     IOTRACE(IO_IRP_DEBUG,
1096             "%s - Canceling IRPs for Thread %p\n",
1097             __FUNCTION__,
1098             Thread);
1099
1100     /* Raise to APC to protect the IrpList */
1101     KeRaiseIrql(APC_LEVEL, &OldIrql);
1102
1103     /* Start by cancelling all the IRPs in the current thread queue. */
1104     ListHead = &Thread->IrpList;
1105     NextEntry = ListHead->Flink;
1106     while (ListHead != NextEntry)
1107     {
1108         /* Get the IRP */
1109         Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
1110
1111         /* Cancel it */
1112         IoCancelIrp(Irp);
1113
1114         /* Move to the next entry */
1115         NextEntry = NextEntry->Flink;
1116     }
1117
1118      /* Wait 100 milliseconds */
1119     Interval.QuadPart = -1000000;
1120
1121     /* Wait till all the IRPs are completed or cancelled. */
1122     while (!IsListEmpty(&Thread->IrpList))
1123     {
1124         /* Now we can lower */
1125         KeLowerIrql(OldIrql);
1126
1127         /* Wait a short while and then look if all our IRPs were completed. */
1128         KeDelayExecutionThread(KernelMode, FALSE, &Interval);
1129
1130         /*
1131          * Don't stay here forever if some broken driver doesn't complete
1132          * the IRP.
1133          */
1134         if (!(Retries--))
1135         {
1136             /* Print out a message and remove the IRP */
1137             DPRINT1("Broken driver did not complete!\n");
1138             IopDisassociateThreadIrp();
1139         }
1140
1141         /* Raise the IRQL Again */
1142         KeRaiseIrql(APC_LEVEL, &OldIrql);
1143     }
1144
1145     /* We're done, lower the IRQL */
1146     KeLowerIrql(OldIrql);
1147 }
1148
1149 /*
1150  * @implemented
1151  */
1152 #undef IoCallDriver
1153 NTSTATUS
1154 NTAPI
1155 IoCallDriver(IN PDEVICE_OBJECT DeviceObject,
1156              IN PIRP Irp)
1157 {
1158     /* Call fastcall */
1159     return IofCallDriver(DeviceObject, Irp);
1160 }
1161
1162 #define IoCallDriver IofCallDriver
1163
1164 /*
1165  * @implemented
1166  */
1167 #undef IoCompleteRequest
1168 VOID
1169 NTAPI
1170 IoCompleteRequest(IN PIRP Irp,
1171                   IN CCHAR PriorityBoost)
1172 {
1173     /* Call the fastcall */
1174     IofCompleteRequest(Irp, PriorityBoost);
1175 }
1176
1177 #define IoCompleteRequest IofCompleteRequest
1178
1179 /*
1180  * @implemented
1181  */
1182 VOID
1183 NTAPI
1184 IoEnqueueIrp(IN PIRP Irp)
1185 {
1186     /* This is the same as calling IoQueueThreadIrp */
1187     IoQueueThreadIrp(Irp);
1188 }
1189
1190 /*
1191  * @implemented
1192  */
1193 NTSTATUS
1194 FASTCALL
1195 IofCallDriver(IN PDEVICE_OBJECT DeviceObject,
1196               IN PIRP Irp)
1197 {
1198     PDRIVER_OBJECT DriverObject;
1199     PIO_STACK_LOCATION StackPtr;
1200
1201     /* Make sure this is a valid IRP */
1202     ASSERT(Irp->Type == IO_TYPE_IRP);
1203
1204     /* Get the Driver Object */
1205     DriverObject = DeviceObject->DriverObject;
1206
1207     /* Decrease the current location and check if */
1208     Irp->CurrentLocation--;
1209     if (Irp->CurrentLocation <= 0)
1210     {
1211         /* This IRP ran out of stack, bugcheck */
1212         KeBugCheckEx(NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR)Irp, 0, 0, 0);
1213     }
1214
1215     /* Now update the stack location */
1216     StackPtr = IoGetNextIrpStackLocation(Irp);
1217     Irp->Tail.Overlay.CurrentStackLocation = StackPtr;
1218
1219     /* Get the Device Object */
1220     StackPtr->DeviceObject = DeviceObject;
1221
1222     /* Call it */
1223     return DriverObject->MajorFunction[StackPtr->MajorFunction](DeviceObject,
1224                                                                 Irp);
1225 }
1226
1227 FORCEINLINE
1228 VOID
1229 IopClearStackLocation(IN PIO_STACK_LOCATION IoStackLocation)
1230 {
1231     IoStackLocation->MinorFunction = 0;
1232     IoStackLocation->Flags = 0;
1233     IoStackLocation->Control &= SL_ERROR_RETURNED;
1234     IoStackLocation->Parameters.Others.Argument1 = 0;
1235     IoStackLocation->Parameters.Others.Argument2 = 0;
1236     IoStackLocation->Parameters.Others.Argument3 = 0;
1237     IoStackLocation->FileObject = NULL;
1238 }
1239
1240 /*
1241  * @implemented
1242  */
1243 VOID
1244 FASTCALL
1245 IofCompleteRequest(IN PIRP Irp,
1246                    IN CCHAR PriorityBoost)
1247 {
1248     PIO_STACK_LOCATION StackPtr, LastStackPtr;
1249     PDEVICE_OBJECT DeviceObject;
1250     PFILE_OBJECT FileObject;
1251     PETHREAD Thread;
1252     NTSTATUS Status;
1253     PMDL Mdl, NextMdl;
1254     ULONG MasterCount;
1255     PIRP MasterIrp;
1256     ULONG Flags;
1257     NTSTATUS ErrorCode = STATUS_SUCCESS;
1258     PREPARSE_DATA_BUFFER DataBuffer = NULL;
1259     IOTRACE(IO_IRP_DEBUG,
1260             "%s - Completing IRP %p\n",
1261             __FUNCTION__,
1262             Irp);
1263
1264     /* Make sure this IRP isn't getting completed twice or is invalid */
1265     if ((Irp->CurrentLocation) > (Irp->StackCount + 1))
1266     {
1267         /* Bugcheck */
1268         KeBugCheckEx(MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR)Irp, 0, 0, 0);
1269     }
1270
1271     /* Some sanity checks */
1272     ASSERT(Irp->Type == IO_TYPE_IRP);
1273     ASSERT(!Irp->CancelRoutine);
1274     ASSERT(Irp->IoStatus.Status != STATUS_PENDING);
1275     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
1276
1277     /* Get the last stack */
1278     LastStackPtr = (PIO_STACK_LOCATION)(Irp + 1);
1279     if (LastStackPtr->Control & SL_ERROR_RETURNED)
1280     {
1281         /* Get the error code */
1282         ErrorCode = PtrToUlong(LastStackPtr->Parameters.Others.Argument4);
1283     }
1284
1285     /*
1286      * Start the loop with the current stack and point the IRP to the next stack
1287      * and then keep incrementing the stack as we loop through. The IRP should
1288      * always point to the next stack location w.r.t the one currently being
1289      * analyzed, so completion routine code will see the appropriate value.
1290      * Because of this, we must loop until the current stack location is +1 of
1291      * the stack count, because when StackPtr is at the end, CurrentLocation is +1.
1292      */
1293     for (StackPtr = IoGetCurrentIrpStackLocation(Irp),
1294          Irp->CurrentLocation++,
1295          Irp->Tail.Overlay.CurrentStackLocation++;
1296          Irp->CurrentLocation <= (Irp->StackCount + 1);
1297          StackPtr++,
1298          Irp->CurrentLocation++,
1299          Irp->Tail.Overlay.CurrentStackLocation++)
1300     {
1301         /* Set Pending Returned */
1302         Irp->PendingReturned = StackPtr->Control & SL_PENDING_RETURNED;
1303
1304         /* Check if we failed */
1305         if (!NT_SUCCESS(Irp->IoStatus.Status))
1306         {
1307             /* Check if it was changed by a completion routine */
1308             if (Irp->IoStatus.Status != ErrorCode)
1309             {
1310                 /* Update the error for the current stack */
1311                 ErrorCode = Irp->IoStatus.Status;
1312                 StackPtr->Control |= SL_ERROR_RETURNED;
1313                 LastStackPtr->Parameters.Others.Argument4 = UlongToPtr(ErrorCode);
1314                 LastStackPtr->Control |= SL_ERROR_RETURNED;
1315             }
1316         }
1317
1318         /* Check if there is a Completion Routine to Call */
1319         if ((NT_SUCCESS(Irp->IoStatus.Status) &&
1320              (StackPtr->Control & SL_INVOKE_ON_SUCCESS)) ||
1321             (!NT_SUCCESS(Irp->IoStatus.Status) &&
1322              (StackPtr->Control & SL_INVOKE_ON_ERROR)) ||
1323             (Irp->Cancel &&
1324              (StackPtr->Control & SL_INVOKE_ON_CANCEL)))
1325         {
1326             /* Clear the stack location */
1327             IopClearStackLocation(StackPtr);
1328
1329             /* Check for highest-level device completion routines */
1330             if (Irp->CurrentLocation == (Irp->StackCount + 1))
1331             {
1332                 /* Clear the DO, since the current stack location is invalid */
1333                 DeviceObject = NULL;
1334             }
1335             else
1336             {
1337                 /* Otherwise, return the real one */
1338                 DeviceObject = IoGetCurrentIrpStackLocation(Irp)->DeviceObject;
1339             }
1340
1341             /* Call the completion routine */
1342             Status = StackPtr->CompletionRoutine(DeviceObject,
1343                                                  Irp,
1344                                                  StackPtr->Context);
1345
1346             /* Don't touch the Packet in this case, since it might be gone! */
1347             if (Status == STATUS_MORE_PROCESSING_REQUIRED) return;
1348         }
1349         else
1350         {
1351             /* Otherwise, check if this is a completed IRP */
1352             if ((Irp->CurrentLocation <= Irp->StackCount) &&
1353                 (Irp->PendingReturned))
1354             {
1355                 /* Mark it as pending */
1356                 IoMarkIrpPending(Irp);
1357             }
1358
1359             /* Clear the stack location */
1360             IopClearStackLocation(StackPtr);
1361         }
1362     }
1363
1364     /* Check if the IRP is an associated IRP */
1365     if (Irp->Flags & IRP_ASSOCIATED_IRP)
1366     {
1367         /* Get the master IRP and count */
1368         MasterIrp = Irp->AssociatedIrp.MasterIrp;
1369         MasterCount = InterlockedDecrement(&MasterIrp->AssociatedIrp.IrpCount);
1370
1371         /* Free the MDLs */
1372         for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
1373         {
1374             /* Go to the next one */
1375             NextMdl = Mdl->Next;
1376             IoFreeMdl(Mdl);
1377         }
1378
1379         /* Free the IRP itself */
1380         IoFreeIrp(Irp);
1381
1382         /* Complete the Master IRP */
1383         if (!MasterCount) IofCompleteRequest(MasterIrp, PriorityBoost);
1384         return;
1385     }
1386
1387     /* Check whether we have to reparse */
1388     if (Irp->IoStatus.Status == STATUS_REPARSE)
1389     {
1390         if (Irp->IoStatus.Information > IO_REMOUNT)
1391         {
1392             /* If that's a reparse tag we understand, save the buffer from deletion */
1393             if (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT)
1394             {
1395                 ASSERT(Irp->Tail.Overlay.AuxiliaryBuffer != NULL);
1396                 DataBuffer = (PREPARSE_DATA_BUFFER)Irp->Tail.Overlay.AuxiliaryBuffer;
1397                 Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1398             }
1399             else
1400             {
1401                 Irp->IoStatus.Status = STATUS_IO_REPARSE_TAG_NOT_HANDLED;
1402             }
1403         }
1404     }
1405
1406     /* Check if we have an auxiliary buffer */
1407     if (Irp->Tail.Overlay.AuxiliaryBuffer)
1408     {
1409         /* Free it */
1410         ExFreePool(Irp->Tail.Overlay.AuxiliaryBuffer);
1411         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1412     }
1413
1414     /* Check if this is a Paging I/O or Close Operation */
1415     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION))
1416     {
1417         /* Handle a Close Operation or Sync Paging I/O */
1418         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION))
1419         {
1420             /* Set the I/O Status and Signal the Event */
1421             Flags = Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_PAGING_IO);
1422             *Irp->UserIosb = Irp->IoStatus;
1423             KeSetEvent(Irp->UserEvent, PriorityBoost, FALSE);
1424
1425             /* Free the IRP for a Paging I/O Only, Close is handled by us */
1426             if (Flags) IoFreeIrp(Irp);
1427         }
1428         else
1429         {
1430 #if 0
1431             /* Page 166 */
1432             KeInitializeApc(&Irp->Tail.Apc
1433                             &Irp->Tail.Overlay.Thread->Tcb,
1434                             Irp->ApcEnvironment,
1435                             IopCompletePageWrite,
1436                             NULL,
1437                             NULL,
1438                             KernelMode,
1439                             NULL);
1440             KeInsertQueueApc(&Irp->Tail.Apc,
1441                              NULL,
1442                              NULL,
1443                              PriorityBoost);
1444 #else
1445             /* Not implemented yet. */
1446             UNIMPLEMENTED_DBGBREAK("Not supported!\n");
1447 #endif
1448         }
1449
1450         /* Get out of here */
1451         return;
1452     }
1453
1454     /* Unlock MDL Pages, page 167. */
1455     Mdl = Irp->MdlAddress;
1456     while (Mdl)
1457     {
1458         MmUnlockPages(Mdl);
1459         Mdl = Mdl->Next;
1460     }
1461
1462     /* Check if we should exit because of a Deferred I/O (page 168) */
1463     if ((Irp->Flags & IRP_DEFER_IO_COMPLETION) && !(Irp->PendingReturned))
1464     {
1465         /* Restore the saved reparse buffer for the caller */
1466         if (Irp->IoStatus.Status == STATUS_REPARSE &&
1467             Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT)
1468         {
1469             Irp->Tail.Overlay.AuxiliaryBuffer = (PCHAR)DataBuffer;
1470         }
1471
1472         /*
1473          * Return without queuing the completion APC, since the caller will
1474          * take care of doing its own optimized completion at PASSIVE_LEVEL.
1475          */
1476         return;
1477     }
1478
1479     /* Get the thread and file object */
1480     Thread = Irp->Tail.Overlay.Thread;
1481     FileObject = Irp->Tail.Overlay.OriginalFileObject;
1482
1483     /* Make sure the IRP isn't canceled */
1484     if (!Irp->Cancel)
1485     {
1486         /* Initialize the APC */
1487         KeInitializeApc(&Irp->Tail.Apc,
1488                         &Thread->Tcb,
1489                         Irp->ApcEnvironment,
1490                         IopCompleteRequest,
1491                         NULL,
1492                         NULL,
1493                         KernelMode,
1494                         NULL);
1495
1496         /* Queue it */
1497         KeInsertQueueApc(&Irp->Tail.Apc,
1498                          FileObject,
1499                          DataBuffer,
1500                          PriorityBoost);
1501     }
1502     else
1503     {
1504         /* The IRP just got canceled... does a thread still own it? */
1505         if (Thread)
1506         {
1507             /* Yes! There is still hope! Initialize the APC */
1508             KeInitializeApc(&Irp->Tail.Apc,
1509                             &Thread->Tcb,
1510                             Irp->ApcEnvironment,
1511                             IopCompleteRequest,
1512                             NULL,
1513                             NULL,
1514                             KernelMode,
1515                             NULL);
1516
1517             /* Queue it */
1518             KeInsertQueueApc(&Irp->Tail.Apc,
1519                              FileObject,
1520                              DataBuffer,
1521                              PriorityBoost);
1522         }
1523         else
1524         {
1525             /* Nothing left for us to do, kill it */
1526             ASSERT(Irp->Cancel);
1527             IopCleanupIrp(Irp, FileObject);
1528         }
1529     }
1530 }
1531
1532 NTSTATUS
1533 NTAPI
1534 IopSynchronousCompletion(IN PDEVICE_OBJECT DeviceObject,
1535                          IN PIRP Irp,
1536                          IN PVOID Context)
1537 {
1538     if (Irp->PendingReturned)
1539         KeSetEvent((PKEVENT)Context, IO_NO_INCREMENT, FALSE);
1540     return STATUS_MORE_PROCESSING_REQUIRED;
1541 }
1542
1543 /*
1544  * @implemented
1545  */
1546 BOOLEAN
1547 NTAPI
1548 IoForwardIrpSynchronously(IN PDEVICE_OBJECT DeviceObject,
1549                           IN PIRP Irp)
1550 {
1551     KEVENT Event;
1552     NTSTATUS Status;
1553
1554     /* Check if next stack location is available */
1555     if (Irp->CurrentLocation < Irp->StackCount)
1556     {
1557         /* No more stack location */
1558         return FALSE;
1559     }
1560
1561     /* Initialize event */
1562     KeInitializeEvent(&Event, NotificationEvent, FALSE);
1563
1564     /* Copy stack location for next driver */
1565     IoCopyCurrentIrpStackLocationToNext(Irp);
1566
1567     /* Set a completion routine, which will signal the event */
1568     IoSetCompletionRoutine(Irp, IopSynchronousCompletion, &Event, TRUE, TRUE, TRUE);
1569
1570     /* Call next driver */
1571     Status = IoCallDriver(DeviceObject, Irp);
1572
1573     /* Check if irp is pending */
1574     if (Status == STATUS_PENDING)
1575     {
1576         /* Yes, wait for its completion */
1577         KeWaitForSingleObject(&Event, Suspended, KernelMode, FALSE, NULL);
1578     }
1579
1580     /* Return success */
1581     return TRUE;
1582 }
1583
1584 /*
1585  * @implemented
1586  */
1587 VOID
1588 NTAPI
1589 IoFreeIrp(IN PIRP Irp)
1590 {
1591     PNPAGED_LOOKASIDE_LIST List;
1592     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
1593     PKPRCB Prcb;
1594     IOTRACE(IO_IRP_DEBUG,
1595             "%s - Freeing IRPs %p\n",
1596             __FUNCTION__,
1597             Irp);
1598
1599     /* Make sure the Thread IRP list is empty and that it OK to free it */
1600     ASSERT(Irp->Type == IO_TYPE_IRP);
1601     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1602     ASSERT(Irp->CurrentLocation >= Irp->StackCount);
1603
1604     /* Get the PRCB */
1605     Prcb = KeGetCurrentPrcb();
1606
1607     /* If this was a lookaside alloc, increment lookaside float */
1608     if (Irp->AllocationFlags & IRP_LOOKASIDE_ALLOCATION)
1609     {
1610         Irp->AllocationFlags &= ~IRP_LOOKASIDE_ALLOCATION;
1611         InterlockedIncrement(&Prcb->LookasideIrpFloat);
1612     }
1613
1614     /* If this was a pool alloc, free it with the pool */
1615     if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
1616     {
1617         /* Free it */
1618         ExFreePoolWithTag(Irp, TAG_IRP);
1619     }
1620     else
1621     {
1622         /* Check if this was a Big IRP */
1623         if (Irp->StackCount != 1) ListType = LookasideLargeIrpList;
1624
1625         /* Use the P List */
1626         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
1627         List->L.TotalFrees++;
1628
1629         /* Check if the Free was within the Depth or not */
1630         if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1631         {
1632             /* Let the balancer know */
1633             List->L.FreeMisses++;
1634
1635             /* Use the L List */
1636             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
1637             List->L.TotalFrees++;
1638
1639             /* Check if the Free was within the Depth or not */
1640             if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1641             {
1642                 /* All lists failed, use the pool */
1643                 List->L.FreeMisses++;
1644                 ExFreePoolWithTag(Irp, TAG_IRP);
1645                 Irp = NULL;
1646             }
1647         }
1648
1649         /* The free was within the Depth */
1650         if (Irp)
1651         {
1652             InterlockedPushEntrySList(&List->L.ListHead,
1653                                       (PSLIST_ENTRY)Irp);
1654         }
1655     }
1656 }
1657
1658 /*
1659  * @implemented
1660  */
1661 IO_PAGING_PRIORITY
1662 FASTCALL
1663 IoGetPagingIoPriority(IN PIRP Irp)
1664 {
1665     IO_PAGING_PRIORITY Priority;
1666     ULONG Flags;
1667
1668     /* Get the flags */
1669     Flags = Irp->Flags;
1670
1671     /* Check what priority it has */
1672     if (Flags & IRP_CLASS_CACHE_OPERATION)
1673     {
1674         /* High priority */
1675         Priority = IoPagingPriorityHigh;
1676     }
1677     else if (Flags & IRP_PAGING_IO)
1678     {
1679         /* Normal priority */
1680         Priority = IoPagingPriorityNormal;
1681     }
1682     else
1683     {
1684         /* Invalid -- not a paging IRP */
1685         Priority = IoPagingPriorityInvalid;
1686     }
1687
1688     /* Return the priority */
1689     return Priority;
1690 }
1691
1692 /*
1693  * @implemented
1694  */
1695 PEPROCESS
1696 NTAPI
1697 IoGetRequestorProcess(IN PIRP Irp)
1698 {
1699     /* Return the requestor process */
1700     if (Irp->Tail.Overlay.Thread)
1701     {
1702         if (Irp->ApcEnvironment == OriginalApcEnvironment)
1703         {
1704             return Irp->Tail.Overlay.Thread->ThreadsProcess;
1705         }
1706         else if (Irp->ApcEnvironment == AttachedApcEnvironment)
1707         {
1708             return (PEPROCESS)Irp->Tail.Overlay.Thread->Tcb.ApcState.Process;
1709         }
1710     }
1711
1712     return NULL;
1713 }
1714
1715 /*
1716  * @implemented
1717  */
1718 ULONG
1719 NTAPI
1720 IoGetRequestorProcessId(IN PIRP Irp)
1721 {
1722     PEPROCESS Process;
1723
1724     /* Return the requestor process' id */
1725     Process = IoGetRequestorProcess(Irp);
1726     if (Process) return PtrToUlong(Process->UniqueProcessId);
1727
1728     return 0;
1729 }
1730
1731 /*
1732  * @implemented
1733  */
1734 NTSTATUS
1735 NTAPI
1736 IoGetRequestorSessionId(IN PIRP Irp,
1737                         OUT PULONG pSessionId)
1738 {
1739     PEPROCESS Process;
1740
1741     /* Return the session */
1742     if (Irp->Tail.Overlay.Thread)
1743     {
1744         Process = Irp->Tail.Overlay.Thread->ThreadsProcess;
1745         *pSessionId = MmGetSessionId(Process);
1746         return STATUS_SUCCESS;
1747     }
1748
1749     *pSessionId = (ULONG)-1;
1750     return STATUS_UNSUCCESSFUL;
1751 }
1752
1753 /*
1754  * @implemented
1755  */
1756 PIRP
1757 NTAPI
1758 IoGetTopLevelIrp(VOID)
1759 {
1760     /* Return the IRP */
1761     return (PIRP)PsGetCurrentThread()->TopLevelIrp;
1762 }
1763
1764 /*
1765  * @implemented
1766  */
1767 VOID
1768 NTAPI
1769 IoInitializeIrp(IN PIRP Irp,
1770                 IN USHORT PacketSize,
1771                 IN CCHAR StackSize)
1772 {
1773     /* Clear it */
1774     IOTRACE(IO_IRP_DEBUG,
1775             "%s - Initializing IRP %p\n",
1776             __FUNCTION__,
1777             Irp);
1778     RtlZeroMemory(Irp, PacketSize);
1779
1780     /* Set the Header and other data */
1781     Irp->Type = IO_TYPE_IRP;
1782     Irp->Size = PacketSize;
1783     Irp->StackCount = StackSize;
1784     Irp->CurrentLocation = StackSize + 1;
1785     Irp->ApcEnvironment =  KeGetCurrentThread()->ApcStateIndex;
1786     Irp->Tail.Overlay.CurrentStackLocation = (PIO_STACK_LOCATION)(Irp + 1) + StackSize;
1787
1788     /* Initialize the Thread List */
1789     InitializeListHead(&Irp->ThreadListEntry);
1790 }
1791
1792 /*
1793  * @implemented
1794  */
1795 BOOLEAN
1796 NTAPI
1797 IoIsOperationSynchronous(IN PIRP Irp)
1798 {
1799     BOOLEAN SynchIO;
1800     BOOLEAN ForceAsync;
1801
1802     /* If the IRP requests synchronous paging I/O, if the file object was opened
1803        for synchronous I/O, if the IRP_SYNCHRONOUS_API flag is set in the IRP
1804        the operation is synchronous */
1805     SynchIO = (IoGetCurrentIrpStackLocation(Irp)->FileObject->Flags & FO_SYNCHRONOUS_IO) ||
1806               (Irp->Flags & IRP_SYNCHRONOUS_API) || (Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1807
1808     /* If the IRP requests asynchronous paging I/O, the operation is asynchronous,
1809        even if one of the above conditions is true */
1810     ForceAsync = (Irp->Flags & IRP_PAGING_IO) && !(Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1811
1812     /* Check the flags */
1813     if (SynchIO && !ForceAsync)
1814     {
1815         /* Synch API or Paging I/O is OK, as is Sync File I/O */
1816         return TRUE;
1817     }
1818
1819     /* Otherwise, it is an asynchronous operation. */
1820     return FALSE;
1821 }
1822
1823 /*
1824  * @unimplemented
1825  */
1826 BOOLEAN
1827 NTAPI
1828 IoIsValidNameGraftingBuffer(IN PIRP Irp,
1829                             IN PREPARSE_DATA_BUFFER ReparseBuffer)
1830 {
1831     UNIMPLEMENTED;
1832     return FALSE;
1833 }
1834
1835 /*
1836  * @implemented
1837  */
1838 PIRP
1839 NTAPI
1840 IoMakeAssociatedIrp(IN PIRP Irp,
1841                     IN CCHAR StackSize)
1842 {
1843     PIRP AssocIrp;
1844     IOTRACE(IO_IRP_DEBUG,
1845             "%s - Associating IRP %p\n",
1846             __FUNCTION__,
1847             Irp);
1848
1849    /* Allocate the IRP */
1850    AssocIrp = IoAllocateIrp(StackSize, FALSE);
1851    if (!AssocIrp) return NULL;
1852
1853    /* Set the Flags */
1854    AssocIrp->Flags |= IRP_ASSOCIATED_IRP;
1855
1856    /* Set the Thread */
1857    AssocIrp->Tail.Overlay.Thread = Irp->Tail.Overlay.Thread;
1858
1859    /* Associate them */
1860    AssocIrp->AssociatedIrp.MasterIrp = Irp;
1861    return AssocIrp;
1862 }
1863
1864 /*
1865  * @implemented
1866  */
1867 VOID
1868 NTAPI
1869 IoQueueThreadIrp(IN PIRP Irp)
1870 {
1871     IOTRACE(IO_IRP_DEBUG,
1872             "%s - Queueing IRP %p\n",
1873             __FUNCTION__,
1874             Irp);
1875
1876     /* Use our inlined routine */
1877     IopQueueIrpToThread(Irp);
1878 }
1879
1880 /*
1881  * @implemented
1882  * Reference: Chris Cant's "Writing WDM Device Drivers"
1883  */
1884 VOID
1885 NTAPI
1886 IoReuseIrp(IN OUT PIRP Irp,
1887            IN NTSTATUS Status)
1888 {
1889     UCHAR AllocationFlags;
1890     IOTRACE(IO_IRP_DEBUG,
1891             "%s - Reusing IRP %p\n",
1892             __FUNCTION__,
1893             Irp);
1894
1895     /* Make sure it's OK to reuse it */
1896     ASSERT(!Irp->CancelRoutine);
1897     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1898
1899     /* Get the old flags */
1900     AllocationFlags = Irp->AllocationFlags;
1901
1902     /* Reinitialize the IRP */
1903     IoInitializeIrp(Irp, Irp->Size, Irp->StackCount);
1904
1905     /* Duplicate the data */
1906     Irp->IoStatus.Status = Status;
1907     Irp->AllocationFlags = AllocationFlags;
1908 }
1909
1910 /*
1911  * @implemented
1912  */
1913 VOID
1914 NTAPI
1915 IoSetTopLevelIrp(IN PIRP Irp)
1916 {
1917     /* Set the IRP */
1918     PsGetCurrentThread()->TopLevelIrp = (ULONG_PTR)Irp;
1919 }
1920
1921 #if defined (_WIN64)
1922 BOOLEAN
1923 NTAPI
1924 IoIs32bitProcess(
1925     IN PIRP Irp OPTIONAL)
1926 {
1927     UNIMPLEMENTED;
1928     return FALSE;
1929 }
1930 #endif