reactos/ntoskrnl/io/iomgr/irp.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/io/iomgr/irp.c
   5  * PURPOSE:         IRP Handling Functions
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Gunnar Dalsnes
   8  *                  Filip Navara (navaraf@reactos.org)
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 PIRP IopDeadIrp;
  18
  19 /* PRIVATE FUNCTIONS  ********************************************************/
  20
  21 VOID
  22 NTAPI
  23 IopFreeIrpKernelApc(IN PKAPC Apc,
  24                     IN PKNORMAL_ROUTINE *NormalRoutine,
  25                     IN PVOID *NormalContext,
  26                     IN PVOID *SystemArgument1,
  27                     IN PVOID *SystemArgument2)
  28 {
  29     /* Free the IRP */
  30     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  31 }
  32
  33 VOID
  34 NTAPI
  35 IopAbortIrpKernelApc(IN PKAPC Apc)
  36 {
  37     /* Free the IRP */
  38     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  39 }
  40
  41 NTSTATUS
  42 NTAPI
  43 IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
  44                     IN PKEVENT EventObject OPTIONAL,
  45                     IN PVOID Buffer OPTIONAL)
  46 {
  47     PAGED_CODE();
  48
  49     /* Dereference the event */
  50     if (EventObject) ObDereferenceObject(EventObject);
  51
  52     /* Free a buffer, if any */
  53     if (Buffer) ExFreePool(Buffer);
  54
  55     /* If this was a file opened for synch I/O, then unlock it */
  56     if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
  57
  58     /* Now dereference it and return */
  59     ObDereferenceObject(FileObject);
  60     return STATUS_INSUFFICIENT_RESOURCES;
  61 }
  62
  63 VOID
  64 NTAPI
  65 IopAbortInterruptedIrp(IN PKEVENT EventObject,
  66                        IN PIRP Irp)
  67 {
  68     KIRQL OldIrql;
  69     BOOLEAN CancelResult;
  70     LARGE_INTEGER Wait;
  71     PAGED_CODE();
  72
  73     /* Raise IRQL to APC */
  74     KeRaiseIrql(APC_LEVEL, &OldIrql);
  75
  76     /* Check if nobody completed it yet */
  77     if (!KeReadStateEvent(EventObject))
  78     {
  79         /* First, cancel it */
  80         CancelResult = IoCancelIrp(Irp);
  81         KeLowerIrql(OldIrql);
  82
  83         /* Check if we cancelled it */
  84         if (CancelResult)
  85         {
  86             /* Wait for the IRP to be cancelled */
  87             Wait.QuadPart = -100000;
  88             while (!KeReadStateEvent(EventObject))
  89             {
  90                 /* Delay indefintely */
  91                 KeDelayExecutionThread(KernelMode, FALSE, &Wait);
  92             }
  93         }
  94         else
  95         {
  96             /* No cancellation done, so wait for the I/O system to kill it */
  97             KeWaitForSingleObject(EventObject,
  98                                   Executive,
  99                                   KernelMode,
 100                                   FALSE,
 101                                   NULL);
 102         }
 103     }
 104     else
 105     {
 106         /* We got preempted, so give up */
 107         KeLowerIrql(OldIrql);
 108     }
 109 }
 110
 111 VOID
 112 NTAPI
 113 IopDisassociateThreadIrp(VOID)
 114 {
 115     KIRQL OldIrql, LockIrql;
 116     PETHREAD IrpThread;
 117     PLIST_ENTRY IrpEntry;
 118     PIO_ERROR_LOG_PACKET ErrorLogEntry;
 119     PDEVICE_OBJECT DeviceObject = NULL;
 120     PIO_STACK_LOCATION IoStackLocation;
 121
 122     /* First, raise to APC to protect IrpList */
 123     KeRaiseIrql(APC_LEVEL, &OldIrql);
 124
 125     /* Get the Thread and check the list */
 126     IrpThread = PsGetCurrentThread();
 127     if (IsListEmpty(&IrpThread->IrpList))
 128     {
 129         /* It got completed now, so quit */
 130         KeLowerIrql(OldIrql);
 131         return;
 132     }
 133
 134     /* Ensure no one will come disturb */
 135     LockIrql = KeAcquireQueuedSpinLock(LockQueueIoCompletionLock);
 136
 137     /* Get the misbehaving IRP */
 138     IrpEntry = IrpThread->IrpList.Flink;
 139     IopDeadIrp = CONTAINING_RECORD(IrpEntry, IRP, ThreadListEntry);
 140     IOTRACE(IO_IRP_DEBUG,
 141             "%s - Deassociating IRP %p for %p\n",
 142             __FUNCTION__,
 143             IopDeadIrp,
 144             IrpThread);
 145
 146     /* Don't cancel the IRP if it's already been completed far */
 147     if (IopDeadIrp->CurrentLocation == (IopDeadIrp->StackCount + 2))
 148     {
 149         /* Return */
 150         KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 151         KeLowerIrql(OldIrql);
 152         return;
 153     }
 154
 155     /* Disown the IRP! */
 156     IopDeadIrp->Tail.Overlay.Thread = NULL;
 157     RemoveHeadList(&IrpThread->IrpList);
 158     InitializeListHead(&IopDeadIrp->ThreadListEntry);
 159
 160     /* Get the stack location and check if it's valid */
 161     IoStackLocation = IoGetCurrentIrpStackLocation(IopDeadIrp);
 162     if (IopDeadIrp->CurrentLocation <= IopDeadIrp->StackCount)
 163     {
 164         /* Get the device object */
 165         DeviceObject = IoStackLocation->DeviceObject;
 166     }
 167
 168     KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 169     /* Lower IRQL now, since we have the pointers we need */
 170     KeLowerIrql(OldIrql);
 171
 172     /* Check if we can send an Error Log Entry*/
 173     if (DeviceObject)
 174     {
 175         /* Allocate an entry */
 176         ErrorLogEntry = IoAllocateErrorLogEntry(DeviceObject,
 177                                                 sizeof(IO_ERROR_LOG_PACKET));
 178         if (ErrorLogEntry)
 179         {
 180             /* Write the entry */
 181             ErrorLogEntry->ErrorCode = IO_DRIVER_CANCEL_TIMEOUT;
 182             IoWriteErrorLogEntry(ErrorLogEntry);
 183         }
 184     }
 185 }
 186
 187 VOID
 188 NTAPI
 189 IopCleanupIrp(IN PIRP Irp,
 190               IN PFILE_OBJECT FileObject)
 191 {
 192     PMDL Mdl;
 193     IOTRACE(IO_IRP_DEBUG,
 194             "%s - Cleaning IRP %p for %p\n",
 195             __FUNCTION__,
 196             Irp,
 197             FileObject);
 198
 199     /* Check if there's an MDL */
 200     while ((Mdl = Irp->MdlAddress))
 201     {
 202         /* Clear all of them */
 203         Irp->MdlAddress = Mdl->Next;
 204         IoFreeMdl(Mdl);
 205     }
 206
 207     /* Check if the IRP has system buffer */
 208     if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 209     {
 210         /* Free the buffer */
 211         ExFreePoolWithTag(Irp->AssociatedIrp.SystemBuffer, TAG_SYS_BUF);
 212     }
 213
 214     /* Check if this IRP has a user event, a file object, and is async */
 215     if ((Irp->UserEvent) &&
 216         !(Irp->Flags & IRP_SYNCHRONOUS_API) &&
 217         (FileObject))
 218     {
 219         /* Dereference the User Event */
 220         ObDereferenceObject(Irp->UserEvent);
 221     }
 222
 223     /* Check if we have a file object and this isn't a create operation */
 224     if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 225     {
 226         /* Dereference the file object */
 227         ObDereferenceObject(FileObject);
 228     }
 229
 230     /* Free the IRP */
 231     IoFreeIrp(Irp);
 232 }
 233
 234 VOID
 235 NTAPI
 236 IopCompleteRequest(IN PKAPC Apc,
 237                    IN PKNORMAL_ROUTINE* NormalRoutine,
 238                    IN PVOID* NormalContext,
 239                    IN PVOID* SystemArgument1,
 240                    IN PVOID* SystemArgument2)
 241 {
 242     PFILE_OBJECT FileObject;
 243     PIRP Irp;
 244     PMDL Mdl, NextMdl;
 245     PVOID Port = NULL, Key = NULL;
 246     BOOLEAN SignaledCreateRequest = FALSE;
 247
 248     /* Get data from the APC */
 249     FileObject = (PFILE_OBJECT)*SystemArgument1;
 250     Irp = CONTAINING_RECORD(Apc, IRP, Tail.Apc);
 251     IOTRACE(IO_IRP_DEBUG,
 252             "%s - Completing IRP %p for %p\n",
 253             __FUNCTION__,
 254             Irp,
 255             FileObject);
 256
 257     /* Sanity check */
 258     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
 259
 260     /* Check if we have a file object */
 261     if (*SystemArgument2)
 262     {
 263         /* Check if we're reparsing */
 264         if ((Irp->IoStatus.Status == STATUS_REPARSE) &&
 265             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT))
 266         {
 267             /* We should never get this yet */
 268             UNIMPLEMENTED_DBGBREAK("Reparse support not yet present!\n");
 269             return;
 270         }
 271     }
 272
 273     /* Handle Buffered case first */
 274     if (Irp->Flags & IRP_BUFFERED_IO)
 275     {
 276         /* Check if we have an input buffer and if we succeeded */
 277         if ((Irp->Flags & IRP_INPUT_OPERATION) &&
 278             (Irp->IoStatus.Status != STATUS_VERIFY_REQUIRED) &&
 279             !(NT_ERROR(Irp->IoStatus.Status)))
 280         {
 281             /* Copy the buffer back to the user */
 282             RtlCopyMemory(Irp->UserBuffer,
 283                           Irp->AssociatedIrp.SystemBuffer,
 284                           Irp->IoStatus.Information);
 285         }
 286
 287         /* Also check if we should de-allocate it */
 288         if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 289         {
 290             /* Deallocate it */
 291             ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 292         }
 293     }
 294
 295     /* Now we got rid of these two... */
 296     Irp->Flags &= ~(IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER);
 297
 298     /* Check if there's an MDL */
 299     for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
 300     {
 301         /* Free it */
 302         NextMdl = Mdl->Next;
 303         IoFreeMdl(Mdl);
 304     }
 305
 306     /* No MDLs left */
 307     Irp->MdlAddress = NULL;
 308
 309     /*
 310      * Check if either the request was completed without any errors
 311      * (but warnings are OK!), or if it was completed with an error, but
 312      * did return from a pending I/O Operation and is not synchronous.
 313      */
 314     if (!(NT_ERROR(Irp->IoStatus.Status)) ||
 315          (NT_ERROR(Irp->IoStatus.Status) &&
 316           (Irp->PendingReturned) &&
 317           !(IsIrpSynchronous(Irp, FileObject))))
 318     {
 319         /* Get any information we need from the FO before we kill it */
 320         if ((FileObject) && (FileObject->CompletionContext))
 321         {
 322             /* Save Completion Data */
 323             Port = FileObject->CompletionContext->Port;
 324             Key = FileObject->CompletionContext->Key;
 325         }
 326
 327         /* Check for UserIos */
 328         if (Irp->UserIosb != NULL)
 329         {
 330             /* Use SEH to make sure we don't write somewhere invalid */
 331             _SEH2_TRY
 332             {
 333                 /*  Save the IOSB Information */
 334                 *Irp->UserIosb = Irp->IoStatus;
 335             }
 336             _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 337             {
 338                 /* Ignore any error */
 339             }
 340             _SEH2_END;
 341         }
 342
 343         /* Check if we have an event or a file object */
 344         if (Irp->UserEvent)
 345         {
 346             /* At the very least, this is a PKEVENT, so signal it always */
 347             KeSetEvent(Irp->UserEvent, 0, FALSE);
 348
 349             /* Check if we also have a File Object */
 350             if (FileObject)
 351             {
 352                 /* Check if this is an Asynch API */
 353                 if (!(Irp->Flags & IRP_SYNCHRONOUS_API))
 354                 {
 355                     /* Dereference the event */
 356                     ObDereferenceObject(Irp->UserEvent);
 357                 }
 358
 359                 /*
 360                  * Now, if this is a Synch I/O File Object, then this event is
 361                  * NOT an actual Executive Event, so we won't dereference it,
 362                  * and instead, we will signal the File Object
 363                  */
 364                 if ((FileObject->Flags & FO_SYNCHRONOUS_IO) &&
 365                     !(Irp->Flags & IRP_OB_QUERY_NAME))
 366                 {
 367                     /* Signal the file object and set the status */
 368                     KeSetEvent(&FileObject->Event, 0, FALSE);
 369                     FileObject->FinalStatus = Irp->IoStatus.Status;
 370                 }
 371
 372                 /*
 373                  * This could also be a create operation, in which case we want
 374                  * to make sure there's no APC fired.
 375                  */
 376                 if (Irp->Flags & IRP_CREATE_OPERATION)
 377                 {
 378                     /* Clear the APC Routine and remember this */
 379                     Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 380                     SignaledCreateRequest = TRUE;
 381                 }
 382             }
 383         }
 384         else if (FileObject)
 385         {
 386             /* Signal the file object and set the status */
 387             KeSetEvent(&FileObject->Event, 0, FALSE);
 388             FileObject->FinalStatus = Irp->IoStatus.Status;
 389
 390             /*
 391             * This could also be a create operation, in which case we want
 392             * to make sure there's no APC fired.
 393             */
 394             if (Irp->Flags & IRP_CREATE_OPERATION)
 395             {
 396                 /* Clear the APC Routine and remember this */
 397                 Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 398                 SignaledCreateRequest = TRUE;
 399             }
 400         }
 401
 402         /* Update transfer count for everything but create operation */
 403         if (!(Irp->Flags & IRP_CREATE_OPERATION))
 404         {
 405             if (Irp->Flags & IRP_WRITE_OPERATION)
 406             {
 407                 /* Update write transfer count */
 408                 IopUpdateTransferCount(IopWriteTransfer,
 409                                        (ULONG)Irp->IoStatus.Information);
 410             }
 411             else if (Irp->Flags & IRP_READ_OPERATION)
 412             {
 413                 /* Update read transfer count */
 414                 IopUpdateTransferCount(IopReadTransfer,
 415                                        (ULONG)Irp->IoStatus.Information);
 416             }
 417             else
 418             {
 419                 /* Update other transfer count */
 420                 IopUpdateTransferCount(IopOtherTransfer,
 421                                        (ULONG)Irp->IoStatus.Information);
 422             }
 423         }
 424
 425         /* Now that we've signaled the events, de-associate the IRP */
 426         IopUnQueueIrpFromThread(Irp);
 427
 428         /* Now check if a User APC Routine was requested */
 429         if (Irp->Overlay.AsynchronousParameters.UserApcRoutine)
 430         {
 431             /* Initialize it */
 432             KeInitializeApc(&Irp->Tail.Apc,
 433                             KeGetCurrentThread(),
 434                             CurrentApcEnvironment,
 435                             IopFreeIrpKernelApc,
 436                             IopAbortIrpKernelApc,
 437                             (PKNORMAL_ROUTINE)Irp->
 438                             Overlay.AsynchronousParameters.UserApcRoutine,
 439                             Irp->RequestorMode,
 440                             Irp->
 441                             Overlay.AsynchronousParameters.UserApcContext);
 442
 443             /* Queue it */
 444             KeInsertQueueApc(&Irp->Tail.Apc, Irp->UserIosb, NULL, 2);
 445         }
 446         else if ((Port) &&
 447                  (Irp->Overlay.AsynchronousParameters.UserApcContext))
 448         {
 449             /* We have an I/O Completion setup... create the special Overlay */
 450             Irp->Tail.CompletionKey = Key;
 451             Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp;
 452             KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry);
 453         }
 454         else
 455         {
 456             /* Free the IRP since we don't need it anymore */
 457             IoFreeIrp(Irp);
 458         }
 459
 460         /* Check if we have a file object that wasn't part of a create */
 461         if ((FileObject) && !(SignaledCreateRequest))
 462         {
 463             /* Dereference it, since it's not needed anymore either */
 464             ObDereferenceObjectDeferDelete(FileObject);
 465         }
 466     }
 467     else
 468     {
 469         /*
 470          * Either we didn't return from the request, or we did return but this
 471          * request was synchronous.
 472          */
 473         if ((Irp->PendingReturned) && (FileObject))
 474         {
 475             /* So we did return with a synch operation, was it the IRP? */
 476             if (Irp->Flags & IRP_SYNCHRONOUS_API)
 477             {
 478                 /* Yes, this IRP was synchronous, so return the I/O Status */
 479                 *Irp->UserIosb = Irp->IoStatus;
 480
 481                 /* Now check if the user gave an event */
 482                 if (Irp->UserEvent)
 483                 {
 484                     /* Signal it */
 485                     KeSetEvent(Irp->UserEvent, 0, FALSE);
 486                 }
 487                 else
 488                 {
 489                     /* No event was given, so signal the FO instead */
 490                     KeSetEvent(&FileObject->Event, 0, FALSE);
 491                 }
 492             }
 493             else
 494             {
 495                 /*
 496                  * It's not the IRP that was synchronous, it was the FO
 497                  * that was opened this way. Signal its event.
 498                  */
 499                 FileObject->FinalStatus = Irp->IoStatus.Status;
 500                 KeSetEvent(&FileObject->Event, 0, FALSE);
 501             }
 502         }
 503
 504         /* Now that we got here, we do this for incomplete I/Os as well */
 505         if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 506         {
 507             /* Dereference the File Object unless this was a create */
 508             ObDereferenceObjectDeferDelete(FileObject);
 509         }
 510
 511         /*
 512          * Check if this was an Executive Event (remember that we know this
 513          * by checking if the IRP is synchronous)
 514          */
 515         if ((Irp->UserEvent) &&
 516             (FileObject) &&
 517             !(Irp->Flags & IRP_SYNCHRONOUS_API))
 518         {
 519             /* This isn't a PKEVENT, so dereference it */
 520             ObDereferenceObject(Irp->UserEvent);
 521         }
 522
 523         /* Now that we've signaled the events, de-associate the IRP */
 524         IopUnQueueIrpFromThread(Irp);
 525
 526         /* Free the IRP as well */
 527         IoFreeIrp(Irp);
 528     }
 529 }
 530
 531 /* FUNCTIONS *****************************************************************/
 532
 533 /*
 534  * @implemented
 535  */
 536 PIRP
 537 NTAPI
 538 IoAllocateIrp(IN CCHAR StackSize,
 539               IN BOOLEAN ChargeQuota)
 540 {
 541     PIRP Irp = NULL;
 542     USHORT Size = IoSizeOfIrp(StackSize);
 543     PKPRCB Prcb;
 544     UCHAR Flags = 0;
 545     PNPAGED_LOOKASIDE_LIST List = NULL;
 546     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
 547
 548     /* Set Charge Quota Flag */
 549     if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
 550
 551     /* FIXME: Implement Lookaside Floats */
 552
 553     /* Figure out which Lookaside List to use */
 554     if ((StackSize <= 8) && (ChargeQuota == FALSE))
 555     {
 556         /* Set Fixed Size Flag */
 557         Flags = IRP_ALLOCATED_FIXED_SIZE;
 558
 559         /* See if we should use big list */
 560         if (StackSize != 1)
 561         {
 562             Size = IoSizeOfIrp(8);
 563             ListType = LookasideLargeIrpList;
 564         }
 565
 566         /* Get the PRCB */
 567         Prcb = KeGetCurrentPrcb();
 568
 569         /* Get the P List First */
 570         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
 571
 572         /* Attempt allocation */
 573         List->L.TotalAllocates++;
 574         Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 575
 576         /* Check if the P List failed */
 577         if (!Irp)
 578         {
 579             /* Let the balancer know */
 580             List->L.AllocateMisses++;
 581
 582             /* Try the L List */
 583             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
 584             List->L.TotalAllocates++;
 585             Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 586         }
 587     }
 588
 589     /* Check if we have to use the pool */
 590     if (!Irp)
 591     {
 592         /* Did we try lookaside and fail? */
 593         if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
 594
 595         /* Check if we should charge quota */
 596         if (ChargeQuota)
 597         {
 598             Irp = ExAllocatePoolWithQuotaTag(NonPagedPool | POOL_QUOTA_FAIL_INSTEAD_OF_RAISE,
 599                                              Size,
 600                                              TAG_IRP);
 601         }
 602         else
 603         {
 604             /* Allocate the IRP with no quota charge */
 605             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 606         }
 607
 608         /* Make sure it was sucessful */
 609         if (!Irp) return NULL;
 610     }
 611     else
 612     {
 613         /* In this case there is no charge quota */
 614         Flags &= ~IRP_QUOTA_CHARGED;
 615     }
 616
 617     /* Now Initialize it */
 618     IoInitializeIrp(Irp, Size, StackSize);
 619
 620     /* Set the Allocation Flags */
 621     Irp->AllocationFlags = Flags;
 622
 623     /* Return it */
 624     IOTRACE(IO_IRP_DEBUG,
 625             "%s - Allocated IRP %p with allocation flags %lx\n",
 626             __FUNCTION__,
 627             Irp,
 628             Flags);
 629     return Irp;
 630 }
 631
 632 /*
 633  * @implemented
 634  */
 635 PIRP
 636 NTAPI
 637 IopAllocateIrpMustSucceed(IN CCHAR StackSize)
 638 {
 639     LONG i;
 640     PIRP Irp;
 641     LARGE_INTEGER Sleep;
 642
 643     /* Try to get an IRP */
 644     Irp = IoAllocateIrp(StackSize, FALSE);
 645     if (Irp)
 646         return Irp;
 647
 648     /* If we fail, start looping till we may get one */
 649     i = LONG_MAX;
 650     do {
 651         i--;
 652
 653         /* First, sleep for 10ms */
 654         Sleep.QuadPart = -10 * 1000 * 10;;
 655         KeDelayExecutionThread(KernelMode, FALSE, &Sleep);
 656
 657         /* Then, retry allocation */
 658         Irp = IoAllocateIrp(StackSize, FALSE);
 659         if (Irp)
 660             return Irp;
 661     } while (i > 0);
 662
 663     return Irp;
 664 }
 665
 666 /*
 667  * @implemented
 668  */
 669 PIRP
 670 NTAPI
 671 IoBuildAsynchronousFsdRequest(IN ULONG MajorFunction,
 672                               IN PDEVICE_OBJECT DeviceObject,
 673                               IN PVOID Buffer,
 674                               IN ULONG Length,
 675                               IN PLARGE_INTEGER StartingOffset,
 676                               IN PIO_STATUS_BLOCK IoStatusBlock)
 677 {
 678     PIRP Irp;
 679     PIO_STACK_LOCATION StackPtr;
 680
 681     /* Allocate IRP */
 682     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 683     if (!Irp) return NULL;
 684
 685     /* Get the Stack */
 686     StackPtr = IoGetNextIrpStackLocation(Irp);
 687
 688     /* Write the Major function and then deal with it */
 689     StackPtr->MajorFunction = (UCHAR)MajorFunction;
 690
 691     /* Do not handle the following here */
 692     if ((MajorFunction != IRP_MJ_FLUSH_BUFFERS) &&
 693         (MajorFunction != IRP_MJ_SHUTDOWN) &&
 694         (MajorFunction != IRP_MJ_PNP) &&
 695         (MajorFunction != IRP_MJ_POWER))
 696     {
 697         /* Check if this is Buffered IO */
 698         if (DeviceObject->Flags & DO_BUFFERED_IO)
 699         {
 700             /* Allocate the System Buffer */
 701             Irp->AssociatedIrp.SystemBuffer =
 702                 ExAllocatePoolWithTag(NonPagedPool, Length, TAG_SYS_BUF);
 703             if (!Irp->AssociatedIrp.SystemBuffer)
 704             {
 705                 /* Free the IRP and fail */
 706                 IoFreeIrp(Irp);
 707                 return NULL;
 708             }
 709
 710             /* Set flags */
 711             Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 712
 713             /* Handle special IRP_MJ_WRITE Case */
 714             if (MajorFunction == IRP_MJ_WRITE)
 715             {
 716                 /* Copy the buffer data */
 717                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, Buffer, Length);
 718             }
 719             else
 720             {
 721                 /* Set the Input Operation flag and set this as a User Buffer */
 722                 Irp->Flags |= IRP_INPUT_OPERATION;
 723                 Irp->UserBuffer = Buffer;
 724             }
 725         }
 726         else if (DeviceObject->Flags & DO_DIRECT_IO)
 727         {
 728             /* Use an MDL for Direct I/O */
 729             Irp->MdlAddress = IoAllocateMdl(Buffer,
 730                                             Length,
 731                                             FALSE,
 732                                             FALSE,
 733                                             NULL);
 734             if (!Irp->MdlAddress)
 735             {
 736                 /* Free the IRP and fail */
 737                 IoFreeIrp(Irp);
 738                 return NULL;
 739             }
 740
 741                         /* Probe and Lock */
 742                         _SEH2_TRY
 743                         {
 744                                 /* Do the probe */
 745                                 MmProbeAndLockPages(Irp->MdlAddress,
 746                                                                         KernelMode,
 747                                                                         MajorFunction == IRP_MJ_READ ?
 748                                                                         IoWriteAccess : IoReadAccess);
 749                         }
 750                         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 751                         {
 752                                 /* Free the IRP and its MDL */
 753                                 IoFreeMdl(Irp->MdlAddress);
 754                                 IoFreeIrp(Irp);
 755
 756                 /* Fail */
 757                                 _SEH2_YIELD(return NULL);
 758                         }
 759                         _SEH2_END;
 760         }
 761         else
 762         {
 763             /* Neither, use the buffer */
 764             Irp->UserBuffer = Buffer;
 765         }
 766
 767         /* Check if this is a read */
 768         if (MajorFunction == IRP_MJ_READ)
 769         {
 770             /* Set the parameters for a read */
 771             StackPtr->Parameters.Read.Length = Length;
 772             StackPtr->Parameters.Read.ByteOffset = *StartingOffset;
 773         }
 774         else if (MajorFunction == IRP_MJ_WRITE)
 775         {
 776             /* Otherwise, set write parameters */
 777             StackPtr->Parameters.Write.Length = Length;
 778             StackPtr->Parameters.Write.ByteOffset = *StartingOffset;
 779         }
 780     }
 781
 782     /* Set the Current Thread and IOSB */
 783     Irp->UserIosb = IoStatusBlock;
 784     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 785
 786     /* Return the IRP */
 787     IOTRACE(IO_IRP_DEBUG,
 788             "%s - Built IRP %p with Major, Buffer, DO %lx %p %p\n",
 789             __FUNCTION__,
 790             Irp,
 791             MajorFunction,
 792             Buffer,
 793             DeviceObject);
 794     return Irp;
 795 }
 796
 797 /*
 798  * @implemented
 799  */
 800 PIRP
 801 NTAPI
 802 IoBuildDeviceIoControlRequest(IN ULONG IoControlCode,
 803                               IN PDEVICE_OBJECT DeviceObject,
 804                               IN PVOID InputBuffer,
 805                               IN ULONG InputBufferLength,
 806                               IN PVOID OutputBuffer,
 807                               IN ULONG OutputBufferLength,
 808                               IN BOOLEAN InternalDeviceIoControl,
 809                               IN PKEVENT Event,
 810                               IN PIO_STATUS_BLOCK IoStatusBlock)
 811 {
 812     PIRP Irp;
 813     PIO_STACK_LOCATION StackPtr;
 814     ULONG BufferLength;
 815
 816     /* Allocate IRP */
 817     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 818     if (!Irp) return NULL;
 819
 820     /* Get the Stack */
 821     StackPtr = IoGetNextIrpStackLocation(Irp);
 822
 823     /* Set the DevCtl Type */
 824     StackPtr->MajorFunction = InternalDeviceIoControl ?
 825                               IRP_MJ_INTERNAL_DEVICE_CONTROL :
 826                               IRP_MJ_DEVICE_CONTROL;
 827
 828     /* Set the IOCTL Data */
 829     StackPtr->Parameters.DeviceIoControl.IoControlCode = IoControlCode;
 830     StackPtr->Parameters.DeviceIoControl.InputBufferLength = InputBufferLength;
 831     StackPtr->Parameters.DeviceIoControl.OutputBufferLength =
 832         OutputBufferLength;
 833
 834     /* Handle the Methods */
 835     switch (IO_METHOD_FROM_CTL_CODE(IoControlCode))
 836     {
 837         /* Buffered I/O */
 838         case METHOD_BUFFERED:
 839
 840             /* Select the right Buffer Length */
 841             BufferLength = InputBufferLength > OutputBufferLength ?
 842                            InputBufferLength : OutputBufferLength;
 843
 844             /* Make sure there is one */
 845             if (BufferLength)
 846             {
 847                 /* Allocate the System Buffer */
 848                 Irp->AssociatedIrp.SystemBuffer =
 849                     ExAllocatePoolWithTag(NonPagedPool,
 850                                           BufferLength,
 851                                           TAG_SYS_BUF);
 852                 if (!Irp->AssociatedIrp.SystemBuffer)
 853                 {
 854                     /* Free the IRP and fail */
 855                     IoFreeIrp(Irp);
 856                     return NULL;
 857                 }
 858
 859                 /* Check if we got a buffer */
 860                 if (InputBuffer)
 861                 {
 862                     /* Copy into the System Buffer */
 863                     RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 864                                   InputBuffer,
 865                                   InputBufferLength);
 866                 }
 867
 868                 /* Write the flags */
 869                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 870                 if (OutputBuffer) Irp->Flags |= IRP_INPUT_OPERATION;
 871
 872                 /* Save the Buffer */
 873                 Irp->UserBuffer = OutputBuffer;
 874             }
 875             else
 876             {
 877                 /* Clear the Flags and Buffer */
 878                 Irp->Flags = 0;
 879                 Irp->UserBuffer = NULL;
 880             }
 881             break;
 882
 883         /* Direct I/O */
 884         case METHOD_IN_DIRECT:
 885         case METHOD_OUT_DIRECT:
 886
 887             /* Check if we got an input buffer */
 888             if (InputBuffer)
 889             {
 890                 /* Allocate the System Buffer */
 891                 Irp->AssociatedIrp.SystemBuffer =
 892                     ExAllocatePoolWithTag(NonPagedPool,
 893                                           InputBufferLength,
 894                                           TAG_SYS_BUF);
 895                 if (!Irp->AssociatedIrp.SystemBuffer)
 896                 {
 897                     /* Free the IRP and fail */
 898                     IoFreeIrp(Irp);
 899                     return NULL;
 900                 }
 901
 902                 /* Copy into the System Buffer */
 903                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 904                               InputBuffer,
 905                               InputBufferLength);
 906
 907                 /* Write the flags */
 908                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 909             }
 910             else
 911             {
 912                 /* Clear the flags */
 913                 Irp->Flags = 0;
 914             }
 915
 916             /* Check if we got an output buffer */
 917             if (OutputBuffer)
 918             {
 919                 /* Allocate the System Buffer */
 920                 Irp->MdlAddress = IoAllocateMdl(OutputBuffer,
 921                                                 OutputBufferLength,
 922                                                 FALSE,
 923                                                 FALSE,
 924                                                 Irp);
 925                 if (!Irp->MdlAddress)
 926                 {
 927                     /* Free the IRP and fail */
 928                     IoFreeIrp(Irp);
 929                     return NULL;
 930                 }
 931
 932                 /* Probe and Lock */
 933                 _SEH2_TRY
 934                 {
 935                     /* Do the probe */
 936                     MmProbeAndLockPages(Irp->MdlAddress,
 937                                         KernelMode,
 938                                         IO_METHOD_FROM_CTL_CODE(IoControlCode) ==
 939                                         METHOD_IN_DIRECT ?
 940                                         IoReadAccess : IoWriteAccess);
 941                 }
 942                 _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 943                 {
 944                     /* Free the MDL */
 945                     IoFreeMdl(Irp->MdlAddress);
 946
 947                     /* Free the input buffer and IRP */
 948                     if (InputBuffer) ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 949                     IoFreeIrp(Irp);
 950
 951                     /* Fail */
 952                     _SEH2_YIELD(return NULL);
 953                 }
 954                 _SEH2_END;
 955             }
 956             break;
 957
 958         case METHOD_NEITHER:
 959
 960             /* Just save the Buffer */
 961             Irp->UserBuffer = OutputBuffer;
 962             StackPtr->Parameters.DeviceIoControl.Type3InputBuffer = InputBuffer;
 963     }
 964
 965     /* Now write the Event and IoSB */
 966     Irp->UserIosb = IoStatusBlock;
 967     Irp->UserEvent = Event;
 968
 969     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 970     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 971     IoQueueThreadIrp(Irp);
 972
 973     /* Return the IRP */
 974     IOTRACE(IO_IRP_DEBUG,
 975             "%s - Built IRP %p with IOCTL, Buffers, DO %lx %p %p %p\n",
 976             __FUNCTION__,
 977             Irp,
 978             IoControlCode,
 979             InputBuffer,
 980             OutputBuffer,
 981             DeviceObject);
 982     return Irp;
 983 }
 984
 985 /*
 986  * @implemented
 987  */
 988 PIRP
 989 NTAPI
 990 IoBuildSynchronousFsdRequest(IN ULONG MajorFunction,
 991                              IN PDEVICE_OBJECT DeviceObject,
 992                              IN PVOID Buffer,
 993                              IN ULONG Length,
 994                              IN PLARGE_INTEGER StartingOffset,
 995                              IN PKEVENT Event,
 996                              IN PIO_STATUS_BLOCK IoStatusBlock)
 997 {
 998     PIRP Irp;
 999
1000     /* Do the big work to set up the IRP */
1001     Irp = IoBuildAsynchronousFsdRequest(MajorFunction,
1002                                         DeviceObject,
1003                                         Buffer,
1004                                         Length,
1005                                         StartingOffset,
1006                                         IoStatusBlock );
1007     if (!Irp) return NULL;
1008
1009     /* Set the Event which makes it Syncronous */
1010     Irp->UserEvent = Event;
1011
1012     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
1013     IoQueueThreadIrp(Irp);
1014     return Irp;
1015 }
1016
1017 /*
1018  * @implemented
1019  */
1020 BOOLEAN
1021 NTAPI
1022 IoCancelIrp(IN PIRP Irp)
1023 {
1024     KIRQL OldIrql;
1025     PDRIVER_CANCEL CancelRoutine;
1026     IOTRACE(IO_IRP_DEBUG,
1027             "%s - Canceling IRP %p\n",
1028             __FUNCTION__,
1029             Irp);
1030     ASSERT(Irp->Type == IO_TYPE_IRP);
1031
1032     /* Acquire the cancel lock and cancel the IRP */
1033     IoAcquireCancelSpinLock(&OldIrql);
1034     Irp->Cancel = TRUE;
1035
1036     /* Clear the cancel routine and get the old one */
1037     CancelRoutine = (PVOID)IoSetCancelRoutine(Irp, NULL);
1038     if (CancelRoutine)
1039     {
1040         /* We had a routine, make sure the IRP isn't completed */
1041         if (Irp->CurrentLocation > (Irp->StackCount + 1))
1042         {
1043             /* It is, bugcheck */
1044             KeBugCheckEx(CANCEL_STATE_IN_COMPLETED_IRP,
1045                          (ULONG_PTR)Irp,
1046                          (ULONG_PTR)CancelRoutine,
1047                          0,
1048                          0);
1049         }
1050
1051         /* Set the cancel IRQL And call the routine */
1052         Irp->CancelIrql = OldIrql;
1053         CancelRoutine(IoGetCurrentIrpStackLocation(Irp)->DeviceObject, Irp);
1054         return TRUE;
1055     }
1056
1057     /* Otherwise, release the cancel lock and fail */
1058     IoReleaseCancelSpinLock(OldIrql);
1059     return FALSE;
1060 }
1061
1062 /*
1063  * @implemented
1064  */
1065 VOID
1066 NTAPI
1067 IoCancelThreadIo(IN PETHREAD Thread)
1068 {
1069     KIRQL OldIrql;
1070     ULONG Retries = 3000;
1071     LARGE_INTEGER Interval;
1072     PLIST_ENTRY ListHead, NextEntry;
1073     PIRP Irp;
1074     PAGED_CODE();
1075
1076     /* Windows isn't using given thread, but using current. */
1077     Thread = PsGetCurrentThread();
1078
1079     IOTRACE(IO_IRP_DEBUG,
1080             "%s - Canceling IRPs for Thread %p\n",
1081             __FUNCTION__,
1082             Thread);
1083
1084     /* Raise to APC to protect the IrpList */
1085     KeRaiseIrql(APC_LEVEL, &OldIrql);
1086
1087     /* Start by cancelling all the IRPs in the current thread queue. */
1088     ListHead = &Thread->IrpList;
1089     NextEntry = ListHead->Flink;
1090     while (ListHead != NextEntry)
1091     {
1092         /* Get the IRP */
1093         Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
1094
1095         /* Cancel it */
1096         IoCancelIrp(Irp);
1097
1098         /* Move to the next entry */
1099         NextEntry = NextEntry->Flink;
1100     }
1101
1102      /* Wait 100 milliseconds */
1103     Interval.QuadPart = -1000000;
1104
1105     /* Wait till all the IRPs are completed or cancelled. */
1106     while (!IsListEmpty(&Thread->IrpList))
1107     {
1108         /* Now we can lower */
1109         KeLowerIrql(OldIrql);
1110
1111         /* Wait a short while and then look if all our IRPs were completed. */
1112         KeDelayExecutionThread(KernelMode, FALSE, &Interval);
1113
1114         /*
1115          * Don't stay here forever if some broken driver doesn't complete
1116          * the IRP.
1117          */
1118         if (!(Retries--))
1119         {
1120             /* Print out a message and remove the IRP */
1121             DPRINT1("Broken driver did not complete!\n");
1122             IopDisassociateThreadIrp();
1123         }
1124
1125         /* Raise the IRQL Again */
1126         KeRaiseIrql(APC_LEVEL, &OldIrql);
1127     }
1128
1129     /* We're done, lower the IRQL */
1130     KeLowerIrql(OldIrql);
1131 }
1132
1133 /*
1134  * @implemented
1135  */
1136 #undef IoCallDriver
1137 NTSTATUS
1138 NTAPI
1139 IoCallDriver(IN PDEVICE_OBJECT DeviceObject,
1140              IN PIRP Irp)
1141 {
1142     /* Call fastcall */
1143     return IofCallDriver(DeviceObject, Irp);
1144 }
1145
1146 #define IoCallDriver IofCallDriver
1147
1148 /*
1149  * @implemented
1150  */
1151 #undef IoCompleteRequest
1152 VOID
1153 NTAPI
1154 IoCompleteRequest(IN PIRP Irp,
1155                   IN CCHAR PriorityBoost)
1156 {
1157     /* Call the fastcall */
1158     IofCompleteRequest(Irp, PriorityBoost);
1159 }
1160
1161 #define IoCompleteRequest IofCompleteRequest
1162
1163 /*
1164  * @implemented
1165  */
1166 VOID
1167 NTAPI
1168 IoEnqueueIrp(IN PIRP Irp)
1169 {
1170     /* This is the same as calling IoQueueThreadIrp */
1171     IoQueueThreadIrp(Irp);
1172 }
1173
1174 /*
1175  * @implemented
1176  */
1177 NTSTATUS
1178 FASTCALL
1179 IofCallDriver(IN PDEVICE_OBJECT DeviceObject,
1180               IN PIRP Irp)
1181 {
1182     PDRIVER_OBJECT DriverObject;
1183     PIO_STACK_LOCATION StackPtr;
1184
1185     /* Make sure this is a valid IRP */
1186     ASSERT(Irp->Type == IO_TYPE_IRP);
1187
1188     /* Get the Driver Object */
1189     DriverObject = DeviceObject->DriverObject;
1190
1191     /* Decrease the current location and check if */
1192     Irp->CurrentLocation--;
1193     if (Irp->CurrentLocation <= 0)
1194     {
1195         /* This IRP ran out of stack, bugcheck */
1196         KeBugCheckEx(NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR)Irp, 0, 0, 0);
1197     }
1198
1199     /* Now update the stack location */
1200     StackPtr = IoGetNextIrpStackLocation(Irp);
1201     Irp->Tail.Overlay.CurrentStackLocation = StackPtr;
1202
1203     /* Get the Device Object */
1204     StackPtr->DeviceObject = DeviceObject;
1205
1206     /* Call it */
1207     return DriverObject->MajorFunction[StackPtr->MajorFunction](DeviceObject,
1208                                                                 Irp);
1209 }
1210
1211 FORCEINLINE
1212 VOID
1213 IopClearStackLocation(IN PIO_STACK_LOCATION IoStackLocation)
1214 {
1215     IoStackLocation->MinorFunction = 0;
1216     IoStackLocation->Flags = 0;
1217     IoStackLocation->Control &= SL_ERROR_RETURNED;
1218     IoStackLocation->Parameters.Others.Argument1 = 0;
1219     IoStackLocation->Parameters.Others.Argument2 = 0;
1220     IoStackLocation->Parameters.Others.Argument3 = 0;
1221     IoStackLocation->FileObject = NULL;
1222 }
1223
1224 /*
1225  * @implemented
1226  */
1227 VOID
1228 FASTCALL
1229 IofCompleteRequest(IN PIRP Irp,
1230                    IN CCHAR PriorityBoost)
1231 {
1232     PIO_STACK_LOCATION StackPtr, LastStackPtr;
1233     PDEVICE_OBJECT DeviceObject;
1234     PFILE_OBJECT FileObject;
1235     PETHREAD Thread;
1236     NTSTATUS Status;
1237     PMDL Mdl, NextMdl;
1238     ULONG MasterCount;
1239     PIRP MasterIrp;
1240     ULONG Flags;
1241     NTSTATUS ErrorCode = STATUS_SUCCESS;
1242     IOTRACE(IO_IRP_DEBUG,
1243             "%s - Completing IRP %p\n",
1244             __FUNCTION__,
1245             Irp);
1246
1247     /* Make sure this IRP isn't getting completed twice or is invalid */
1248     if ((Irp->CurrentLocation) > (Irp->StackCount + 1))
1249     {
1250         /* Bugcheck */
1251         KeBugCheckEx(MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR)Irp, 0, 0, 0);
1252     }
1253
1254     /* Some sanity checks */
1255     ASSERT(Irp->Type == IO_TYPE_IRP);
1256     ASSERT(!Irp->CancelRoutine);
1257     ASSERT(Irp->IoStatus.Status != STATUS_PENDING);
1258     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
1259
1260     /* Get the last stack */
1261     LastStackPtr = (PIO_STACK_LOCATION)(Irp + 1);
1262     if (LastStackPtr->Control & SL_ERROR_RETURNED)
1263     {
1264         /* Get the error code */
1265         ErrorCode = PtrToUlong(LastStackPtr->Parameters.Others.Argument4);
1266     }
1267
1268     /*
1269      * Start the loop with the current stack and point the IRP to the next stack
1270      * and then keep incrementing the stack as we loop through. The IRP should
1271      * always point to the next stack location w.r.t the one currently being
1272      * analyzed, so completion routine code will see the appropriate value.
1273      * Because of this, we must loop until the current stack location is +1 of
1274      * the stack count, because when StackPtr is at the end, CurrentLocation is +1.
1275      */
1276     for (StackPtr = IoGetCurrentIrpStackLocation(Irp),
1277          Irp->CurrentLocation++,
1278          Irp->Tail.Overlay.CurrentStackLocation++;
1279          Irp->CurrentLocation <= (Irp->StackCount + 1);
1280          StackPtr++,
1281          Irp->CurrentLocation++,
1282          Irp->Tail.Overlay.CurrentStackLocation++)
1283     {
1284         /* Set Pending Returned */
1285         Irp->PendingReturned = StackPtr->Control & SL_PENDING_RETURNED;
1286
1287         /* Check if we failed */
1288         if (!NT_SUCCESS(Irp->IoStatus.Status))
1289         {
1290             /* Check if it was changed by a completion routine */
1291             if (Irp->IoStatus.Status != ErrorCode)
1292             {
1293                 /* Update the error for the current stack */
1294                 ErrorCode = Irp->IoStatus.Status;
1295                 StackPtr->Control |= SL_ERROR_RETURNED;
1296                 LastStackPtr->Parameters.Others.Argument4 = UlongToPtr(ErrorCode);
1297                 LastStackPtr->Control |= SL_ERROR_RETURNED;
1298             }
1299         }
1300
1301         /* Check if there is a Completion Routine to Call */
1302         if ((NT_SUCCESS(Irp->IoStatus.Status) &&
1303              (StackPtr->Control & SL_INVOKE_ON_SUCCESS)) ||
1304             (!NT_SUCCESS(Irp->IoStatus.Status) &&
1305              (StackPtr->Control & SL_INVOKE_ON_ERROR)) ||
1306             (Irp->Cancel &&
1307              (StackPtr->Control & SL_INVOKE_ON_CANCEL)))
1308         {
1309             /* Clear the stack location */
1310             IopClearStackLocation(StackPtr);
1311
1312             /* Check for highest-level device completion routines */
1313             if (Irp->CurrentLocation == (Irp->StackCount + 1))
1314             {
1315                 /* Clear the DO, since the current stack location is invalid */
1316                 DeviceObject = NULL;
1317             }
1318             else
1319             {
1320                 /* Otherwise, return the real one */
1321                 DeviceObject = IoGetCurrentIrpStackLocation(Irp)->DeviceObject;
1322             }
1323
1324             /* Call the completion routine */
1325             Status = StackPtr->CompletionRoutine(DeviceObject,
1326                                                  Irp,
1327                                                  StackPtr->Context);
1328
1329             /* Don't touch the Packet in this case, since it might be gone! */
1330             if (Status == STATUS_MORE_PROCESSING_REQUIRED) return;
1331         }
1332         else
1333         {
1334             /* Otherwise, check if this is a completed IRP */
1335             if ((Irp->CurrentLocation <= Irp->StackCount) &&
1336                 (Irp->PendingReturned))
1337             {
1338                 /* Mark it as pending */
1339                 IoMarkIrpPending(Irp);
1340             }
1341
1342             /* Clear the stack location */
1343             IopClearStackLocation(StackPtr);
1344         }
1345     }
1346
1347     /* Check if the IRP is an associated IRP */
1348     if (Irp->Flags & IRP_ASSOCIATED_IRP)
1349     {
1350         /* Get the master IRP and count */
1351         MasterIrp = Irp->AssociatedIrp.MasterIrp;
1352         MasterCount = InterlockedDecrement(&MasterIrp->AssociatedIrp.IrpCount);
1353
1354         /* Free the MDLs */
1355         for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
1356         {
1357             /* Go to the next one */
1358             NextMdl = Mdl->Next;
1359             IoFreeMdl(Mdl);
1360         }
1361
1362         /* Free the IRP itself */
1363         IoFreeIrp(Irp);
1364
1365         /* Complete the Master IRP */
1366         if (!MasterCount) IofCompleteRequest(MasterIrp, PriorityBoost);
1367         return;
1368     }
1369
1370     /* We don't support this yet */
1371     ASSERT(Irp->IoStatus.Status != STATUS_REPARSE);
1372
1373     /* Check if we have an auxiliary buffer */
1374     if (Irp->Tail.Overlay.AuxiliaryBuffer)
1375     {
1376         /* Free it */
1377         ExFreePool(Irp->Tail.Overlay.AuxiliaryBuffer);
1378         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1379     }
1380
1381     /* Check if this is a Paging I/O or Close Operation */
1382     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION))
1383     {
1384         /* Handle a Close Operation or Sync Paging I/O */
1385         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION))
1386         {
1387             /* Set the I/O Status and Signal the Event */
1388             Flags = Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_PAGING_IO);
1389             *Irp->UserIosb = Irp->IoStatus;
1390             KeSetEvent(Irp->UserEvent, PriorityBoost, FALSE);
1391
1392             /* Free the IRP for a Paging I/O Only, Close is handled by us */
1393             if (Flags) IoFreeIrp(Irp);
1394         }
1395         else
1396         {
1397 #if 0
1398             /* Page 166 */
1399             KeInitializeApc(&Irp->Tail.Apc
1400                             &Irp->Tail.Overlay.Thread->Tcb,
1401                             Irp->ApcEnvironment,
1402                             IopCompletePageWrite,
1403                             NULL,
1404                             NULL,
1405                             KernelMode,
1406                             NULL);
1407             KeInsertQueueApc(&Irp->Tail.Apc,
1408                              NULL,
1409                              NULL,
1410                              PriorityBoost);
1411 #else
1412             /* Not implemented yet. */
1413             UNIMPLEMENTED_DBGBREAK("Not supported!\n");
1414 #endif
1415         }
1416
1417         /* Get out of here */
1418         return;
1419     }
1420
1421     /* Unlock MDL Pages, page 167. */
1422     Mdl = Irp->MdlAddress;
1423     while (Mdl)
1424     {
1425                 MmUnlockPages(Mdl);
1426         Mdl = Mdl->Next;
1427     }
1428
1429     /* Check if we should exit because of a Deferred I/O (page 168) */
1430     if ((Irp->Flags & IRP_DEFER_IO_COMPLETION) && !(Irp->PendingReturned))
1431     {
1432         /*
1433          * Return without queuing the completion APC, since the caller will
1434          * take care of doing its own optimized completion at PASSIVE_LEVEL.
1435          */
1436         return;
1437     }
1438
1439     /* Get the thread and file object */
1440     Thread = Irp->Tail.Overlay.Thread;
1441     FileObject = Irp->Tail.Overlay.OriginalFileObject;
1442
1443     /* Make sure the IRP isn't canceled */
1444     if (!Irp->Cancel)
1445     {
1446         /* Initialize the APC */
1447         KeInitializeApc(&Irp->Tail.Apc,
1448                         &Thread->Tcb,
1449                         Irp->ApcEnvironment,
1450                         IopCompleteRequest,
1451                         NULL,
1452                         NULL,
1453                         KernelMode,
1454                         NULL);
1455
1456         /* Queue it */
1457         KeInsertQueueApc(&Irp->Tail.Apc,
1458                          FileObject,
1459                          NULL, /* This is used for REPARSE stuff */
1460                          PriorityBoost);
1461     }
1462     else
1463     {
1464         /* The IRP just got canceled... does a thread still own it? */
1465         if (Thread)
1466         {
1467             /* Yes! There is still hope! Initialize the APC */
1468             KeInitializeApc(&Irp->Tail.Apc,
1469                             &Thread->Tcb,
1470                             Irp->ApcEnvironment,
1471                             IopCompleteRequest,
1472                             NULL,
1473                             NULL,
1474                             KernelMode,
1475                             NULL);
1476
1477             /* Queue it */
1478             KeInsertQueueApc(&Irp->Tail.Apc,
1479                              FileObject,
1480                              NULL, /* This is used for REPARSE stuff */
1481                              PriorityBoost);
1482         }
1483         else
1484         {
1485             /* Nothing left for us to do, kill it */
1486             ASSERT(Irp->Cancel);
1487             IopCleanupIrp(Irp, FileObject);
1488         }
1489     }
1490 }
1491
1492 NTSTATUS
1493 NTAPI
1494 IopSynchronousCompletion(IN PDEVICE_OBJECT DeviceObject,
1495                          IN PIRP Irp,
1496                          IN PVOID Context)
1497 {
1498     if (Irp->PendingReturned)
1499         KeSetEvent((PKEVENT)Context, IO_NO_INCREMENT, FALSE);
1500     return STATUS_MORE_PROCESSING_REQUIRED;
1501 }
1502
1503 /*
1504  * @implemented
1505  */
1506 BOOLEAN
1507 NTAPI
1508 IoForwardIrpSynchronously(IN PDEVICE_OBJECT DeviceObject,
1509                           IN PIRP Irp)
1510 {
1511     KEVENT Event;
1512     NTSTATUS Status;
1513
1514     /* Check if next stack location is available */
1515     if (Irp->CurrentLocation < Irp->StackCount)
1516     {
1517         /* No more stack location */
1518         return FALSE;
1519     }
1520
1521     /* Initialize event */
1522     KeInitializeEvent(&Event, NotificationEvent, FALSE);
1523
1524     /* Copy stack location for next driver */
1525     IoCopyCurrentIrpStackLocationToNext(Irp);
1526
1527     /* Set a completion routine, which will signal the event */
1528     IoSetCompletionRoutine(Irp, IopSynchronousCompletion, &Event, TRUE, TRUE, TRUE);
1529
1530     /* Call next driver */
1531     Status = IoCallDriver(DeviceObject, Irp);
1532
1533     /* Check if irp is pending */
1534     if (Status == STATUS_PENDING)
1535     {
1536         /* Yes, wait for its completion */
1537         KeWaitForSingleObject(&Event, Suspended, KernelMode, FALSE, NULL);
1538     }
1539
1540     /* Return success */
1541     return TRUE;
1542 }
1543
1544 /*
1545  * @implemented
1546  */
1547 VOID
1548 NTAPI
1549 IoFreeIrp(IN PIRP Irp)
1550 {
1551     PNPAGED_LOOKASIDE_LIST List;
1552     PP_NPAGED_LOOKASIDE_NUMBER ListType =  LookasideSmallIrpList;
1553     PKPRCB Prcb;
1554     IOTRACE(IO_IRP_DEBUG,
1555             "%s - Freeing IRPs %p\n",
1556             __FUNCTION__,
1557             Irp);
1558
1559     /* Make sure the Thread IRP list is empty and that it OK to free it */
1560     ASSERT(Irp->Type == IO_TYPE_IRP);
1561     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1562     ASSERT(Irp->CurrentLocation >= Irp->StackCount);
1563
1564     /* If this was a pool alloc, free it with the pool */
1565     if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
1566     {
1567         /* Free it */
1568         ExFreePoolWithTag(Irp, TAG_IRP);
1569     }
1570     else
1571     {
1572         /* Check if this was a Big IRP */
1573         if (Irp->StackCount != 1) ListType = LookasideLargeIrpList;
1574
1575         /* Get the PRCB */
1576         Prcb = KeGetCurrentPrcb();
1577
1578         /* Use the P List */
1579         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
1580         List->L.TotalFrees++;
1581
1582         /* Check if the Free was within the Depth or not */
1583         if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1584         {
1585             /* Let the balancer know */
1586             List->L.FreeMisses++;
1587
1588             /* Use the L List */
1589             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
1590             List->L.TotalFrees++;
1591
1592             /* Check if the Free was within the Depth or not */
1593             if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1594             {
1595                 /* All lists failed, use the pool */
1596                 List->L.FreeMisses++;
1597                 ExFreePoolWithTag(Irp, TAG_IRP);
1598                 Irp = NULL;
1599             }
1600         }
1601
1602         /* The free was within the Depth */
1603         if (Irp)
1604         {
1605            InterlockedPushEntrySList(&List->L.ListHead,
1606                                      (PSLIST_ENTRY)Irp);
1607         }
1608     }
1609 }
1610
1611 /*
1612  * @implemented
1613  */
1614 IO_PAGING_PRIORITY
1615 FASTCALL
1616 IoGetPagingIoPriority(IN PIRP Irp)
1617 {
1618     IO_PAGING_PRIORITY Priority;
1619     ULONG Flags;
1620
1621     /* Get the flags */
1622     Flags = Irp->Flags;
1623
1624     /* Check what priority it has */
1625     if (Flags & IRP_CLASS_CACHE_OPERATION)
1626     {
1627         /* High priority */
1628         Priority = IoPagingPriorityHigh;
1629     }
1630     else if (Flags & IRP_PAGING_IO)
1631     {
1632         /* Normal priority */
1633         Priority = IoPagingPriorityNormal;
1634     }
1635     else
1636     {
1637         /* Invalid -- not a paging IRP */
1638         Priority = IoPagingPriorityInvalid;
1639     }
1640
1641     /* Return the priority */
1642     return Priority;
1643 }
1644
1645 /*
1646  * @implemented
1647  */
1648 PEPROCESS
1649 NTAPI
1650 IoGetRequestorProcess(IN PIRP Irp)
1651 {
1652     /* Return the requestor process */
1653     if (Irp->Tail.Overlay.Thread)
1654     {
1655         if (Irp->ApcEnvironment == OriginalApcEnvironment)
1656         {
1657             return Irp->Tail.Overlay.Thread->ThreadsProcess;
1658         }
1659         else if (Irp->ApcEnvironment == AttachedApcEnvironment)
1660         {
1661             return (PEPROCESS)Irp->Tail.Overlay.Thread->Tcb.ApcState.Process;
1662         }
1663     }
1664
1665     return NULL;
1666 }
1667
1668 /*
1669  * @implemented
1670  */
1671 ULONG
1672 NTAPI
1673 IoGetRequestorProcessId(IN PIRP Irp)
1674 {
1675     PEPROCESS Process;
1676
1677     /* Return the requestor process' id */
1678     Process = IoGetRequestorProcess(Irp);
1679     if (Process) return PtrToUlong(Process->UniqueProcessId);
1680
1681     return 0;
1682 }
1683
1684 /*
1685  * @implemented
1686  */
1687 NTSTATUS
1688 NTAPI
1689 IoGetRequestorSessionId(IN PIRP Irp,
1690                         OUT PULONG pSessionId)
1691 {
1692     PEPROCESS Process;
1693
1694     /* Return the session */
1695     if (Irp->Tail.Overlay.Thread)
1696     {
1697         Process = Irp->Tail.Overlay.Thread->ThreadsProcess;
1698         *pSessionId = MmGetSessionId(Process);
1699         return STATUS_SUCCESS;
1700     }
1701
1702     *pSessionId = (ULONG)-1;
1703     return STATUS_UNSUCCESSFUL;
1704 }
1705
1706 /*
1707  * @implemented
1708  */
1709 PIRP
1710 NTAPI
1711 IoGetTopLevelIrp(VOID)
1712 {
1713     /* Return the IRP */
1714     return (PIRP)PsGetCurrentThread()->TopLevelIrp;
1715 }
1716
1717 /*
1718  * @implemented
1719  */
1720 VOID
1721 NTAPI
1722 IoInitializeIrp(IN PIRP Irp,
1723                 IN USHORT PacketSize,
1724                 IN CCHAR StackSize)
1725 {
1726     /* Clear it */
1727     IOTRACE(IO_IRP_DEBUG,
1728             "%s - Initializing IRP %p\n",
1729             __FUNCTION__,
1730             Irp);
1731     RtlZeroMemory(Irp, PacketSize);
1732
1733     /* Set the Header and other data */
1734     Irp->Type = IO_TYPE_IRP;
1735     Irp->Size = PacketSize;
1736     Irp->StackCount = StackSize;
1737     Irp->CurrentLocation = StackSize + 1;
1738     Irp->ApcEnvironment =  KeGetCurrentThread()->ApcStateIndex;
1739     Irp->Tail.Overlay.CurrentStackLocation = (PIO_STACK_LOCATION)(Irp + 1) + StackSize;
1740
1741     /* Initialize the Thread List */
1742     InitializeListHead(&Irp->ThreadListEntry);
1743 }
1744
1745 /*
1746  * @implemented
1747  */
1748 BOOLEAN
1749 NTAPI
1750 IoIsOperationSynchronous(IN PIRP Irp)
1751 {
1752     BOOLEAN SynchIO;
1753     BOOLEAN ForceAsync;
1754
1755     /* If the IRP requests synchronous paging I/O, if the file object was opened
1756        for synchronous I/O, if the IRP_SYNCHRONOUS_API flag is set in the IRP
1757        the operation is synchronous */
1758     SynchIO = (IoGetCurrentIrpStackLocation(Irp)->FileObject->Flags & FO_SYNCHRONOUS_IO) ||
1759               (Irp->Flags & IRP_SYNCHRONOUS_API) || (Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1760
1761     /* If the IRP requests asynchronous paging I/O, the operation is asynchronous,
1762        even if one of the above conditions is true */
1763     ForceAsync = (Irp->Flags & IRP_PAGING_IO) && !(Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1764
1765     /* Check the flags */
1766     if (SynchIO && !ForceAsync)
1767     {
1768         /* Synch API or Paging I/O is OK, as is Sync File I/O */
1769         return TRUE;
1770     }
1771
1772     /* Otherwise, it is an asynchronous operation. */
1773     return FALSE;
1774 }
1775
1776 /*
1777  * @unimplemented
1778  */
1779 BOOLEAN
1780 NTAPI
1781 IoIsValidNameGraftingBuffer(IN PIRP Irp,
1782                             IN PREPARSE_DATA_BUFFER ReparseBuffer)
1783 {
1784     UNIMPLEMENTED;
1785     return FALSE;
1786 }
1787
1788 /*
1789  * @implemented
1790  */
1791 PIRP
1792 NTAPI
1793 IoMakeAssociatedIrp(IN PIRP Irp,
1794                     IN CCHAR StackSize)
1795 {
1796     PIRP AssocIrp;
1797     IOTRACE(IO_IRP_DEBUG,
1798             "%s - Associating IRP %p\n",
1799             __FUNCTION__,
1800             Irp);
1801
1802    /* Allocate the IRP */
1803    AssocIrp = IoAllocateIrp(StackSize, FALSE);
1804    if (!AssocIrp) return NULL;
1805
1806    /* Set the Flags */
1807    AssocIrp->Flags |= IRP_ASSOCIATED_IRP;
1808
1809    /* Set the Thread */
1810    AssocIrp->Tail.Overlay.Thread = Irp->Tail.Overlay.Thread;
1811
1812    /* Associate them */
1813    AssocIrp->AssociatedIrp.MasterIrp = Irp;
1814    return AssocIrp;
1815 }
1816
1817 /*
1818  * @implemented
1819  */
1820 VOID
1821 NTAPI
1822 IoQueueThreadIrp(IN PIRP Irp)
1823 {
1824     IOTRACE(IO_IRP_DEBUG,
1825             "%s - Queueing IRP %p\n",
1826             __FUNCTION__,
1827             Irp);
1828
1829     /* Use our inlined routine */
1830     IopQueueIrpToThread(Irp);
1831 }
1832
1833 /*
1834  * @implemented
1835  * Reference: Chris Cant's "Writing WDM Device Drivers"
1836  */
1837 VOID
1838 NTAPI
1839 IoReuseIrp(IN OUT PIRP Irp,
1840            IN NTSTATUS Status)
1841 {
1842     UCHAR AllocationFlags;
1843     IOTRACE(IO_IRP_DEBUG,
1844             "%s - Reusing IRP %p\n",
1845             __FUNCTION__,
1846             Irp);
1847
1848     /* Make sure it's OK to reuse it */
1849     ASSERT(!Irp->CancelRoutine);
1850     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1851
1852     /* Get the old flags */
1853     AllocationFlags = Irp->AllocationFlags;
1854
1855     /* Reinitialize the IRP */
1856     IoInitializeIrp(Irp, Irp->Size, Irp->StackCount);
1857
1858     /* Duplicate the data */
1859     Irp->IoStatus.Status = Status;
1860     Irp->AllocationFlags = AllocationFlags;
1861 }
1862
1863 /*
1864  * @implemented
1865  */
1866 VOID
1867 NTAPI
1868 IoSetTopLevelIrp(IN PIRP Irp)
1869 {
1870     /* Set the IRP */
1871     PsGetCurrentThread()->TopLevelIrp = (ULONG_PTR)Irp;
1872 }
1873
1874 #if defined (_WIN64)
1875 BOOLEAN
1876 NTAPI
1877 IoIs32bitProcess(
1878     IN PIRP Irp OPTIONAL)
1879 {
1880     UNIMPLEMENTED;
1881     return FALSE;
1882 }
1883 #endif