reactos/ntoskrnl/io/iomgr/irp.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/io/iomgr/irp.c
   5  * PURPOSE:         IRP Handling Functions
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Gunnar Dalsnes
   8  *                  Filip Navara (navaraf@reactos.org)
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 PIRP IopDeadIrp;
  18
  19 /* PRIVATE FUNCTIONS  ********************************************************/
  20
  21 VOID
  22 NTAPI
  23 IopFreeIrpKernelApc(IN PKAPC Apc,
  24                     IN PKNORMAL_ROUTINE *NormalRoutine,
  25                     IN PVOID *NormalContext,
  26                     IN PVOID *SystemArgument1,
  27                     IN PVOID *SystemArgument2)
  28 {
  29     /* Free the IRP */
  30     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  31 }
  32
  33 VOID
  34 NTAPI
  35 IopAbortIrpKernelApc(IN PKAPC Apc)
  36 {
  37     /* Free the IRP */
  38     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  39 }
  40
  41 NTSTATUS
  42 NTAPI
  43 IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
  44                     IN PKEVENT EventObject OPTIONAL,
  45                     IN PVOID Buffer OPTIONAL)
  46 {
  47     PAGED_CODE();
  48
  49     /* Dereference the event */
  50     if (EventObject) ObDereferenceObject(EventObject);
  51
  52     /* Free a buffer, if any */
  53     if (Buffer) ExFreePool(Buffer);
  54
  55     /* If this was a file opened for synch I/O, then unlock it */
  56     if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
  57
  58     /* Now dereference it and return */
  59     ObDereferenceObject(FileObject);
  60     return STATUS_INSUFFICIENT_RESOURCES;
  61 }
  62
  63 VOID
  64 NTAPI
  65 IopAbortInterruptedIrp(IN PKEVENT EventObject,
  66                        IN PIRP Irp)
  67 {
  68     KIRQL OldIrql;
  69     BOOLEAN CancelResult;
  70     LARGE_INTEGER Wait;
  71     PAGED_CODE();
  72
  73     /* Raise IRQL to APC */
  74     KeRaiseIrql(APC_LEVEL, &OldIrql);
  75
  76     /* Check if nobody completed it yet */
  77     if (!KeReadStateEvent(EventObject))
  78     {
  79         /* First, cancel it */
  80         CancelResult = IoCancelIrp(Irp);
  81         KeLowerIrql(OldIrql);
  82
  83         /* Check if we cancelled it */
  84         if (CancelResult)
  85         {
  86             /* Wait for the IRP to be cancelled */
  87             Wait.QuadPart = -100000;
  88             while (!KeReadStateEvent(EventObject))
  89             {
  90                 /* Delay indefintely */
  91                 KeDelayExecutionThread(KernelMode, FALSE, &Wait);
  92             }
  93         }
  94         else
  95         {
  96             /* No cancellation done, so wait for the I/O system to kill it */
  97             KeWaitForSingleObject(EventObject,
  98                                   Executive,
  99                                   KernelMode,
 100                                   FALSE,
 101                                   NULL);
 102         }
 103     }
 104     else
 105     {
 106         /* We got preempted, so give up */
 107         KeLowerIrql(OldIrql);
 108     }
 109 }
 110
 111 VOID
 112 NTAPI
 113 IopDisassociateThreadIrp(VOID)
 114 {
 115     KIRQL OldIrql, LockIrql;
 116     PETHREAD IrpThread;
 117     PLIST_ENTRY IrpEntry;
 118     PIO_ERROR_LOG_PACKET ErrorLogEntry;
 119     PDEVICE_OBJECT DeviceObject = NULL;
 120     PIO_STACK_LOCATION IoStackLocation;
 121
 122     /* First, raise to APC to protect IrpList */
 123     KeRaiseIrql(APC_LEVEL, &OldIrql);
 124
 125     /* Get the Thread and check the list */
 126     IrpThread = PsGetCurrentThread();
 127     if (IsListEmpty(&IrpThread->IrpList))
 128     {
 129         /* It got completed now, so quit */
 130         KeLowerIrql(OldIrql);
 131         return;
 132     }
 133
 134     /* Ensure no one will come disturb */
 135     LockIrql = KeAcquireQueuedSpinLock(LockQueueIoCompletionLock);
 136
 137     /* Get the misbehaving IRP */
 138     IrpEntry = IrpThread->IrpList.Flink;
 139     IopDeadIrp = CONTAINING_RECORD(IrpEntry, IRP, ThreadListEntry);
 140     IOTRACE(IO_IRP_DEBUG,
 141             "%s - Deassociating IRP %p for %p\n",
 142             __FUNCTION__,
 143             IopDeadIrp,
 144             IrpThread);
 145
 146     /* Don't cancel the IRP if it's already been completed far */
 147     if (IopDeadIrp->CurrentLocation == (IopDeadIrp->StackCount + 2))
 148     {
 149         /* Return */
 150         KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 151         KeLowerIrql(OldIrql);
 152         return;
 153     }
 154
 155     /* Disown the IRP! */
 156     IopDeadIrp->Tail.Overlay.Thread = NULL;
 157     RemoveHeadList(&IrpThread->IrpList);
 158     InitializeListHead(&IopDeadIrp->ThreadListEntry);
 159
 160     /* Get the stack location and check if it's valid */
 161     IoStackLocation = IoGetCurrentIrpStackLocation(IopDeadIrp);
 162     if (IopDeadIrp->CurrentLocation <= IopDeadIrp->StackCount)
 163     {
 164         /* Get the device object */
 165         DeviceObject = IoStackLocation->DeviceObject;
 166     }
 167
 168     KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 169     /* Lower IRQL now, since we have the pointers we need */
 170     KeLowerIrql(OldIrql);
 171
 172     /* Check if we can send an Error Log Entry*/
 173     if (DeviceObject)
 174     {
 175         /* Allocate an entry */
 176         ErrorLogEntry = IoAllocateErrorLogEntry(DeviceObject,
 177                                                 sizeof(IO_ERROR_LOG_PACKET));
 178         if (ErrorLogEntry)
 179         {
 180             /* Write the entry */
 181             ErrorLogEntry->ErrorCode = IO_DRIVER_CANCEL_TIMEOUT;
 182             IoWriteErrorLogEntry(ErrorLogEntry);
 183         }
 184     }
 185 }
 186
 187 VOID
 188 NTAPI
 189 IopCleanupIrp(IN PIRP Irp,
 190               IN PFILE_OBJECT FileObject)
 191 {
 192     PMDL Mdl;
 193     IOTRACE(IO_IRP_DEBUG,
 194             "%s - Cleaning IRP %p for %p\n",
 195             __FUNCTION__,
 196             Irp,
 197             FileObject);
 198
 199     /* Check if there's an MDL */
 200     while ((Mdl = Irp->MdlAddress))
 201     {
 202         /* Clear all of them */
 203         Irp->MdlAddress = Mdl->Next;
 204         IoFreeMdl(Mdl);
 205     }
 206
 207     /* Check if the IRP has system buffer */
 208     if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 209     {
 210         /* Free the buffer */
 211         ExFreePoolWithTag(Irp->AssociatedIrp.SystemBuffer, TAG_SYS_BUF);
 212     }
 213
 214     /* Check if this IRP has a user event, a file object, and is async */
 215     if ((Irp->UserEvent) &&
 216         !(Irp->Flags & IRP_SYNCHRONOUS_API) &&
 217         (FileObject))
 218     {
 219         /* Dereference the User Event */
 220         ObDereferenceObject(Irp->UserEvent);
 221     }
 222
 223     /* Check if we have a file object and this isn't a create operation */
 224     if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 225     {
 226         /* Dereference the file object */
 227         ObDereferenceObject(FileObject);
 228     }
 229
 230     /* Free the IRP */
 231     IoFreeIrp(Irp);
 232 }
 233
 234 VOID
 235 NTAPI
 236 IopCompleteRequest(IN PKAPC Apc,
 237                    IN PKNORMAL_ROUTINE* NormalRoutine,
 238                    IN PVOID* NormalContext,
 239                    IN PVOID* SystemArgument1,
 240                    IN PVOID* SystemArgument2)
 241 {
 242     PFILE_OBJECT FileObject;
 243     PIRP Irp;
 244     PMDL Mdl, NextMdl;
 245     PVOID Port = NULL, Key = NULL;
 246     BOOLEAN SignaledCreateRequest = FALSE;
 247
 248     /* Get data from the APC */
 249     FileObject = (PFILE_OBJECT)*SystemArgument1;
 250     Irp = CONTAINING_RECORD(Apc, IRP, Tail.Apc);
 251     IOTRACE(IO_IRP_DEBUG,
 252             "%s - Completing IRP %p for %p\n",
 253             __FUNCTION__,
 254             Irp,
 255             FileObject);
 256
 257     /* Sanity check */
 258     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
 259
 260     /* Check if we have a file object */
 261     if (*SystemArgument2)
 262     {
 263         /* Check if we're reparsing */
 264         if ((Irp->IoStatus.Status == STATUS_REPARSE) &&
 265             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT))
 266         {
 267             /* We should never get this yet */
 268             UNIMPLEMENTED_DBGBREAK("Reparse support not yet present!\n");
 269             return;
 270         }
 271     }
 272
 273     /* Handle Buffered case first */
 274     if (Irp->Flags & IRP_BUFFERED_IO)
 275     {
 276         /* Check if we have an input buffer and if we succeeded */
 277         if ((Irp->Flags & IRP_INPUT_OPERATION) &&
 278             (Irp->IoStatus.Status != STATUS_VERIFY_REQUIRED) &&
 279             !(NT_ERROR(Irp->IoStatus.Status)))
 280         {
 281             /* Copy the buffer back to the user */
 282             RtlCopyMemory(Irp->UserBuffer,
 283                           Irp->AssociatedIrp.SystemBuffer,
 284                           Irp->IoStatus.Information);
 285         }
 286
 287         /* Also check if we should de-allocate it */
 288         if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 289         {
 290             /* Deallocate it */
 291             ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 292         }
 293     }
 294
 295     /* Now we got rid of these two... */
 296     Irp->Flags &= ~(IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER);
 297
 298     /* Check if there's an MDL */
 299     for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
 300     {
 301         /* Free it */
 302         NextMdl = Mdl->Next;
 303         IoFreeMdl(Mdl);
 304     }
 305
 306     /* No MDLs left */
 307     Irp->MdlAddress = NULL;
 308
 309     /*
 310      * Check if either the request was completed without any errors
 311      * (but warnings are OK!), or if it was completed with an error, but
 312      * did return from a pending I/O Operation and is not synchronous.
 313      */
 314     if (!(NT_ERROR(Irp->IoStatus.Status)) ||
 315          (NT_ERROR(Irp->IoStatus.Status) &&
 316           (Irp->PendingReturned) &&
 317           !(IsIrpSynchronous(Irp, FileObject))))
 318     {
 319         /* Get any information we need from the FO before we kill it */
 320         if ((FileObject) && (FileObject->CompletionContext))
 321         {
 322             /* Save Completion Data */
 323             Port = FileObject->CompletionContext->Port;
 324             Key = FileObject->CompletionContext->Key;
 325         }
 326
 327         /* Use SEH to make sure we don't write somewhere invalid */
 328         _SEH2_TRY
 329         {
 330             /*  Save the IOSB Information */
 331             *Irp->UserIosb = Irp->IoStatus;
 332         }
 333         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 334         {
 335             /* Ignore any error */
 336         }
 337         _SEH2_END;
 338
 339         /* Check if we have an event or a file object */
 340         if (Irp->UserEvent)
 341         {
 342             /* At the very least, this is a PKEVENT, so signal it always */
 343             KeSetEvent(Irp->UserEvent, 0, FALSE);
 344
 345             /* Check if we also have a File Object */
 346             if (FileObject)
 347             {
 348                 /* Check if this is an Asynch API */
 349                 if (!(Irp->Flags & IRP_SYNCHRONOUS_API))
 350                 {
 351                     /* Dereference the event */
 352                     ObDereferenceObject(Irp->UserEvent);
 353                 }
 354
 355                 /*
 356                  * Now, if this is a Synch I/O File Object, then this event is
 357                  * NOT an actual Executive Event, so we won't dereference it,
 358                  * and instead, we will signal the File Object
 359                  */
 360                 if ((FileObject->Flags & FO_SYNCHRONOUS_IO) &&
 361                     !(Irp->Flags & IRP_OB_QUERY_NAME))
 362                 {
 363                     /* Signal the file object and set the status */
 364                     KeSetEvent(&FileObject->Event, 0, FALSE);
 365                     FileObject->FinalStatus = Irp->IoStatus.Status;
 366                 }
 367
 368                 /*
 369                  * This could also be a create operation, in which case we want
 370                  * to make sure there's no APC fired.
 371                  */
 372                 if (Irp->Flags & IRP_CREATE_OPERATION)
 373                 {
 374                     /* Clear the APC Routine and remember this */
 375                     Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 376                     SignaledCreateRequest = TRUE;
 377                 }
 378             }
 379         }
 380         else if (FileObject)
 381         {
 382             /* Signal the file object and set the status */
 383             KeSetEvent(&FileObject->Event, 0, FALSE);
 384             FileObject->FinalStatus = Irp->IoStatus.Status;
 385
 386             /*
 387             * This could also be a create operation, in which case we want
 388             * to make sure there's no APC fired.
 389             */
 390             if (Irp->Flags & IRP_CREATE_OPERATION)
 391             {
 392                 /* Clear the APC Routine and remember this */
 393                 Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 394                 SignaledCreateRequest = TRUE;
 395             }
 396         }
 397
 398         /* Update transfer count for everything but create operation */
 399         if (!(Irp->Flags & IRP_CREATE_OPERATION))
 400         {
 401             if (Irp->Flags & IRP_WRITE_OPERATION)
 402             {
 403                 /* Update write transfer count */
 404                 IopUpdateTransferCount(IopWriteTransfer,
 405                                        (ULONG)Irp->IoStatus.Information);
 406             }
 407             else if (Irp->Flags & IRP_READ_OPERATION)
 408             {
 409                 /* Update read transfer count */
 410                 IopUpdateTransferCount(IopReadTransfer,
 411                                        (ULONG)Irp->IoStatus.Information);
 412             }
 413             else
 414             {
 415                 /* Update other transfer count */
 416                 IopUpdateTransferCount(IopOtherTransfer,
 417                                        (ULONG)Irp->IoStatus.Information);
 418             }
 419         }
 420
 421         /* Now that we've signaled the events, de-associate the IRP */
 422         IopUnQueueIrpFromThread(Irp);
 423
 424         /* Now check if a User APC Routine was requested */
 425         if (Irp->Overlay.AsynchronousParameters.UserApcRoutine)
 426         {
 427             /* Initialize it */
 428             KeInitializeApc(&Irp->Tail.Apc,
 429                             KeGetCurrentThread(),
 430                             CurrentApcEnvironment,
 431                             IopFreeIrpKernelApc,
 432                             IopAbortIrpKernelApc,
 433                             (PKNORMAL_ROUTINE)Irp->
 434                             Overlay.AsynchronousParameters.UserApcRoutine,
 435                             Irp->RequestorMode,
 436                             Irp->
 437                             Overlay.AsynchronousParameters.UserApcContext);
 438
 439             /* Queue it */
 440             KeInsertQueueApc(&Irp->Tail.Apc, Irp->UserIosb, NULL, 2);
 441         }
 442         else if ((Port) &&
 443                  (Irp->Overlay.AsynchronousParameters.UserApcContext))
 444         {
 445             /* We have an I/O Completion setup... create the special Overlay */
 446             Irp->Tail.CompletionKey = Key;
 447             Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp;
 448             KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry);
 449         }
 450         else
 451         {
 452             /* Free the IRP since we don't need it anymore */
 453             IoFreeIrp(Irp);
 454         }
 455
 456         /* Check if we have a file object that wasn't part of a create */
 457         if ((FileObject) && !(SignaledCreateRequest))
 458         {
 459             /* Dereference it, since it's not needed anymore either */
 460             ObDereferenceObjectDeferDelete(FileObject);
 461         }
 462     }
 463     else
 464     {
 465         /*
 466          * Either we didn't return from the request, or we did return but this
 467          * request was synchronous.
 468          */
 469         if ((Irp->PendingReturned) && (FileObject))
 470         {
 471             /* So we did return with a synch operation, was it the IRP? */
 472             if (Irp->Flags & IRP_SYNCHRONOUS_API)
 473             {
 474                 /* Yes, this IRP was synchronous, so return the I/O Status */
 475                 *Irp->UserIosb = Irp->IoStatus;
 476
 477                 /* Now check if the user gave an event */
 478                 if (Irp->UserEvent)
 479                 {
 480                     /* Signal it */
 481                     KeSetEvent(Irp->UserEvent, 0, FALSE);
 482                 }
 483                 else
 484                 {
 485                     /* No event was given, so signal the FO instead */
 486                     KeSetEvent(&FileObject->Event, 0, FALSE);
 487                 }
 488             }
 489             else
 490             {
 491                 /*
 492                  * It's not the IRP that was synchronous, it was the FO
 493                  * that was opened this way. Signal its event.
 494                  */
 495                 FileObject->FinalStatus = Irp->IoStatus.Status;
 496                 KeSetEvent(&FileObject->Event, 0, FALSE);
 497             }
 498         }
 499
 500         /* Now that we got here, we do this for incomplete I/Os as well */
 501         if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 502         {
 503             /* Dereference the File Object unless this was a create */
 504             ObDereferenceObjectDeferDelete(FileObject);
 505         }
 506
 507         /*
 508          * Check if this was an Executive Event (remember that we know this
 509          * by checking if the IRP is synchronous)
 510          */
 511         if ((Irp->UserEvent) &&
 512             (FileObject) &&
 513             !(Irp->Flags & IRP_SYNCHRONOUS_API))
 514         {
 515             /* This isn't a PKEVENT, so dereference it */
 516             ObDereferenceObject(Irp->UserEvent);
 517         }
 518
 519         /* Now that we've signaled the events, de-associate the IRP */
 520         IopUnQueueIrpFromThread(Irp);
 521
 522         /* Free the IRP as well */
 523         IoFreeIrp(Irp);
 524     }
 525 }
 526
 527 /* FUNCTIONS *****************************************************************/
 528
 529 /*
 530  * @implemented
 531  */
 532 PIRP
 533 NTAPI
 534 IoAllocateIrp(IN CCHAR StackSize,
 535               IN BOOLEAN ChargeQuota)
 536 {
 537     PIRP Irp = NULL;
 538     USHORT Size = IoSizeOfIrp(StackSize);
 539     PKPRCB Prcb;
 540     UCHAR Flags = 0;
 541     PNPAGED_LOOKASIDE_LIST List = NULL;
 542     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
 543
 544     /* Set Charge Quota Flag */
 545     if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
 546
 547     /* FIXME: Implement Lookaside Floats */
 548
 549     /* Figure out which Lookaside List to use */
 550     if ((StackSize <= 8) && (ChargeQuota == FALSE))
 551     {
 552         /* Set Fixed Size Flag */
 553         Flags = IRP_ALLOCATED_FIXED_SIZE;
 554
 555         /* See if we should use big list */
 556         if (StackSize != 1)
 557         {
 558             Size = IoSizeOfIrp(8);
 559             ListType = LookasideLargeIrpList;
 560         }
 561
 562         /* Get the PRCB */
 563         Prcb = KeGetCurrentPrcb();
 564
 565         /* Get the P List First */
 566         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
 567
 568         /* Attempt allocation */
 569         List->L.TotalAllocates++;
 570         Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 571
 572         /* Check if the P List failed */
 573         if (!Irp)
 574         {
 575             /* Let the balancer know */
 576             List->L.AllocateMisses++;
 577
 578             /* Try the L List */
 579             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
 580             List->L.TotalAllocates++;
 581             Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 582         }
 583     }
 584
 585     /* Check if we have to use the pool */
 586     if (!Irp)
 587     {
 588         /* Did we try lookaside and fail? */
 589         if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
 590
 591         /* Check if we should charge quota */
 592         if (ChargeQuota)
 593         {
 594             /* Irp = ExAllocatePoolWithQuotaTag(NonPagedPool, Size, TAG_IRP); */
 595             /* FIXME */
 596             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 597         }
 598         else
 599         {
 600             /* Allocate the IRP With no Quota charge */
 601             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 602         }
 603
 604         /* Make sure it was sucessful */
 605         if (!Irp) return(NULL);
 606     }
 607     else
 608     {
 609         /* In this case there is no charge quota */
 610         Flags &= ~IRP_QUOTA_CHARGED;
 611     }
 612
 613     /* Now Initialize it */
 614     IoInitializeIrp(Irp, Size, StackSize);
 615
 616     /* Set the Allocation Flags */
 617     Irp->AllocationFlags = Flags;
 618
 619     /* Return it */
 620     IOTRACE(IO_IRP_DEBUG,
 621             "%s - Allocated IRP %p with allocation flags %lx\n",
 622             __FUNCTION__,
 623             Irp,
 624             Flags);
 625     return Irp;
 626 }
 627
 628 /*
 629  * @implemented
 630  */
 631 PIRP
 632 NTAPI
 633 IopAllocateIrpMustSucceed(IN CCHAR StackSize)
 634 {
 635     LONG i;
 636     PIRP Irp;
 637     LARGE_INTEGER Sleep;
 638
 639     /* Try to get an IRP */
 640     Irp = IoAllocateIrp(StackSize, FALSE);
 641     if (Irp)
 642         return Irp;
 643
 644     /* If we fail, start looping till we may get one */
 645     i = LONG_MAX;
 646     do {
 647         i--;
 648
 649         /* First, sleep for 10ms */
 650         Sleep.QuadPart = -10 * 1000 * 10;;
 651         KeDelayExecutionThread(KernelMode, FALSE, &Sleep);
 652
 653         /* Then, retry allocation */
 654         Irp = IoAllocateIrp(StackSize, FALSE);
 655         if (Irp)
 656             return Irp;
 657     } while (i > 0);
 658
 659     return Irp;
 660 }
 661
 662 /*
 663  * @implemented
 664  */
 665 PIRP
 666 NTAPI
 667 IoBuildAsynchronousFsdRequest(IN ULONG MajorFunction,
 668                               IN PDEVICE_OBJECT DeviceObject,
 669                               IN PVOID Buffer,
 670                               IN ULONG Length,
 671                               IN PLARGE_INTEGER StartingOffset,
 672                               IN PIO_STATUS_BLOCK IoStatusBlock)
 673 {
 674     PIRP Irp;
 675     PIO_STACK_LOCATION StackPtr;
 676
 677     /* Allocate IRP */
 678     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 679     if (!Irp) return NULL;
 680
 681     /* Get the Stack */
 682     StackPtr = IoGetNextIrpStackLocation(Irp);
 683
 684     /* Write the Major function and then deal with it */
 685     StackPtr->MajorFunction = (UCHAR)MajorFunction;
 686
 687     /* Do not handle the following here */
 688     if ((MajorFunction != IRP_MJ_FLUSH_BUFFERS) &&
 689         (MajorFunction != IRP_MJ_SHUTDOWN) &&
 690         (MajorFunction != IRP_MJ_PNP) &&
 691         (MajorFunction != IRP_MJ_POWER))
 692     {
 693         /* Check if this is Buffered IO */
 694         if (DeviceObject->Flags & DO_BUFFERED_IO)
 695         {
 696             /* Allocate the System Buffer */
 697             Irp->AssociatedIrp.SystemBuffer =
 698                 ExAllocatePoolWithTag(NonPagedPool, Length, TAG_SYS_BUF);
 699             if (!Irp->AssociatedIrp.SystemBuffer)
 700             {
 701                 /* Free the IRP and fail */
 702                 IoFreeIrp(Irp);
 703                 return NULL;
 704             }
 705
 706             /* Set flags */
 707             Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 708
 709             /* Handle special IRP_MJ_WRITE Case */
 710             if (MajorFunction == IRP_MJ_WRITE)
 711             {
 712                 /* Copy the buffer data */
 713                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, Buffer, Length);
 714             }
 715             else
 716             {
 717                 /* Set the Input Operation flag and set this as a User Buffer */
 718                 Irp->Flags |= IRP_INPUT_OPERATION;
 719                 Irp->UserBuffer = Buffer;
 720             }
 721         }
 722         else if (DeviceObject->Flags & DO_DIRECT_IO)
 723         {
 724             /* Use an MDL for Direct I/O */
 725             Irp->MdlAddress = IoAllocateMdl(Buffer,
 726                                             Length,
 727                                             FALSE,
 728                                             FALSE,
 729                                             NULL);
 730             if (!Irp->MdlAddress)
 731             {
 732                 /* Free the IRP and fail */
 733                 IoFreeIrp(Irp);
 734                 return NULL;
 735             }
 736
 737                         /* Probe and Lock */
 738                         _SEH2_TRY
 739                         {
 740                                 /* Do the probe */
 741                                 MmProbeAndLockPages(Irp->MdlAddress,
 742                                                                         KernelMode,
 743                                                                         MajorFunction == IRP_MJ_READ ?
 744                                                                         IoWriteAccess : IoReadAccess);
 745                         }
 746                         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 747                         {
 748                                 /* Free the IRP and its MDL */
 749                                 IoFreeMdl(Irp->MdlAddress);
 750                                 IoFreeIrp(Irp);
 751
 752                 /* Fail */
 753                                 _SEH2_YIELD(return NULL);
 754                         }
 755                         _SEH2_END;
 756         }
 757         else
 758         {
 759             /* Neither, use the buffer */
 760             Irp->UserBuffer = Buffer;
 761         }
 762
 763         /* Check if this is a read */
 764         if (MajorFunction == IRP_MJ_READ)
 765         {
 766             /* Set the parameters for a read */
 767             StackPtr->Parameters.Read.Length = Length;
 768             StackPtr->Parameters.Read.ByteOffset = *StartingOffset;
 769         }
 770         else if (MajorFunction == IRP_MJ_WRITE)
 771         {
 772             /* Otherwise, set write parameters */
 773             StackPtr->Parameters.Write.Length = Length;
 774             StackPtr->Parameters.Write.ByteOffset = *StartingOffset;
 775         }
 776     }
 777
 778     /* Set the Current Thread and IOSB */
 779     Irp->UserIosb = IoStatusBlock;
 780     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 781
 782     /* Return the IRP */
 783     IOTRACE(IO_IRP_DEBUG,
 784             "%s - Built IRP %p with Major, Buffer, DO %lx %p %p\n",
 785             __FUNCTION__,
 786             Irp,
 787             MajorFunction,
 788             Buffer,
 789             DeviceObject);
 790     return Irp;
 791 }
 792
 793 /*
 794  * @implemented
 795  */
 796 PIRP
 797 NTAPI
 798 IoBuildDeviceIoControlRequest(IN ULONG IoControlCode,
 799                               IN PDEVICE_OBJECT DeviceObject,
 800                               IN PVOID InputBuffer,
 801                               IN ULONG InputBufferLength,
 802                               IN PVOID OutputBuffer,
 803                               IN ULONG OutputBufferLength,
 804                               IN BOOLEAN InternalDeviceIoControl,
 805                               IN PKEVENT Event,
 806                               IN PIO_STATUS_BLOCK IoStatusBlock)
 807 {
 808     PIRP Irp;
 809     PIO_STACK_LOCATION StackPtr;
 810     ULONG BufferLength;
 811
 812     /* Allocate IRP */
 813     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 814     if (!Irp) return NULL;
 815
 816     /* Get the Stack */
 817     StackPtr = IoGetNextIrpStackLocation(Irp);
 818
 819     /* Set the DevCtl Type */
 820     StackPtr->MajorFunction = InternalDeviceIoControl ?
 821                               IRP_MJ_INTERNAL_DEVICE_CONTROL :
 822                               IRP_MJ_DEVICE_CONTROL;
 823
 824     /* Set the IOCTL Data */
 825     StackPtr->Parameters.DeviceIoControl.IoControlCode = IoControlCode;
 826     StackPtr->Parameters.DeviceIoControl.InputBufferLength = InputBufferLength;
 827     StackPtr->Parameters.DeviceIoControl.OutputBufferLength =
 828         OutputBufferLength;
 829
 830     /* Handle the Methods */
 831     switch (IO_METHOD_FROM_CTL_CODE(IoControlCode))
 832     {
 833         /* Buffered I/O */
 834         case METHOD_BUFFERED:
 835
 836             /* Select the right Buffer Length */
 837             BufferLength = InputBufferLength > OutputBufferLength ?
 838                            InputBufferLength : OutputBufferLength;
 839
 840             /* Make sure there is one */
 841             if (BufferLength)
 842             {
 843                 /* Allocate the System Buffer */
 844                 Irp->AssociatedIrp.SystemBuffer =
 845                     ExAllocatePoolWithTag(NonPagedPool,
 846                                           BufferLength,
 847                                           TAG_SYS_BUF);
 848                 if (!Irp->AssociatedIrp.SystemBuffer)
 849                 {
 850                     /* Free the IRP and fail */
 851                     IoFreeIrp(Irp);
 852                     return NULL;
 853                 }
 854
 855                 /* Check if we got a buffer */
 856                 if (InputBuffer)
 857                 {
 858                     /* Copy into the System Buffer */
 859                     RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 860                                   InputBuffer,
 861                                   InputBufferLength);
 862                 }
 863
 864                 /* Write the flags */
 865                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 866                 if (OutputBuffer) Irp->Flags |= IRP_INPUT_OPERATION;
 867
 868                 /* Save the Buffer */
 869                 Irp->UserBuffer = OutputBuffer;
 870             }
 871             else
 872             {
 873                 /* Clear the Flags and Buffer */
 874                 Irp->Flags = 0;
 875                 Irp->UserBuffer = NULL;
 876             }
 877             break;
 878
 879         /* Direct I/O */
 880         case METHOD_IN_DIRECT:
 881         case METHOD_OUT_DIRECT:
 882
 883             /* Check if we got an input buffer */
 884             if (InputBuffer)
 885             {
 886                 /* Allocate the System Buffer */
 887                 Irp->AssociatedIrp.SystemBuffer =
 888                     ExAllocatePoolWithTag(NonPagedPool,
 889                                           InputBufferLength,
 890                                           TAG_SYS_BUF);
 891                 if (!Irp->AssociatedIrp.SystemBuffer)
 892                 {
 893                     /* Free the IRP and fail */
 894                     IoFreeIrp(Irp);
 895                     return NULL;
 896                 }
 897
 898                 /* Copy into the System Buffer */
 899                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 900                               InputBuffer,
 901                               InputBufferLength);
 902
 903                 /* Write the flags */
 904                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 905             }
 906             else
 907             {
 908                 /* Clear the flags */
 909                 Irp->Flags = 0;
 910             }
 911
 912             /* Check if we got an output buffer */
 913             if (OutputBuffer)
 914             {
 915                 /* Allocate the System Buffer */
 916                 Irp->MdlAddress = IoAllocateMdl(OutputBuffer,
 917                                                 OutputBufferLength,
 918                                                 FALSE,
 919                                                 FALSE,
 920                                                 Irp);
 921                 if (!Irp->MdlAddress)
 922                 {
 923                     /* Free the IRP and fail */
 924                     IoFreeIrp(Irp);
 925                     return NULL;
 926                 }
 927
 928                 /* Probe and Lock */
 929                 _SEH2_TRY
 930                 {
 931                     /* Do the probe */
 932                     MmProbeAndLockPages(Irp->MdlAddress,
 933                                         KernelMode,
 934                                         IO_METHOD_FROM_CTL_CODE(IoControlCode) ==
 935                                         METHOD_IN_DIRECT ?
 936                                         IoReadAccess : IoWriteAccess);
 937                 }
 938                 _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 939                 {
 940                     /* Free the MDL */
 941                     IoFreeMdl(Irp->MdlAddress);
 942
 943                     /* Free the input buffer and IRP */
 944                     if (InputBuffer) ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 945                     IoFreeIrp(Irp);
 946
 947                     /* Fail */
 948                     _SEH2_YIELD(return NULL);
 949                 }
 950                 _SEH2_END;
 951             }
 952             break;
 953
 954         case METHOD_NEITHER:
 955
 956             /* Just save the Buffer */
 957             Irp->UserBuffer = OutputBuffer;
 958             StackPtr->Parameters.DeviceIoControl.Type3InputBuffer = InputBuffer;
 959     }
 960
 961     /* Now write the Event and IoSB */
 962     Irp->UserIosb = IoStatusBlock;
 963     Irp->UserEvent = Event;
 964
 965     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 966     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 967     IoQueueThreadIrp(Irp);
 968
 969     /* Return the IRP */
 970     IOTRACE(IO_IRP_DEBUG,
 971             "%s - Built IRP %p with IOCTL, Buffers, DO %lx %p %p %p\n",
 972             __FUNCTION__,
 973             Irp,
 974             IoControlCode,
 975             InputBuffer,
 976             OutputBuffer,
 977             DeviceObject);
 978     return Irp;
 979 }
 980
 981 /*
 982  * @implemented
 983  */
 984 PIRP
 985 NTAPI
 986 IoBuildSynchronousFsdRequest(IN ULONG MajorFunction,
 987                              IN PDEVICE_OBJECT DeviceObject,
 988                              IN PVOID Buffer,
 989                              IN ULONG Length,
 990                              IN PLARGE_INTEGER StartingOffset,
 991                              IN PKEVENT Event,
 992                              IN PIO_STATUS_BLOCK IoStatusBlock)
 993 {
 994     PIRP Irp;
 995
 996     /* Do the big work to set up the IRP */
 997     Irp = IoBuildAsynchronousFsdRequest(MajorFunction,
 998                                         DeviceObject,
 999                                         Buffer,
1000                                         Length,
1001                                         StartingOffset,
1002                                         IoStatusBlock );
1003     if (!Irp) return NULL;
1004
1005     /* Set the Event which makes it Syncronous */
1006     Irp->UserEvent = Event;
1007
1008     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
1009     IoQueueThreadIrp(Irp);
1010     return Irp;
1011 }
1012
1013 /*
1014  * @implemented
1015  */
1016 BOOLEAN
1017 NTAPI
1018 IoCancelIrp(IN PIRP Irp)
1019 {
1020     KIRQL OldIrql;
1021     PDRIVER_CANCEL CancelRoutine;
1022     IOTRACE(IO_IRP_DEBUG,
1023             "%s - Canceling IRP %p\n",
1024             __FUNCTION__,
1025             Irp);
1026     ASSERT(Irp->Type == IO_TYPE_IRP);
1027
1028     /* Acquire the cancel lock and cancel the IRP */
1029     IoAcquireCancelSpinLock(&OldIrql);
1030     Irp->Cancel = TRUE;
1031
1032     /* Clear the cancel routine and get the old one */
1033     CancelRoutine = (PVOID)IoSetCancelRoutine(Irp, NULL);
1034     if (CancelRoutine)
1035     {
1036         /* We had a routine, make sure the IRP isn't completed */
1037         if (Irp->CurrentLocation > (Irp->StackCount + 1))
1038         {
1039             /* It is, bugcheck */
1040             KeBugCheckEx(CANCEL_STATE_IN_COMPLETED_IRP,
1041                          (ULONG_PTR)Irp,
1042                          (ULONG_PTR)CancelRoutine,
1043                          0,
1044                          0);
1045         }
1046
1047         /* Set the cancel IRQL And call the routine */
1048         Irp->CancelIrql = OldIrql;
1049         CancelRoutine(IoGetCurrentIrpStackLocation(Irp)->DeviceObject, Irp);
1050         return TRUE;
1051     }
1052
1053     /* Otherwise, release the cancel lock and fail */
1054     IoReleaseCancelSpinLock(OldIrql);
1055     return FALSE;
1056 }
1057
1058 /*
1059  * @implemented
1060  */
1061 VOID
1062 NTAPI
1063 IoCancelThreadIo(IN PETHREAD Thread)
1064 {
1065     KIRQL OldIrql;
1066     ULONG Retries = 3000;
1067     LARGE_INTEGER Interval;
1068     PLIST_ENTRY ListHead, NextEntry;
1069     PIRP Irp;
1070     PAGED_CODE();
1071
1072     /* Windows isn't using given thread, but using current. */
1073     Thread = PsGetCurrentThread();
1074
1075     IOTRACE(IO_IRP_DEBUG,
1076             "%s - Canceling IRPs for Thread %p\n",
1077             __FUNCTION__,
1078             Thread);
1079
1080     /* Raise to APC to protect the IrpList */
1081     KeRaiseIrql(APC_LEVEL, &OldIrql);
1082
1083     /* Start by cancelling all the IRPs in the current thread queue. */
1084     ListHead = &Thread->IrpList;
1085     NextEntry = ListHead->Flink;
1086     while (ListHead != NextEntry)
1087     {
1088         /* Get the IRP */
1089         Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
1090
1091         /* Cancel it */
1092         IoCancelIrp(Irp);
1093
1094         /* Move to the next entry */
1095         NextEntry = NextEntry->Flink;
1096     }
1097
1098      /* Wait 100 milliseconds */
1099     Interval.QuadPart = -1000000;
1100
1101     /* Wait till all the IRPs are completed or cancelled. */
1102     while (!IsListEmpty(&Thread->IrpList))
1103     {
1104         /* Now we can lower */
1105         KeLowerIrql(OldIrql);
1106
1107         /* Wait a short while and then look if all our IRPs were completed. */
1108         KeDelayExecutionThread(KernelMode, FALSE, &Interval);
1109
1110         /*
1111          * Don't stay here forever if some broken driver doesn't complete
1112          * the IRP.
1113          */
1114         if (!(Retries--))
1115         {
1116             /* Print out a message and remove the IRP */
1117             DPRINT1("Broken driver did not complete!\n");
1118             IopDisassociateThreadIrp();
1119         }
1120
1121         /* Raise the IRQL Again */
1122         KeRaiseIrql(APC_LEVEL, &OldIrql);
1123     }
1124
1125     /* We're done, lower the IRQL */
1126     KeLowerIrql(OldIrql);
1127 }
1128
1129 /*
1130  * @implemented
1131  */
1132 #undef IoCallDriver
1133 NTSTATUS
1134 NTAPI
1135 IoCallDriver(IN PDEVICE_OBJECT DeviceObject,
1136              IN PIRP Irp)
1137 {
1138     /* Call fastcall */
1139     return IofCallDriver(DeviceObject, Irp);
1140 }
1141
1142 #define IoCallDriver IofCallDriver
1143
1144 /*
1145  * @implemented
1146  */
1147 #undef IoCompleteRequest
1148 VOID
1149 NTAPI
1150 IoCompleteRequest(IN PIRP Irp,
1151                   IN CCHAR PriorityBoost)
1152 {
1153     /* Call the fastcall */
1154     IofCompleteRequest(Irp, PriorityBoost);
1155 }
1156
1157 #define IoCompleteRequest IofCompleteRequest
1158
1159 /*
1160  * @implemented
1161  */
1162 VOID
1163 NTAPI
1164 IoEnqueueIrp(IN PIRP Irp)
1165 {
1166     /* This is the same as calling IoQueueThreadIrp */
1167     IoQueueThreadIrp(Irp);
1168 }
1169
1170 /*
1171  * @implemented
1172  */
1173 NTSTATUS
1174 FASTCALL
1175 IofCallDriver(IN PDEVICE_OBJECT DeviceObject,
1176               IN PIRP Irp)
1177 {
1178     PDRIVER_OBJECT DriverObject;
1179     PIO_STACK_LOCATION StackPtr;
1180
1181     /* Make sure this is a valid IRP */
1182     ASSERT(Irp->Type == IO_TYPE_IRP);
1183
1184     /* Get the Driver Object */
1185     DriverObject = DeviceObject->DriverObject;
1186
1187     /* Decrease the current location and check if */
1188     Irp->CurrentLocation--;
1189     if (Irp->CurrentLocation <= 0)
1190     {
1191         /* This IRP ran out of stack, bugcheck */
1192         KeBugCheckEx(NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR)Irp, 0, 0, 0);
1193     }
1194
1195     /* Now update the stack location */
1196     StackPtr = IoGetNextIrpStackLocation(Irp);
1197     Irp->Tail.Overlay.CurrentStackLocation = StackPtr;
1198
1199     /* Get the Device Object */
1200     StackPtr->DeviceObject = DeviceObject;
1201
1202     /* Call it */
1203     return DriverObject->MajorFunction[StackPtr->MajorFunction](DeviceObject,
1204                                                                 Irp);
1205 }
1206
1207 FORCEINLINE
1208 VOID
1209 IopClearStackLocation(IN PIO_STACK_LOCATION IoStackLocation)
1210 {
1211     IoStackLocation->MinorFunction = 0;
1212     IoStackLocation->Flags = 0;
1213     IoStackLocation->Control &= SL_ERROR_RETURNED;
1214     IoStackLocation->Parameters.Others.Argument1 = 0;
1215     IoStackLocation->Parameters.Others.Argument2 = 0;
1216     IoStackLocation->Parameters.Others.Argument3 = 0;
1217     IoStackLocation->FileObject = NULL;
1218 }
1219
1220 /*
1221  * @implemented
1222  */
1223 VOID
1224 FASTCALL
1225 IofCompleteRequest(IN PIRP Irp,
1226                    IN CCHAR PriorityBoost)
1227 {
1228     PIO_STACK_LOCATION StackPtr, LastStackPtr;
1229     PDEVICE_OBJECT DeviceObject;
1230     PFILE_OBJECT FileObject;
1231     PETHREAD Thread;
1232     NTSTATUS Status;
1233     PMDL Mdl, NextMdl;
1234     ULONG MasterCount;
1235     PIRP MasterIrp;
1236     ULONG Flags;
1237     NTSTATUS ErrorCode = STATUS_SUCCESS;
1238     IOTRACE(IO_IRP_DEBUG,
1239             "%s - Completing IRP %p\n",
1240             __FUNCTION__,
1241             Irp);
1242
1243     /* Make sure this IRP isn't getting completed twice or is invalid */
1244     if ((Irp->CurrentLocation) > (Irp->StackCount + 1))
1245     {
1246         /* Bugcheck */
1247         KeBugCheckEx(MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR)Irp, 0, 0, 0);
1248     }
1249
1250     /* Some sanity checks */
1251     ASSERT(Irp->Type == IO_TYPE_IRP);
1252     ASSERT(!Irp->CancelRoutine);
1253     ASSERT(Irp->IoStatus.Status != STATUS_PENDING);
1254     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
1255
1256     /* Get the last stack */
1257     LastStackPtr = (PIO_STACK_LOCATION)(Irp + 1);
1258     if (LastStackPtr->Control & SL_ERROR_RETURNED)
1259     {
1260         /* Get the error code */
1261         ErrorCode = PtrToUlong(LastStackPtr->Parameters.Others.Argument4);
1262     }
1263
1264     /*
1265      * Start the loop with the current stack and point the IRP to the next stack
1266      * and then keep incrementing the stack as we loop through. The IRP should
1267      * always point to the next stack location w.r.t the one currently being
1268      * analyzed, so completion routine code will see the appropriate value.
1269      * Because of this, we must loop until the current stack location is +1 of
1270      * the stack count, because when StackPtr is at the end, CurrentLocation is +1.
1271      */
1272     for (StackPtr = IoGetCurrentIrpStackLocation(Irp),
1273          Irp->CurrentLocation++,
1274          Irp->Tail.Overlay.CurrentStackLocation++;
1275          Irp->CurrentLocation <= (Irp->StackCount + 1);
1276          StackPtr++,
1277          Irp->CurrentLocation++,
1278          Irp->Tail.Overlay.CurrentStackLocation++)
1279     {
1280         /* Set Pending Returned */
1281         Irp->PendingReturned = StackPtr->Control & SL_PENDING_RETURNED;
1282
1283         /* Check if we failed */
1284         if (!NT_SUCCESS(Irp->IoStatus.Status))
1285         {
1286             /* Check if it was changed by a completion routine */
1287             if (Irp->IoStatus.Status != ErrorCode)
1288             {
1289                 /* Update the error for the current stack */
1290                 ErrorCode = Irp->IoStatus.Status;
1291                 StackPtr->Control |= SL_ERROR_RETURNED;
1292                 LastStackPtr->Parameters.Others.Argument4 = UlongToPtr(ErrorCode);
1293                 LastStackPtr->Control |= SL_ERROR_RETURNED;
1294             }
1295         }
1296
1297         /* Check if there is a Completion Routine to Call */
1298         if ((NT_SUCCESS(Irp->IoStatus.Status) &&
1299              (StackPtr->Control & SL_INVOKE_ON_SUCCESS)) ||
1300             (!NT_SUCCESS(Irp->IoStatus.Status) &&
1301              (StackPtr->Control & SL_INVOKE_ON_ERROR)) ||
1302             (Irp->Cancel &&
1303              (StackPtr->Control & SL_INVOKE_ON_CANCEL)))
1304         {
1305             /* Clear the stack location */
1306             IopClearStackLocation(StackPtr);
1307
1308             /* Check for highest-level device completion routines */
1309             if (Irp->CurrentLocation == (Irp->StackCount + 1))
1310             {
1311                 /* Clear the DO, since the current stack location is invalid */
1312                 DeviceObject = NULL;
1313             }
1314             else
1315             {
1316                 /* Otherwise, return the real one */
1317                 DeviceObject = IoGetCurrentIrpStackLocation(Irp)->DeviceObject;
1318             }
1319
1320             /* Call the completion routine */
1321             Status = StackPtr->CompletionRoutine(DeviceObject,
1322                                                  Irp,
1323                                                  StackPtr->Context);
1324
1325             /* Don't touch the Packet in this case, since it might be gone! */
1326             if (Status == STATUS_MORE_PROCESSING_REQUIRED) return;
1327         }
1328         else
1329         {
1330             /* Otherwise, check if this is a completed IRP */
1331             if ((Irp->CurrentLocation <= Irp->StackCount) &&
1332                 (Irp->PendingReturned))
1333             {
1334                 /* Mark it as pending */
1335                 IoMarkIrpPending(Irp);
1336             }
1337
1338             /* Clear the stack location */
1339             IopClearStackLocation(StackPtr);
1340         }
1341     }
1342
1343     /* Check if the IRP is an associated IRP */
1344     if (Irp->Flags & IRP_ASSOCIATED_IRP)
1345     {
1346         /* Get the master IRP and count */
1347         MasterIrp = Irp->AssociatedIrp.MasterIrp;
1348         MasterCount = InterlockedDecrement(&MasterIrp->AssociatedIrp.IrpCount);
1349
1350         /* Free the MDLs */
1351         for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
1352         {
1353             /* Go to the next one */
1354             NextMdl = Mdl->Next;
1355             IoFreeMdl(Mdl);
1356         }
1357
1358         /* Free the IRP itself */
1359         IoFreeIrp(Irp);
1360
1361         /* Complete the Master IRP */
1362         if (!MasterCount) IofCompleteRequest(MasterIrp, PriorityBoost);
1363         return;
1364     }
1365
1366     /* We don't support this yet */
1367     ASSERT(Irp->IoStatus.Status != STATUS_REPARSE);
1368
1369     /* Check if we have an auxiliary buffer */
1370     if (Irp->Tail.Overlay.AuxiliaryBuffer)
1371     {
1372         /* Free it */
1373         ExFreePool(Irp->Tail.Overlay.AuxiliaryBuffer);
1374         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1375     }
1376
1377     /* Check if this is a Paging I/O or Close Operation */
1378     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION))
1379     {
1380         /* Handle a Close Operation or Sync Paging I/O */
1381         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION))
1382         {
1383             /* Set the I/O Status and Signal the Event */
1384             Flags = Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_PAGING_IO);
1385             *Irp->UserIosb = Irp->IoStatus;
1386             KeSetEvent(Irp->UserEvent, PriorityBoost, FALSE);
1387
1388             /* Free the IRP for a Paging I/O Only, Close is handled by us */
1389             if (Flags) IoFreeIrp(Irp);
1390         }
1391         else
1392         {
1393 #if 0
1394             /* Page 166 */
1395             KeInitializeApc(&Irp->Tail.Apc
1396                             &Irp->Tail.Overlay.Thread->Tcb,
1397                             Irp->ApcEnvironment,
1398                             IopCompletePageWrite,
1399                             NULL,
1400                             NULL,
1401                             KernelMode,
1402                             NULL);
1403             KeInsertQueueApc(&Irp->Tail.Apc,
1404                              NULL,
1405                              NULL,
1406                              PriorityBoost);
1407 #else
1408             /* Not implemented yet. */
1409             UNIMPLEMENTED_DBGBREAK("Not supported!\n");
1410 #endif
1411         }
1412
1413         /* Get out of here */
1414         return;
1415     }
1416
1417     /* Unlock MDL Pages, page 167. */
1418     Mdl = Irp->MdlAddress;
1419     while (Mdl)
1420     {
1421                 MmUnlockPages(Mdl);
1422         Mdl = Mdl->Next;
1423     }
1424
1425     /* Check if we should exit because of a Deferred I/O (page 168) */
1426     if ((Irp->Flags & IRP_DEFER_IO_COMPLETION) && !(Irp->PendingReturned))
1427     {
1428         /*
1429          * Return without queuing the completion APC, since the caller will
1430          * take care of doing its own optimized completion at PASSIVE_LEVEL.
1431          */
1432         return;
1433     }
1434
1435     /* Get the thread and file object */
1436     Thread = Irp->Tail.Overlay.Thread;
1437     FileObject = Irp->Tail.Overlay.OriginalFileObject;
1438
1439     /* Make sure the IRP isn't canceled */
1440     if (!Irp->Cancel)
1441     {
1442         /* Initialize the APC */
1443         KeInitializeApc(&Irp->Tail.Apc,
1444                         &Thread->Tcb,
1445                         Irp->ApcEnvironment,
1446                         IopCompleteRequest,
1447                         NULL,
1448                         NULL,
1449                         KernelMode,
1450                         NULL);
1451
1452         /* Queue it */
1453         KeInsertQueueApc(&Irp->Tail.Apc,
1454                          FileObject,
1455                          NULL, /* This is used for REPARSE stuff */
1456                          PriorityBoost);
1457     }
1458     else
1459     {
1460         /* The IRP just got canceled... does a thread still own it? */
1461         if (Thread)
1462         {
1463             /* Yes! There is still hope! Initialize the APC */
1464             KeInitializeApc(&Irp->Tail.Apc,
1465                             &Thread->Tcb,
1466                             Irp->ApcEnvironment,
1467                             IopCompleteRequest,
1468                             NULL,
1469                             NULL,
1470                             KernelMode,
1471                             NULL);
1472
1473             /* Queue it */
1474             KeInsertQueueApc(&Irp->Tail.Apc,
1475                              FileObject,
1476                              NULL, /* This is used for REPARSE stuff */
1477                              PriorityBoost);
1478         }
1479         else
1480         {
1481             /* Nothing left for us to do, kill it */
1482             ASSERT(Irp->Cancel);
1483             IopCleanupIrp(Irp, FileObject);
1484         }
1485     }
1486 }
1487
1488 NTSTATUS
1489 NTAPI
1490 IopSynchronousCompletion(IN PDEVICE_OBJECT DeviceObject,
1491                          IN PIRP Irp,
1492                          IN PVOID Context)
1493 {
1494     if (Irp->PendingReturned)
1495         KeSetEvent((PKEVENT)Context, IO_NO_INCREMENT, FALSE);
1496     return STATUS_MORE_PROCESSING_REQUIRED;
1497 }
1498
1499 /*
1500  * @implemented
1501  */
1502 BOOLEAN
1503 NTAPI
1504 IoForwardIrpSynchronously(IN PDEVICE_OBJECT DeviceObject,
1505                           IN PIRP Irp)
1506 {
1507     KEVENT Event;
1508     NTSTATUS Status;
1509
1510     /* Check if next stack location is available */
1511     if (Irp->CurrentLocation < Irp->StackCount)
1512     {
1513         /* No more stack location */
1514         return FALSE;
1515     }
1516
1517     /* Initialize event */
1518     KeInitializeEvent(&Event, NotificationEvent, FALSE);
1519
1520     /* Copy stack location for next driver */
1521     IoCopyCurrentIrpStackLocationToNext(Irp);
1522
1523     /* Set a completion routine, which will signal the event */
1524     IoSetCompletionRoutine(Irp, IopSynchronousCompletion, &Event, TRUE, TRUE, TRUE);
1525
1526     /* Call next driver */
1527     Status = IoCallDriver(DeviceObject, Irp);
1528
1529     /* Check if irp is pending */
1530     if (Status == STATUS_PENDING)
1531     {
1532         /* Yes, wait for its completion */
1533         KeWaitForSingleObject(&Event, Suspended, KernelMode, FALSE, NULL);
1534     }
1535
1536     /* Return success */
1537     return TRUE;
1538 }
1539
1540 /*
1541  * @implemented
1542  */
1543 VOID
1544 NTAPI
1545 IoFreeIrp(IN PIRP Irp)
1546 {
1547     PNPAGED_LOOKASIDE_LIST List;
1548     PP_NPAGED_LOOKASIDE_NUMBER ListType =  LookasideSmallIrpList;
1549     PKPRCB Prcb;
1550     IOTRACE(IO_IRP_DEBUG,
1551             "%s - Freeing IRPs %p\n",
1552             __FUNCTION__,
1553             Irp);
1554
1555     /* Make sure the Thread IRP list is empty and that it OK to free it */
1556     ASSERT(Irp->Type == IO_TYPE_IRP);
1557     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1558     ASSERT(Irp->CurrentLocation >= Irp->StackCount);
1559
1560     /* If this was a pool alloc, free it with the pool */
1561     if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
1562     {
1563         /* Free it */
1564         ExFreePoolWithTag(Irp, TAG_IRP);
1565     }
1566     else
1567     {
1568         /* Check if this was a Big IRP */
1569         if (Irp->StackCount != 1) ListType = LookasideLargeIrpList;
1570
1571         /* Get the PRCB */
1572         Prcb = KeGetCurrentPrcb();
1573
1574         /* Use the P List */
1575         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
1576         List->L.TotalFrees++;
1577
1578         /* Check if the Free was within the Depth or not */
1579         if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1580         {
1581             /* Let the balancer know */
1582             List->L.FreeMisses++;
1583
1584             /* Use the L List */
1585             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
1586             List->L.TotalFrees++;
1587
1588             /* Check if the Free was within the Depth or not */
1589             if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1590             {
1591                 /* All lists failed, use the pool */
1592                 List->L.FreeMisses++;
1593                 ExFreePoolWithTag(Irp, TAG_IRP);
1594                 Irp = NULL;
1595             }
1596         }
1597
1598         /* The free was within the Depth */
1599         if (Irp)
1600         {
1601            InterlockedPushEntrySList(&List->L.ListHead,
1602                                      (PSLIST_ENTRY)Irp);
1603         }
1604     }
1605 }
1606
1607 /*
1608  * @implemented
1609  */
1610 IO_PAGING_PRIORITY
1611 FASTCALL
1612 IoGetPagingIoPriority(IN PIRP Irp)
1613 {
1614     IO_PAGING_PRIORITY Priority;
1615     ULONG Flags;
1616
1617     /* Get the flags */
1618     Flags = Irp->Flags;
1619
1620     /* Check what priority it has */
1621     if (Flags & IRP_CLASS_CACHE_OPERATION)
1622     {
1623         /* High priority */
1624         Priority = IoPagingPriorityHigh;
1625     }
1626     else if (Flags & IRP_PAGING_IO)
1627     {
1628         /* Normal priority */
1629         Priority = IoPagingPriorityNormal;
1630     }
1631     else
1632     {
1633         /* Invalid -- not a paging IRP */
1634         Priority = IoPagingPriorityInvalid;
1635     }
1636
1637     /* Return the priority */
1638     return Priority;
1639 }
1640
1641 /*
1642  * @implemented
1643  */
1644 PEPROCESS
1645 NTAPI
1646 IoGetRequestorProcess(IN PIRP Irp)
1647 {
1648     /* Return the requestor process */
1649     if (Irp->Tail.Overlay.Thread)
1650     {
1651         if (Irp->ApcEnvironment == OriginalApcEnvironment)
1652         {
1653             return Irp->Tail.Overlay.Thread->ThreadsProcess;
1654         }
1655         else if (Irp->ApcEnvironment == AttachedApcEnvironment)
1656         {
1657             return (PEPROCESS)Irp->Tail.Overlay.Thread->Tcb.ApcState.Process;
1658         }
1659     }
1660
1661     return NULL;
1662 }
1663
1664 /*
1665  * @implemented
1666  */
1667 ULONG
1668 NTAPI
1669 IoGetRequestorProcessId(IN PIRP Irp)
1670 {
1671     PEPROCESS Process;
1672
1673     /* Return the requestor process' id */
1674     Process = IoGetRequestorProcess(Irp);
1675     if (Process) return PtrToUlong(Process->UniqueProcessId);
1676
1677     return 0;
1678 }
1679
1680 /*
1681  * @implemented
1682  */
1683 NTSTATUS
1684 NTAPI
1685 IoGetRequestorSessionId(IN PIRP Irp,
1686                         OUT PULONG pSessionId)
1687 {
1688     PEPROCESS Process;
1689
1690     /* Return the session */
1691     if (Irp->Tail.Overlay.Thread)
1692     {
1693         Process = Irp->Tail.Overlay.Thread->ThreadsProcess;
1694         *pSessionId = MmGetSessionId(Process);
1695         return STATUS_SUCCESS;
1696     }
1697
1698     *pSessionId = (ULONG)-1;
1699     return STATUS_UNSUCCESSFUL;
1700 }
1701
1702 /*
1703  * @implemented
1704  */
1705 PIRP
1706 NTAPI
1707 IoGetTopLevelIrp(VOID)
1708 {
1709     /* Return the IRP */
1710     return (PIRP)PsGetCurrentThread()->TopLevelIrp;
1711 }
1712
1713 /*
1714  * @implemented
1715  */
1716 VOID
1717 NTAPI
1718 IoInitializeIrp(IN PIRP Irp,
1719                 IN USHORT PacketSize,
1720                 IN CCHAR StackSize)
1721 {
1722     /* Clear it */
1723     IOTRACE(IO_IRP_DEBUG,
1724             "%s - Initializing IRP %p\n",
1725             __FUNCTION__,
1726             Irp);
1727     RtlZeroMemory(Irp, PacketSize);
1728
1729     /* Set the Header and other data */
1730     Irp->Type = IO_TYPE_IRP;
1731     Irp->Size = PacketSize;
1732     Irp->StackCount = StackSize;
1733     Irp->CurrentLocation = StackSize + 1;
1734     Irp->ApcEnvironment =  KeGetCurrentThread()->ApcStateIndex;
1735     Irp->Tail.Overlay.CurrentStackLocation = (PIO_STACK_LOCATION)(Irp + 1) + StackSize;
1736
1737     /* Initialize the Thread List */
1738     InitializeListHead(&Irp->ThreadListEntry);
1739 }
1740
1741 /*
1742  * @implemented
1743  */
1744 BOOLEAN
1745 NTAPI
1746 IoIsOperationSynchronous(IN PIRP Irp)
1747 {
1748     BOOLEAN SynchIO;
1749     BOOLEAN ForceAsync;
1750
1751     /* If the IRP requests synchronous paging I/O, if the file object was opened
1752        for synchronous I/O, if the IRP_SYNCHRONOUS_API flag is set in the IRP
1753        the operation is synchronous */
1754     SynchIO = (IoGetCurrentIrpStackLocation(Irp)->FileObject->Flags & FO_SYNCHRONOUS_IO) ||
1755               (Irp->Flags & IRP_SYNCHRONOUS_API) || (Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1756
1757     /* If the IRP requests asynchronous paging I/O, the operation is asynchronous,
1758        even if one of the above conditions is true */
1759     ForceAsync = (Irp->Flags & IRP_PAGING_IO) && !(Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO);
1760
1761     /* Check the flags */
1762     if (SynchIO && !ForceAsync)
1763     {
1764         /* Synch API or Paging I/O is OK, as is Sync File I/O */
1765         return TRUE;
1766     }
1767
1768     /* Otherwise, it is an asynchronous operation. */
1769     return FALSE;
1770 }
1771
1772 /*
1773  * @unimplemented
1774  */
1775 BOOLEAN
1776 NTAPI
1777 IoIsValidNameGraftingBuffer(IN PIRP Irp,
1778                             IN PREPARSE_DATA_BUFFER ReparseBuffer)
1779 {
1780     UNIMPLEMENTED;
1781     return FALSE;
1782 }
1783
1784 /*
1785  * @implemented
1786  */
1787 PIRP
1788 NTAPI
1789 IoMakeAssociatedIrp(IN PIRP Irp,
1790                     IN CCHAR StackSize)
1791 {
1792     PIRP AssocIrp;
1793     IOTRACE(IO_IRP_DEBUG,
1794             "%s - Associating IRP %p\n",
1795             __FUNCTION__,
1796             Irp);
1797
1798    /* Allocate the IRP */
1799    AssocIrp = IoAllocateIrp(StackSize, FALSE);
1800    if (!AssocIrp) return NULL;
1801
1802    /* Set the Flags */
1803    AssocIrp->Flags |= IRP_ASSOCIATED_IRP;
1804
1805    /* Set the Thread */
1806    AssocIrp->Tail.Overlay.Thread = Irp->Tail.Overlay.Thread;
1807
1808    /* Associate them */
1809    AssocIrp->AssociatedIrp.MasterIrp = Irp;
1810    return AssocIrp;
1811 }
1812
1813 /*
1814  * @implemented
1815  */
1816 VOID
1817 NTAPI
1818 IoQueueThreadIrp(IN PIRP Irp)
1819 {
1820     IOTRACE(IO_IRP_DEBUG,
1821             "%s - Queueing IRP %p\n",
1822             __FUNCTION__,
1823             Irp);
1824
1825     /* Use our inlined routine */
1826     IopQueueIrpToThread(Irp);
1827 }
1828
1829 /*
1830  * @implemented
1831  * Reference: Chris Cant's "Writing WDM Device Drivers"
1832  */
1833 VOID
1834 NTAPI
1835 IoReuseIrp(IN OUT PIRP Irp,
1836            IN NTSTATUS Status)
1837 {
1838     UCHAR AllocationFlags;
1839     IOTRACE(IO_IRP_DEBUG,
1840             "%s - Reusing IRP %p\n",
1841             __FUNCTION__,
1842             Irp);
1843
1844     /* Make sure it's OK to reuse it */
1845     ASSERT(!Irp->CancelRoutine);
1846     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1847
1848     /* Get the old flags */
1849     AllocationFlags = Irp->AllocationFlags;
1850
1851     /* Reinitialize the IRP */
1852     IoInitializeIrp(Irp, Irp->Size, Irp->StackCount);
1853
1854     /* Duplicate the data */
1855     Irp->IoStatus.Status = Status;
1856     Irp->AllocationFlags = AllocationFlags;
1857 }
1858
1859 /*
1860  * @implemented
1861  */
1862 VOID
1863 NTAPI
1864 IoSetTopLevelIrp(IN PIRP Irp)
1865 {
1866     /* Set the IRP */
1867     PsGetCurrentThread()->TopLevelIrp = (ULONG_PTR)Irp;
1868 }
1869
1870 #if defined (_WIN64)
1871 BOOLEAN
1872 NTAPI
1873 IoIs32bitProcess(
1874     IN PIRP Irp OPTIONAL)
1875 {
1876     UNIMPLEMENTED;
1877     return FALSE;
1878 }
1879 #endif