reactos/ntoskrnl/io/iomgr/irp.c

   1 /*
   2  * PROJECT:         ReactOS Kernel
   3  * LICENSE:         GPL - See COPYING in the top level directory
   4  * FILE:            ntoskrnl/io/iomgr/irp.c
   5  * PURPOSE:         IRP Handling Functions
   6  * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
   7  *                  Gunnar Dalsnes
   8  *                  Filip Navara (navaraf@reactos.org)
   9  */
  10
  11 /* INCLUDES ****************************************************************/
  12
  13 #include <ntoskrnl.h>
  14 #define NDEBUG
  15 #include <debug.h>
  16
  17 /* Undefine some macros we implement here */
  18 #undef IoCallDriver
  19 #undef IoCompleteRequest
  20
  21 PIRP IopDeadIrp;
  22
  23 /* PRIVATE FUNCTIONS  ********************************************************/
  24
  25 VOID
  26 NTAPI
  27 IopFreeIrpKernelApc(IN PKAPC Apc,
  28                     IN PKNORMAL_ROUTINE *NormalRoutine,
  29                     IN PVOID *NormalContext,
  30                     IN PVOID *SystemArgument1,
  31                     IN PVOID *SystemArgument2)
  32 {
  33     /* Free the IRP */
  34     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  35 }
  36
  37 VOID
  38 NTAPI
  39 IopAbortIrpKernelApc(IN PKAPC Apc)
  40 {
  41     /* Free the IRP */
  42     IoFreeIrp(CONTAINING_RECORD(Apc, IRP, Tail.Apc));
  43 }
  44
  45 NTSTATUS
  46 NTAPI
  47 IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
  48                     IN PKEVENT EventObject OPTIONAL,
  49                     IN PVOID Buffer OPTIONAL)
  50 {
  51     PAGED_CODE();
  52
  53     /* Dereference the event */
  54     if (EventObject) ObDereferenceObject(EventObject);
  55
  56     /* Free a buffer, if any */
  57     if (Buffer) ExFreePool(Buffer);
  58
  59     /* If this was a file opened for synch I/O, then unlock it */
  60     if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
  61
  62     /* Now dereference it and return */
  63     ObDereferenceObject(FileObject);
  64     return STATUS_INSUFFICIENT_RESOURCES;
  65 }
  66
  67 VOID
  68 NTAPI
  69 IopAbortInterruptedIrp(IN PKEVENT EventObject,
  70                        IN PIRP Irp)
  71 {
  72     KIRQL OldIrql;
  73     BOOLEAN CancelResult;
  74     LARGE_INTEGER Wait;
  75     PAGED_CODE();
  76
  77     /* Raise IRQL to APC */
  78     KeRaiseIrql(APC_LEVEL, &OldIrql);
  79
  80     /* Check if nobody completed it yet */
  81     if (!KeReadStateEvent(EventObject))
  82     {
  83         /* First, cancel it */
  84         CancelResult = IoCancelIrp(Irp);
  85         KeLowerIrql(OldIrql);
  86
  87         /* Check if we cancelled it */
  88         if (CancelResult)
  89         {
  90             /* Wait for the IRP to be cancelled */
  91             Wait.QuadPart = -100000;
  92             while (!KeReadStateEvent(EventObject))
  93             {
  94                 /* Delay indefintely */
  95                 KeDelayExecutionThread(KernelMode, FALSE, &Wait);
  96             }
  97         }
  98         else
  99         {
 100             /* No cancellation done, so wait for the I/O system to kill it */
 101             KeWaitForSingleObject(EventObject,
 102                                   Executive,
 103                                   KernelMode,
 104                                   FALSE,
 105                                   NULL);
 106         }
 107     }
 108     else
 109     {
 110         /* We got preempted, so give up */
 111         KeLowerIrql(OldIrql);
 112     }
 113 }
 114
 115 VOID
 116 NTAPI
 117 IopDisassociateThreadIrp(VOID)
 118 {
 119     KIRQL OldIrql, LockIrql;
 120     PETHREAD IrpThread;
 121     PLIST_ENTRY IrpEntry;
 122     PIO_ERROR_LOG_PACKET ErrorLogEntry;
 123     PDEVICE_OBJECT DeviceObject = NULL;
 124     PIO_STACK_LOCATION IoStackLocation;
 125
 126     /* First, raise to APC to protect IrpList */
 127     KeRaiseIrql(APC_LEVEL, &OldIrql);
 128
 129     /* Get the Thread and check the list */
 130     IrpThread = PsGetCurrentThread();
 131     if (IsListEmpty(&IrpThread->IrpList))
 132     {
 133         /* It got completed now, so quit */
 134         KeLowerIrql(OldIrql);
 135         return;
 136     }
 137
 138     /* Ensure no one will come disturb */
 139     LockIrql = KeAcquireQueuedSpinLock(LockQueueIoCompletionLock);
 140
 141     /* Get the misbehaving IRP */
 142     IrpEntry = IrpThread->IrpList.Flink;
 143     IopDeadIrp = CONTAINING_RECORD(IrpEntry, IRP, ThreadListEntry);
 144     IOTRACE(IO_IRP_DEBUG,
 145             "%s - Deassociating IRP %p for %p\n",
 146             __FUNCTION__,
 147             IopDeadIrp,
 148             IrpThread);
 149
 150     /* Don't cancel the IRP if it's already been completed far */
 151     if (IopDeadIrp->CurrentLocation == (IopDeadIrp->StackCount + 2))
 152     {
 153         /* Return */
 154         KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 155         KeLowerIrql(OldIrql);
 156         return;
 157     }
 158
 159     /* Disown the IRP! */
 160     IopDeadIrp->Tail.Overlay.Thread = NULL;
 161     RemoveHeadList(&IrpThread->IrpList);
 162     InitializeListHead(&IopDeadIrp->ThreadListEntry);
 163
 164     /* Get the stack location and check if it's valid */
 165     IoStackLocation = IoGetCurrentIrpStackLocation(IopDeadIrp);
 166     if (IopDeadIrp->CurrentLocation <= IopDeadIrp->StackCount)
 167     {
 168         /* Get the device object */
 169         DeviceObject = IoStackLocation->DeviceObject;
 170     }
 171
 172     KeReleaseQueuedSpinLock(LockQueueIoCompletionLock, LockIrql);
 173     /* Lower IRQL now, since we have the pointers we need */
 174     KeLowerIrql(OldIrql);
 175
 176     /* Check if we can send an Error Log Entry*/
 177     if (DeviceObject)
 178     {
 179         /* Allocate an entry */
 180         ErrorLogEntry = IoAllocateErrorLogEntry(DeviceObject,
 181                                                 sizeof(IO_ERROR_LOG_PACKET));
 182         if (ErrorLogEntry)
 183         {
 184             /* Write the entry */
 185             ErrorLogEntry->ErrorCode = IO_DRIVER_CANCEL_TIMEOUT;
 186             IoWriteErrorLogEntry(ErrorLogEntry);
 187         }
 188     }
 189 }
 190
 191 VOID
 192 NTAPI
 193 IopCleanupIrp(IN PIRP Irp,
 194               IN PFILE_OBJECT FileObject)
 195 {
 196     PMDL Mdl;
 197     IOTRACE(IO_IRP_DEBUG,
 198             "%s - Cleaning IRP %p for %p\n",
 199             __FUNCTION__,
 200             Irp,
 201             FileObject);
 202
 203     /* Check if there's an MDL */
 204     while ((Mdl = Irp->MdlAddress))
 205     {
 206         /* Clear all of them */
 207         Irp->MdlAddress = Mdl->Next;
 208         IoFreeMdl(Mdl);
 209     }
 210
 211     /* Check if the IRP has system buffer */
 212     if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 213     {
 214         /* Free the buffer */
 215         ExFreePoolWithTag(Irp->AssociatedIrp.SystemBuffer, TAG_SYS_BUF);
 216     }
 217
 218     /* Check if this IRP has a user event, a file object, and is async */
 219     if ((Irp->UserEvent) &&
 220         !(Irp->Flags & IRP_SYNCHRONOUS_API) &&
 221         (FileObject))
 222     {
 223         /* Dereference the User Event */
 224         ObDereferenceObject(Irp->UserEvent);
 225     }
 226
 227     /* Check if we have a file object and this isn't a create operation */
 228     if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 229     {
 230         /* Dereference the file object */
 231         ObDereferenceObject(FileObject);
 232     }
 233
 234     /* Free the IRP */
 235     IoFreeIrp(Irp);
 236 }
 237
 238 VOID
 239 NTAPI
 240 IopCompleteRequest(IN PKAPC Apc,
 241                    IN PKNORMAL_ROUTINE* NormalRoutine,
 242                    IN PVOID* NormalContext,
 243                    IN PVOID* SystemArgument1,
 244                    IN PVOID* SystemArgument2)
 245 {
 246     PFILE_OBJECT FileObject;
 247     PIRP Irp;
 248     PMDL Mdl, NextMdl;
 249     PVOID Port = NULL, Key = NULL;
 250     BOOLEAN SignaledCreateRequest = FALSE;
 251
 252     /* Get data from the APC */
 253     FileObject = (PFILE_OBJECT)*SystemArgument1;
 254     Irp = CONTAINING_RECORD(Apc, IRP, Tail.Apc);
 255     IOTRACE(IO_IRP_DEBUG,
 256             "%s - Completing IRP %p for %p\n",
 257             __FUNCTION__,
 258             Irp,
 259             FileObject);
 260
 261     /* Sanity check */
 262     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
 263
 264     /* Check if we have a file object */
 265     if (*SystemArgument2)
 266     {
 267         /* Check if we're reparsing */
 268         if ((Irp->IoStatus.Status == STATUS_REPARSE) &&
 269             (Irp->IoStatus.Information == IO_REPARSE_TAG_MOUNT_POINT))
 270         {
 271             /* We should never get this yet */
 272             DPRINT1("Reparse support not yet present!\n");
 273             while (TRUE);
 274         }
 275     }
 276
 277     /* Handle Buffered case first */
 278     if (Irp->Flags & IRP_BUFFERED_IO)
 279     {
 280         /* Check if we have an input buffer and if we succeeded */
 281         if ((Irp->Flags & IRP_INPUT_OPERATION) &&
 282             (Irp->IoStatus.Status != STATUS_VERIFY_REQUIRED) &&
 283             !(NT_ERROR(Irp->IoStatus.Status)))
 284         {
 285             /* Copy the buffer back to the user */
 286             RtlCopyMemory(Irp->UserBuffer,
 287                           Irp->AssociatedIrp.SystemBuffer,
 288                           Irp->IoStatus.Information);
 289         }
 290
 291         /* Also check if we should de-allocate it */
 292         if (Irp->Flags & IRP_DEALLOCATE_BUFFER)
 293         {
 294             /* Deallocate it */
 295             ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 296         }
 297     }
 298
 299     /* Now we got rid of these two... */
 300     Irp->Flags &= ~(IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER);
 301
 302     /* Check if there's an MDL */
 303     for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
 304     {
 305         /* Free it */
 306         NextMdl = Mdl->Next;
 307         IoFreeMdl(Mdl);
 308     }
 309
 310     /* No MDLs left */
 311     Irp->MdlAddress = NULL;
 312
 313     /*
 314      * Check if either the request was completed without any errors
 315      * (but warnings are OK!), or if it was completed with an error, but
 316      * did return from a pending I/O Operation and is not synchronous.
 317      */
 318     if (!(NT_ERROR(Irp->IoStatus.Status)) ||
 319          (NT_ERROR(Irp->IoStatus.Status) &&
 320           (Irp->PendingReturned) &&
 321           !(IsIrpSynchronous(Irp, FileObject))))
 322     {
 323         /* Get any information we need from the FO before we kill it */
 324         if ((FileObject) && (FileObject->CompletionContext))
 325         {
 326             /* Save Completion Data */
 327             Port = FileObject->CompletionContext->Port;
 328             Key = FileObject->CompletionContext->Key;
 329         }
 330
 331         /* Use SEH to make sure we don't write somewhere invalid */
 332         _SEH2_TRY
 333         {
 334             /*  Save the IOSB Information */
 335             *Irp->UserIosb = Irp->IoStatus;
 336         }
 337         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 338         {
 339             /* Ignore any error */
 340         }
 341         _SEH2_END;
 342
 343         /* Check if we have an event or a file object */
 344         if (Irp->UserEvent)
 345         {
 346             /* At the very least, this is a PKEVENT, so signal it always */
 347             KeSetEvent(Irp->UserEvent, 0, FALSE);
 348
 349             /* Check if we also have a File Object */
 350             if (FileObject)
 351             {
 352                 /* Check if this is an Asynch API */
 353                 if (!(Irp->Flags & IRP_SYNCHRONOUS_API))
 354                 {
 355                     /* Dereference the event */
 356                     ObDereferenceObject(Irp->UserEvent);
 357                 }
 358
 359                 /*
 360                  * Now, if this is a Synch I/O File Object, then this event is
 361                  * NOT an actual Executive Event, so we won't dereference it,
 362                  * and instead, we will signal the File Object
 363                  */
 364                 if ((FileObject->Flags & FO_SYNCHRONOUS_IO) &&
 365                     !(Irp->Flags & IRP_OB_QUERY_NAME))
 366                 {
 367                     /* Signal the file object and set the status */
 368                     KeSetEvent(&FileObject->Event, 0, FALSE);
 369                     FileObject->FinalStatus = Irp->IoStatus.Status;
 370                 }
 371
 372                 /*
 373                  * This could also be a create operation, in which case we want
 374                  * to make sure there's no APC fired.
 375                  */
 376                 if (Irp->Flags & IRP_CREATE_OPERATION)
 377                 {
 378                     /* Clear the APC Routine and remember this */
 379                     Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 380                     SignaledCreateRequest = TRUE;
 381                 }
 382             }
 383         }
 384         else if (FileObject)
 385         {
 386             /* Signal the file object and set the status */
 387             KeSetEvent(&FileObject->Event, 0, FALSE);
 388             FileObject->FinalStatus = Irp->IoStatus.Status;
 389
 390             /*
 391             * This could also be a create operation, in which case we want
 392             * to make sure there's no APC fired.
 393             */
 394             if (Irp->Flags & IRP_CREATE_OPERATION)
 395             {
 396                 /* Clear the APC Routine and remember this */
 397                 Irp->Overlay.AsynchronousParameters.UserApcRoutine = NULL;
 398                 SignaledCreateRequest = TRUE;
 399             }
 400         }
 401
 402         /* Update transfer count for everything but create operation */
 403         if (!(Irp->Flags & IRP_CREATE_OPERATION))
 404         {
 405             if (Irp->Flags & IRP_WRITE_OPERATION)
 406             {
 407                 /* Update write transfer count */
 408                 IopUpdateTransferCount(IopWriteTransfer,
 409                                        (ULONG)Irp->IoStatus.Information);
 410             }
 411             else if (Irp->Flags & IRP_READ_OPERATION)
 412             {
 413                 /* Update read transfer count */
 414                 IopUpdateTransferCount(IopReadTransfer,
 415                                        (ULONG)Irp->IoStatus.Information);
 416             }
 417             else
 418             {
 419                 /* Update other transfer count */
 420                 IopUpdateTransferCount(IopOtherTransfer,
 421                                        (ULONG)Irp->IoStatus.Information);
 422             }
 423         }
 424
 425         /* Now that we've signaled the events, de-associate the IRP */
 426         IopUnQueueIrpFromThread(Irp);
 427
 428         /* Now check if a User APC Routine was requested */
 429         if (Irp->Overlay.AsynchronousParameters.UserApcRoutine)
 430         {
 431             /* Initialize it */
 432             KeInitializeApc(&Irp->Tail.Apc,
 433                             KeGetCurrentThread(),
 434                             CurrentApcEnvironment,
 435                             IopFreeIrpKernelApc,
 436                             IopAbortIrpKernelApc,
 437                             (PKNORMAL_ROUTINE)Irp->
 438                             Overlay.AsynchronousParameters.UserApcRoutine,
 439                             Irp->RequestorMode,
 440                             Irp->
 441                             Overlay.AsynchronousParameters.UserApcContext);
 442
 443             /* Queue it */
 444             KeInsertQueueApc(&Irp->Tail.Apc, Irp->UserIosb, NULL, 2);
 445         }
 446         else if ((Port) &&
 447                  (Irp->Overlay.AsynchronousParameters.UserApcContext))
 448         {
 449             /* We have an I/O Completion setup... create the special Overlay */
 450             Irp->Tail.CompletionKey = Key;
 451             Irp->Tail.Overlay.PacketType = IopCompletionPacketIrp;
 452             KeInsertQueue(Port, &Irp->Tail.Overlay.ListEntry);
 453         }
 454         else
 455         {
 456             /* Free the IRP since we don't need it anymore */
 457             IoFreeIrp(Irp);
 458         }
 459
 460         /* Check if we have a file object that wasn't part of a create */
 461         if ((FileObject) && !(SignaledCreateRequest))
 462         {
 463             /* Dereference it, since it's not needed anymore either */
 464             ObDereferenceObjectDeferDelete(FileObject);
 465         }
 466     }
 467     else
 468     {
 469         /*
 470          * Either we didn't return from the request, or we did return but this
 471          * request was synchronous.
 472          */
 473         if ((Irp->PendingReturned) && (FileObject))
 474         {
 475             /* So we did return with a synch operation, was it the IRP? */
 476             if (Irp->Flags & IRP_SYNCHRONOUS_API)
 477             {
 478                 /* Yes, this IRP was synchronous, so return the I/O Status */
 479                 *Irp->UserIosb = Irp->IoStatus;
 480
 481                 /* Now check if the user gave an event */
 482                 if (Irp->UserEvent)
 483                 {
 484                     /* Signal it */
 485                     KeSetEvent(Irp->UserEvent, 0, FALSE);
 486                 }
 487                 else
 488                 {
 489                     /* No event was given, so signal the FO instead */
 490                     KeSetEvent(&FileObject->Event, 0, FALSE);
 491                 }
 492             }
 493             else
 494             {
 495                 /*
 496                  * It's not the IRP that was synchronous, it was the FO
 497                  * that was opened this way. Signal its event.
 498                  */
 499                 FileObject->FinalStatus = Irp->IoStatus.Status;
 500                 KeSetEvent(&FileObject->Event, 0, FALSE);
 501             }
 502         }
 503
 504         /* Now that we got here, we do this for incomplete I/Os as well */
 505         if ((FileObject) && !(Irp->Flags & IRP_CREATE_OPERATION))
 506         {
 507             /* Dereference the File Object unless this was a create */
 508             ObDereferenceObjectDeferDelete(FileObject);
 509         }
 510
 511         /*
 512          * Check if this was an Executive Event (remember that we know this
 513          * by checking if the IRP is synchronous)
 514          */
 515         if ((Irp->UserEvent) &&
 516             (FileObject) &&
 517             !(Irp->Flags & IRP_SYNCHRONOUS_API))
 518         {
 519             /* This isn't a PKEVENT, so dereference it */
 520             ObDereferenceObject(Irp->UserEvent);
 521         }
 522
 523         /* Now that we've signaled the events, de-associate the IRP */
 524         IopUnQueueIrpFromThread(Irp);
 525
 526         /* Free the IRP as well */
 527         IoFreeIrp(Irp);
 528     }
 529 }
 530
 531 /* FUNCTIONS *****************************************************************/
 532
 533 /*
 534  * @implemented
 535  */
 536 PIRP
 537 NTAPI
 538 IoAllocateIrp(IN CCHAR StackSize,
 539               IN BOOLEAN ChargeQuota)
 540 {
 541     PIRP Irp = NULL;
 542     USHORT Size = IoSizeOfIrp(StackSize);
 543     PKPRCB Prcb;
 544     UCHAR Flags = 0;
 545     PNPAGED_LOOKASIDE_LIST List = NULL;
 546     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;
 547
 548     /* Set Charge Quota Flag */
 549     if (ChargeQuota) Flags |= IRP_QUOTA_CHARGED;
 550
 551     /* FIXME: Implement Lookaside Floats */
 552
 553     /* Figure out which Lookaside List to use */
 554     if ((StackSize <= 8) && (ChargeQuota == FALSE))
 555     {
 556         /* Set Fixed Size Flag */
 557         Flags = IRP_ALLOCATED_FIXED_SIZE;
 558
 559         /* See if we should use big list */
 560         if (StackSize != 1)
 561         {
 562             Size = IoSizeOfIrp(8);
 563             ListType = LookasideLargeIrpList;
 564         }
 565
 566         /* Get the PRCB */
 567         Prcb = KeGetCurrentPrcb();
 568
 569         /* Get the P List First */
 570         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
 571
 572         /* Attempt allocation */
 573         List->L.TotalAllocates++;
 574         Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 575
 576         /* Check if the P List failed */
 577         if (!Irp)
 578         {
 579             /* Let the balancer know */
 580             List->L.AllocateMisses++;
 581
 582             /* Try the L List */
 583             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
 584             List->L.TotalAllocates++;
 585             Irp = (PIRP)InterlockedPopEntrySList(&List->L.ListHead);
 586         }
 587     }
 588
 589     /* Check if we have to use the pool */
 590     if (!Irp)
 591     {
 592         /* Did we try lookaside and fail? */
 593         if (Flags & IRP_ALLOCATED_FIXED_SIZE) List->L.AllocateMisses++;
 594
 595         /* Check if we should charge quota */
 596         if (ChargeQuota)
 597         {
 598             /* Irp = ExAllocatePoolWithQuotaTag(NonPagedPool, Size, TAG_IRP); */
 599             /* FIXME */
 600             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 601         }
 602         else
 603         {
 604             /* Allocate the IRP With no Quota charge */
 605             Irp = ExAllocatePoolWithTag(NonPagedPool, Size, TAG_IRP);
 606         }
 607
 608         /* Make sure it was sucessful */
 609         if (!Irp) return(NULL);
 610     }
 611     else
 612     {
 613         /* In this case there is no charge quota */
 614         Flags &= ~IRP_QUOTA_CHARGED;
 615     }
 616
 617     /* Now Initialize it */
 618     IoInitializeIrp(Irp, Size, StackSize);
 619
 620     /* Set the Allocation Flags */
 621     Irp->AllocationFlags = Flags;
 622
 623     /* Return it */
 624     IOTRACE(IO_IRP_DEBUG,
 625             "%s - Allocated IRP %p with allocation flags %lx\n",
 626             __FUNCTION__,
 627             Irp,
 628             Flags);
 629     return Irp;
 630 }
 631
 632 /*
 633  * @implemented
 634  */
 635 PIRP
 636 NTAPI
 637 IoBuildAsynchronousFsdRequest(IN ULONG MajorFunction,
 638                               IN PDEVICE_OBJECT DeviceObject,
 639                               IN PVOID Buffer,
 640                               IN ULONG Length,
 641                               IN PLARGE_INTEGER StartingOffset,
 642                               IN PIO_STATUS_BLOCK IoStatusBlock)
 643 {
 644     PIRP Irp;
 645     PIO_STACK_LOCATION StackPtr;
 646
 647     /* Allocate IRP */
 648     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 649     if (!Irp) return NULL;
 650
 651     /* Get the Stack */
 652     StackPtr = IoGetNextIrpStackLocation(Irp);
 653
 654     /* Write the Major function and then deal with it */
 655     StackPtr->MajorFunction = (UCHAR)MajorFunction;
 656
 657     /* Do not handle the following here */
 658     if ((MajorFunction != IRP_MJ_FLUSH_BUFFERS) &&
 659         (MajorFunction != IRP_MJ_SHUTDOWN) &&
 660         (MajorFunction != IRP_MJ_PNP) &&
 661         (MajorFunction != IRP_MJ_POWER))
 662     {
 663         /* Check if this is Buffered IO */
 664         if (DeviceObject->Flags & DO_BUFFERED_IO)
 665         {
 666             /* Allocate the System Buffer */
 667             Irp->AssociatedIrp.SystemBuffer =
 668                 ExAllocatePoolWithTag(NonPagedPool, Length, TAG_SYS_BUF);
 669             if (!Irp->AssociatedIrp.SystemBuffer)
 670             {
 671                 /* Free the IRP and fail */
 672                 IoFreeIrp(Irp);
 673                 return NULL;
 674             }
 675
 676             /* Set flags */
 677             Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 678
 679             /* Handle special IRP_MJ_WRITE Case */
 680             if (MajorFunction == IRP_MJ_WRITE)
 681             {
 682                 /* Copy the buffer data */
 683                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer, Buffer, Length);
 684             }
 685             else
 686             {
 687                 /* Set the Input Operation flag and set this as a User Buffer */
 688                 Irp->Flags |= IRP_INPUT_OPERATION;
 689                 Irp->UserBuffer = Buffer;
 690             }
 691         }
 692         else if (DeviceObject->Flags & DO_DIRECT_IO)
 693         {
 694             /* Use an MDL for Direct I/O */
 695             Irp->MdlAddress = IoAllocateMdl(Buffer,
 696                                             Length,
 697                                             FALSE,
 698                                             FALSE,
 699                                             NULL);
 700             if (!Irp->MdlAddress)
 701             {
 702                 /* Free the IRP and fail */
 703                 IoFreeIrp(Irp);
 704                 return NULL;
 705             }
 706
 707                         /* Probe and Lock */
 708                         _SEH2_TRY
 709                         {
 710                                 /* Do the probe */
 711                                 MmProbeAndLockPages(Irp->MdlAddress,
 712                                                                         KernelMode,
 713                                                                         MajorFunction == IRP_MJ_READ ?
 714                                                                         IoWriteAccess : IoReadAccess);
 715                         }
 716                         _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 717                         {
 718                                 /* Free the IRP and its MDL */
 719                                 IoFreeMdl(Irp->MdlAddress);
 720                                 IoFreeIrp(Irp);
 721
 722                 /* Fail */
 723                                 _SEH2_YIELD(return NULL);
 724                         }
 725                         _SEH2_END;
 726         }
 727         else
 728         {
 729             /* Neither, use the buffer */
 730             Irp->UserBuffer = Buffer;
 731         }
 732
 733         /* Check if this is a read */
 734         if (MajorFunction == IRP_MJ_READ)
 735         {
 736             /* Set the parameters for a read */
 737             StackPtr->Parameters.Read.Length = Length;
 738             StackPtr->Parameters.Read.ByteOffset = *StartingOffset;
 739         }
 740         else if (MajorFunction == IRP_MJ_WRITE)
 741         {
 742             /* Otherwise, set write parameters */
 743             StackPtr->Parameters.Write.Length = Length;
 744             StackPtr->Parameters.Write.ByteOffset = *StartingOffset;
 745         }
 746     }
 747
 748     /* Set the Current Thread and IOSB */
 749     Irp->UserIosb = IoStatusBlock;
 750     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 751
 752     /* Return the IRP */
 753     IOTRACE(IO_IRP_DEBUG,
 754             "%s - Built IRP %p with Major, Buffer, DO %lx %p %p\n",
 755             __FUNCTION__,
 756             Irp,
 757             MajorFunction,
 758             Buffer,
 759             DeviceObject);
 760     return Irp;
 761 }
 762
 763 /*
 764  * @implemented
 765  */
 766 PIRP
 767 NTAPI
 768 IoBuildDeviceIoControlRequest(IN ULONG IoControlCode,
 769                               IN PDEVICE_OBJECT DeviceObject,
 770                               IN PVOID InputBuffer,
 771                               IN ULONG InputBufferLength,
 772                               IN PVOID OutputBuffer,
 773                               IN ULONG OutputBufferLength,
 774                               IN BOOLEAN InternalDeviceIoControl,
 775                               IN PKEVENT Event,
 776                               IN PIO_STATUS_BLOCK IoStatusBlock)
 777 {
 778     PIRP Irp;
 779     PIO_STACK_LOCATION StackPtr;
 780     ULONG BufferLength;
 781
 782     /* Allocate IRP */
 783     Irp = IoAllocateIrp(DeviceObject->StackSize, FALSE);
 784     if (!Irp) return NULL;
 785
 786     /* Get the Stack */
 787     StackPtr = IoGetNextIrpStackLocation(Irp);
 788
 789     /* Set the DevCtl Type */
 790     StackPtr->MajorFunction = InternalDeviceIoControl ?
 791                               IRP_MJ_INTERNAL_DEVICE_CONTROL :
 792                               IRP_MJ_DEVICE_CONTROL;
 793
 794     /* Set the IOCTL Data */
 795     StackPtr->Parameters.DeviceIoControl.IoControlCode = IoControlCode;
 796     StackPtr->Parameters.DeviceIoControl.InputBufferLength = InputBufferLength;
 797     StackPtr->Parameters.DeviceIoControl.OutputBufferLength =
 798         OutputBufferLength;
 799
 800     /* Handle the Methods */
 801     switch (IO_METHOD_FROM_CTL_CODE(IoControlCode))
 802     {
 803         /* Buffered I/O */
 804         case METHOD_BUFFERED:
 805
 806             /* Select the right Buffer Length */
 807             BufferLength = InputBufferLength > OutputBufferLength ?
 808                            InputBufferLength : OutputBufferLength;
 809
 810             /* Make sure there is one */
 811             if (BufferLength)
 812             {
 813                 /* Allocate the System Buffer */
 814                 Irp->AssociatedIrp.SystemBuffer =
 815                     ExAllocatePoolWithTag(NonPagedPool,
 816                                           BufferLength,
 817                                           TAG_SYS_BUF);
 818                 if (!Irp->AssociatedIrp.SystemBuffer)
 819                 {
 820                     /* Free the IRP and fail */
 821                     IoFreeIrp(Irp);
 822                     return NULL;
 823                 }
 824
 825                 /* Check if we got a buffer */
 826                 if (InputBuffer)
 827                 {
 828                     /* Copy into the System Buffer */
 829                     RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 830                                   InputBuffer,
 831                                   InputBufferLength);
 832                 }
 833
 834                 /* Write the flags */
 835                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 836                 if (OutputBuffer) Irp->Flags |= IRP_INPUT_OPERATION;
 837
 838                 /* Save the Buffer */
 839                 Irp->UserBuffer = OutputBuffer;
 840             }
 841             else
 842             {
 843                 /* Clear the Flags and Buffer */
 844                 Irp->Flags = 0;
 845                 Irp->UserBuffer = NULL;
 846             }
 847             break;
 848
 849         /* Direct I/O */
 850         case METHOD_IN_DIRECT:
 851         case METHOD_OUT_DIRECT:
 852
 853             /* Check if we got an input buffer */
 854             if (InputBuffer)
 855             {
 856                 /* Allocate the System Buffer */
 857                 Irp->AssociatedIrp.SystemBuffer =
 858                     ExAllocatePoolWithTag(NonPagedPool,
 859                                           InputBufferLength,
 860                                           TAG_SYS_BUF);
 861                 if (!Irp->AssociatedIrp.SystemBuffer)
 862                 {
 863                     /* Free the IRP and fail */
 864                     IoFreeIrp(Irp);
 865                     return NULL;
 866                 }
 867
 868                 /* Copy into the System Buffer */
 869                 RtlCopyMemory(Irp->AssociatedIrp.SystemBuffer,
 870                               InputBuffer,
 871                               InputBufferLength);
 872
 873                 /* Write the flags */
 874                 Irp->Flags = IRP_BUFFERED_IO | IRP_DEALLOCATE_BUFFER;
 875             }
 876             else
 877             {
 878                 /* Clear the flags */
 879                 Irp->Flags = 0;
 880             }
 881
 882             /* Check if we got an output buffer */
 883             if (OutputBuffer)
 884             {
 885                 /* Allocate the System Buffer */
 886                 Irp->MdlAddress = IoAllocateMdl(OutputBuffer,
 887                                                 OutputBufferLength,
 888                                                 FALSE,
 889                                                 FALSE,
 890                                                 Irp);
 891                 if (!Irp->MdlAddress)
 892                 {
 893                     /* Free the IRP and fail */
 894                     IoFreeIrp(Irp);
 895                     return NULL;
 896                 }
 897
 898                 /* Probe and Lock */
 899                 _SEH2_TRY
 900                 {
 901                     /* Do the probe */
 902                     MmProbeAndLockPages(Irp->MdlAddress,
 903                                         KernelMode,
 904                                         IO_METHOD_FROM_CTL_CODE(IoControlCode) ==
 905                                         METHOD_IN_DIRECT ?
 906                                         IoReadAccess : IoWriteAccess);
 907                 }
 908                 _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER)
 909                 {
 910                     /* Free the MDL */
 911                     IoFreeMdl(Irp->MdlAddress);
 912
 913                     /* Free the input buffer and IRP */
 914                     if (InputBuffer) ExFreePool(Irp->AssociatedIrp.SystemBuffer);
 915                     IoFreeIrp(Irp);
 916
 917                     /* Fail */
 918                     _SEH2_YIELD(return NULL);
 919                 }
 920                 _SEH2_END;
 921             }
 922             break;
 923
 924         case METHOD_NEITHER:
 925
 926             /* Just save the Buffer */
 927             Irp->UserBuffer = OutputBuffer;
 928             StackPtr->Parameters.DeviceIoControl.Type3InputBuffer = InputBuffer;
 929     }
 930
 931     /* Now write the Event and IoSB */
 932     Irp->UserIosb = IoStatusBlock;
 933     Irp->UserEvent = Event;
 934
 935     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 936     Irp->Tail.Overlay.Thread = PsGetCurrentThread();
 937     IoQueueThreadIrp(Irp);
 938
 939     /* Return the IRP */
 940     IOTRACE(IO_IRP_DEBUG,
 941             "%s - Built IRP %p with IOCTL, Buffers, DO %lx %p %p %p\n",
 942             __FUNCTION__,
 943             Irp,
 944             IoControlCode,
 945             InputBuffer,
 946             OutputBuffer,
 947             DeviceObject);
 948     return Irp;
 949 }
 950
 951 /*
 952  * @implemented
 953  */
 954 PIRP
 955 NTAPI
 956 IoBuildSynchronousFsdRequest(IN ULONG MajorFunction,
 957                              IN PDEVICE_OBJECT DeviceObject,
 958                              IN PVOID Buffer,
 959                              IN ULONG Length,
 960                              IN PLARGE_INTEGER StartingOffset,
 961                              IN PKEVENT Event,
 962                              IN PIO_STATUS_BLOCK IoStatusBlock)
 963 {
 964     PIRP Irp;
 965
 966     /* Do the big work to set up the IRP */
 967     Irp = IoBuildAsynchronousFsdRequest(MajorFunction,
 968                                         DeviceObject,
 969                                         Buffer,
 970                                         Length,
 971                                         StartingOffset,
 972                                         IoStatusBlock );
 973     if (!Irp) return NULL;
 974
 975     /* Set the Event which makes it Syncronous */
 976     Irp->UserEvent = Event;
 977
 978     /* Sync IRPs are queued to requestor thread's irp cancel/cleanup list */
 979     IoQueueThreadIrp(Irp);
 980     return Irp;
 981 }
 982
 983 /*
 984  * @implemented
 985  */
 986 BOOLEAN
 987 NTAPI
 988 IoCancelIrp(IN PIRP Irp)
 989 {
 990     KIRQL OldIrql;
 991     PDRIVER_CANCEL CancelRoutine;
 992     IOTRACE(IO_IRP_DEBUG,
 993             "%s - Canceling IRP %p\n",
 994             __FUNCTION__,
 995             Irp);
 996     ASSERT(Irp->Type == IO_TYPE_IRP);
 997
 998     /* Acquire the cancel lock and cancel the IRP */
 999     IoAcquireCancelSpinLock(&OldIrql);
1000     Irp->Cancel = TRUE;
1001
1002     /* Clear the cancel routine and get the old one */
1003     CancelRoutine = (PVOID)IoSetCancelRoutine(Irp, NULL);
1004     if (CancelRoutine)
1005     {
1006         /* We had a routine, make sure the IRP isn't completed */
1007         if (Irp->CurrentLocation > (Irp->StackCount + 1))
1008         {
1009             /* It is, bugcheck */
1010             KeBugCheckEx(CANCEL_STATE_IN_COMPLETED_IRP,
1011                          (ULONG_PTR)Irp,
1012                          (ULONG_PTR)CancelRoutine,
1013                          0,
1014                          0);
1015         }
1016
1017         /* Set the cancel IRQL And call the routine */
1018         Irp->CancelIrql = OldIrql;
1019         CancelRoutine(IoGetCurrentIrpStackLocation(Irp)->DeviceObject, Irp);
1020         return TRUE;
1021     }
1022
1023     /* Otherwise, release the cancel lock and fail */
1024     IoReleaseCancelSpinLock(OldIrql);
1025     return FALSE;
1026 }
1027
1028 /*
1029  * @implemented
1030  */
1031 VOID
1032 NTAPI
1033 IoCancelThreadIo(IN PETHREAD Thread)
1034 {
1035     KIRQL OldIrql;
1036     ULONG Retries = 3000;
1037     LARGE_INTEGER Interval;
1038     PLIST_ENTRY ListHead, NextEntry;
1039     PIRP Irp;
1040     PAGED_CODE();
1041
1042     /* Windows isn't using given thread, but using current. */
1043     Thread = PsGetCurrentThread();
1044
1045     IOTRACE(IO_IRP_DEBUG,
1046             "%s - Canceling IRPs for Thread %p\n",
1047             __FUNCTION__,
1048             Thread);
1049
1050     /* Raise to APC to protect the IrpList */
1051     KeRaiseIrql(APC_LEVEL, &OldIrql);
1052
1053     /* Start by cancelling all the IRPs in the current thread queue. */
1054     ListHead = &Thread->IrpList;
1055     NextEntry = ListHead->Flink;
1056     while (ListHead != NextEntry)
1057     {
1058         /* Get the IRP */
1059         Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
1060
1061         /* Cancel it */
1062         IoCancelIrp(Irp);
1063
1064         /* Move to the next entry */
1065         NextEntry = NextEntry->Flink;
1066     }
1067
1068      /* Wait 100 milliseconds */
1069     Interval.QuadPart = -1000000;
1070
1071     /* Wait till all the IRPs are completed or cancelled. */
1072     while (!IsListEmpty(&Thread->IrpList))
1073     {
1074         /* Now we can lower */
1075         KeLowerIrql(OldIrql);
1076
1077         /* Wait a short while and then look if all our IRPs were completed. */
1078         KeDelayExecutionThread(KernelMode, FALSE, &Interval);
1079
1080         /*
1081          * Don't stay here forever if some broken driver doesn't complete
1082          * the IRP.
1083          */
1084         if (!(Retries--))
1085         {
1086             /* Print out a message and remove the IRP */
1087             DPRINT1("Broken driver did not complete!\n");
1088             IopDisassociateThreadIrp();
1089         }
1090
1091         /* Raise the IRQL Again */
1092         KeRaiseIrql(APC_LEVEL, &OldIrql);
1093     }
1094
1095     /* We're done, lower the IRQL */
1096     KeLowerIrql(OldIrql);
1097 }
1098
1099 /*
1100  * @implemented
1101  */
1102 NTSTATUS
1103 NTAPI
1104 IoCallDriver(IN PDEVICE_OBJECT DeviceObject,
1105              IN PIRP Irp)
1106 {
1107     /* Call fastcall */
1108     return IofCallDriver(DeviceObject, Irp);
1109 }
1110
1111 /*
1112  * @implemented
1113  */
1114 VOID
1115 NTAPI
1116 IoCompleteRequest(IN PIRP Irp,
1117                   IN CCHAR PriorityBoost)
1118 {
1119     /* Call the fastcall */
1120     IofCompleteRequest(Irp, PriorityBoost);
1121 }
1122
1123 /*
1124  * @implemented
1125  */
1126 VOID
1127 NTAPI
1128 IoEnqueueIrp(IN PIRP Irp)
1129 {
1130     /* This is the same as calling IoQueueThreadIrp */
1131     IoQueueThreadIrp(Irp);
1132 }
1133
1134 /*
1135  * @implemented
1136  */
1137 NTSTATUS
1138 FASTCALL
1139 IofCallDriver(IN PDEVICE_OBJECT DeviceObject,
1140               IN PIRP Irp)
1141 {
1142     PDRIVER_OBJECT DriverObject;
1143     PIO_STACK_LOCATION StackPtr;
1144
1145     /* Make sure this is a valid IRP */
1146     ASSERT(Irp->Type == IO_TYPE_IRP);
1147
1148     /* Get the Driver Object */
1149     DriverObject = DeviceObject->DriverObject;
1150
1151     /* Decrease the current location and check if */
1152     Irp->CurrentLocation--;
1153     if (Irp->CurrentLocation <= 0)
1154     {
1155         /* This IRP ran out of stack, bugcheck */
1156         KeBugCheckEx(NO_MORE_IRP_STACK_LOCATIONS, (ULONG_PTR)Irp, 0, 0, 0);
1157     }
1158
1159     /* Now update the stack location */
1160     StackPtr = IoGetNextIrpStackLocation(Irp);
1161     Irp->Tail.Overlay.CurrentStackLocation = StackPtr;
1162
1163     /* Get the Device Object */
1164     StackPtr->DeviceObject = DeviceObject;
1165
1166     /* Call it */
1167     return DriverObject->MajorFunction[StackPtr->MajorFunction](DeviceObject,
1168                                                                 Irp);
1169 }
1170
1171 FORCEINLINE
1172 VOID
1173 IopClearStackLocation(IN PIO_STACK_LOCATION IoStackLocation)
1174 {
1175     IoStackLocation->MinorFunction = 0;
1176     IoStackLocation->Flags = 0;
1177     IoStackLocation->Control &= SL_ERROR_RETURNED;
1178     IoStackLocation->Parameters.Others.Argument1 = 0;
1179     IoStackLocation->Parameters.Others.Argument2 = 0;
1180     IoStackLocation->Parameters.Others.Argument3 = 0;
1181     IoStackLocation->FileObject = NULL;
1182 }
1183
1184 /*
1185  * @implemented
1186  */
1187 VOID
1188 FASTCALL
1189 IofCompleteRequest(IN PIRP Irp,
1190                    IN CCHAR PriorityBoost)
1191 {
1192     PIO_STACK_LOCATION StackPtr, LastStackPtr;
1193     PDEVICE_OBJECT DeviceObject;
1194     PFILE_OBJECT FileObject;
1195     PETHREAD Thread;
1196     NTSTATUS Status;
1197     PMDL Mdl, NextMdl;
1198     ULONG MasterCount;
1199     PIRP MasterIrp;
1200     ULONG Flags;
1201     NTSTATUS ErrorCode = STATUS_SUCCESS;
1202     IOTRACE(IO_IRP_DEBUG,
1203             "%s - Completing IRP %p\n",
1204             __FUNCTION__,
1205             Irp);
1206
1207     /* Make sure this IRP isn't getting completed twice or is invalid */
1208     if ((Irp->CurrentLocation) > (Irp->StackCount + 1))
1209     {
1210         /* Bugcheck */
1211         KeBugCheckEx(MULTIPLE_IRP_COMPLETE_REQUESTS, (ULONG_PTR)Irp, 0, 0, 0);
1212     }
1213
1214     /* Some sanity checks */
1215     ASSERT(Irp->Type == IO_TYPE_IRP);
1216     ASSERT(!Irp->CancelRoutine);
1217     ASSERT(Irp->IoStatus.Status != STATUS_PENDING);
1218     ASSERT(Irp->IoStatus.Status != (NTSTATUS)0xFFFFFFFF);
1219
1220     /* Get the last stack */
1221     LastStackPtr = (PIO_STACK_LOCATION)(Irp + 1);
1222     if (LastStackPtr->Control & SL_ERROR_RETURNED)
1223     {
1224         /* Get the error code */
1225         ErrorCode = PtrToUlong(LastStackPtr->Parameters.Others.Argument4);
1226     }
1227
1228     /*
1229      * Start the loop with the current stack and point the IRP to the next stack
1230      * and then keep incrementing the stack as we loop through. The IRP should
1231      * always point to the next stack location w.r.t the one currently being
1232      * analyzed, so completion routine code will see the appropriate value.
1233      * Because of this, we must loop until the current stack location is +1 of
1234      * the stack count, because when StackPtr is at the end, CurrentLocation is +1.
1235      */
1236     for (StackPtr = IoGetCurrentIrpStackLocation(Irp),
1237          Irp->CurrentLocation++,
1238          Irp->Tail.Overlay.CurrentStackLocation++;
1239          Irp->CurrentLocation <= (Irp->StackCount + 1);
1240          StackPtr++,
1241          Irp->CurrentLocation++,
1242          Irp->Tail.Overlay.CurrentStackLocation++)
1243     {
1244         /* Set Pending Returned */
1245         Irp->PendingReturned = StackPtr->Control & SL_PENDING_RETURNED;
1246
1247         /* Check if we failed */
1248         if (!NT_SUCCESS(Irp->IoStatus.Status))
1249         {
1250             /* Check if it was changed by a completion routine */
1251             if (Irp->IoStatus.Status != ErrorCode)
1252             {
1253                 /* Update the error for the current stack */
1254                 ErrorCode = Irp->IoStatus.Status;
1255                 StackPtr->Control |= SL_ERROR_RETURNED;
1256                 LastStackPtr->Parameters.Others.Argument4 = UlongToPtr(ErrorCode);
1257                 LastStackPtr->Control |= SL_ERROR_RETURNED;
1258             }
1259         }
1260
1261         /* Check if there is a Completion Routine to Call */
1262         if ((NT_SUCCESS(Irp->IoStatus.Status) &&
1263              (StackPtr->Control & SL_INVOKE_ON_SUCCESS)) ||
1264             (!NT_SUCCESS(Irp->IoStatus.Status) &&
1265              (StackPtr->Control & SL_INVOKE_ON_ERROR)) ||
1266             (Irp->Cancel &&
1267              (StackPtr->Control & SL_INVOKE_ON_CANCEL)))
1268         {
1269             /* Clear the stack location */
1270             IopClearStackLocation(StackPtr);
1271
1272             /* Check for highest-level device completion routines */
1273             if (Irp->CurrentLocation == (Irp->StackCount + 1))
1274             {
1275                 /* Clear the DO, since the current stack location is invalid */
1276                 DeviceObject = NULL;
1277             }
1278             else
1279             {
1280                 /* Otherwise, return the real one */
1281                 DeviceObject = IoGetCurrentIrpStackLocation(Irp)->DeviceObject;
1282             }
1283
1284             /* Call the completion routine */
1285             Status = StackPtr->CompletionRoutine(DeviceObject,
1286                                                  Irp,
1287                                                  StackPtr->Context);
1288
1289             /* Don't touch the Packet in this case, since it might be gone! */
1290             if (Status == STATUS_MORE_PROCESSING_REQUIRED) return;
1291         }
1292         else
1293         {
1294             /* Otherwise, check if this is a completed IRP */
1295             if ((Irp->CurrentLocation <= Irp->StackCount) &&
1296                 (Irp->PendingReturned))
1297             {
1298                 /* Mark it as pending */
1299                 IoMarkIrpPending(Irp);
1300             }
1301
1302             /* Clear the stack location */
1303             IopClearStackLocation(StackPtr);
1304         }
1305     }
1306
1307     /* Check if the IRP is an associated IRP */
1308     if (Irp->Flags & IRP_ASSOCIATED_IRP)
1309     {
1310         /* Get the master IRP and count */
1311         MasterIrp = Irp->AssociatedIrp.MasterIrp;
1312         MasterCount = InterlockedDecrement(&MasterIrp->AssociatedIrp.IrpCount);
1313
1314         /* Free the MDLs */
1315         for (Mdl = Irp->MdlAddress; Mdl; Mdl = NextMdl)
1316         {
1317             /* Go to the next one */
1318             NextMdl = Mdl->Next;
1319             IoFreeMdl(Mdl);
1320         }
1321
1322         /* Free the IRP itself */
1323         IoFreeIrp(Irp);
1324
1325         /* Complete the Master IRP */
1326         if (!MasterCount) IofCompleteRequest(MasterIrp, PriorityBoost);
1327         return;
1328     }
1329
1330     /* We don't support this yet */
1331     ASSERT(Irp->IoStatus.Status != STATUS_REPARSE);
1332
1333     /* Check if we have an auxiliary buffer */
1334     if (Irp->Tail.Overlay.AuxiliaryBuffer)
1335     {
1336         /* Free it */
1337         ExFreePool(Irp->Tail.Overlay.AuxiliaryBuffer);
1338         Irp->Tail.Overlay.AuxiliaryBuffer = NULL;
1339     }
1340
1341     /* Check if this is a Paging I/O or Close Operation */
1342     if (Irp->Flags & (IRP_PAGING_IO | IRP_CLOSE_OPERATION))
1343     {
1344         /* Handle a Close Operation or Sync Paging I/O */
1345         if (Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_CLOSE_OPERATION))
1346         {
1347             /* Set the I/O Status and Signal the Event */
1348             Flags = Irp->Flags & (IRP_SYNCHRONOUS_PAGING_IO | IRP_PAGING_IO);
1349             *Irp->UserIosb = Irp->IoStatus;
1350             KeSetEvent(Irp->UserEvent, PriorityBoost, FALSE);
1351
1352             /* Free the IRP for a Paging I/O Only, Close is handled by us */
1353             if (Flags) IoFreeIrp(Irp);
1354         }
1355         else
1356         {
1357 #if 0
1358             /* Page 166 */
1359             KeInitializeApc(&Irp->Tail.Apc
1360                             &Irp->Tail.Overlay.Thread->Tcb,
1361                             Irp->ApcEnvironment,
1362                             IopCompletePageWrite,
1363                             NULL,
1364                             NULL,
1365                             KernelMode,
1366                             NULL);
1367             KeInsertQueueApc(&Irp->Tail.Apc,
1368                              NULL,
1369                              NULL,
1370                              PriorityBoost);
1371 #else
1372             /* Not implemented yet. */
1373             DPRINT1("Not supported!\n");
1374             while (TRUE);
1375 #endif
1376         }
1377
1378         /* Get out of here */
1379         return;
1380     }
1381
1382     /* Unlock MDL Pages, page 167. */
1383     Mdl = Irp->MdlAddress;
1384     while (Mdl)
1385     {
1386                 MmUnlockPages(Mdl);
1387         Mdl = Mdl->Next;
1388     }
1389
1390     /* Check if we should exit because of a Deferred I/O (page 168) */
1391     if ((Irp->Flags & IRP_DEFER_IO_COMPLETION) && !(Irp->PendingReturned))
1392     {
1393         /*
1394          * Return without queuing the completion APC, since the caller will
1395          * take care of doing its own optimized completion at PASSIVE_LEVEL.
1396          */
1397         return;
1398     }
1399
1400     /* Get the thread and file object */
1401     Thread = Irp->Tail.Overlay.Thread;
1402     FileObject = Irp->Tail.Overlay.OriginalFileObject;
1403
1404     /* Make sure the IRP isn't canceled */
1405     if (!Irp->Cancel)
1406     {
1407         /* Initialize the APC */
1408         KeInitializeApc(&Irp->Tail.Apc,
1409                         &Thread->Tcb,
1410                         Irp->ApcEnvironment,
1411                         IopCompleteRequest,
1412                         NULL,
1413                         NULL,
1414                         KernelMode,
1415                         NULL);
1416
1417         /* Queue it */
1418         KeInsertQueueApc(&Irp->Tail.Apc,
1419                          FileObject,
1420                          NULL, /* This is used for REPARSE stuff */
1421                          PriorityBoost);
1422     }
1423     else
1424     {
1425         /* The IRP just got canceled... does a thread still own it? */
1426         if (Thread)
1427         {
1428             /* Yes! There is still hope! Initialize the APC */
1429             KeInitializeApc(&Irp->Tail.Apc,
1430                             &Thread->Tcb,
1431                             Irp->ApcEnvironment,
1432                             IopCompleteRequest,
1433                             NULL,
1434                             NULL,
1435                             KernelMode,
1436                             NULL);
1437
1438             /* Queue it */
1439             KeInsertQueueApc(&Irp->Tail.Apc,
1440                              FileObject,
1441                              NULL, /* This is used for REPARSE stuff */
1442                              PriorityBoost);
1443         }
1444         else
1445         {
1446             /* Nothing left for us to do, kill it */
1447             ASSERT(Irp->Cancel);
1448             IopCleanupIrp(Irp, FileObject);
1449         }
1450     }
1451 }
1452
1453 NTSTATUS
1454 NTAPI
1455 IopSynchronousCompletion(IN PDEVICE_OBJECT DeviceObject,
1456                          IN PIRP Irp,
1457                          IN PVOID Context)
1458 {
1459     if (Irp->PendingReturned)
1460         KeSetEvent((PKEVENT)Context, IO_NO_INCREMENT, FALSE);
1461     return STATUS_MORE_PROCESSING_REQUIRED;
1462 }
1463
1464 /*
1465  * @implemented
1466  */
1467 BOOLEAN
1468 NTAPI
1469 IoForwardIrpSynchronously(IN PDEVICE_OBJECT DeviceObject,
1470                           IN PIRP Irp)
1471 {
1472     KEVENT Event;
1473     NTSTATUS Status;
1474
1475     /* Check if next stack location is available */
1476     if (Irp->CurrentLocation < Irp->StackCount)
1477     {
1478         /* No more stack location */
1479         return FALSE;
1480     }
1481
1482     /* Initialize event */
1483     KeInitializeEvent(&Event, NotificationEvent, FALSE);
1484
1485     /* Copy stack location for next driver */
1486     IoCopyCurrentIrpStackLocationToNext(Irp);
1487
1488     /* Set a completion routine, which will signal the event */
1489     IoSetCompletionRoutine(Irp, IopSynchronousCompletion, &Event, TRUE, TRUE, TRUE);
1490
1491     /* Call next driver */
1492     Status = IoCallDriver(DeviceObject, Irp);
1493
1494     /* Check if irp is pending */
1495     if (Status == STATUS_PENDING)
1496     {
1497         /* Yes, wait for its completion */
1498         KeWaitForSingleObject(&Event, Suspended, KernelMode, FALSE, NULL);
1499     }
1500
1501     /* Return success */
1502     return TRUE;
1503 }
1504
1505 /*
1506  * @implemented
1507  */
1508 VOID
1509 NTAPI
1510 IoFreeIrp(IN PIRP Irp)
1511 {
1512     PNPAGED_LOOKASIDE_LIST List;
1513     PP_NPAGED_LOOKASIDE_NUMBER ListType =  LookasideSmallIrpList;
1514     PKPRCB Prcb;
1515     IOTRACE(IO_IRP_DEBUG,
1516             "%s - Freeing IRPs %p\n",
1517             __FUNCTION__,
1518             Irp);
1519
1520     /* Make sure the Thread IRP list is empty and that it OK to free it */
1521     ASSERT(Irp->Type == IO_TYPE_IRP);
1522     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1523     ASSERT(Irp->CurrentLocation >= Irp->StackCount);
1524
1525     /* If this was a pool alloc, free it with the pool */
1526     if (!(Irp->AllocationFlags & IRP_ALLOCATED_FIXED_SIZE))
1527     {
1528         /* Free it */
1529         ExFreePoolWithTag(Irp, TAG_IRP);
1530     }
1531     else
1532     {
1533         /* Check if this was a Big IRP */
1534         if (Irp->StackCount != 1) ListType = LookasideLargeIrpList;
1535
1536         /* Get the PRCB */
1537         Prcb = KeGetCurrentPrcb();
1538
1539         /* Use the P List */
1540         List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].P;
1541         List->L.TotalFrees++;
1542
1543         /* Check if the Free was within the Depth or not */
1544         if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1545         {
1546             /* Let the balancer know */
1547             List->L.FreeMisses++;
1548
1549             /* Use the L List */
1550             List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[ListType].L;
1551             List->L.TotalFrees++;
1552
1553             /* Check if the Free was within the Depth or not */
1554             if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
1555             {
1556                 /* All lists failed, use the pool */
1557                 List->L.FreeMisses++;
1558                 ExFreePoolWithTag(Irp, TAG_IRP);
1559                 Irp = NULL;
1560             }
1561         }
1562
1563         /* The free was within the Depth */
1564         if (Irp)
1565         {
1566            InterlockedPushEntrySList(&List->L.ListHead,
1567                                      (PSLIST_ENTRY)Irp);
1568         }
1569     }
1570 }
1571
1572 /*
1573  * @implemented
1574  */
1575 IO_PAGING_PRIORITY
1576 FASTCALL
1577 IoGetPagingIoPriority(IN PIRP Irp)
1578 {
1579     IO_PAGING_PRIORITY Priority;
1580     ULONG Flags;
1581
1582     /* Get the flags */
1583     Flags = Irp->Flags;
1584
1585     /* Check what priority it has */
1586     if (Flags & IRP_CLASS_CACHE_OPERATION)
1587     {
1588         /* High priority */
1589         Priority = IoPagingPriorityHigh;
1590     }
1591     else if (Flags & IRP_PAGING_IO)
1592     {
1593         /* Normal priority */
1594         Priority = IoPagingPriorityNormal;
1595     }
1596     else
1597     {
1598         /* Invalid -- not a paging IRP */
1599         Priority = IoPagingPriorityInvalid;
1600     }
1601
1602     /* Return the priority */
1603     return Priority;
1604 }
1605
1606 /*
1607  * @implemented
1608  */
1609 PEPROCESS
1610 NTAPI
1611 IoGetRequestorProcess(IN PIRP Irp)
1612 {
1613     /* Return the requestor process */
1614     if (Irp->Tail.Overlay.Thread)
1615     {
1616         return Irp->Tail.Overlay.Thread->ThreadsProcess;
1617     }
1618
1619     return NULL;
1620 }
1621
1622 /*
1623  * @implemented
1624  */
1625 ULONG
1626 NTAPI
1627 IoGetRequestorProcessId(IN PIRP Irp)
1628 {
1629     PEPROCESS Process;
1630
1631     /* Return the requestor process' id */
1632     if ((Process = IoGetRequestorProcess(Irp)))
1633     {
1634         return PtrToUlong(Process->UniqueProcessId);
1635     }
1636
1637     return 0;
1638 }
1639
1640 /*
1641  * @implemented
1642  */
1643 NTSTATUS
1644 NTAPI
1645 IoGetRequestorSessionId(IN PIRP Irp,
1646                         OUT PULONG pSessionId)
1647 {
1648     PEPROCESS Process;
1649
1650     /* Return the session */
1651     if ((Process = IoGetRequestorProcess(Irp)))
1652     {
1653         *pSessionId = Process->Session;
1654         return STATUS_SUCCESS;
1655     }
1656
1657     *pSessionId = (ULONG)-1;
1658     return STATUS_UNSUCCESSFUL;
1659 }
1660
1661 /*
1662  * @implemented
1663  */
1664 PIRP
1665 NTAPI
1666 IoGetTopLevelIrp(VOID)
1667 {
1668     /* Return the IRP */
1669     return (PIRP)PsGetCurrentThread()->TopLevelIrp;
1670 }
1671
1672 /*
1673  * @implemented
1674  */
1675 VOID
1676 NTAPI
1677 IoInitializeIrp(IN PIRP Irp,
1678                 IN USHORT PacketSize,
1679                 IN CCHAR StackSize)
1680 {
1681     /* Clear it */
1682     IOTRACE(IO_IRP_DEBUG,
1683             "%s - Initializing IRP %p\n",
1684             __FUNCTION__,
1685             Irp);
1686     RtlZeroMemory(Irp, PacketSize);
1687
1688     /* Set the Header and other data */
1689     Irp->Type = IO_TYPE_IRP;
1690     Irp->Size = PacketSize;
1691     Irp->StackCount = StackSize;
1692     Irp->CurrentLocation = StackSize + 1;
1693     Irp->ApcEnvironment =  KeGetCurrentThread()->ApcStateIndex;
1694     Irp->Tail.Overlay.CurrentStackLocation = (PIO_STACK_LOCATION)(Irp + 1) + StackSize;
1695
1696     /* Initialize the Thread List */
1697     InitializeListHead(&Irp->ThreadListEntry);
1698 }
1699
1700 /*
1701  * @implemented
1702  */
1703 BOOLEAN
1704 NTAPI
1705 IoIsOperationSynchronous(IN PIRP Irp)
1706 {
1707     /* Check the flags */
1708     if (!(Irp->Flags & (IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO)) &&
1709         ((Irp->Flags & IRP_SYNCHRONOUS_PAGING_IO) ||
1710          (Irp->Flags & IRP_SYNCHRONOUS_API) ||
1711          (IoGetCurrentIrpStackLocation(Irp)->FileObject->Flags &
1712           FO_SYNCHRONOUS_IO)))
1713     {
1714         /* Synch API or Paging I/O is OK, as is Sync File I/O */
1715         return TRUE;
1716     }
1717
1718     /* Otherwise, it is an asynchronous operation. */
1719     return FALSE;
1720 }
1721
1722 /*
1723  * @unimplemented
1724  */
1725 BOOLEAN
1726 NTAPI
1727 IoIsValidNameGraftingBuffer(IN PIRP Irp,
1728                             IN PREPARSE_DATA_BUFFER ReparseBuffer)
1729 {
1730     UNIMPLEMENTED;
1731     return FALSE;
1732 }
1733
1734 /*
1735  * @implemented
1736  */
1737 PIRP
1738 NTAPI
1739 IoMakeAssociatedIrp(IN PIRP Irp,
1740                     IN CCHAR StackSize)
1741 {
1742     PIRP AssocIrp;
1743     IOTRACE(IO_IRP_DEBUG,
1744             "%s - Associating IRP %p\n",
1745             __FUNCTION__,
1746             Irp);
1747
1748    /* Allocate the IRP */
1749    AssocIrp = IoAllocateIrp(StackSize, FALSE);
1750    if (!AssocIrp) return NULL;
1751
1752    /* Set the Flags */
1753    AssocIrp->Flags |= IRP_ASSOCIATED_IRP;
1754
1755    /* Set the Thread */
1756    AssocIrp->Tail.Overlay.Thread = Irp->Tail.Overlay.Thread;
1757
1758    /* Associate them */
1759    AssocIrp->AssociatedIrp.MasterIrp = Irp;
1760    return AssocIrp;
1761 }
1762
1763 /*
1764  * @implemented
1765  */
1766 VOID
1767 NTAPI
1768 IoQueueThreadIrp(IN PIRP Irp)
1769 {
1770     IOTRACE(IO_IRP_DEBUG,
1771             "%s - Queueing IRP %p\n",
1772             __FUNCTION__,
1773             Irp);
1774
1775     /* Use our inlined routine */
1776     IopQueueIrpToThread(Irp);
1777 }
1778
1779 /*
1780  * @implemented
1781  * Reference: Chris Cant's "Writing WDM Device Drivers"
1782  */
1783 VOID
1784 NTAPI
1785 IoReuseIrp(IN OUT PIRP Irp,
1786            IN NTSTATUS Status)
1787 {
1788     UCHAR AllocationFlags;
1789     IOTRACE(IO_IRP_DEBUG,
1790             "%s - Reusing IRP %p\n",
1791             __FUNCTION__,
1792             Irp);
1793
1794     /* Make sure it's OK to reuse it */
1795     ASSERT(!Irp->CancelRoutine);
1796     ASSERT(IsListEmpty(&Irp->ThreadListEntry));
1797
1798     /* Get the old flags */
1799     AllocationFlags = Irp->AllocationFlags;
1800
1801     /* Reinitialize the IRP */
1802     IoInitializeIrp(Irp, Irp->Size, Irp->StackCount);
1803
1804     /* Duplicate the data */
1805     Irp->IoStatus.Status = Status;
1806     Irp->AllocationFlags = AllocationFlags;
1807 }
1808
1809 /*
1810  * @implemented
1811  */
1812 VOID
1813 NTAPI
1814 IoSetTopLevelIrp(IN PIRP Irp)
1815 {
1816     /* Set the IRP */
1817     PsGetCurrentThread()->TopLevelIrp = (ULONG_PTR)Irp;
1818 }