reactos/lib/rtl/amd64/slist.S

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS system libraries
   4  * FILE:            lib/rtl/amd64/interlck.S
   5  * PURPOSE:         Rtl Interlocked Functions for amd64
   6  * PROGRAMMERS:     Timo Kreuzer
   7  */
   8
   9 #include <asm.inc>
  10 #include <ksamd64.inc>
  11
  12 EXTERN RtlpUse16ByteSLists:BYTE
  13
  14 /*
  15 typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
  16 {
  17     struct
  18     {
  19         ULONGLONG Alignment;
  20         ULONGLONG Region;
  21     };
  22     struct
  23     {
  24         ULONGLONG Depth:16;
  25         ULONGLONG Sequence:9;
  26         ULONGLONG NextEntry:39;
  27         ULONGLONG HeaderType:1;
  28         ULONGLONG Init:1;
  29         ULONGLONG Reserved:59;
  30         ULONGLONG Region:3;
  31     } Header8;
  32     struct
  33     {
  34         ULONGLONG Depth:16;
  35         ULONGLONG Sequence:48;
  36         ULONGLONG HeaderType:1;
  37         ULONGLONG Init:1;
  38         ULONGLONG Reserved:2;
  39         ULONGLONG NextEntry:60;
  40     } Header16;
  41     struct
  42     {
  43         ULONGLONG Depth:16;
  44         ULONGLONG Sequence:48;
  45         ULONGLONG HeaderType:1;
  46         ULONGLONG Reserved:3;
  47         ULONGLONG NextEntry:60;
  48     } HeaderX64;
  49 } SLIST_HEADER, *PSLIST_HEADER;
  50 */
  51
  52 #define SLIST8A_DEPTH_MASK       HEX(000000000000FFFF)
  53 #define SLIST8A_DEPTH_INC        HEX(0000000000000001)
  54 #define SLIST8A_SEQUENCE_MASK    HEX(0000000001FF0000)
  55 #define SLIST8A_SEQUENCE_INC     HEX(0000000000010000)
  56 #define SLIST8A_NEXTENTRY_MASK   HEX(FFFFFFFFFE000000)
  57 #define SLIST8A_NEXTENTRY_SHIFT  21
  58 #define SLIST8B_HEADERTYPE_MASK  HEX(0000000000000001)
  59 #define SLIST8B_INIT_MASK        HEX(0000000000000002)
  60 #define SLIST8B_REGION_MASK      HEX(E000000000000000)
  61 #define SLIST8_POINTER_MASK      HEX(000007FFFFFFFFFF)
  62
  63 #define SLIST16A_DEPTH_MASK      HEX(000000000000FFFF)
  64 #define SLIST16A_DEPTH_INC       HEX(0000000000000001)
  65 #define SLIST16A_SEQUENCE_MASK   HEX(FFFFFFFFFFFF0000)
  66 #define SLIST16A_SEQUENCE_INC    HEX(0000000000010000)
  67 #define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
  68 #define SLIST16B_INIT_MASK       HEX(0000000000000002)
  69 #define SLIST16B_NEXTENTRY_MASK  HEX(FFFFFFFFFFFFFFF0)
  70
  71
  72 /* FUNCTIONS ****************************************************************/
  73
  74 .code64
  75
  76 PUBLIC ExpInterlockedPopEntrySList
  77 PUBLIC ExpInterlockedPopEntrySListResume
  78 PUBLIC ExpInterlockedPopEntrySListFault
  79 PUBLIC ExpInterlockedPopEntrySListEnd
  80 PUBLIC ExpInterlockedPopEntrySListResume16
  81 PUBLIC ExpInterlockedPopEntrySListFault16
  82 PUBLIC ExpInterlockedPopEntrySListEnd16
  83 PUBLIC ExpInterlockedPushEntrySList
  84 PUBLIC ExpInterlockedFlushSList
  85
  86 PUBLIC RtlInterlockedFlushSList
  87 PUBLIC RtlInterlockedPopEntrySList
  88 PUBLIC RtlInterlockedPushEntrySList
  89
  90 /* PSLIST_ENTRY
  91  * NTAPI
  92  * RtlInterlockedPopEntrySList(
  93  *     IN PSLIST_HEADER ListHead);
  94  */
  95 RtlInterlockedPopEntrySList:
  96 ExpInterlockedPopEntrySList:
  97
  98     /* Load ListHead->Region into rdx */
  99     mov rdx, [rcx + 8]
 100
 101     /* Load ListHead->Alignment into rax */
 102     mov rax, [rcx]
 103
 104     /* Check for 16 byte SList support */
 105     cmp byte ptr [RtlpUse16ByteSLists], 0
 106     jne RtlInterlockedPopEntrySList16
 107
 108     /* Use the 8 byte header */
 109
 110 ExpInterlockedPopEntrySListResume:
 111
 112     /* Check if ListHead->NextEntry is NULL */
 113     mov r9, rax
 114     and r9, SLIST8A_NEXTENTRY_MASK
 115     jz RtlInterlockedPopEntrySListEmpty
 116
 117     /* Copy Depth and Sequence number and adjust Depth */
 118     lea r8, [rax - SLIST8A_DEPTH_INC]
 119     and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
 120
 121     /* Create a pointer template from rcx in rdx */
 122     mov rdx, (NOT SLIST8_POINTER_MASK)
 123     and rdx, rcx
 124
 125     /* Shift the NextEntry pointer */
 126     shr r9, SLIST8A_NEXTENTRY_SHIFT
 127
 128     /* Combine to new pointer in rdx */
 129     or rdx, r9
 130
 131 ExpInterlockedPopEntrySListFault:
 132
 133     /* Load the next NextEntry pointer to r9 */
 134     mov r9, [rdx]
 135
 136     /* Shift bits in place */
 137     shl r9, SLIST8A_NEXTENTRY_SHIFT
 138
 139     /* Combine into r8 */
 140     or r8, r9
 141
 142 ExpInterlockedPopEntrySListEnd:
 143
 144     /* If [rcx] equals rax, exchange it with r8 */
 145     lock cmpxchg [rcx], r8
 146
 147     /* If not equal, retry with rax, being the content of [rcx] now */
 148     jne ExpInterlockedPopEntrySListResume
 149
 150     /* Shift the pointer bits in place */
 151     and rax, SLIST8A_NEXTENTRY_MASK
 152     shr rax, SLIST8A_NEXTENTRY_SHIFT
 153
 154     /* Use rcx as pointer template */
 155     mov rdx, (NOT SLIST8_POINTER_MASK)
 156     and rdx, rcx
 157
 158     /* Combine result and return */
 159     or rax, rdx
 160     ret
 161
 162 RtlInterlockedPopEntrySListEmpty:
 163     xor rax, rax
 164     ret
 165
 166 RtlInterlockedPopEntrySList16:
 167     /* This is a 16 byte header
 168        rcx == ListHead
 169        rdx == ListHead->Region
 170        rax == ListHead->Alignment */
 171
 172     /* Save rbx */
 173     push rbx
 174
 175     /* Copy rcx to r8, as we need rcx for the exchange */
 176     mov r8, rcx
 177
 178 ExpInterlockedPopEntrySListResume16:
 179
 180     /* Set r9 = ListHead->NextEntry and check if it is NULL */
 181     mov r9, rdx
 182     and r9, SLIST16B_NEXTENTRY_MASK
 183     jz RtlInterlockedPopEntrySListEmpty16
 184
 185 ExpInterlockedPopEntrySListFault16:
 186
 187     /* Set NewListHead.Next = ListHead->NextEntry->Next */
 188     mov rcx, [r9]
 189
 190     /* Set NewListHead.HeaderType = 1 and  NewListHead.Init = 1 */
 191     or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 192
 193     /* Copy Depth and Sequence number and adjust Depth */
 194     lea rbx, [rax - SLIST16A_DEPTH_INC]
 195
 196 ExpInterlockedPopEntrySListEnd16:
 197
 198     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 199     lock cmpxchg16b [r8]
 200
 201     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 202     jne ExpInterlockedPopEntrySListResume16
 203
 204     /* Copy the old NextEntry pointer to rax */
 205     mov rax, rdx
 206     and rax, SLIST16B_NEXTENTRY_MASK
 207
 208     /* Return */
 209     pop rbx
 210     ret
 211
 212 RtlInterlockedPopEntrySListEmpty16:
 213     xor rax, rax
 214     pop rbx
 215     ret
 216
 217
 218 /* PSLIST_ENTRY
 219  * NTAPI
 220  * RtlInterlockedPushEntrySList(
 221  *     IN PSLIST_HEADER ListHead,
 222  *     IN PSLIST_ENTRY ListEntry);
 223  */
 224 RtlInterlockedPushEntrySList:
 225 ExpInterlockedPushEntrySList:
 226
 227 #if DBG
 228     /* Make sure the ListEntry is 16 bytes aligned */
 229     test rdx, HEX(0F)
 230     jz ExpInterlockedPushEntrySListChecked
 231     /* Not aligned, raise an assertion */
 232     int HEX(2C)
 233 ExpInterlockedPushEntrySListChecked:
 234
 235     /* Make sure RtlpUse16ByteSLists is initialized */
 236     cmp byte ptr [RtlpUse16ByteSLists], HEX(FF)
 237     jne ExpInterlockedPushEntrySListChecked2
 238     /* Not initialized, raise an assertion */
 239     int HEX(2C)
 240 ExpInterlockedPushEntrySListChecked2:
 241 #endif
 242
 243     /* Load ListHead->Alignment into rax */
 244     mov rax, [rcx]
 245
 246     /* Load ListHead->Region into r9 */
 247     mov r9, [rcx + 8]
 248
 249     /* Check for 16 byte SList support */
 250     cmp byte ptr [RtlpUse16ByteSLists], 0
 251     jne RtlInterlockedPushEntrySList16
 252
 253     /* Use the 8 byte header */
 254
 255 RtlInterlockedPushEntrySListLoop:
 256
 257     /* Get ListHead->NextEntry */
 258     mov r8, rax
 259     and r8, SLIST8A_NEXTENTRY_MASK
 260     jz RtlInterlockedPushEntrySListEmpty
 261
 262     /* Shift the NextEntry pointer */
 263     shr r8, SLIST8A_NEXTENTRY_SHIFT
 264
 265     /* Create a pointer template from rcx in rdx */
 266     mov r9, (NOT SLIST8_POINTER_MASK)
 267     and r9, rcx
 268
 269     /* Combine to new pointer and save as ListEntry->NextEntry */
 270     or r8, r9
 271
 272 RtlInterlockedPushEntrySListEmpty:
 273     /* Store the NextEntry pointer in the new ListEntry */
 274     mov [rdx], r8
 275
 276     /* Shift and mask the new ListEntry pointer */
 277     mov r8, rdx
 278     shl r8, SLIST8A_NEXTENTRY_SHIFT
 279     and r8, SLIST8A_NEXTENTRY_MASK
 280
 281     /* Copy and adjust depth and sequence number */
 282     lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
 283     and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
 284
 285     /* Combine to exchange value in r8 */
 286     or r8, r9
 287
 288     /* Save the NextEntry in r9 */
 289     mov r9, [rdx]
 290
 291     /* If [rcx] equals rax, exchange it with r8 */
 292     lock cmpxchg [rcx], r8
 293
 294     /* If not equal, retry with rax, being the content of [rcx] now */
 295     jne RtlInterlockedPushEntrySListLoop
 296
 297     /* Return the old NextEntry pointer */
 298     mov rax, r9
 299     ret
 300
 301 RtlInterlockedPushEntrySList16:
 302     /* This is a 16 byte header
 303        rcx = ListHead
 304        rdx = ListEntry
 305        rax = ListHead->Alignment
 306        r9 = ListHead->Region */
 307
 308     /* Save rbx */
 309     push rbx
 310
 311     /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
 312     mov r8, rcx
 313     mov r9, rdx
 314
 315     /* Set NewListHead.NextEntry = ListEntry */
 316     mov rcx, rdx
 317
 318     /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
 319     or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 320
 321     /* Set rdx = ListHead->Region */
 322     mov rdx, [r8 + 8]
 323
 324 RtlInterlockedPushEntrySListLoop16:
 325     /* r8 = ListHead
 326        r9 = ListEntry
 327        rax = ListHead->Alignment
 328        rdx = ListHead->Region
 329     */
 330
 331     /* Move ListHead->NextEntry to rbx */
 332     mov rbx, rdx
 333     and rbx, SLIST16B_NEXTENTRY_MASK
 334
 335     /* Store next pointer in ListEntry->Next */
 336     mov [r9], rbx
 337
 338     /* Copy and increment Depth and Sequence number to rbx */
 339     lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
 340
 341     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 342     lock cmpxchg16b [r8]
 343
 344     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 345     jne RtlInterlockedPushEntrySListLoop16
 346
 347     /* Copy the old NextEntry pointer to rax */
 348     mov rax, rdx
 349     and rax, SLIST16B_NEXTENTRY_MASK
 350
 351     /* Return */
 352     pop rbx
 353     ret
 354
 355
 356 /* PSLIST_ENTRY
 357  * NTAPI
 358  * RtlInterlockedFlushSList(
 359  *     IN PSLIST_HEADER ListHead);
 360  */
 361 RtlInterlockedFlushSList:
 362 ExpInterlockedFlushSList:
 363
 364     /* Load ListHead->Region into rdx */
 365     mov rdx, [rcx + 8]
 366
 367     /* Load ListHead->Alignment into rax */
 368     mov rax, [rcx]
 369
 370     /* Check for 16 byte SList support */
 371     cmp byte ptr [RtlpUse16ByteSLists], 0
 372     jne RtlInterlockedFlushSList16
 373
 374     /* Use the 8 byte header */
 375
 376 RtlInterlockedFlushSListLoop:
 377
 378     /* Zero NewListHead.Alignment */
 379     xor r8, r8
 380
 381     /* If [rcx] equals rax, exchange it with r8 */
 382     lock cmpxchg [rcx], r8
 383
 384     /* If not equal, retry with rax, being the content of [rcx] now */
 385     jne RtlInterlockedFlushSListLoop
 386
 387     /* Create a pointer template from rcx in rdx */
 388     mov rdx, (NOT SLIST8_POINTER_MASK)
 389     and rdx, rcx
 390
 391     /* Load the old NextEntry pointer into rax */
 392     and rax, SLIST8A_NEXTENTRY_MASK
 393     shr rax, SLIST8A_NEXTENTRY_SHIFT
 394
 395     /* Combine result and return */
 396     or rax, rdx
 397     ret
 398
 399 RtlInterlockedFlushSList16:
 400     /* We have a 16 byte header
 401         rcx = ListHead
 402         rax = ListHead->Alignment
 403         rdx = ListHead->Region
 404     */
 405
 406     /* Save rbx */
 407     push rbx
 408
 409     /* Load ListHead into r8, as we need rcx for the exchange */
 410     mov r8, rcx
 411
 412     /* Initialize an ampty NewListHead in rcx:rbx */
 413     xor rbx, rbx
 414     mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 415
 416 RtlInterlockedFlushSListLoop16:
 417
 418     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 419     lock cmpxchg16b [r8]
 420
 421     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 422     jne RtlInterlockedFlushSListLoop16
 423
 424     /* Copy the old NextEntry pointer to rax */
 425     mov rax, rdx
 426     and rax, SLIST16B_NEXTENTRY_MASK
 427
 428     /* Return */
 429     pop rbx
 430     ret
 431
 432 END
 433