reactos/lib/rtl/amd64/slist.S

   1 /*
   2  * COPYRIGHT:       See COPYING in the top level directory
   3  * PROJECT:         ReactOS system libraries
   4  * FILE:            lib/rtl/amd64/interlck.S
   5  * PURPOSE:         Rtl Interlocked Functions for amd64
   6  * PROGRAMMERS:     Timo Kreuzer
   7  */
   8
   9 #include <asm.inc>
  10 #include <ksamd64.inc>
  11
  12 /*
  13 typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
  14 {
  15     struct
  16     {
  17         ULONGLONG Alignment;
  18         ULONGLONG Region;
  19     };
  20     struct
  21     {
  22         ULONGLONG Depth:16;
  23         ULONGLONG Sequence:9;
  24         ULONGLONG NextEntry:39;
  25         ULONGLONG HeaderType:1;
  26         ULONGLONG Init:1;
  27         ULONGLONG Reserved:59;
  28         ULONGLONG Region:3;
  29     } Header8;
  30     struct
  31     {
  32         ULONGLONG Depth:16;
  33         ULONGLONG Sequence:48;
  34         ULONGLONG HeaderType:1;
  35         ULONGLONG Init:1;
  36         ULONGLONG Reserved:2;
  37         ULONGLONG NextEntry:60;
  38     } Header16;
  39     struct
  40     {
  41         ULONGLONG Depth:16;
  42         ULONGLONG Sequence:48;
  43         ULONGLONG HeaderType:1;
  44         ULONGLONG Reserved:3;
  45         ULONGLONG NextEntry:60;
  46     } HeaderX64;
  47 } SLIST_HEADER, *PSLIST_HEADER;
  48 */
  49
  50 #define SLIST8A_DEPTH_MASK       HEX(000000000000FFFF)
  51 #define SLIST8A_DEPTH_INC        HEX(0000000000000001)
  52 #define SLIST8A_SEQUENCE_MASK    HEX(0000000001FF0000)
  53 #define SLIST8A_SEQUENCE_INC     HEX(0000000000010000)
  54 #define SLIST8A_NEXTENTRY_MASK   HEX(FFFFFFFFFE000000)
  55 #define SLIST8A_NEXTENTRY_SHIFT  21
  56 #define SLIST8B_HEADERTYPE_MASK  HEX(0000000000000001)
  57 #define SLIST8B_INIT_MASK        HEX(0000000000000002)
  58 #define SLIST8B_REGION_MASK      HEX(E000000000000000)
  59 #define SLIST8_POINTER_MASK      HEX(000007FFFFFFFFFF)
  60
  61 #define SLIST16A_DEPTH_MASK      HEX(000000000000FFFF)
  62 #define SLIST16A_DEPTH_INC       HEX(0000000000000001)
  63 #define SLIST16A_SEQUENCE_MASK   HEX(FFFFFFFFFFFF0000)
  64 #define SLIST16A_SEQUENCE_INC    HEX(0000000000010000)
  65 #define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
  66 #define SLIST16B_INIT_MASK       HEX(0000000000000002)
  67 #define SLIST16B_NEXTENTRY_MASK  HEX(FFFFFFFFFFFFFFF0)
  68
  69
  70 /* FUNCTIONS ****************************************************************/
  71
  72 .code64
  73
  74 PUBLIC ExpInterlockedPopEntrySList
  75 PUBLIC ExpInterlockedPopEntrySListResume
  76 PUBLIC ExpInterlockedPopEntrySListFault
  77 PUBLIC ExpInterlockedPopEntrySListEnd
  78 PUBLIC ExpInterlockedPopEntrySListResume16
  79 PUBLIC ExpInterlockedPopEntrySListFault16
  80 PUBLIC ExpInterlockedPopEntrySListEnd16
  81 PUBLIC ExpInterlockedPushEntrySList
  82 PUBLIC ExpInterlockedFlushSList
  83
  84 PUBLIC RtlInterlockedFlushSList
  85 PUBLIC RtlInterlockedPopEntrySList
  86 PUBLIC RtlInterlockedPushEntrySList
  87
  88 /* PSLIST_ENTRY
  89  * NTAPI
  90  * RtlInterlockedPopEntrySList(
  91  *     IN PSLIST_HEADER ListHead);
  92  */
  93 RtlInterlockedPopEntrySList:
  94 ExpInterlockedPopEntrySList:
  95
  96     /* Load ListHead->Region into rdx */
  97     mov rdx, [rcx + 8]
  98
  99     /* Load ListHead->Alignment into rax */
 100     mov rax, [rcx]
 101
 102     /* Check what kind of header this is */
 103     test rdx, SLIST8B_HEADERTYPE_MASK
 104     jnz RtlInterlockedPopEntrySList16
 105
 106     /* We have an 8 byte header */
 107
 108 ExpInterlockedPopEntrySListResume:
 109
 110     /* Check if ListHead->NextEntry is NULL */
 111     mov r9, rax
 112     and r9, SLIST8A_NEXTENTRY_MASK
 113     jz RtlInterlockedPopEntrySListEmpty
 114
 115     /* Copy Depth and Sequence number and adjust Depth */
 116     lea r8, [rax - SLIST8A_DEPTH_INC]
 117     and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
 118
 119     /* Create a pointer template from rcx in rdx */
 120     mov rdx, (NOT SLIST8_POINTER_MASK)
 121     and rdx, rcx
 122
 123     /* Shift the NextEntry pointer */
 124     shr r9, SLIST8A_NEXTENTRY_SHIFT
 125
 126     /* Combine to new pointer in rdx */
 127     or rdx, r9
 128
 129 ExpInterlockedPopEntrySListFault:
 130
 131     /* Load the next NextEntry pointer to r9 */
 132     mov r9, [rdx]
 133
 134     /* Shift bits in place */
 135     shl r9, SLIST8A_NEXTENTRY_SHIFT
 136
 137     /* Combine into r8 */
 138     or r8, r9
 139
 140 ExpInterlockedPopEntrySListEnd:
 141
 142     /* If [rcx] equals rax, exchange it with r8 */
 143     lock cmpxchg [rcx], r8
 144
 145     /* If not equal, retry with rax, being the content of [rcx] now */
 146     jne ExpInterlockedPopEntrySListResume
 147
 148     /* Shift the pointer bits in place */
 149     and rax, SLIST8A_NEXTENTRY_MASK
 150     shr rax, SLIST8A_NEXTENTRY_SHIFT
 151
 152     /* Use rcx as pointer template */
 153     mov rdx, (NOT SLIST8_POINTER_MASK)
 154     and rdx, rcx
 155
 156     /* Combine result and return */
 157     or rax, rdx
 158     ret
 159
 160 RtlInterlockedPopEntrySListEmpty:
 161     xor rax, rax
 162     ret
 163
 164 RtlInterlockedPopEntrySList16:
 165     /* This is a 16 byte header
 166        rcx == ListHead
 167        rdx == ListHead->Region
 168        rax == ListHead->Alignment */
 169
 170     /* Save rbx */
 171     push rbx
 172
 173     /* Copy rcx to r8, as we need rcx for the exchange */
 174     mov r8, rcx
 175
 176 ExpInterlockedPopEntrySListResume16:
 177
 178     /* Set r9 = ListHead->NextEntry and check if it is NULL */
 179     mov r9, rdx
 180     and r9, SLIST16B_NEXTENTRY_MASK
 181     jz RtlInterlockedPopEntrySListEmpty16
 182
 183 ExpInterlockedPopEntrySListFault16:
 184
 185     /* Set NewListHead.Next = ListHead->NextEntry->Next */
 186     mov rcx, [r9]
 187
 188     /* Set NewListHead.HeaderType = 1 and  NewListHead.Init = 1 */
 189     or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 190
 191     /* Copy Depth and Sequence number and adjust Depth */
 192     lea rbx, [rax - SLIST16A_DEPTH_INC]
 193
 194 ExpInterlockedPopEntrySListEnd16:
 195
 196     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 197     lock cmpxchg16b [r8]
 198
 199     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 200     jne ExpInterlockedPopEntrySListResume16
 201
 202     /* Copy the old NextEntry pointer to rax */
 203     mov rax, rdx
 204     and rax, SLIST16B_NEXTENTRY_MASK
 205
 206     /* Return */
 207     pop rbx
 208     ret
 209
 210 RtlInterlockedPopEntrySListEmpty16:
 211     xor rax, rax
 212     pop rbx
 213     ret
 214
 215
 216 /* PSLIST_ENTRY
 217  * NTAPI
 218  * RtlInterlockedPushEntrySList(
 219  *     IN PSLIST_HEADER ListHead,
 220  *     IN PSLIST_ENTRY ListEntry);
 221  */
 222 RtlInterlockedPushEntrySList:
 223 ExpInterlockedPushEntrySList:
 224
 225 #if DBG
 226     /* Make sure the ListEntry is 16 bytes aligned */
 227     test rdx, HEX(0F)
 228     jz ExpInterlockedPushEntrySListChecked
 229     /* Not aligned, raise an assertion */
 230     int HEX(2C)
 231 ExpInterlockedPushEntrySListChecked:
 232 #endif
 233
 234     /* Load ListHead->Alignment into rax */
 235     mov rax, [rcx]
 236
 237     /* Load ListHead->Region into r9 */
 238     mov r9, [rcx + 8]
 239
 240     /* Check what kind of header this is */
 241     test r9, SLIST8B_HEADERTYPE_MASK
 242     jnz RtlInterlockedPushEntrySList16
 243
 244     /* We have an 8 byte header */
 245
 246 RtlInterlockedPushEntrySListLoop:
 247
 248     /* Get ListHead->NextEntry */
 249     mov r8, rax
 250     and r8, SLIST8A_NEXTENTRY_MASK
 251     jz RtlInterlockedPushEntrySListEmpty
 252
 253     /* Shift the NextEntry pointer */
 254     shr r8, SLIST8A_NEXTENTRY_SHIFT
 255
 256     /* Create a pointer template from rcx in rdx */
 257     mov r9, (NOT SLIST8_POINTER_MASK)
 258     and r9, rcx
 259
 260     /* Combine to new pointer and save as ListEntry->NextEntry */
 261     or r8, r9
 262
 263 RtlInterlockedPushEntrySListEmpty:
 264     /* Store the NextEntry pointer in the new ListEntry */
 265     mov [rdx], r8
 266
 267     /* Shift and mask the new ListEntry pointer */
 268     mov r8, rdx
 269     shl r8, SLIST8A_NEXTENTRY_SHIFT
 270     and r8, SLIST8A_NEXTENTRY_MASK
 271
 272     /* Copy and adjust depth and sequence number */
 273     lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
 274     and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
 275
 276     /* Combine to exchange value in r8 */
 277     or r8, r9
 278
 279     /* Save the NextEntry in r9 */
 280     mov r9, [rdx]
 281
 282     /* If [rcx] equals rax, exchange it with r8 */
 283     lock cmpxchg [rcx], r8
 284
 285     /* If not equal, retry with rax, being the content of [rcx] now */
 286     jne RtlInterlockedPushEntrySListLoop
 287
 288     /* Return the old NextEntry pointer */
 289     mov rax, r9
 290     ret
 291
 292 RtlInterlockedPushEntrySList16:
 293     /* This is a 16 byte header
 294        rcx = ListHead
 295        rdx = ListEntry
 296        rax = ListHead->Alignment
 297        r9 = ListHead->Region */
 298
 299     /* Save rbx */
 300     push rbx
 301
 302     /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
 303     mov r8, rcx
 304     mov r9, rdx
 305
 306     /* Set NewListHead.NextEntry = ListEntry */
 307     mov rcx, rdx
 308
 309     /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
 310     or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 311
 312     /* Set rdx = ListHead->Region */
 313     mov rdx, [r8 + 8]
 314
 315 RtlInterlockedPushEntrySListLoop16:
 316     /* r8 = ListHead
 317        r9 = ListEntry
 318        rax = ListHead->Alignment
 319        rdx = ListHead->Region
 320     */
 321
 322     /* Move ListHead->NextEntry to rbx */
 323     mov rbx, rdx
 324     and rbx, SLIST16B_NEXTENTRY_MASK
 325
 326     /* Store next pointer in ListEntry->Next */
 327     mov [r9], rbx
 328
 329     /* Copy and increment Depth and Sequence number to rbx */
 330     lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
 331
 332     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 333     lock cmpxchg16b [r8]
 334
 335     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 336     jne RtlInterlockedPushEntrySListLoop16
 337
 338     /* Copy the old NextEntry pointer to rax */
 339     mov rax, rdx
 340     and rax, SLIST16B_NEXTENTRY_MASK
 341
 342     /* Return */
 343     pop rbx
 344     ret
 345
 346
 347 /* PSLIST_ENTRY
 348  * NTAPI
 349  * RtlInterlockedFlushSList(
 350  *     IN PSLIST_HEADER ListHead);
 351  */
 352 RtlInterlockedFlushSList:
 353 ExpInterlockedFlushSList:
 354
 355     /* Load ListHead->Region into rdx */
 356     mov rdx, [rcx + 8]
 357
 358     /* Load ListHead->Alignment into rax */
 359     mov rax, [rcx]
 360
 361     /* Check what kind of header this is */
 362     test rdx, SLIST8B_HEADERTYPE_MASK
 363     jnz RtlInterlockedFlushSList16
 364
 365     /* We have an 8 byte header */
 366
 367 RtlInterlockedFlushSListLoop:
 368
 369     /* Zero NewListHead.Alignment */
 370     xor r8, r8
 371
 372     /* If [rcx] equals rax, exchange it with r8 */
 373     lock cmpxchg [rcx], r8
 374
 375     /* If not equal, retry with rax, being the content of [rcx] now */
 376     jne RtlInterlockedFlushSListLoop
 377
 378     /* Create a pointer template from rcx in rdx */
 379     mov rdx, (NOT SLIST8_POINTER_MASK)
 380     and rdx, rcx
 381
 382     /* Load the old NextEntry pointer into rax */
 383     and rax, SLIST8A_NEXTENTRY_MASK
 384     shr rax, SLIST8A_NEXTENTRY_SHIFT
 385
 386     /* Combine result and return */
 387     or rax, rdx
 388     ret
 389
 390 RtlInterlockedFlushSList16:
 391     /* We have a 16 byte header
 392         rcx = ListHead
 393         rax = ListHead->Alignment
 394         rdx = ListHead->Region
 395     */
 396
 397     /* Save rbx */
 398     push rbx
 399
 400     /* Load ListHead into r8, as we need rcx for the exchange */
 401     mov r8, rcx
 402
 403     /* Initialize an ampty NewListHead in rcx:rbx */
 404     xor rbx, rbx
 405     mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
 406
 407 RtlInterlockedFlushSListLoop16:
 408
 409     /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
 410     lock cmpxchg16b [r8]
 411
 412     /* If not equal, retry with rdx:rax, being the content of [r8] now */
 413     jne RtlInterlockedFlushSListLoop16
 414
 415     /* Copy the old NextEntry pointer to rax */
 416     mov rax, rdx
 417     and rax, SLIST16B_NEXTENTRY_MASK
 418
 419     /* Return */
 420     pop rbx
 421     ret
 422
 423 END
 424