[NTOSKRNL] Handle some more KeFeatureFlags in amd64/cpu.c and set RtlpUse16ByteSLists
[reactos.git] / reactos / lib / rtl / amd64 / slist.S
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS system libraries
4 * FILE: lib/rtl/amd64/interlck.S
5 * PURPOSE: Rtl Interlocked Functions for amd64
6 * PROGRAMMERS: Timo Kreuzer
7 */
8
9 #include <asm.inc>
10 #include <ksamd64.inc>
11
12 EXTERN RtlpUse16ByteSLists:BYTE
13
14 /*
15 typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
16 {
17 struct
18 {
19 ULONGLONG Alignment;
20 ULONGLONG Region;
21 };
22 struct
23 {
24 ULONGLONG Depth:16;
25 ULONGLONG Sequence:9;
26 ULONGLONG NextEntry:39;
27 ULONGLONG HeaderType:1;
28 ULONGLONG Init:1;
29 ULONGLONG Reserved:59;
30 ULONGLONG Region:3;
31 } Header8;
32 struct
33 {
34 ULONGLONG Depth:16;
35 ULONGLONG Sequence:48;
36 ULONGLONG HeaderType:1;
37 ULONGLONG Init:1;
38 ULONGLONG Reserved:2;
39 ULONGLONG NextEntry:60;
40 } Header16;
41 struct
42 {
43 ULONGLONG Depth:16;
44 ULONGLONG Sequence:48;
45 ULONGLONG HeaderType:1;
46 ULONGLONG Reserved:3;
47 ULONGLONG NextEntry:60;
48 } HeaderX64;
49 } SLIST_HEADER, *PSLIST_HEADER;
50 */
51
52 #define SLIST8A_DEPTH_MASK HEX(000000000000FFFF)
53 #define SLIST8A_DEPTH_INC HEX(0000000000000001)
54 #define SLIST8A_SEQUENCE_MASK HEX(0000000001FF0000)
55 #define SLIST8A_SEQUENCE_INC HEX(0000000000010000)
56 #define SLIST8A_NEXTENTRY_MASK HEX(FFFFFFFFFE000000)
57 #define SLIST8A_NEXTENTRY_SHIFT 21
58 #define SLIST8B_HEADERTYPE_MASK HEX(0000000000000001)
59 #define SLIST8B_INIT_MASK HEX(0000000000000002)
60 #define SLIST8B_REGION_MASK HEX(E000000000000000)
61 #define SLIST8_POINTER_MASK HEX(000007FFFFFFFFFF)
62
63 #define SLIST16A_DEPTH_MASK HEX(000000000000FFFF)
64 #define SLIST16A_DEPTH_INC HEX(0000000000000001)
65 #define SLIST16A_SEQUENCE_MASK HEX(FFFFFFFFFFFF0000)
66 #define SLIST16A_SEQUENCE_INC HEX(0000000000010000)
67 #define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
68 #define SLIST16B_INIT_MASK HEX(0000000000000002)
69 #define SLIST16B_NEXTENTRY_MASK HEX(FFFFFFFFFFFFFFF0)
70
71
72 /* FUNCTIONS ****************************************************************/
73
74 .code64
75
76 PUBLIC ExpInterlockedPopEntrySList
77 PUBLIC ExpInterlockedPopEntrySListResume
78 PUBLIC ExpInterlockedPopEntrySListFault
79 PUBLIC ExpInterlockedPopEntrySListEnd
80 PUBLIC ExpInterlockedPopEntrySListResume16
81 PUBLIC ExpInterlockedPopEntrySListFault16
82 PUBLIC ExpInterlockedPopEntrySListEnd16
83 PUBLIC ExpInterlockedPushEntrySList
84 PUBLIC ExpInterlockedFlushSList
85
86 PUBLIC RtlInterlockedFlushSList
87 PUBLIC RtlInterlockedPopEntrySList
88 PUBLIC RtlInterlockedPushEntrySList
89
90 /* PSLIST_ENTRY
91 * NTAPI
92 * RtlInterlockedPopEntrySList(
93 * IN PSLIST_HEADER ListHead);
94 */
95 RtlInterlockedPopEntrySList:
96 ExpInterlockedPopEntrySList:
97
98 /* Load ListHead->Region into rdx */
99 mov rdx, [rcx + 8]
100
101 /* Load ListHead->Alignment into rax */
102 mov rax, [rcx]
103
104 /* Check for 16 byte SList support */
105 cmp byte ptr [RtlpUse16ByteSLists], 0
106 jne RtlInterlockedPopEntrySList16
107
108 /* Use the 8 byte header */
109
110 ExpInterlockedPopEntrySListResume:
111
112 /* Check if ListHead->NextEntry is NULL */
113 mov r9, rax
114 and r9, SLIST8A_NEXTENTRY_MASK
115 jz RtlInterlockedPopEntrySListEmpty
116
117 /* Copy Depth and Sequence number and adjust Depth */
118 lea r8, [rax - SLIST8A_DEPTH_INC]
119 and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
120
121 /* Create a pointer template from rcx in rdx */
122 mov rdx, (NOT SLIST8_POINTER_MASK)
123 and rdx, rcx
124
125 /* Shift the NextEntry pointer */
126 shr r9, SLIST8A_NEXTENTRY_SHIFT
127
128 /* Combine to new pointer in rdx */
129 or rdx, r9
130
131 ExpInterlockedPopEntrySListFault:
132
133 /* Load the next NextEntry pointer to r9 */
134 mov r9, [rdx]
135
136 /* Shift bits in place */
137 shl r9, SLIST8A_NEXTENTRY_SHIFT
138
139 /* Combine into r8 */
140 or r8, r9
141
142 ExpInterlockedPopEntrySListEnd:
143
144 /* If [rcx] equals rax, exchange it with r8 */
145 lock cmpxchg [rcx], r8
146
147 /* If not equal, retry with rax, being the content of [rcx] now */
148 jne ExpInterlockedPopEntrySListResume
149
150 /* Shift the pointer bits in place */
151 and rax, SLIST8A_NEXTENTRY_MASK
152 shr rax, SLIST8A_NEXTENTRY_SHIFT
153
154 /* Use rcx as pointer template */
155 mov rdx, (NOT SLIST8_POINTER_MASK)
156 and rdx, rcx
157
158 /* Combine result and return */
159 or rax, rdx
160 ret
161
162 RtlInterlockedPopEntrySListEmpty:
163 xor rax, rax
164 ret
165
166 RtlInterlockedPopEntrySList16:
167 /* This is a 16 byte header
168 rcx == ListHead
169 rdx == ListHead->Region
170 rax == ListHead->Alignment */
171
172 /* Save rbx */
173 push rbx
174
175 /* Copy rcx to r8, as we need rcx for the exchange */
176 mov r8, rcx
177
178 ExpInterlockedPopEntrySListResume16:
179
180 /* Set r9 = ListHead->NextEntry and check if it is NULL */
181 mov r9, rdx
182 and r9, SLIST16B_NEXTENTRY_MASK
183 jz RtlInterlockedPopEntrySListEmpty16
184
185 ExpInterlockedPopEntrySListFault16:
186
187 /* Set NewListHead.Next = ListHead->NextEntry->Next */
188 mov rcx, [r9]
189
190 /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
191 or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
192
193 /* Copy Depth and Sequence number and adjust Depth */
194 lea rbx, [rax - SLIST16A_DEPTH_INC]
195
196 ExpInterlockedPopEntrySListEnd16:
197
198 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
199 lock cmpxchg16b [r8]
200
201 /* If not equal, retry with rdx:rax, being the content of [r8] now */
202 jne ExpInterlockedPopEntrySListResume16
203
204 /* Copy the old NextEntry pointer to rax */
205 mov rax, rdx
206 and rax, SLIST16B_NEXTENTRY_MASK
207
208 /* Return */
209 pop rbx
210 ret
211
212 RtlInterlockedPopEntrySListEmpty16:
213 xor rax, rax
214 pop rbx
215 ret
216
217
218 /* PSLIST_ENTRY
219 * NTAPI
220 * RtlInterlockedPushEntrySList(
221 * IN PSLIST_HEADER ListHead,
222 * IN PSLIST_ENTRY ListEntry);
223 */
224 RtlInterlockedPushEntrySList:
225 ExpInterlockedPushEntrySList:
226
227 #if DBG
228 /* Make sure the ListEntry is 16 bytes aligned */
229 test rdx, HEX(0F)
230 jz ExpInterlockedPushEntrySListChecked
231 /* Not aligned, raise an assertion */
232 int HEX(2C)
233 ExpInterlockedPushEntrySListChecked:
234
235 /* Make sure RtlpUse16ByteSLists is initialized */
236 cmp byte ptr [RtlpUse16ByteSLists], HEX(FF)
237 jne ExpInterlockedPushEntrySListChecked2
238 /* Not initialized, raise an assertion */
239 int HEX(2C)
240 ExpInterlockedPushEntrySListChecked2:
241 #endif
242
243 /* Load ListHead->Alignment into rax */
244 mov rax, [rcx]
245
246 /* Load ListHead->Region into r9 */
247 mov r9, [rcx + 8]
248
249 /* Check for 16 byte SList support */
250 cmp byte ptr [RtlpUse16ByteSLists], 0
251 jne RtlInterlockedPushEntrySList16
252
253 /* Use the 8 byte header */
254
255 RtlInterlockedPushEntrySListLoop:
256
257 /* Get ListHead->NextEntry */
258 mov r8, rax
259 and r8, SLIST8A_NEXTENTRY_MASK
260 jz RtlInterlockedPushEntrySListEmpty
261
262 /* Shift the NextEntry pointer */
263 shr r8, SLIST8A_NEXTENTRY_SHIFT
264
265 /* Create a pointer template from rcx in rdx */
266 mov r9, (NOT SLIST8_POINTER_MASK)
267 and r9, rcx
268
269 /* Combine to new pointer and save as ListEntry->NextEntry */
270 or r8, r9
271
272 RtlInterlockedPushEntrySListEmpty:
273 /* Store the NextEntry pointer in the new ListEntry */
274 mov [rdx], r8
275
276 /* Shift and mask the new ListEntry pointer */
277 mov r8, rdx
278 shl r8, SLIST8A_NEXTENTRY_SHIFT
279 and r8, SLIST8A_NEXTENTRY_MASK
280
281 /* Copy and adjust depth and sequence number */
282 lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
283 and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
284
285 /* Combine to exchange value in r8 */
286 or r8, r9
287
288 /* Save the NextEntry in r9 */
289 mov r9, [rdx]
290
291 /* If [rcx] equals rax, exchange it with r8 */
292 lock cmpxchg [rcx], r8
293
294 /* If not equal, retry with rax, being the content of [rcx] now */
295 jne RtlInterlockedPushEntrySListLoop
296
297 /* Return the old NextEntry pointer */
298 mov rax, r9
299 ret
300
301 RtlInterlockedPushEntrySList16:
302 /* This is a 16 byte header
303 rcx = ListHead
304 rdx = ListEntry
305 rax = ListHead->Alignment
306 r9 = ListHead->Region */
307
308 /* Save rbx */
309 push rbx
310
311 /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
312 mov r8, rcx
313 mov r9, rdx
314
315 /* Set NewListHead.NextEntry = ListEntry */
316 mov rcx, rdx
317
318 /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
319 or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
320
321 /* Set rdx = ListHead->Region */
322 mov rdx, [r8 + 8]
323
324 RtlInterlockedPushEntrySListLoop16:
325 /* r8 = ListHead
326 r9 = ListEntry
327 rax = ListHead->Alignment
328 rdx = ListHead->Region
329 */
330
331 /* Move ListHead->NextEntry to rbx */
332 mov rbx, rdx
333 and rbx, SLIST16B_NEXTENTRY_MASK
334
335 /* Store next pointer in ListEntry->Next */
336 mov [r9], rbx
337
338 /* Copy and increment Depth and Sequence number to rbx */
339 lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
340
341 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
342 lock cmpxchg16b [r8]
343
344 /* If not equal, retry with rdx:rax, being the content of [r8] now */
345 jne RtlInterlockedPushEntrySListLoop16
346
347 /* Copy the old NextEntry pointer to rax */
348 mov rax, rdx
349 and rax, SLIST16B_NEXTENTRY_MASK
350
351 /* Return */
352 pop rbx
353 ret
354
355
356 /* PSLIST_ENTRY
357 * NTAPI
358 * RtlInterlockedFlushSList(
359 * IN PSLIST_HEADER ListHead);
360 */
361 RtlInterlockedFlushSList:
362 ExpInterlockedFlushSList:
363
364 /* Load ListHead->Region into rdx */
365 mov rdx, [rcx + 8]
366
367 /* Load ListHead->Alignment into rax */
368 mov rax, [rcx]
369
370 /* Check for 16 byte SList support */
371 cmp byte ptr [RtlpUse16ByteSLists], 0
372 jne RtlInterlockedFlushSList16
373
374 /* Use the 8 byte header */
375
376 RtlInterlockedFlushSListLoop:
377
378 /* Zero NewListHead.Alignment */
379 xor r8, r8
380
381 /* If [rcx] equals rax, exchange it with r8 */
382 lock cmpxchg [rcx], r8
383
384 /* If not equal, retry with rax, being the content of [rcx] now */
385 jne RtlInterlockedFlushSListLoop
386
387 /* Create a pointer template from rcx in rdx */
388 mov rdx, (NOT SLIST8_POINTER_MASK)
389 and rdx, rcx
390
391 /* Load the old NextEntry pointer into rax */
392 and rax, SLIST8A_NEXTENTRY_MASK
393 shr rax, SLIST8A_NEXTENTRY_SHIFT
394
395 /* Combine result and return */
396 or rax, rdx
397 ret
398
399 RtlInterlockedFlushSList16:
400 /* We have a 16 byte header
401 rcx = ListHead
402 rax = ListHead->Alignment
403 rdx = ListHead->Region
404 */
405
406 /* Save rbx */
407 push rbx
408
409 /* Load ListHead into r8, as we need rcx for the exchange */
410 mov r8, rcx
411
412 /* Initialize an ampty NewListHead in rcx:rbx */
413 xor rbx, rbx
414 mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
415
416 RtlInterlockedFlushSListLoop16:
417
418 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
419 lock cmpxchg16b [r8]
420
421 /* If not equal, retry with rdx:rax, being the content of [r8] now */
422 jne RtlInterlockedFlushSListLoop16
423
424 /* Copy the old NextEntry pointer to rax */
425 mov rax, rdx
426 and rax, SLIST16B_NEXTENTRY_MASK
427
428 /* Return */
429 pop rbx
430 ret
431
432 END
433