[RTL]
[reactos.git] / reactos / lib / rtl / amd64 / slist.S
1 /*
2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS system libraries
4 * FILE: lib/rtl/amd64/interlck.S
5 * PURPOSE: Rtl Interlocked Functions for amd64
6 * PROGRAMMERS: Timo Kreuzer
7 */
8
9 #include <asm.inc>
10 #include <ksamd64.inc>
11
12 /*
13 typedef union DECLSPEC_ALIGN(16) _SLIST_HEADER
14 {
15 struct
16 {
17 ULONGLONG Alignment;
18 ULONGLONG Region;
19 };
20 struct
21 {
22 ULONGLONG Depth:16;
23 ULONGLONG Sequence:9;
24 ULONGLONG NextEntry:39;
25 ULONGLONG HeaderType:1;
26 ULONGLONG Init:1;
27 ULONGLONG Reserved:59;
28 ULONGLONG Region:3;
29 } Header8;
30 struct
31 {
32 ULONGLONG Depth:16;
33 ULONGLONG Sequence:48;
34 ULONGLONG HeaderType:1;
35 ULONGLONG Init:1;
36 ULONGLONG Reserved:2;
37 ULONGLONG NextEntry:60;
38 } Header16;
39 struct
40 {
41 ULONGLONG Depth:16;
42 ULONGLONG Sequence:48;
43 ULONGLONG HeaderType:1;
44 ULONGLONG Reserved:3;
45 ULONGLONG NextEntry:60;
46 } HeaderX64;
47 } SLIST_HEADER, *PSLIST_HEADER;
48 */
49
50 #define SLIST8A_DEPTH_MASK HEX(000000000000FFFF)
51 #define SLIST8A_DEPTH_INC HEX(0000000000000001)
52 #define SLIST8A_SEQUENCE_MASK HEX(0000000001FF0000)
53 #define SLIST8A_SEQUENCE_INC HEX(0000000000010000)
54 #define SLIST8A_NEXTENTRY_MASK HEX(FFFFFFFFFE000000)
55 #define SLIST8A_NEXTENTRY_SHIFT 21
56 #define SLIST8B_HEADERTYPE_MASK HEX(0000000000000001)
57 #define SLIST8B_INIT_MASK HEX(0000000000000002)
58 #define SLIST8B_REGION_MASK HEX(E000000000000000)
59 #define SLIST8_POINTER_MASK HEX(000007FFFFFFFFFF)
60
61 #define SLIST16A_DEPTH_MASK HEX(000000000000FFFF)
62 #define SLIST16A_DEPTH_INC HEX(0000000000000001)
63 #define SLIST16A_SEQUENCE_MASK HEX(FFFFFFFFFFFF0000)
64 #define SLIST16A_SEQUENCE_INC HEX(0000000000010000)
65 #define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
66 #define SLIST16B_INIT_MASK HEX(0000000000000002)
67 #define SLIST16B_NEXTENTRY_MASK HEX(FFFFFFFFFFFFFFF0)
68
69
70 /* FUNCTIONS ****************************************************************/
71
72 .code64
73
74 PUBLIC ExpInterlockedPopEntrySList
75 PUBLIC ExpInterlockedPopEntrySListResume
76 PUBLIC ExpInterlockedPopEntrySListFault
77 PUBLIC ExpInterlockedPopEntrySListEnd
78 PUBLIC ExpInterlockedPopEntrySListResume16
79 PUBLIC ExpInterlockedPopEntrySListFault16
80 PUBLIC ExpInterlockedPopEntrySListEnd16
81 PUBLIC ExpInterlockedPushEntrySList
82 PUBLIC ExpInterlockedFlushSList
83
84 PUBLIC RtlInterlockedFlushSList
85 PUBLIC RtlInterlockedPopEntrySList
86 PUBLIC RtlInterlockedPushEntrySList
87
88 /* PSLIST_ENTRY
89 * NTAPI
90 * RtlInterlockedPopEntrySList(
91 * IN PSLIST_HEADER ListHead);
92 */
93 RtlInterlockedPopEntrySList:
94 ExpInterlockedPopEntrySList:
95
96 /* Load ListHead->Region into rdx */
97 mov rdx, [rcx + 8]
98
99 /* Load ListHead->Alignment into rax */
100 mov rax, [rcx]
101
102 /* Check what kind of header this is */
103 test rdx, SLIST8B_HEADERTYPE_MASK
104 jnz RtlInterlockedPopEntrySList16
105
106 /* We have an 8 byte header */
107
108 ExpInterlockedPopEntrySListResume:
109
110 /* Check if ListHead->NextEntry is NULL */
111 mov r9, rax
112 and r9, SLIST8A_NEXTENTRY_MASK
113 jz RtlInterlockedPopEntrySListEmpty
114
115 /* Copy Depth and Sequence number and adjust Depth */
116 lea r8, [rax - SLIST8A_DEPTH_INC]
117 and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
118
119 /* Create a pointer template from rcx in rdx */
120 mov rdx, (NOT SLIST8_POINTER_MASK)
121 and rdx, rcx
122
123 /* Shift the NextEntry pointer */
124 shr r9, SLIST8A_NEXTENTRY_SHIFT
125
126 /* Combine to new pointer in rdx */
127 or rdx, r9
128
129 ExpInterlockedPopEntrySListFault:
130
131 /* Load the next NextEntry pointer to r9 */
132 mov r9, [rdx]
133
134 /* Shift bits in place */
135 shl r9, SLIST8A_NEXTENTRY_SHIFT
136
137 /* Combine into r8 */
138 or r8, r9
139
140 ExpInterlockedPopEntrySListEnd:
141
142 /* If [rcx] equals rax, exchange it with r8 */
143 lock cmpxchg [rcx], r8
144
145 /* If not equal, retry with rax, being the content of [rcx] now */
146 jne ExpInterlockedPopEntrySListResume
147
148 /* Shift the pointer bits in place */
149 and rax, SLIST8A_NEXTENTRY_MASK
150 shr rax, SLIST8A_NEXTENTRY_SHIFT
151
152 /* Use rcx as pointer template */
153 mov rdx, (NOT SLIST8_POINTER_MASK)
154 and rdx, rcx
155
156 /* Combine result and return */
157 or rax, rdx
158 ret
159
160 RtlInterlockedPopEntrySListEmpty:
161 xor rax, rax
162 ret
163
164 RtlInterlockedPopEntrySList16:
165 /* This is a 16 byte header
166 rcx == ListHead
167 rdx == ListHead->Region
168 rax == ListHead->Alignment */
169
170 /* Save rbx */
171 push rbx
172
173 /* Copy rcx to r8, as we need rcx for the exchange */
174 mov r8, rcx
175
176 ExpInterlockedPopEntrySListResume16:
177
178 /* Set r9 = ListHead->NextEntry and check if it is NULL */
179 mov r9, rdx
180 and r9, SLIST16B_NEXTENTRY_MASK
181 jz RtlInterlockedPopEntrySListEmpty16
182
183 ExpInterlockedPopEntrySListFault16:
184
185 /* Set NewListHead.Next = ListHead->NextEntry->Next */
186 mov rcx, [r9]
187
188 /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
189 or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
190
191 /* Copy Depth and Sequence number and adjust Depth */
192 lea rbx, [rax - SLIST16A_DEPTH_INC]
193
194 ExpInterlockedPopEntrySListEnd16:
195
196 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
197 lock cmpxchg16b [r8]
198
199 /* If not equal, retry with rdx:rax, being the content of [r8] now */
200 jne ExpInterlockedPopEntrySListResume16
201
202 /* Copy the old NextEntry pointer to rax */
203 mov rax, rdx
204 and rax, SLIST16B_NEXTENTRY_MASK
205
206 /* Return */
207 pop rbx
208 ret
209
210 RtlInterlockedPopEntrySListEmpty16:
211 xor rax, rax
212 pop rbx
213 ret
214
215
216 /* PSLIST_ENTRY
217 * NTAPI
218 * RtlInterlockedPushEntrySList(
219 * IN PSLIST_HEADER ListHead,
220 * IN PSLIST_ENTRY ListEntry);
221 */
222 RtlInterlockedPushEntrySList:
223 ExpInterlockedPushEntrySList:
224
225 #if DBG
226 /* Make sure the ListEntry is 16 bytes aligned */
227 test rdx, HEX(0F)
228 jz ExpInterlockedPushEntrySListChecked
229 /* Not aligned, raise an assertion */
230 int HEX(2C)
231 ExpInterlockedPushEntrySListChecked:
232 #endif
233
234 /* Load ListHead->Alignment into rax */
235 mov rax, [rcx]
236
237 /* Load ListHead->Region into r9 */
238 mov r9, [rcx + 8]
239
240 /* Check what kind of header this is */
241 test r9, SLIST8B_HEADERTYPE_MASK
242 jnz RtlInterlockedPushEntrySList16
243
244 /* We have an 8 byte header */
245
246 RtlInterlockedPushEntrySListLoop:
247
248 /* Get ListHead->NextEntry */
249 mov r8, rax
250 and r8, SLIST8A_NEXTENTRY_MASK
251 jz RtlInterlockedPushEntrySListEmpty
252
253 /* Shift the NextEntry pointer */
254 shr r8, SLIST8A_NEXTENTRY_SHIFT
255
256 /* Create a pointer template from rcx in rdx */
257 mov r9, (NOT SLIST8_POINTER_MASK)
258 and r9, rcx
259
260 /* Combine to new pointer and save as ListEntry->NextEntry */
261 or r8, r9
262
263 RtlInterlockedPushEntrySListEmpty:
264 /* Store the NextEntry pointer in the new ListEntry */
265 mov [rdx], r8
266
267 /* Shift and mask the new ListEntry pointer */
268 mov r8, rdx
269 shl r8, SLIST8A_NEXTENTRY_SHIFT
270 and r8, SLIST8A_NEXTENTRY_MASK
271
272 /* Copy and adjust depth and sequence number */
273 lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
274 and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
275
276 /* Combine to exchange value in r8 */
277 or r8, r9
278
279 /* Save the NextEntry in r9 */
280 mov r9, [rdx]
281
282 /* If [rcx] equals rax, exchange it with r8 */
283 lock cmpxchg [rcx], r8
284
285 /* If not equal, retry with rax, being the content of [rcx] now */
286 jne RtlInterlockedPushEntrySListLoop
287
288 /* Return the old NextEntry pointer */
289 mov rax, r9
290 ret
291
292 RtlInterlockedPushEntrySList16:
293 /* This is a 16 byte header
294 rcx = ListHead
295 rdx = ListEntry
296 rax = ListHead->Alignment
297 r9 = ListHead->Region */
298
299 /* Save rbx */
300 push rbx
301
302 /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
303 mov r8, rcx
304 mov r9, rdx
305
306 /* Set NewListHead.NextEntry = ListEntry */
307 mov rcx, rdx
308
309 /* Set NewListHead.HeaderType = 1 and NewListHead.Init = 1 */
310 or rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
311
312 /* Set rdx = ListHead->Region */
313 mov rdx, [r8 + 8]
314
315 RtlInterlockedPushEntrySListLoop16:
316 /* r8 = ListHead
317 r9 = ListEntry
318 rax = ListHead->Alignment
319 rdx = ListHead->Region
320 */
321
322 /* Move ListHead->NextEntry to rbx */
323 mov rbx, rdx
324 and rbx, SLIST16B_NEXTENTRY_MASK
325
326 /* Store next pointer in ListEntry->Next */
327 mov [r9], rbx
328
329 /* Copy and increment Depth and Sequence number to rbx */
330 lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
331
332 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
333 lock cmpxchg16b [r8]
334
335 /* If not equal, retry with rdx:rax, being the content of [r8] now */
336 jne RtlInterlockedPushEntrySListLoop16
337
338 /* Copy the old NextEntry pointer to rax */
339 mov rax, rdx
340 and rax, SLIST16B_NEXTENTRY_MASK
341
342 /* Return */
343 pop rbx
344 ret
345
346
347 /* PSLIST_ENTRY
348 * NTAPI
349 * RtlInterlockedFlushSList(
350 * IN PSLIST_HEADER ListHead);
351 */
352 RtlInterlockedFlushSList:
353 ExpInterlockedFlushSList:
354
355 /* Load ListHead->Region into rdx */
356 mov rdx, [rcx + 8]
357
358 /* Load ListHead->Alignment into rax */
359 mov rax, [rcx]
360
361 /* Check what kind of header this is */
362 test rdx, SLIST8B_HEADERTYPE_MASK
363 jnz RtlInterlockedFlushSList16
364
365 /* We have an 8 byte header */
366
367 RtlInterlockedFlushSListLoop:
368
369 /* Zero NewListHead.Alignment */
370 xor r8, r8
371
372 /* If [rcx] equals rax, exchange it with r8 */
373 lock cmpxchg [rcx], r8
374
375 /* If not equal, retry with rax, being the content of [rcx] now */
376 jne RtlInterlockedFlushSListLoop
377
378 /* Create a pointer template from rcx in rdx */
379 mov rdx, (NOT SLIST8_POINTER_MASK)
380 and rdx, rcx
381
382 /* Load the old NextEntry pointer into rax */
383 and rax, SLIST8A_NEXTENTRY_MASK
384 shr rax, SLIST8A_NEXTENTRY_SHIFT
385
386 /* Combine result and return */
387 or rax, rdx
388 ret
389
390 RtlInterlockedFlushSList16:
391 /* We have a 16 byte header
392 rcx = ListHead
393 rax = ListHead->Alignment
394 rdx = ListHead->Region
395 */
396
397 /* Save rbx */
398 push rbx
399
400 /* Load ListHead into r8, as we need rcx for the exchange */
401 mov r8, rcx
402
403 /* Initialize an ampty NewListHead in rcx:rbx */
404 xor rbx, rbx
405 mov rcx, (SLIST16B_HEADERTYPE_MASK or SLIST16B_INIT_MASK)
406
407 RtlInterlockedFlushSListLoop16:
408
409 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
410 lock cmpxchg16b [r8]
411
412 /* If not equal, retry with rdx:rax, being the content of [r8] now */
413 jne RtlInterlockedFlushSListLoop16
414
415 /* Copy the old NextEntry pointer to rax */
416 mov rax, rdx
417 and rax, SLIST16B_NEXTENTRY_MASK
418
419 /* Return */
420 pop rbx
421 ret
422
423 END
424