2 * COPYRIGHT: See COPYING in the top level directory
3 * PROJECT: ReactOS system libraries
4 * FILE: lib/rtl/amd64/interlck.S
5 * PURPOSE: Rtl Interlocked Functions for amd64
6 * PROGRAMMERS: Timo Kreuzer
10 #include <ksamd64.inc>
12 #define SLIST8A_DEPTH_MASK HEX(000000000000FFFF)
13 #define SLIST8A_DEPTH_INC HEX(0000000000000001)
14 #define SLIST8A_SEQUENCE_MASK HEX(0000000001FF0000)
15 #define SLIST8A_SEQUENCE_INC HEX(0000000000010000)
16 #define SLIST8A_NEXTENTRY_MASK HEX(FFFFFFFFFE000000)
17 #define SLIST8A_NEXTENTRY_SHIFT 21
18 #define SLIST8B_HEADERTYPE_MASK HEX(0000000000000001)
19 #define SLIST8B_INIT_MASK HEX(0000000000000002)
20 #define SLIST8B_REGION_MASK HEX(E000000000000000)
21 #define SLIST8_POINTER_MASK HEX(000007FFFFFFFFFF)
23 #define SLIST16A_DEPTH_MASK HEX(000000000000FFFF)
24 #define SLIST16A_DEPTH_INC HEX(0000000000000001)
25 #define SLIST16A_SEQUENCE_MASK HEX(FFFFFFFFFFFF0000)
26 #define SLIST16A_SEQUENCE_INC HEX(0000000000010000)
27 #define SLIST16B_HEADERTYPE_MASK HEX(0000000000000001)
28 #define SLIST16B_INIT_MASK HEX(0000000000000002)
29 #define SLIST16B_NEXTENTY_MASK HEX(FFFFFFFFFFFFFFF0)
32 /* FUNCTIONS ****************************************************************/
36 PUBLIC ExpInterlockedPopEntrySList
37 PUBLIC ExpInterlockedPopEntrySListResume
38 PUBLIC ExpInterlockedPopEntrySListFault
39 PUBLIC ExpInterlockedPopEntrySListEnd
40 PUBLIC ExpInterlockedPopEntrySListResume16
41 PUBLIC ExpInterlockedPopEntrySListFault16
42 PUBLIC ExpInterlockedPopEntrySListEnd16
43 PUBLIC ExpInterlockedPushEntrySList
44 PUBLIC ExpInterlockedFlushSList
46 PUBLIC RtlInterlockedFlushSList
47 PUBLIC RtlInterlockedPopEntrySList
48 PUBLIC RtlInterlockedPushEntrySList
52 * RtlInterlockedPopEntrySList(
53 * IN PSLIST_HEADER ListHead);
55 RtlInterlockedPopEntrySList:
56 ExpInterlockedPopEntrySList:
58 /* Load ListHead->Region into rdx */
61 /* Load ListHead->Alignment into rax */
64 /* Check what kind of header this is */
65 test rdx, SLIST8B_HEADERTYPE_MASK
66 jnz RtlInterlockedPopEntrySList16
68 /* We have an 8 byte header */
70 ExpInterlockedPopEntrySListResume:
72 /* Check if ListHead->NextEntry is NULL */
74 and r9, SLIST8A_NEXTENTRY_MASK
75 jz RtlInterlockedPopEntrySListEmpty
77 /* Copy Depth and Sequence number and adjust Depth */
78 lea r8, [rax - SLIST8A_DEPTH_INC]
79 and r8, (SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK)
81 /* Create a pointer template from rcx in rdx */
82 mov rdx, (NOT SLIST8_POINTER_MASK)
85 /* Shift the NextEntry pointer */
86 shr r9, SLIST8A_NEXTENTRY_SHIFT
88 /* Combine to new pointer in rdx */
91 ExpInterlockedPopEntrySListFault:
93 /* Load the next NextEntry pointer to r9 */
96 /* Shift bits in place */
97 shl r9, SLIST8A_NEXTENTRY_SHIFT
102 ExpInterlockedPopEntrySListEnd:
104 /* If [rcx] equals rax, exchange it with r8 */
105 lock cmpxchg [rcx], r8
107 /* If not equal, retry with rax, being the content of [rcx] now */
108 jnz ExpInterlockedPopEntrySListResume
110 /* Shift the pointer bits in place */
111 and rax, SLIST8A_NEXTENTRY_MASK
112 shr rax, SLIST8A_NEXTENTRY_SHIFT
114 /* Use rcx as pointer template */
115 mov rdx, (NOT SLIST8_POINTER_MASK)
118 /* Combine result and return */
122 RtlInterlockedPopEntrySListEmpty:
126 RtlInterlockedPopEntrySList16:
127 /* This is a 16 byte header */
132 /* Copy rcx to r8, as we need rcx for the exchange */
135 ExpInterlockedPopEntrySListResume16:
137 /* Check if ListHead->NextEntry is NULL */
139 and r9, SLIST16B_NEXTENTY_MASK
140 jz RtlInterlockedPopEntrySListEmpty16
142 ExpInterlockedPopEntrySListFault16:
144 /* Get next pointer */
147 /* Set ListHead->HeaderType = 1 and ListHead->Init = 1 */
150 /* Copy Depth and Sequence number and adjust Depth */
151 lea rbx, [rax - SLIST16A_DEPTH_INC]
153 ExpInterlockedPopEntrySListEnd16:
155 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
158 /* If not equal, retry with rdx:rax, being the content of [r8] now */
159 jnz ExpInterlockedPopEntrySListResume16
161 /* Copy the old NextEntry pointer to rax */
163 and rax, SLIST16B_NEXTENTY_MASK
169 RtlInterlockedPopEntrySListEmpty16:
177 * RtlInterlockedPushEntrySList(
178 * IN PSLIST_HEADER ListHead,
179 * IN PSLIST_ENTRY ListEntry);
181 RtlInterlockedPushEntrySList:
182 ExpInterlockedPushEntrySList:
185 /* Make sure the ListEntry is 16 bytes aligned */
187 jz ExpInterlockedPushEntrySListChecked
188 /* Not aligned, raise an assertion */
190 ExpInterlockedPushEntrySListChecked:
193 /* Load ListHead->Alignment into rax */
196 /* Load ListHead->Region into rdx */
199 /* Check what kind of header this is */
200 test r9, SLIST8B_HEADERTYPE_MASK
201 jnz RtlInterlockedPushEntrySList16
203 /* We have an 8 byte header */
205 RtlInterlockedPushEntrySListLoop:
207 /* Get ListHead->NextEntry */
209 and r8, SLIST8A_NEXTENTRY_MASK
210 jz RtlInterlockedPushEntrySListEmpty
212 /* Shift the NextEntry pointer */
213 shr r8, SLIST8A_NEXTENTRY_SHIFT
215 /* Create a pointer template from rcx in rdx */
216 mov r9, (NOT SLIST8_POINTER_MASK)
219 /* Combine to new pointer and save as ListEntry->NextEntry */
222 RtlInterlockedPushEntrySListEmpty:
223 /* Store the NextEntry pointer in the new ListEntry */
226 /* Shift and mask the new ListEntry pointer */
228 shl r8, SLIST8A_NEXTENTRY_SHIFT
229 and r8, SLIST8A_NEXTENTRY_MASK
231 /* Copy and adjust depth and sequence number */
232 lea r9, [rax + SLIST8A_DEPTH_INC + SLIST8A_SEQUENCE_INC]
233 and r9, SLIST8A_SEQUENCE_MASK OR SLIST8A_DEPTH_MASK
235 /* Combine to exchange value in r8 */
238 /* Save the NextEntry in r9 */
241 /* If [rcx] equals rax, exchange it with r8 */
242 lock cmpxchg [rcx], r8
244 /* If not equal, retry with rax, being the content of [rcx] now */
245 jnz RtlInterlockedPushEntrySListLoop
247 /* Return the old NextEntry pointer */
251 RtlInterlockedPushEntrySList16:
252 /* This is a 16 byte header */
257 /* Copy rcx/rdx to r8/r9, as we need rcx/rdx for the exchange */
261 /* Set ListHead->HeaderType = 1 and ListHead->Init = 1 */
267 RtlInterlockedPushEntrySListLoop16:
269 /* Move ListHead->NextEntry to rbx */
271 and rbx, SLIST16B_NEXTENTY_MASK
273 /* Store next pointer in ListEntry->NextEntry */
276 /* Copy Depth and Sequence number and adjust Depth */
277 lea rbx, [rax + SLIST16A_DEPTH_INC + SLIST16A_SEQUENCE_INC]
279 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
282 /* If not equal, retry with rdx:rax, being the content of [r8] now */
283 jnz RtlInterlockedPushEntrySListLoop16
285 /* Copy the old NextEntry pointer to rax */
287 and rax, SLIST16B_NEXTENTY_MASK
296 * RtlInterlockedFlushSList(
297 * IN PSINGLE_LIST_ENTRY ListHead);
299 RtlInterlockedFlushSList:
300 ExpInterlockedFlushSList:
302 /* Load ListHead->Region into rdx */
305 /* Check what kind of header this is */
306 test rax, SLIST8B_HEADERTYPE_MASK
307 jnz RtlInterlockedFlushSList16
309 /* We have an 8 byte header */
311 RtlInterlockedFlushSListLoop:
313 /* Zero ListHead->Alignment */
316 /* If [rcx] equals rax, exchange it with r8 */
317 lock cmpxchg [rcx], r8
319 /* If not equal, retry with rax, being the content of [rcx] now */
320 jnz RtlInterlockedFlushSListLoop
322 /* Use rcx as pointer template */
323 mov rdx, (not SLIST8_POINTER_MASK)
326 /* Combine result and return */
330 RtlInterlockedFlushSList16:
331 /* We have a 16 byte header */
338 RtlInterlockedFlushSListLoop16:
340 /* If [r8] equals rdx:rax, exchange it with rcx:rbx */
343 /* If not equal, retry with rdx:rax, being the content of [r8] now */
344 jnz RtlInterlockedFlushSListLoop16
346 /* Copy the old NextEntry pointer to rax */
348 and rax, SLIST16B_NEXTENTY_MASK