73fee1097705bb9c2145a2c5ab9ef81e20392849
[reactos.git] / dll / apisets / update.py
1 '''
2 PROJECT: ReactOS apisets generator
3 LICENSE: MIT (https://spdx.org/licenses/MIT)
4 PURPOSE: Create apiset forwarders based on Wine apisets
5 COPYRIGHT: Copyright 2017,2018 Mark Jansen (mark.jansen@reactos.org)
6 '''
7
8 import os
9 import re
10 import sys
11 from collections import defaultdict
12 import subprocess
13
14
15 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
16
17 NL_CHAR = '\n'
18
19 IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64', '-register', '-private',
20 '-noname', '-ordinal', '-i386', '-arch=', '-stub', '-version=')
21
22 # Figure these out later
23 FUNCTION_BLACKLIST = [
24 # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
25 # error C2169: '_abs64': intrinsic function, cannot be defined
26 '_abs64',
27 '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
28 '_rotl64', '_rotr64',
29 ]
30
31 SPEC_HEADER = [
32 '\n',
33 '# This file is autogenerated by update.py\n',
34 '\n'
35 ]
36
37 ALIAS_DLL = {
38 'ucrtbase': 'msvcrt',
39 'kernelbase': 'kernel32',
40 'shcore': 'shlwapi',
41 'combase': 'ole32',
42
43 # These modules cannot be linked against in ROS, so forward it
44 'cfgmgr32': 'setupapi', # Forward everything
45 'wmi': 'advapi32', # Forward everything
46 }
47
48 class InvalidSpecError(Exception):
49 def __init__(self, message):
50 Exception.__init__(self, message)
51
52 class Arch(object):
53 none = 0
54 i386 = 1
55 x86_64 = 2
56 arm = 4
57 arm64 = 8
58 Any = i386 | x86_64 | arm | arm64
59
60 FROM_STR = {
61 'i386': i386,
62 'x86_64': x86_64,
63 'arm': arm,
64 'arm64': arm64,
65 'any': Any,
66 'win32': i386,
67 'win64': x86_64,
68 }
69
70 TO_STR = {
71 i386: 'i386',
72 x86_64: 'x86_64',
73 arm: 'arm',
74 arm64: 'arm64',
75 }
76
77 def __init__(self, initial=none):
78 self._val = initial
79
80 def add(self, text):
81 self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
82 assert self._val != 0
83
84 def has(self, val):
85 return (self._val & val) != 0
86
87 def to_str(self):
88 arch_str = []
89 for value in Arch.TO_STR:
90 if value & self._val:
91 arch_str.append(Arch.TO_STR[value])
92 return ','.join(arch_str)
93
94 def __len__(self):
95 return bin(self._val).count("1")
96
97 def __add__(self, other):
98 return Arch(self._val | other._val) # pylint: disable=W0212
99
100 def __sub__(self, other):
101 return Arch(self._val & ~other._val) # pylint: disable=W0212
102
103 def __gt__(self, other):
104 return self._val > other._val # pylint: disable=W0212
105
106 def __lt__(self, other):
107 return self._val < other._val # pylint: disable=W0212
108
109 def __eq__(self, other):
110 return self._val == other._val # pylint: disable=W0212
111
112 def __ne__(self, other):
113 return not self.__eq__(other)
114
115
116 class VersionExpr(object):
117 def __init__(self, text):
118 self.text = text
119 self.parse()
120
121 def parse(self):
122 pass
123
124 def to_str(self):
125 if self.text:
126 return '-version={}'.format(self.text)
127 return ''
128
129
130 class SpecEntry(object):
131 def __init__(self, text, spec):
132 self.spec = spec
133 self._ord = None
134 self.callconv = None
135 self.name = None
136 self.arch = Arch()
137 self.version = None
138 self._forwarder = None
139 self.init(text)
140 self.noname = False
141 if self.name == '@':
142 self.noname = True
143 if self._forwarder:
144 self.name = self._forwarder[1]
145
146 def init(self, text):
147 tokens = re.split(r'([\s\(\)#;])', text.strip())
148 tokens = [token for token in tokens if token and not token.isspace()]
149 idx = []
150 for comment in ['#', ';']:
151 if comment in tokens:
152 idx.append(tokens.index(comment))
153 idx = sorted(idx)
154 if idx:
155 tokens = tokens[:idx[0]]
156 if not tokens:
157 raise InvalidSpecError(text)
158 self._ord = tokens[0]
159 assert self._ord == '@' or self._ord.isdigit(), text
160 tokens = tokens[1:]
161 self.callconv = tokens.pop(0)
162 self.name = tokens.pop(0)
163 while self.name.startswith(IGNORE_OPTIONS):
164 if self.name.startswith('-arch='):
165 self.arch.add(self.name[6:])
166 elif self.name.startswith('-version='):
167 self.version = VersionExpr(self.name[9:])
168 elif self.name == '-i386':
169 self.arch.add('i386')
170 self.name = tokens.pop(0)
171 if not self.arch:
172 self.arch = Arch(Arch.Any)
173 assert not self.name.startswith('-'), text
174 if not tokens:
175 return
176 if tokens[0] == '(':
177 assert ')' in tokens, text
178 arg = tokens.pop(0)
179 while True:
180 arg = tokens.pop(0)
181 if arg == ')':
182 break
183 if not tokens:
184 return
185 assert len(tokens) == 1, text
186 self._forwarder = tokens.pop(0).split('.', 2)
187 if len(self._forwarder) == 1:
188 self._forwarder = ['self', self._forwarder[0]]
189 assert len(self._forwarder) in (0, 2), self._forwarder
190 if self._forwarder[0] in ALIAS_DLL:
191 self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
192
193 def resolve_forwarders(self, module_lookup, try_modules):
194 if self._forwarder:
195 assert self._forwarder[1] == self.name, '{}:{}'.format(self._forwarder[1], self.name)
196 if self.noname and self.name == '@':
197 return 0 # cannot search for this function
198 self._forwarder = []
199 self.arch = Arch()
200 for module_name in try_modules:
201 assert module_name in module_lookup, module_name
202 module = module_lookup[module_name]
203 fwd_arch = module.find_arch(self.name)
204 callconv = module.find_callconv(self.name)
205 version = module.find_version(self.name)
206 if fwd_arch:
207 self.arch = fwd_arch
208 self._forwarder = [module_name, self.name]
209 self.callconv = callconv
210 self.version = version
211 return 1
212 return 0
213
214 def extra_forwarders(self, function_lookup, module_lookup):
215 if self._forwarder:
216 return 1
217 if self.noname and self.name == '@':
218 return 0 # cannot search for this function
219 lst = function_lookup.get(self.name, None)
220 if lst:
221 modules = list(set([func.spec.name for func in lst]))
222 if len(modules) > 1:
223 mod = None
224 arch = Arch()
225 for module in modules:
226 mod_arch = module_lookup[module].find_arch(self.name)
227 if mod is None or mod_arch > arch:
228 mod = module
229 arch = mod_arch
230 modules = [mod]
231 mod = modules[0]
232 self._forwarder = [mod, self.name]
233 mod = module_lookup[mod]
234 self.arch = mod.find_arch(self.name)
235 self.callconv = mod.find_callconv(self.name)
236 self.version = mod.find_version(self.name)
237 return 1
238 return 0
239
240 def forwarder_module(self):
241 if self._forwarder:
242 return self._forwarder[0]
243
244 def forwarder(self):
245 if self._forwarder:
246 return 1
247 return 0
248
249 def write(self, spec_file):
250 name = self.name
251 opts = ''
252 estimate_size = 0
253 if self.noname:
254 opts = '{} -noname'.format(opts)
255 if self.version:
256 opts = '{} {}'.format(opts, self.version.to_str())
257 if self.name == '@':
258 assert self._ord != '@'
259 name = 'Ordinal' + self._ord
260 if not self._forwarder:
261 spec_file.write('{} stub{} {}{}'.format(self._ord, opts, name, NL_CHAR))
262 estimate_size += 0x1000
263 else:
264 assert self.arch != Arch(), self.name
265 args = '()'
266 callconv = 'stdcall'
267 fwd = '.'.join(self._forwarder)
268 name = self.name if not self.noname else '@'
269 arch = self.arch
270 if self.callconv == 'extern':
271 args = ''
272 callconv = 'extern -stub' # HACK
273 fwd += ' # the -stub is a HACK to fix VS < 2017 build!'
274 if arch != Arch(Arch.Any):
275 opts = '{} -arch={}'.format(opts, arch.to_str())
276 spec_file.write('{ord} {cc}{opts} {name}{args} {fwd}{nl}'.format(ord=self._ord,
277 cc=callconv,
278 opts=opts,
279 name=name,
280 args=args,
281 fwd=fwd,
282 nl=NL_CHAR))
283 estimate_size += 0x100
284 return estimate_size
285
286
287
288 class SpecFile(object):
289 def __init__(self, fullpath, name):
290 self._path = fullpath
291 self.name = name
292 self._entries = []
293 self._functions = defaultdict(list)
294 self._estimate_size = 0
295
296 def parse(self):
297 with open(self._path, 'rb') as specfile:
298 for line in specfile.readlines():
299 if line:
300 try:
301 entry = SpecEntry(line, self)
302 self._entries.append(entry)
303 self._functions[entry.name].append(entry)
304 except InvalidSpecError:
305 pass
306 return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
307
308 def add_functions(self, function_lookup):
309 for entry in self._entries:
310 function_lookup[entry.name].append(entry)
311
312 def find(self, name):
313 return self._functions.get(name, None)
314
315 def find_arch(self, name):
316 functions = self.find(name)
317 arch = Arch()
318 if functions:
319 for func in functions:
320 arch += func.arch
321 return arch
322
323 def find_callconv(self, name):
324 functions = self.find(name)
325 callconv = None
326 if functions:
327 for func in functions:
328 if not callconv:
329 callconv = func.callconv
330 elif callconv != func.callconv:
331 assert callconv != 'extern', 'Cannot have data/function with same name'
332 callconv = func.callconv
333 return callconv
334
335 def find_version(self, name):
336 functions = self.find(name)
337 version = None
338 if functions:
339 for func in functions:
340 if not func.version:
341 continue
342 if version:
343 assert version.text == func.version.text
344 version = func.version
345 return version
346
347 def resolve_forwarders(self, module_lookup):
348 modules = self.forwarder_modules()
349 total = 0
350 for entry in self._entries:
351 total += entry.resolve_forwarders(module_lookup, modules)
352 return (total, len(self._entries))
353
354 def extra_forwarders(self, function_lookup, module_lookup):
355 total = 0
356 for entry in self._entries:
357 total += entry.extra_forwarders(function_lookup, module_lookup)
358 return (total, len(self._entries))
359
360 def forwarder_modules(self):
361 modules = defaultdict(int)
362 for entry in self._entries:
363 module = entry.forwarder_module()
364 if module:
365 modules[module] += 1
366 return sorted(modules, key=modules.get, reverse=True)
367
368 def write(self, spec_file):
369 written = set(FUNCTION_BLACKLIST)
370 self._estimate_size = 0
371 for entry in self._entries:
372 if entry.name not in written:
373 self._estimate_size += entry.write(spec_file)
374 written.add(entry.name)
375
376 def write_cmake(self, cmakelists, baseaddress):
377 seen = set()
378 # ntdll and kernel32 are linked against everything, self = internal,
379 # we cannot link cfgmgr32 and wmi?
380 ignore = ['ntdll', 'kernel32', 'self', 'cfgmgr32', 'wmi']
381 forwarders = self.forwarder_modules()
382 fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or seen.add(x))]
383 fwd_strings = ' '.join(fwd_strings)
384 name = self.name
385 baseaddress = '0x{:8x}'.format(baseaddress)
386 cmakelists.write('add_apiset({} {} {}){}'.format(name, baseaddress, fwd_strings, NL_CHAR))
387 return self._estimate_size
388
389
390
391 def generate_specnames(dll_dir):
392 win32 = os.path.join(dll_dir, 'win32')
393 for dirname in os.listdir(win32):
394 fullpath = os.path.join(win32, dirname, dirname + '.spec')
395 if not os.path.isfile(fullpath):
396 if '.' in dirname:
397 fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0] + '.spec')
398 if not os.path.isfile(fullpath):
399 continue
400 else:
401 continue
402 yield (fullpath, dirname)
403 # Special cases
404 yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'), 'ntdll')
405 yield (os.path.join(dll_dir, 'appcompat', 'apphelp', 'apphelp.spec'), 'apphelp')
406 yield (os.path.join(dll_dir, '..', 'win32ss', 'user', 'user32', 'user32.spec'), 'user32')
407 yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32', 'gdi32.spec'), 'gdi32')
408
409 def run(wineroot):
410 global NL_CHAR
411 wine_apisets = []
412 ros_modules = []
413
414 module_lookup = {}
415 function_lookup = defaultdict(list)
416
417 version = subprocess.check_output(["git", "describe"], cwd=wineroot).strip()
418
419 print 'Reading Wine apisets for', version
420 wine_apiset_path = os.path.join(wineroot, 'dlls')
421 for dirname in os.listdir(wine_apiset_path):
422 if not dirname.startswith('api-'):
423 continue
424 if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
425 continue
426 fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
427 spec = SpecFile(fullpath, dirname)
428 wine_apisets.append(spec)
429
430 print 'Parsing Wine apisets,',
431 total = (0, 0)
432 for apiset in wine_apisets:
433 total = tuple(map(sum, zip(apiset.parse(), total)))
434 print 'found', total[0], '/', total[1], 'forwarders'
435
436 print 'Reading ReactOS modules'
437 for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
438 spec = SpecFile(fullpath, dllname)
439 ros_modules.append(spec)
440
441 print 'Parsing ReactOS modules'
442 for module in ros_modules:
443 module.parse()
444 assert module.name not in module_lookup, module.name
445 module_lookup[module.name] = module
446 module.add_functions(function_lookup)
447
448 print 'First pass, resolving forwarders,',
449 total = (0, 0)
450 for apiset in wine_apisets:
451 total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
452 print 'found', total[0], '/', total[1], 'forwarders'
453
454 print 'Second pass, searching extra forwarders,',
455 total = (0, 0)
456 for apiset in wine_apisets:
457 total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup, module_lookup), total)))
458 print 'found', total[0], '/', total[1], 'forwarders'
459
460 with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as template:
461 cmake_template = template.read()
462 cmake_template = cmake_template.replace('%WINE_GIT_VERSION%', version)
463 # Detect the checkout newline settings
464 if '\r\n' in cmake_template:
465 NL_CHAR = '\r\n'
466
467 manifest_files = []
468 print 'Writing apisets'
469 spec_header = [line.replace('\n', NL_CHAR) for line in SPEC_HEADER]
470 for apiset in wine_apisets:
471 with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb') as out_spec:
472 out_spec.writelines(spec_header)
473 apiset.write(out_spec)
474 manifest_files.append(' <file name="{}.dll"/>'.format(apiset.name))
475
476 print 'Generating manifest'
477 manifest_name = 'x86_reactos.apisets_6595b64144ccf1df_1.0.0.0_none_deadbeef.manifest'
478 with open(os.path.join(SCRIPT_DIR, manifest_name + '.in'), 'rb') as template:
479 manifest_template = template.read()
480 manifest_template = manifest_template.replace('%WINE_GIT_VERSION%', version)
481 file_list = '\r\n'.join(manifest_files)
482 manifest_template = manifest_template.replace('%MANIFEST_FILE_LIST%', file_list)
483 with open(os.path.join(SCRIPT_DIR, manifest_name), 'wb') as manifest:
484 manifest.write(manifest_template)
485
486 print 'Writing CMakeLists.txt'
487 baseaddress = 0x60000000
488 with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as cmakelists:
489 cmakelists.write(cmake_template)
490 for apiset in wine_apisets:
491 baseaddress += apiset.write_cmake(cmakelists, baseaddress)
492 baseaddress += (0x10000 - baseaddress) % 0x10000
493 print 'Done'
494
495 def main(paths):
496 for path in paths:
497 if path:
498 run(path)
499 return
500 print 'No path specified,'
501 print 'either pass it as argument, or set the environment variable "WINE_SRC_ROOT"'
502
503 if __name__ == '__main__':
504 main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])