dll/apisets/update.py

   1 '''
   2 PROJECT:     ReactOS apisets generator
   3 LICENSE:     MIT (https://spdx.org/licenses/MIT)
   4 PURPOSE:     Create apiset forwarders based on Wine apisets
   5 COPYRIGHT:   Copyright 2017,2018 Mark Jansen (mark.jansen@reactos.org)
   6 '''
   7
   8 import os
   9 import re
  10 import sys
  11 from collections import defaultdict
  12 import subprocess
  13
  14
  15 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  16
  17 NL_CHAR = '\n'
  18
  19 IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64', '-register', '-private',
  20                   '-noname', '-ordinal', '-i386', '-arch=', '-stub')
  21
  22 # Figure these out later
  23 FUNCTION_BLACKLIST = [
  24     # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
  25     # error C2169: '_abs64': intrinsic function, cannot be defined
  26     '_abs64',
  27     '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
  28     '_rotl64', '_rotr64',
  29 ]
  30
  31 SPEC_HEADER = [
  32     '\n',
  33     '# This file is autogenerated by update.py\n',
  34     '\n'
  35 ]
  36
  37
  38 class InvalidSpecError(Exception):
  39     def __init__(self, message):
  40         Exception.__init__(self, message)
  41
  42 class Arch(object):
  43     none = 0
  44     i386 = 1
  45     x86_64 = 2
  46     arm = 4
  47     arm64 = 8
  48     Any = i386 | x86_64 | arm | arm64
  49
  50     FROM_STR = {
  51         'i386': i386,
  52         'x86_64': x86_64,
  53         'arm': arm,
  54         'arm64': arm64,
  55         'any': Any,
  56         'win32': i386,
  57         'win64': x86_64,
  58     }
  59
  60     TO_STR = {
  61         i386: 'i386',
  62         x86_64: 'x86_64',
  63         arm: 'arm',
  64         arm64: 'arm64',
  65     }
  66
  67     def __init__(self, initial=none):
  68         self._val = initial
  69
  70     def add(self, text):
  71         self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
  72         assert self._val != 0
  73
  74     def has(self, val):
  75         return (self._val & val) != 0
  76
  77     def to_str(self):
  78         arch_str = []
  79         for value in Arch.TO_STR:
  80             if value & self._val:
  81                 arch_str.append(Arch.TO_STR[value])
  82         return ','.join(arch_str)
  83
  84     def __len__(self):
  85         return bin(self._val).count("1")
  86
  87     def __add__(self, other):
  88         return Arch(self._val | other._val) # pylint: disable=W0212
  89
  90     def __sub__(self, other):
  91         return Arch(self._val & ~other._val) # pylint: disable=W0212
  92
  93     def __gt__(self, other):
  94         return self._val > other._val       # pylint: disable=W0212
  95
  96     def __lt__(self, other):
  97         return self._val < other._val       # pylint: disable=W0212
  98
  99     def __eq__(self, other):
 100         return self._val == other._val      # pylint: disable=W0212
 101
 102     def __ne__(self, other):
 103         return not self.__eq__(other)
 104
 105 ALIAS_DLL = {
 106     'ucrtbase': 'msvcrt',
 107     'kernelbase': 'kernel32',
 108     'shcore': 'shlwapi',
 109     'combase': 'ole32',
 110
 111     # These modules cannot be linked against in ROS, so forward it
 112     'cfgmgr32': 'setupapi', # Forward everything
 113     'wmi': 'advapi32',      # Forward everything
 114 }
 115
 116 class SpecEntry(object):
 117     def __init__(self, text, spec):
 118         self.spec = spec
 119         self._ord = None
 120         self.callconv = None
 121         self.name = None
 122         self.arch = Arch()
 123         self._forwarder = None
 124         self.init(text)
 125         self.noname = False
 126         if self.name == '@':
 127             self.noname = True
 128             if self._forwarder:
 129                 self.name = self._forwarder[1]
 130
 131     def init(self, text):
 132         tokens = re.split(r'([\s\(\)#;])', text.strip())
 133         tokens = [token for token in tokens if token and not token.isspace()]
 134         idx = []
 135         for comment in ['#', ';']:
 136             if comment in tokens:
 137                 idx.append(tokens.index(comment))
 138         idx = sorted(idx)
 139         if idx:
 140             tokens = tokens[:idx[0]]
 141         if not tokens:
 142             raise InvalidSpecError(text)
 143         self._ord = tokens[0]
 144         assert self._ord == '@' or self._ord.isdigit(), text
 145         tokens = tokens[1:]
 146         self.callconv = tokens.pop(0)
 147         self.name = tokens.pop(0)
 148         while self.name.startswith(IGNORE_OPTIONS):
 149             if self.name.startswith('-arch='):
 150                 self.arch.add(self.name[6:])
 151             elif self.name == '-i386':
 152                 self.arch.add('i386')
 153             self.name = tokens.pop(0)
 154         if not self.arch:
 155             self.arch = Arch(Arch.Any)
 156         assert not self.name.startswith('-'), text
 157         if not tokens:
 158             return
 159         if tokens[0] == '(':
 160             assert ')' in tokens, text
 161             arg = tokens.pop(0)
 162             while True:
 163                 arg = tokens.pop(0)
 164                 if arg == ')':
 165                     break
 166         if not tokens:
 167             return
 168         assert len(tokens) == 1, text
 169         self._forwarder = tokens.pop(0).split('.', 2)
 170         if len(self._forwarder) == 1:
 171             self._forwarder = ['self', self._forwarder[0]]
 172         assert len(self._forwarder) in (0, 2), self._forwarder
 173         if self._forwarder[0] in ALIAS_DLL:
 174             self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
 175
 176     def resolve_forwarders(self, module_lookup, try_modules):
 177         if self._forwarder:
 178             assert self._forwarder[1] == self.name, '{}:{}'.format(self._forwarder[1], self.name)
 179         if self.noname and self.name == '@':
 180             return 0    # cannot search for this function
 181         self._forwarder = []
 182         self.arch = Arch()
 183         for module_name in try_modules:
 184             assert module_name in module_lookup, module_name
 185             module = module_lookup[module_name]
 186             fwd_arch = module.find_arch(self.name)
 187             callconv = module.find_callconv(self.name)
 188             if fwd_arch:
 189                 self.arch = fwd_arch
 190                 self._forwarder = [module_name, self.name]
 191                 self.callconv = callconv
 192                 return 1
 193         return 0
 194
 195     def extra_forwarders(self, function_lookup, module_lookup):
 196         if self._forwarder:
 197             return 1
 198         if self.noname and self.name == '@':
 199             return 0    # cannot search for this function
 200         lst = function_lookup.get(self.name, None)
 201         if lst:
 202             modules = list(set([func.spec.name for func in lst]))
 203             if len(modules) > 1:
 204                 mod = None
 205                 arch = Arch()
 206                 for module in modules:
 207                     mod_arch = module_lookup[module].find_arch(self.name)
 208                     if mod is None or mod_arch > arch:
 209                         mod = module
 210                         arch = mod_arch
 211                 modules = [mod]
 212             mod = modules[0]
 213             self._forwarder = [mod, self.name]
 214             mod = module_lookup[mod]
 215             self.arch = mod.find_arch(self.name)
 216             self.callconv = mod.find_callconv(self.name)
 217             return 1
 218         return 0
 219
 220     def forwarder_module(self):
 221         if self._forwarder:
 222             return self._forwarder[0]
 223
 224     def forwarder(self):
 225         if self._forwarder:
 226             return 1
 227         return 0
 228
 229     def write(self, spec_file):
 230         name = self.name
 231         opts = ''
 232         estimate_size = 0
 233         if self.noname:
 234             opts = '{} -noname'.format(opts)
 235         if self.name == '@':
 236             assert self._ord != '@'
 237             name = 'Ordinal' + self._ord
 238         if not self._forwarder:
 239             spec_file.write('{} stub{} {}{}'.format(self._ord, opts, name, NL_CHAR))
 240             estimate_size += 0x1000
 241         else:
 242             assert self.arch != Arch(), self.name
 243             args = '()'
 244             callconv = 'stdcall'
 245             fwd = '.'.join(self._forwarder)
 246             name = self.name if not self.noname else '@'
 247             arch = self.arch
 248             if self.callconv == 'extern':
 249                 args = ''
 250                 callconv = 'extern -stub'   # HACK
 251                 fwd += ' # the -stub is a HACK to fix VS < 2017 build!'
 252             if arch != Arch(Arch.Any):
 253                 opts = '{} -arch={}'.format(opts, arch.to_str())
 254             spec_file.write('{ord} {cc}{opts} {name}{args} {fwd}{nl}'.format(ord=self._ord,
 255                                                                              cc=callconv,
 256                                                                              opts=opts,
 257                                                                              name=name,
 258                                                                              args=args,
 259                                                                              fwd=fwd,
 260                                                                              nl=NL_CHAR))
 261             estimate_size += 0x100
 262         return estimate_size
 263
 264
 265
 266 class SpecFile(object):
 267     def __init__(self, fullpath, name):
 268         self._path = fullpath
 269         self.name = name
 270         self._entries = []
 271         self._functions = defaultdict(list)
 272         self._estimate_size = 0
 273
 274     def parse(self):
 275         with open(self._path, 'rb') as specfile:
 276             for line in specfile.readlines():
 277                 if line:
 278                     try:
 279                         entry = SpecEntry(line, self)
 280                         self._entries.append(entry)
 281                         self._functions[entry.name].append(entry)
 282                     except InvalidSpecError:
 283                         pass
 284         return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
 285
 286     def add_functions(self, function_lookup):
 287         for entry in self._entries:
 288             function_lookup[entry.name].append(entry)
 289
 290     def find(self, name):
 291         return self._functions.get(name, None)
 292
 293     def find_arch(self, name):
 294         functions = self.find(name)
 295         arch = Arch()
 296         if functions:
 297             for func in functions:
 298                 arch += func.arch
 299         return arch
 300
 301     def find_callconv(self, name):
 302         functions = self.find(name)
 303         callconv = None
 304         if functions:
 305             for func in functions:
 306                 if not callconv:
 307                     callconv = func.callconv
 308                 elif callconv != func.callconv:
 309                     assert callconv != 'extern', 'Cannot have data/function with same name'
 310                     callconv = func.callconv
 311         return callconv
 312
 313     def resolve_forwarders(self, module_lookup):
 314         modules = self.forwarder_modules()
 315         total = 0
 316         for entry in self._entries:
 317             total += entry.resolve_forwarders(module_lookup, modules)
 318         return (total, len(self._entries))
 319
 320     def extra_forwarders(self, function_lookup, module_lookup):
 321         total = 0
 322         for entry in self._entries:
 323             total += entry.extra_forwarders(function_lookup, module_lookup)
 324         return (total, len(self._entries))
 325
 326     def forwarder_modules(self):
 327         modules = defaultdict(int)
 328         for entry in self._entries:
 329             module = entry.forwarder_module()
 330             if module:
 331                 modules[module] += 1
 332         return sorted(modules, key=modules.get, reverse=True)
 333
 334     def write(self, spec_file):
 335         written = set(FUNCTION_BLACKLIST)
 336         self._estimate_size = 0
 337         for entry in self._entries:
 338             if entry.name not in written:
 339                 self._estimate_size += entry.write(spec_file)
 340                 written.add(entry.name)
 341
 342     def write_cmake(self, cmakelists, baseaddress):
 343         seen = set()
 344         # ntdll and kernel32 are linked against everything, self = internal,
 345         # we cannot link cfgmgr32 and wmi?
 346         ignore = ['ntdll', 'kernel32', 'self', 'cfgmgr32', 'wmi']
 347         forwarders = self.forwarder_modules()
 348         fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or seen.add(x))]
 349         fwd_strings = ' '.join(fwd_strings)
 350         name = self.name
 351         baseaddress = '0x{:8x}'.format(baseaddress)
 352         cmakelists.write('add_apiset({} {} {}){}'.format(name, baseaddress, fwd_strings, NL_CHAR))
 353         return self._estimate_size
 354
 355
 356
 357 def generate_specnames(dll_dir):
 358     win32 = os.path.join(dll_dir, 'win32')
 359     for dirname in os.listdir(win32):
 360         fullpath = os.path.join(win32, dirname, dirname + '.spec')
 361         if not os.path.isfile(fullpath):
 362             if '.' in dirname:
 363                 fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0] + '.spec')
 364                 if not os.path.isfile(fullpath):
 365                     continue
 366             else:
 367                 continue
 368         yield (fullpath, dirname)
 369     # Special cases
 370     yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'), 'ntdll')
 371     yield (os.path.join(dll_dir, 'appcompat', 'apphelp', 'apphelp.spec'), 'apphelp')
 372     yield (os.path.join(dll_dir, '..', 'win32ss', 'user', 'user32', 'user32.spec'), 'user32')
 373     yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32', 'gdi32.spec'), 'gdi32')
 374
 375 def run(wineroot):
 376     global NL_CHAR
 377     wine_apisets = []
 378     ros_modules = []
 379
 380     module_lookup = {}
 381     function_lookup = defaultdict(list)
 382
 383     version = subprocess.check_output(["git", "describe"], cwd=wineroot).strip()
 384
 385     print 'Reading Wine apisets for', version
 386     wine_apiset_path = os.path.join(wineroot, 'dlls')
 387     for dirname in os.listdir(wine_apiset_path):
 388         if not dirname.startswith('api-'):
 389             continue
 390         if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
 391             continue
 392         fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
 393         spec = SpecFile(fullpath, dirname)
 394         wine_apisets.append(spec)
 395
 396     print 'Parsing Wine apisets,',
 397     total = (0, 0)
 398     for apiset in wine_apisets:
 399         total = tuple(map(sum, zip(apiset.parse(), total)))
 400     print 'found', total[0], '/', total[1], 'forwarders'
 401
 402     print 'Reading ReactOS modules'
 403     for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
 404         spec = SpecFile(fullpath, dllname)
 405         ros_modules.append(spec)
 406
 407     print 'Parsing ReactOS modules'
 408     for module in ros_modules:
 409         module.parse()
 410         assert module.name not in module_lookup, module.name
 411         module_lookup[module.name] = module
 412         module.add_functions(function_lookup)
 413
 414     print 'First pass, resolving forwarders,',
 415     total = (0, 0)
 416     for apiset in wine_apisets:
 417         total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
 418     print 'found', total[0], '/', total[1], 'forwarders'
 419
 420     print 'Second pass, searching extra forwarders,',
 421     total = (0, 0)
 422     for apiset in wine_apisets:
 423         total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup, module_lookup), total)))
 424     print 'found', total[0], '/', total[1], 'forwarders'
 425
 426     with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as template:
 427         data = template.read()
 428         data = data.replace('%WINE_GIT_VERSION%', version)
 429         # Detect the checkout newline settings
 430         if '\r\n' in data:
 431             NL_CHAR = '\r\n'
 432
 433     print 'Writing apisets'
 434     spec_header = [line.replace('\n', NL_CHAR) for line in SPEC_HEADER]
 435     for apiset in wine_apisets:
 436         with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb') as out_spec:
 437             out_spec.writelines(spec_header)
 438             apiset.write(out_spec)
 439
 440     print 'Writing CMakeLists.txt'
 441     baseaddress = 0x60000000
 442     with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as cmakelists:
 443         cmakelists.write(data)
 444         for apiset in wine_apisets:
 445             baseaddress += apiset.write_cmake(cmakelists, baseaddress)
 446             baseaddress += (0x10000 - baseaddress) % 0x10000
 447     print 'Done'
 448
 449 def main(paths):
 450     for path in paths:
 451         if path:
 452             run(path)
 453             return
 454     print 'No path specified,'
 455     print 'either pass it as argument, or set the environment variable "WINE_SRC_ROOT"'
 456
 457 if __name__ == '__main__':
 458     main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])