dll/apisets/update.py

   1 '''
   2 PROJECT:     ReactOS apisets generator
   3 LICENSE:     MIT (https://spdx.org/licenses/MIT)
   4 PURPOSE:     Create apiset forwarders based on Wine apisets
   5 COPYRIGHT:   Copyright 2017,2018 Mark Jansen (mark.jansen@reactos.org)
   6 '''
   7
   8 import os
   9 import re
  10 import sys
  11 from collections import defaultdict
  12 import subprocess
  13
  14
  15 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  16
  17 NL_CHAR = '\n'
  18
  19 IGNORE_OPTIONS = ('-norelay', '-ret16', '-ret64', '-register', '-private',
  20                   '-noname', '-ordinal', '-i386', '-arch=', '-stub', '-version=')
  21
  22 # Figure these out later
  23 FUNCTION_BLACKLIST = [
  24     # api-ms-win-crt-utility-l1-1-0_stubs.c(6):
  25     # error C2169: '_abs64': intrinsic function, cannot be defined
  26     '_abs64',
  27     '_byteswap_uint64', '_byteswap_ulong', '_byteswap_ushort',
  28     '_rotl64', '_rotr64',
  29 ]
  30
  31 SPEC_HEADER = [
  32     '\n',
  33     '# This file is autogenerated by update.py\n',
  34     '\n'
  35 ]
  36
  37 ALIAS_DLL = {
  38     'ucrtbase': 'msvcrt',
  39     'kernelbase': 'kernel32',
  40     'shcore': 'shlwapi',
  41     'combase': 'ole32',
  42
  43     # These modules cannot be linked against in ROS, so forward it
  44     'cfgmgr32': 'setupapi', # Forward everything
  45     'wmi': 'advapi32',      # Forward everything
  46 }
  47
  48 class InvalidSpecError(Exception):
  49     def __init__(self, message):
  50         Exception.__init__(self, message)
  51
  52 class Arch(object):
  53     none = 0
  54     i386 = 1
  55     x86_64 = 2
  56     arm = 4
  57     arm64 = 8
  58     Any = i386 | x86_64 | arm | arm64
  59
  60     FROM_STR = {
  61         'i386': i386,
  62         'x86_64': x86_64,
  63         'arm': arm,
  64         'arm64': arm64,
  65         'any': Any,
  66         'win32': i386,
  67         'win64': x86_64,
  68     }
  69
  70     TO_STR = {
  71         i386: 'i386',
  72         x86_64: 'x86_64',
  73         arm: 'arm',
  74         arm64: 'arm64',
  75     }
  76
  77     def __init__(self, initial=none):
  78         self._val = initial
  79
  80     def add(self, text):
  81         self._val |= sum([Arch.FROM_STR[arch] for arch in text.split(',')])
  82         assert self._val != 0
  83
  84     def has(self, val):
  85         return (self._val & val) != 0
  86
  87     def to_str(self):
  88         arch_str = []
  89         for value in Arch.TO_STR:
  90             if value & self._val:
  91                 arch_str.append(Arch.TO_STR[value])
  92         return ','.join(arch_str)
  93
  94     def __len__(self):
  95         return bin(self._val).count("1")
  96
  97     def __add__(self, other):
  98         return Arch(self._val | other._val) # pylint: disable=W0212
  99
 100     def __sub__(self, other):
 101         return Arch(self._val & ~other._val) # pylint: disable=W0212
 102
 103     def __gt__(self, other):
 104         return self._val > other._val       # pylint: disable=W0212
 105
 106     def __lt__(self, other):
 107         return self._val < other._val       # pylint: disable=W0212
 108
 109     def __eq__(self, other):
 110         return self._val == other._val      # pylint: disable=W0212
 111
 112     def __ne__(self, other):
 113         return not self.__eq__(other)
 114
 115
 116 class VersionExpr(object):
 117     def __init__(self, text):
 118         self.text = text
 119         self.parse()
 120
 121     def parse(self):
 122         pass
 123
 124     def to_str(self):
 125         if self.text:
 126             return '-version={}'.format(self.text)
 127         return ''
 128
 129
 130 class SpecEntry(object):
 131     def __init__(self, text, spec):
 132         self.spec = spec
 133         self._ord = None
 134         self.callconv = None
 135         self.name = None
 136         self.arch = Arch()
 137         self.version = None
 138         self._forwarder = None
 139         self.init(text)
 140         self.noname = False
 141         if self.name == '@':
 142             self.noname = True
 143             if self._forwarder:
 144                 self.name = self._forwarder[1]
 145
 146     def init(self, text):
 147         tokens = re.split(r'([\s\(\)#;])', text.strip())
 148         tokens = [token for token in tokens if token and not token.isspace()]
 149         idx = []
 150         for comment in ['#', ';']:
 151             if comment in tokens:
 152                 idx.append(tokens.index(comment))
 153         idx = sorted(idx)
 154         if idx:
 155             tokens = tokens[:idx[0]]
 156         if not tokens:
 157             raise InvalidSpecError(text)
 158         self._ord = tokens[0]
 159         assert self._ord == '@' or self._ord.isdigit(), text
 160         tokens = tokens[1:]
 161         self.callconv = tokens.pop(0)
 162         self.name = tokens.pop(0)
 163         while self.name.startswith(IGNORE_OPTIONS):
 164             if self.name.startswith('-arch='):
 165                 self.arch.add(self.name[6:])
 166             elif self.name.startswith('-version='):
 167                 self.version = VersionExpr(self.name[9:])
 168             elif self.name == '-i386':
 169                 self.arch.add('i386')
 170             self.name = tokens.pop(0)
 171         if not self.arch:
 172             self.arch = Arch(Arch.Any)
 173         assert not self.name.startswith('-'), text
 174         if not tokens:
 175             return
 176         if tokens[0] == '(':
 177             assert ')' in tokens, text
 178             arg = tokens.pop(0)
 179             while True:
 180                 arg = tokens.pop(0)
 181                 if arg == ')':
 182                     break
 183         if not tokens:
 184             return
 185         assert len(tokens) == 1, text
 186         self._forwarder = tokens.pop(0).split('.', 2)
 187         if len(self._forwarder) == 1:
 188             self._forwarder = ['self', self._forwarder[0]]
 189         assert len(self._forwarder) in (0, 2), self._forwarder
 190         if self._forwarder[0] in ALIAS_DLL:
 191             self._forwarder[0] = ALIAS_DLL[self._forwarder[0]]
 192
 193     def resolve_forwarders(self, module_lookup, try_modules):
 194         if self._forwarder:
 195             assert self._forwarder[1] == self.name, '{}:{}'.format(self._forwarder[1], self.name)
 196         if self.noname and self.name == '@':
 197             return 0    # cannot search for this function
 198         self._forwarder = []
 199         self.arch = Arch()
 200         for module_name in try_modules:
 201             assert module_name in module_lookup, module_name
 202             module = module_lookup[module_name]
 203             fwd_arch = module.find_arch(self.name)
 204             callconv = module.find_callconv(self.name)
 205             version = module.find_version(self.name)
 206             if fwd_arch:
 207                 self.arch = fwd_arch
 208                 self._forwarder = [module_name, self.name]
 209                 self.callconv = callconv
 210                 self.version = version
 211                 return 1
 212         return 0
 213
 214     def extra_forwarders(self, function_lookup, module_lookup):
 215         if self._forwarder:
 216             return 1
 217         if self.noname and self.name == '@':
 218             return 0    # cannot search for this function
 219         lst = function_lookup.get(self.name, None)
 220         if lst:
 221             modules = list(set([func.spec.name for func in lst]))
 222             if len(modules) > 1:
 223                 mod = None
 224                 arch = Arch()
 225                 for module in modules:
 226                     mod_arch = module_lookup[module].find_arch(self.name)
 227                     if mod is None or mod_arch > arch:
 228                         mod = module
 229                         arch = mod_arch
 230                 modules = [mod]
 231             mod = modules[0]
 232             self._forwarder = [mod, self.name]
 233             mod = module_lookup[mod]
 234             self.arch = mod.find_arch(self.name)
 235             self.callconv = mod.find_callconv(self.name)
 236             self.version = mod.find_version(self.name)
 237             return 1
 238         return 0
 239
 240     def forwarder_module(self):
 241         if self._forwarder:
 242             return self._forwarder[0]
 243
 244     def forwarder(self):
 245         if self._forwarder:
 246             return 1
 247         return 0
 248
 249     def write(self, spec_file):
 250         name = self.name
 251         opts = ''
 252         estimate_size = 0
 253         if self.noname:
 254             opts = '{} -noname'.format(opts)
 255         if self.version:
 256             opts = '{} {}'.format(opts, self.version.to_str())
 257         if self.name == '@':
 258             assert self._ord != '@'
 259             name = 'Ordinal' + self._ord
 260         if not self._forwarder:
 261             spec_file.write('{} stub{} {}{}'.format(self._ord, opts, name, NL_CHAR))
 262             estimate_size += 0x1000
 263         else:
 264             assert self.arch != Arch(), self.name
 265             args = '()'
 266             callconv = 'stdcall'
 267             fwd = '.'.join(self._forwarder)
 268             name = self.name if not self.noname else '@'
 269             arch = self.arch
 270             if self.callconv == 'extern':
 271                 args = ''
 272                 callconv = 'extern -stub'   # HACK
 273                 fwd += ' # the -stub is a HACK to fix VS < 2017 build!'
 274             if arch != Arch(Arch.Any):
 275                 opts = '{} -arch={}'.format(opts, arch.to_str())
 276             spec_file.write('{ord} {cc}{opts} {name}{args} {fwd}{nl}'.format(ord=self._ord,
 277                                                                              cc=callconv,
 278                                                                              opts=opts,
 279                                                                              name=name,
 280                                                                              args=args,
 281                                                                              fwd=fwd,
 282                                                                              nl=NL_CHAR))
 283             estimate_size += 0x100
 284         return estimate_size
 285
 286
 287
 288 class SpecFile(object):
 289     def __init__(self, fullpath, name):
 290         self._path = fullpath
 291         self.name = name
 292         self._entries = []
 293         self._functions = defaultdict(list)
 294         self._estimate_size = 0
 295
 296     def parse(self):
 297         with open(self._path, 'rb') as specfile:
 298             for line in specfile.readlines():
 299                 if line:
 300                     try:
 301                         entry = SpecEntry(line, self)
 302                         self._entries.append(entry)
 303                         self._functions[entry.name].append(entry)
 304                     except InvalidSpecError:
 305                         pass
 306         return (sum([entry.forwarder() for entry in self._entries]), len(self._entries))
 307
 308     def add_functions(self, function_lookup):
 309         for entry in self._entries:
 310             function_lookup[entry.name].append(entry)
 311
 312     def find(self, name):
 313         return self._functions.get(name, None)
 314
 315     def find_arch(self, name):
 316         functions = self.find(name)
 317         arch = Arch()
 318         if functions:
 319             for func in functions:
 320                 arch += func.arch
 321         return arch
 322
 323     def find_callconv(self, name):
 324         functions = self.find(name)
 325         callconv = None
 326         if functions:
 327             for func in functions:
 328                 if not callconv:
 329                     callconv = func.callconv
 330                 elif callconv != func.callconv:
 331                     assert callconv != 'extern', 'Cannot have data/function with same name'
 332                     callconv = func.callconv
 333         return callconv
 334
 335     def find_version(self, name):
 336         functions = self.find(name)
 337         version = None
 338         if functions:
 339             for func in functions:
 340                 if not func.version:
 341                     continue
 342                 if version:
 343                     assert version.text == func.version.text
 344                 version = func.version
 345         return version
 346
 347     def resolve_forwarders(self, module_lookup):
 348         modules = self.forwarder_modules()
 349         total = 0
 350         for entry in self._entries:
 351             total += entry.resolve_forwarders(module_lookup, modules)
 352         return (total, len(self._entries))
 353
 354     def extra_forwarders(self, function_lookup, module_lookup):
 355         total = 0
 356         for entry in self._entries:
 357             total += entry.extra_forwarders(function_lookup, module_lookup)
 358         return (total, len(self._entries))
 359
 360     def forwarder_modules(self):
 361         modules = defaultdict(int)
 362         for entry in self._entries:
 363             module = entry.forwarder_module()
 364             if module:
 365                 modules[module] += 1
 366         return sorted(modules, key=modules.get, reverse=True)
 367
 368     def write(self, spec_file):
 369         written = set(FUNCTION_BLACKLIST)
 370         self._estimate_size = 0
 371         for entry in self._entries:
 372             if entry.name not in written:
 373                 self._estimate_size += entry.write(spec_file)
 374                 written.add(entry.name)
 375
 376     def write_cmake(self, cmakelists, baseaddress):
 377         seen = set()
 378         # ntdll and kernel32 are linked against everything, self = internal,
 379         # we cannot link cfgmgr32 and wmi?
 380         ignore = ['ntdll', 'kernel32', 'self', 'cfgmgr32', 'wmi']
 381         forwarders = self.forwarder_modules()
 382         fwd_strings = [x for x in forwarders if not (x in seen or x in ignore or seen.add(x))]
 383         fwd_strings = ' '.join(fwd_strings)
 384         name = self.name
 385         baseaddress = '0x{:8x}'.format(baseaddress)
 386         cmakelists.write('add_apiset({} {} {}){}'.format(name, baseaddress, fwd_strings, NL_CHAR))
 387         return self._estimate_size
 388
 389
 390
 391 def generate_specnames(dll_dir):
 392     win32 = os.path.join(dll_dir, 'win32')
 393     for dirname in os.listdir(win32):
 394         fullpath = os.path.join(win32, dirname, dirname + '.spec')
 395         if not os.path.isfile(fullpath):
 396             if '.' in dirname:
 397                 fullpath = os.path.join(win32, dirname, dirname.rsplit('.', 1)[0] + '.spec')
 398                 if not os.path.isfile(fullpath):
 399                     continue
 400             else:
 401                 continue
 402         yield (fullpath, dirname)
 403     # Special cases
 404     yield (os.path.join(dll_dir, 'ntdll', 'def', 'ntdll.spec'), 'ntdll')
 405     yield (os.path.join(dll_dir, 'appcompat', 'apphelp', 'apphelp.spec'), 'apphelp')
 406     yield (os.path.join(dll_dir, '..', 'win32ss', 'user', 'user32', 'user32.spec'), 'user32')
 407     yield (os.path.join(dll_dir, '..', 'win32ss', 'gdi', 'gdi32', 'gdi32.spec'), 'gdi32')
 408
 409 def run(wineroot):
 410     global NL_CHAR
 411     wine_apisets = []
 412     ros_modules = []
 413
 414     module_lookup = {}
 415     function_lookup = defaultdict(list)
 416
 417     version = subprocess.check_output(["git", "describe"], cwd=wineroot).strip()
 418
 419     print 'Reading Wine apisets for', version
 420     wine_apiset_path = os.path.join(wineroot, 'dlls')
 421     for dirname in os.listdir(wine_apiset_path):
 422         if not dirname.startswith('api-'):
 423             continue
 424         if not os.path.isdir(os.path.join(wine_apiset_path, dirname)):
 425             continue
 426         fullpath = os.path.join(wine_apiset_path, dirname, dirname + '.spec')
 427         spec = SpecFile(fullpath, dirname)
 428         wine_apisets.append(spec)
 429
 430     print 'Parsing Wine apisets,',
 431     total = (0, 0)
 432     for apiset in wine_apisets:
 433         total = tuple(map(sum, zip(apiset.parse(), total)))
 434     print 'found', total[0], '/', total[1], 'forwarders'
 435
 436     print 'Reading ReactOS modules'
 437     for fullpath, dllname in generate_specnames(os.path.dirname(SCRIPT_DIR)):
 438         spec = SpecFile(fullpath, dllname)
 439         ros_modules.append(spec)
 440
 441     print 'Parsing ReactOS modules'
 442     for module in ros_modules:
 443         module.parse()
 444         assert module.name not in module_lookup, module.name
 445         module_lookup[module.name] = module
 446         module.add_functions(function_lookup)
 447
 448     print 'First pass, resolving forwarders,',
 449     total = (0, 0)
 450     for apiset in wine_apisets:
 451         total = tuple(map(sum, zip(apiset.resolve_forwarders(module_lookup), total)))
 452     print 'found', total[0], '/', total[1], 'forwarders'
 453
 454     print 'Second pass, searching extra forwarders,',
 455     total = (0, 0)
 456     for apiset in wine_apisets:
 457         total = tuple(map(sum, zip(apiset.extra_forwarders(function_lookup, module_lookup), total)))
 458     print 'found', total[0], '/', total[1], 'forwarders'
 459
 460     with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt.in'), 'rb') as template:
 461         data = template.read()
 462         data = data.replace('%WINE_GIT_VERSION%', version)
 463         # Detect the checkout newline settings
 464         if '\r\n' in data:
 465             NL_CHAR = '\r\n'
 466
 467     print 'Writing apisets'
 468     spec_header = [line.replace('\n', NL_CHAR) for line in SPEC_HEADER]
 469     for apiset in wine_apisets:
 470         with open(os.path.join(SCRIPT_DIR, apiset.name + '.spec'), 'wb') as out_spec:
 471             out_spec.writelines(spec_header)
 472             apiset.write(out_spec)
 473
 474     print 'Writing CMakeLists.txt'
 475     baseaddress = 0x60000000
 476     with open(os.path.join(SCRIPT_DIR, 'CMakeLists.txt'), 'wb') as cmakelists:
 477         cmakelists.write(data)
 478         for apiset in wine_apisets:
 479             baseaddress += apiset.write_cmake(cmakelists, baseaddress)
 480             baseaddress += (0x10000 - baseaddress) % 0x10000
 481     print 'Done'
 482
 483 def main(paths):
 484     for path in paths:
 485         if path:
 486             run(path)
 487             return
 488     print 'No path specified,'
 489     print 'either pass it as argument, or set the environment variable "WINE_SRC_ROOT"'
 490
 491 if __name__ == '__main__':
 492     main(sys.argv[1:] + [os.environ.get('WINE_SRC_ROOT')])