sdk/tools/wpp/ppl.l

   1 /* -*-C-*-
   2  * Wrc preprocessor lexical analysis
   3  *
   4  * Copyright 1999-2000  Bertho A. Stultiens (BS)
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  *
  20  * History:
  21  * 24-Apr-2000 BS       - Started from scratch to restructure everything
  22  *                        and reintegrate the source into the wine-tree.
  23  * 04-Jan-2000 BS       - Added comments about the lexicographical
  24  *                        grammar to give some insight in the complexity.
  25  * 28-Dec-1999 BS       - Eliminated backing-up of the flexer by running
  26  *                        `flex -b' on the source. This results in some
  27  *                        weirdo extra rules, but a much faster scanner.
  28  * 23-Dec-1999 BS       - Started this file
  29  *
  30  *-------------------------------------------------------------------------
  31  * The preprocessor's lexographical grammar (approximately):
  32  *
  33  * pp           := {ws} # {ws} if {ws} {expr} {ws} \n
  34  *              |  {ws} # {ws} ifdef {ws} {id} {ws} \n
  35  *              |  {ws} # {ws} ifndef {ws} {id} {ws} \n
  36  *              |  {ws} # {ws} elif {ws} {expr} {ws} \n
  37  *              |  {ws} # {ws} else {ws} \n
  38  *              |  {ws} # {ws} endif {ws} \n
  39  *              |  {ws} # {ws} include {ws} < {anytext} > \n
  40  *              |  {ws} # {ws} include {ws} " {anytext} " \n
  41  *              |  {ws} # {ws} define {ws} {anytext} \n
  42  *              |  {ws} # {ws} define( {arglist} ) {ws} {expansion} \n
  43  *              |  {ws} # {ws} pragma {ws} {anytext} \n
  44  *              |  {ws} # {ws} ident {ws} {anytext} \n
  45  *              |  {ws} # {ws} error {ws} {anytext} \n
  46  *              |  {ws} # {ws} warning {ws} {anytext} \n
  47  *              |  {ws} # {ws} line {ws} " {anytext} " {number} \n
  48  *              |  {ws} # {ws} {number} " {anytext} " {number} [ {number} [{number}] ] \n
  49  *              |  {ws} # {ws} \n
  50  *
  51  * ws           := [ \t\r\f\v]*
  52  *
  53  * expr         := {expr} [+-*%^/|&] {expr}
  54  *              |  {expr} {logor|logand} {expr}
  55  *              |  [!~+-] {expr}
  56  *              |  {expr} ? {expr} : {expr}
  57  *
  58  * logor        := ||
  59  *
  60  * logand       := &&
  61  *
  62  * id           := [a-zA-Z_][a-zA-Z0-9_]*
  63  *
  64  * anytext      := [^\n]*       (see note)
  65  *
  66  * arglist      :=
  67  *              |  {id}
  68  *              |  {arglist} , {id}
  69  *              |  {arglist} , {id} ...
  70  *
  71  * expansion    := {id}
  72  *              |  # {id}
  73  *              |  {anytext}
  74  *              |  {anytext} ## {anytext}
  75  *
  76  * number       := [0-9]+
  77  *
  78  * Note: "anytext" is not always "[^\n]*". This is because the
  79  *       trailing context must be considered as well.
  80  *
  81  * The only certain assumption for the preprocessor to make is that
  82  * directives start at the beginning of the line, followed by a '#'
  83  * and end with a newline.
  84  * Any directive may be suffixed with a line-continuation. Also
  85  * classical comment / *...* / (note: no comments within comments,
  86  * therefore spaces) is considered to be a line-continuation
  87  * (according to gcc and egcs AFAIK, ANSI is a bit vague).
  88  * Comments have not been added to the above grammar for simplicity
  89  * reasons. However, it is allowed to enter comment anywhere within
  90  * the directives as long as they do not interfere with the context.
  91  * All comments are considered to be deletable whitespace (both
  92  * classical form "/ *...* /" and C++ form "//...\n").
  93  *
  94  * All recursive scans, except for macro-expansion, are done by the
  95  * parser, whereas the simple state transitions of non-recursive
  96  * directives are done in the scanner. This results in the many
  97  * exclusive start-conditions of the scanner.
  98  *
  99  * Macro expansions are slightly more difficult because they have to
 100  * prescan the arguments. Parameter substitution is literal if the
 101  * substitution is # or ## (either side). This enables new identifiers
 102  * to be created (see 'info cpp' node Macro|Pitfalls|Prescan for more
 103  * information).
 104  *
 105  * FIXME: Variable macro parameters is recognized, but not yet
 106  * expanded. I have to reread the ANSI standard on the subject (yes,
 107  * ANSI defines it).
 108  *
 109  * The following special defines are supported:
 110  * __FILE__     -> "thissource.c"
 111  * __LINE__     -> 123
 112  * __DATE__     -> "May  1 2000"
 113  * __TIME__     -> "23:59:59"
 114  * These macros expand, as expected, into their ANSI defined values.
 115  *
 116  * The same include prevention is implemented as gcc and egcs does.
 117  * This results in faster processing because we do not read the text
 118  * at all. Some wine-sources attempt to include the same file 4 or 5
 119  * times. This strategy also saves a lot blank output-lines, which in
 120  * its turn improves the real resource scanner/parser.
 121  *
 122  */
 123
 124 %top{
 125 #include "config.h"
 126 #include "wine/port.h"
 127 }
 128
 129 /*
 130  * Special flex options and exclusive scanner start-conditions
 131  */
 132 %option stack
 133 %option 8bit never-interactive
 134 %option noinput nounput
 135 %option prefix="ppy_"
 136
 137 %x pp_pp
 138 %x pp_eol
 139 %x pp_inc
 140 %x pp_dqs
 141 %x pp_sqs
 142 %x pp_iqs
 143 %x pp_comment
 144 %x pp_def
 145 %x pp_define
 146 %x pp_macro
 147 %x pp_mbody
 148 %x pp_macign
 149 %x pp_macscan
 150 %x pp_macexp
 151 %x pp_if
 152 %x pp_ifd
 153 %x pp_endif
 154 %x pp_line
 155 %x pp_defined
 156 %x pp_ignore
 157 %x RCINCL
 158
 159 ws      [ \v\f\t\r]
 160 cident  [a-zA-Z_][0-9a-zA-Z_]*
 161 ul      [uUlL]|[uUlL][lL]|[lL][uU]|[lL][lL][uU]|[uU][lL][lL]|[lL][uU][lL]
 162
 163 %{
 164 #include <stdio.h>
 165 #include <stdlib.h>
 166 #include <string.h>
 167 #include <ctype.h>
 168 #include <assert.h>
 169 #include <errno.h>
 170 #include <limits.h>
 171
 172 #ifndef LLONG_MAX
 173 # define LLONG_MAX  ((long long)0x7fffffff << 32 | 0xffffffff)
 174 # define LLONG_MIN  (-LLONG_MAX - 1)
 175 #endif
 176 #ifndef ULLONG_MAX
 177 # define ULLONG_MAX ((long long)0xffffffff << 32 | 0xffffffff)
 178 #endif
 179
 180 #ifndef HAVE_UNISTD_H
 181 #define YY_NO_UNISTD_H
 182 #endif
 183
 184 #include "wine/wpp.h"
 185 #include "wpp_private.h"
 186 #include "ppy.tab.h"
 187
 188 /*
 189  * Make sure that we are running an appropriate version of flex.
 190  */
 191 #if !defined(YY_FLEX_MAJOR_VERSION) || (1000 * YY_FLEX_MAJOR_VERSION + YY_FLEX_MINOR_VERSION < 2005)
 192 #error Must use flex version 2.5.1 or higher (yy_scan_* routines are required).
 193 #endif
 194
 195 #define YY_READ_BUF_SIZE        65536           /* So we read most of a file at once */
 196
 197 #define yy_current_state()      YY_START
 198 #define yy_pp_state(x)          yy_pop_state(); yy_push_state(x)
 199
 200 /*
 201  * Always update the current character position within a line
 202  */
 203 #define YY_USER_ACTION  pp_status.char_number+=ppy_leng;
 204
 205 /*
 206  * Buffer management for includes and expansions
 207  */
 208 #define MAXBUFFERSTACK  128     /* Nesting more than 128 includes or macro expansion textss is insane */
 209
 210 typedef struct bufferstackentry {
 211         YY_BUFFER_STATE bufferstate;    /* Buffer to switch back to */
 212         void            *filehandle;    /* Handle to be used with wpp_callbacks->read */
 213         pp_entry_t      *define;        /* Points to expanding define or NULL if handling includes */
 214         int             line_number;    /* Line that we were handling */
 215         int             char_number;    /* The current position on that line */
 216         char            *filename;      /* Filename that we were handling */
 217         int             if_depth;       /* How many #if:s deep to check matching #endif:s */
 218         int             ncontinuations; /* Remember the continuation state */
 219         int             should_pop;     /* Set if we must pop the start-state on EOF */
 220         /* Include management */
 221         include_state_t incl;
 222         char            *include_filename;
 223 } bufferstackentry_t;
 224
 225 #define ALLOCBLOCKSIZE  (1 << 10)       /* Allocate these chunks at a time for string-buffers */
 226
 227 /*
 228  * Macro expansion nesting
 229  * We need the stack to handle expansions while scanning
 230  * a macro's arguments. The TOS must always be the macro
 231  * that receives the current expansion from the scanner.
 232  */
 233 #define MAXMACEXPSTACK  128     /* Nesting more than 128 macro expansions is insane */
 234
 235 typedef struct macexpstackentry {
 236         pp_entry_t      *ppp;           /* This macro we are scanning */
 237         char            **args;         /* With these arguments */
 238         char            **ppargs;       /* Resulting in these preprocessed arguments */
 239         int             *nnls;          /* Number of newlines per argument */
 240         int             nargs;          /* And this many arguments scanned */
 241         int             parentheses;    /* Nesting level of () */
 242         int             curargsize;     /* Current scanning argument's size */
 243         int             curargalloc;    /* Current scanning argument's block allocated */
 244         char            *curarg;        /* Current scanning argument's content */
 245 } macexpstackentry_t;
 246
 247 #define MACROPARENTHESES()      (top_macro()->parentheses)
 248
 249 /*
 250  * Prototypes
 251  */
 252 static void newline(int);
 253 static int make_number(int radix, YYSTYPE *val, const char *str, int len);
 254 static void put_buffer(const char *s, int len);
 255 /* Buffer management */
 256 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop);
 257 static bufferstackentry_t *pop_buffer(void);
 258 /* String functions */
 259 static void new_string(void);
 260 static void add_string(const char *str, int len);
 261 static char *get_string(void);
 262 static void put_string(void);
 263 static int string_start(void);
 264 /* Macro functions */
 265 static void push_macro(pp_entry_t *ppp);
 266 static macexpstackentry_t *top_macro(void);
 267 static macexpstackentry_t *pop_macro(void);
 268 static void free_macro(macexpstackentry_t *mep);
 269 static void add_text_to_macro(const char *text, int len);
 270 static void macro_add_arg(int last);
 271 static void macro_add_expansion(void);
 272 /* Expansion */
 273 static void expand_special(pp_entry_t *ppp);
 274 static void expand_define(pp_entry_t *ppp);
 275 static void expand_macro(macexpstackentry_t *mep);
 276
 277 /*
 278  * Local variables
 279  */
 280 static int ncontinuations;
 281
 282 static int strbuf_idx = 0;
 283 static int strbuf_alloc = 0;
 284 static char *strbuffer = NULL;
 285 static int str_startline;
 286
 287 static macexpstackentry_t *macexpstack[MAXMACEXPSTACK];
 288 static int macexpstackidx = 0;
 289
 290 static bufferstackentry_t bufferstack[MAXBUFFERSTACK];
 291 static int bufferstackidx = 0;
 292
 293 /*
 294  * Global variables
 295  */
 296 include_state_t pp_incl_state =
 297 {
 298     -1,    /* state */
 299     NULL,  /* ppp */
 300     0,     /* ifdepth */
 301     0      /* seen_junk */
 302 };
 303
 304 includelogicentry_t *pp_includelogiclist = NULL;
 305
 306 #define YY_INPUT(buf,result,max_size)                                        \
 307         {                                                                    \
 308                 result = wpp_callbacks->read(pp_status.file, buf, max_size); \
 309         }
 310
 311 #define BUFFERINITIALCAPACITY 256
 312
 313 void pp_writestring(const char *format, ...)
 314 {
 315         va_list valist;
 316         int len;
 317         static char *buffer;
 318         static int buffercapacity;
 319         char *new_buffer;
 320
 321         if(buffercapacity == 0)
 322         {
 323                 buffer = pp_xmalloc(BUFFERINITIALCAPACITY);
 324                 if(buffer == NULL)
 325                         return;
 326                 buffercapacity = BUFFERINITIALCAPACITY;
 327         }
 328
 329         va_start(valist, format);
 330         len = vsnprintf(buffer, buffercapacity,
 331                         format, valist);
 332         va_end(valist);
 333         /* If the string is longer than buffersize, vsnprintf returns
 334          * the string length with glibc >= 2.1, -1 with glibc < 2.1 */
 335         while(len > buffercapacity || len < 0)
 336         {
 337                 do
 338                 {
 339                         buffercapacity *= 2;
 340                 } while(len > buffercapacity);
 341
 342                 new_buffer = pp_xrealloc(buffer, buffercapacity);
 343                 if(new_buffer == NULL)
 344                 {
 345                         va_end(valist);
 346                         return;
 347                 }
 348                 buffer = new_buffer;
 349                 va_start(valist, format);
 350                 len = vsnprintf(buffer, buffercapacity,
 351                                 format, valist);
 352                 va_end(valist);
 353         }
 354
 355         wpp_callbacks->write(buffer, len);
 356 }
 357
 358 %}
 359
 360 /*
 361  **************************************************************************
 362  * The scanner starts here
 363  **************************************************************************
 364  */
 365
 366 %%
 367         /*
 368          * Catch line-continuations.
 369          * Note: Gcc keeps the line-continuations in, for example, strings
 370          * intact. However, I prefer to remove them all so that the next
 371          * scanner will not need to reduce the continuation state.
 372          *
 373          * <*>\\\n              newline(0);
 374          */
 375
 376         /*
 377          * Detect the leading # of a preprocessor directive.
 378          */
 379 <INITIAL,pp_ignore>^{ws}*#      pp_incl_state.seen_junk++; yy_push_state(pp_pp);
 380
 381         /*
 382          * Scan for the preprocessor directives
 383          */
 384 <pp_pp>{ws}*include{ws}*        if(yy_top_state() != pp_ignore) {yy_pp_state(pp_inc); return tINCLUDE;} else {yy_pp_state(pp_eol);}
 385 <pp_pp>{ws}*define{ws}*         yy_pp_state(yy_current_state() != pp_ignore ? pp_def : pp_eol);
 386 <pp_pp>{ws}*error{ws}*          yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tERROR;
 387 <pp_pp>{ws}*warning{ws}*        yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tWARNING;
 388 <pp_pp>{ws}*pragma{ws}*         yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tPRAGMA;
 389 <pp_pp>{ws}*ident{ws}*          yy_pp_state(pp_eol);    if(yy_top_state() != pp_ignore) return tPPIDENT;
 390 <pp_pp>{ws}*undef{ws}*          if(yy_top_state() != pp_ignore) {yy_pp_state(pp_ifd); return tUNDEF;} else {yy_pp_state(pp_eol);}
 391 <pp_pp>{ws}*ifdef{ws}*          yy_pp_state(pp_ifd);    return tIFDEF;
 392 <pp_pp>{ws}*ifndef{ws}*         pp_incl_state.seen_junk--; yy_pp_state(pp_ifd); return tIFNDEF;
 393 <pp_pp>{ws}*if{ws}*             yy_pp_state(pp_if);     return tIF;
 394 <pp_pp>{ws}*elif{ws}*           yy_pp_state(pp_if);     return tELIF;
 395 <pp_pp>{ws}*else{ws}*           yy_pp_state(pp_endif);  return tELSE;
 396 <pp_pp>{ws}*endif{ws}*          yy_pp_state(pp_endif);  return tENDIF;
 397 <pp_pp>{ws}*line{ws}*           if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tLINE;} else {yy_pp_state(pp_eol);}
 398 <pp_pp>{ws}+                    if(yy_top_state() != pp_ignore) {yy_pp_state(pp_line); return tGCCLINE;} else {yy_pp_state(pp_eol);}
 399 <pp_pp>{ws}*[a-z]+              ppy_error("Invalid preprocessor token '%s'", ppy_text);
 400 <pp_pp>\r?\n                    newline(1); yy_pop_state(); return tNL; /* This could be the null-token */
 401 <pp_pp>\\\r?\n                  newline(0);
 402 <pp_pp>\\\r?                    ppy_error("Preprocessor junk '%s'", ppy_text);
 403 <pp_pp>.                        return *ppy_text;
 404
 405         /*
 406          * Handle #include and #line
 407          */
 408 <pp_line>[0-9]+                 return make_number(10, &ppy_lval, ppy_text, ppy_leng);
 409 <pp_inc>\<                      new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_iqs);
 410 <pp_inc,pp_line>\"              new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 411 <pp_inc,pp_line>{ws}+           ;
 412 <pp_inc,pp_line>\n              newline(1); yy_pop_state(); return tNL;
 413 <pp_inc,pp_line>\\\r?\n         newline(0);
 414 <pp_inc,pp_line>(\\\r?)|(.)     ppy_error(yy_current_state() == pp_inc ? "Trailing junk in #include" : "Trailing junk in #line");
 415
 416         /*
 417          * Ignore all input when a false clause is parsed
 418          */
 419 <pp_ignore>[^#/\\\n]+           ;
 420 <pp_ignore>\n                   newline(1);
 421 <pp_ignore>\\\r?\n              newline(0);
 422 <pp_ignore>(\\\r?)|(.)          ;
 423
 424         /*
 425          * Handle #if and #elif.
 426          * These require conditionals to be evaluated, but we do not
 427          * want to jam the scanner normally when we see these tokens.
 428          * Note: tIDENT is handled below.
 429          */
 430
 431 <pp_if>0[0-7]*{ul}?             return make_number(8, &ppy_lval, ppy_text, ppy_leng);
 432 <pp_if>0[0-7]*[8-9]+{ul}?       ppy_error("Invalid octal digit");
 433 <pp_if>[1-9][0-9]*{ul}?         return make_number(10, &ppy_lval, ppy_text, ppy_leng);
 434 <pp_if>0[xX][0-9a-fA-F]+{ul}?   return make_number(16, &ppy_lval, ppy_text, ppy_leng);
 435 <pp_if>0[xX]                    ppy_error("Invalid hex number");
 436 <pp_if>defined                  yy_push_state(pp_defined); return tDEFINED;
 437 <pp_if>"<<"                     return tLSHIFT;
 438 <pp_if>">>"                     return tRSHIFT;
 439 <pp_if>"&&"                     return tLOGAND;
 440 <pp_if>"||"                     return tLOGOR;
 441 <pp_if>"=="                     return tEQ;
 442 <pp_if>"!="                     return tNE;
 443 <pp_if>"<="                     return tLTE;
 444 <pp_if>">="                     return tGTE;
 445 <pp_if>\n                       newline(1); yy_pop_state(); return tNL;
 446 <pp_if>\\\r?\n                  newline(0);
 447 <pp_if>\\\r?                    ppy_error("Junk in conditional expression");
 448 <pp_if>{ws}+                    ;
 449 <pp_if>\'                       new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 450 <pp_if>\"                       ppy_error("String constants not allowed in conditionals");
 451 <pp_if>.                        return *ppy_text;
 452
 453         /*
 454          * Handle #ifdef, #ifndef and #undef
 455          * to get only an untranslated/unexpanded identifier
 456          */
 457 <pp_ifd>{cident}        ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 458 <pp_ifd>{ws}+           ;
 459 <pp_ifd>\n              newline(1); yy_pop_state(); return tNL;
 460 <pp_ifd>\\\r?\n         newline(0);
 461 <pp_ifd>(\\\r?)|(.)     ppy_error("Identifier expected");
 462
 463         /*
 464          * Handle #else and #endif.
 465          */
 466 <pp_endif>{ws}+         ;
 467 <pp_endif>\n            newline(1); yy_pop_state(); return tNL;
 468 <pp_endif>\\\r?\n       newline(0);
 469 <pp_endif>.             ppy_error("Garbage after #else or #endif.");
 470
 471         /*
 472          * Handle the special 'defined' keyword.
 473          * This is necessary to get the identifier prior to any
 474          * substitutions.
 475          */
 476 <pp_defined>{cident}            yy_pop_state(); ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 477 <pp_defined>{ws}+               ;
 478 <pp_defined>(\()|(\))           return *ppy_text;
 479 <pp_defined>\\\r?\n             newline(0);
 480 <pp_defined>(\\.)|(\n)|(.)      ppy_error("Identifier expected");
 481
 482         /*
 483          * Handle #error, #warning, #pragma and #ident.
 484          * Pass everything literally to the parser, which
 485          * will act appropriately.
 486          * Comments are stripped from the literal text.
 487          */
 488 <pp_eol>[^/\\\n]+               if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 489 <pp_eol>\/[^/\\\n*]*            if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 490 <pp_eol>(\\\r?)|(\/[^/*])       if(yy_top_state() != pp_ignore) { ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL; }
 491 <pp_eol>\n                      newline(1); yy_pop_state(); if(yy_current_state() != pp_ignore) { return tNL; }
 492 <pp_eol>\\\r?\n                 newline(0);
 493
 494         /*
 495          * Handle left side of #define
 496          */
 497 <pp_def>{cident}\(              ppy_lval.cptr = pp_xstrdup(ppy_text); if(ppy_lval.cptr) ppy_lval.cptr[ppy_leng-1] = '\0'; yy_pp_state(pp_macro);  return tMACRO;
 498 <pp_def>{cident}                ppy_lval.cptr = pp_xstrdup(ppy_text); yy_pp_state(pp_define); return tDEFINE;
 499 <pp_def>{ws}+                   ;
 500 <pp_def>\\\r?\n                 newline(0);
 501 <pp_def>(\\\r?)|(\n)|(.)        perror("Identifier expected");
 502
 503         /*
 504          * Scan the substitution of a define
 505          */
 506 <pp_define>[^'"/\\\n]+          ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 507 <pp_define>(\\\r?)|(\/[^/*])    ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 508 <pp_define>\\\r?\n{ws}+         newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
 509 <pp_define>\\\r?\n              newline(0);
 510 <pp_define>\n                   newline(1); yy_pop_state(); return tNL;
 511 <pp_define>\'                   new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 512 <pp_define>\"                   new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 513
 514         /*
 515          * Scan the definition macro arguments
 516          */
 517 <pp_macro>\){ws}*               yy_pp_state(pp_mbody); return tMACROEND;
 518 <pp_macro>{ws}+                 ;
 519 <pp_macro>{cident}              ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 520 <pp_macro>,                     return ',';
 521 <pp_macro>"..."                 return tELIPSIS;
 522 <pp_macro>(\\\r?)|(\n)|(.)|(\.\.?)      ppy_error("Argument identifier expected");
 523 <pp_macro>\\\r?\n               newline(0);
 524
 525         /*
 526          * Scan the substitution of a macro
 527          */
 528 <pp_mbody>[^a-zA-Z0-9'"#/\\\n]+ ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 529 <pp_mbody>{cident}              ppy_lval.cptr = pp_xstrdup(ppy_text); return tIDENT;
 530 <pp_mbody>\#\#                  return tCONCAT;
 531 <pp_mbody>\#                    return tSTRINGIZE;
 532 <pp_mbody>[0-9][a-zA-Z0-9]*[^a-zA-Z0-9'"#/\\\n]*        ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 533 <pp_mbody>(\\\r?)|(\/[^/*'"#\\\n]*)     ppy_lval.cptr = pp_xstrdup(ppy_text); return tLITERAL;
 534 <pp_mbody>\\\r?\n{ws}+          newline(0); ppy_lval.cptr = pp_xstrdup(" "); return tLITERAL;
 535 <pp_mbody>\\\r?\n               newline(0);
 536 <pp_mbody>\n                    newline(1); yy_pop_state(); return tNL;
 537 <pp_mbody>\'                    new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 538 <pp_mbody>\"                    new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 539
 540         /*
 541          * Macro expansion text scanning.
 542          * This state is active just after the identifier is scanned
 543          * that triggers an expansion. We *must* delete the leading
 544          * whitespace before we can start scanning for arguments.
 545          *
 546          * If we do not see a '(' as next trailing token, then we have
 547          * a false alarm. We just continue with a nose-bleed...
 548          */
 549 <pp_macign>{ws}*/\(     yy_pp_state(pp_macscan);
 550 <pp_macign>{ws}*\n      {
 551                 if(yy_top_state() != pp_macscan)
 552                         newline(0);
 553         }
 554 <pp_macign>{ws}*\\\r?\n newline(0);
 555 <pp_macign>{ws}+|{ws}*\\\r?|.   {
 556                 macexpstackentry_t *mac = pop_macro();
 557                 yy_pop_state();
 558                 put_buffer(mac->ppp->ident, strlen(mac->ppp->ident));
 559                 put_buffer(ppy_text, ppy_leng);
 560                 free_macro(mac);
 561         }
 562
 563         /*
 564          * Macro expansion argument text scanning.
 565          * This state is active when a macro's arguments are being read for expansion.
 566          */
 567 <pp_macscan>\(  {
 568                 if(++MACROPARENTHESES() > 1)
 569                         add_text_to_macro(ppy_text, ppy_leng);
 570         }
 571 <pp_macscan>\)  {
 572                 if(--MACROPARENTHESES() == 0)
 573                 {
 574                         yy_pop_state();
 575                         macro_add_arg(1);
 576                 }
 577                 else
 578                         add_text_to_macro(ppy_text, ppy_leng);
 579         }
 580 <pp_macscan>,           {
 581                 if(MACROPARENTHESES() > 1)
 582                         add_text_to_macro(ppy_text, ppy_leng);
 583                 else
 584                         macro_add_arg(0);
 585         }
 586 <pp_macscan>\"          new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 587 <pp_macscan>\'          new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 588 <pp_macscan>"/*"        yy_push_state(pp_comment); add_text_to_macro(" ", 1);
 589 <pp_macscan>\n          pp_status.line_number++; pp_status.char_number = 1; add_text_to_macro(ppy_text, ppy_leng);
 590 <pp_macscan>([^/(),\\\n"']+)|(\/[^/*(),\\\n'"]*)|(\\\r?)|(.)    add_text_to_macro(ppy_text, ppy_leng);
 591 <pp_macscan>\\\r?\n     newline(0);
 592
 593         /*
 594          * Comment handling (almost all start-conditions)
 595          */
 596 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,RCINCL>"/*" yy_push_state(pp_comment);
 597 <pp_comment>[^*\n]*|"*"+[^*/\n]*        ;
 598 <pp_comment>\n                          newline(0);
 599 <pp_comment>"*"+"/"                     yy_pop_state();
 600
 601         /*
 602          * Remove C++ style comment (almost all start-conditions)
 603          */
 604 <INITIAL,pp_pp,pp_ignore,pp_eol,pp_inc,pp_if,pp_ifd,pp_endif,pp_defined,pp_def,pp_define,pp_macro,pp_mbody,pp_macscan,RCINCL>"//"[^\n]* {
 605                 if(ppy_text[ppy_leng-1] == '\\')
 606                         ppy_warning("C++ style comment ends with an escaped newline (escape ignored)");
 607         }
 608
 609         /*
 610          * Single, double and <> quoted constants
 611          */
 612 <INITIAL,pp_macexp>\"           pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_dqs);
 613 <INITIAL,pp_macexp>\'           pp_incl_state.seen_junk++; new_string(); add_string(ppy_text, ppy_leng); yy_push_state(pp_sqs);
 614 <pp_dqs>[^"\\\n]+               add_string(ppy_text, ppy_leng);
 615 <pp_dqs>\"                      {
 616                 add_string(ppy_text, ppy_leng);
 617                 yy_pop_state();
 618                 switch(yy_current_state())
 619                 {
 620                 case pp_pp:
 621                 case pp_define:
 622                 case pp_mbody:
 623                 case pp_inc:
 624                 case RCINCL:
 625                         if (yy_current_state()==RCINCL) yy_pop_state();
 626                         ppy_lval.cptr = get_string();
 627                         return tDQSTRING;
 628                 case pp_line:
 629                         ppy_lval.cptr = get_string();
 630                         return tDQSTRING;
 631                 default:
 632                         put_string();
 633                 }
 634         }
 635 <pp_sqs>[^'\\\n]+               add_string(ppy_text, ppy_leng);
 636 <pp_sqs>\'                      {
 637                 add_string(ppy_text, ppy_leng);
 638                 yy_pop_state();
 639                 switch(yy_current_state())
 640                 {
 641                 case pp_if:
 642                 case pp_define:
 643                 case pp_mbody:
 644                         ppy_lval.cptr = get_string();
 645                         return tSQSTRING;
 646                 default:
 647                         put_string();
 648                 }
 649         }
 650 <pp_iqs>[^\>\\\n]+              add_string(ppy_text, ppy_leng);
 651 <pp_iqs>\>                      {
 652                 add_string(ppy_text, ppy_leng);
 653                 yy_pop_state();
 654                 ppy_lval.cptr = get_string();
 655                 return tIQSTRING;
 656         }
 657 <pp_dqs>\\\r?\n         {
 658                 /*
 659                  * This is tricky; we need to remove the line-continuation
 660                  * from preprocessor strings, but OTOH retain them in all
 661                  * other strings. This is because the resource grammar is
 662                  * even more braindead than initially analysed and line-
 663                  * continuations in strings introduce, sigh, newlines in
 664                  * the output. There goes the concept of non-breaking, non-
 665                  * spacing whitespace.
 666                  */
 667                 switch(yy_top_state())
 668                 {
 669                 case pp_pp:
 670                 case pp_define:
 671                 case pp_mbody:
 672                 case pp_inc:
 673                 case pp_line:
 674                         newline(0);
 675                         break;
 676                 default:
 677                         add_string(ppy_text, ppy_leng);
 678                         newline(-1);
 679                 }
 680         }
 681 <pp_iqs,pp_dqs,pp_sqs>\\.       add_string(ppy_text, ppy_leng);
 682 <pp_iqs,pp_dqs,pp_sqs>\n        {
 683                 newline(1);
 684                 add_string(ppy_text, ppy_leng);
 685                 ppy_warning("Newline in string constant encountered (started line %d)", string_start());
 686         }
 687
 688         /*
 689          * Identifier scanning
 690          */
 691 <INITIAL,pp_if,pp_inc,pp_macexp>{cident}        {
 692                 pp_entry_t *ppp;
 693                 pp_incl_state.seen_junk++;
 694                 if(!(ppp = pplookup(ppy_text)))
 695                 {
 696                         if(yy_current_state() == pp_inc)
 697                                 ppy_error("Expected include filename");
 698
 699                         else if(yy_current_state() == pp_if)
 700                         {
 701                                 ppy_lval.cptr = pp_xstrdup(ppy_text);
 702                                 return tIDENT;
 703                         }
 704                         else {
 705                                 if((yy_current_state()==INITIAL) && (strcasecmp(ppy_text,"RCINCLUDE")==0)){
 706                                         yy_push_state(RCINCL);
 707                                         return tRCINCLUDE;
 708                                 }
 709                                 else put_buffer(ppy_text, ppy_leng);
 710                         }
 711                 }
 712                 else if(!ppp->expanding)
 713                 {
 714                         switch(ppp->type)
 715                         {
 716                         case def_special:
 717                                 expand_special(ppp);
 718                                 break;
 719                         case def_define:
 720                                 expand_define(ppp);
 721                                 break;
 722                         case def_macro:
 723                                 yy_push_state(pp_macign);
 724                                 push_macro(ppp);
 725                                 break;
 726                         default:
 727                                 pp_internal_error(__FILE__, __LINE__, "Invalid define type %d\n", ppp->type);
 728                         }
 729                 }
 730                 else put_buffer(ppy_text, ppy_leng);
 731         }
 732
 733         /*
 734          * Everything else that needs to be passed and
 735          * newline and continuation handling
 736          */
 737 <INITIAL,pp_macexp>[^a-zA-Z_#'"/\\\n \r\t\f\v]+|(\/|\\)[^a-zA-Z_/*'"\\\n \r\t\v\f]*     pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
 738 <INITIAL,pp_macexp>{ws}+        put_buffer(ppy_text, ppy_leng);
 739 <INITIAL>\n                     newline(1);
 740 <INITIAL>\\\r?\n                newline(0);
 741 <INITIAL>\\\r?                  pp_incl_state.seen_junk++; put_buffer(ppy_text, ppy_leng);
 742
 743         /*
 744          * Special catcher for macro argmument expansion to prevent
 745          * newlines to propagate to the output or admin.
 746          */
 747 <pp_macexp>(\n)|(.)|(\\\r?(\n|.))       put_buffer(ppy_text, ppy_leng);
 748
 749 <RCINCL>[A-Za-z0-9_\.\\/]+ {
 750                 ppy_lval.cptr=pp_xstrdup(ppy_text);
 751                 yy_pop_state();
 752                 return tRCINCLUDEPATH;
 753         }
 754
 755 <RCINCL>{ws}+ ;
 756
 757 <RCINCL>\"              {
 758                 new_string(); add_string(ppy_text,ppy_leng);yy_push_state(pp_dqs);
 759         }
 760
 761         /*
 762          * This is a 'catch-all' rule to discover errors in the scanner
 763          * in an orderly manner.
 764          */
 765 <*>.            pp_incl_state.seen_junk++; ppy_warning("Unmatched text '%c' (0x%02x); please report\n", isprint(*ppy_text & 0xff) ? *ppy_text : ' ', *ppy_text);
 766
 767 <<EOF>> {
 768                 YY_BUFFER_STATE b = YY_CURRENT_BUFFER;
 769                 bufferstackentry_t *bep = pop_buffer();
 770
 771                 if((!bep && pp_get_if_depth()) || (bep && pp_get_if_depth() != bep->if_depth))
 772                         ppy_warning("Unmatched #if/#endif at end of file");
 773
 774                 if(!bep)
 775                 {
 776                         if(YY_START != INITIAL)
 777                         {
 778                                 ppy_error("Unexpected end of file during preprocessing");
 779                                 BEGIN(INITIAL);
 780                         }
 781                         yyterminate();
 782                 }
 783                 else if(bep->should_pop == 2)
 784                 {
 785                         macexpstackentry_t *mac;
 786                         mac = pop_macro();
 787                         expand_macro(mac);
 788                 }
 789                 ppy__delete_buffer(b);
 790         }
 791
 792 %%
 793 /*
 794  **************************************************************************
 795  * Support functions
 796  **************************************************************************
 797  */
 798
 799 #ifndef ppy_wrap
 800 int ppy_wrap(void)
 801 {
 802         return 1;
 803 }
 804 #endif
 805
 806
 807 /*
 808  *-------------------------------------------------------------------------
 809  * Output newlines or set them as continuations
 810  *
 811  * Input: -1 - Don't count this one, but update local position (see pp_dqs)
 812  *         0 - Line-continuation seen and cache output
 813  *         1 - Newline seen and flush output
 814  *-------------------------------------------------------------------------
 815  */
 816 static void newline(int dowrite)
 817 {
 818         pp_status.line_number++;
 819         pp_status.char_number = 1;
 820
 821         if(dowrite == -1)
 822                 return;
 823
 824         ncontinuations++;
 825         if(dowrite)
 826         {
 827                 for(;ncontinuations; ncontinuations--)
 828                         put_buffer("\n", 1);
 829         }
 830 }
 831
 832
 833 /*
 834  *-------------------------------------------------------------------------
 835  * Make a number out of an any-base and suffixed string
 836  *
 837  * Possible number extensions:
 838  * - ""         int
 839  * - "L"        long int
 840  * - "LL"       long long int
 841  * - "U"        unsigned int
 842  * - "UL"       unsigned long int
 843  * - "ULL"      unsigned long long int
 844  * - "LU"       unsigned long int
 845  * - "LLU"      unsigned long long int
 846  * - "LUL"      invalid
 847  *
 848  * FIXME:
 849  * The sizes of resulting 'int' and 'long' are compiler specific.
 850  * I depend on sizeof(int) > 2 here (although a relatively safe
 851  * assumption).
 852  * Long longs are not yet implemented because this is very compiler
 853  * specific and I don't want to think too much about the problems.
 854  *
 855  *-------------------------------------------------------------------------
 856  */
 857 static int make_number(int radix, YYSTYPE *val, const char *str, int len)
 858 {
 859         int is_l  = 0;
 860         int is_ll = 0;
 861         int is_u  = 0;
 862         char ext[4];
 863         long l;
 864
 865         ext[3] = '\0';
 866         ext[2] = toupper(str[len-1]);
 867         ext[1] = len > 1 ? toupper(str[len-2]) : ' ';
 868         ext[0] = len > 2 ? toupper(str[len-3]) : ' ';
 869
 870         if(!strcmp(ext, "LUL"))
 871         {
 872                 ppy_error("Invalid constant suffix");
 873                 return 0;
 874         }
 875         else if(!strcmp(ext, "LLU") || !strcmp(ext, "ULL"))
 876         {
 877                 is_ll++;
 878                 is_u++;
 879         }
 880         else if(!strcmp(ext+1, "LU") || !strcmp(ext+1, "UL"))
 881         {
 882                 is_l++;
 883                 is_u++;
 884         }
 885         else if(!strcmp(ext+1, "LL"))
 886         {
 887                 is_ll++;
 888         }
 889         else if(!strcmp(ext+2, "L"))
 890         {
 891                 is_l++;
 892         }
 893         else if(!strcmp(ext+2, "U"))
 894         {
 895                 is_u++;
 896         }
 897
 898         if(is_ll)
 899         {
 900 /* Assume as in the declaration of wrc_ull_t and wrc_sll_t */
 901 #ifdef HAVE_LONG_LONG
 902                 if (is_u)
 903                 {
 904                         errno = 0;
 905                         val->ull = strtoull(str, NULL, radix);
 906                         if (val->ull == ULLONG_MAX && errno == ERANGE)
 907                                 ppy_error("integer constant %s is too large\n", str);
 908                         return tULONGLONG;
 909                 }
 910                 else
 911                 {
 912                         errno = 0;
 913                         val->sll = strtoll(str, NULL, radix);
 914                         if ((val->sll == LLONG_MIN || val->sll == LLONG_MAX) && errno == ERANGE)
 915                                 ppy_error("integer constant %s is too large\n", str);
 916                         return tSLONGLONG;
 917                 }
 918 #else
 919                 pp_internal_error(__FILE__, __LINE__, "long long constants not supported on this platform");
 920 #endif
 921         }
 922         else if(is_u && is_l)
 923         {
 924                 errno = 0;
 925                 val->ulong = strtoul(str, NULL, radix);
 926                 if (val->ulong == ULONG_MAX && errno == ERANGE)
 927                         ppy_error("integer constant %s is too large\n", str);
 928                 return tULONG;
 929         }
 930         else if(!is_u && is_l)
 931         {
 932                 errno = 0;
 933                 val->slong = strtol(str, NULL, radix);
 934                 if ((val->slong == LONG_MIN || val->slong == LONG_MAX) && errno == ERANGE)
 935                         ppy_error("integer constant %s is too large\n", str);
 936                 return tSLONG;
 937         }
 938         else if(is_u && !is_l)
 939         {
 940                 unsigned long ul;
 941                 errno = 0;
 942                 ul = strtoul(str, NULL, radix);
 943                 if ((ul == ULONG_MAX && errno == ERANGE) || (ul > UINT_MAX))
 944                         ppy_error("integer constant %s is too large\n", str);
 945                 val->uint = (unsigned int)ul;
 946                 return tUINT;
 947         }
 948
 949         /* Else it must be an int... */
 950         errno = 0;
 951         l = strtol(str, NULL, radix);
 952         if (((l == LONG_MIN || l == LONG_MAX) && errno == ERANGE) ||
 953                 (l > INT_MAX) || (l < INT_MIN))
 954                 ppy_error("integer constant %s is too large\n", str);
 955         val->sint = (int)l;
 956         return tSINT;
 957 }
 958
 959
 960 /*
 961  *-------------------------------------------------------------------------
 962  * Macro and define expansion support
 963  *
 964  * FIXME: Variable macro arguments.
 965  *-------------------------------------------------------------------------
 966  */
 967 static void expand_special(pp_entry_t *ppp)
 968 {
 969         static char *buf = NULL;
 970         char *new_buf;
 971
 972         assert(ppp->type == def_special);
 973
 974         if(!strcmp(ppp->ident, "__LINE__"))
 975         {
 976                 new_buf = pp_xrealloc(buf, 32);
 977                 if(!new_buf)
 978                         return;
 979                 buf = new_buf;
 980                 sprintf(buf, "%d", pp_status.line_number);
 981         }
 982         else if(!strcmp(ppp->ident, "__FILE__"))
 983         {
 984                 new_buf = pp_xrealloc(buf, strlen(pp_status.input) + 3);
 985                 if(!new_buf)
 986                         return;
 987                 buf = new_buf;
 988                 sprintf(buf, "\"%s\"", pp_status.input);
 989         }
 990         else
 991                 pp_internal_error(__FILE__, __LINE__, "Special macro '%s' not found...\n", ppp->ident);
 992
 993         if(pp_flex_debug)
 994                 fprintf(stderr, "expand_special(%d): %s:%d: '%s' -> '%s'\n",
 995                         macexpstackidx,
 996                         pp_status.input,
 997                         pp_status.line_number,
 998                         ppp->ident,
 999                         buf ? buf : "");
1000
1001         if(buf && buf[0])
1002         {
1003                 push_buffer(ppp, NULL, NULL, 0);
1004                 yy_scan_string(buf);
1005         }
1006 }
1007
1008 static void expand_define(pp_entry_t *ppp)
1009 {
1010         assert(ppp->type == def_define);
1011
1012         if(pp_flex_debug)
1013                 fprintf(stderr, "expand_define(%d): %s:%d: '%s' -> '%s'\n",
1014                         macexpstackidx,
1015                         pp_status.input,
1016                         pp_status.line_number,
1017                         ppp->ident,
1018                         ppp->subst.text);
1019         if(ppp->subst.text && ppp->subst.text[0])
1020         {
1021                 push_buffer(ppp, NULL, NULL, 0);
1022                 yy_scan_string(ppp->subst.text);
1023         }
1024 }
1025
1026 static int curdef_idx = 0;
1027 static int curdef_alloc = 0;
1028 static char *curdef_text = NULL;
1029
1030 static void add_text(const char *str, int len)
1031 {
1032         int new_alloc;
1033         char *new_text;
1034
1035         if(len == 0)
1036                 return;
1037         if(curdef_idx >= curdef_alloc || curdef_alloc - curdef_idx < len)
1038         {
1039                 new_alloc = curdef_alloc + ((len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1));
1040                 new_text = pp_xrealloc(curdef_text, new_alloc * sizeof(curdef_text[0]));
1041                 if(!new_text)
1042                         return;
1043                 curdef_text = new_text;
1044                 curdef_alloc = new_alloc;
1045                 if(curdef_alloc > 65536)
1046                         ppy_warning("Reallocating macro-expansion buffer larger than 64kB");
1047         }
1048         memcpy(&curdef_text[curdef_idx], str, len);
1049         curdef_idx += len;
1050 }
1051
1052 static mtext_t *add_expand_text(mtext_t *mtp, macexpstackentry_t *mep, int *nnl)
1053 {
1054         char *cptr;
1055         char *exp;
1056         int tag;
1057         int n;
1058
1059         if(mtp == NULL)
1060                 return NULL;
1061
1062         switch(mtp->type)
1063         {
1064         case exp_text:
1065                 if(pp_flex_debug)
1066                         fprintf(stderr, "add_expand_text: exp_text: '%s'\n", mtp->subst.text);
1067                 add_text(mtp->subst.text, strlen(mtp->subst.text));
1068                 break;
1069
1070         case exp_stringize:
1071                 if(pp_flex_debug)
1072                         fprintf(stderr, "add_expand_text: exp_stringize(%d): '%s'\n",
1073                                 mtp->subst.argidx,
1074                                 mep->args[mtp->subst.argidx]);
1075                 cptr = mep->args[mtp->subst.argidx];
1076                 add_text("\"", 1);
1077                 while(*cptr)
1078                 {
1079                         if(*cptr == '"' || *cptr == '\\')
1080                                 add_text("\\", 1);
1081                         add_text(cptr, 1);
1082                         cptr++;
1083                 }
1084                 add_text("\"", 1);
1085                 break;
1086
1087         case exp_concat:
1088                 if(pp_flex_debug)
1089                         fprintf(stderr, "add_expand_text: exp_concat\n");
1090                 /* Remove trailing whitespace from current expansion text */
1091                 while(curdef_idx)
1092                 {
1093                         if(isspace(curdef_text[curdef_idx-1] & 0xff))
1094                                 curdef_idx--;
1095                         else
1096                                 break;
1097                 }
1098                 /* tag current position and recursively expand the next part */
1099                 tag = curdef_idx;
1100                 mtp = add_expand_text(mtp->next, mep, nnl);
1101
1102                 /* Now get rid of the leading space of the expansion */
1103                 cptr = &curdef_text[tag];
1104                 n = curdef_idx - tag;
1105                 while(n)
1106                 {
1107                         if(isspace(*cptr & 0xff))
1108                         {
1109                                 cptr++;
1110                                 n--;
1111                         }
1112                         else
1113                                 break;
1114                 }
1115                 if(cptr != &curdef_text[tag])
1116                 {
1117                         memmove(&curdef_text[tag], cptr, n);
1118                         curdef_idx -= (curdef_idx - tag) - n;
1119                 }
1120                 break;
1121
1122         case exp_subst:
1123                 if((mtp->next && mtp->next->type == exp_concat) || (mtp->prev && mtp->prev->type == exp_concat))
1124                         exp = mep->args[mtp->subst.argidx];
1125                 else
1126                         exp = mep->ppargs[mtp->subst.argidx];
1127                 if(exp)
1128                 {
1129                         add_text(exp, strlen(exp));
1130                         *nnl -= mep->nnls[mtp->subst.argidx];
1131                         cptr = strchr(exp, '\n');
1132                         while(cptr)
1133                         {
1134                                 *cptr = ' ';
1135                                 cptr = strchr(cptr+1, '\n');
1136                         }
1137                         mep->nnls[mtp->subst.argidx] = 0;
1138                 }
1139                 if(pp_flex_debug)
1140                         fprintf(stderr, "add_expand_text: exp_subst(%d): '%s'\n", mtp->subst.argidx, exp);
1141                 break;
1142
1143         default:
1144                 pp_internal_error(__FILE__, __LINE__, "Invalid expansion type (%d) in macro expansion\n", mtp->type);
1145         }
1146         return mtp;
1147 }
1148
1149 static void expand_macro(macexpstackentry_t *mep)
1150 {
1151         mtext_t *mtp;
1152         int n, k;
1153         char *cptr;
1154         int nnl = 0;
1155         pp_entry_t *ppp = mep->ppp;
1156         int nargs = mep->nargs;
1157
1158         assert(ppp->type == def_macro);
1159         assert(ppp->expanding == 0);
1160
1161         if((ppp->nargs >= 0 && nargs != ppp->nargs) || (ppp->nargs < 0 && nargs < -ppp->nargs))
1162         {
1163                 ppy_error("Too %s macro arguments (%d)", nargs < abs(ppp->nargs) ? "few" : "many", nargs);
1164                 return;
1165         }
1166
1167         for(n = 0; n < nargs; n++)
1168                 nnl += mep->nnls[n];
1169
1170         if(pp_flex_debug)
1171                 fprintf(stderr, "expand_macro(%d): %s:%d: '%s'(%d,%d) -> ...\n",
1172                         macexpstackidx,
1173                         pp_status.input,
1174                         pp_status.line_number,
1175                         ppp->ident,
1176                         mep->nargs,
1177                         nnl);
1178
1179         curdef_idx = 0;
1180
1181         for(mtp = ppp->subst.mtext; mtp; mtp = mtp->next)
1182         {
1183                 if(!(mtp = add_expand_text(mtp, mep, &nnl)))
1184                         break;
1185         }
1186
1187         for(n = 0; n < nnl; n++)
1188                 add_text("\n", 1);
1189
1190         /* To make sure there is room and termination (see below) */
1191         add_text(" \0", 2);
1192
1193         /* Strip trailing whitespace from expansion */
1194         for(k = curdef_idx, cptr = &curdef_text[curdef_idx-1]; k > 0; k--, cptr--)
1195         {
1196                 if(!isspace(*cptr & 0xff))
1197                         break;
1198         }
1199
1200         /*
1201          * We must add *one* whitespace to make sure that there
1202          * is a token-separation after the expansion.
1203          */
1204         *(++cptr) = ' ';
1205         *(++cptr) = '\0';
1206         k++;
1207
1208         /* Strip leading whitespace from expansion */
1209         for(n = 0, cptr = curdef_text; n < k; n++, cptr++)
1210         {
1211                 if(!isspace(*cptr & 0xff))
1212                         break;
1213         }
1214
1215         if(k - n > 0)
1216         {
1217                 if(pp_flex_debug)
1218                         fprintf(stderr, "expand_text: '%s'\n", curdef_text + n);
1219                 push_buffer(ppp, NULL, NULL, 0);
1220                 /*yy_scan_bytes(curdef_text + n, k - n);*/
1221                 yy_scan_string(curdef_text + n);
1222         }
1223 }
1224
1225 /*
1226  *-------------------------------------------------------------------------
1227  * String collection routines
1228  *-------------------------------------------------------------------------
1229  */
1230 static void new_string(void)
1231 {
1232 #ifdef DEBUG
1233         if(strbuf_idx)
1234                 ppy_warning("new_string: strbuf_idx != 0");
1235 #endif
1236         strbuf_idx = 0;
1237         str_startline = pp_status.line_number;
1238 }
1239
1240 static void add_string(const char *str, int len)
1241 {
1242         int new_alloc;
1243         char *new_buffer;
1244
1245         if(len == 0)
1246                 return;
1247         if(strbuf_idx >= strbuf_alloc || strbuf_alloc - strbuf_idx < len)
1248         {
1249                 new_alloc = strbuf_alloc + ((len + ALLOCBLOCKSIZE-1) & ~(ALLOCBLOCKSIZE-1));
1250                 new_buffer = pp_xrealloc(strbuffer, new_alloc * sizeof(strbuffer[0]));
1251                 if(!new_buffer)
1252                         return;
1253                 strbuffer = new_buffer;
1254                 strbuf_alloc = new_alloc;
1255                 if(strbuf_alloc > 65536)
1256                         ppy_warning("Reallocating string buffer larger than 64kB");
1257         }
1258         memcpy(&strbuffer[strbuf_idx], str, len);
1259         strbuf_idx += len;
1260 }
1261
1262 static char *get_string(void)
1263 {
1264         char *str = pp_xmalloc(strbuf_idx + 1);
1265         if(!str)
1266                 return NULL;
1267         memcpy(str, strbuffer, strbuf_idx);
1268         str[strbuf_idx] = '\0';
1269 #ifdef DEBUG
1270         strbuf_idx = 0;
1271 #endif
1272         return str;
1273 }
1274
1275 static void put_string(void)
1276 {
1277         put_buffer(strbuffer, strbuf_idx);
1278 #ifdef DEBUG
1279         strbuf_idx = 0;
1280 #endif
1281 }
1282
1283 static int string_start(void)
1284 {
1285         return str_startline;
1286 }
1287
1288
1289 /*
1290  *-------------------------------------------------------------------------
1291  * Buffer management
1292  *-------------------------------------------------------------------------
1293  */
1294 static void push_buffer(pp_entry_t *ppp, char *filename, char *incname, int pop)
1295 {
1296         if(ppy_debug)
1297                 printf("push_buffer(%d): %p %p %p %d\n", bufferstackidx, ppp, filename, incname, pop);
1298         if(bufferstackidx >= MAXBUFFERSTACK)
1299                 pp_internal_error(__FILE__, __LINE__, "Buffer stack overflow");
1300
1301         memset(&bufferstack[bufferstackidx], 0, sizeof(bufferstack[0]));
1302         bufferstack[bufferstackidx].bufferstate = YY_CURRENT_BUFFER;
1303         bufferstack[bufferstackidx].filehandle  = pp_status.file;
1304         bufferstack[bufferstackidx].define      = ppp;
1305         bufferstack[bufferstackidx].line_number = pp_status.line_number;
1306         bufferstack[bufferstackidx].char_number = pp_status.char_number;
1307         bufferstack[bufferstackidx].if_depth    = pp_get_if_depth();
1308         bufferstack[bufferstackidx].should_pop  = pop;
1309         bufferstack[bufferstackidx].filename    = pp_status.input;
1310         bufferstack[bufferstackidx].ncontinuations      = ncontinuations;
1311         bufferstack[bufferstackidx].incl                = pp_incl_state;
1312         bufferstack[bufferstackidx].include_filename    = incname;
1313
1314         if(ppp)
1315                 ppp->expanding = 1;
1316         else if(filename)
1317         {
1318                 /* These will track the ppy_error to the correct file and line */
1319                 pp_status.line_number = 1;
1320                 pp_status.char_number = 1;
1321                 pp_status.input  = filename;
1322                 ncontinuations = 0;
1323         }
1324         else if(!pop)
1325                 pp_internal_error(__FILE__, __LINE__, "Pushing buffer without knowing where to go to");
1326         bufferstackidx++;
1327 }
1328
1329 static bufferstackentry_t *pop_buffer(void)
1330 {
1331         if(bufferstackidx < 0)
1332                 pp_internal_error(__FILE__, __LINE__, "Bufferstack underflow?");
1333
1334         if(bufferstackidx == 0)
1335                 return NULL;
1336
1337         bufferstackidx--;
1338
1339         if(bufferstack[bufferstackidx].define)
1340                 bufferstack[bufferstackidx].define->expanding = 0;
1341         else
1342         {
1343                 includelogicentry_t *iep = NULL;
1344
1345                 if(!bufferstack[bufferstackidx].should_pop)
1346                 {
1347                         wpp_callbacks->close(pp_status.file);
1348                         pp_writestring("# %d \"%s\" 2\n", bufferstack[bufferstackidx].line_number, bufferstack[bufferstackidx].filename);
1349
1350                         /* We have EOF, check the include logic */
1351                         if(pp_incl_state.state == 2 && !pp_incl_state.seen_junk && pp_incl_state.ppp)
1352                         {
1353                                 pp_entry_t *ppp = pplookup(pp_incl_state.ppp);
1354                                 if(ppp)
1355                                 {
1356                                         iep = pp_xmalloc(sizeof(includelogicentry_t));
1357                                         if (iep)
1358                                         {
1359                                                 iep->ppp = ppp;
1360                                                 ppp->iep = iep;
1361                                                 iep->filename = bufferstack[bufferstackidx].include_filename;
1362                                                 iep->prev = NULL;
1363                                                 iep->next = pp_includelogiclist;
1364                                                 if(iep->next)
1365                                                         iep->next->prev = iep;
1366                                                 pp_includelogiclist = iep;
1367                                                 if(pp_status.debug)
1368                                                         fprintf(stderr, "pop_buffer: %s:%d: includelogic added, include_ppp='%s', file='%s'\n",
1369                                                                 bufferstack[bufferstackidx].filename, bufferstack[bufferstackidx].line_number, pp_incl_state.ppp, iep->filename);
1370                                         }
1371                                 }
1372                         }
1373                         free(pp_incl_state.ppp);
1374                         pp_incl_state   = bufferstack[bufferstackidx].incl;
1375
1376                 }
1377                 if (bufferstack[bufferstackidx].include_filename)
1378                 {
1379                         free(pp_status.input);
1380                         pp_status.input = bufferstack[bufferstackidx].filename;
1381                 }
1382                 pp_status.line_number = bufferstack[bufferstackidx].line_number;
1383                 pp_status.char_number = bufferstack[bufferstackidx].char_number;
1384                 ncontinuations = bufferstack[bufferstackidx].ncontinuations;
1385                 if (!iep)
1386                         free(bufferstack[bufferstackidx].include_filename);
1387         }
1388
1389         if(ppy_debug)
1390                 printf("pop_buffer(%d): %p %p (%d, %d, %d) %p %d\n",
1391                         bufferstackidx,
1392                         bufferstack[bufferstackidx].bufferstate,
1393                         bufferstack[bufferstackidx].define,
1394                         bufferstack[bufferstackidx].line_number,
1395                         bufferstack[bufferstackidx].char_number,
1396                         bufferstack[bufferstackidx].if_depth,
1397                         bufferstack[bufferstackidx].filename,
1398                         bufferstack[bufferstackidx].should_pop);
1399
1400         pp_status.file = bufferstack[bufferstackidx].filehandle;
1401         ppy__switch_to_buffer(bufferstack[bufferstackidx].bufferstate);
1402
1403         if(bufferstack[bufferstackidx].should_pop)
1404         {
1405                 if(yy_current_state() == pp_macexp)
1406                         macro_add_expansion();
1407                 else
1408                         pp_internal_error(__FILE__, __LINE__, "Pop buffer and state without macro expansion state");
1409                 yy_pop_state();
1410         }
1411
1412         return &bufferstack[bufferstackidx];
1413 }
1414
1415
1416 /*
1417  *-------------------------------------------------------------------------
1418  * Macro nestng support
1419  *-------------------------------------------------------------------------
1420  */
1421 static void push_macro(pp_entry_t *ppp)
1422 {
1423         if(macexpstackidx >= MAXMACEXPSTACK)
1424         {
1425                 ppy_error("Too many nested macros");
1426                 return;
1427         }
1428
1429         macexpstack[macexpstackidx] = pp_xmalloc(sizeof(macexpstack[0][0]));
1430         if(!macexpstack[macexpstackidx])
1431                 return;
1432         memset( macexpstack[macexpstackidx], 0, sizeof(macexpstack[0][0]));
1433         macexpstack[macexpstackidx]->ppp = ppp;
1434         macexpstackidx++;
1435 }
1436
1437 static macexpstackentry_t *top_macro(void)
1438 {
1439         return macexpstackidx > 0 ? macexpstack[macexpstackidx-1] : NULL;
1440 }
1441
1442 static macexpstackentry_t *pop_macro(void)
1443 {
1444         if(macexpstackidx <= 0)
1445                 pp_internal_error(__FILE__, __LINE__, "Macro expansion stack underflow\n");
1446         return macexpstack[--macexpstackidx];
1447 }
1448
1449 static void free_macro(macexpstackentry_t *mep)
1450 {
1451         int i;
1452
1453         for(i = 0; i < mep->nargs; i++)
1454                 free(mep->args[i]);
1455         free(mep->args);
1456         free(mep->nnls);
1457         free(mep->curarg);
1458         free(mep);
1459 }
1460
1461 static void add_text_to_macro(const char *text, int len)
1462 {
1463         macexpstackentry_t *mep = top_macro();
1464
1465         assert(mep->ppp->expanding == 0);
1466
1467         if(mep->curargalloc - mep->curargsize <= len+1) /* +1 for '\0' */
1468         {
1469                 char *new_curarg;
1470                 int new_alloc = mep->curargalloc + ((ALLOCBLOCKSIZE > len+1) ? ALLOCBLOCKSIZE : len+1);
1471                 new_curarg = pp_xrealloc(mep->curarg, new_alloc * sizeof(mep->curarg[0]));
1472                 if(!new_curarg)
1473                         return;
1474                 mep->curarg = new_curarg;
1475                 mep->curargalloc = new_alloc;
1476         }
1477         memcpy(mep->curarg + mep->curargsize, text, len);
1478         mep->curargsize += len;
1479         mep->curarg[mep->curargsize] = '\0';
1480 }
1481
1482 static void macro_add_arg(int last)
1483 {
1484         int nnl = 0;
1485         char *cptr;
1486         char **new_args, **new_ppargs;
1487         int *new_nnls;
1488         macexpstackentry_t *mep = top_macro();
1489
1490         assert(mep->ppp->expanding == 0);
1491
1492         new_args = pp_xrealloc(mep->args, (mep->nargs+1) * sizeof(mep->args[0]));
1493         if(!new_args)
1494                 return;
1495         mep->args = new_args;
1496
1497         new_ppargs = pp_xrealloc(mep->ppargs, (mep->nargs+1) * sizeof(mep->ppargs[0]));
1498         if(!new_ppargs)
1499                 return;
1500         mep->ppargs = new_ppargs;
1501
1502         new_nnls = pp_xrealloc(mep->nnls, (mep->nargs+1) * sizeof(mep->nnls[0]));
1503         if(!new_nnls)
1504                 return;
1505         mep->nnls = new_nnls;
1506
1507         mep->args[mep->nargs] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1508         if(!mep->args[mep->nargs])
1509                 return;
1510         cptr = mep->args[mep->nargs]-1;
1511         while((cptr = strchr(cptr+1, '\n')))
1512         {
1513                 nnl++;
1514         }
1515         mep->nnls[mep->nargs] = nnl;
1516         mep->nargs++;
1517         free(mep->curarg);
1518         mep->curargalloc = mep->curargsize = 0;
1519         mep->curarg = NULL;
1520
1521         if(pp_flex_debug)
1522                 fprintf(stderr, "macro_add_arg: %s:%d: %d -> '%s'\n",
1523                         pp_status.input,
1524                         pp_status.line_number,
1525                         mep->nargs-1,
1526                         mep->args[mep->nargs-1]);
1527
1528         /* Each macro argument must be expanded to cope with stingize */
1529         if(last || mep->args[mep->nargs-1][0])
1530         {
1531                 yy_push_state(pp_macexp);
1532                 push_buffer(NULL, NULL, NULL, last ? 2 : 1);
1533                 yy_scan_string(mep->args[mep->nargs-1]);
1534                 /*mep->bufferstackidx = bufferstackidx;  But not nested! */
1535         }
1536 }
1537
1538 static void macro_add_expansion(void)
1539 {
1540         macexpstackentry_t *mep = top_macro();
1541
1542         assert(mep->ppp->expanding == 0);
1543
1544         mep->ppargs[mep->nargs-1] = pp_xstrdup(mep->curarg ? mep->curarg : "");
1545         free(mep->curarg);
1546         mep->curargalloc = mep->curargsize = 0;
1547         mep->curarg = NULL;
1548
1549         if(pp_flex_debug)
1550                 fprintf(stderr, "macro_add_expansion: %s:%d: %d -> '%s'\n",
1551                         pp_status.input,
1552                         pp_status.line_number,
1553                         mep->nargs-1,
1554                         mep->ppargs[mep->nargs-1] ? mep->ppargs[mep->nargs-1] : "");
1555 }
1556
1557
1558 /*
1559  *-------------------------------------------------------------------------
1560  * Output management
1561  *-------------------------------------------------------------------------
1562  */
1563 static void put_buffer(const char *s, int len)
1564 {
1565         if(top_macro())
1566                 add_text_to_macro(s, len);
1567         else
1568                 wpp_callbacks->write(s, len);
1569 }
1570
1571
1572 /*
1573  *-------------------------------------------------------------------------
1574  * Include management
1575  *-------------------------------------------------------------------------
1576  */
1577 void pp_do_include(char *fname, int type)
1578 {
1579         char *newpath;
1580         int n;
1581         includelogicentry_t *iep;
1582         void *fp;
1583
1584         if(!fname)
1585                 return;
1586
1587         for(iep = pp_includelogiclist; iep; iep = iep->next)
1588         {
1589                 if(!strcmp(iep->filename, fname))
1590                 {
1591                         /*
1592                          * We are done. The file was included before.
1593                          * If the define was deleted, then this entry would have
1594                          * been deleted too.
1595                          */
1596                         free(fname);
1597                         return;
1598                 }
1599         }
1600
1601         n = strlen(fname);
1602
1603         if(n <= 2)
1604         {
1605                 ppy_error("Empty include filename");
1606                 free(fname);
1607                 return;
1608         }
1609
1610         /* Undo the effect of the quotation */
1611         fname[n-1] = '\0';
1612
1613         if((fp = pp_open_include(fname+1, type, pp_status.input, &newpath)) == NULL)
1614         {
1615                 ppy_error("Unable to open include file %s", fname+1);
1616                 free(fname);
1617                 return;
1618         }
1619
1620         fname[n-1] = *fname;    /* Redo the quotes */
1621         push_buffer(NULL, newpath, fname, 0);
1622         pp_incl_state.seen_junk = 0;
1623         pp_incl_state.state = 0;
1624         pp_incl_state.ppp = NULL;
1625
1626         if(pp_status.debug)
1627                 fprintf(stderr, "pp_do_include: %s:%d: include_state=%d, include_ppp='%s', include_ifdepth=%d\n",
1628                         pp_status.input, pp_status.line_number, pp_incl_state.state, pp_incl_state.ppp, pp_incl_state.ifdepth);
1629         pp_status.file = fp;
1630         ppy__switch_to_buffer(ppy__create_buffer(NULL, YY_BUF_SIZE));
1631
1632         pp_writestring("# 1 \"%s\" 1%s\n", newpath, type ? "" : " 3");
1633 }
1634
1635 /*
1636  *-------------------------------------------------------------------------
1637  * Push/pop preprocessor ignore state when processing conditionals
1638  * which are false.
1639  *-------------------------------------------------------------------------
1640  */
1641 void pp_push_ignore_state(void)
1642 {
1643         yy_push_state(pp_ignore);
1644 }
1645
1646 void pp_pop_ignore_state(void)
1647 {
1648         yy_pop_state();
1649 }