2 * Wine Message Compiler lexical scanner
4 * Copyright 2000 Bertho A. Stultiens (BS)
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
35 * Keywords are case insenitive. All normal input is treated as
36 * being in codepage iso-8859-1 for ascii input files (unicode
37 * page 0) and as equivalent unicode if unicode input is selected.
38 * All normal input, which is not part of a message text, is
39 * enforced to be unicode page 0. Otherwise an error will be
40 * generated. The normal file data should only be ASCII because
41 * that is the basic definition of the grammar.
43 * Byteorder or unicode input is determined automatically by
44 * reading the first 8 bytes and checking them against unicode
45 * page 0 byteorder (hibyte must be 0).
47 * Alternatively, the input is checked against a special byte
48 * sequence to identify the file.
63 * Default added identifiers for classes:
73 * The 'Codepages' keyword is a wmc extension.
76 static WCHAR ustr_application
[] = { 'A', 'p', 'p', 'l', 'i', 'c', 'a', 't', 'i', 'o', 'n', 0 };
77 static WCHAR ustr_codepages
[] = { 'C', 'o', 'd', 'e', 'p', 'a', 'g', 'e', 's', 0 };
78 static WCHAR ustr_english
[] = { 'E', 'n', 'g', 'l', 'i', 's', 'h', 0 };
79 static WCHAR ustr_error
[] = { 'E', 'r', 'r', 'o', 'r', 0 };
80 static WCHAR ustr_facility
[] = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 0 };
81 static WCHAR ustr_facilitynames
[] = { 'F', 'a', 'c', 'i', 'l', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
82 static WCHAR ustr_informational
[] = { 'I', 'n', 'f', 'o', 'r', 'm', 'a', 't', 'i', 'o', 'n', 'a', 'l', 0 };
83 static WCHAR ustr_language
[] = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 0};
84 static WCHAR ustr_languagenames
[] = { 'L', 'a', 'n', 'g', 'u', 'a', 'g', 'e', 'N', 'a', 'm', 'e', 's', 0};
85 static WCHAR ustr_messageid
[] = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 0 };
86 static WCHAR ustr_messageidtypedef
[] = { 'M', 'e', 's', 's', 'a', 'g', 'e', 'I', 'd', 'T', 'y', 'p', 'e', 'd', 'e', 'f', 0 };
87 static WCHAR ustr_outputbase
[] = { 'O', 'u', 't', 'p', 'u', 't', 'B', 'a', 's', 'e', 0 };
88 static WCHAR ustr_severity
[] = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 0 };
89 static WCHAR ustr_severitynames
[] = { 'S', 'e', 'v', 'e', 'r', 'i', 't', 'y', 'N', 'a', 'm', 'e', 's', 0 };
90 static WCHAR ustr_success
[] = { 'S', 'u', 'c', 'c', 'e', 's', 's', 0 };
91 static WCHAR ustr_symbolicname
[] = { 'S', 'y', 'm', 'b', 'o', 'l', 'i', 'c', 'N', 'a', 'm', 'e', 0 };
92 static WCHAR ustr_system
[] = { 'S', 'y', 's', 't', 'e', 'm', 0 };
93 static WCHAR ustr_warning
[] = { 'W', 'a', 'r', 'n', 'i', 'n', 'g', 0 };
94 static WCHAR ustr_msg00001
[] = { 'm', 's', 'g', '0', '0', '0', '0', '1', 0 };
96 * This table is to beat any form of "expression building" to check for
97 * correct filename characters. It is also used for ident checks.
98 * FIXME: use it more consistently.
101 #define CH_SHORTNAME 0x01
102 #define CH_LONGNAME 0x02
103 #define CH_IDENT 0x04
104 #define CH_NUMBER 0x08
105 /*#define CH_WILDCARD 0x10*/
106 /*#define CH_DOT 0x20*/
107 #define CH_PUNCT 0x40
108 #define CH_INVALID 0x80
110 static const char char_table
[256] = {
111 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x00 - 0x07 */
112 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x08 - 0x0F */
113 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x10 - 0x17 */
114 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, /* 0x18 - 0x1F */
115 0x80, 0x03, 0x80, 0x03, 0x03, 0x03, 0x03, 0x03, /* 0x20 - 0x27 " !"#$%&'" */
116 0x43, 0x43, 0x10, 0x80, 0x03, 0x03, 0x22, 0x80, /* 0x28 - 0x2F "()*+,-./" */
117 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, /* 0x30 - 0x37 "01234567" */
118 0x0b, 0x0b, 0xc0, 0x80, 0x80, 0x80, 0x80, 0x10, /* 0x38 - 0x3F "89:;<=>?" */
119 0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x40 - 0x47 "@ABCDEFG" */
120 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x48 - 0x4F "HIJKLMNO" */
121 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x50 - 0x57 "PQRSTUVW" */
122 0x07, 0x07, 0x07, 0x80, 0x80, 0x80, 0x80, 0x07, /* 0x58 - 0x5F "XYZ[\]^_" */
123 0x03, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x60 - 0x67 "`abcdefg" */
124 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x68 - 0x6F "hijklmno" */
125 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, /* 0x70 - 0x77 "pqrstuvw" */
126 0x07, 0x07, 0x07, 0x03, 0x80, 0x03, 0x03, 0x80, /* 0x78 - 0x7F "xyz{|}~ " */
127 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x80 - 0x87 */
128 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x88 - 0x8F */
129 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x90 - 0x97 */
130 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0x98 - 0x9F */
131 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA0 - 0xA7 */
132 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xA8 - 0xAF */
133 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB0 - 0xB7 */
134 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xB8 - 0xBF */
135 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC0 - 0xC7 */
136 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xC8 - 0xCF */
137 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD0 - 0xD7 */
138 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xD8 - 0xDF */
139 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE0 - 0xE7 */
140 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xE8 - 0xEF */
141 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 0xF0 - 0xF7 */
142 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x80, /* 0xF8 - 0xFF */
145 static int isisochar(int ch
)
147 return !(ch
& (~0xff));
151 //static const union cptable *codepage_def;
153 void set_codepage(int cp
)
157 codepage_def
= find_codepage(codepage
);
159 xyyerror("Codepage %d not found; cannot process", codepage
);
166 static int nungetstack
= 0;
167 static int allocungetstack
= 0;
168 static char *ungetstack
= NULL
;
169 static int ninputbuffer
= 0;
170 static WCHAR
*inputbuffer
= NULL
;
171 static char *xlatebuffer
= NULL
;
173 #define INPUTBUFFER_SIZE 2048 /* Must be larger than 4 and approx. large enough to hold a line */
176 * Fill the input buffer with *one* line of input.
177 * The line is '\n' terminated so that scanning
178 * messages with translation works as expected
179 * (otherwise we cannot pre-translate because the
180 * language is first known one line before the
183 static int fill_inputbuffer(void)
186 static char err_fatalread
[] = "Fatal: reading input failed";
187 static int endian
= -1;
191 inputbuffer
= xmalloc(INPUTBUFFER_SIZE
);
192 xlatebuffer
= xmalloc(INPUTBUFFER_SIZE
);
199 cptr
= fgets(xlatebuffer
, INPUTBUFFER_SIZE
, yyin
);
200 if(!cptr
&& ferror(yyin
))
201 xyyerror(err_fatalread
);
204 // assert(codepage_def != NULL);
205 // n = cp_mbstowcs(codepage_def, 0, xlatebuffer, strlen(xlatebuffer)+1, inputbuffer, INPUTBUFFER_SIZE);
206 n
= MultiByteToWideChar(codepage
, 0, xlatebuffer
, strlen(xlatebuffer
)+1, inputbuffer
, INPUTBUFFER_SIZE
); if(n
< 0)
207 internal_error(__FILE__
, __LINE__
, "Could not translate to unicode (%d)", n
);
209 goto try_again
; /* Should not hapen */
210 n
--; /* Strip added conversion '\0' from input length */
213 * Detect UTF-8 in the first time we read some bytes by
214 * checking the special sequence "FE..." or something like
215 * that. I need to check www.unicode.org for details.
222 n
= fread(inputbuffer
, 1, 8, yyin
);
225 if(!n
&& ferror(yyin
))
226 xyyerror(err_fatalread
);
228 xyyerror("Fatal: file to short to determine byteorder (should never happen)");
230 if(isisochar(inputbuffer
[0]) &&
231 isisochar(inputbuffer
[1]) &&
232 isisochar(inputbuffer
[2]) &&
233 isisochar(inputbuffer
[3]))
235 #ifdef WORDS_BIGENDIAN
238 endian
= WMC_BO_LITTLE
;
241 else if(isisochar(BYTESWAP_WORD(inputbuffer
[0])) &&
242 isisochar(BYTESWAP_WORD(inputbuffer
[1])) &&
243 isisochar(BYTESWAP_WORD(inputbuffer
[2])) &&
244 isisochar(BYTESWAP_WORD(inputbuffer
[3])))
246 #ifdef WORDS_BIGENDIAN
247 endian
= WMC_BO_LITTLE
;
253 xyyerror("Fatal: cannot determine file's byteorder");
255 * Determine the file-endian with the leader-bytes
256 * "FF FE..."; can't remember the exact sequence.
259 #ifdef WORDS_BIGENDIAN
260 if(endian
== WMC_BO_LITTLE
)
262 if(endian
== WMC_BO_BIG
)
265 inputbuffer
[0] = BYTESWAP_WORD(inputbuffer
[0]);
266 inputbuffer
[1] = BYTESWAP_WORD(inputbuffer
[1]);
267 inputbuffer
[2] = BYTESWAP_WORD(inputbuffer
[2]);
268 inputbuffer
[3] = BYTESWAP_WORD(inputbuffer
[3]);
276 for(i
= 0; i
< INPUTBUFFER_SIZE
; i
++)
279 t
= fread(&inputbuffer
[i
], 2, 1, yyin
);
280 if(!t
&& ferror(yyin
))
281 xyyerror(err_fatalread
);
285 #ifdef WORDS_BIGENDIAN
286 if(endian
== WMC_BO_LITTLE
)
288 if(endian
== WMC_BO_BIG
)
291 if((inputbuffer
[i
] = BYTESWAP_WORD(inputbuffer
[i
])) == '\n')
296 if(inputbuffer
[i
] == '\n')
306 yywarning("Re-read line (input was or converted to zilch)");
307 goto try_again
; /* Should not happen, but could be due to stdin reading and a signal */
314 static int get_unichar(void)
316 static WCHAR
*b
= NULL
;
320 return ungetstack
[--nungetstack
];
324 if(!fill_inputbuffer())
330 return (int)(*b
++ & 0xffff);
333 static void unget_unichar(int ch
)
340 if(nungetstack
== allocungetstack
)
342 allocungetstack
+= 32;
343 ungetstack
= xrealloc(ungetstack
, allocungetstack
* sizeof(*ungetstack
));
346 ungetstack
[nungetstack
++] = (WCHAR
)ch
;
351 * Normal character stack.
352 * Used for number scanning.
354 static int ncharstack
= 0;
355 static int alloccharstack
= 0;
356 static char *charstack
= NULL
;
358 static void empty_char_stack(void)
363 static void push_char(int ch
)
365 if(ncharstack
== alloccharstack
)
367 alloccharstack
+= 32;
368 charstack
= xrealloc(charstack
, alloccharstack
* sizeof(*charstack
));
370 charstack
[ncharstack
++] = (char)ch
;
373 static int tos_char_stack(void)
378 return (int)(charstack
[ncharstack
-1] & 0xff);
381 static char *get_char_stack(void)
387 * Unicode character stack.
388 * Used for general scanner.
390 static int nunicharstack
= 0;
391 static int allocunicharstack
= 0;
392 static WCHAR
*unicharstack
= NULL
;
394 static void empty_unichar_stack(void)
399 static void push_unichar(int ch
)
401 if(nunicharstack
== allocunicharstack
)
403 allocunicharstack
+= 128;
404 unicharstack
= xrealloc(unicharstack
, allocunicharstack
* sizeof(*unicharstack
));
406 unicharstack
[nunicharstack
++] = (WCHAR
)ch
;
410 static int tos_unichar_stack(void)
415 return (int)(unicharstack
[nunicharstack
-1] & 0xffff);
419 static WCHAR
*get_unichar_stack(void)
427 * state | ch | next state
428 * ------+-----------------+--------------------------
431 * 0 | . | error (should never occur)
434 * 1 | [89a-wyzA-WYZ_] | error invalid digit
436 * 2 | [0-9a-fA-F] | 2
437 * 2 | [g-zG-Z_] | error invalid hex digit
438 * 2 | . | return (hex-number) if TOS != [xX] else error
440 * 3 | [89a-zA-Z_] | error invalid octal digit
441 * 3 | . | return (octal-number)
443 * 4 | [a-zA-Z_] | error invalid decimal digit
444 * 4 | . | return (decimal-number)
446 * All non-identifier characters [^a-zA-Z_0-9] terminate the scan
447 * and return the value. This is not entirely correct, but close
448 * enough (should check punctuators as trailing context, but the
449 * char_table is not adapted to that and it is questionable whether
450 * it is worth the trouble).
451 * All non-iso-8859-1 characters are an error.
453 static int scan_number(int ch
)
462 xyyerror("Invalid digit");
476 internal_error(__FILE__
, __LINE__
, "Non-digit in first number-scanner state");
479 if(ch
== 'x' || ch
== 'X')
484 else if(ch
>= '0' && ch
<= '7')
489 else if(isalpha(ch
) || ch
== '_')
490 xyyerror("Invalid number digit");
501 else if(isalpha(ch
) || ch
== '_' || !isxdigit(tos_char_stack()))
502 xyyerror("Invalid hex digit");
510 if(ch
>= '0' && ch
<= '7')
512 else if(isalnum(ch
) || ch
== '_')
513 xyyerror("Invalid octal digit");
523 else if(isalnum(ch
) || ch
== '_')
524 xyyerror("Invalid decimal digit");
532 internal_error(__FILE__
, __LINE__
, "Invalid state in number-scanner");
539 yylval
.num
= strtoul(get_char_stack(), NULL
, base
);
543 static void newline(void)
549 static int unisort(const void *p1
, const void *p2
)
551 return unistricmp(((token_t
*)p1
)->name
, ((token_t
*)p2
)->name
);
554 static token_t
*tokentable
= NULL
;
555 static int ntokentable
= 0;
557 token_t
*lookup_token(const WCHAR
*s
)
562 return (token_t
*)bsearch(&tok
, tokentable
, ntokentable
, sizeof(*tokentable
), unisort
);
565 void add_token(tok_e type
, const WCHAR
*name
, int tok
, int cp
, const WCHAR
*alias
, int fix
)
568 tokentable
= xrealloc(tokentable
, ntokentable
* sizeof(*tokentable
));
569 tokentable
[ntokentable
-1].type
= type
;
570 tokentable
[ntokentable
-1].name
= name
;
571 tokentable
[ntokentable
-1].token
= tok
;
572 tokentable
[ntokentable
-1].codepage
= cp
;
573 tokentable
[ntokentable
-1].alias
= alias
;
574 tokentable
[ntokentable
-1].fixed
= fix
;
575 qsort(tokentable
, ntokentable
, sizeof(*tokentable
), unisort
);
578 void get_tokentable(token_t
**tab
, int *len
)
592 static WCHAR ustr_dot1
[] = { '.', '\n', 0 };
593 static WCHAR ustr_dot2
[] = { '.', '\r', '\n', 0 };
594 static int isinit
= 0;
600 set_codepage(WMC_DEFAULT_CODEPAGE
);
601 add_token(tok_keyword
, ustr_codepages
, tCODEPAGE
, 0, NULL
, 0);
602 add_token(tok_keyword
, ustr_facility
, tFACILITY
, 0, NULL
, 1);
603 add_token(tok_keyword
, ustr_facilitynames
, tFACNAMES
, 0, NULL
, 1);
604 add_token(tok_keyword
, ustr_language
, tLANGUAGE
, 0, NULL
, 1);
605 add_token(tok_keyword
, ustr_languagenames
, tLANNAMES
, 0, NULL
, 1);
606 add_token(tok_keyword
, ustr_messageid
, tMSGID
, 0, NULL
, 1);
607 add_token(tok_keyword
, ustr_messageidtypedef
, tTYPEDEF
, 0, NULL
, 1);
608 add_token(tok_keyword
, ustr_outputbase
, tBASE
, 0, NULL
, 1);
609 add_token(tok_keyword
, ustr_severity
, tSEVERITY
, 0, NULL
, 1);
610 add_token(tok_keyword
, ustr_severitynames
, tSEVNAMES
, 0, NULL
, 1);
611 add_token(tok_keyword
, ustr_symbolicname
, tSYMNAME
, 0, NULL
, 1);
612 add_token(tok_severity
, ustr_error
, 0x03, 0, NULL
, 0);
613 add_token(tok_severity
, ustr_warning
, 0x02, 0, NULL
, 0);
614 add_token(tok_severity
, ustr_informational
, 0x01, 0, NULL
, 0);
615 add_token(tok_severity
, ustr_success
, 0x00, 0, NULL
, 0);
616 add_token(tok_facility
, ustr_application
, 0xFFF, 0, NULL
, 0);
617 add_token(tok_facility
, ustr_system
, 0x0FF, 0, NULL
, 0);
618 add_token(tok_language
, ustr_english
, 0x409, 437, ustr_msg00001
, 0);
621 empty_unichar_stack();
627 while((ch
= get_unichar()) != '\n')
630 xyyerror("Unexpected EOF");
636 if(!unistrcmp(ustr_dot1
, get_unichar_stack()) || !unistrcmp(ustr_dot2
, get_unichar_stack()))
639 /* Reset the codepage to our default after each message */
640 set_codepage(WMC_DEFAULT_CODEPAGE
);
643 yylval
.str
= xunistrdup(get_unichar_stack());
668 while(n
< 8 && isisochar(ch
))
670 int t
= char_table
[ch
];
671 if((t
& CH_PUNCT
) || !(t
& CH_SHORTNAME
))
681 yylval
.str
= xunistrdup(get_unichar_stack());
685 if(char_table
[ch
] & CH_IDENT
)
688 while(isisochar(ch
) && (char_table
[ch
] & (CH_IDENT
|CH_NUMBER
)))
695 if(!(tok
= lookup_token(get_unichar_stack())))
697 yylval
.str
= xunistrdup(get_unichar_stack());
706 codepage
= tok
->codepage
;
714 internal_error(__FILE__
, __LINE__
, "Invalid token type encountered");
718 if(isspace(ch
)) /* Ignore space */
722 return scan_number(ch
);
734 while(ch
!= '\n' && ch
!= EOF
)
740 push_unichar(ch
); /* Include the newline */
742 yylval
.str
= xunistrdup(get_unichar_stack());
745 xyyerror("Invalid character '%c' (0x%04x)", isisochar(ch
) && isprint(ch
) ? ch
: '.', ch
);