28bdde309a5ce548a9b391e9409caa2b658a472a
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
11 *************************************************************************
12 ** A tokenizer for SQL
14 ** This file contains C code that splits an SQL input string up into
15 ** individual tokens and sends those tokens one-by-one over to the
16 ** parser for analysis.
19 #define WIN32_NO_STATUS
21 #define COM_NO_WINDOWS_H
28 //#include "winbase.h"
29 #include <wine/unicode.h>
34 ** All the keywords of the SQL language are stored as in a hash
35 ** table composed of instances of the following structure.
37 typedef struct Keyword Keyword
;
39 const WCHAR
*name
; /* The keyword name */
41 int tokenType
; /* The token value for this keyword */
44 #define MAX_TOKEN_LEN 11
46 static const WCHAR addW
[] = {'A','D','D'};
47 static const WCHAR alterW
[] = {'A','L','T','E','R'};
48 static const WCHAR andW
[] = {'A','N','D'};
49 static const WCHAR byW
[] = {'B','Y'};
50 static const WCHAR charW
[] = {'C','H','A','R'};
51 static const WCHAR characterW
[] = {'C','H','A','R','A','C','T','E','R'};
52 static const WCHAR createW
[] = {'C','R','E','A','T','E'};
53 static const WCHAR deleteW
[] = {'D','E','L','E','T','E'};
54 static const WCHAR distinctW
[] = {'D','I','S','T','I','N','C','T'};
55 static const WCHAR dropW
[] = {'D','R','O','P'};
56 static const WCHAR freeW
[] = {'F','R','E','E'};
57 static const WCHAR fromW
[] = {'F','R','O','M'};
58 static const WCHAR holdW
[] = {'H','O','L','D'};
59 static const WCHAR insertW
[] = {'I','N','S','E','R','T'};
60 static const WCHAR intW
[] = {'I','N','T'};
61 static const WCHAR integerW
[] = {'I','N','T','E','G','E','R'};
62 static const WCHAR intoW
[] = {'I','N','T','O'};
63 static const WCHAR isW
[] = {'I','S'};
64 static const WCHAR keyW
[] = {'K','E','Y'};
65 static const WCHAR likeW
[] = {'L','I','K','E'};
66 static const WCHAR localizableW
[] = {'L','O','C','A','L','I','Z','A','B','L','E'};
67 static const WCHAR longW
[] = {'L','O','N','G'};
68 static const WCHAR longcharW
[] = {'L','O','N','G','C','H','A','R'};
69 static const WCHAR notW
[] = {'N','O','T'};
70 static const WCHAR nullW
[] = {'N','U','L','L'};
71 static const WCHAR objectW
[] = {'O','B','J','E','C','T'};
72 static const WCHAR orW
[] = {'O','R'};
73 static const WCHAR orderW
[] = {'O','R','D','E','R'};
74 static const WCHAR primaryW
[] = {'P','R','I','M','A','R','Y'};
75 static const WCHAR selectW
[] = {'S','E','L','E','C','T'};
76 static const WCHAR setW
[] = {'S','E','T'};
77 static const WCHAR shortW
[] = {'S','H','O','R','T'};
78 static const WCHAR tableW
[] = {'T','A','B','L','E'};
79 static const WCHAR temporaryW
[] = {'T','E','M','P','O','R','A','R','Y'};
80 static const WCHAR updateW
[] = {'U','P','D','A','T','E'};
81 static const WCHAR valuesW
[] = {'V','A','L','U','E','S'};
82 static const WCHAR whereW
[] = {'W','H','E','R','E'};
84 #define ARRAY_SIZE(array) (sizeof(array)/sizeof((array)[0]))
87 ** These are the keywords
88 ** They MUST be in alphabetical order
90 static const Keyword aKeywordTable
[] = {
91 { addW
, ARRAY_SIZE(addW
), TK_ADD
},
92 { alterW
, ARRAY_SIZE(alterW
), TK_ALTER
},
93 { andW
, ARRAY_SIZE(andW
), TK_AND
},
94 { byW
, ARRAY_SIZE(byW
), TK_BY
},
95 { charW
, ARRAY_SIZE(charW
), TK_CHAR
},
96 { characterW
, ARRAY_SIZE(characterW
), TK_CHAR
},
97 { createW
, ARRAY_SIZE(createW
), TK_CREATE
},
98 { deleteW
, ARRAY_SIZE(deleteW
), TK_DELETE
},
99 { distinctW
, ARRAY_SIZE(distinctW
), TK_DISTINCT
},
100 { dropW
, ARRAY_SIZE(dropW
), TK_DROP
},
101 { freeW
, ARRAY_SIZE(freeW
), TK_FREE
},
102 { fromW
, ARRAY_SIZE(fromW
), TK_FROM
},
103 { holdW
, ARRAY_SIZE(holdW
), TK_HOLD
},
104 { insertW
, ARRAY_SIZE(insertW
), TK_INSERT
},
105 { intW
, ARRAY_SIZE(intW
), TK_INT
},
106 { integerW
, ARRAY_SIZE(integerW
), TK_INT
},
107 { intoW
, ARRAY_SIZE(intoW
), TK_INTO
},
108 { isW
, ARRAY_SIZE(isW
), TK_IS
},
109 { keyW
, ARRAY_SIZE(keyW
), TK_KEY
},
110 { likeW
, ARRAY_SIZE(likeW
), TK_LIKE
},
111 { localizableW
, ARRAY_SIZE(localizableW
), TK_LOCALIZABLE
},
112 { longW
, ARRAY_SIZE(longW
), TK_LONG
},
113 { longcharW
, ARRAY_SIZE(longcharW
), TK_LONGCHAR
},
114 { notW
, ARRAY_SIZE(notW
), TK_NOT
},
115 { nullW
, ARRAY_SIZE(nullW
), TK_NULL
},
116 { objectW
, ARRAY_SIZE(objectW
), TK_OBJECT
},
117 { orW
, ARRAY_SIZE(orW
), TK_OR
},
118 { orderW
, ARRAY_SIZE(orderW
), TK_ORDER
},
119 { primaryW
, ARRAY_SIZE(primaryW
), TK_PRIMARY
},
120 { selectW
, ARRAY_SIZE(selectW
), TK_SELECT
},
121 { setW
, ARRAY_SIZE(setW
), TK_SET
},
122 { shortW
, ARRAY_SIZE(shortW
), TK_SHORT
},
123 { tableW
, ARRAY_SIZE(tableW
), TK_TABLE
},
124 { temporaryW
, ARRAY_SIZE(temporaryW
), TK_TEMPORARY
},
125 { updateW
, ARRAY_SIZE(updateW
), TK_UPDATE
},
126 { valuesW
, ARRAY_SIZE(valuesW
), TK_VALUES
},
127 { whereW
, ARRAY_SIZE(whereW
), TK_WHERE
},
131 ** Comparison function for binary search.
133 static int compKeyword(const void *m1
, const void *m2
){
134 const Keyword
*k1
= m1
, *k2
= m2
;
135 int ret
, len
= min( k1
->len
, k2
->len
);
137 if ((ret
= memicmpW( k1
->name
, k2
->name
, len
))) return ret
;
138 if (k1
->len
< k2
->len
) return -1;
139 else if (k1
->len
> k2
->len
) return 1;
144 ** This function looks up an identifier to determine if it is a
145 ** keyword. If it is a keyword, the token code of that keyword is
146 ** returned. If the input is not a keyword, TK_ID is returned.
148 static int sqliteKeywordCode(const WCHAR
*z
, int n
){
151 if( n
>MAX_TOKEN_LEN
)
157 r
= bsearch( &key
, aKeywordTable
, ARRAY_SIZE(aKeywordTable
), sizeof(Keyword
), compKeyword
);
165 ** If X is a character that can be used in an identifier then
166 ** isIdChar[X] will be 1. Otherwise isIdChar[X] will be 0.
168 ** In this implementation, an identifier can be a string of
169 ** alphabetic characters, digits, and "_" plus any character
170 ** with the high-order bit set. The latter rule means that
171 ** any sequence of UTF-8 characters or characters taken from
172 ** an extended ISO8859 character set can form an identifier.
174 static const char isIdChar
[] = {
175 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
176 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
177 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
178 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, /* 2x */
179 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
180 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
181 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
182 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
183 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
184 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 8x */
185 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 9x */
186 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ax */
187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Bx */
188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Cx */
189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Dx */
190 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Ex */
191 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* Fx */
196 ** Return the length of the token that begins at z[0]. Return
197 ** -1 if the token is (or might be) incomplete. Store the token
198 ** type in *tokenType before returning.
200 int sqliteGetToken(const WCHAR
*z
, int *tokenType
, int *skip
){
205 case ' ': case '\t': case '\n': case '\f':
206 for(i
=1; isspace(z
[i
]) && z
[i
] != '\r'; i
++){}
207 *tokenType
= TK_SPACE
;
210 if( z
[1]==0 ) return -1;
211 *tokenType
= TK_MINUS
;
220 *tokenType
= TK_STAR
;
229 }else if( z
[1]=='>' ){
246 *tokenType
= TK_ILLEGAL
;
253 *tokenType
= TK_WILDCARD
;
256 *tokenType
= TK_COMMA
;
258 case '`': case '\'': {
268 *tokenType
= TK_STRING
;
272 if( !isdigit(z
[1]) ){
277 case '0': case '1': case '2': case '3': case '4':
278 case '5': case '6': case '7': case '8': case '9':
279 *tokenType
= TK_INTEGER
;
280 for(i
=1; isdigit(z
[i
]); i
++){}
283 for(i
=1; z
[i
] && z
[i
-1]!=']'; i
++){}
290 for(i
=1; isIdChar
[z
[i
]]; i
++){}
291 *tokenType
= sqliteKeywordCode(z
, i
);
292 if( *tokenType
== TK_ID
&& z
[i
] == '`' ) *skip
= 1;
295 *tokenType
= TK_ILLEGAL
;