4 #pragma warning ( disable : 4786 )
18 void tokenize ( const string
& text
, vector
<string
>& tokens
)
27 // check for literal string
41 ASSERT(0); // come back to this....
51 else if ( *end
== '\"' )
59 tokens
.push_back ( string ( p
, end
-p
) );
62 else if ( __iscsymf(*p
) )
65 while ( __iscsym ( *end
) )
67 tokens
.push_back ( string ( p
, end
-p
) );
70 else if ( isdigit(*p
) || *p
== '.' )
73 while ( isdigit(*end
) )
79 while ( isdigit(*end
) )
83 if ( *end
== 'f' || *end
== 'F' )
85 else if ( !f
&& ( *end
== 'l' || *end
== 'L' ) )
87 tokens
.push_back ( string ( p
, end
-p
) );
93 tokens
.push_back ( "." );
97 tokens
.push_back ( "," );
101 tokens
.push_back ( "(" );
105 tokens
.push_back ( ")" );
109 tokens
.push_back ( "{" );
113 tokens
.push_back ( "}" );
117 tokens
.push_back ( "[" );
121 tokens
.push_back ( "]" );
125 tokens
.push_back ( ";" );
132 tokens
.push_back ( string ( p
, 2 ) );
136 ASSERT(0); // shouldn't hit here, I think
137 tokens
.push_back ( "\\" );
146 tokens
.push_back ( string ( p
, 2 ) );
150 tokens
.push_back ( "|" );
159 tokens
.push_back ( string ( p
, 2 ) );
163 tokens
.push_back ( "&" );
173 tokens
.push_back ( string ( p
, 3 ) ), p
+= 3;
175 tokens
.push_back ( string ( p
, 2 ) ), p
+= 2;
178 tokens
.push_back ( string ( p
, 2 ) );
182 tokens
.push_back ( "<" );
192 tokens
.push_back ( string ( p
, 3 ) ), p
+= 3;
194 tokens
.push_back ( string ( p
, 2 ) ), p
+= 2;
197 tokens
.push_back ( string ( p
, 2 ) );
201 tokens
.push_back ( ">" );
210 tokens
.push_back ( string ( p
, 2 ) );
214 tokens
.push_back ( "!" );
223 tokens
.push_back ( string ( p
, 2 ) );
227 tokens
.push_back ( "=" );
236 tokens
.push_back ( string ( p
, 2 ) );
240 tokens
.push_back ( ":" );
249 tokens
.push_back ( string ( p
, 2 ) );
253 tokens
.push_back ( "*" );
262 tokens
.push_back ( string ( p
, 2 ) );
266 tokens
.push_back ( "/" );
276 tokens
.push_back ( string ( p
, 2 ) );
280 tokens
.push_back ( "+" );
290 tokens
.push_back ( string ( p
, 2 ) );
294 tokens
.push_back ( "-" );
300 while ( *p
&& *p
!= '\n' )
306 printf ( "choked on '%c' in tokenize() - press any key to continue\n", *p
);