Improved the tokenizer.

2025-04-29 00:24:13 -07:00
parent 0fc10f4c76
commit ff6b5785ae
7 changed files with 148 additions and 74 deletions
--- a/include/arctyx/compiler/Token.h
+++ b/include/arctyx/compiler/Token.h
@@ -5,7 +5,10 @@
 enum class TokenT : ehs::UInt_8
 {
 	UNKNOWN,
-	VALUE,
+	SEPARATOR,
+	NUMBER,
+	STRING,
+	CHARACTER,
 	KEYWORD,
 	TYPE,
 	IDENTIFIER,