Improved the tokenizer.

include/arctyx/compiler/Compiler.h

@@ -30,6 +30,8 @@ public:
 	ehs::Array<ehs::Byte> Compile(const ehs::Str_8 &code) const;
 
 private:
+	static bool IsEOL(const ehs::Array<ehs::Char_8> &eols, const ehs::Char_8 *c);
+
 	static bool IsSeparator(const ehs::Array<ehs::Char_8> &separators, const ehs::Char_8 *c);
 
 	static bool IsPrimitive(const ehs::Array<Primitive> &primitives, const ehs::Str_8 &value);
@@ -38,7 +40,5 @@ private:
 
 	static const Operator *IsOperator(const ehs::Array<Operator> &operators, const ehs::Str_8 &value);
 
-	Token ParseValue(const ehs::Array<Primitive> &primitives, const ehs::Array<ehs::Str_8> &keywords, const ehs::Array<Operator> &operators, const ehs::Str_8 &value) const;
-
 	ehs::Vector<Token> Parse(const ehs::Str_8 &code) const;
 };

include/arctyx/compiler/Language.h

@@ -17,7 +17,7 @@ private:
 	ehs::UInt_64 id;
 	ehs::Str_8 name;
 	ehs::Version version;
-	ehs::Char_8 eol;
+	ehs::Array<ehs::Char_8> eols;
 	ehs::Array<ehs::Char_8> separators;
 	ehs::Array<Primitive> primitives;
 	ehs::Array<ehs::Str_8> keywords;
@@ -45,9 +45,11 @@ public:
 
 	ehs::Version GetVersion() const;
 
-	void SetEOL(const ehs::Char_8 &newEOL);
+	ehs::Array<ehs::Char_8> GetEOLs() const;
 
-	ehs::Char_8 GetEOL() const;
+	bool HasEOL(const ehs::Char_8 &eol) const;
+
+	bool AddEOL(const ehs::Char_8 &eol);
 
 	ehs::Array<ehs::Char_8> GetSeparators() const;
 
@@ -77,7 +79,6 @@ public:
 
 	bool HasOperator(const ehs::Str_8 &delimeter) const;
 
-
 	const Operator *GetOperator(const ehs::Str_8 &delimeter) const;
 
 	bool AddOperator(Operator primitive);

include/arctyx/compiler/Token.h

@@ -5,7 +5,10 @@
 enum class TokenT : ehs::UInt_8
 {
 	UNKNOWN,
-	VALUE,
+	SEPARATOR,
+	NUMBER,
+	STRING,
+	CHARACTER,
 	KEYWORD,
 	TYPE,
 	IDENTIFIER,