SilverMoonStudio/Arctyx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved the tokenizer.

Browse Source
This commit is contained in:
Karutoh
2025-04-29 00:24:13 -07:00
parent 0fc10f4c76
commit ff6b5785ae
7 changed files with 148 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
src/compiler/Compiler.cpp
View File

@@ -1,5 +1,8 @@
#include "arctyx/compiler/Compiler.h"
#include <iterator>
#include <ehs/io/Console.h>
Compiler::Compiler()
: architecture(nullptr), language(nullptr)
{
@@ -65,13 +68,51 @@ Compiler& Compiler::operator=(const Compiler& other)
ehs::Array<ehs::Byte> Compiler::Compile(const ehs::Str_8 &code) const
{
ehs::Console::Write_8("Code:");
ehs::Console::Write_8(code);
ehs::Console::Write_8("Tokens:");
ehs::Vector<Token> tokens = Parse(code);
for (ehs::UInt_64 i = 0; i < tokens.Size(); ++i)
{
if (tokens[i].GetType() == TokenT::UNKNOWN)
ehs::Console::Write_8("UNKNOWN, ", false);
else if (tokens[i].GetType() == TokenT::SEPARATOR)
ehs::Console::Write_8("SEPARATOR, ", false);
else if (tokens[i].GetType() == TokenT::NUMBER)
ehs::Console::Write_8("NUMBER, ", false);
else if (tokens[i].GetType() == TokenT::STRING)
ehs::Console::Write_8("STRING, ", false);
else if (tokens[i].GetType() == TokenT::CHARACTER)
ehs::Console::Write_8("CHARACTER, ", false);
else if (tokens[i].GetType() == TokenT::KEYWORD)
ehs::Console::Write_8("KEYWORD, ", false);
else if (tokens[i].GetType() == TokenT::TYPE)
ehs::Console::Write_8("TYPE, ", false);
else if (tokens[i].GetType() == TokenT::IDENTIFIER)
ehs::Console::Write_8("IDENTIFIER, ", false);
else if (tokens[i].GetType() == TokenT::UNARY_OPERATOR)
ehs::Console::Write_8("UNARY_OPERATOR, ", false);
else if (tokens[i].GetType() == TokenT::COMPOUND_OPERATOR)
ehs::Console::Write_8("COMPOUND_OPERATOR, ", false);
else if (tokens[i].GetType() == TokenT::EOL)
ehs::Console::Write_8("EOL");
}
ehs::Array<ehs::Byte> machineCode;
return machineCode;
}
bool Compiler::IsEOL(const ehs::Array<ehs::Char_8>& eols, const ehs::Char_8* c)
{
for (ehs::UInt_64 s = 0; s < eols.Size(); ++s)
if (*c == eols[s])
return true;
return false;
}
bool Compiler::IsSeparator(const ehs::Array<ehs::Char_8> &separators, const ehs::Char_8 *c)
{
for (ehs::UInt_64 s = 0; s < separators.Size(); ++s)
@@ -108,33 +149,11 @@ const Operator *Compiler::IsOperator(const ehs::Array<Operator>& operators, cons
return nullptr;
}
Token Compiler::ParseValue(const ehs::Array<Primitive> &primitives, const ehs::Array<ehs::Str_8> &keywords, const ehs::Array<Operator> &operators, const ehs::Str_8& value) const
{
if (IsPrimitive(primitives, value))
return {TokenT::TYPE, value};
if (IsKeyword(keywords, value))
return {TokenT::KEYWORD, value};
const Operator *op = IsOperator(operators, value);
if (op)
{
if (op->IsUnary())
return {TokenT::UNARY_OPERATOR, value};
else
return {TokenT::COMPOUND_OPERATOR, value};
}
if (value[0] >= '0' && value[0] <= '9')
return {TokenT::VALUE, value};
return {TokenT::IDENTIFIER, value};
}
ehs::Vector<Token> Compiler::Parse(const ehs::Str_8 &code) const
{
ehs::Vector<Token> tokens;
const ehs::Array<ehs::Char_8> eols = language->GetEOLs();
const ehs::Array<ehs::Char_8> separators = language->GetSeparators();
const ehs::Array<Primitive> primitives = language->GetPrimitives();
const ehs::Array<ehs::Str_8> keywords = language->GetKeywords();
@@ -142,45 +161,78 @@ ehs::Vector<Token> Compiler::Parse(const ehs::Str_8 &code) const
for (ehs::Char_8 *i = &code[0], *start = i; i < &code[code.Size()]; ++i)
{
if (*i == language->GetEOL())
{
if (start != i)
tokens.Push(ParseValue(primitives, keywords, operators, ehs::Str_8(start, i - start)));
tokens.Push({TokenT::EOL, {i, 1}});
start = i + 1;
continue;
}
if (*start == '\"')
{
if (i != start && *i == '\"')
if (*i == '\"')
{
tokens.Push({TokenT::VALUE, {start, (ehs::UInt_64)(i - (start - 1))}});
++start;
tokens.Push({TokenT::STRING, ehs::Str_8(start, i - 1 - start)});
if (i + 1 < &code[code.Size()])
start = i + 1;
}
}
else if (*start == '\'')
{
if (*i == '\'')
{
++start;
const ehs::UInt_64 size = i - 1 - start;
if (size > 1)
{
EHS_LOG(ehs::LogType::ERR, 0, "Characters cannot extend to more than one.");
return {};
}
tokens.Push({TokenT::CHARACTER, {start, size}});
if (i + 1 < &code[code.Size()])
start = i + 1;
}
}
else if (const bool isEOL = IsEOL(eols, i), isSep = IsSeparator(separators, i); *i == ' ' || *i == '\t' || isEOL || isSep)
{
if ((*start == ' ' || *start == '\t') && i + 1 < &code[code.Size()])
{
start = i + 1;
continue;
}
continue;
}
const ehs::Str_8 value(start, i - start);
if (!IsSeparator(separators, i))
continue;
if (value.Size())
{
if (IsPrimitive(primitives, value))
tokens.Push({TokenT::TYPE, value});
else if (IsKeyword(keywords, value))
tokens.Push({TokenT::KEYWORD, value});
else if (const Operator *op = IsOperator(operators, value); op)
{
if (op->IsUnary())
tokens.Push({TokenT::UNARY_OPERATOR, value});
else
tokens.Push({TokenT::COMPOUND_OPERATOR, value});
}
else if (value[0] >= '0' && value[0] <= '9')
tokens.Push({TokenT::NUMBER, value});
else
tokens.Push({TokenT::IDENTIFIER, value});
}
if (isEOL)
tokens.Push({TokenT::EOL, {i, 1}});
else if (isSep)
tokens.Push({TokenT::SEPARATOR, {i, 1}});
if (i - (start - 1) == 1 && IsSeparator(separators, start))
{
start = i + 1;
continue;
}
tokens.Push(ParseValue(primitives, keywords, operators, ehs::Str_8(start, i - start)));
start = i + 1;
}
EHS_LOG_SUCCESS();
return tokens;
}

36
src/compiler/Language.cpp
View File

@@ -3,28 +3,27 @@
ehs::Array<const Language *> Language::languages;
Language::Language()
: id(0), eol('\n')
: id(0)
{
}
Language::Language(ehs::Str_8 name, const ehs::Version& version)
: id(name.Hash_64()), name((ehs::Str_8 &&)name), version(version), eol('\n')
: id(name.Hash_64()), name((ehs::Str_8 &&)name), version(version)
{
}
Language::Language(Language&& lang) noexcept
: id(lang.id), name((ehs::Str_8 &&)lang.name), version(lang.version), eol(lang.eol),
: id(lang.id), name((ehs::Str_8 &&)lang.name), version(lang.version), eols((ehs::Array<ehs::Char_8> &&)lang.eols),
separators((ehs::Array<ehs::Char_8> &&)lang.separators), primitives((ehs::Array<Primitive> &&)lang.primitives),
keywords((ehs::Array<ehs::Str_8> &&)lang.keywords), operators((ehs::Array<Operator> &&)lang.operators),
interpretations((ehs::Array<Interpretation> &&)lang.interpretations)
{
lang.id = 0;
lang.version = {};
lang.eol = '\n';
}
Language::Language(const Language& lang)
: id(lang.id), name(lang.name), version(lang.version), eol(lang.eol), separators(lang.separators),
: id(lang.id), name(lang.name), version(lang.version), eols(lang.eols), separators(lang.separators),
primitives(lang.primitives), keywords(lang.keywords), operators(lang.operators),
interpretations(lang.interpretations)
{
@@ -38,7 +37,7 @@ Language& Language::operator=(Language&& lang) noexcept
id = lang.id;
name = (ehs::Str_8 &&)lang.name;
version = lang.version;
eol = lang.eol;
eols = (ehs::Array<ehs::Char_8> &&)lang.eols;
separators = (ehs::Array<ehs::Char_8> &&)lang.separators;
primitives = (ehs::Array<Primitive> &&)lang.primitives;
keywords = (ehs::Array<ehs::Str_8> &&)lang.keywords;
@@ -47,7 +46,6 @@ Language& Language::operator=(Language&& lang) noexcept
lang.id = 0;
lang.version = {};
lang.eol = '\n';
return *this;
}
@@ -60,7 +58,7 @@ Language& Language::operator=(const Language& lang)
id = lang.id;
name = lang.name;
version = lang.version;
eol = lang.eol;
eols = lang.eols;
separators = lang.separators;
primitives = lang.primitives;
keywords = lang.keywords;
@@ -85,14 +83,28 @@ ehs::Version Language::GetVersion() const
return version;
}
void Language::SetEOL(const ehs::Char_8& newEOL)
ehs::Array<ehs::Char_8> Language::GetEOLs() const
{
eol = newEOL;
return eols;
}
ehs::Char_8 Language::GetEOL() const
bool Language::HasEOL(const ehs::Char_8 &eol) const
{
return eol;
for (ehs::UInt_64 i = 0; i < eols.Size(); ++i)
if (eols[i] == eol)
return true;
return false;
}
bool Language::AddEOL(const ehs::Char_8 &eol)
{
if (HasEOL(eol))
return false;
eols.Push(eol);
return true;
}
ehs::Array<ehs::Char_8> Language::GetSeparators() const