haxala1r/mash
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change lexer code to use an option instead of ignoring the variant when no value is needed

Browse Source
This commit is contained in:
haxala1r
2025-10-02 17:19:51 +03:00
committed by Emin Arslan
parent d0eae97771
commit 47f33f3dc0
3 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/include/lex.hpp
View File

@@ -3,6 +3,7 @@
#include <vector>
#include <stdint.h>
#include <variant>
#include <optional>
enum TokenType {
OpenParen,
@@ -18,7 +19,7 @@ enum TokenType {
// Plain Old Data
struct Token {
enum TokenType type;
std::variant<int64_t, double, std::string> value;
std::optional<std::variant<int64_t, double, std::string>> value;
};
bool operator==(Token const& one, Token const& other);
std::ostream &operator<<(std::ostream &os, Token const &t);

16
src/lex.cpp
View File

@@ -12,10 +12,10 @@ std::ostream &operator<<(std::ostream &os, Token const &t) {
case TokenType::OpenParen: os << "OpenParen)"; break;
case TokenType::CloseParen: os << "CloseParen)"; break;
case TokenType::Dollar: os << "Dollar)"; break;
case TokenType::Symbol: os << "Symbol, " << get<string>(t.value) << ")"; break;
case TokenType::String: os << "String, \"" << get<string>(t.value) << "\")"; break;
case TokenType::Int: os << "Int, " << get<int64_t>(t.value) << ")"; break;
case TokenType::Double: os << "Double, " << get<double>(t.value) << ")"; break;
case TokenType::Symbol: os << "Symbol, " << get<string>(*t.value) << ")"; break;
case TokenType::String: os << "String, \"" << get<string>(*t.value) << "\")"; break;
case TokenType::Int: os << "Int, " << get<int64_t>(*t.value) << ")"; break;
case TokenType::Double: os << "Double, " << get<double>(*t.value) << ")"; break;
case TokenType::End: os << "END)"; break;
default:
os << ")";
@@ -131,14 +131,14 @@ Token Lexer::next() {
// character while at EOF, even if we have exhausted the stream.
char c = ss.get();
if (ss.eof())
return {TokenType::End};
return {TokenType::End, nullopt};
if (isspace(c))
continue;
switch (c) {
case '(': return {TokenType::OpenParen};
case ')': return {TokenType::CloseParen};
case '$': return {TokenType::Dollar};
case '(': return {TokenType::OpenParen, nullopt};
case ')': return {TokenType::CloseParen, nullopt};
case '$': return {TokenType::Dollar, nullopt};
default:
ss.unget();
return lexNonSpecial();

4
src/tests/test.cpp
View File

@@ -6,7 +6,7 @@ TEST_CASE("Lexer lexes doubles correctly", "[Lexer]") {
SECTION("double and negative syntax") {
Lexer l("(1.0 0.1 -.1 -1. . - -. .-)");
REQUIRE(l.next() == Token({OpenParen}));
REQUIRE(l.next() == Token({OpenParen, nullopt}));
REQUIRE(l.next() == Token({Double, 1.0}));
REQUIRE(l.next() == Token({Double, 0.1}));
REQUIRE(l.next() == Token({Double, -0.1}));
@@ -15,7 +15,7 @@ TEST_CASE("Lexer lexes doubles correctly", "[Lexer]") {
REQUIRE(l.next() == Token({Symbol, "-"}));
REQUIRE(l.next() == Token({Symbol, "-."}));
REQUIRE(l.next() == Token({Symbol, ".-"}));
REQUIRE(l.next() == Token({CloseParen}));
REQUIRE(l.next() == Token({CloseParen, nullopt}));
}
}