.idea/editor.xml

@@ -27,6 +27,7 @@
     <option name="/Default/CodeStyle/CodeFormatting/CppFormatting/SIMPLE_EMBEDDED_STATEMENT_STYLE/@EntryValue" value="LINE_BREAK" type="string" />
     <option name="/Default/CodeStyle/CodeFormatting/CppFormatting/SPACE_AFTER_CAST_EXPRESSION_PARENTHESES/@EntryValue" value="true" type="bool" />
     <option name="/Default/CodeStyle/CodeFormatting/CppFormatting/WRAP_LINES/@EntryValue" value="false" type="bool" />
+    <option name="/Default/CodeStyle/Generate/=CppEqualityOperators/Options/=GenerateSecondaryOperators/@EntryIndexedValue" value="False" type="string" />
     <option name="/Default/CodeStyle/Naming/CppNaming/Rules/=Classes_0020and_0020structs/@EntryIndexedValue" value="&lt;NamingElement Priority=&quot;1&quot;&gt;&lt;Descriptor Static=&quot;Indeterminate&quot; Constexpr=&quot;Indeterminate&quot; Const=&quot;Indeterminate&quot; Volatile=&quot;Indeterminate&quot; Accessibility=&quot;NOT_APPLICABLE&quot;&gt;&lt;type Name=&quot;__interface&quot; /&gt;&lt;type Name=&quot;class&quot; /&gt;&lt;type Name=&quot;struct&quot; /&gt;&lt;/Descriptor&gt;&lt;Policy Inspect=&quot;True&quot; Prefix=&quot;&quot; Suffix=&quot;&quot; Style=&quot;AaBb&quot; /&gt;&lt;/NamingElement&gt;" type="string" />
     <option name="/Default/CodeStyle/Naming/CppNaming/Rules/=Concepts/@EntryIndexedValue" value="&lt;NamingElement Priority=&quot;2&quot;&gt;&lt;Descriptor Static=&quot;Indeterminate&quot; Constexpr=&quot;Indeterminate&quot; Const=&quot;Indeterminate&quot; Volatile=&quot;Indeterminate&quot; Accessibility=&quot;NOT_APPLICABLE&quot;&gt;&lt;type Name=&quot;concept&quot; /&gt;&lt;/Descriptor&gt;&lt;Policy Inspect=&quot;True&quot; Prefix=&quot;&quot; Suffix=&quot;&quot; Style=&quot;AaBb&quot; /&gt;&lt;/NamingElement&gt;" type="string" />
     <option name="/Default/CodeStyle/Naming/CppNaming/Rules/=Enum_0020members/@EntryIndexedValue" value="&lt;NamingElement Priority=&quot;14&quot;&gt;&lt;Descriptor Static=&quot;Indeterminate&quot; Constexpr=&quot;Indeterminate&quot; Const=&quot;Indeterminate&quot; Volatile=&quot;Indeterminate&quot; Accessibility=&quot;NOT_APPLICABLE&quot;&gt;&lt;type Name=&quot;scoped enumerator&quot; /&gt;&lt;type Name=&quot;unscoped enumerator&quot; /&gt;&lt;/Descriptor&gt;&lt;Policy Inspect=&quot;True&quot; Prefix=&quot;&quot; Suffix=&quot;&quot; Style=&quot;AaBb&quot; /&gt;&lt;/NamingElement&gt;" type="string" />

src/assembler/lexer.cc

@@ -12,12 +12,12 @@ void Lexer::reset()
 
 Token Lexer::peek() const
 {
-    return look_ahead_;
+    return current_token_;
 }
 
 Token Lexer::ingest()
 {
-    Token t = look_ahead_;
+    Token t = current_token_;
     ingest_next_token();
     return t;
 }
@@ -26,21 +26,54 @@ void Lexer::ingest_next_token()
 {
     char c = source_.at(pos_);
 
+    TokenType type {};
+    std::string token;
+
     if (pos_ >= source_.size()) {
-        look_ahead_ = { TokenType::EOF_, "" };
+        type = TokenType::EOF_;
     } else if (c == '.') {
-
+        type = TokenType::Directive;
+        token += '.';
+        while (c = source_.at(++pos_), isalpha(c) || c == '_')
+            token += c;
     } else if (c == '"') {
-
+        type = TokenType::String;
-    } else if (isdigit(c)) {
+        ++pos_;
-
+        while (true) {
+            if (source_.at(pos_) == '\'') {   // TODO - improve this for special characters
+                ++pos_;
+            } else if (source_.at(pos_) == '"') {
+                ++pos_;
+                break;
+            } else if (pos_ >= source_.size()) {
+                throw AssemblyError("Unterminated string");
+            }
+            token += source_.at(pos_++);
+        }
+    } else if (isdigit(c) || c == '-') {
+        type = TokenType::Integer;
+        token += c;
+        while (c = source_.at(++pos_), isdigit(c) || c == '.') {
+            token += c;
+            if (c == '.') {
+                if (type == TokenType::Integer)
+                    type = TokenType::Float;
+                else
+                    throw AssemblyError("Double point in floating point number");
+            }
+        }
     } else if (isalpha(c)) {
-
+        type = TokenType::Instruction;
+        token += c;
+        while (c = source_.at(++pos_), isalpha(c))
+            token += c;
     } else if (c == '\n') {
-
+        type = TokenType::Enter;
     } else if (c != ' ' && c != '\t' && c != '\r') {
         throw AssemblyError(std::string("Unexpected character '") + c + "' (ascii: " + std::to_string((int) c) + ")");
     }
+
+    current_token_ = { .type = type, .token = token };
 }
 
 } // tyche

src/assembler/lexer.hh

@@ -6,11 +6,13 @@
 
 namespace tyche::as {
 
-enum class TokenType { BOF, Directive, Instruction, Number, Float, String, Enter, EOF_ };
+enum class TokenType { BOF, Directive, Instruction, Integer, Float, String, Enter, EOF_ };
 
 struct Token {
     TokenType   type;
     std::string token;
+
+    friend bool operator==(Token const& lhs, Token const& rhs) { return std::tie(lhs.type, lhs.token) == std::tie(rhs.type, rhs.token); }
 };
 
 class Lexer {
@@ -24,7 +26,7 @@ public:
 private:
     const std::string source_;
     size_t pos_ = 0;
-    Token look_ahead_;
+    Token current_token_ { TokenType::BOF, "" };
 
     void ingest_next_token();
 };

src/assembler/tests.cc

@@ -10,6 +10,14 @@ using namespace tyche::as;
 using namespace tyche::bc;
 using namespace tyche::vm;
 
+TEST(Lexer, Lexer)
+{
+    Lexer lexer(".dir push 382 -12 3.14 -12.8 \"Hello\" \"Hel\"lo\"\n");
+
+    ASSERT_EQ(lexer.ingest(), (Token { TokenType::Directive, ".dir" }));
+    ASSERT_EQ(lexer.ingest(), (Token { TokenType::Instruction, "push" }));
+}
+
 TEST(Assember, Assembler)
 {
     BytecodePrototype bp;