MXVM 1.8.1
Virtual Machine, Compiler, and Pascal Frontend
Loading...
Searching...
No Matches
scanner.hpp
Go to the documentation of this file.
1
6#ifndef _SCANNER_H_
7#define _SCANNER_H_
8
9#include "string_buffer.hpp"
10#include <iostream>
11#include <optional>
12#include <string>
13#include <vector>
14
15namespace scan {
16
21
28 class Scanner {
29 public:
34 Scanner(const TString &b);
39 uint64_t scan();
41 std::optional<TToken> grabId();
43 std::optional<TToken> grabDigits();
45 std::optional<TToken> grabSymbols();
47 std::optional<TToken> grabString();
49 std::optional<TToken> grabSingle();
50
56 TToken &operator[](size_t index);
58 size_t size() const;
60 void removeEOL();
62 std::vector<TToken> &getTokens() { return tokens; }
63
64 private:
67 std::vector<TToken> tokens;
68
70 bool is_c_sym(const StringType &str);
72 uint64_t parseLineNumber();
74 std::string parseFileName();
75 };
76
80 class ScanExcept {
81 public:
82 ScanExcept() = default;
89 StringType why() const { return msg; }
90
91 private:
93 };
94} // namespace scan
95
96std::ostream &operator<<(std::ostream &out, const types::CharType &c);
98std::ostream &operator<<(std::ostream &out, const types::TokenType &t);
99
100#endif
ScanExcept(const StringType &why)
Construct a scan exception with a reason string.
Definition scanner.hpp:87
StringType why() const
Get the error description.
Definition scanner.hpp:89
ScanExcept()=default
StringType msg
Definition scanner.hpp:92
std::optional< TToken > grabSingle()
Try to grab a single-quoted character literal token.
Definition scanner.cpp:420
std::optional< TToken > grabDigits()
Try to grab a numeric literal token.
Definition scanner.cpp:211
uint64_t parseLineNumber()
Parse a line directive line number.
Definition scanner.cpp:22
size_t size() const
Return the total number of scanned tokens.
Definition scanner.cpp:487
std::vector< TToken > & getTokens()
Get the underlying token vector.
Definition scanner.hpp:62
TToken & operator[](size_t index)
Access a token by index.
Definition scanner.cpp:480
std::optional< TToken > grabId()
Try to grab an identifier or keyword token.
Definition scanner.cpp:185
std::string parseFileName()
Parse a line directive filename.
Definition scanner.cpp:44
TString string_buffer
source text buffer
Definition scanner.hpp:65
std::optional< TToken > grabString()
Try to grab a double-quoted string literal token.
Definition scanner.cpp:367
std::vector< TToken > tokens
scanned token list
Definition scanner.hpp:67
Scanner(const TString &b)
Construct a scanner from a string buffer.
Definition scanner.cpp:12
uint64_t scan()
Tokenize the entire source buffer.
Definition scanner.cpp:62
void removeEOL()
Remove end-of-line tokens from the token list.
Definition scanner.cpp:14
bool is_c_sym(const StringType &str)
Check if a string is a recognized C-style symbol.
Definition scanner.cpp:283
std::optional< TToken > grabSymbols()
Try to grab one or more symbol characters as a token.
Definition scanner.cpp:292
TMap token_map
character classification map
Definition scanner.hpp:66
Character-by-character string buffer with line/column tracking.
String string_type
string type alias
Maps characters to their lexical category (letter, digit, symbol, etc.).
A scanned token with type, value, and source position.
buffer::StringBuffer< char >::string_type StringType
Underlying string type.
Definition scanner.hpp:18
token::TokenMap< char > TMap
Default character classification map.
Definition scanner.hpp:20
token::Token< char > TToken
Default token type.
Definition scanner.hpp:19
buffer::StringBuffer< char > TString
Default character string buffer.
Definition scanner.hpp:17
TokenType
Classification of scanned tokens.
Definition types.hpp:16
CharType
Classification of individual characters during lexing.
Definition types.hpp:26
std::ostream & operator<<(std::ostream &out, const types::CharType &c)
Stream output operator for CharType.
Character-by-character string buffer with line/column tracking and Token/TokenMap types.