toml_parser/lexer/
token.rs

1//! Lexed TOML tokens
2
3use super::Span;
4use super::APOSTROPHE;
5use super::COMMENT_START_SYMBOL;
6use super::QUOTATION_MARK;
7use super::WSCHAR;
8use crate::decoder::Encoding;
9
10/// An unvalidated TOML Token
11#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
12pub struct Token {
13    pub(super) kind: TokenKind,
14    pub(super) span: Span,
15}
16
17impl Token {
18    pub(super) fn new(kind: TokenKind, span: Span) -> Self {
19        Self { kind, span }
20    }
21
22    #[inline(always)]
23    pub fn kind(&self) -> TokenKind {
24        self.kind
25    }
26
27    #[inline(always)]
28    pub fn span(&self) -> Span {
29        self.span
30    }
31}
32
33#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
34#[repr(u8)]
35pub enum TokenKind {
36    /// Either for dotted-key or float
37    Dot = b'.',
38    /// Key-value separator
39    Equals = b'=',
40    /// Value separator
41    Comma = b',',
42    /// Either array or standard-table start
43    LeftSquareBracket = b'[',
44    /// Either array or standard-table end
45    RightSquareBracket = b']',
46    /// Inline table start
47    LeftCurlyBracket = b'{',
48    /// Inline table end
49    RightCurlyBracket = b'}',
50    Whitespace = WSCHAR.0,
51    Comment = COMMENT_START_SYMBOL,
52    Newline = b'\n',
53    LiteralString = APOSTROPHE,
54    BasicString = QUOTATION_MARK,
55    MlLiteralString = 1,
56    MlBasicString,
57
58    /// Anything else
59    Atom,
60
61    Eof,
62}
63
64impl TokenKind {
65    pub const fn description(&self) -> &'static str {
66        match self {
67            Self::Dot => "`.`",
68            Self::Equals => "`=`",
69            Self::Comma => "`,`",
70            Self::LeftSquareBracket => "`[`",
71            Self::RightSquareBracket => "`]`",
72            Self::LeftCurlyBracket => "`{`",
73            Self::RightCurlyBracket => "`}`",
74            Self::Whitespace => "whitespace",
75            Self::Comment => "comment",
76            Self::Newline => "newline",
77            Self::LiteralString => "literal string",
78            Self::BasicString => "basic string",
79            Self::MlLiteralString => "multi-line literal string",
80            Self::MlBasicString => "multi-line basic string",
81            Self::Atom => "token",
82            Self::Eof => "end-of-input",
83        }
84    }
85
86    pub fn encoding(&self) -> Option<Encoding> {
87        match self {
88            Self::LiteralString => Some(Encoding::LiteralString),
89            Self::BasicString => Some(Encoding::BasicString),
90            Self::MlLiteralString => Some(Encoding::MlLiteralString),
91            Self::MlBasicString => Some(Encoding::MlBasicString),
92            Self::Atom
93            | Self::LeftSquareBracket
94            | Self::RightSquareBracket
95            | Self::Dot
96            | Self::Equals
97            | Self::Comma
98            | Self::RightCurlyBracket
99            | Self::LeftCurlyBracket
100            | Self::Whitespace
101            | Self::Newline
102            | Self::Comment
103            | Self::Eof => None,
104        }
105    }
106}
toml_parser/lexer/token.rs

toml_parser/lexer/
token.rs
