huff_neo_utils/
token.rs

1use crate::file::file_source::FileSource;
2use crate::file::span::Span;
3use crate::{opcodes::Opcode, types::PrimitiveEVMType};
4use std::fmt;
5use std::sync::Arc;
6
7/// A 32-byte array representing a hexadecimal literal value
8pub type Literal = [u8; 32];
9
10/// A single Token
11#[derive(Debug, PartialEq, Eq, Clone)]
12pub struct Token {
13    /// The kind of token
14    pub kind: TokenKind,
15    /// An associated Span
16    pub span: Span,
17}
18
19impl Token {
20    /// Public associated function that instantiates a Token.
21    pub fn new(kind: TokenKind, span: Span) -> Self {
22        Self { kind, span }
23    }
24}
25
26/// The kind of token
27#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
28pub enum TokenKind {
29    /// Hexadecimal literal (e.g., 0x42, 0x1234) - hex string with even number of characters
30    HexLiteral(String),
31    /// EOF Token
32    Eof,
33    /// A Comment
34    Comment(String),
35    /// Division
36    /// Lexing done at the comment level due to clash
37    Div,
38    /// "#define" keyword
39    Define,
40    /// "#include" keyword
41    Include,
42    /// "macro" keyword
43    Macro,
44    /// "fn" keyword
45    Fn,
46    /// "test" keyword
47    Test,
48    /// "function" keyword
49    Function,
50    /// "event" keyword
51    Event,
52    /// "constant" keyword
53    Constant,
54    /// "error" keyword
55    Error,
56    /// "takes" keyword
57    Takes,
58    /// "returns" keyword
59    Returns,
60    /// "view" keyword
61    View,
62    /// "pure" keyword
63    Pure,
64    /// "payable" keyword
65    Payable,
66    /// "nonpayable" keyword
67    NonPayable,
68    /// "indexed" keyword
69    Indexed,
70    /// "FREE_STORAGE_POINTER()" keyword
71    FreeStoragePointer,
72    /// "__NOOP" builtin constant
73    Noop,
74    /// An Identifier
75    Ident(String),
76    /// Equal Sign
77    Assign,
78    /// An open parenthesis
79    OpenParen,
80    /// A close parenthesis
81    CloseParen,
82    /// An open bracket
83    OpenBracket,
84    /// A close bracket
85    CloseBracket,
86    /// An open brace
87    OpenBrace,
88    /// A close brace
89    CloseBrace,
90    /// A Less-Than Angle Bracket
91    LeftAngle,
92    /// A Greater-Than Angle Bracket
93    RightAngle,
94    /// Addition
95    Add,
96    /// Subtraction
97    Sub,
98    /// Multiplication
99    Mul,
100    /// Modulo
101    Mod,
102    /// Equal comparison (==)
103    EqualEqual,
104    /// Not equal comparison (!=)
105    NotEqual,
106    /// Less than or equal (<=)
107    LessEqual,
108    /// Greater than or equal (>=)
109    GreaterEqual,
110    /// Logical NOT (!)
111    Not,
112    /// A comma
113    Comma,
114    /// A Colon
115    Colon,
116    /// A pound
117    Pound,
118    /// Integer literal (decimal number)
119    Integer(usize),
120    /// A Space
121    Whitespace,
122    /// A string literal
123    Str(String),
124    /// Opcode
125    Opcode(Opcode),
126    /// Huff label (aka PC)
127    Label(String),
128    // TODO: recursive dependency resolution at the lexing level?
129    // Import path
130    // Path(String),
131    /// EVM Type
132    PrimitiveType(PrimitiveEVMType),
133    /// Array of EVM Types
134    /// `uint256[5][2][3]` => `ArrayType(PrimitiveEVMType::Uint(256)`, `[5, 2, 3]`)
135    ArrayType(PrimitiveEVMType, Vec<usize>),
136    /// A Jump Table
137    JumpTable,
138    /// A Packed Jump Table
139    JumpTablePacked,
140    /// A Code Table
141    CodeTable,
142    /// A builtin function (__codesize, __tablesize, __tablestart)
143    BuiltinFunction(String),
144    /// Calldata Data Location
145    Calldata,
146    /// Memory Data Location
147    Memory,
148    /// Storage Data Location
149    Storage,
150    /// "for" keyword
151    For,
152    /// "in" keyword
153    In,
154    /// "step" keyword
155    Step,
156    /// ".." range operator
157    DoubleDot,
158    /// "if" keyword
159    If,
160    /// "else" keyword
161    Else,
162}
163
164impl TokenKind {
165    /// Transform a single char TokenKind into a Token given a single position
166    pub fn into_single_span(self, position: u32, file: Option<Arc<FileSource>>) -> Token {
167        self.into_span(position, position, file)
168    }
169
170    /// Transform a TokenKind into a Token given a start and end position
171    pub fn into_span(self, start: u32, end: u32, file: Option<Arc<FileSource>>) -> Token {
172        Token { kind: self, span: Span { start: start as usize, end: end as usize, file } }
173    }
174
175    /// Transform a TokenKind into a Token given a Span
176    pub fn into_token_with_span(self, span: Span) -> Token {
177        Token { kind: self, span }
178    }
179}
180
181impl fmt::Display for TokenKind {
182    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
183        let x = match self {
184            TokenKind::HexLiteral(s) => return write!(f, "0x{s}"),
185            TokenKind::Eof => "EOF",
186            TokenKind::Comment(s) => return write!(f, "Comment({s})"),
187            TokenKind::Div => "/",
188            TokenKind::Define => "#define",
189            TokenKind::Include => "#include",
190            TokenKind::Macro => "macro",
191            TokenKind::Fn => "fn",
192            TokenKind::Test => "test",
193            TokenKind::Function => "function",
194            TokenKind::Event => "event",
195            TokenKind::Constant => "constant",
196            TokenKind::Error => "error",
197            TokenKind::View => "view",
198            TokenKind::Pure => "pure",
199            TokenKind::Payable => "payable",
200            TokenKind::NonPayable => "nonpayable",
201            TokenKind::Indexed => "indexed",
202            TokenKind::Takes => "takes",
203            TokenKind::Returns => "returns",
204            TokenKind::FreeStoragePointer => "FREE_STORAGE_POINTER()",
205            TokenKind::Noop => "__NOOP",
206            TokenKind::Ident(s) => return write!(f, "{s}"),
207            TokenKind::Assign => "=",
208            TokenKind::OpenParen => "(",
209            TokenKind::CloseParen => ")",
210            TokenKind::OpenBracket => "[",
211            TokenKind::CloseBracket => "]",
212            TokenKind::OpenBrace => "{",
213            TokenKind::CloseBrace => "}",
214            TokenKind::LeftAngle => "<",
215            TokenKind::RightAngle => ">",
216            TokenKind::Add => "+",
217            TokenKind::Sub => "-",
218            TokenKind::Mul => "*",
219            TokenKind::Mod => "%",
220            TokenKind::EqualEqual => "==",
221            TokenKind::NotEqual => "!=",
222            TokenKind::LessEqual => "<=",
223            TokenKind::GreaterEqual => ">=",
224            TokenKind::Not => "!",
225            TokenKind::Colon => ":",
226            TokenKind::Comma => ",",
227            TokenKind::Pound => "#",
228            TokenKind::Integer(num) => return write!(f, "{num}"),
229            TokenKind::Whitespace => " ",
230            TokenKind::Str(str) => str,
231            TokenKind::Opcode(o) => return write!(f, "{o}"),
232            TokenKind::Label(s) => return write!(f, "{s}"),
233            TokenKind::PrimitiveType(pt) => return write!(f, "{pt}"),
234            TokenKind::ArrayType(pt, size_vec) => {
235                let mut s = String::new();
236                for size in size_vec {
237                    let brackets = if size > &0 { format!("[{size}]") } else { "[]".to_string() };
238                    s.push_str(&brackets);
239                }
240                return write!(f, "{pt}{s}");
241            }
242            TokenKind::JumpTable => "jumptable",
243            TokenKind::JumpTablePacked => "jumptable__packed",
244            TokenKind::CodeTable => "table",
245            TokenKind::BuiltinFunction(s) => return write!(f, "BuiltinFunction({s})"),
246            TokenKind::Calldata => return write!(f, "calldata"),
247            TokenKind::Memory => return write!(f, "memory"),
248            TokenKind::Storage => return write!(f, "storage"),
249            TokenKind::For => "for",
250            TokenKind::In => "in",
251            TokenKind::Step => "step",
252            TokenKind::DoubleDot => "..",
253            TokenKind::If => "if",
254            TokenKind::Else => "else",
255        };
256
257        write!(f, "{x}")
258    }
259}
huff_neo_utils/token.rs

huff_neo_utils/
token.rs
