You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
86 lines
2.4 KiB
Rust
86 lines
2.4 KiB
Rust
/// This module reads characters in arithmetic expression and converts them to tokens.
|
|
/// The allowed tokens are defined in ast module.
|
|
// Standard lib
|
|
use std::iter::Peekable;
|
|
use std::str::Chars;
|
|
|
|
//Other internal modules
|
|
use super::token::Token;
|
|
|
|
// Other structs
|
|
|
|
// Tokenizer struct contains a Peekable iterator on the arithmetic expression
|
|
pub struct Tokenizer<'a> {
|
|
expr: Peekable<Chars<'a>>,
|
|
}
|
|
|
|
// Constructs a new instance of Tokenizer
|
|
impl<'a> Tokenizer<'a> {
|
|
pub fn new(new_expr: &'a str) -> Self {
|
|
Tokenizer {
|
|
expr: new_expr.chars().peekable(),
|
|
}
|
|
}
|
|
}
|
|
|
|
// Implement Iterator trait for Tokenizer struct.
|
|
// With this, we can use next() method on tokenier to retrieve the next token from arithmetic expression
|
|
|
|
impl<'a> Iterator for Tokenizer<'a> {
|
|
type Item = Token;
|
|
|
|
fn next(&mut self) -> Option<Token> {
|
|
let next_char = self.expr.next();
|
|
|
|
match next_char {
|
|
Some('0'..='9') => {
|
|
let mut number = next_char?.to_string();
|
|
|
|
while let Some(next_char) = self.expr.peek() {
|
|
if next_char.is_numeric() || next_char == &'.' {
|
|
number.push(self.expr.next()?);
|
|
} else if next_char == &'(' {
|
|
return None;
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
Some(Token::Num(number.parse::<f64>().unwrap()))
|
|
}
|
|
Some('+') => Some(Token::Add),
|
|
Some('-') => Some(Token::Subtract),
|
|
Some('*') => Some(Token::Multiply),
|
|
Some('/') => Some(Token::Divide),
|
|
Some('^') => Some(Token::Caret),
|
|
Some('(') => Some(Token::LeftParen),
|
|
Some(')') => Some(Token::RightParen),
|
|
None => Some(Token::EOF),
|
|
Some(_) => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
// Unit tests
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_positive_integer() {
|
|
let mut tokenizer = Tokenizer::new("34");
|
|
assert_eq!(tokenizer.next().unwrap(), Token::Num(34.0))
|
|
}
|
|
#[test]
|
|
fn test_decimal_number() {
|
|
let mut tokenizer = Tokenizer::new("34.5");
|
|
assert_eq!(tokenizer.next().unwrap(), Token::Num(34.5))
|
|
}
|
|
#[test]
|
|
#[ignore]
|
|
fn test_invalid_char() {
|
|
let mut tokenizer = Tokenizer::new("#$%");
|
|
assert_eq!(tokenizer.next().unwrap(), Token::Num(34.5));
|
|
}
|
|
}
|