Mochaccino Sprint 2
Share
Explore
Mochaccino Sprint 2

icon picker
Tokeniser

tokeniser.dart
⁠

Tokeniser

tokenise

scanToken

addToken

advance

match

peek

tokeniseString

tokeniseNumber

tokeniseIdentifier

atEnd


Token

toString


TokenType


keywords


Final File

part of cortado;

enum TokenType {
// SINGLE-CHARACTER TOKENS
LEFT_PAREN,
RIGHT_PAREN,
LEFT_BRACK,
RIGHT_BRACK,
LEFT_BRACE,
RIGHT_BRACE,
COMMA,
DOT,
MINUS,
PLUS,
SEMICOLON,
SLASH,
STAR,
// ONE- or TWO-CHARACTER TOKENS
BANG,
BANG_EQUAL,
EQUAL,
EQUAL_EQUAL,
ANGLED_RIGHT,
ANGLED_RIGHT_EQUAL,
ANGLED_LEFT,
ANGLED_LEFT_EQUAL,
PIPE_PIPE,
AMPERSAND_AMPERSAND,
// LITERALS
IDENTIFIER,
STRING,
NUMBER,
BOOL,
NULL,
// KEYWORDS
OK,
NOTOK,
NAMED,
PACKAGE,
INCLUDE,
FROM,
AS,
GUARDED,
EXCEPT,
VAR,
RETURN,
ASYNC,
STATIC,
MODULE,
STRUCT,
FUNC,
DOCK,
EXTENDS,
IMPLEMENTS,
IF,
ELIF,
ELSE,
FOR,
WHILE,
PROP,
GET,
SET,
// DEBUG FLAGS
DEBUG_FLAG,
STRUCT_ANNOTATION,
// EOF
EOF,
}

class Tokeniser {
final String source;
final List<String> lines = [];
final List<Token> tokens = [];
int start = 0;
int current = 0;
int lineNo = 0;
Tokeniser(this.source) {
lines.addAll(source.split('\n'));
}
List<Token> tokenise() {
while (!atEnd) {
start = current;
scanToken();
}
return tokens;
}
void scanToken() {
String c = advance();
switch (c) {
case '(':
addToken(TokenType.LEFT_PAREN);
break;
case ')':
addToken(TokenType.RIGHT_PAREN);
break;
case '{':
addToken(TokenType.LEFT_BRACE);
break;
case '}':
addToken(TokenType.RIGHT_BRACE);
break;
case ',':
addToken(TokenType.COMMA);
break;
case '.':
addToken(TokenType.DOT);
break;
case '-':
addToken(TokenType.MINUS);
break;
case '+':
addToken(TokenType.PLUS);
break;
case ';':
addToken(TokenType.SEMICOLON);
break;
case '*':
addToken(TokenType.STAR);
break;
case '!':
addToken(match('=') ? TokenType.BANG_EQUAL : TokenType.BANG);
break;
case '=':
addToken(match('=') ? TokenType.EQUAL_EQUAL : TokenType.EQUAL);
break;
case '<':
addToken(match('=') ? TokenType.ANGLED_LEFT_EQUAL : TokenType.ANGLED_LEFT);
break;
case '>':
addToken(match('=') ? TokenType.ANGLED_RIGHT_EQUAL : TokenType.ANGLED_RIGHT);
break;
case '/':
if (match('/')) {
while (peek() != '\n' && !atEnd) {
advance();
}
} else {
addToken(TokenType.SLASH);
}
break;
case ' ':
case '\r':
case '\t':
// Ignore whitespace.
break;
case '\n':
lineNo++;
break;
case '"':
tokeniseString();
break;
default:
if (c.isDigit) {
tokeniseNumber();
} else if (c.isAlpha) {
tokeniseIdentifier();
} else {
ErrorHandler.issues.add(
Issue(
IssueType.SyntaxError,
IssueTitle.unexpectedChar(c),
lineNo: lineNo,
start: current,
offendingLine: lines[lineNo],
description: "Tokeniser could not produce a token for '$c'. It may be an illegal character in the given context.",
),
);
}
break;
}
}
void addToken(TokenType tokenType, [Object? literal]) {
tokens.add(
Token(
tokenType,
source.substring(start, current),
lineNo: lineNo,
start: start,
literal: literal,
),
);
}
String advance() {
return source.charAt(current++);
}
bool match(String expected) {
if (atEnd) return false;
if (source.charAt(current) != expected) return false;
current++;
return true;
}
String peek([int lookaheadCount = 0]) {
if (current + lookaheadCount >= source.length) return 'EOF';
return source.charAt(current + lookaheadCount);
}
void tokeniseString() {
while (peek() != '"' && !atEnd) {
if (peek() == '\n') lineNo++;
advance();
}
if (atEnd) {
// unterminated string error
ErrorHandler.issues.add(
Issue(
IssueType.SyntaxError,
IssueTitle.unterminatedPair('"'),
lineNo: lineNo,
start: start,
offendingLine: lines[lineNo],
description: 'Unterminated string literal.',
),
);
Gallery
Share
 
Want to print your doc?
This is not the way.
Try clicking the ⋯ next to your doc name or using a keyboard shortcut (
CtrlP
) instead.