basic_tokenizer.h File Reference

#include <kernel.h>

Macros
#define	MAX_VARNAME   50
#define	GENERATE_ENUM(ENUM)   ENUM,
#define	GENERATE_STRING(STRING)   #STRING,
#define	GENERATE_ENUM_LIST(MACRO, NAME)   enum NAME { MACRO(GENERATE_ENUM) };
#define	GENERATE_ENUM_STRING_NAMES(MACRO, NAME)   const char* NAME [] = { MACRO(GENERATE_STRING) };
#define	TOKEN(T)
	All tokens recognised by the interpreter. Note that built in function names are NOT tokens, they are parsed like user functions, just with a hard coded handler instead of redirecting into the user program. More...

Enumerations
enum	token_t {   ERROR , ENDOFINPUT , NUMBER , HEXNUMBER ,   STRING , VARIABLE , LET , PRINT ,   IF , THEN , ELSE , CHAIN ,   FOR , STEP , TO , NEXT ,   CURSOR , GOTO , GOSUB , RETURN ,   CALL , INPUT , COLOUR , COLOR ,   BACKGROUND , EVAL , OPENIN , READ ,   CLOSE , EOF , DEF , PROC ,   RETPROC , FN , END , REM ,   COMMA , SEMICOLON , PLUS , MINUS ,   AND , OR , NOT , EOR ,   ASTERISK , SLASH , MOD , OPENBRACKET ,   CLOSEBRACKET , LESSTHAN , GREATERTHAN , EQUALS ,   NEWLINE , AMPERSAND , TILDE , GLOBAL ,   SOCKREAD , SOCKWRITE , CONNECT , SOCKCLOSE ,   CLS , GCOL , LINE , TRIANGLE ,   RECTANGLE , CIRCLE , POINT , OPENOUT ,   OPENUP , WRITE , MKDIR , RMDIR ,   DELETE , REPEAT , UNTIL , DIM ,   REDIM , PUSH , POP , LOCAL ,   CHDIR , LIBRARY , YIELD , MOUNT }

Functions
*brief Initialise tokenizer **param program program text *param ctx context *void	tokenizer_init (const char *program, struct basic_ctx *ctx)
void	tokenizer_next (struct basic_ctx *ctx)
	advance to next token More...
int	tokenizer_token (struct basic_ctx *ctx)
	peek to next token More...
int64_t	tokenizer_num (struct basic_ctx *ctx, int token)
	Get integer number as next token (do not advance the pointer) More...
void	tokenizer_fnum (struct basic_ctx *ctx, int token, double *f)
	Get real number as next token (do not advance the pointer) More...
const char *	tokenizer_variable_name (struct basic_ctx *ctx)
	Get a variable name as next token (do not advance the pointer) More...
bool	tokenizer_string (char *dest, int len, struct basic_ctx *ctx)
	Get a string constant as the next token (do not advance the pointer) More...
int	tokenizer_finished (struct basic_ctx *ctx)
	Returns true if the program is finished (does not advance the pointer) More...
void	tokenizer_error_print (struct basic_ctx *ctx, const char *error)
	display an error to the terminal and end the program More...
int	get_next_token (struct basic_ctx *ctx)
	Get the next token (advances the pointer past the end of the token) More...
bool	tokenizer_decimal_number (struct basic_ctx *ctx)
	Check if a decimal number is at the current program pointer. (does not advance the pointer) More...

Macro Definition Documentation

GENERATE_ENUM

#define GENERATE_ENUM

(

ENUM

)

ENUM,

These macros generate both an enum and an array of strings, these are used as part of the tokenizer to parse the names of the keywords out of the BASIC program without having to remember to match up a string and its token enum value in two places. It is also used for error reporting.

GENERATE_ENUM_LIST

#define GENERATE_ENUM_LIST	(		MACRO,
			NAME
	)		enum NAME { MACRO(GENERATE_ENUM) };

GENERATE_ENUM_STRING_NAMES

#define GENERATE_ENUM_STRING_NAMES	(		MACRO,
			NAME
	)		const char* NAME [] = { MACRO(GENERATE_STRING) };

GENERATE_STRING

#define GENERATE_STRING

(

STRING

)

#STRING,

MAX_VARNAME

#define MAX_VARNAME   50

TOKEN

#define TOKEN

(

T

)

All tokens recognised by the interpreter. Note that built in function names are NOT tokens, they are parsed like user functions, just with a hard coded handler instead of redirecting into the user program.

The #define below builds an enum, and can also build an array of strings of the names in the enum, which is built and used within basic.c for tokenization.

Enumeration Type Documentation

token_t

enum token_t

Enumerator
ERROR
ENDOFINPUT
NUMBER
HEXNUMBER
STRING
VARIABLE
LET
PRINT
IF
THEN
ELSE
CHAIN
FOR
STEP
TO
NEXT
CURSOR
GOTO
GOSUB
RETURN
CALL
INPUT
COLOUR
COLOR
BACKGROUND
EVAL
OPENIN
READ
CLOSE
EOF
DEF
PROC
RETPROC
FN
END
REM
COMMA
SEMICOLON
PLUS
MINUS
AND
OR
NOT
EOR
ASTERISK
SLASH
MOD
OPENBRACKET
CLOSEBRACKET
LESSTHAN
GREATERTHAN
EQUALS
NEWLINE
AMPERSAND
TILDE
GLOBAL
SOCKREAD
SOCKWRITE
CONNECT
SOCKCLOSE
CLS
GCOL
LINE
TRIANGLE
RECTANGLE
CIRCLE
POINT
OPENOUT
OPENUP
WRITE
MKDIR
RMDIR
DELETE
REPEAT
UNTIL
DIM
REDIM
PUSH
POP
LOCAL
CHDIR
LIBRARY
YIELD
MOUNT

Function Documentation

get_next_token()

int get_next_token

(

struct basic_ctx *

ctx

)

Get the next token (advances the pointer past the end of the token)

Parameters

ctx

context

Returns

int token found

tokenizer_decimal_number()

bool tokenizer_decimal_number

(

struct basic_ctx *

ctx

)

Check if a decimal number is at the current program pointer. (does not advance the pointer)

Parameters

ctx

context

Returns

true if pointer points at a decimal number

tokenizer_error_print()

void tokenizer_error_print	(	struct basic_ctx *	ctx,
		const char *	error
	)

display an error to the terminal and end the program

Note

If the program is running an EVAL, the error is printed but the program is not ended, instead ERROR$ and ERROR are set.

Parameters

ctx	context
error	error message

tokenizer_finished()

int tokenizer_finished

(

struct basic_ctx *

ctx

)

Returns true if the program is finished (does not advance the pointer)

Parameters

ctx

context

Returns

int true if the program has finished

tokenizer_fnum()

void tokenizer_fnum	(	struct basic_ctx *	ctx,
		int	token,
		double *	f
	)

Get real number as next token (do not advance the pointer)

Parameters

ctx	context
token	token (NUMBER)
f	number read from program

tokenizer_init()

* brief Initialise tokenizer* * param program program text* param ctx context* void tokenizer_init	(	const char *	program,
		struct basic_ctx *	ctx
	)

tokenizer_next()

void tokenizer_next

(

struct basic_ctx *

ctx

)

advance to next token

Parameters

ctx

context

tokenizer_num()

int64_t tokenizer_num	(	struct basic_ctx *	ctx,
		int	token
	)

Get integer number as next token (do not advance the pointer)

Parameters

ctx	context
token	token (NUMBER or HEXNUMBER)

Returns

int64_t number read from program

tokenizer_string()

bool tokenizer_string	(	char *	dest,
		int	len,
		struct basic_ctx *	ctx
	)

Get a string constant as the next token (do not advance the pointer)

Parameters

dest	destination string buffer
len	length of destination buffer
ctx	context

Returns

true if succesfully found a string constant

tokenizer_token()

int tokenizer_token

(

struct basic_ctx *

ctx

)

peek to next token

Parameters

ctx

context

Returns

int token

tokenizer_variable_name()

const char* tokenizer_variable_name

(

struct basic_ctx *

ctx

)

Get a variable name as next token (do not advance the pointer)

Parameters

ctx

context

Returns

const char* variable name