tokenizer.h File Reference

#include <kernel.h>

Macros
#define	MAX_VARNAME   50
#define	GENERATE_ENUM(ENUM)   ENUM,
#define	GENERATE_STRING(STRING)   #STRING,
#define	GENERATE_ENUM_LIST(MACRO, NAME)   enum NAME { MACRO(GENERATE_ENUM) };
#define	GENERATE_ENUM_STRING_NAMES(MACRO, NAME)   const char* NAME [] = { MACRO(GENERATE_STRING) };
#define	TOKEN(T)
	All tokens recognised by the interpreter. Note that built in function names are NOT tokens, they are parsed like user functions, just with a hard coded handler instead of redirecting into the user program. More...

Enumerations
enum	token_t {   ERROR , ENDOFINPUT , NUMBER , HEXNUMBER ,   STRING , VARIABLE , LET , PRINT ,   IF , THEN , ELSE , CHAIN ,   FOR , STEP , TO , NEXT ,   CURSOR , GOTO , GOSUB , RETURN ,   CALL , INPUT , COLOUR , COLOR ,   BACKGROUND , EVAL , OPENIN , READ ,   CLOSE , EOF , DEF , PROC ,   RETPROC , FN , END , REM ,   COMMA , SEMICOLON , PLUS , MINUS ,   AND , OR , ASTERISK , SLASH ,   MOD , OPENBRACKET , CLOSEBRACKET , LESSTHAN ,   GREATERTHAN , EQUALS , NEWLINE , AMPERSAND ,   TILDE , GLOBAL , SOCKREAD , SOCKWRITE ,   CONNECT , SOCKCLOSE , CLS , GCOL ,   LINE , TRIANGLE , RECTANGLE , CIRCLE ,   POINT , OPENOUT , OPENUP , WRITE ,   MKDIR , RMDIR , DELETE , REPEAT ,   UNTIL , DIM , REDIM , PUSH ,   POP , LOCAL , CHDIR , MOUNT }

Functions
*brief Initialise tokenizer **param program program text *param ctx context *void	tokenizer_init (const char *program, struct ubasic_ctx *ctx)
void	tokenizer_next (struct ubasic_ctx *ctx)
	advance to next token More...
int	tokenizer_token (struct ubasic_ctx *ctx)
	peek to next token More...
int64_t	tokenizer_num (struct ubasic_ctx *ctx, int token)
	Get integer number as next token (do not advance the pointer) More...
void	tokenizer_fnum (struct ubasic_ctx *ctx, int token, double *f)
	Get real number as next token (do not advance the pointer) More...
const char *	tokenizer_variable_name (struct ubasic_ctx *ctx)
	Get a variable name as next token (do not advance the pointer) More...
bool	tokenizer_string (char *dest, int len, struct ubasic_ctx *ctx)
	Get a string constant as the next token (do not advance the pointer) More...
int	tokenizer_finished (struct ubasic_ctx *ctx)
	Returns true if the program is finished (does not advance the pointer) More...
void	tokenizer_error_print (struct ubasic_ctx *ctx, const char *error)
	display an error to the terminal and end the program More...
int	get_next_token (struct ubasic_ctx *ctx)
	Get the next token (advances the pointer past the end of the token) More...
bool	tokenizer_decimal_number (struct ubasic_ctx *ctx)
	Check if a decimal number is at the current program pointer. (does not advance the pointer) More...

Detailed Description

BBC BASIC interpreter, Retro Rocket OS Project (C) Craig Edwards 2012. loosely based on uBASIC (Copyright (c) 2006, Adam Dunkels, All rights reserved).

THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS `‘AS IS’' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Macro Definition Documentation

GENERATE_ENUM

#define GENERATE_ENUM

(

ENUM

)

ENUM,

These macros generate both an enum and an array of strings, these are used as part of the tokenizer to parse the names of the keywords out of the BASIC program without having to remember to match up a string and its token enum value in two places. It is also used for error reporting.

GENERATE_ENUM_LIST

#define GENERATE_ENUM_LIST	(		MACRO,
			NAME
	)		enum NAME { MACRO(GENERATE_ENUM) };

GENERATE_ENUM_STRING_NAMES

#define GENERATE_ENUM_STRING_NAMES	(		MACRO,
			NAME
	)		const char* NAME [] = { MACRO(GENERATE_STRING) };

GENERATE_STRING

#define GENERATE_STRING

(

STRING

)

#STRING,

MAX_VARNAME

#define MAX_VARNAME   50

TOKEN

#define TOKEN

(

T

)

All tokens recognised by the interpreter. Note that built in function names are NOT tokens, they are parsed like user functions, just with a hard coded handler instead of redirecting into the user program.

The #define below builds an enum, and can also build an array of strings of the names in the enum, which is built and used within ubasic.c for tokenization.

Enumeration Type Documentation

token_t

enum token_t

Enumerator
ERROR
ENDOFINPUT
NUMBER
HEXNUMBER
STRING
VARIABLE
LET
PRINT
IF
THEN
ELSE
CHAIN
FOR
STEP
TO
NEXT
CURSOR
GOTO
GOSUB
RETURN
CALL
INPUT
COLOUR
COLOR
BACKGROUND
EVAL
OPENIN
READ
CLOSE
EOF
DEF
PROC
RETPROC
FN
END
REM
COMMA
SEMICOLON
PLUS
MINUS
AND
OR
ASTERISK
SLASH
MOD
OPENBRACKET
CLOSEBRACKET
LESSTHAN
GREATERTHAN
EQUALS
NEWLINE
AMPERSAND
TILDE
GLOBAL
SOCKREAD
SOCKWRITE
CONNECT
SOCKCLOSE
CLS
GCOL
LINE
TRIANGLE
RECTANGLE
CIRCLE
POINT
OPENOUT
OPENUP
WRITE
MKDIR
RMDIR
DELETE
REPEAT
UNTIL
DIM
REDIM
PUSH
POP
LOCAL
CHDIR
MOUNT

Function Documentation

get_next_token()

int get_next_token

(

struct ubasic_ctx *

ctx

)

Get the next token (advances the pointer past the end of the token)

Parameters

ctx

context

Returns

int token found

tokenizer_decimal_number()

bool tokenizer_decimal_number

(

struct ubasic_ctx *

ctx

)

Check if a decimal number is at the current program pointer. (does not advance the pointer)

Parameters

ctx

context

Returns

true if pointer points at a decimal number

tokenizer_error_print()

void tokenizer_error_print	(	struct ubasic_ctx *	ctx,
		const char *	error
	)

display an error to the terminal and end the program

Note

If the program is running an EVAL, the error is printed but the program is not ended, instead ERROR$ and ERROR are set.

Parameters

ctx	context
error	error message

tokenizer_finished()

int tokenizer_finished

(

struct ubasic_ctx *

ctx

)

Returns true if the program is finished (does not advance the pointer)

Parameters

ctx

context

Returns

int true if the program has finished

tokenizer_fnum()

void tokenizer_fnum	(	struct ubasic_ctx *	ctx,
		int	token,
		double *	f
	)

Get real number as next token (do not advance the pointer)

Parameters

ctx	context
token	token (NUMBER)
f	number read from program

tokenizer_init()

* brief Initialise tokenizer* * param program program text* param ctx context* void tokenizer_init	(	const char *	program,
		struct ubasic_ctx *	ctx
	)

tokenizer_next()

void tokenizer_next

(

struct ubasic_ctx *

ctx

)

advance to next token

Parameters

ctx

context

tokenizer_num()

int64_t tokenizer_num	(	struct ubasic_ctx *	ctx,
		int	token
	)

Get integer number as next token (do not advance the pointer)

Parameters

ctx	context
token	token (NUMBER or HEXNUMBER)

Returns

int64_t number read from program

tokenizer_string()

bool tokenizer_string	(	char *	dest,
		int	len,
		struct ubasic_ctx *	ctx
	)

Get a string constant as the next token (do not advance the pointer)

Parameters

dest	destination string buffer
len	length of destination buffer
ctx	context

Returns

true if succesfully found a string constant

tokenizer_token()

int tokenizer_token

(

struct ubasic_ctx *

ctx

)

peek to next token

Parameters

ctx

context

Returns

int token

tokenizer_variable_name()

const char* tokenizer_variable_name

(

struct ubasic_ctx *

ctx

)

Get a variable name as next token (do not advance the pointer)

Parameters

ctx

context

Returns

const char* variable name