\hypertarget{tokenizer_8h}{}\doxysection{libgymbo/tokenizer.h File Reference}
\label{tokenizer_8h}\index{libgymbo/tokenizer.h@{libgymbo/tokenizer.h}}


Implementation of tokenizer.  


{\ttfamily \#include $<$ctype.\+h$>$}\newline
{\ttfamily \#include $<$stdarg.\+h$>$}\newline
{\ttfamily \#include $<$stdbool.\+h$>$}\newline
{\ttfamily \#include $<$stdio.\+h$>$}\newline
{\ttfamily \#include $<$stdlib.\+h$>$}\newline
{\ttfamily \#include $<$string.\+h$>$}\newline
{\ttfamily \#include $<$cstdlib$>$}\newline
{\ttfamily \#include $<$cstring$>$}\newline
{\ttfamily \#include $<$string$>$}\newline
{\ttfamily \#include $<$unordered\+\_\+map$>$}\newline
Include dependency graph for tokenizer.\+h\+:\nopagebreak
\begin{figure}[H]
\begin{center}
\leavevmode
\includegraphics[width=350pt]{tokenizer_8h__incl}
\end{center}
\end{figure}
This graph shows which files directly or indirectly include this file\+:\nopagebreak
\begin{figure}[H]
\begin{center}
\leavevmode
\includegraphics[width=199pt]{tokenizer_8h__dep__incl}
\end{center}
\end{figure}
\doxysubsection*{Classes}
\begin{DoxyCompactItemize}
\item 
struct \mbox{\hyperlink{structgymbo_1_1Token}{gymbo\+::\+Token}}
\begin{DoxyCompactList}\small\item\em Structure representing a token. \end{DoxyCompactList}\end{DoxyCompactItemize}
\doxysubsection*{Namespaces}
\begin{DoxyCompactItemize}
\item 
 \mbox{\hyperlink{namespacegymbo}{gymbo}}
\end{DoxyCompactItemize}
\doxysubsection*{Enumerations}
\begin{DoxyCompactItemize}
\item 
enum \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646}{gymbo\+::\+Token\+Kind}} \{ \newline
\mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646ac7e847d9a71b84b9d707022ede462c26}{gymbo\+::\+TOKEN\+\_\+\+RESERVED}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646a8170c440f4a465a9d4a8d69c6934b7b2}{gymbo\+::\+TOKEN\+\_\+\+RETURN}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646ac53e34e4445f94b1c47dd34d9676b209}{gymbo\+::\+TOKEN\+\_\+\+IF}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646a352b101ce5bfbf86b28d9a18ef217a61}{gymbo\+::\+TOKEN\+\_\+\+ELSE}}
, \newline
\mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646ad616d279f986a51a6774fa10d77aa415}{gymbo\+::\+TOKEN\+\_\+\+FOR}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646ac453ec427086ba98563ee6fb8582f042}{gymbo\+::\+TOKEN\+\_\+\+IDENT}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646a041d42319df0f64d0688bd8398a5f6f2}{gymbo\+::\+TOKEN\+\_\+\+NUM}}
, \mbox{\hyperlink{namespacegymbo_a886e58c66782cda4ff74ad10b5a6b646a87451f2ad23351cb715502a4890397bf}{gymbo\+::\+TOKEN\+\_\+\+EOF}}
 \}
\begin{DoxyCompactList}\small\item\em Enumeration representing different token kinds. \end{DoxyCompactList}\end{DoxyCompactItemize}
\doxysubsection*{Functions}
\begin{DoxyCompactItemize}
\item 
bool \mbox{\hyperlink{namespacegymbo_aab49b70724bec7c8b7ee2a6a2fd31771}{gymbo\+::is\+\_\+alpha}} (char c)
\begin{DoxyCompactList}\small\item\em Checks if a character is an alphabetical character or underscore. \end{DoxyCompactList}\item 
bool \mbox{\hyperlink{namespacegymbo_a9f75e72416f1f4de36d005b6935c0136}{gymbo\+::is\+\_\+alnum}} (char c)
\begin{DoxyCompactList}\small\item\em Checks if a character is an alphanumeric character. \end{DoxyCompactList}\item 
void \mbox{\hyperlink{namespacegymbo_afd45221e5385a56b69288384e8e393aa}{gymbo\+::error}} (char $\ast$fmt,...)
\begin{DoxyCompactList}\small\item\em Reports an error and exits the program. \end{DoxyCompactList}\item 
void \mbox{\hyperlink{namespacegymbo_a74bef918aba1493fe3b0f94b0c34a92e}{gymbo\+::error\+\_\+at}} (char $\ast$user\+\_\+input, char $\ast$loc, char $\ast$fmt,...)
\begin{DoxyCompactList}\small\item\em Reports an error location and exits the program. \end{DoxyCompactList}\item 
bool \mbox{\hyperlink{namespacegymbo_a351aaa8f55935e9d65f1c33b143ffdc1}{gymbo\+::consume}} (Token $\ast$\&token, char $\ast$op)
\begin{DoxyCompactList}\small\item\em Consumes the current token if it matches {\ttfamily op}. \end{DoxyCompactList}\item 
bool \mbox{\hyperlink{namespacegymbo_a05bc470a265b7b40aaddd5a073c5a1d7}{gymbo\+::consume\+\_\+tok}} (Token $\ast$\&token, Token\+Kind tok)
\begin{DoxyCompactList}\small\item\em Consumes the current token if it matches {\ttfamily tok}. \end{DoxyCompactList}\item 
Token $\ast$ \mbox{\hyperlink{namespacegymbo_aea8e842b4ef04a10618bf3f95b2de229}{gymbo\+::consume\+\_\+ident}} (Token $\ast$\&token)
\begin{DoxyCompactList}\small\item\em Consumes the current token if it is an identifier. \end{DoxyCompactList}\item 
void \mbox{\hyperlink{namespacegymbo_acdabc32fe6aa65db48889984321294b4}{gymbo\+::expect}} (Token $\ast$\&token, char $\ast$user\+\_\+input, char $\ast$op)
\begin{DoxyCompactList}\small\item\em Ensures that the current token matches {\ttfamily op}. \end{DoxyCompactList}\item 
float \mbox{\hyperlink{namespacegymbo_a82e491fd1648ba276b4c2873e7c8b4b1}{gymbo\+::expect\+\_\+number}} (Token $\ast$\&token, char $\ast$user\+\_\+input)
\begin{DoxyCompactList}\small\item\em Ensures that the current token is a number. \end{DoxyCompactList}\item 
bool \mbox{\hyperlink{namespacegymbo_a573f6b3a170a995133014dec63f330dc}{gymbo\+::at\+\_\+eof}} (Token $\ast$token)
\begin{DoxyCompactList}\small\item\em Checks if the current token is at the end of the program. \end{DoxyCompactList}\item 
Token $\ast$ \mbox{\hyperlink{namespacegymbo_a09b17fb4e9dfe3e5c6e03ad57bcfecf4}{gymbo\+::new\+\_\+token}} (Token\+Kind kind, Token $\ast$cur, char $\ast$str, int len)
\begin{DoxyCompactList}\small\item\em Creates a new token and adds it as the next token of {\ttfamily cur}. \end{DoxyCompactList}\item 
bool \mbox{\hyperlink{namespacegymbo_a086916a92edc60e687a056603995182e}{gymbo\+::startswith}} (char $\ast$p, char $\ast$q)
\begin{DoxyCompactList}\small\item\em Checks if the string {\ttfamily p} starts with the string {\ttfamily q}. \end{DoxyCompactList}\item 
Token $\ast$ \mbox{\hyperlink{namespacegymbo_a89c8cb7e3b04d3d02f0cba69ab593981}{gymbo\+::tokenize}} (char $\ast$user\+\_\+input, std\+::unordered\+\_\+map$<$ std\+::string, int $>$ \&var\+\_\+counter)
\begin{DoxyCompactList}\small\item\em Tokenizes a given string and returns a linked list of tokens. \end{DoxyCompactList}\end{DoxyCompactItemize}


\doxysubsection{Detailed Description}
Implementation of tokenizer. 

\begin{DoxyAuthor}{Author}
Hideaki Takahashi 
\end{DoxyAuthor}
