bshell: first-pass implementation of a syntax lexer/parser

bshell/parse/lex.h

@@ -0,0 +1,82 @@
+#ifndef LEX_H_
+#define LEX_H_
+
+#include "../status.h"
+
+#include <fx/queue.h>
+#include <fx/string.h>
+#include <fx/stringstream.h>
+
+struct lex_token;
+struct line_source;
+
+enum lex_flags {
+	LEX_PRINT_TOKENS = 0x01u,
+
+	/* these flags are for lex_ctx_peek and lex_ctx_claim */
+	LEX_ENABLE_KEYWORD = 0x0100u,
+	LEX_ENABLE_INT = 0x0200u,
+	LEX_ENABLE_SYMBOL = 0x0400u,
+};
+
+enum lex_token_flags {
+	LEX_TOKEN_ENABLE_IN_STRING = 0x01u,
+	LEX_TOKEN_ENABLE_IN_WORD = 0x02u,
+};
+
+struct lex_token_def {
+	int id;
+	const char *name;
+	uint64_t name_hash;
+	enum lex_token_flags flags;
+};
+
+struct lex_symbol_node {
+	char s_char;
+	struct lex_token_def *s_def;
+
+	fx_queue_entry s_entry;
+	fx_queue s_children;
+};
+
+enum lex_state_type {
+	LEX_STATE_NORMAL = 0,
+	LEX_STATE_WORD,
+	LEX_STATE_STRING,
+	LEX_STATE_INTERPOLATION,
+};
+
+struct lex_state {
+	enum lex_state_type s_type;
+	unsigned int s_paren_depth;
+	fx_queue_entry s_entry;
+};
+
+struct lex_ctx {
+	enum lex_flags lex_flags;
+	fx_queue lex_tokens;
+	struct line_source *lex_src;
+	fx_stringstream *lex_buf;
+	fx_string *lex_tmp;
+	fx_wchar lex_ch;
+	struct lex_token *lex_alt;
+	fx_queue lex_state;
+	struct lex_symbol_node *lex_sym_tree;
+	enum bshell_status lex_status;
+};
+
+extern enum bshell_status lex_ctx_init(
+	struct lex_ctx *ctx,
+	enum lex_flags flags,
+	struct line_source *src);
+extern enum bshell_status lex_ctx_cleanup(struct lex_ctx *ctx);
+
+extern struct lex_token *lex_ctx_peek(
+	struct lex_ctx *ctx,
+	enum lex_flags flags);
+extern struct lex_token *lex_ctx_claim(
+	struct lex_ctx *ctx,
+	enum lex_flags flags);
+extern void lex_ctx_discard(struct lex_ctx *ctx, enum lex_flags flags);
+
+#endif

bshell/parse/parse.c

@@ -0,0 +1,30 @@
+#include "parse.h"
+
+#include "../ast/ast.h"
+#include "lex.h"
+#include "syntax.h"
+#include "token.h"
+
+#include <stdio.h>
+#include <string.h>
+
+enum bshell_status parse_ctx_init(struct parse_ctx *ctx, struct lex_ctx *src)
+{
+	memset(ctx, 0x0, sizeof *ctx);
+
+	ctx->p_src = src;
+
+	return BSHELL_SUCCESS;
+}
+
+void parse_ctx_cleanup(struct parse_ctx *ctx)
+{
+}
+
+struct ast_node *parse_ctx_read_node(struct parse_ctx *ctx)
+{
+	struct ast_node *result = NULL;
+	bool ok = parse_expr(ctx, &result);
+
+	return ok ? result : NULL;
+}

bshell/parse/parse.h

@@ -0,0 +1,21 @@
+#ifndef PARSE_H_
+#define PARSE_H_
+
+#include "../status.h"
+
+struct lex_ctx;
+struct ast_node;
+
+struct parse_ctx {
+	struct lex_ctx *p_src;
+	enum bshell_status p_status;
+};
+
+extern enum bshell_status parse_ctx_init(
+	struct parse_ctx *ctx,
+	struct lex_ctx *src);
+extern void parse_ctx_cleanup(struct parse_ctx *ctx);
+
+extern struct ast_node *parse_ctx_read_node(struct parse_ctx *ctx);
+
+#endif

bshell/parse/syntax.h

@@ -0,0 +1,53 @@
+#ifndef PARSE_SYNTAX_H_
+#define PARSE_SYNTAX_H_
+
+#include "../ast/ast.h"
+#include "lex.h"
+#include "parse.h"
+#include "token.h"
+
+#include <stdbool.h>
+
+enum parse_operand_flags {
+	OPERAND_BASIC = 0x01u,
+};
+
+extern struct lex_token *peek_token(
+	struct parse_ctx *ctx,
+	enum lex_flags flags);
+extern enum token_type peek_token_type(
+	struct parse_ctx *ctx,
+	enum lex_flags flags);
+extern enum token_keyword peek_unknown_keyword(struct parse_ctx *ctx);
+extern enum token_symbol peek_unknown_symbol(struct parse_ctx *ctx);
+extern bool peek_int(struct parse_ctx *ctx);
+
+extern struct lex_token *claim_token(
+	struct parse_ctx *ctx,
+	enum lex_flags flags);
+extern void discard_token(struct parse_ctx *ctx);
+
+extern bool peek_linefeed(struct parse_ctx *ctx);
+extern bool peek_symbol(struct parse_ctx *ctx, enum token_symbol sym);
+
+extern bool parse_linefeed(struct parse_ctx *ctx);
+extern bool parse_symbol(struct parse_ctx *ctx, enum token_symbol sym);
+extern bool parse_keyword(struct parse_ctx *ctx, enum token_keyword kw);
+extern bool parse_int(struct parse_ctx *ctx, long long *out);
+extern bool parse_flag(struct parse_ctx *ctx, struct lex_token **out);
+
+extern bool peek_arith_expr(struct parse_ctx *ctx);
+extern bool parse_arith_expr(struct parse_ctx *ctx, struct ast_node **out);
+extern bool parse_operand(
+	struct parse_ctx *ctx,
+	enum parse_operand_flags flags,
+	struct ast_node **out);
+
+extern bool parse_expr(struct parse_ctx *ctx, struct ast_node **out);
+
+extern bool peek_command(struct parse_ctx *ctx);
+extern bool parse_command(struct parse_ctx *ctx, struct ast_node **out);
+extern bool parse_cmdcall(struct parse_ctx *ctx, struct ast_node **out);
+extern bool parse_redirect(struct parse_ctx *ctx, struct ast_node **out);
+
+#endif

bshell/parse/syntax/arith.c

@@ -0,0 +1,11 @@
+#include "../syntax.h"
+
+bool peek_arith_expr(struct parse_ctx *ctx)
+{
+	return false;
+}
+
+bool parse_arith_expr(struct parse_ctx *ctx, struct ast_node **out)
+{
+	return false;
+}

bshell/parse/syntax/command.c

@@ -0,0 +1,385 @@
+#include "../syntax.h"
+
+#include <fx/encoding.h>
+
+static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out)
+{
+	struct lex_token *tok = peek_token(ctx, 0);
+	if (!tok) {
+		return false;
+	}
+
+	struct ast_node *arg = NULL;
+
+	switch (tok->tok_type) {
+	case TOK_WORD: {
+		struct word_ast_node *n
+			= (struct word_ast_node *)ast_node_create(AST_WORD);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_value = claim_token(ctx, 0);
+		*out = (struct ast_node *)n;
+		return true;
+	}
+
+	case TOK_FLAG: {
+		struct word_ast_node *n
+			= (struct word_ast_node *)ast_node_create(AST_WORD);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_value = claim_token(ctx, 0);
+		*out = (struct ast_node *)n;
+		return true;
+	}
+
+	case TOK_VAR: {
+		struct var_ast_node *n
+			= (struct var_ast_node *)ast_node_create(AST_VAR);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_ident = claim_token(ctx, 0);
+		*out = (struct ast_node *)n;
+		return true;
+	}
+
+	case TOK_VAR_SPLAT: {
+		struct var_splat_ast_node *n
+			= (struct var_splat_ast_node *)ast_node_create(
+				AST_VAR_SPLAT);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_ident = claim_token(ctx, 0);
+		*out = (struct ast_node *)n;
+		return true;
+	}
+
+	case TOK_STRING: {
+		struct string_ast_node *n
+			= (struct string_ast_node *)ast_node_create(AST_STRING);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_value = claim_token(ctx, 0);
+		*out = (struct ast_node *)n;
+		return true;
+	}
+
+	default:
+		return false;
+	}
+
+	return true;
+}
+
+static bool parse_redirect_to_fd(
+	struct parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	struct ast_node **out)
+{
+	struct redirection_ast_node *redirect
+		= (struct redirection_ast_node *)ast_node_create(
+			AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+
+	if (!parse_symbol(ctx, SYM_AMPERSAND)) {
+		ast_node_destroy((struct ast_node *)redirect);
+		return false;
+	}
+
+	long long out_fd = 0;
+	if (!parse_int(ctx, &out_fd)) {
+		ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+		ast_node_destroy((struct ast_node *)redirect);
+		return false;
+	}
+
+	redirect->n_out_is_fd = true;
+	redirect->n_out_is_expr = false;
+	redirect->n_out = (unsigned int)out_fd;
+	*out = (struct ast_node *)redirect;
+
+	return true;
+}
+
+static bool parse_redirect_to_file_squashed(
+	struct parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	const char *str,
+	struct ast_node **out)
+{
+	struct lex_token *tok = peek_token(ctx, 0);
+	if (*str == '\0') {
+		return false;
+	}
+
+	struct redirection_ast_node *redirect
+		= (struct redirection_ast_node *)ast_node_create(
+			AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+	redirect->n_out_is_fd = false;
+	redirect->n_out_is_expr = false;
+	redirect->n_out_path = str;
+
+	redirect->n_out_tok = claim_token(ctx, 0);
+
+	*out = (struct ast_node *)redirect;
+	return true;
+}
+
+static bool parse_redirect_to_file_separate(
+	struct parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	struct ast_node **out)
+{
+	struct ast_node *out_path = NULL;
+	if (!parse_cmdcall_arg(ctx, &out_path)) {
+		ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+		return false;
+	}
+
+	struct redirection_ast_node *redirect
+		= (struct redirection_ast_node *)ast_node_create(
+			AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+	redirect->n_out_is_fd = false;
+	redirect->n_out_is_expr = true;
+	redirect->n_out_path_expr = out_path;
+
+	*out = (struct ast_node *)redirect;
+	return true;
+}
+
+bool parse_redirect(struct parse_ctx *ctx, struct ast_node **out)
+{
+	struct lex_token *tok = peek_token(ctx, 0);
+	if (!tok || tok->tok_type != TOK_WORD) {
+		return false;
+	}
+
+	unsigned int in_fd = 1;
+	const char *str = tok->tok_str;
+	bool append = false;
+
+	if (fx_wchar_is_number(*str)) {
+		in_fd = *str - '0';
+		str++;
+	}
+
+	if (*str != '>') {
+		return false;
+	}
+
+	str++;
+	if (*str == '>') {
+		append = true;
+		str++;
+	}
+
+	if (*str != '\0') {
+		return parse_redirect_to_file_squashed(
+			ctx,
+			in_fd,
+			append,
+			str,
+			out);
+	}
+
+	discard_token(ctx);
+
+	if (parse_redirect_to_fd(ctx, in_fd, append, out)) {
+		return true;
+	}
+
+	if (parse_redirect_to_file_separate(ctx, in_fd, append, out)) {
+		return true;
+	}
+
+	return false;
+}
+
+static bool peek_cmdcall_item(struct parse_ctx *ctx, bool unrestricted)
+{
+	/* each token type falls into one of three categories:
+	 * - cmdcall item: the token can be used as part of a command call. the
+	 *   token indicates the start of a command call.
+	 * - NOT a cmdcall item: the token cannot be used as part of a command
+	 *   call, usually because it as a cmdcall operator like | or &.
+	 *   encountering one of these tokens ends the cmdcall currently being
+	 *   parsed.
+	 * - RESTRICTED cmdcall item: the token can be used as part of a
+	 *   command, but will not be considered the start of a cmdcall. to run
+	 *   a command with this token as its name, the call operator must be
+	 *   used.
+	 */
+	switch (peek_token_type(ctx, LEX_ENABLE_INT | LEX_ENABLE_KEYWORD)) {
+	case TOK_KEYWORD:
+	case TOK_INT:
+	case TOK_DOUBLE:
+	case TOK_VAR:
+	case TOK_VAR_SPLAT:
+	case TOK_STRING:
+	case TOK_STR_START:
+		return unrestricted;
+	case TOK_SYMBOL:
+		switch (peek_unknown_symbol(ctx)) {
+		case SYM_PLUS:
+		case SYM_HYPHEN:
+			return unrestricted;
+		case SYM_PIPE:
+		case SYM_AMPERSAND:
+		case SYM_SEMICOLON:
+			return false;
+		default:
+			return true;
+		}
+	case TOK_NONE:
+	case TOK_LINEFEED:
+		return false;
+	default:
+		return true;
+	}
+}
+
+bool parse_cmdcall(struct parse_ctx *ctx, struct ast_node **out)
+{
+	struct cmdcall_ast_node *node
+		= (struct cmdcall_ast_node *)ast_node_create(AST_CMDCALL);
+	if (!node) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	struct ast_node *child = NULL;
+	bool unrestricted = false;
+	bool ok = true;
+	bool stop = false;
+
+	if (parse_symbol(ctx, SYM_AMPERSAND)) {
+		unrestricted = true;
+	}
+
+	if (!peek_cmdcall_item(ctx, unrestricted)) {
+		return false;
+	}
+
+	struct lex_token *tok = peek_token(ctx, 0);
+	if (!tok) {
+		return false;
+	}
+
+	if (!parse_cmdcall_arg(ctx, &child)) {
+		return false;
+	}
+
+	fx_queue_push_back(&node->n_args, &child->n_entry);
+
+	while (ok && !stop) {
+		if (!peek_cmdcall_item(ctx, true)) {
+			break;
+		}
+
+		struct lex_token *tok = peek_token(ctx, 0);
+		if (!tok) {
+			break;
+		}
+
+		if (parse_redirect(ctx, &child)) {
+			fx_queue_push_back(&node->n_redirect, &child->n_entry);
+		} else if (parse_cmdcall_arg(ctx, &child)) {
+			fx_queue_push_back(&node->n_args, &child->n_entry);
+		} else {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+			ok = false;
+			break;
+		}
+	}
+
+	if (!ok) {
+		ast_node_destroy((struct ast_node *)node);
+		node = NULL;
+	}
+
+	*out = (struct ast_node *)node;
+	return ok;
+}
+
+bool peek_command(struct parse_ctx *ctx)
+{
+	if (peek_symbol(ctx, SYM_AMPERSAND)) {
+		return true;
+	}
+
+	return peek_cmdcall_item(ctx, false);
+}
+
+bool parse_command(struct parse_ctx *ctx, struct ast_node **out)
+{
+	struct ast_node *cmdcall = NULL;
+	if (!parse_cmdcall(ctx, &cmdcall)) {
+		return false;
+	}
+
+	struct pipeline_ast_node *pipeline = NULL;
+
+	while (1) {
+		if (parse_symbol(ctx, SYM_SEMICOLON) || parse_linefeed(ctx)) {
+			break;
+		}
+
+		if (!parse_symbol(ctx, SYM_PIPE)) {
+			break;
+		}
+
+		if (!pipeline) {
+			pipeline = (struct pipeline_ast_node *)ast_node_create(
+				AST_PIPELINE);
+			if (!pipeline) {
+				ctx->p_status = BSHELL_ERR_NO_MEMORY;
+				ast_node_destroy(cmdcall);
+				return false;
+			}
+
+			fx_queue_push_back(
+				&pipeline->n_stages,
+				&cmdcall->n_entry);
+		}
+
+		if (!parse_cmdcall(ctx, &cmdcall)) {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+			return false;
+		}
+
+		fx_queue_push_back(&pipeline->n_stages, &cmdcall->n_entry);
+	}
+
+	if (pipeline) {
+		*out = (struct ast_node *)pipeline;
+	} else {
+		*out = cmdcall;
+	}
+
+	return true;
+}

bshell/parse/syntax/expr.c

@@ -0,0 +1,15 @@
+#include "../syntax.h"
+
+bool parse_expr(struct parse_ctx *ctx, struct ast_node **out)
+{
+	bool ok = false;
+	if (peek_command(ctx)) {
+		ok = parse_command(ctx, out);
+	}
+
+	if (!ok && peek_arith_expr(ctx)) {
+		ok = parse_arith_expr(ctx, out);
+	}
+
+	return ok;
+}

bshell/parse/syntax/generic.c

@@ -0,0 +1,135 @@
+#include "../lex.h"
+#include "../parse.h"
+#include "../syntax.h"
+#include "../token.h"
+
+#define DEFAULT_LEX_FLAGS                                                      \
+	(LEX_ENABLE_INT | LEX_ENABLE_KEYWORD | LEX_ENABLE_SYMBOL)
+
+struct lex_token *claim_token(struct parse_ctx *ctx, enum lex_flags flags)
+{
+	return lex_ctx_claim(ctx->p_src, flags);
+}
+
+void discard_token(struct parse_ctx *ctx)
+{
+	return lex_ctx_discard(ctx->p_src, DEFAULT_LEX_FLAGS);
+}
+
+struct lex_token *peek_token(struct parse_ctx *ctx, enum lex_flags flags)
+{
+	return lex_ctx_peek(ctx->p_src, flags);
+}
+
+enum token_type peek_token_type(struct parse_ctx *ctx, enum lex_flags flags)
+{
+	struct lex_token *tok = peek_token(ctx, flags);
+	return tok ? tok->tok_type : TOK_NONE;
+}
+
+enum token_symbol peek_unknown_symbol(struct parse_ctx *ctx)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	return (tok && tok->tok_type == TOK_SYMBOL) ? tok->tok_symbol
+	                                            : SYM_NONE;
+}
+
+enum token_keyword peek_unknown_keyword(struct parse_ctx *ctx)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	return (tok && tok->tok_type == TOK_KEYWORD) ? tok->tok_keyword
+	                                             : KW_NONE;
+}
+
+bool peek_linefeed(struct parse_ctx *ctx)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (tok && tok->tok_type == TOK_LINEFEED) {
+		return true;
+	}
+
+	return false;
+}
+
+bool peek_symbol(struct parse_ctx *ctx, enum token_symbol sym)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != TOK_SYMBOL) {
+		return false;
+	}
+
+	if (tok->tok_symbol != sym) {
+		return false;
+	}
+
+	return true;
+}
+
+bool parse_linefeed(struct parse_ctx *ctx)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (tok && tok->tok_type == TOK_LINEFEED) {
+		discard_token(ctx);
+		return true;
+	}
+
+	return false;
+}
+
+bool parse_symbol(struct parse_ctx *ctx, enum token_symbol sym)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != TOK_SYMBOL) {
+		return false;
+	}
+
+	if (tok->tok_symbol != sym) {
+		return false;
+	}
+
+	discard_token(ctx);
+	return true;
+}
+
+bool parse_keyword(struct parse_ctx *ctx, enum token_keyword kw)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != TOK_KEYWORD) {
+		return false;
+	}
+
+	if (tok->tok_keyword != kw) {
+		return false;
+	}
+
+	discard_token(ctx);
+	return true;
+}
+
+bool parse_int(struct parse_ctx *ctx, long long *out)
+{
+	struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != TOK_INT) {
+		return false;
+	}
+
+	*out = tok->tok_int;
+	discard_token(ctx);
+	return true;
+}

bshell/parse/token.c

@@ -0,0 +1,148 @@
+#include "token.h"
+
+#include <fx/string.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct lex_token *lex_token_create(enum token_type type)
+{
+	struct lex_token *out = malloc(sizeof *out);
+	if (!out) {
+		return NULL;
+	}
+
+	memset(out, 0x0, sizeof *out);
+
+	out->tok_type = type;
+
+	return out;
+}
+
+struct lex_token *lex_token_create_with_string(
+	enum token_type type,
+	const char *s)
+{
+	struct lex_token *tok = lex_token_create(type);
+	if (!tok) {
+		return NULL;
+	}
+
+	tok->tok_str = fx_strdup(s);
+	if (!tok->tok_str) {
+		free(tok);
+		return NULL;
+	}
+
+	return tok;
+}
+
+void lex_token_destroy(struct lex_token *tok)
+{
+	switch (tok->tok_type) {
+	case TOK_WORD:
+	case TOK_FLAG:
+	case TOK_STRING:
+		if (tok->tok_str) {
+			free(tok->tok_str);
+		}
+
+		break;
+	default:
+		break;
+	}
+
+	free(tok);
+}
+
+struct lex_token *lex_token_change_type(
+	struct lex_token *tok,
+	enum token_type new_type)
+{
+	switch (tok->tok_type) {
+	case TOK_WORD:
+	case TOK_FLAG:
+	case TOK_STRING:
+		if (tok->tok_str) {
+			free(tok->tok_str);
+			tok->tok_str = NULL;
+		}
+		break;
+	default:
+		break;
+	}
+
+	tok->tok_type = new_type;
+	return tok;
+}
+
+#define ENUM_STR(x)                                                            \
+	case x:                                                                \
+		return #x
+
+const char *token_type_to_string(enum token_type type)
+{
+	switch (type) {
+		ENUM_STR(TOK_NONE);
+		ENUM_STR(TOK_KEYWORD);
+		ENUM_STR(TOK_SYMBOL);
+		ENUM_STR(TOK_INT);
+		ENUM_STR(TOK_DOUBLE);
+		ENUM_STR(TOK_WORD);
+		ENUM_STR(TOK_VAR);
+		ENUM_STR(TOK_VAR_SPLAT);
+		ENUM_STR(TOK_FLAG);
+		ENUM_STR(TOK_STRING);
+		ENUM_STR(TOK_STR_START);
+		ENUM_STR(TOK_STR_END);
+		ENUM_STR(TOK_LINEFEED);
+	default:
+		return "<unknown>";
+	}
+}
+
+const char *token_keyword_to_string(enum token_keyword keyword)
+{
+	switch (keyword) {
+		ENUM_STR(KW_NONE);
+		ENUM_STR(KW_FUNC);
+	default:
+		return "<unknown>";
+	}
+}
+
+const char *token_symbol_to_string(enum token_symbol sym)
+{
+	switch (sym) {
+		ENUM_STR(SYM_NONE);
+		ENUM_STR(SYM_PLUS);
+		ENUM_STR(SYM_HYPHEN);
+		ENUM_STR(SYM_FORWARD_SLASH);
+		ENUM_STR(SYM_ASTERISK);
+		ENUM_STR(SYM_AMPERSAND);
+		ENUM_STR(SYM_PERCENT);
+		ENUM_STR(SYM_SQUOTE);
+		ENUM_STR(SYM_DQUOTE);
+		ENUM_STR(SYM_HASH);
+		ENUM_STR(SYM_SEMICOLON);
+		ENUM_STR(SYM_COMMA);
+		ENUM_STR(SYM_DOLLAR);
+		ENUM_STR(SYM_DOLLAR_LEFT_PAREN);
+		ENUM_STR(SYM_PIPE);
+		ENUM_STR(SYM_AT);
+		ENUM_STR(SYM_AT_LEFT_BRACE);
+		ENUM_STR(SYM_LEFT_BRACE);
+		ENUM_STR(SYM_RIGHT_BRACE);
+		ENUM_STR(SYM_LEFT_BRACKET);
+		ENUM_STR(SYM_RIGHT_BRACKET);
+		ENUM_STR(SYM_LEFT_PAREN);
+		ENUM_STR(SYM_RIGHT_PAREN);
+		ENUM_STR(SYM_EQUAL);
+		ENUM_STR(SYM_PLUS_EQUAL);
+		ENUM_STR(SYM_HYPHEN_EQUAL);
+		ENUM_STR(SYM_FORWARD_SLASH_EQUAL);
+		ENUM_STR(SYM_ASTERISK_EQUAL);
+		ENUM_STR(SYM_PERCENT_EQUAL);
+	default:
+		return "<unknown>";
+	}
+}

bshell/parse/token.h

@@ -0,0 +1,131 @@
+#ifndef IVY_LANG_LEX_H_
+#define IVY_LANG_LEX_H_
+
+#include <fx/queue.h>
+#include <stdbool.h>
+
+struct char_cell {
+	unsigned long c_row, c_col;
+};
+
+enum token_type {
+	TOK_NONE = 0,
+	__TOK_INDEX_BASE = 100,
+	TOK_KEYWORD,
+	TOK_SYMBOL,
+	TOK_INT,
+	TOK_DOUBLE,
+	TOK_WORD,
+	TOK_FLAG,
+	TOK_VAR,
+	TOK_VAR_SPLAT,
+	TOK_STRING,
+	TOK_STR_START,
+	TOK_STR_END,
+	TOK_LINEFEED,
+	__TOK_INDEX_LIMIT,
+};
+
+enum token_keyword {
+	KW_NONE = 0,
+	__KW_INDEX_BASE = 200,
+	KW_FUNC,
+	__KW_INDEX_LIMIT,
+};
+
+enum token_symbol {
+	SYM_NONE = 0,
+	__SYM_INDEX_BASE = 300,
+	SYM_PLUS,
+	SYM_HYPHEN,
+	SYM_FORWARD_SLASH,
+	SYM_ASTERISK,
+	SYM_AMPERSAND,
+	SYM_PERCENT,
+	SYM_SQUOTE,
+	SYM_DQUOTE,
+	SYM_HASH,
+	SYM_SEMICOLON,
+	SYM_COMMA,
+	SYM_DOLLAR,
+	SYM_DOLLAR_LEFT_PAREN,
+	SYM_DOLLAR_LEFT_BRACE,
+	SYM_PIPE,
+	SYM_AT,
+	SYM_AT_LEFT_BRACE,
+	SYM_LEFT_BRACE,
+	SYM_RIGHT_BRACE,
+	SYM_LEFT_BRACKET,
+	SYM_RIGHT_BRACKET,
+	SYM_LEFT_PAREN,
+	SYM_RIGHT_PAREN,
+	SYM_EQUAL,
+	SYM_PLUS_EQUAL,
+	SYM_HYPHEN_EQUAL,
+	SYM_ASTERISK_EQUAL,
+	SYM_FORWARD_SLASH_EQUAL,
+	SYM_PERCENT_EQUAL,
+	__SYM_INDEX_LIMIT,
+};
+
+struct lex_token {
+	enum token_type tok_type;
+
+	struct char_cell tok_start, tok_end;
+
+	fx_queue_entry tok_entry;
+
+	union {
+		enum token_keyword tok_keyword;
+		enum token_symbol tok_symbol;
+		long long tok_int;
+		double tok_double;
+		char *tok_str;
+	};
+};
+
+extern struct lex_token *lex_token_create(enum token_type type);
+extern struct lex_token *lex_token_create_with_string(
+	enum token_type type,
+	const char *s);
+extern void lex_token_destroy(struct lex_token *tok);
+
+extern struct lex_token *lex_token_change_type(
+	struct lex_token *tok,
+	enum token_type new_type);
+
+static inline bool lex_token_is_symbol(
+	struct lex_token *tok,
+	enum token_symbol sym)
+{
+	return (tok->tok_type == TOK_SYMBOL && tok->tok_symbol == sym);
+}
+static inline bool lex_token_is_keyword(
+	struct lex_token *tok,
+	enum token_keyword kw)
+{
+	return (tok->tok_type == TOK_KEYWORD && tok->tok_keyword == kw);
+}
+static inline bool lex_token_type_has_string_value(enum token_type type)
+{
+	switch (type) {
+	case TOK_WORD:
+	case TOK_STRING:
+	case TOK_FLAG:
+	case TOK_VAR:
+	case TOK_VAR_SPLAT:
+		return true;
+	default:
+		return false;
+	}
+}
+static inline bool lex_token_has_string_value(const struct lex_token *tok)
+{
+	return lex_token_type_has_string_value(tok->tok_type);
+}
+
+extern const char *token_type_to_string(enum token_type type);
+extern const char *token_keyword_to_string(enum token_keyword keyword);
+extern const char *token_symbol_to_string(enum token_symbol sym);
+
+#endif