bshell: re-organise build into three separate components

bshell: the front-end binary
bshell.runtime: contains the parser, compiler, and classes needed to run bshell scripts
bshell.core: contains the builtin commandlets and aliases

bshell.runtime/parse/lex/arithmetic.c

@@ -0,0 +1,232 @@
+#include "lex-internal.h"
+
+static enum bshell_status arithmetic_hyphen(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	if (!fx_wchar_is_alnum(c)) {
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+		handle_lex_state_transition(ctx, BSHELL_SYM_HYPHEN);
+		return BSHELL_SUCCESS;
+	}
+
+	struct bshell_lex_token *tok = NULL;
+	enum bshell_status status = read_word(
+		ctx,
+		READ_NO_SET_TOKEN_START | READ_APPEND_HYPHEN,
+		&tok);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	unsigned int token_type = BSHELL_TOK_WORD;
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
+
+		/* because of APPEND_HYPHEN (which is needed to ensure operator
+		 * tokens are detected properly), the resulting number will be
+		 * negative.
+		 * this token will be preceded by a HYPHEN token, so the number
+		 * must be positive */
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+	} else if (convert_word_to_operator(ctx, tok)) {
+		token_type = tok->tok_operator;
+	}
+
+	enqueue_token(ctx, tok);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status arithmetic_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	handle_lex_state_transition(ctx, sym->id);
+
+	struct bshell_lex_token *tok = NULL;
+	switch (sym->id) {
+	case BSHELL_SYM_DQUOTE:
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_SQUOTE:
+		status = read_literal_string(ctx, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+		enqueue_token(ctx, tok);
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_HYPHEN:
+		return arithmetic_hyphen(ctx);
+	case BSHELL_SYM_HASH:
+		return read_line_comment(ctx);
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_DOLLAR_LEFT_BRACE:
+		status = read_braced_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	push_symbol(ctx, sym->id);
+
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status arithmetic_word(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *word = NULL;
+	enum bshell_status status = read_word(ctx, 0, &word);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	unsigned int token_type = BSHELL_TOK_WORD;
+	bool kw = false, number = false;
+	if (convert_word_to_keyword(word)) {
+		token_type = word->tok_keyword;
+	} else if (convert_word_to_number(word)) {
+		token_type = word->tok_type;
+	}
+
+	handle_lex_state_transition(ctx, token_type);
+
+	enqueue_token(ctx, word);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status arithmetic_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	bool newline = false;
+
+	set_token_start(ctx);
+	while (fx_wchar_is_space(c)) {
+		if (c == '\n') {
+			newline = true;
+		}
+
+		set_token_end(ctx);
+		advance_char_noread(ctx);
+		c = peek_char_noread(ctx);
+	}
+
+	if (newline) {
+		struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_LINEFEED);
+		enqueue_token(ctx, tok);
+		handle_lex_state_transition(ctx, BSHELL_TOK_LINEFEED);
+		return BSHELL_SUCCESS;
+	}
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return arithmetic_symbol(ctx);
+	}
+
+	return arithmetic_word(ctx);
+}
+
+static const struct bshell_lex_state_link links[] = {
+	LINK_CHANGE(BSHELL_SYM_EQUAL, BSHELL_LEX_STATE_STATEMENT),
+	LINK_PUSH(BSHELL_SYM_DQUOTE, BSHELL_LEX_STATE_STRING, 0),
+	LINK_PUSH(BSHELL_SYM_DOLLAR_LEFT_PAREN, BSHELL_LEX_STATE_STATEMENT, 0),
+	LINK_POP(BSHELL_SYM_RIGHT_PAREN),
+	LINK_CHANGE(BSHELL_SYM_SEMICOLON, BSHELL_LEX_STATE_STATEMENT),
+	LINK_CHANGE(BSHELL_TOK_LINEFEED, BSHELL_LEX_STATE_STATEMENT),
+	LINK_CHANGE(BSHELL_SYM_PIPE, BSHELL_LEX_STATE_STATEMENT),
+	LINK_PUSH(BSHELL_SYM_AT_LEFT_BRACE, BSHELL_LEX_STATE_HASHTABLE, 0),
+	LINK_PUSH(
+		BSHELL_SYM_LEFT_PAREN,
+		BSHELL_LEX_STATE_STATEMENT,
+		STATEMENT_F_DISABLE_KEYWORDS),
+	LINK_END,
+};
+
+static const unsigned int keywords[] = {
+	BSHELL_KW_IF,
+	BSHELL_KW_ELSEIF,
+	BSHELL_KW_ELSE,
+	BSHELL_KW_NONE,
+};
+
+static const unsigned int operators[] = {
+	BSHELL_TKOP_F,        BSHELL_TKOP_BAND,        BSHELL_TKOP_BOR,      BSHELL_TKOP_BXOR,
+	BSHELL_TKOP_BNOT,     BSHELL_TKOP_SHL,         BSHELL_TKOP_SHR,      BSHELL_TKOP_EQ,
+	BSHELL_TKOP_NE,       BSHELL_TKOP_GT,          BSHELL_TKOP_LT,       BSHELL_TKOP_GE,
+	BSHELL_TKOP_LE,       BSHELL_TKOP_MATCH,       BSHELL_TKOP_NOTMATCH, BSHELL_TKOP_REPLACE,
+	BSHELL_TKOP_LIKE,     BSHELL_TKOP_NOTLIKE,     BSHELL_TKOP_IN,       BSHELL_TKOP_NOTIN,
+	BSHELL_TKOP_CONTAINS, BSHELL_TKOP_NOTCONTAINS, BSHELL_TKOP_AND,      BSHELL_TKOP_OR,
+	BSHELL_TKOP_XOR,      BSHELL_TKOP_NOT,         BSHELL_TKOP_SPLIT,    BSHELL_TKOP_JOIN,
+	BSHELL_TKOP_IS,       BSHELL_TKOP_ISNOT,       BSHELL_TKOP_AS,       BSHELL_TKOP_NONE,
+};
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_BANG,
+	BSHELL_SYM_PLUS,
+	BSHELL_SYM_HYPHEN,
+	BSHELL_SYM_FORWARD_SLASH,
+	BSHELL_SYM_ASTERISK,
+	BSHELL_SYM_AMPERSAND,
+	BSHELL_SYM_PERCENT,
+	BSHELL_SYM_SQUOTE,
+	BSHELL_SYM_DQUOTE,
+	BSHELL_SYM_HASH,
+	BSHELL_SYM_DOLLAR,
+	BSHELL_SYM_DOLLAR_LEFT_PAREN,
+	BSHELL_SYM_DOLLAR_LEFT_BRACE,
+	BSHELL_SYM_AT,
+	BSHELL_SYM_AT_LEFT_BRACE,
+	BSHELL_SYM_PIPE,
+	BSHELL_SYM_COMMA,
+	BSHELL_SYM_SEMICOLON,
+	BSHELL_SYM_LEFT_PAREN,
+	BSHELL_SYM_RIGHT_PAREN,
+	BSHELL_SYM_LEFT_BRACE,
+	BSHELL_SYM_RIGHT_BRACE,
+	BSHELL_SYM_LEFT_BRACKET,
+	BSHELL_SYM_RIGHT_BRACKET,
+	BSHELL_SYM_QUESTION_DOT,
+	BSHELL_SYM_QUESTION_LEFT_BRACKET,
+	BSHELL_SYM_EQUAL,
+	BSHELL_SYM_PLUS_EQUAL,
+	BSHELL_SYM_HYPHEN_EQUAL,
+	BSHELL_SYM_FORWARD_SLASH_EQUAL,
+	BSHELL_SYM_ASTERISK_EQUAL,
+	BSHELL_SYM_PERCENT_EQUAL,
+	BSHELL_SYM_DOT,
+	BSHELL_SYM_DOT_DOT,
+	BSHELL_SYM_COLON_COLON,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_arithmetic_state = {
+	.s_id = BSHELL_LEX_STATE_ARITHMETIC,
+	.s_pump_token = arithmetic_pump_token,
+	.s_links = links,
+	.s_keywords = keywords,
+	.s_operators = operators,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/lex/command.c

@@ -0,0 +1,228 @@
+#include "lex-internal.h"
+
+#include <bshell/parse/token.h>
+
+static bool char_can_continue_word(struct bshell_lex_ctx *ctx, fx_wchar c)
+{
+	if (fx_wchar_is_alnum(c)) {
+		return true;
+	}
+
+	if (fx_wchar_is_space(c)) {
+		return false;
+	}
+
+	if (c == '$') {
+		return true;
+	}
+
+	if (char_can_begin_symbol_in_state(ctx, c, BSHELL_LEX_STATE_WORD)) {
+		return false;
+	}
+
+	return true;
+}
+
+static enum bshell_status command_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	handle_lex_state_transition(ctx, sym->id);
+
+	struct bshell_lex_token *tok = NULL;
+	switch (sym->id) {
+	case BSHELL_SYM_DQUOTE:
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_SQUOTE:
+		status = read_literal_string(ctx, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+		enqueue_token(ctx, tok);
+		return BSHELL_SUCCESS;
+
+	case BSHELL_SYM_HASH:
+		return read_line_comment(ctx);
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		if (char_can_continue_word(ctx, peek_char(ctx))) {
+			lex_state_push(ctx, BSHELL_LEX_STATE_WORD, 0);
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_DOLLAR_LEFT_BRACE:
+		status = read_braced_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		if (char_can_continue_word(ctx, peek_char(ctx))) {
+			lex_state_push(ctx, BSHELL_LEX_STATE_WORD, 0);
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	push_symbol(ctx, sym->id);
+
+	return BSHELL_SUCCESS;
+}
+
+static bool string_is_redirection(const char *s)
+{
+	if (!*s) {
+		return false;
+	}
+
+	if (!strcmp(s, ">") || !strcmp(s, ">>")) {
+		return true;
+	}
+
+	long nr_angles = 0;
+	for (size_t i = 0; s[i];) {
+		fx_wchar c = fx_wchar_utf8_codepoint_decode(s);
+		if (fx_wchar_is_number(c)) {
+			if (nr_angles) {
+				return false;
+			}
+		} else if (c == '>') {
+			nr_angles++;
+
+			if (nr_angles > 2) {
+				return false;
+			}
+		} else {
+			return false;
+		}
+
+		s += fx_wchar_utf8_codepoint_stride(s);
+	}
+
+	return true;
+}
+
+static enum bshell_status command_word(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *word = NULL;
+	enum bshell_status status
+		= read_word(ctx, READ_NO_NUMBER_RECOGNITION, &word);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	bool continue_word = false;
+
+	fx_wchar c = peek_char(ctx);
+	const char *s = word->tok_str;
+	if (char_can_begin_symbol_in_state(ctx, c, BSHELL_LEX_STATE_WORD)) {
+		continue_word = true;
+	}
+
+	if (char_has_flags(ctx, c, BSHELL_LEX_TOKEN_TERMINATES_WORD)) {
+		continue_word = false;
+	}
+
+	if (string_is_redirection(s)) {
+		continue_word = false;
+	}
+
+	if (continue_word) {
+		lex_state_push(ctx, BSHELL_LEX_STATE_WORD, 0);
+	}
+
+	enqueue_token(ctx, word);
+	return BSHELL_SUCCESS;
+}
+
+enum bshell_status command_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	bool newline = false;
+
+	set_token_start(ctx);
+	while (fx_wchar_is_space(c)) {
+		if (c == '\n') {
+			newline = true;
+		}
+
+		set_token_end(ctx);
+		advance_char_noread(ctx);
+		c = peek_char_noread(ctx);
+	}
+
+	if (newline) {
+		struct bshell_lex_token *tok
+			= bshell_lex_token_create(BSHELL_TOK_LINEFEED);
+		enqueue_token(ctx, tok);
+		handle_lex_state_transition(ctx, BSHELL_TOK_LINEFEED);
+		return BSHELL_SUCCESS;
+	}
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return command_symbol(ctx);
+	}
+
+	return command_word(ctx);
+}
+
+const struct bshell_lex_state_link links[] = {
+	LINK_PUSH(BSHELL_SYM_DQUOTE, BSHELL_LEX_STATE_STRING, 0),
+	LINK_PUSH(
+		BSHELL_SYM_LEFT_PAREN,
+		BSHELL_LEX_STATE_STATEMENT,
+		STATEMENT_F_DISABLE_KEYWORDS),
+	LINK_PUSH(BSHELL_SYM_DOLLAR_LEFT_PAREN, BSHELL_LEX_STATE_STATEMENT, 0),
+	LINK_POP(BSHELL_SYM_RIGHT_PAREN),
+	LINK_POP(BSHELL_SYM_RIGHT_BRACE),
+	LINK_CHANGE(BSHELL_SYM_SEMICOLON, BSHELL_LEX_STATE_STATEMENT),
+	LINK_PUSH(BSHELL_SYM_AT_LEFT_BRACE, BSHELL_LEX_STATE_HASHTABLE, 0),
+	LINK_CHANGE(BSHELL_TOK_LINEFEED, BSHELL_LEX_STATE_STATEMENT),
+	LINK_END,
+};
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_DQUOTE,
+	BSHELL_SYM_SQUOTE,
+	BSHELL_SYM_DOLLAR,
+	BSHELL_SYM_DOLLAR_LEFT_PAREN,
+	BSHELL_SYM_DOLLAR_LEFT_BRACE,
+	BSHELL_SYM_AT,
+	BSHELL_SYM_AT_LEFT_BRACE,
+	BSHELL_SYM_AT_LEFT_PAREN,
+	BSHELL_SYM_AMPERSAND,
+	BSHELL_SYM_PIPE,
+	BSHELL_SYM_SEMICOLON,
+	BSHELL_SYM_RIGHT_PAREN,
+	BSHELL_SYM_LEFT_PAREN,
+	BSHELL_SYM_LEFT_BRACE,
+	BSHELL_SYM_RIGHT_BRACE,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_command_state = {
+	.s_id = BSHELL_LEX_STATE_COMMAND,
+	.s_pump_token = command_pump_token,
+	.s_links = links,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/lex/hashtable.c

@@ -0,0 +1,185 @@
+#include "lex-internal.h"
+
+static enum bshell_status hashtable_hyphen(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	if (!fx_wchar_is_alnum(c)) {
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+		handle_lex_state_transition(ctx, BSHELL_SYM_HYPHEN);
+		return BSHELL_SUCCESS;
+	}
+
+	struct bshell_lex_token *tok = NULL;
+	enum bshell_status status = read_word(
+		ctx,
+		READ_NO_SET_TOKEN_START | READ_APPEND_HYPHEN,
+		&tok);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	unsigned int token_type = BSHELL_TOK_WORD;
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
+		/* because of APPEND_HYPHEN (which is needed to ensure operator
+		 * tokens are detected properly), the resulting number will be
+		 * negative.
+		 * this token will be preceded by a HYPHEN token, so the number
+		 * must be positive */
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+	} else if (convert_word_to_operator(ctx, tok)) {
+		token_type = tok->tok_operator;
+	}
+
+	handle_lex_state_transition(ctx, token_type);
+	enqueue_token(ctx, tok);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status hashtable_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	handle_lex_state_transition(ctx, sym->id);
+
+	struct bshell_lex_token *tok = NULL;
+	switch (sym->id) {
+	case BSHELL_SYM_SQUOTE:
+		status = read_literal_string(ctx, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+		enqueue_token(ctx, tok);
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_HYPHEN:
+		return hashtable_hyphen(ctx);
+	case BSHELL_SYM_HASH:
+		return read_line_comment(ctx);
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_DOLLAR_LEFT_BRACE:
+		status = read_braced_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	push_symbol(ctx, sym->id);
+
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status hashtable_word(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *word = NULL;
+	enum bshell_status status = read_word(ctx, 0, &word);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	convert_word_to_number(word);
+
+	handle_lex_state_transition(ctx, word->tok_type);
+	enqueue_token(ctx, word);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status hashtable_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	bool newline = false;
+
+	set_token_start(ctx);
+	while (fx_wchar_is_space(c)) {
+		if (c == '\n') {
+			newline = true;
+		}
+
+		set_token_end(ctx);
+		advance_char_noread(ctx);
+		c = peek_char_noread(ctx);
+	}
+
+#if 1
+	if (newline) {
+		struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_LINEFEED);
+		enqueue_token(ctx, tok);
+		return BSHELL_SUCCESS;
+	}
+#endif
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return hashtable_symbol(ctx);
+	}
+
+	return hashtable_word(ctx);
+}
+
+static const struct bshell_lex_state_link links[] = {
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_EQUAL,
+		BSHELL_LEX_STATE_STATEMENT,
+		0,
+		BSHELL_SYM_RIGHT_BRACE,
+		BSHELL_SYM_SEMICOLON,
+		BSHELL_TOK_LINEFEED),
+	LINK_PUSH_WITH_TERM(
+		BSHELL_TOK_LINEFEED,
+		BSHELL_LEX_STATE_STATEMENT,
+		0,
+		BSHELL_SYM_SEMICOLON,
+		BSHELL_TOK_LINEFEED),
+	LINK_PUSH(BSHELL_SYM_DQUOTE, BSHELL_LEX_STATE_STRING, 0),
+	LINK_PUSH(
+		BSHELL_SYM_LEFT_PAREN,
+		BSHELL_LEX_STATE_STATEMENT,
+		STATEMENT_F_DISABLE_KEYWORDS),
+	LINK_PUSH(BSHELL_SYM_DOLLAR_LEFT_PAREN, BSHELL_LEX_STATE_STATEMENT, 0),
+	LINK_POP2(BSHELL_SYM_RIGHT_BRACE, LINK_ALLOW_RECURSION),
+	LINK_END,
+};
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_EQUAL,
+	BSHELL_SYM_DQUOTE,
+	BSHELL_SYM_SQUOTE,
+	BSHELL_SYM_SEMICOLON,
+	BSHELL_SYM_RIGHT_BRACE,
+	BSHELL_SYM_DOLLAR_LEFT_PAREN,
+	BSHELL_SYM_LEFT_PAREN,
+	BSHELL_SYM_HASH,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_hashtable_state = {
+	.s_id = BSHELL_LEX_STATE_HASHTABLE,
+	.s_pump_token = hashtable_pump_token,
+	.s_links = links,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/lex/lex-internal.h

@@ -0,0 +1,204 @@
+#ifndef PARSE_LEX_INTERNAL_H_
+#define PARSE_LEX_INTERNAL_H_
+
+#include <bshell/parse/lex.h>
+#include <bshell/parse/token.h>
+#include <bshell/status.h>
+
+struct bshell_lex_ctx;
+
+enum state_flags {
+	/* statement: don't convert matching words to keywords */
+	STATEMENT_F_DISABLE_KEYWORDS = 0x01u,
+	/* arithmetic: don't switch back to statement mode even when
+	 * encountering a token that would otherwise require it. */
+	ARITHMETIC_F_DISABLE_STATEMENTS = 0x01u,
+};
+
+enum read_flags {
+	READ_APPEND_HYPHEN = 0x01u,
+	READ_NO_SET_TOKEN_START = 0x02u,
+	READ_NO_NUMBER_RECOGNITION = 0x04u,
+};
+
+enum link_flags {
+	LINK_ALLOW_RECURSION = 0x01u,
+};
+
+#define LINK_PUSH(tok, target, flags)                                          \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_PUSH,                          \
+		.l_target = (target),                                          \
+		.l_target_flags = (flags),                                     \
+	})
+#define LINK_PUSH_WITH_TERM(tok, target, flags, ...)                           \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_PUSH,                          \
+		.l_target = (target),                                          \
+		.l_target_flags = (flags),                                     \
+		.l_terminators = {__VA_ARGS__, BSHELL_TOK_NONE},               \
+	})
+#define LINK_CHANGE(tok, target)                                               \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_CHANGE,                        \
+		.l_target = (target),                                          \
+	})
+#define LINK_POP(tok)                                                          \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_POP,                           \
+	})
+#define LINK_POP2(tok, flags)                                                  \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_POP,                           \
+		.l_flags = (flags),                                            \
+	})
+#define LINK_NONE(tok)                                                         \
+	((struct bshell_lex_state_link) {                                      \
+		.l_token = (tok),                                              \
+		.l_type = BSHELL_LEX_STATE_LINK_NONE,                          \
+	})
+#define LINK_END ((struct bshell_lex_state_link) {})
+
+struct bshell_lex_state_link {
+	unsigned int l_token;
+	enum {
+		BSHELL_LEX_STATE_LINK_NONE,
+		BSHELL_LEX_STATE_LINK_PUSH,
+		BSHELL_LEX_STATE_LINK_CHANGE,
+		BSHELL_LEX_STATE_LINK_POP,
+	} l_type;
+	enum link_flags l_flags;
+	enum bshell_lex_state_id l_target;
+	enum state_flags l_target_flags;
+	unsigned int l_terminators[BSHELL_LEX_STATE_MAX_TERMINATORS];
+};
+
+typedef enum bshell_status (*lex_state_pump_token)(struct bshell_lex_ctx *);
+typedef enum bshell_status (*lex_state_begin)(struct bshell_lex_ctx *);
+typedef enum bshell_status (*lex_state_end)(struct bshell_lex_ctx *);
+
+struct bshell_lex_state_type {
+	enum bshell_lex_state_id s_id;
+	lex_state_pump_token s_pump_token;
+	lex_state_begin s_begin;
+	lex_state_end s_end;
+
+	const unsigned int *s_keywords;
+	const unsigned int *s_operators;
+	const unsigned int *s_symbols;
+	const struct bshell_lex_state_link *s_links;
+};
+
+extern enum bshell_status pump_token_statement(struct bshell_lex_ctx *ctx);
+extern enum bshell_status pump_token_expression(struct bshell_lex_ctx *ctx);
+extern enum bshell_status pump_token_command(struct bshell_lex_ctx *ctx);
+extern enum bshell_status pump_token_arithmetic(struct bshell_lex_ctx *ctx);
+extern enum bshell_status pump_token_string(struct bshell_lex_ctx *ctx);
+
+extern void set_token_start(struct bshell_lex_ctx *ctx);
+extern void set_token_end(struct bshell_lex_ctx *ctx);
+
+extern struct bshell_lex_state *lex_state_push(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_state_id state_type,
+	enum state_flags flags);
+extern void lex_state_pop(struct bshell_lex_ctx *ctx);
+extern struct bshell_lex_state *lex_state_get(struct bshell_lex_ctx *ctx);
+extern void lex_state_change(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_state_id type);
+extern fx_string *lex_state_get_tempstr(struct bshell_lex_ctx *ctx);
+extern void lex_state_add_terminator(
+	struct bshell_lex_state *state,
+	unsigned int tok);
+extern bool lex_state_terminates_at_token(
+	struct bshell_lex_ctx *ctx,
+	unsigned int tok);
+
+extern fx_wchar peek_char(struct bshell_lex_ctx *ctx);
+extern fx_wchar peek_char_noread(struct bshell_lex_ctx *ctx);
+extern fx_wchar peek2_char(struct bshell_lex_ctx *ctx);
+extern fx_wchar peek2_char_noread(struct bshell_lex_ctx *ctx);
+extern void advance_char(struct bshell_lex_ctx *ctx);
+extern void advance_char_noread(struct bshell_lex_ctx *ctx);
+
+extern bool string_is_valid_number(
+	const char *s,
+	fx_value *out,
+	enum bshell_lex_token_type *out_type);
+extern bool convert_word_to_number(struct bshell_lex_token *tok);
+extern bool convert_word_to_keyword(struct bshell_lex_token *tok);
+extern bool convert_word_to_operator(
+	struct bshell_lex_ctx *ctx,
+	struct bshell_lex_token *tok);
+
+extern void enqueue_token(
+	struct bshell_lex_ctx *ctx,
+	struct bshell_lex_token *tok);
+extern void enqueue_token_with_coordinates(
+	struct bshell_lex_ctx *ctx,
+	struct bshell_lex_token *tok,
+	const struct bshell_char_cell *start,
+	const struct bshell_char_cell *end);
+
+extern enum bshell_status read_word(
+	struct bshell_lex_ctx *ctx,
+	enum read_flags flags,
+	struct bshell_lex_token **out);
+extern enum bshell_status read_symbol(
+	struct bshell_lex_ctx *ctx,
+	const struct bshell_lex_token_definition **out);
+extern enum bshell_status read_literal_string(
+	struct bshell_lex_ctx *ctx,
+	struct bshell_lex_token **out);
+extern enum bshell_status read_line_comment(struct bshell_lex_ctx *lex);
+extern enum bshell_status read_var(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_token_type type,
+	struct bshell_lex_token **out);
+extern enum bshell_status read_braced_var(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_token_type type,
+	struct bshell_lex_token **out);
+
+extern enum bshell_status push_symbol(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_symbol sym);
+
+extern bool char_can_begin_symbol(struct bshell_lex_ctx *ctx, char c);
+extern bool char_can_begin_symbol_in_state(
+	struct bshell_lex_ctx *ctx,
+	char c,
+	enum bshell_lex_state_id state_type);
+extern bool char_has_flags(
+	struct bshell_lex_ctx *ctx,
+	char c,
+	enum bshell_lex_token_flags flags);
+extern bool keyword_has_flags(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_keyword kw,
+	enum bshell_lex_token_flags flags);
+extern enum bshell_lex_token_flags keyword_get_flags(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_keyword kw);
+extern bool symbol_has_flags(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_symbol sym,
+	enum bshell_lex_token_flags flags);
+extern enum bshell_lex_token_flags symbol_get_flags(
+	struct bshell_lex_ctx *ctx,
+	enum bshell_lex_symbol sym);
+extern enum bshell_lex_operator get_bshell_operator_with_string(
+	struct bshell_lex_ctx *ctx,
+	const char *s);
+
+extern void handle_lex_state_transition(
+	struct bshell_lex_ctx *ctx,
+	unsigned int token);
+
+#endif

bshell.runtime/parse/lex/statement.c

@@ -0,0 +1,254 @@
+#include "lex-internal.h"
+
+static enum bshell_status statement_hyphen(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	if (!fx_wchar_is_alnum(c)) {
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+		handle_lex_state_transition(ctx, BSHELL_SYM_HYPHEN);
+		return BSHELL_SUCCESS;
+	}
+
+	struct bshell_lex_token *tok = NULL;
+	enum bshell_status status = read_word(
+		ctx,
+		READ_NO_SET_TOKEN_START | READ_APPEND_HYPHEN,
+		&tok);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	unsigned int token_type = BSHELL_TOK_WORD;
+
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
+
+		/* because of APPEND_HYPHEN (which is needed to ensure operator
+		 * tokens are detected properly), the resulting number will be
+		 * negative.
+		 * this token will be preceded by a HYPHEN token, so the number
+		 * must be positive */
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
+		push_symbol(ctx, BSHELL_SYM_HYPHEN);
+	} else if (convert_word_to_operator(ctx, tok)) {
+		token_type = tok->tok_operator;
+	}
+
+	handle_lex_state_transition(ctx, token_type);
+	enqueue_token(ctx, tok);
+
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status statement_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	handle_lex_state_transition(ctx, sym->id);
+
+	struct bshell_lex_token *tok = NULL;
+	switch (sym->id) {
+	case BSHELL_SYM_DQUOTE:
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_HYPHEN:
+		return statement_hyphen(ctx);
+	case BSHELL_SYM_SQUOTE:
+		status = read_literal_string(ctx, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+		enqueue_token(ctx, tok);
+		return BSHELL_SUCCESS;
+
+	case BSHELL_SYM_HASH:
+		return read_line_comment(ctx);
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_DOLLAR_LEFT_BRACE:
+		status = read_braced_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	push_symbol(ctx, sym->id);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status statement_word(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *word = NULL;
+	enum bshell_status status = read_word(ctx, 0, &word);
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	struct bshell_lex_state *state = lex_state_get(ctx);
+
+	bool enable_keywords = !(state->s_flags & STATEMENT_F_DISABLE_KEYWORDS);
+	unsigned int token = BSHELL_TOK_WORD;
+
+	if (enable_keywords && convert_word_to_keyword(word)) {
+		token = word->tok_keyword;
+	} else if (convert_word_to_number(word)) {
+		token = word->tok_type;
+	}
+
+	handle_lex_state_transition(ctx, token);
+
+	enqueue_token(ctx, word);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status statement_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+	bool newline = false;
+
+	set_token_start(ctx);
+	while (fx_wchar_is_space(c)) {
+		if (c == '\n') {
+			newline = true;
+		}
+
+		set_token_end(ctx);
+		advance_char_noread(ctx);
+		c = peek_char_noread(ctx);
+	}
+
+	if (newline) {
+		struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_LINEFEED);
+		enqueue_token(ctx, tok);
+		handle_lex_state_transition(ctx, BSHELL_TOK_LINEFEED);
+		return BSHELL_SUCCESS;
+	}
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return statement_symbol(ctx);
+	}
+
+	if (char_has_flags(ctx, c, BSHELL_LEX_TOKEN_UNARY_ARITHMETIC)) {
+		lex_state_change(ctx, BSHELL_LEX_STATE_ARITHMETIC);
+		return BSHELL_SUCCESS;
+	}
+
+	return statement_word(ctx);
+}
+
+static const struct bshell_lex_state_link links[] = {
+	LINK_PUSH(BSHELL_SYM_DQUOTE, BSHELL_LEX_STATE_STRING, 0),
+	/* arithmetic tokens */
+	LINK_CHANGE(BSHELL_TOK_KEYWORD, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_SYM_SQUOTE, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_TOK_INT, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_TOK_DOUBLE, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_PUSH(BSHELL_SYM_DOLLAR, BSHELL_LEX_STATE_ARITHMETIC, 0),
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_DOLLAR_LEFT_BRACE,
+		BSHELL_LEX_STATE_ARITHMETIC,
+		0,
+		BSHELL_SYM_RIGHT_BRACE),
+	LINK_CHANGE(BSHELL_SYM_AT_LEFT_BRACE, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_SYM_AT_LEFT_PAREN, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_AT_LEFT_BRACE,
+		BSHELL_LEX_STATE_HASHTABLE,
+		0,
+		BSHELL_SYM_RIGHT_BRACE),
+	LINK_PUSH(BSHELL_SYM_AT, BSHELL_LEX_STATE_ARITHMETIC, 0),
+	LINK_CHANGE(BSHELL_SYM_LEFT_PAREN, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_SYM_LEFT_BRACKET, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(BSHELL_SYM_BANG, BSHELL_LEX_STATE_ARITHMETIC),
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_LEFT_PAREN,
+		BSHELL_LEX_STATE_STATEMENT,
+		STATEMENT_F_DISABLE_KEYWORDS,
+		BSHELL_SYM_RIGHT_PAREN),
+
+	/* statement tokens */
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_LEFT_BRACE,
+		BSHELL_LEX_STATE_STATEMENT,
+		0,
+		BSHELL_SYM_RIGHT_BRACE),
+	LINK_PUSH_WITH_TERM(
+		BSHELL_SYM_DOLLAR_LEFT_PAREN,
+		BSHELL_LEX_STATE_STATEMENT,
+		0,
+		BSHELL_SYM_RIGHT_PAREN),
+
+	/* command tokens */
+	LINK_CHANGE(BSHELL_KW_FUNC, BSHELL_LEX_STATE_COMMAND),
+	LINK_CHANGE(BSHELL_SYM_AMPERSAND, BSHELL_LEX_STATE_COMMAND),
+	LINK_CHANGE(BSHELL_TOK_WORD, BSHELL_LEX_STATE_COMMAND),
+	LINK_END,
+};
+
+static const unsigned int keywords[] = {
+	BSHELL_KW_FUNC,
+	BSHELL_KW_IF,
+	BSHELL_KW_ELSEIF,
+	BSHELL_KW_ELSE,
+	BSHELL_KW_NONE,
+};
+
+static const unsigned int operators[] = {
+	BSHELL_TKOP_BNOT,
+	BSHELL_TKOP_NOT,
+	BSHELL_TKOP_NONE,
+};
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_AMPERSAND,
+	BSHELL_SYM_BANG,
+	BSHELL_SYM_SQUOTE,
+	BSHELL_SYM_DQUOTE,
+	BSHELL_SYM_HASH,
+	BSHELL_SYM_AT,
+	BSHELL_SYM_AT_LEFT_PAREN,
+	BSHELL_SYM_AT_LEFT_BRACE,
+	BSHELL_SYM_PIPE,
+	BSHELL_SYM_COMMA,
+	BSHELL_SYM_SEMICOLON,
+	BSHELL_SYM_LEFT_BRACE,
+	BSHELL_SYM_RIGHT_BRACE,
+	BSHELL_SYM_LEFT_BRACKET,
+	BSHELL_SYM_RIGHT_BRACKET,
+	BSHELL_SYM_LEFT_PAREN,
+	BSHELL_SYM_RIGHT_PAREN,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_statement_state = {
+	.s_id = BSHELL_LEX_STATE_STATEMENT,
+	.s_pump_token = statement_pump_token,
+	.s_links = links,
+	.s_keywords = keywords,
+	.s_operators = operators,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/lex/string.c

@@ -0,0 +1,141 @@
+#include "lex-internal.h"
+
+static enum bshell_status string_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	handle_lex_state_transition(ctx, sym->id);
+
+	struct bshell_lex_token *tok = NULL;
+
+	switch (sym->id) {
+	case BSHELL_SYM_DQUOTE:
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_DOLLAR_LEFT_PAREN:
+		return push_symbol(ctx, sym->id);
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_DOLLAR_LEFT_BRACE:
+		status = read_braced_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	return BSHELL_ERR_BAD_SYNTAX;
+}
+
+static enum bshell_status string_content(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = FX_WCHAR_INVALID;
+	fx_string *temp = lex_state_get_tempstr(ctx);
+	set_token_start(ctx);
+	fx_string_clear(temp);
+
+	while (1) {
+		c = peek_char(ctx);
+		if (c == FX_WCHAR_INVALID) {
+			/* EOF without end of string */
+			ctx->lex_status = BSHELL_ERR_BAD_SYNTAX;
+		}
+
+		if (char_can_begin_symbol(ctx, c)) {
+			break;
+		}
+
+		fx_string_append_wc(temp, c);
+		set_token_end(ctx);
+		advance_char(ctx);
+	}
+
+	if (fx_string_get_size(temp, FX_STRLEN_NORMAL) == 0) {
+		return BSHELL_SUCCESS;
+	}
+
+	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
+		BSHELL_TOK_STRING,
+		fx_string_get_cstr(temp));
+	enqueue_token(ctx, tok);
+
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status string_begin(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_STR_START);
+	if (!tok) {
+		return BSHELL_ERR_NO_MEMORY;
+	}
+
+	enqueue_token(ctx, tok);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status string_end(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_STR_END);
+	if (!tok) {
+		return BSHELL_ERR_NO_MEMORY;
+	}
+
+	enqueue_token(ctx, tok);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status string_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return string_symbol(ctx);
+	}
+
+	return string_content(ctx);
+}
+
+static const struct bshell_lex_state_link links[] = {
+	LINK_PUSH(BSHELL_SYM_DOLLAR_LEFT_PAREN, BSHELL_LEX_STATE_STATEMENT, 0),
+	LINK_POP(BSHELL_SYM_DQUOTE),
+	LINK_END,
+};
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_DOLLAR,
+	BSHELL_SYM_DOLLAR_LEFT_PAREN,
+	BSHELL_SYM_DOLLAR_LEFT_BRACE,
+	BSHELL_SYM_DQUOTE,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_string_state = {
+	.s_id = BSHELL_LEX_STATE_STRING,
+	.s_begin = string_begin,
+	.s_end = string_end,
+	.s_pump_token = string_pump_token,
+	.s_links = links,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/lex/word.c

@@ -0,0 +1,162 @@
+#include "lex-internal.h"
+
+static enum bshell_status word_symbol(struct bshell_lex_ctx *ctx)
+{
+	const struct bshell_lex_token_definition *sym = NULL;
+	enum bshell_status status = read_symbol(ctx, &sym);
+
+	if (status != BSHELL_SUCCESS) {
+		return status;
+	}
+
+	struct bshell_lex_token *tok = NULL;
+
+	switch (sym->id) {
+	case BSHELL_SYM_DOLLAR_LEFT_PAREN:
+		status = push_symbol(ctx, sym->id);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		lex_state_push(ctx, BSHELL_LEX_STATE_STATEMENT, 0);
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_RIGHT_PAREN:
+		lex_state_pop(ctx);
+
+		status = push_symbol(ctx, sym->id);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+		return BSHELL_SUCCESS;
+	case BSHELL_SYM_DOLLAR:
+		status = read_var(ctx, BSHELL_TOK_VAR, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	case BSHELL_SYM_AT:
+		status = read_var(ctx, BSHELL_TOK_VAR_SPLAT, &tok);
+		if (status != BSHELL_SUCCESS) {
+			return status;
+		}
+
+		enqueue_token(ctx, tok);
+		return status;
+	default:
+		break;
+	}
+
+	return BSHELL_ERR_BAD_SYNTAX;
+}
+
+static enum bshell_status word_content(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = FX_WCHAR_INVALID;
+	fx_string *temp = lex_state_get_tempstr(ctx);
+	set_token_start(ctx);
+	fx_string_clear(temp);
+
+	while (1) {
+		c = peek_char(ctx);
+		if (c == FX_WCHAR_INVALID) {
+			/* EOF without end of word */
+			ctx->lex_status = BSHELL_ERR_BAD_SYNTAX;
+		}
+
+		if (fx_wchar_is_space(c)) {
+			break;
+		}
+
+		if (char_can_begin_symbol(ctx, c)) {
+			break;
+		}
+
+		fx_string_append_wc(temp, c);
+		set_token_end(ctx);
+		advance_char(ctx);
+	}
+
+	if (fx_string_get_size(temp, FX_STRLEN_NORMAL) == 0) {
+		return BSHELL_SUCCESS;
+	}
+
+	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
+		BSHELL_TOK_WORD,
+		fx_string_get_cstr(temp));
+	enqueue_token(ctx, tok);
+
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status word_begin(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_WORD_START);
+	if (!tok) {
+		return BSHELL_ERR_NO_MEMORY;
+	}
+
+	enqueue_token_with_coordinates(
+		ctx,
+		tok,
+		&ctx->lex_start,
+		&ctx->lex_start);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status word_end(struct bshell_lex_ctx *ctx)
+{
+	struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_WORD_END);
+	if (!tok) {
+		return BSHELL_ERR_NO_MEMORY;
+	}
+
+	enqueue_token_with_coordinates(ctx, tok, &ctx->lex_end, &ctx->lex_end);
+	return BSHELL_SUCCESS;
+}
+
+static enum bshell_status word_pump_token(struct bshell_lex_ctx *ctx)
+{
+	fx_wchar c = peek_char(ctx);
+
+	if (fx_wchar_is_space(c)) {
+		lex_state_pop(ctx);
+		return BSHELL_SUCCESS;
+	}
+
+	if (char_has_flags(ctx, c, BSHELL_LEX_TOKEN_TERMINATES_WORD)) {
+		lex_state_pop(ctx);
+		return BSHELL_SUCCESS;
+	}
+
+	if (char_can_begin_symbol(ctx, c)) {
+		return word_symbol(ctx);
+	}
+
+	return word_content(ctx);
+}
+
+static const unsigned int symbols[] = {
+	BSHELL_SYM_AMPERSAND,
+	BSHELL_SYM_HASH,
+	BSHELL_SYM_DOLLAR,
+	BSHELL_SYM_DOLLAR_LEFT_PAREN,
+	BSHELL_SYM_DOLLAR_LEFT_BRACE,
+	BSHELL_SYM_PIPE,
+	BSHELL_SYM_COMMA,
+	BSHELL_SYM_SEMICOLON,
+	BSHELL_SYM_LEFT_BRACE,
+	BSHELL_SYM_RIGHT_BRACE,
+	BSHELL_SYM_LEFT_PAREN,
+	BSHELL_SYM_RIGHT_PAREN,
+	BSHELL_SYM_NONE,
+};
+
+const struct bshell_lex_state_type lex_word_state = {
+	.s_id = BSHELL_LEX_STATE_WORD,
+	.s_begin = word_begin,
+	.s_end = word_end,
+	.s_pump_token = word_pump_token,
+	.s_symbols = symbols,
+};

bshell.runtime/parse/parse.c

@@ -0,0 +1,55 @@
+#include "syntax.h"
+
+#include <bshell/ast.h>
+#include <bshell/parse/lex.h>
+#include <bshell/parse/parse.h>
+#include <bshell/parse/token.h>
+#include <stdio.h>
+#include <string.h>
+
+enum bshell_status bshell_parse_ctx_init(struct bshell_parse_ctx *ctx, struct bshell_lex_ctx *src)
+{
+	memset(ctx, 0x0, sizeof *ctx);
+
+	ctx->p_src = src;
+
+	return BSHELL_SUCCESS;
+}
+
+void bshell_parse_ctx_cleanup(struct bshell_parse_ctx *ctx)
+{
+}
+
+struct bshell_ast_node *bshell_parse_ctx_read_node(struct bshell_parse_ctx *ctx)
+{
+	parse_symbol(ctx, BSHELL_SYM_SEMICOLON);
+	parse_linefeed(ctx);
+
+	struct bshell_ast_node *result = NULL;
+	bool ok = parse_statement(ctx, &result);
+
+	return ok ? result : NULL;
+}
+
+void report_error(struct bshell_parse_ctx *ctx, const char *format, ...)
+{
+	ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+	fprintf(stderr, "PARSE: ");
+
+	va_list arg;
+	va_start(arg, format);
+	vfprintf(stderr, format, arg);
+	va_end(arg);
+
+	fprintf(stderr, "\n");
+
+	struct bshell_lex_token *tok = peek_token(ctx);
+	fprintf(stderr, "  peek_token = ");
+#if 0
+	if (tok) {
+		print_bshell_lex_token(tok);
+	} else {
+		fprintf(stderr, " EOF\n");
+	}
+#endif
+}

bshell.runtime/parse/syntax.h

@@ -0,0 +1,73 @@
+#ifndef PARSE_SYNTAX_H_
+#define PARSE_SYNTAX_H_
+
+#include <bshell/ast.h>
+#include <bshell/parse/lex.h>
+#include <bshell/parse/parse.h>
+#include <bshell/parse/token.h>
+#include <bshell/runtime/operator.h>
+#include <stdbool.h>
+#include <stdio.h>
+
+extern void report_error(struct bshell_parse_ctx *ctx, const char *format, ...);
+
+extern struct bshell_lex_token *peek_token(struct bshell_parse_ctx *ctx);
+extern enum bshell_lex_token_type peek_token_type(struct bshell_parse_ctx *ctx);
+extern enum bshell_lex_keyword peek_unknown_keyword(struct bshell_parse_ctx *ctx);
+extern enum bshell_lex_symbol peek_unknown_symbol(struct bshell_parse_ctx *ctx);
+
+extern struct bshell_lex_token *claim_token(struct bshell_parse_ctx *ctx);
+extern void discard_token(struct bshell_parse_ctx *ctx);
+
+extern bool peek_linefeed(struct bshell_parse_ctx *ctx);
+extern bool peek_symbol(struct bshell_parse_ctx *ctx, enum bshell_lex_symbol sym);
+extern bool peek_word(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out);
+extern bool peek_int(struct bshell_parse_ctx *ctx);
+
+extern bool parse_linefeed(struct bshell_parse_ctx *ctx);
+extern bool parse_symbol(struct bshell_parse_ctx *ctx, enum bshell_lex_symbol sym);
+extern bool parse_keyword(struct bshell_parse_ctx *ctx, enum bshell_lex_keyword kw);
+extern bool parse_word(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out);
+extern bool parse_var(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out);
+extern bool parse_flag(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out);
+
+extern bool peek_arith_expr(struct bshell_parse_ctx *ctx);
+extern bool parse_arith_value(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out);
+extern bool parse_arith_expr(
+	struct bshell_parse_ctx *ctx,
+	enum bshell_operator_precedence minimum_precedence,
+	struct bshell_ast_node **out);
+
+extern bool peek_keyword_expr(struct bshell_parse_ctx *ctx);
+extern bool parse_keyword_expr(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out);
+
+extern bool parse_if(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+extern bool parse_func(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+
+extern bool parse_fstring(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+
+extern bool parse_block(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+
+extern bool peek_command(struct bshell_parse_ctx *ctx);
+extern bool parse_pipeline(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node *first_item,
+	struct bshell_ast_node **out);
+extern bool parse_command(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+extern bool parse_cmdcall(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+extern bool parse_redirect(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+
+extern bool parse_expr(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out);
+extern bool peek_statement(struct bshell_parse_ctx *ctx);
+extern bool parse_statement(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out);
+extern bool parse_statement_list(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out);
+
+#endif

bshell.runtime/parse/syntax/arith.c

@@ -0,0 +1,923 @@
+#include "../syntax.h"
+
+#include <bshell/runtime/operator.h>
+#include <fx/queue.h>
+
+enum expr_component {
+	EXPR_C_NONE = 0,
+	EXPR_C_OPERAND,
+	EXPR_C_BINARY_OP,
+	EXPR_C_UNARY_OP,
+};
+
+struct expr_bshell_parse_ctx {
+	fx_queue expr_bshell_operator_stack, expr_out_queue;
+	enum expr_component expr_prev;
+	unsigned int expr_prev_symbol;
+	enum bshell_operator_precedence expr_minimum_precedence;
+	bool expr_done, expr_fail;
+};
+
+static bool op_node_is_complete(struct bshell_op_ast_node *node)
+{
+	if (!node->n_op) {
+		return false;
+	}
+
+	switch (node->n_op->op_arity) {
+	case BSHELL_OPA_UNARY:
+		return node->n_right != NULL;
+	case BSHELL_OPA_BINARY:
+		return (node->n_left != NULL && node->n_right != NULL);
+	default:
+		return false;
+	}
+}
+
+static bool finalise_expr(
+	struct expr_bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out,
+	enum bshell_operator_precedence minimum_precedence)
+{
+	fx_queue_entry *entry = NULL;
+	while (true) {
+		entry = fx_queue_pop_back(&ctx->expr_bshell_operator_stack);
+		if (!entry) {
+			break;
+		}
+
+		struct bshell_op_ast_node *node = fx_unbox(
+			struct bshell_op_ast_node,
+			entry,
+			n_base.n_entry);
+		if (!node) {
+			/* this should never happen */
+			return false;
+		}
+
+		const struct bshell_operator_info *op = node->n_op;
+
+		/* if we aren't processing operators below a certain precedence
+		 * then leave them on the stack and stop here. */
+		if (op->op_precedence < minimum_precedence) {
+			fx_queue_push_back(
+				&ctx->expr_bshell_operator_stack,
+				entry);
+			break;
+		}
+
+		fx_queue_push_back(&ctx->expr_out_queue, entry);
+	}
+
+	fx_queue q = FX_QUEUE_INIT;
+	fx_queue_entry *tmp = NULL;
+	entry = fx_queue_first(&ctx->expr_out_queue);
+	int i = 0;
+
+	while (entry) {
+		struct bshell_ast_node *item
+			= fx_unbox(struct bshell_ast_node, entry, n_entry);
+		fx_queue_entry *next = fx_queue_next(entry);
+		fx_queue_delete(&ctx->expr_out_queue, entry);
+
+		/* if the node is an operand, just push it to a
+		 * temporary queue and come back to it later. */
+		if (item->n_type != BSHELL_AST_OP) {
+			/* operand */
+			fx_queue_push_back(&q, &item->n_entry);
+			goto next;
+		}
+
+		const struct bshell_operator_info *op = NULL;
+
+		struct bshell_op_ast_node *op_node
+			= (struct bshell_op_ast_node *)item;
+
+		/* if an operator node is already complete (i.e. it
+		 * already has all the operands it needs, it can be
+		 * pushed to the operand queue as-is */
+		if (op_node_is_complete(op_node)) {
+			fx_queue_push_back(&q, &item->n_entry);
+			goto next;
+		}
+
+		/* otherwise, pop the relevant operands from the operand
+		 * queue... */
+		op = op_node->n_op;
+		tmp = fx_queue_pop_back(&q);
+		op_node->n_right
+			= fx_unbox(struct bshell_ast_node, tmp, n_entry);
+
+		if (op_node->n_right) {
+			op_node->n_right->n_parent
+				= (struct bshell_ast_node *)op_node;
+
+#if 0
+			bshell_ast_node_extend_bounds_recursive(
+				(struct ivy_bshell_ast_node *)op_node,
+				(struct ivy_bshell_ast_node *)tmp);
+#endif
+		}
+
+		if (op->op_arity == BSHELL_OPA_BINARY) {
+			tmp = fx_queue_pop_back(&q);
+			op_node->n_left = fx_unbox(
+				struct bshell_ast_node,
+				tmp,
+				n_entry);
+
+			if (op_node->n_left) {
+				op_node->n_left->n_parent
+					= (struct bshell_ast_node *)op_node;
+
+#if 0
+				bshell_ast_node_extend_bounds_recursive(
+					(struct ivy_bshell_ast_node *)op_node,
+					(struct ivy_bshell_ast_node *)tmp);
+#endif
+			}
+		}
+
+		/* ...and push the newly-completed operator node to the
+		 * operand queue */
+		fx_queue_push_back(&q, &op_node->n_base.n_entry);
+	next:
+		entry = next;
+	}
+
+#if 0
+	debug_printf("** after hierarchisation:\n");
+	print_expr_queues(state);
+#endif
+
+	/* if we are not processing operators below a certain precedence,
+	 * i.e. when determining the recipient of a keyword-message), these
+	 * operators will still be on the parser state's operator stack, but
+	 * their operands have just been moved to the temporary operand stack
+	 * used above. move them back to the parser state's output queue here
+	 * so they can be used later. */
+	entry = fx_queue_first(&ctx->expr_bshell_operator_stack);
+	while (entry) {
+		fx_queue_entry *entry2 = fx_queue_pop_front(&q);
+		if (!entry2) {
+			return false;
+		}
+
+		fx_queue_push_back(&ctx->expr_out_queue, entry2);
+		entry = fx_queue_next(entry);
+	}
+
+#if 0
+	debug_printf("** after de-linearisation:\n");
+	print_expr_queues(state);
+	ivy_bshell_ast_node_print(*expr_tree);
+	debug_printf("------\n");
+#endif
+
+	/* the final node remaining on the temp operand stack is the
+	 * root node of the new expression tree */
+	tmp = fx_queue_pop_back(&q);
+	*out = fx_unbox(struct bshell_ast_node, tmp, n_entry);
+
+	return true;
+}
+
+bool peek_arith_expr(struct bshell_parse_ctx *ctx)
+{
+	switch (peek_token_type(ctx)) {
+	case BSHELL_TOK_SYMBOL:
+		return bshell_operator_get_by_token(peek_unknown_symbol(ctx));
+	case BSHELL_TOK_INT:
+	case BSHELL_TOK_DOUBLE:
+	case BSHELL_TOK_STRING:
+	case BSHELL_TOK_VAR:
+	case BSHELL_TOK_STR_START:
+	case BSHELL_TOK_OPERATOR:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool parse_subexpr(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_symbol(ctx, BSHELL_SYM_LEFT_PAREN)) {
+		report_error(ctx, "expected `(`");
+	}
+
+	struct bshell_ast_node *v = NULL;
+	if (!parse_expr(ctx, &v)) {
+		report_error(ctx, "error while parsing parenthesis expression");
+		return false;
+	}
+
+	if (!parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+		report_error(ctx, "expected `)` after parenthesis expression");
+		return false;
+	}
+
+	*out = v;
+	return true;
+}
+
+static bool parse_stmt_block(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out)
+{
+	if (!parse_symbol(ctx, BSHELL_SYM_DOLLAR_LEFT_PAREN)) {
+		report_error(ctx, "expected `$(`");
+		return false;
+	}
+
+	if (parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+		*out = bshell_ast_node_create(BSHELL_AST_NULL);
+		return true;
+	}
+
+	struct bshell_ast_node *v = NULL;
+	if (!parse_statement_list(ctx, &v)) {
+		return false;
+	}
+
+	if (!parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+		report_error(ctx, "expected ')' after subexpression");
+		bshell_ast_node_destroy(v);
+		return false;
+	}
+
+	*out = v;
+	return true;
+}
+
+static bool parse_hashtable(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_symbol(ctx, BSHELL_SYM_AT_LEFT_BRACE)) {
+		report_error(ctx, "expected `@{`");
+		return false;
+	}
+
+	parse_linefeed(ctx);
+
+	struct bshell_hashtable_ast_node *table
+		= (struct bshell_hashtable_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_HASHTABLE);
+	if (!table) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	size_t nr_items = 0;
+	bool ok = true;
+	while (ok) {
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_BRACE)) {
+			break;
+		}
+
+		parse_linefeed(ctx);
+
+		struct bshell_hashtable_item_ast_node *item
+			= (struct bshell_hashtable_item_ast_node *)
+				bshell_ast_node_create(
+					BSHELL_AST_HASHTABLE_ITEM);
+		struct bshell_lex_token *tok = NULL;
+		if (parse_word(ctx, &tok)) {
+			struct bshell_string_ast_node *v
+				= (struct bshell_string_ast_node *)
+					bshell_ast_node_create(
+						BSHELL_AST_STRING);
+			v->n_value = tok;
+			item->n_key = (struct bshell_ast_node *)v;
+		} else if (!parse_arith_value(ctx, &item->n_key)) {
+			report_error(ctx, "failed to parse hashtable key");
+			bshell_ast_node_destroy((struct bshell_ast_node *)item);
+			ok = false;
+			break;
+		}
+
+		if (!parse_symbol(ctx, BSHELL_SYM_EQUAL)) {
+			report_error(ctx, "expected `=` after hashtable key");
+			bshell_ast_node_destroy((struct bshell_ast_node *)item);
+			ok = false;
+			break;
+		}
+
+		if (!parse_expr(ctx, &item->n_value)) {
+			report_error(ctx, "failed to parse hashtable value");
+			bshell_ast_node_destroy((struct bshell_ast_node *)item);
+			ok = false;
+			break;
+		}
+
+		fx_queue_push_back(&table->n_items, &item->n_base.n_entry);
+		nr_items++;
+
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_BRACE)) {
+			break;
+		}
+
+		if (!parse_linefeed(ctx)
+		    && !parse_symbol(ctx, BSHELL_SYM_SEMICOLON)) {
+			report_error(
+				ctx,
+				"expected `;`, `}`, or linefeed after "
+				"hashtable value");
+			ok = false;
+			break;
+		}
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)table);
+		return false;
+	}
+
+	*out = (struct bshell_ast_node *)table;
+	return true;
+}
+
+static bool parse_array(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_symbol(ctx, BSHELL_SYM_AT_LEFT_PAREN)) {
+		report_error(ctx, "expected `@(`");
+		return false;
+	}
+
+	struct bshell_array_ast_node *array
+		= (struct bshell_array_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_ARRAY);
+	if (!array) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	size_t nr_items = 0;
+	bool ok = true;
+	while (ok) {
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+			break;
+		}
+
+		if (nr_items && !parse_symbol(ctx, BSHELL_SYM_COMMA)) {
+			report_error(
+				ctx,
+				"expected `,` or `)` after array value");
+			ok = false;
+		}
+
+		struct bshell_ast_node *item = NULL;
+		if (!parse_arith_value(ctx, &item)) {
+			report_error(ctx, "failed to parse array item");
+			ok = false;
+			break;
+		}
+
+		fx_queue_push_back(&array->n_items, &item->n_entry);
+		nr_items++;
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)array);
+		return false;
+	}
+
+	*out = (struct bshell_ast_node *)array;
+	return true;
+}
+
+bool parse_fstring(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (peek_token_type(ctx) != BSHELL_TOK_STR_START) {
+		return false;
+	}
+
+	discard_token(ctx);
+
+	struct bshell_fstring_ast_node *fstring
+		= (struct bshell_fstring_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_FSTRING);
+	if (!fstring) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	bool ok = true;
+	while (ok) {
+		if (peek_token_type(ctx) == BSHELL_TOK_STR_END) {
+			discard_token(ctx);
+			break;
+		}
+
+		struct bshell_ast_node *item = NULL;
+		if (!parse_arith_value(ctx, &item)) {
+			ok = false;
+			break;
+		}
+
+		fx_queue_push_back(&fstring->n_elements, &item->n_entry);
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)fstring);
+		fstring = NULL;
+	}
+
+	*out = (struct bshell_ast_node *)fstring;
+	return ok;
+}
+
+bool parse_arith_value(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	switch (tok->tok_type) {
+	case BSHELL_TOK_INT: {
+		struct bshell_int_ast_node *v
+			= (struct bshell_int_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_INT);
+		v->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)v;
+		return true;
+	}
+	case BSHELL_TOK_DOUBLE: {
+		struct bshell_double_ast_node *v
+			= (struct bshell_double_ast_node *)
+				bshell_ast_node_create(BSHELL_AST_DOUBLE);
+		v->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)v;
+		return true;
+	}
+	case BSHELL_TOK_STRING: {
+		struct bshell_string_ast_node *v
+			= (struct bshell_string_ast_node *)
+				bshell_ast_node_create(BSHELL_AST_STRING);
+		v->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)v;
+		return true;
+	}
+	case BSHELL_TOK_WORD: {
+		struct bshell_word_ast_node *v
+			= (struct bshell_word_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_WORD);
+		v->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)v;
+		return true;
+	}
+	case BSHELL_TOK_VAR: {
+		struct bshell_var_ast_node *v
+			= (struct bshell_var_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_VAR);
+		v->n_ident = claim_token(ctx);
+		*out = (struct bshell_ast_node *)v;
+		return true;
+	}
+	case BSHELL_TOK_STR_START:
+		return parse_fstring(ctx, out);
+	case BSHELL_TOK_SYMBOL:
+		switch (tok->tok_symbol) {
+		case BSHELL_SYM_LEFT_PAREN:
+			return parse_subexpr(ctx, out);
+		case BSHELL_SYM_DOLLAR_LEFT_PAREN:
+			return parse_stmt_block(ctx, out);
+		case BSHELL_SYM_AT_LEFT_BRACE:
+			return parse_hashtable(ctx, out);
+		case BSHELL_SYM_AT_LEFT_PAREN:
+			return parse_array(ctx, out);
+		case BSHELL_SYM_LEFT_BRACE:
+			return parse_block(ctx, out);
+		default:
+			report_error(ctx, "token is not a valid operand");
+			return false;
+		}
+		break;
+	default:
+		report_error(ctx, "token is not a valid operand");
+		return false;
+	}
+}
+
+static bool parse_operand(struct bshell_parse_ctx *ctx, struct expr_bshell_parse_ctx *expr)
+{
+	if (expr->expr_prev == EXPR_C_OPERAND) {
+		report_error(ctx, "encountered two operands in a row");
+		return false;
+	}
+
+	expr->expr_prev = EXPR_C_OPERAND;
+
+	struct bshell_ast_node *v = NULL;
+	if (!parse_arith_value(ctx, &v)) {
+		return false;
+	}
+
+	fx_queue_push_back(&expr->expr_out_queue, &v->n_entry);
+	return true;
+}
+
+void arith_push_operator(
+	struct expr_bshell_parse_ctx *state,
+	struct bshell_op_ast_node *node)
+{
+	const struct bshell_operator_info *op = node->n_op;
+	if (!op) {
+		return;
+	}
+
+	while (true) {
+		fx_queue_entry *top
+			= fx_queue_last(&state->expr_bshell_operator_stack);
+		if (!top) {
+			break;
+		}
+
+		struct bshell_ast_node *top_node
+			= fx_unbox(struct bshell_ast_node, top, n_entry);
+		const struct bshell_operator_info *top_op = NULL;
+
+		switch (top_node->n_type) {
+		case BSHELL_AST_OP: {
+			struct bshell_op_ast_node *op_node
+				= (struct bshell_op_ast_node *)top_node;
+			top_op = op_node->n_op;
+			break;
+		}
+		default:
+			return;
+		}
+
+		if (top_op->op_precedence < op->op_precedence
+		    || (top_op->op_precedence == op->op_precedence
+		        && op->op_associativity != BSHELL_ASSOCIATIVITY_LEFT)) {
+			break;
+		}
+
+		fx_queue_delete(&state->expr_bshell_operator_stack, top);
+		fx_queue_push_back(&state->expr_out_queue, top);
+	}
+
+	fx_queue_push_back(
+		&state->expr_bshell_operator_stack,
+		&node->n_base.n_entry);
+}
+
+static bool parse_unary_operator(
+	struct bshell_parse_ctx *ctx,
+	struct expr_bshell_parse_ctx *expr)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+
+	const struct bshell_operator_info *op = NULL;
+	switch (tok->tok_type) {
+	case BSHELL_TOK_SYMBOL:
+		op = bshell_operator_get_by_token(tok->tok_symbol);
+		break;
+	case BSHELL_TOK_OPERATOR:
+		switch (tok->tok_operator) {
+		case BSHELL_TKOP_SPLIT:
+			op = bshell_operator_get_by_id(BSHELL_OP_USPLIT);
+			break;
+		case BSHELL_TKOP_JOIN:
+			op = bshell_operator_get_by_id(BSHELL_OP_USPLIT);
+			break;
+		default:
+			op = bshell_operator_get_by_token(tok->tok_operator);
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (expr->expr_prev == EXPR_C_OPERAND
+	    && op->op_location == BSHELL_OPL_PREFIX) {
+		report_error(
+			ctx,
+			"unexpected operand before unary "
+			"operator");
+		return false;
+	}
+
+	if (!op) {
+		report_error(ctx, "unknown unary operator");
+		return false;
+	}
+
+	if (op->op_precedence < expr->expr_minimum_precedence) {
+		expr->expr_done = true;
+		return true;
+	}
+
+	expr->expr_prev = EXPR_C_BINARY_OP;
+
+	struct bshell_op_ast_node *op_node
+		= (struct bshell_op_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_OP);
+	if (!op_node) {
+		return false;
+	}
+
+	op_node->n_op = op;
+	discard_token(ctx);
+	arith_push_operator(expr, op_node);
+	return true;
+}
+
+static bool parse_binary_operator(
+	struct bshell_parse_ctx *ctx,
+	struct expr_bshell_parse_ctx *expr)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+
+	const struct bshell_operator_info *op = NULL;
+	switch (tok->tok_type) {
+	case BSHELL_TOK_SYMBOL:
+		op = bshell_operator_get_by_token(tok->tok_symbol);
+		break;
+	case BSHELL_TOK_OPERATOR:
+		switch (tok->tok_operator) {
+		case BSHELL_TKOP_SPLIT:
+			op = bshell_operator_get_by_id(BSHELL_OP_BSPLIT);
+			break;
+		case BSHELL_TKOP_JOIN:
+			op = bshell_operator_get_by_id(BSHELL_OP_BJOIN);
+			break;
+		default:
+			op = bshell_operator_get_by_token(tok->tok_operator);
+			break;
+		}
+	default:
+		break;
+	}
+
+	if (!op) {
+		report_error(ctx, "unknown binary operator");
+		return false;
+	}
+
+	if (op->op_precedence < expr->expr_minimum_precedence) {
+		expr->expr_done = true;
+		return true;
+	}
+
+	if (expr->expr_prev != EXPR_C_OPERAND) {
+		switch (op->op_id) {
+		case BSHELL_OP_PAREN:
+			break;
+		default:
+			report_error(
+				ctx,
+				"expected operand before binary "
+				"operator");
+			return false;
+		}
+	}
+
+	expr->expr_prev = EXPR_C_BINARY_OP;
+
+	struct bshell_op_ast_node *op_node
+		= (struct bshell_op_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_OP);
+	if (!op_node) {
+		return false;
+	}
+
+	op_node->n_op = op;
+	discard_token(ctx);
+	arith_push_operator(expr, op_node);
+	return true;
+}
+
+static bool parse_call(struct bshell_parse_ctx *ctx, struct expr_bshell_parse_ctx *expr)
+{
+	return false;
+}
+
+static bool parse_comma(struct bshell_parse_ctx *ctx, struct expr_bshell_parse_ctx *expr)
+{
+	if (BSHELL_PRECEDENCE_ARRAY < expr->expr_minimum_precedence) {
+		expr->expr_done = true;
+		return true;
+	}
+
+	struct bshell_ast_node *item = NULL;
+	if (!finalise_expr(expr, &item, BSHELL_PRECEDENCE_ARRAY)) {
+		report_error(ctx, "failed to collect first array item.");
+		return false;
+	}
+
+	struct bshell_array_ast_node *array
+		= (struct bshell_array_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_ARRAY);
+	if (!array) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		bshell_ast_node_destroy(item);
+		return false;
+	}
+
+	if (item) {
+		fx_queue_push_back(&array->n_items, &item->n_entry);
+	}
+
+	while (1) {
+		if (!parse_symbol(ctx, BSHELL_SYM_COMMA)) {
+			break;
+		}
+
+		if (!parse_arith_expr(
+			    ctx,
+			    BSHELL_PRECEDENCE_ARRAY + 1,
+			    &item)) {
+			report_error(ctx, "failed to parse array item.");
+			bshell_ast_node_destroy(
+				(struct bshell_ast_node *)array);
+			return false;
+		}
+
+		fx_queue_push_back(&array->n_items, &item->n_entry);
+	}
+
+	fx_queue_push_back(&expr->expr_out_queue, &array->n_base.n_entry);
+	expr->expr_prev = EXPR_C_OPERAND;
+
+	return true;
+}
+
+static bool can_use_command(struct expr_bshell_parse_ctx *ctx)
+{
+	switch (ctx->expr_prev_symbol) {
+	case BSHELL_TOK_NONE:
+	case BSHELL_SYM_EQUAL:
+	case BSHELL_SYM_PLUS_EQUAL:
+	case BSHELL_SYM_HYPHEN_EQUAL:
+	case BSHELL_SYM_ASTERISK_EQUAL:
+	case BSHELL_SYM_FORWARD_SLASH_EQUAL:
+	case BSHELL_SYM_PERCENT_EQUAL:
+		return true;
+	default:
+		return false;
+	}
+}
+
+bool parse_arith_expr(
+	struct bshell_parse_ctx *ctx,
+	enum bshell_operator_precedence minimum_precedence,
+	struct bshell_ast_node **out)
+{
+	struct expr_bshell_parse_ctx expr = {
+		.expr_minimum_precedence = minimum_precedence,
+	};
+
+	while (!expr.expr_fail && !expr.expr_done) {
+		struct bshell_lex_token *tok = peek_token(ctx);
+		if (!tok) {
+			break;
+		}
+
+		switch (tok->tok_type) {
+		case BSHELL_TOK_LINEFEED:
+			expr.expr_prev_symbol = tok->tok_type;
+			expr.expr_done = true;
+			break;
+		case BSHELL_TOK_WORD: {
+			if (!can_use_command(&expr)) {
+				expr.expr_fail = !parse_operand(ctx, &expr);
+				break;
+			}
+
+			expr.expr_prev_symbol = tok->tok_type;
+			struct bshell_ast_node *value = NULL;
+			if (!parse_command(ctx, &value)) {
+				expr.expr_fail = true;
+				break;
+			}
+
+			fx_queue_push_back(
+				&expr.expr_out_queue,
+				&value->n_entry);
+			break;
+		}
+
+		case BSHELL_TOK_VAR:
+		case BSHELL_TOK_INT:
+		case BSHELL_TOK_DOUBLE:
+		case BSHELL_TOK_STRING:
+		case BSHELL_TOK_STR_START:
+			expr.expr_prev_symbol = tok->tok_type;
+			expr.expr_fail = !parse_operand(ctx, &expr);
+			expr.expr_prev_symbol = tok->tok_type;
+			break;
+		case BSHELL_TOK_OPERATOR:
+			expr.expr_prev_symbol = tok->tok_operator;
+			switch (tok->tok_operator) {
+			/* these two are special cases, as they are both
+			 * unary AND binary operators */
+			case BSHELL_TKOP_SPLIT:
+			case BSHELL_TKOP_JOIN:
+				if (expr.expr_prev == EXPR_C_OPERAND) {
+					expr.expr_fail = !parse_binary_operator(
+						ctx,
+						&expr);
+				} else {
+					expr.expr_fail = !parse_unary_operator(
+						ctx,
+						&expr);
+				}
+
+				break;
+			case BSHELL_TKOP_BNOT:
+			case BSHELL_TKOP_NOT:
+				expr.expr_fail
+					= !parse_unary_operator(ctx, &expr);
+				break;
+			default:
+				expr.expr_fail
+					= !parse_binary_operator(ctx, &expr);
+				break;
+			}
+			expr.expr_prev_symbol = tok->tok_operator;
+			break;
+		case BSHELL_TOK_SYMBOL:
+			expr.expr_prev_symbol = tok->tok_symbol;
+
+			switch (tok->tok_symbol) {
+			case BSHELL_SYM_SEMICOLON:
+			case BSHELL_SYM_AMPERSAND:
+			case BSHELL_SYM_PIPE:
+			case BSHELL_SYM_RIGHT_PAREN:
+			case BSHELL_SYM_RIGHT_BRACE:
+			case BSHELL_SYM_RIGHT_BRACKET:
+				expr.expr_done = true;
+				break;
+			case BSHELL_SYM_COMMA:
+				expr.expr_fail = !parse_comma(ctx, &expr);
+				break;
+			case BSHELL_SYM_LEFT_PAREN: {
+				if (expr.expr_prev == EXPR_C_OPERAND) {
+					return parse_call(ctx, &expr);
+				}
+
+				struct bshell_ast_node *v = NULL;
+				expr.expr_fail = !parse_subexpr(ctx, &v);
+				if (expr.expr_fail) {
+					break;
+				}
+
+				fx_queue_push_back(
+					&expr.expr_out_queue,
+					&v->n_entry);
+				expr.expr_prev = EXPR_C_OPERAND;
+				break;
+			}
+			case BSHELL_SYM_DOLLAR_LEFT_PAREN:
+			case BSHELL_SYM_AT_LEFT_PAREN:
+			case BSHELL_SYM_AT_LEFT_BRACE:
+			case BSHELL_SYM_LEFT_BRACE:
+				expr.expr_fail = !parse_operand(ctx, &expr);
+				break;
+			default: {
+				const struct bshell_operator_info *op
+					= bshell_operator_get_by_token(
+						tok->tok_symbol);
+				if (op->op_arity == BSHELL_OPA_BINARY) {
+					expr.expr_fail = !parse_binary_operator(
+						ctx,
+						&expr);
+				} else {
+					expr.expr_fail = !parse_unary_operator(
+						ctx,
+						&expr);
+				}
+				break;
+			}
+			}
+
+			break;
+		default:
+			report_error(
+				ctx,
+				"unexpected token in arithmetic "
+				"expression");
+			expr.expr_fail = true;
+			break;
+		}
+	}
+
+	if (expr.expr_fail) {
+		/* TODO cleanup */
+		return false;
+	}
+
+	struct bshell_ast_node *value = NULL;
+	if (!finalise_expr(&expr, &value, BSHELL_PRECEDENCE_ASSIGN)) {
+		report_error(ctx, "failed to convert expression to AST");
+		/* TODO cleanup */
+		return false;
+	}
+
+	if (BSHELL_PRECEDENCE_PIPELINE >= expr.expr_minimum_precedence) {
+		if (peek_symbol(ctx, BSHELL_SYM_PIPE)) {
+			return parse_pipeline(ctx, value, out);
+		}
+	}
+
+	*out = value;
+	return true;
+}

bshell.runtime/parse/syntax/block.c

@@ -0,0 +1,53 @@
+#include "../syntax.h"
+
+bool parse_block(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_symbol(ctx, BSHELL_SYM_LEFT_BRACE)) {
+		return false;
+	}
+
+	parse_linefeed(ctx);
+
+	struct bshell_block_ast_node *block
+		= (struct bshell_block_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_BLOCK);
+
+	bool ok = true;
+	while (1) {
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_BRACE)) {
+			break;
+		}
+
+		parse_linefeed(ctx);
+
+		struct bshell_ast_node *stmt = NULL;
+		if (!parse_statement(ctx, &stmt)) {
+			ok = false;
+			break;
+		}
+
+		fx_queue_push_back(&block->n_statements, &stmt->n_entry);
+
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_BRACE)) {
+			break;
+		}
+
+		if (!parse_linefeed(ctx)
+		    && !parse_symbol(ctx, BSHELL_SYM_SEMICOLON)) {
+			report_error(
+				ctx,
+				"expected `;`, `}`, or linefeed after "
+				"statement");
+			ok = false;
+			break;
+		}
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)block);
+		block = NULL;
+	}
+
+	*out = (struct bshell_ast_node *)block;
+	return true;
+}

bshell.runtime/parse/syntax/command.c

@@ -0,0 +1,523 @@
+#include "../syntax.h"
+
+#include <fx/encoding.h>
+
+static bool parse_fword(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (peek_token_type(ctx) != BSHELL_TOK_WORD_START) {
+		return false;
+	}
+
+	discard_token(ctx);
+
+	struct bshell_fstring_ast_node *fstring
+		= (struct bshell_fstring_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_FSTRING);
+	if (!fstring) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	bool ok = true;
+	while (ok) {
+		if (peek_token_type(ctx) == BSHELL_TOK_WORD_END) {
+			discard_token(ctx);
+			break;
+		}
+
+		struct bshell_ast_node *item = NULL;
+		if (peek_token_type(ctx) == BSHELL_TOK_WORD) {
+			struct bshell_word_ast_node *n
+				= (struct bshell_word_ast_node *)
+					bshell_ast_node_create(BSHELL_AST_WORD);
+			if (!n) {
+				ctx->p_status = BSHELL_ERR_NO_MEMORY;
+				ok = false;
+				break;
+			}
+
+			n->n_value = claim_token(ctx);
+			item = (struct bshell_ast_node *)n;
+		} else {
+			if (!parse_arith_value(ctx, &item)) {
+				ok = false;
+				break;
+			}
+		}
+
+		fx_queue_push_back(&fstring->n_elements, &item->n_entry);
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)fstring);
+		fstring = NULL;
+	}
+
+	*out = (struct bshell_ast_node *)fstring;
+	return ok;
+	return false;
+}
+
+static bool parse_cmdcall_arg(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node **out)
+{
+	if (ctx->p_status != BSHELL_SUCCESS) {
+		return false;
+	}
+
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	struct bshell_ast_node *arg = NULL;
+
+	switch (tok->tok_type) {
+	case BSHELL_TOK_WORD_START:
+		return parse_fword(ctx, out);
+	case BSHELL_TOK_STR_START:
+		return parse_fstring(ctx, out);
+	case BSHELL_TOK_WORD: {
+		struct bshell_word_ast_node *n
+			= (struct bshell_word_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_WORD);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)n;
+		return true;
+	}
+
+	case BSHELL_TOK_VAR: {
+		struct bshell_var_ast_node *n
+			= (struct bshell_var_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_VAR);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_ident = claim_token(ctx);
+		*out = (struct bshell_ast_node *)n;
+		return true;
+	}
+
+	case BSHELL_TOK_VAR_SPLAT: {
+		struct bshell_var_splat_ast_node *n
+			= (struct bshell_var_splat_ast_node *)
+				bshell_ast_node_create(BSHELL_AST_VAR_SPLAT);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_ident = claim_token(ctx);
+		*out = (struct bshell_ast_node *)n;
+		return true;
+	}
+
+	case BSHELL_TOK_STRING: {
+		struct bshell_string_ast_node *n
+			= (struct bshell_string_ast_node *)
+				bshell_ast_node_create(BSHELL_AST_STRING);
+		if (!n) {
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			return false;
+		}
+
+		n->n_value = claim_token(ctx);
+		*out = (struct bshell_ast_node *)n;
+		return true;
+	}
+
+	case BSHELL_TOK_SYMBOL:
+		switch (tok->tok_symbol) {
+		case BSHELL_SYM_LEFT_PAREN:
+		case BSHELL_SYM_LEFT_BRACE:
+		case BSHELL_SYM_DOLLAR_LEFT_PAREN:
+		case BSHELL_SYM_AT_LEFT_BRACE:
+		case BSHELL_SYM_AT_LEFT_PAREN:
+			return parse_arith_value(ctx, out);
+		default:
+			report_error(
+				ctx,
+				"encountered unsupported command arg");
+			return false;
+		}
+		break;
+	default:
+		report_error(ctx, "encountered unsupported command arg");
+		return false;
+	}
+
+	return true;
+}
+
+static bool parse_redirect_to_fd(
+	struct bshell_parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	struct bshell_ast_node **out)
+{
+	if (ctx->p_status != BSHELL_SUCCESS) {
+		return false;
+	}
+
+	struct bshell_redirection_ast_node *redirect
+		= (struct bshell_redirection_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+
+	if (!parse_symbol(ctx, BSHELL_SYM_AMPERSAND)) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)redirect);
+		return false;
+	}
+
+	struct bshell_lex_token *out_tok = NULL;
+	struct bshell_ast_node *out_expr = NULL;
+	long long out_fd = -1;
+
+	if (peek_word(ctx, &out_tok)) {
+		const char *s = out_tok->tok_str;
+		char *ep;
+		out_fd = strtoll(s, &ep, 10);
+		if (*ep == '\0') {
+			discard_token(ctx);
+			out_tok = NULL;
+		} else {
+			out_fd = -1;
+		}
+	} else if (!parse_cmdcall_arg(ctx, &out_expr)) {
+		return false;
+	}
+
+	redirect->n_out_is_fd = (out_fd >= 0) || out_expr;
+	redirect->n_out_is_expr = out_expr != NULL;
+	redirect->n_out = (unsigned int)out_fd;
+	redirect->n_out_path_expr = out_expr;
+	if (out_tok) {
+		redirect->n_out_tok = claim_token(ctx);
+		redirect->n_out_path = out_tok->tok_str;
+	}
+
+	*out = (struct bshell_ast_node *)redirect;
+	return true;
+}
+
+static bool parse_redirect_to_file_squashed(
+	struct bshell_parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	const char *str,
+	struct bshell_ast_node **out)
+{
+	if (ctx->p_status != BSHELL_SUCCESS) {
+		return false;
+	}
+
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (*str == '\0') {
+		return false;
+	}
+
+	struct bshell_redirection_ast_node *redirect
+		= (struct bshell_redirection_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+	redirect->n_out_is_fd = false;
+	redirect->n_out_is_expr = false;
+	redirect->n_out_path = str;
+
+	redirect->n_out_tok = claim_token(ctx);
+
+	*out = (struct bshell_ast_node *)redirect;
+	return true;
+}
+
+static bool parse_redirect_to_file_separate(
+	struct bshell_parse_ctx *ctx,
+	unsigned int in_fd,
+	bool append,
+	struct bshell_ast_node **out)
+{
+	if (ctx->p_status != BSHELL_SUCCESS) {
+		return false;
+	}
+
+	struct bshell_ast_node *out_path = NULL;
+	if (!parse_cmdcall_arg(ctx, &out_path)) {
+		ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+		return false;
+	}
+
+	struct bshell_redirection_ast_node *redirect
+		= (struct bshell_redirection_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_REDIRECTION);
+
+	redirect->n_in = in_fd;
+	redirect->n_append = append;
+	redirect->n_out_is_fd = false;
+	redirect->n_out_is_expr = true;
+	redirect->n_out_path_expr = out_path;
+
+	*out = (struct bshell_ast_node *)redirect;
+	return true;
+}
+
+bool parse_redirect(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok || tok->tok_type != BSHELL_TOK_WORD) {
+		return false;
+	}
+
+	unsigned int in_fd = 1;
+	const char *str = tok->tok_str;
+	bool append = false;
+
+	if (fx_wchar_is_number(*str)) {
+		in_fd = 0;
+		while (fx_wchar_is_number(*str)) {
+			in_fd *= 10;
+			in_fd += *str - '0';
+			str++;
+		}
+	}
+
+	if (*str != '>') {
+		return false;
+	}
+
+	str++;
+	if (*str == '>') {
+		append = true;
+		str++;
+	}
+
+	if (*str != '\0') {
+		return parse_redirect_to_file_squashed(
+			ctx,
+			in_fd,
+			append,
+			str,
+			out);
+	}
+
+	discard_token(ctx);
+
+	if (parse_redirect_to_fd(ctx, in_fd, append, out)) {
+		return true;
+	}
+
+	if (parse_redirect_to_file_separate(ctx, in_fd, append, out)) {
+		return true;
+	}
+
+	return false;
+}
+
+static bool peek_cmdcall_item(struct bshell_parse_ctx *ctx, bool unrestricted)
+{
+	/* each token type falls into one of three categories:
+	 * - cmdcall item: the token can be used as part of a command call. the
+	 *   token indicates the start of a command call.
+	 * - NOT a cmdcall item: the token cannot be used as part of a command
+	 *   call, usually because it as a cmdcall operator like | or &.
+	 *   encountering one of these tokens ends the cmdcall currently being
+	 *   parsed.
+	 * - RESTRICTED cmdcall item: the token can be used as part of a
+	 *   command, but will not be considered the start of a cmdcall. to run
+	 *   a command with this token as its name, the call operator must be
+	 *   used.
+	 */
+	switch (peek_token_type(ctx)) {
+	case BSHELL_TOK_KEYWORD:
+	case BSHELL_TOK_INT:
+	case BSHELL_TOK_DOUBLE:
+	case BSHELL_TOK_VAR:
+	case BSHELL_TOK_VAR_SPLAT:
+	case BSHELL_TOK_STRING:
+	case BSHELL_TOK_WORD_START:
+		return unrestricted;
+	case BSHELL_TOK_SYMBOL:
+		switch (peek_unknown_symbol(ctx)) {
+		case BSHELL_SYM_PLUS:
+		case BSHELL_SYM_HYPHEN:
+			return unrestricted;
+		case BSHELL_SYM_PIPE:
+		case BSHELL_SYM_AMPERSAND:
+		case BSHELL_SYM_SEMICOLON:
+		case BSHELL_SYM_RIGHT_PAREN:
+		case BSHELL_SYM_RIGHT_BRACE:
+		case BSHELL_SYM_RIGHT_BRACKET:
+			return false;
+		default:
+			return true;
+		}
+	case BSHELL_TOK_NONE:
+	case BSHELL_TOK_LINEFEED:
+		return false;
+	default:
+		return true;
+	}
+}
+
+bool parse_cmdcall(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	struct bshell_cmdcall_ast_node *node
+		= (struct bshell_cmdcall_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_CMDCALL);
+	if (!node) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		return false;
+	}
+
+	struct bshell_ast_node *child = NULL;
+	bool unrestricted = false;
+	bool ok = true;
+	bool stop = false;
+
+	if (parse_symbol(ctx, BSHELL_SYM_AMPERSAND)) {
+		unrestricted = true;
+	}
+
+	if (!peek_cmdcall_item(ctx, unrestricted)) {
+		return false;
+	}
+
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (!parse_cmdcall_arg(ctx, &child)) {
+		return false;
+	}
+
+	fx_queue_push_back(&node->n_args, &child->n_entry);
+
+	while (ok && !stop) {
+		if (!peek_cmdcall_item(ctx, true)) {
+			break;
+		}
+
+		struct bshell_lex_token *tok = peek_token(ctx);
+		if (!tok) {
+			break;
+		}
+
+		if (parse_redirect(ctx, &child)) {
+			fx_queue_push_back(&node->n_redirect, &child->n_entry);
+		} else if (parse_cmdcall_arg(ctx, &child)) {
+			fx_queue_push_back(&node->n_args, &child->n_entry);
+		} else {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+			ok = false;
+			break;
+		}
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)node);
+		node = NULL;
+	}
+
+	*out = (struct bshell_ast_node *)node;
+	return ok;
+}
+
+bool peek_command(struct bshell_parse_ctx *ctx)
+{
+	if (peek_symbol(ctx, BSHELL_SYM_AMPERSAND)) {
+		return true;
+	}
+
+	return peek_cmdcall_item(ctx, false);
+}
+
+bool parse_command(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	struct bshell_ast_node *cmdcall = NULL;
+	if (!parse_cmdcall(ctx, &cmdcall)) {
+		return false;
+	}
+
+	struct bshell_pipeline_ast_node *pipeline = NULL;
+
+	while (1) {
+		if (peek_symbol(ctx, BSHELL_SYM_SEMICOLON)
+		    || peek_linefeed(ctx)) {
+			break;
+		}
+
+		if (!parse_symbol(ctx, BSHELL_SYM_PIPE)) {
+			break;
+		}
+
+		if (!pipeline) {
+			pipeline = (struct bshell_pipeline_ast_node *)
+				bshell_ast_node_create(BSHELL_AST_PIPELINE);
+			if (!pipeline) {
+				ctx->p_status = BSHELL_ERR_NO_MEMORY;
+				bshell_ast_node_destroy(cmdcall);
+				return false;
+			}
+
+			fx_queue_push_back(
+				&pipeline->n_stages,
+				&cmdcall->n_entry);
+		}
+
+		if (!parse_cmdcall(ctx, &cmdcall)) {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+			return false;
+		}
+
+		fx_queue_push_back(&pipeline->n_stages, &cmdcall->n_entry);
+	}
+
+	if (pipeline) {
+		*out = (struct bshell_ast_node *)pipeline;
+	} else {
+		*out = cmdcall;
+	}
+
+	return true;
+}
+
+bool parse_pipeline(
+	struct bshell_parse_ctx *ctx,
+	struct bshell_ast_node *first_item,
+	struct bshell_ast_node **out)
+{
+	struct bshell_pipeline_ast_node *pipeline
+		= (struct bshell_pipeline_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_PIPELINE);
+
+	fx_queue_push_back(&pipeline->n_stages, &first_item->n_entry);
+
+	while (1) {
+		if (!parse_symbol(ctx, BSHELL_SYM_PIPE)) {
+			break;
+		}
+
+		struct bshell_ast_node *cmdcall = NULL;
+		if (!parse_cmdcall(ctx, &cmdcall)) {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+			return false;
+		}
+
+		fx_queue_push_back(&pipeline->n_stages, &cmdcall->n_entry);
+	}
+
+	*out = (struct bshell_ast_node *)pipeline;
+	return true;
+}

bshell.runtime/parse/syntax/expr.c

@@ -0,0 +1,15 @@
+#include "../syntax.h"
+
+bool parse_expr(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	bool ok = false;
+	if (!ok && peek_arith_expr(ctx)) {
+		ok = parse_arith_expr(ctx, BSHELL_PRECEDENCE_MINIMUM, out);
+	}
+
+	if (!ok && peek_command(ctx)) {
+		ok = parse_command(ctx, out);
+	}
+
+	return ok;
+}

bshell.runtime/parse/syntax/func.c

@@ -0,0 +1,88 @@
+#include "../syntax.h"
+
+bool parse_func(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_keyword(ctx, BSHELL_KW_FUNC)) {
+		return false;
+	}
+
+	struct bshell_lex_token *name = NULL;
+	if (!parse_word(ctx, &name)) {
+		report_error(ctx, "expected function identifier");
+		return false;
+	}
+
+	struct bshell_func_ast_node *func
+		= (struct bshell_func_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_FUNC);
+	if (!func) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		bshell_lex_token_destroy(name);
+		return false;
+	}
+
+	func->n_name = name;
+
+	if (!parse_symbol(ctx, BSHELL_SYM_LEFT_PAREN)) {
+		report_error(ctx, "expected `(` after function identifier");
+		bshell_ast_node_destroy((struct bshell_ast_node *)func);
+		return false;
+	}
+
+	size_t nr_args = 0;
+	bool ok = true;
+	while (1) {
+		if (parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+			break;
+		}
+
+		if (nr_args > 0 && !parse_symbol(ctx, BSHELL_SYM_COMMA)) {
+			report_error(
+				ctx,
+				"expected `,` or `)` after parameter name");
+			ok = false;
+			break;
+		}
+
+		struct bshell_lex_token *param_token = NULL;
+		struct bshell_var_ast_node *param_node = NULL;
+		if (!parse_var(ctx, &param_token)) {
+			report_error(ctx, "expected parameter variable");
+			ok = false;
+			break;
+		}
+
+		param_node
+			= (struct bshell_var_ast_node *)bshell_ast_node_create(
+				BSHELL_AST_VAR);
+		if (!param_node) {
+			ok = false;
+			ctx->p_status = BSHELL_ERR_NO_MEMORY;
+			bshell_lex_token_destroy(param_token);
+			break;
+		}
+
+		param_node->n_ident = param_token;
+		fx_queue_push_back(
+			&func->n_params,
+			&param_node->n_base.n_entry);
+	}
+
+	if (!ok) {
+		if (ctx->p_status == BSHELL_SUCCESS) {
+			ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+		}
+
+		bshell_ast_node_destroy((struct bshell_ast_node *)func);
+		return false;
+	}
+
+	if (!parse_block(ctx, &func->n_body)) {
+		report_error(ctx, "failed to parse function body");
+		bshell_ast_node_destroy((struct bshell_ast_node *)func);
+		return false;
+	}
+
+	*out = (struct bshell_ast_node *)func;
+	return true;
+}

bshell.runtime/parse/syntax/generic.c

@@ -0,0 +1,158 @@
+#include "../syntax.h"
+
+#include <bshell/parse/lex.h>
+#include <bshell/parse/parse.h>
+#include <bshell/parse/token.h>
+
+struct bshell_lex_token *claim_token(struct bshell_parse_ctx *ctx)
+{
+	return bshell_lex_ctx_claim(ctx->p_src);
+}
+
+void discard_token(struct bshell_parse_ctx *ctx)
+{
+	return bshell_lex_ctx_discard(ctx->p_src);
+}
+
+struct bshell_lex_token *peek_token(struct bshell_parse_ctx *ctx)
+{
+	return bshell_lex_ctx_peek(ctx->p_src);
+}
+
+enum bshell_lex_token_type peek_token_type(struct bshell_parse_ctx *ctx)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	return tok ? tok->tok_type : BSHELL_TOK_NONE;
+}
+
+enum bshell_lex_symbol peek_unknown_symbol(struct bshell_parse_ctx *ctx)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	return (tok && tok->tok_type == BSHELL_TOK_SYMBOL) ? tok->tok_symbol
+	                                                   : BSHELL_SYM_NONE;
+}
+
+enum bshell_lex_keyword peek_unknown_keyword(struct bshell_parse_ctx *ctx)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	return (tok && tok->tok_type == BSHELL_TOK_KEYWORD) ? tok->tok_keyword
+	                                                    : BSHELL_KW_NONE;
+}
+
+bool peek_word(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (tok && tok->tok_type == BSHELL_TOK_WORD) {
+		*out = tok;
+		return true;
+	}
+
+	return false;
+}
+
+bool peek_linefeed(struct bshell_parse_ctx *ctx)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (tok && tok->tok_type == BSHELL_TOK_LINEFEED) {
+		return true;
+	}
+
+	return false;
+}
+
+bool peek_symbol(struct bshell_parse_ctx *ctx, enum bshell_lex_symbol sym)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != BSHELL_TOK_SYMBOL) {
+		return false;
+	}
+
+	if (tok->tok_symbol != sym) {
+		return false;
+	}
+
+	return true;
+}
+
+bool parse_linefeed(struct bshell_parse_ctx *ctx)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (tok && tok->tok_type == BSHELL_TOK_LINEFEED) {
+		discard_token(ctx);
+		return true;
+	}
+
+	return false;
+}
+
+bool parse_symbol(struct bshell_parse_ctx *ctx, enum bshell_lex_symbol sym)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != BSHELL_TOK_SYMBOL) {
+		return false;
+	}
+
+	if (tok->tok_symbol != sym) {
+		return false;
+	}
+
+	discard_token(ctx);
+	return true;
+}
+
+bool parse_keyword(struct bshell_parse_ctx *ctx, enum bshell_lex_keyword kw)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != BSHELL_TOK_KEYWORD) {
+		return false;
+	}
+
+	if (tok->tok_keyword != kw) {
+		return false;
+	}
+
+	discard_token(ctx);
+	return true;
+}
+
+bool parse_word(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != BSHELL_TOK_WORD) {
+		return false;
+	}
+
+	*out = claim_token(ctx);
+	return true;
+}
+
+bool parse_var(struct bshell_parse_ctx *ctx, struct bshell_lex_token **out)
+{
+	struct bshell_lex_token *tok = peek_token(ctx);
+	if (!tok) {
+		return false;
+	}
+
+	if (tok->tok_type != BSHELL_TOK_VAR) {
+		return false;
+	}
+
+	*out = claim_token(ctx);
+	return true;
+}

bshell.runtime/parse/syntax/if-else.c

@@ -0,0 +1,130 @@
+#include "../syntax.h"
+
+static bool add_branch(
+	struct bshell_if_ast_node *group,
+	struct bshell_ast_node *cond,
+	struct bshell_ast_node *body)
+{
+	struct bshell_if_branch_ast_node *branch
+		= (struct bshell_if_branch_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_IF_BRANCH);
+	if (!branch) {
+		return false;
+	}
+
+	branch->n_cond = cond;
+	branch->n_body = body;
+	fx_queue_push_back(&group->n_branches, &branch->n_base.n_entry);
+
+	return true;
+}
+
+bool parse_if(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!parse_keyword(ctx, BSHELL_KW_IF)) {
+		return false;
+	}
+
+	if (!parse_symbol(ctx, BSHELL_SYM_LEFT_PAREN)) {
+		report_error(ctx, "expected `(` after `if`");
+		return false;
+	}
+
+	struct bshell_ast_node *if_cond = NULL, *if_body = NULL;
+	if (!parse_expr(ctx, &if_cond)) {
+		report_error(ctx, "invalid if condition");
+		return false;
+	}
+
+	if (!parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+		report_error(ctx, "expected `)` after if-condition");
+		bshell_ast_node_destroy(if_cond);
+		return false;
+	}
+
+	if (!parse_block(ctx, &if_body)) {
+		report_error(ctx, "invalid if body");
+		bshell_ast_node_destroy(if_cond);
+		return false;
+	}
+
+	struct bshell_if_ast_node *if_group
+		= (struct bshell_if_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_IF);
+	if (!if_group) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		bshell_ast_node_destroy(if_cond);
+		bshell_ast_node_destroy(if_body);
+		return false;
+	}
+
+	if (!add_branch(if_group, if_cond, if_body)) {
+		ctx->p_status = BSHELL_ERR_NO_MEMORY;
+		bshell_ast_node_destroy(if_cond);
+		bshell_ast_node_destroy(if_body);
+		bshell_ast_node_destroy((struct bshell_ast_node *)if_group);
+		return false;
+	}
+
+	bool done = false;
+	while (!done) {
+		struct bshell_ast_node *cond = NULL, *body = NULL;
+		if (parse_keyword(ctx, BSHELL_KW_ELSE)) {
+			done = true;
+		} else if (parse_keyword(ctx, BSHELL_KW_ELSEIF)) {
+			if (!parse_symbol(ctx, BSHELL_SYM_LEFT_PAREN)) {
+				report_error(
+					ctx,
+					"expected `(` after `elseif`");
+				return false;
+			}
+
+			if (!parse_expr(ctx, &cond)) {
+				report_error(
+					ctx,
+					"invalid conditional expression");
+				bshell_ast_node_destroy(
+					(struct bshell_ast_node *)if_group);
+				return false;
+			}
+
+			if (!parse_symbol(ctx, BSHELL_SYM_RIGHT_PAREN)) {
+				report_error(
+					ctx,
+					"expected `)` after elseif-condition");
+				bshell_ast_node_destroy(if_cond);
+				return false;
+			}
+		} else {
+			done = true;
+			break;
+		}
+
+		if (!parse_block(ctx, &body)) {
+			report_error(ctx, "invalid conditional body");
+			if (cond) {
+				bshell_ast_node_destroy(cond);
+			}
+
+			bshell_ast_node_destroy(
+				(struct bshell_ast_node *)if_group);
+			return false;
+		}
+
+		if (!add_branch(if_group, cond, body)) {
+			report_error(ctx, "failed to add branch to if-group");
+			if (cond) {
+				bshell_ast_node_destroy(cond);
+			}
+
+			bshell_ast_node_destroy(body);
+			bshell_ast_node_destroy(
+				(struct bshell_ast_node *)if_group);
+			return false;
+		}
+	}
+
+	*out = (struct bshell_ast_node *)if_group;
+
+	return true;
+}

bshell.runtime/parse/syntax/keyword.c

@@ -0,0 +1,21 @@
+#include "../syntax.h"
+
+bool peek_keyword_expr(struct bshell_parse_ctx *ctx)
+{
+	return peek_unknown_keyword(ctx) != BSHELL_KW_NONE;
+}
+
+bool parse_keyword_expr(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	switch (peek_unknown_keyword(ctx)) {
+	case BSHELL_KW_NONE:
+		return false;
+	case BSHELL_KW_IF:
+		return parse_if(ctx, out);
+	case BSHELL_KW_FUNC:
+		return parse_func(ctx, out);
+	default:
+		ctx->p_status = BSHELL_ERR_BAD_SYNTAX;
+		return false;
+	}
+}

bshell.runtime/parse/syntax/statement.c

@@ -0,0 +1,100 @@
+#include "../syntax.h"
+
+bool peek_statement(struct bshell_parse_ctx *ctx)
+{
+	if (peek_keyword_expr(ctx)) {
+		return true;
+	}
+
+	if (peek_arith_expr(ctx)) {
+		return true;
+	}
+
+	if (peek_command(ctx)) {
+		return true;
+	}
+
+	return false;
+}
+
+bool parse_statement(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	if (!peek_token(ctx)) {
+		/* error, or EOF */
+		return false;
+	}
+
+	bool unknown = true;
+	bool ok = false;
+	if (peek_keyword_expr(ctx)) {
+		unknown = false;
+		ok = parse_keyword_expr(ctx, out);
+	}
+
+	if (!ok && peek_arith_expr(ctx)) {
+		unknown = false;
+		ok = parse_arith_expr(ctx, BSHELL_PRECEDENCE_MINIMUM, out);
+	}
+
+	if (!ok && peek_command(ctx)) {
+		unknown = false;
+		ok = parse_command(ctx, out);
+	}
+
+	if (!ok && unknown) {
+		report_error(
+			ctx,
+			"encountered unknown token while parsing statement");
+		return false;
+	}
+
+	return ok;
+}
+
+static struct bshell_ast_node *convert_single_statement(
+	struct bshell_stmt_list_ast_node *list)
+{
+	fx_queue_entry *first_entry = fx_queue_first(&list->n_statements);
+	if (!first_entry || fx_queue_next(first_entry)) {
+		return (struct bshell_ast_node *)list;
+	}
+
+	fx_queue_delete(&list->n_statements, first_entry);
+	struct bshell_ast_node *first
+		= fx_unbox(struct bshell_ast_node, first_entry, n_entry);
+	bshell_ast_node_destroy((struct bshell_ast_node *)list);
+
+	return first;
+}
+
+bool parse_statement_list(struct bshell_parse_ctx *ctx, struct bshell_ast_node **out)
+{
+	struct bshell_stmt_list_ast_node *stmt_list
+		= (struct bshell_stmt_list_ast_node *)bshell_ast_node_create(
+			BSHELL_AST_STMT_LIST);
+
+	bool ok = true;
+	while (ok) {
+		parse_linefeed(ctx);
+
+		struct bshell_ast_node *stmt = NULL;
+		if (!parse_statement(ctx, &stmt)) {
+			ok = false;
+			break;
+		}
+
+		fx_queue_push_back(&stmt_list->n_statements, &stmt->n_entry);
+
+		if (!parse_symbol(ctx, BSHELL_SYM_SEMICOLON)) {
+			break;
+		}
+	}
+
+	if (!ok) {
+		bshell_ast_node_destroy((struct bshell_ast_node *)stmt_list);
+		return false;
+	}
+
+	*out = convert_single_statement(stmt_list);
+	return true;
+}

bshell.runtime/parse/token.c

@@ -0,0 +1,213 @@
+#include <bshell/parse/token.h>
+#include <fx/string.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct bshell_lex_token *bshell_lex_token_create(
+	enum bshell_lex_token_type type)
+{
+	struct bshell_lex_token *out = malloc(sizeof *out);
+	if (!out) {
+		return NULL;
+	}
+
+	memset(out, 0x0, sizeof *out);
+
+	out->tok_type = type;
+
+	return out;
+}
+
+struct bshell_lex_token *bshell_lex_token_create_with_string(
+	enum bshell_lex_token_type type,
+	const char *s)
+{
+	struct bshell_lex_token *tok = bshell_lex_token_create(type);
+	if (!tok) {
+		return NULL;
+	}
+
+	tok->tok_str = fx_strdup(s);
+	if (!tok->tok_str) {
+		free(tok);
+		return NULL;
+	}
+
+	return tok;
+}
+
+void bshell_lex_token_destroy(struct bshell_lex_token *tok)
+{
+	switch (tok->tok_type) {
+	case BSHELL_TOK_WORD:
+	case BSHELL_TOK_FLAG:
+	case BSHELL_TOK_STRING:
+		if (tok->tok_str) {
+			free(tok->tok_str);
+		}
+
+		break;
+	default:
+		break;
+	}
+
+	free(tok);
+}
+
+struct bshell_lex_token *bshell_lex_token_change_type(
+	struct bshell_lex_token *tok,
+	enum bshell_lex_token_type new_type)
+{
+	switch (tok->tok_type) {
+	case BSHELL_TOK_WORD:
+	case BSHELL_TOK_FLAG:
+	case BSHELL_TOK_STRING:
+		if (tok->tok_str) {
+			free(tok->tok_str);
+			tok->tok_str = NULL;
+		}
+		break;
+	default:
+		break;
+	}
+
+	tok->tok_type = new_type;
+	return tok;
+}
+
+void bshell_lex_token_change_string(struct bshell_lex_token *tok, const char *s)
+{
+	if (!bshell_lex_token_has_string_value(tok)) {
+		return;
+	}
+
+	if (tok->tok_str) {
+		free(tok->tok_str);
+	}
+
+	tok->tok_str = fx_strdup(s);
+}
+
+#define ENUM_STR(x)                                                            \
+	case x:                                                                \
+		return #x
+
+const char *bshell_token_type_to_string(enum bshell_lex_token_type type)
+{
+	switch (type) {
+		ENUM_STR(BSHELL_TOK_NONE);
+		ENUM_STR(BSHELL_TOK_KEYWORD);
+		ENUM_STR(BSHELL_TOK_SYMBOL);
+		ENUM_STR(BSHELL_TOK_INT);
+		ENUM_STR(BSHELL_TOK_DOUBLE);
+		ENUM_STR(BSHELL_TOK_WORD);
+		ENUM_STR(BSHELL_TOK_WORD_START);
+		ENUM_STR(BSHELL_TOK_WORD_END);
+		ENUM_STR(BSHELL_TOK_OPERATOR);
+		ENUM_STR(BSHELL_TOK_VAR);
+		ENUM_STR(BSHELL_TOK_VAR_SPLAT);
+		ENUM_STR(BSHELL_TOK_FLAG);
+		ENUM_STR(BSHELL_TOK_STRING);
+		ENUM_STR(BSHELL_TOK_STR_START);
+		ENUM_STR(BSHELL_TOK_STR_END);
+		ENUM_STR(BSHELL_TOK_LINEFEED);
+	default:
+		return "<unknown>";
+	}
+}
+
+const char *bshell_lex_keyword_to_string(enum bshell_lex_keyword keyword)
+{
+	switch (keyword) {
+		ENUM_STR(BSHELL_KW_NONE);
+		ENUM_STR(BSHELL_KW_FUNC);
+		ENUM_STR(BSHELL_KW_IF);
+		ENUM_STR(BSHELL_KW_ELSEIF);
+		ENUM_STR(BSHELL_KW_ELSE);
+	default:
+		return "<unknown>";
+	}
+}
+
+const char *bshell_lex_symbol_to_string(enum bshell_lex_symbol sym)
+{
+	switch (sym) {
+		ENUM_STR(BSHELL_SYM_NONE);
+		ENUM_STR(BSHELL_SYM_PLUS);
+		ENUM_STR(BSHELL_SYM_HYPHEN);
+		ENUM_STR(BSHELL_SYM_FORWARD_SLASH);
+		ENUM_STR(BSHELL_SYM_ASTERISK);
+		ENUM_STR(BSHELL_SYM_AMPERSAND);
+		ENUM_STR(BSHELL_SYM_PERCENT);
+		ENUM_STR(BSHELL_SYM_SQUOTE);
+		ENUM_STR(BSHELL_SYM_DQUOTE);
+		ENUM_STR(BSHELL_SYM_HASH);
+		ENUM_STR(BSHELL_SYM_COLON_COLON);
+		ENUM_STR(BSHELL_SYM_SEMICOLON);
+		ENUM_STR(BSHELL_SYM_COMMA);
+		ENUM_STR(BSHELL_SYM_DOLLAR);
+		ENUM_STR(BSHELL_SYM_DOLLAR_LEFT_PAREN);
+		ENUM_STR(BSHELL_SYM_DOLLAR_LEFT_BRACE);
+		ENUM_STR(BSHELL_SYM_DOT);
+		ENUM_STR(BSHELL_SYM_DOT_DOT);
+		ENUM_STR(BSHELL_SYM_PIPE);
+		ENUM_STR(BSHELL_SYM_AT);
+		ENUM_STR(BSHELL_SYM_AT_LEFT_PAREN);
+		ENUM_STR(BSHELL_SYM_AT_LEFT_BRACE);
+		ENUM_STR(BSHELL_SYM_LEFT_BRACE);
+		ENUM_STR(BSHELL_SYM_RIGHT_BRACE);
+		ENUM_STR(BSHELL_SYM_LEFT_BRACKET);
+		ENUM_STR(BSHELL_SYM_RIGHT_BRACKET);
+		ENUM_STR(BSHELL_SYM_LEFT_PAREN);
+		ENUM_STR(BSHELL_SYM_RIGHT_PAREN);
+		ENUM_STR(BSHELL_SYM_EQUAL);
+		ENUM_STR(BSHELL_SYM_PLUS_EQUAL);
+		ENUM_STR(BSHELL_SYM_HYPHEN_EQUAL);
+		ENUM_STR(BSHELL_SYM_ASTERISK_EQUAL);
+		ENUM_STR(BSHELL_SYM_FORWARD_SLASH_EQUAL);
+		ENUM_STR(BSHELL_SYM_PERCENT_EQUAL);
+		ENUM_STR(BSHELL_SYM_QUESTION_DOT);
+		ENUM_STR(BSHELL_SYM_QUESTION_LEFT_BRACKET);
+	default:
+		return "<unknown>";
+	}
+}
+
+const char *bshell_lex_operator_to_string(enum bshell_lex_operator op)
+{
+	switch (op) {
+		ENUM_STR(BSHELL_TKOP_BAND);
+		ENUM_STR(BSHELL_TKOP_BOR);
+		ENUM_STR(BSHELL_TKOP_BXOR);
+		ENUM_STR(BSHELL_TKOP_BNOT);
+		ENUM_STR(BSHELL_TKOP_SHL);
+		ENUM_STR(BSHELL_TKOP_SHR);
+		ENUM_STR(BSHELL_TKOP_EQ);
+		ENUM_STR(BSHELL_TKOP_NE);
+		ENUM_STR(BSHELL_TKOP_GT);
+		ENUM_STR(BSHELL_TKOP_LT);
+		ENUM_STR(BSHELL_TKOP_GE);
+		ENUM_STR(BSHELL_TKOP_LE);
+		ENUM_STR(BSHELL_TKOP_MATCH);
+		ENUM_STR(BSHELL_TKOP_NOTMATCH);
+		ENUM_STR(BSHELL_TKOP_REPLACE);
+		ENUM_STR(BSHELL_TKOP_LIKE);
+		ENUM_STR(BSHELL_TKOP_NOTLIKE);
+		ENUM_STR(BSHELL_TKOP_IN);
+		ENUM_STR(BSHELL_TKOP_F);
+		ENUM_STR(BSHELL_TKOP_NOTIN);
+		ENUM_STR(BSHELL_TKOP_CONTAINS);
+		ENUM_STR(BSHELL_TKOP_NOTCONTAINS);
+		ENUM_STR(BSHELL_TKOP_AND);
+		ENUM_STR(BSHELL_TKOP_OR);
+		ENUM_STR(BSHELL_TKOP_XOR);
+		ENUM_STR(BSHELL_TKOP_NOT);
+		ENUM_STR(BSHELL_TKOP_SPLIT);
+		ENUM_STR(BSHELL_TKOP_JOIN);
+		ENUM_STR(BSHELL_TKOP_IS);
+		ENUM_STR(BSHELL_TKOP_ISNOT);
+		ENUM_STR(BSHELL_TKOP_AS);
+	default:
+		return "<unknown>";
+	}
+}