bshell/bshell/parse/lex/lex.c

#include "../lex.h"

#include "../../debug.h"
#include "../../line-source.h"
#include "../token.h"
#include "lex-internal.h"

#define LEX_TOKEN_DEF(i, n, s) {.id = (i), .name = (n), .enabled_states = (s)}

#define CONVERSION_REQUESTED(flags)                                            \
	((flags) & (LEX_ENABLE_INT | LEX_ENABLE_KEYWORD))

static struct lex_token_def keywords[] = {
	LEX_TOKEN_DEF(KW_FUNC, "func", LEX_STATE_STATEMENT),
	LEX_TOKEN_DEF(KW_IF, "if", LEX_STATE_STATEMENT),
	LEX_TOKEN_DEF(KW_ELSE, "else", LEX_STATE_STATEMENT),
};
static const size_t nr_keywords = sizeof keywords / sizeof keywords[0];

#define LEX_STATES(states) (LEX_STATE_STATEMENT | LEX_STATE_EXPRESSION | states)
#define LEX_STATE_ALL                                                          \
	(LEX_STATE_ARITHMETIC | LEX_STATE_STATEMENT | LEX_STATE_COMMAND        \
	 | LEX_STATE_STRING | LEX_STATE_EXPRESSION)

static struct lex_token_def symbols[] = {
	LEX_TOKEN_DEF(SYM_PLUS, "+", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(SYM_HYPHEN, "-", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(SYM_FORWARD_SLASH, "/", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(SYM_ASTERISK, "*", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(
		SYM_AMPERSAND,
		"&",
		LEX_STATES(LEX_STATE_ARITHMETIC | LEX_STATE_COMMAND)),
	LEX_TOKEN_DEF(SYM_PERCENT, "%", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(
		SYM_SQUOTE,
		"'",
		LEX_STATES(LEX_STATE_ARITHMETIC | LEX_STATE_COMMAND)),
	LEX_TOKEN_DEF(SYM_DQUOTE, "\"", LEX_STATE_ALL),
	LEX_TOKEN_DEF(
		SYM_HASH,
		"#",
		LEX_STATES(LEX_STATE_ARITHMETIC | LEX_STATE_COMMAND)),
	LEX_TOKEN_DEF(
		SYM_DOLLAR,
		"$",
		LEX_STATES(
			LEX_STATE_ARITHMETIC | LEX_STATE_COMMAND
			| LEX_STATE_STRING)),
	LEX_TOKEN_DEF(SYM_DOLLAR_LEFT_PAREN, "$(", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_DOLLAR_LEFT_BRACE, "${", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_AT, "@", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_PIPE, "|", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_COMMA, ",", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_SEMICOLON, ";", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_AT_LEFT_BRACE, "@{", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_LEFT_BRACE, "{", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_RIGHT_BRACE, "}", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_LEFT_BRACKET, "[", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(SYM_RIGHT_BRACKET, "]", LEX_STATES(LEX_STATE_ARITHMETIC)),
	LEX_TOKEN_DEF(SYM_LEFT_PAREN, "(", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_RIGHT_PAREN, ")", LEX_STATE_ALL),
	LEX_TOKEN_DEF(SYM_EQUAL, "=", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(SYM_PLUS_EQUAL, "+=", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(SYM_HYPHEN_EQUAL, "-=", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(SYM_FORWARD_SLASH_EQUAL, "/=", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(SYM_ASTERISK_EQUAL, "*=", LEX_STATE_ARITHMETIC),
	LEX_TOKEN_DEF(SYM_PERCENT_EQUAL, "%=", LEX_STATE_ARITHMETIC),
};
static const size_t nr_symbols = sizeof symbols / sizeof symbols[0];

extern const struct lex_state_type lex_statement_state;
extern const struct lex_state_type lex_expression_state;
extern const struct lex_state_type lex_command_state;
extern const struct lex_state_type lex_arithmetic_state;
extern const struct lex_state_type lex_string_state;

static const struct lex_state_type *state_types[] = {
	[LEX_STATE_STATEMENT] = &lex_statement_state,
	[LEX_STATE_EXPRESSION] = &lex_expression_state,
	[LEX_STATE_COMMAND] = &lex_command_state,
	[LEX_STATE_ARITHMETIC] = &lex_arithmetic_state,
	[LEX_STATE_STRING] = &lex_string_state,
};

struct lex_state *lex_state_push(
	struct lex_ctx *ctx,
	enum lex_state_type_id state_type)
{
	struct lex_state *state = malloc(sizeof *state);
	if (!state) {
		return NULL;
	}

	memset(state, 0x0, sizeof *state);

	state->s_type = state_types[state_type];
	fx_queue_push_back(&ctx->lex_state, &state->s_entry);

	if (state->s_type->s_begin) {
		state->s_type->s_begin(ctx);
	}

	return state;
}

void lex_state_pop(struct lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_last(&ctx->lex_state);
	if (!entry || !fx_queue_prev(entry)) {
		/* don't pop if this is the root state */
		return;
	}

	struct lex_state *state = fx_unbox(struct lex_state, entry, s_entry);

	if (state->s_type->s_end) {
		state->s_type->s_end(ctx);
	}

	fx_queue_pop_back(&ctx->lex_state);

	if (state->s_tempstr) {
		fx_string_unref(state->s_tempstr);
	}

	free(state);
}

struct lex_state *lex_state_get(struct lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_last(&ctx->lex_state);
	if (!entry) {
		return NULL;
	}

	return fx_unbox(struct lex_state, entry, s_entry);
}

void lex_state_change(struct lex_ctx *ctx, enum lex_state_type_id type)
{
	struct lex_state *state = lex_state_get(ctx);
	if (!state) {
		return;
	}

	if (state->s_type->s_end) {
		state->s_type->s_end(ctx);
	}

	state->s_type = state_types[type];

	if (state->s_type->s_begin) {
		state->s_type->s_begin(ctx);
	}
}

fx_string *lex_state_get_tempstr(struct lex_ctx *ctx)
{
	struct lex_state *state = lex_state_get(ctx);
	if (!state) {
		return NULL;
	}

	if (!state->s_tempstr) {
		state->s_tempstr = fx_string_create();
	}

	if (!state->s_tempstr) {
		return NULL;
	}

	return state->s_tempstr;
}

static struct lex_symbol_node *get_symbol_node(
	struct lex_symbol_node *node,
	char c)
{
	fx_queue_entry *entry = fx_queue_first(&node->s_children);
	while (entry) {
		struct lex_symbol_node *child
			= fx_unbox(struct lex_symbol_node, entry, s_entry);
		if (child->s_char == c) {
			return child;
		}

		entry = fx_queue_next(entry);
	}

	return NULL;
}

static enum bshell_status put_symbol(
	struct lex_symbol_node *tree,
	struct lex_token_def *sym)
{
	for (size_t i = 0; sym->name[i]; i++) {
		char c = sym->name[i];
		struct lex_symbol_node *child = get_symbol_node(tree, c);
		if (child) {
			tree = child;
			continue;
		}

		child = malloc(sizeof *child);
		if (!child) {
			return BSHELL_ERR_NO_MEMORY;
		}

		memset(child, 0x0, sizeof *child);

		child->s_def = NULL;
		child->s_char = c;

		fx_queue_push_back(&tree->s_children, &child->s_entry);
		tree = child;
	}

	tree->s_def = sym;
	return BSHELL_SUCCESS;
}

static void destroy_symbol_tree(struct lex_symbol_node *tree)
{
	fx_queue_entry *entry = fx_queue_first(&tree->s_children);
	while (entry) {
		struct lex_symbol_node *node
			= fx_unbox(struct lex_symbol_node, entry, s_entry);
		fx_queue_entry *next = fx_queue_next(entry);
		fx_queue_delete(&tree->s_children, entry);

		destroy_symbol_tree(node);

		entry = next;
	}

	free(tree);
}

static struct lex_symbol_node *build_symbol_tree(void)
{
	struct lex_symbol_node *root = malloc(sizeof *root);
	if (!root) {
		return NULL;
	}

	memset(root, 0x0, sizeof *root);
	root->s_def = NULL;

	enum bshell_status status = BSHELL_SUCCESS;
	for (size_t i = 0; i < nr_symbols; i++) {
		status = put_symbol(root, &symbols[i]);

		if (status != BSHELL_SUCCESS) {
			destroy_symbol_tree(root);
			return NULL;
		}
	}

	return root;
}

enum bshell_status lex_ctx_init(
	struct lex_ctx *ctx,
	enum lex_flags flags,
	struct line_source *src)
{
	memset(ctx, 0x0, sizeof *ctx);

	ctx->lex_flags = flags;
	ctx->lex_status = BSHELL_SUCCESS;
	ctx->lex_buf = fx_stringstream_create();
	ctx->lex_sym_tree = build_symbol_tree();
	lex_state_push(ctx, LEX_STATE_STATEMENT);
	ctx->lex_src = src;
	ctx->lex_ch = FX_WCHAR_INVALID;

	return BSHELL_SUCCESS;
}

enum bshell_status lex_ctx_cleanup(struct lex_ctx *ctx)
{
	if (ctx->lex_sym_tree) {
		destroy_symbol_tree(ctx->lex_sym_tree);
	}

	if (ctx->lex_buf) {
		fx_stringstream_unref(ctx->lex_buf);
	}

	if (ctx->lex_tmp) {
		fx_string_unref(ctx->lex_tmp);
	}

	memset(ctx, 0x0, sizeof *ctx);
	return BSHELL_SUCCESS;
}

static enum bshell_status refill_buffer(struct lex_ctx *ctx)
{
	fx_stringstream_reset(ctx->lex_buf);
	ctx->lex_ch = FX_WCHAR_INVALID;

	return line_source_readline(ctx->lex_src, ctx->lex_buf);
}

static fx_wchar __peek_char(struct lex_ctx *ctx, bool noread)
{
	if (ctx->lex_status != BSHELL_SUCCESS) {
		return FX_WCHAR_INVALID;
	}

	if (ctx->lex_ch != FX_WCHAR_INVALID) {
		return ctx->lex_ch;
	}

	fx_status status = fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
	if (!FX_OK(status) && !noread) {
		enum bshell_status status2 = refill_buffer(ctx);
		if (status2 != BSHELL_SUCCESS) {
			ctx->lex_status = status2;
			ctx->lex_ch = FX_WCHAR_INVALID;
		} else {
			fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
		}
	}

	return ctx->lex_ch;
}

fx_wchar peek_char(struct lex_ctx *ctx)
{
	return __peek_char(ctx, false);
}

fx_wchar peek_char_noread(struct lex_ctx *ctx)
{
	return __peek_char(ctx, true);
}

static void __advance_char(struct lex_ctx *ctx, bool noread)
{
	if (ctx->lex_ch != FX_WCHAR_INVALID) {
		ctx->lex_ch = FX_WCHAR_INVALID;
		return;
	}

	if (ctx->lex_status != BSHELL_SUCCESS) {
		return;
	}

	fx_status status = fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
	if (!FX_OK(status) && !noread) {
		enum bshell_status status2 = refill_buffer(ctx);
		if (status2 != BSHELL_SUCCESS) {
			ctx->lex_status = status2;
			ctx->lex_ch = FX_WCHAR_INVALID;
		} else {
			fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
		}
	}
}

void advance_char(struct lex_ctx *ctx)
{
	return __advance_char(ctx, false);
}

void advance_char_noread(struct lex_ctx *ctx)
{
	return __advance_char(ctx, true);
}

bool convert_word_to_keyword(struct lex_token *tok)
{
	if (!lex_token_has_string_value(tok)) {
		return false;
	}

	for (size_t i = 0; i < nr_keywords; i++) {
		const char *kw_str = keywords[i].name;
		if (strcmp(kw_str, tok->tok_str) != 0) {
			continue;
		}

		lex_token_change_type(tok, TOK_KEYWORD);
		tok->tok_keyword = keywords[i].id;
		return true;
	}

	return false;
}

static int get_int_base_by_prefix(const char **s)
{
#define CH(x) (tolower(value[x]))
	const char *value = *s;
	if (CH(0) != '0') {
		return 10;
	}

	switch (CH(1)) {
	case 'x':
		*s += 2;
		return 16;
	case 'b':
		*s += 2;
		return 2;
	default:
		*s += 1;
		return 8;
	}
#undef CH
}

static size_t get_int_multiplier_by_suffix(const char *suffix)
{
#define CH(x) (tolower(suffix[x]))
	if (CH(1) != 'b' || CH(2) != 0) {
		return 0;
	}

	switch (CH(0)) {
	case 'k':
		return 0x400;
	case 'm':
		return 0x100000;
	case 'g':
		return 0x40000000;
	case 't':
		return 0x10000000000;
	case 'b':
		return 0x4000000000000;
	default:
		return 0;
	}

#undef CH
	return 0;
}

bool string_is_valid_number(const char *s, long long *out)
{
	int base = get_int_base_by_prefix(&s);

	char *ep = NULL;
	long long value = strtoll(s, &ep, base);
	if (*ep == '\0') {
		out && (*out = value);
		return true;
	}

	size_t multiplier = get_int_multiplier_by_suffix(ep);
	if (multiplier != 0) {
		out && (*out = value * multiplier);
		return true;
	}

	return false;
}

bool convert_word_to_int(struct lex_token *tok)
{
	if (!lex_token_has_string_value(tok)) {
		return false;
	}

	const char *s = tok->tok_str;
	long long value = 0;
	if (!string_is_valid_number(s, &value)) {
		return false;
	}

	lex_token_change_type(tok, TOK_INT);
	tok->tok_int = value;

	return true;
}

static struct lex_token *get_next_token(struct lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_first(&ctx->lex_tokens);
	return fx_unbox(struct lex_token, entry, tok_entry);
}

void enqueue_token(struct lex_ctx *ctx, struct lex_token *tok)
{
	if (tok && (ctx->lex_flags & LEX_PRINT_TOKENS)) {
		print_lex_token(tok);
	}

	fx_queue_push_back(&ctx->lex_tokens, &tok->tok_entry);
}

static struct lex_token *dequeue_next_token(struct lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_pop_front(&ctx->lex_tokens);
	return fx_unbox(struct lex_token, entry, tok_entry);
}

static fx_string *get_temp_string(struct lex_ctx *ctx)
{
	if (!ctx->lex_tmp) {
		ctx->lex_tmp = fx_string_create();
	}

	fx_string_clear(ctx->lex_tmp);
	return ctx->lex_tmp;
}

enum bshell_status push_symbol(struct lex_ctx *ctx, enum token_symbol sym)
{
	struct lex_token *tok = lex_token_create(TOK_SYMBOL);
	if (!tok) {
		return BSHELL_ERR_NO_MEMORY;
	}

	tok->tok_symbol = sym;
	enqueue_token(ctx, tok);
	return BSHELL_SUCCESS;
}

enum bshell_status read_var(
	struct lex_ctx *ctx,
	enum token_type type,
	struct lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);

	bool done = false;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		bool valid = fx_wchar_is_alnum(c) || (c == '_') || (c == ':');

		if (!valid) {
			break;
		}

		if (done) {
			break;
		}

		fx_string_append_wc(tmp, c);
		advance_char(ctx);
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		return ctx->lex_status;
	}

	struct lex_token *tok
		= lex_token_create_with_string(type, fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_braced_var(
	struct lex_ctx *ctx,
	enum token_type type,
	struct lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);
	bool ok = false;

	while (1) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (c == '}') {
			ok = true;
			advance_char(ctx);
			break;
		}

		fx_string_append_wc(tmp, c);
		advance_char(ctx);
	}

	if (!ok) {
		return BSHELL_ERR_BAD_SYNTAX;
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		return ctx->lex_status;
	}

	struct lex_token *tok
		= lex_token_create_with_string(type, fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

#if 0
static enum bshell_status read_flag(struct lex_ctx *ctx)
{
	fx_string *tmp = get_temp_string(ctx);

	bool done = false;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (fx_wchar_is_space(c)) {
			break;
		}

		switch (c) {
		case '{':
		case '}':
		case '(':
		case ')':
		case ';':
		case ',':
		case '|':
		case '&':
		case '$':
			done = true;
			break;
		default:
			break;
		}

		if (done) {
			break;
		}

		fx_string_append_wc(tmp, c);
		advance_char(ctx);
	}

	struct lex_token *tok = NULL;
	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 1) {
		tok = lex_token_create(TOK_SYMBOL);
		tok->tok_symbol = SYM_HYPHEN;
	} else {
		tok = lex_token_create_with_string(
			TOK_FLAG,
			fx_string_get_cstr(tmp));
	}

	if (!tok) {
		return BSHELL_ERR_NO_MEMORY;
	}

#if 0
	if (convert_word_to_int(tok)) {
		tok->tok_int *= -1;
		struct lex_token *prefix = lex_token_create(TOK_SYMBOL);
		prefix->tok_symbol = SYM_HYPHEN;
		enqueue_token(ctx, prefix);
	}
#endif

	enqueue_token(ctx, tok);
	return BSHELL_SUCCESS;
}

static enum bshell_status read_interpolation_marker(struct lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;
	struct lex_state *state = lex_state_get(ctx);

	struct lex_token *tok = NULL;

	if (state->s_type != LEX_STATE_STRING) {
		return BSHELL_ERR_INTERNAL_FAILURE;
	}

	/* start of a new interpolation */
	if (!lex_state_push(ctx, LEX_STATE_STATEMENT)) {
		return BSHELL_ERR_NO_MEMORY;
	}

	return BSHELL_SUCCESS;
}
#endif

enum bshell_status read_literal_string(
	struct lex_ctx *ctx,
	struct lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);

	bool done = false;
	bool fail = true;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (c == '\'') {
			fail = false;
			done = true;
			advance_char(ctx);
			break;
		}

		fx_string_append_wc(tmp, c);
		advance_char(ctx);
	}

	struct lex_token *tok = lex_token_create_with_string(
		TOK_STRING,
		fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_line_comment(struct lex_ctx *lex)
{
	while (true) {
		fx_wchar c = peek_char(lex);

		if (c == FX_WCHAR_INVALID) {
			break;
		}

		advance_char(lex);

		if (c == '\n') {
			break;
		}
	}

	return BSHELL_SUCCESS;
}

#if 0
enum bshell_status read_dquote_marker(struct lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;
	struct lex_state *state = lex_state_get(ctx);

	struct lex_token *tok = NULL;

	if (state->s_type == LEX_STATE_STRING) {
		/* already within an fstring */
		lex_state_pop(ctx);
		tok = lex_token_create(TOK_STR_END);
		enqueue_token(ctx, tok);
		return BSHELL_SUCCESS;
	}

	/* start of a new fstring */
	tok = lex_token_create(TOK_STR_START);
	enqueue_token(ctx, tok);

	if (!lex_state_push(ctx, LEX_STATE_STRING)) {
		return BSHELL_ERR_NO_MEMORY;
	}

	return BSHELL_SUCCESS;
}
#endif

enum bshell_status read_word(struct lex_ctx *ctx, struct lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);
	bool word_is_number = false;

	bool done = false;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (fx_wchar_is_space(c)) {
			done = true;
			break;
		}

		if (word_is_number && char_can_begin_symbol(ctx, c)) {
			done = true;
			break;
		}

		if (char_can_begin_symbol(ctx, c)) {
			done = true;
			break;
		}

		switch (c) {
		case '{':
		case '}':
		case '(':
		case ')':
		case ';':
		case ',':
		case '|':
		case '&':
		case '$':
			done = true;
			break;
		default:
			break;
		}

		if (done) {
			break;
		}

		fx_string_append_wc(tmp, c);
		word_is_number
			= string_is_valid_number(fx_string_get_cstr(tmp), NULL);
		advance_char(ctx);
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		if (ctx->lex_status == BSHELL_SUCCESS) {
			return BSHELL_ERR_BAD_SYNTAX;
		}

		return ctx->lex_status;
	}

	struct lex_token *tok = lex_token_create_with_string(
		TOK_WORD,
		fx_string_get_cstr(tmp));
#if 0
	bool converted = convert_word_to_keyword(tok);
	if (!converted) {
		converted = convert_word_to_int(tok);
	}
#endif

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_symbol(
	struct lex_ctx *ctx,
	const struct lex_token_def **out)
{
	struct lex_state *state = lex_state_get(ctx);

	struct lex_symbol_node *node = ctx->lex_sym_tree;
	char prev = 0;

	while (true) {
		fx_wchar c = peek_char(ctx);
		if (c < 0) {
			break;
		}

		struct lex_symbol_node *next = get_symbol_node(node, c);
		if (!next
		    || !(next->s_def->enabled_states & state->s_type->s_id)) {
			prev = c;
			break;
		}

		node = next;
		advance_char(ctx);
		prev = c;
	}

	if (!node || node->s_def == NULL) {
		return BSHELL_ERR_BAD_SYNTAX;
	}

#if 0
	struct lex_token *tok = NULL;
	switch (node->s_def->id) {
	case SYM_SQUOTE:
		return read_literal_string(ctx);
	case SYM_DQUOTE:
		return read_dquote_marker(ctx);
	case SYM_DOLLAR_LEFT_PAREN:
		push_symbol(ctx, SYM_DOLLAR_LEFT_PAREN);
		if (state->s_type == LEX_STATE_STRING) {
			lex_state_push(ctx, LEX_STATE_STRING);
		}
		break;
	case SYM_DOLLAR_LEFT_BRACE:
		return read_braced_var(ctx, TOK_VAR);
	case SYM_HASH:
		return read_line_comment(ctx);
	case SYM_LEFT_PAREN:
		push_symbol(ctx, SYM_LEFT_PAREN);
		lex_state_push(ctx, LEX_STATE_EXPRESSION);
		break;
	case SYM_RIGHT_PAREN:
		push_symbol(ctx, SYM_RIGHT_PAREN);
		lex_state_pop(ctx);
		break;
	case SYM_DOLLAR:
		return read_var(ctx, TOK_VAR);
	case SYM_AT:
		return read_var(ctx, TOK_VAR_SPLAT);
	default:
		push_symbol(ctx, node->s_def->id);
		break;
	}
#endif
	*out = node->s_def;
	return BSHELL_SUCCESS;
}

bool char_can_begin_symbol_in_state(
	struct lex_ctx *ctx,
	char c,
	enum lex_state_type_id state_type)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (symbols[i].name[0] != c) {
			continue;
		}

		if (symbols[i].enabled_states & state_type) {
			return true;
		}
	}

	return false;
}

bool char_can_begin_symbol(struct lex_ctx *ctx, char c)
{
	struct lex_state *state = lex_state_get(ctx);
	return char_can_begin_symbol_in_state(ctx, c, state->s_type->s_id);
}

static enum bshell_status read_string_content(struct lex_ctx *ctx)
{
	fx_wchar c = FX_WCHAR_INVALID;
	fx_string *str = get_temp_string(ctx);
	struct lex_state *state = lex_state_get(ctx);

	if (!str) {
		return BSHELL_ERR_NO_MEMORY;
	}

	while (true) {
		c = peek_char(ctx);
		if (c < 0) {
			break;
		}

		if (char_can_begin_symbol(ctx, c)) {
			break;
		}

		fx_string_append_wc(str, c);
		// set_token_end(lex);
		advance_char(ctx);
	}

	if (fx_string_get_size(str, FX_STRLEN_NORMAL) == 0) {
		return BSHELL_SUCCESS;
	}

	struct lex_token *tok = lex_token_create_with_string(
		TOK_STRING,
		fx_string_get_cstr(str));
	enqueue_token(ctx, tok);

	return BSHELL_SUCCESS;
}

#if 0
static enum bshell_status do_pump_token_string(struct lex_ctx *ctx)
{
	fx_wchar c = peek_char(ctx);
	enum bshell_status status = BSHELL_SUCCESS;
	bool ok = false;

	if (char_can_begin_symbol(ctx, c)) {
		status = read_symbol(ctx);
		ok = true;
	}

	if (status != BSHELL_SUCCESS || !ok) {
		status = read_string_content(ctx);
	}

	return status;
}

static enum bshell_status do_pump_token_normal(struct lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;

	fx_wchar c = peek_char(ctx);
	bool newline = false;

	while (fx_wchar_is_space(c)) {
		if (c == '\n') {
			newline = true;
		}

		advance_char_noread(ctx);
		c = peek_char_noread(ctx);
	}

	if (newline) {
		struct lex_token *tok = lex_token_create(TOK_LINEFEED);
		enqueue_token(ctx, tok);
		return BSHELL_SUCCESS;
	}

	if (c == '-') {
		return read_flag(ctx);
	}

	if (char_can_begin_symbol(ctx, c)) {
		return read_symbol(ctx);
	}

	return read_word(ctx);
}
#endif

static enum bshell_status pump_tokens(struct lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;
	while (fx_queue_empty(&ctx->lex_tokens) && status == BSHELL_SUCCESS) {
		struct lex_state *state = lex_state_get(ctx);
		status = state->s_type->s_pump_token(ctx);
	}

	return status;
}

static void discard_all_tokens(struct lex_ctx *ctx)
{
	fx_queue_entry *cur = fx_queue_first(&ctx->lex_tokens);
	while (cur) {
		struct lex_token *tok
			= fx_unbox(struct lex_token, cur, tok_entry);
		if (tok->tok_type == TOK_LINEFEED) {
			break;
		}

		fx_queue_pop_front(&ctx->lex_tokens);
		lex_token_destroy(tok);
		cur = fx_queue_first(&ctx->lex_tokens);
	}
}

struct lex_token *lex_ctx_peek(struct lex_ctx *ctx)
{
	struct lex_token *tok = get_next_token(ctx);
	if (tok) {
		return tok;
	}

	pump_tokens(ctx);
	tok = get_next_token(ctx);

	return tok;
}

struct lex_token *lex_ctx_claim(struct lex_ctx *ctx)
{
	struct lex_token *tok = dequeue_next_token(ctx);
	if (tok) {
		return tok;
	}

	if (fx_queue_empty(&ctx->lex_tokens)) {
		pump_tokens(ctx);

		tok = get_next_token(ctx);
	}

	return dequeue_next_token(ctx);
}

void lex_ctx_discard(struct lex_ctx *ctx)
{
	struct lex_token *tok = dequeue_next_token(ctx);
	if (tok) {
		lex_token_destroy(tok);
		return;
	}

	/* request more tokens if necessary */
	if (fx_queue_empty(&ctx->lex_tokens)) {
		pump_tokens(ctx);
	}
}