bshell/bshell.runtime/parse/lex/lex.c

#include "lex-internal.h"

#include <assert.h>
#include <bshell/line-source.h>
#include <bshell/parse/lex.h>
#include <bshell/parse/token.h>

#define SYMBOL_DEF(i, n, f)                                                    \
	[i - __BSHELL_SYM_INDEX_BASE] = {                                      \
		.id = (i),                                                     \
		.name = (n),                                                   \
		.flags = (f),                                                  \
	}
#define BSHELL_KW_DEF(i, n, f)                                                 \
	[i - __BSHELL_KW_INDEX_BASE] = {                                       \
		.id = (i),                                                     \
		.name = (n),                                                   \
		.flags = (f),                                                  \
	}
#define BSHELL_TKOP_DEF(i, n, f)                                               \
	[i - __BSHELL_TKOP_INDEX_BASE] = {                                     \
		.id = (i),                                                     \
		.name = (n),                                                   \
		.flags = (f),                                                  \
	}

static struct bshell_lex_token_definition keywords[] = {
	BSHELL_KW_DEF(BSHELL_KW_FUNC, "func", BSHELL_LEX_TOKEN_COMMAND_MODE),
	BSHELL_KW_DEF(BSHELL_KW_IF, "if", 0),
	BSHELL_KW_DEF(BSHELL_KW_ELSEIF, "elseif", 0),
	BSHELL_KW_DEF(BSHELL_KW_ELSE, "else", 0),
};
static const size_t nr_keywords = sizeof keywords / sizeof keywords[0];

static struct bshell_lex_token_definition operators[] = {
	BSHELL_TKOP_DEF(BSHELL_TKOP_BAND, "-band", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_BOR, "-bor", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_BXOR, "-bxor", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_BNOT, "-bnot", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_SHL, "-shl", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_SHR, "-shr", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_EQ, "-eq", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_NE, "-ne", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_GT, "-gt", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_LT, "-lt", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_GE, "-ge", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_LE, "-le", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_MATCH, "-match", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_NOTMATCH, "-notmatch", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_REPLACE, "-replace", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_LIKE, "-like", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_NOTLIKE, "-notlike", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_CONTAINS, "-contains", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_NOTCONTAINS, "-notcontains", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_AND, "-and", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_OR, "-or", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_XOR, "-xor", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_NOT, "-not", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_SPLIT, "-split", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_JOIN, "-join", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_IS, "-is", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_ISNOT, "-isnot", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_AS, "-as", 0),
	BSHELL_TKOP_DEF(BSHELL_TKOP_F, "-f", 0),
};
static const size_t nr_operators = sizeof operators / sizeof operators[0];

static struct bshell_lex_token_definition symbols[] = {
	SYMBOL_DEF(BSHELL_SYM_BANG, "!", BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(BSHELL_SYM_PLUS, "+", BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(BSHELL_SYM_HYPHEN, "-", BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(BSHELL_SYM_FORWARD_SLASH, "/", 0),
	SYMBOL_DEF(BSHELL_SYM_ASTERISK, "*", 0),
	SYMBOL_DEF(BSHELL_SYM_AMPERSAND, "&", 0),
	SYMBOL_DEF(BSHELL_SYM_PERCENT, "%", 0),
	SYMBOL_DEF(BSHELL_SYM_SQUOTE, "'", 0),
	SYMBOL_DEF(BSHELL_SYM_DQUOTE, "\"", 0),
	SYMBOL_DEF(BSHELL_SYM_HASH, "#", 0),
	SYMBOL_DEF(BSHELL_SYM_DOLLAR, "$", BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(
		BSHELL_SYM_DOLLAR_LEFT_PAREN,
		"$(",
		BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(
		BSHELL_SYM_DOLLAR_LEFT_BRACE,
		"${",
		BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(BSHELL_SYM_AT, "@", 0),
	SYMBOL_DEF(BSHELL_SYM_PIPE, "|", BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(BSHELL_SYM_COMMA, ",", BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(BSHELL_SYM_SEMICOLON, ";", BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(
		BSHELL_SYM_AT_LEFT_BRACE,
		"@{",
		BSHELL_LEX_TOKEN_UNARY_ARITHMETIC),
	SYMBOL_DEF(BSHELL_SYM_AT_LEFT_PAREN, "@(", 0),
	SYMBOL_DEF(
		BSHELL_SYM_LEFT_BRACE,
		"{",
		BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(
		BSHELL_SYM_RIGHT_BRACE,
		"}",
		BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(BSHELL_SYM_LEFT_BRACKET, "[", 0),
	SYMBOL_DEF(BSHELL_SYM_RIGHT_BRACKET, "]", 0),
	SYMBOL_DEF(BSHELL_SYM_QUESTION_LEFT_BRACKET, "?[", 0),
	SYMBOL_DEF(
		BSHELL_SYM_LEFT_PAREN,
		"(",
		BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(
		BSHELL_SYM_RIGHT_PAREN,
		")",
		BSHELL_LEX_TOKEN_TERMINATES_WORD),
	SYMBOL_DEF(BSHELL_SYM_EQUAL, "=", 0),
	SYMBOL_DEF(BSHELL_SYM_PLUS_EQUAL, "+=", 0),
	SYMBOL_DEF(BSHELL_SYM_HYPHEN_EQUAL, "-=", 0),
	SYMBOL_DEF(BSHELL_SYM_FORWARD_SLASH_EQUAL, "/=", 0),
	SYMBOL_DEF(BSHELL_SYM_ASTERISK_EQUAL, "*=", 0),
	SYMBOL_DEF(BSHELL_SYM_PERCENT_EQUAL, "%=", 0),
	SYMBOL_DEF(BSHELL_SYM_DOT, ".", 0),
	SYMBOL_DEF(BSHELL_SYM_COLON_COLON, "::", 0),
	SYMBOL_DEF(BSHELL_SYM_DOT_DOT, "..", 0),
	SYMBOL_DEF(BSHELL_SYM_QUESTION_DOT, "?.", 0),
};
static const size_t nr_symbols = sizeof symbols / sizeof symbols[0];

extern const struct bshell_lex_state_type lex_statement_state;
extern const struct bshell_lex_state_type lex_command_state;
extern const struct bshell_lex_state_type lex_arithmetic_state;
extern const struct bshell_lex_state_type lex_string_state;
extern const struct bshell_lex_state_type lex_word_state;
extern const struct bshell_lex_state_type lex_hashtable_state;

static const struct bshell_lex_state_type *state_types[] = {
	[BSHELL_LEX_STATE_STATEMENT] = &lex_statement_state,
	[BSHELL_LEX_STATE_COMMAND] = &lex_command_state,
	[BSHELL_LEX_STATE_ARITHMETIC] = &lex_arithmetic_state,
	[BSHELL_LEX_STATE_STRING] = &lex_string_state,
	[BSHELL_LEX_STATE_WORD] = &lex_word_state,
	[BSHELL_LEX_STATE_HASHTABLE] = &lex_hashtable_state,
};

void set_token_start(struct bshell_lex_ctx *ctx)
{
	memcpy(&ctx->lex_start, &ctx->lex_cursor, sizeof ctx->lex_cursor);
}

void set_token_end(struct bshell_lex_ctx *ctx)
{
	memcpy(&ctx->lex_end, &ctx->lex_cursor, sizeof ctx->lex_cursor);
}

static const char *lex_state_type_id_to_string(enum bshell_lex_state_id id)
{
#define ENUM_STR(v)                                                            \
	case v:                                                                \
		return #v
	switch (id) {
		ENUM_STR(BSHELL_LEX_STATE_STATEMENT);
		ENUM_STR(BSHELL_LEX_STATE_COMMAND);
		ENUM_STR(BSHELL_LEX_STATE_ARITHMETIC);
		ENUM_STR(BSHELL_LEX_STATE_STRING);
		ENUM_STR(BSHELL_LEX_STATE_WORD);
		ENUM_STR(BSHELL_LEX_STATE_HASHTABLE);
	default:
		return "<unknown>";
	}
#undef ENUM_STR
}

struct bshell_lex_state *lex_state_push(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_state_id state_type,
	enum state_flags flags)
{
	struct bshell_lex_state *state = malloc(sizeof *state);
	if (!state) {
		return NULL;
	}

#if defined(VERBOSE)
	printf("push(%s, 0x%04x)\n",
	       lex_state_type_id_to_string(state_type),
	       flags);
#endif
	memset(state, 0x0, sizeof *state);

	state->s_type = state_types[state_type];
	state->s_flags = flags;
	fx_queue_push_back(&ctx->lex_state, &state->s_entry);

	if (state->s_type->s_begin) {
		state->s_type->s_begin(ctx);
	}

	return state;
}

void lex_state_pop(struct bshell_lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_last(&ctx->lex_state);
	if (!entry || !fx_queue_prev(entry)) {
		/* don't pop if this is the root state */
		return;
	}

	struct bshell_lex_state *state
		= fx_unbox(struct bshell_lex_state, entry, s_entry);

#if defined(VERBOSE)
	printf("pop(%s) -> %s\n",
	       lex_state_type_id_to_string(state->s_type->s_id),
	       lex_state_type_id_to_string(lex_state_get(ctx)->s_type->s_id));
#endif

	if (state->s_type->s_end) {
		state->s_type->s_end(ctx);
	}

	fx_queue_pop_back(&ctx->lex_state);

	if (state->s_tempstr) {
		fx_string_unref(state->s_tempstr);
	}

	free(state);
}

struct bshell_lex_state *lex_state_get(struct bshell_lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_last(&ctx->lex_state);
	if (!entry) {
		return NULL;
	}

	return fx_unbox(struct bshell_lex_state, entry, s_entry);
}

void lex_state_change(struct bshell_lex_ctx *ctx, enum bshell_lex_state_id type)
{
	struct bshell_lex_state *state = lex_state_get(ctx);
	if (!state) {
		return;
	}

#if defined(VERBOSE)
	printf("change(%s -> %s)\n",
	       lex_state_type_id_to_string(state->s_type->s_id),
	       lex_state_type_id_to_string(type));
#endif

	if (state->s_type->s_end) {
		state->s_type->s_end(ctx);
	}

	state->s_type = state_types[type];

	if (state->s_type->s_begin) {
		state->s_type->s_begin(ctx);
	}
}

fx_string *lex_state_get_tempstr(struct bshell_lex_ctx *ctx)
{
	struct bshell_lex_state *state = lex_state_get(ctx);
	if (!state) {
		return NULL;
	}

	if (!state->s_tempstr) {
		state->s_tempstr = fx_string_create();
	}

	if (!state->s_tempstr) {
		return NULL;
	}

	return state->s_tempstr;
}

void lex_state_add_terminator(struct bshell_lex_state *state, unsigned int tok)
{
	if (state->s_nr_terminators < BSHELL_LEX_STATE_MAX_TERMINATORS) {
		state->s_terminators[state->s_nr_terminators++] = tok;
	}
}

static struct bshell_lex_symbol_node *get_symbol_node(
	struct bshell_lex_symbol_node *node,
	char c)
{
	fx_queue_entry *entry = fx_queue_first(&node->s_children);
	while (entry) {
		struct bshell_lex_symbol_node *child = fx_unbox(
			struct bshell_lex_symbol_node,
			entry,
			s_entry);
		if (child->s_char == c) {
			return child;
		}

		entry = fx_queue_next(entry);
	}

	return NULL;
}

static enum bshell_status put_symbol(
	struct bshell_lex_symbol_node *tree,
	struct bshell_lex_token_definition *sym)
{
	for (size_t i = 0; sym->name[i]; i++) {
		char c = sym->name[i];
		struct bshell_lex_symbol_node *child = get_symbol_node(tree, c);
		if (child) {
			tree = child;
			continue;
		}

		child = malloc(sizeof *child);
		if (!child) {
			return BSHELL_ERR_NO_MEMORY;
		}

		memset(child, 0x0, sizeof *child);

		child->s_def = NULL;
		child->s_char = c;

		fx_queue_push_back(&tree->s_children, &child->s_entry);
		tree = child;
	}

	tree->s_def = sym;
	return BSHELL_SUCCESS;
}

static void destroy_symbol_tree(struct bshell_lex_symbol_node *tree)
{
	fx_queue_entry *entry = fx_queue_first(&tree->s_children);
	while (entry) {
		struct bshell_lex_symbol_node *node = fx_unbox(
			struct bshell_lex_symbol_node,
			entry,
			s_entry);
		fx_queue_entry *next = fx_queue_next(entry);
		fx_queue_delete(&tree->s_children, entry);

		destroy_symbol_tree(node);

		entry = next;
	}

	free(tree);
}

static struct bshell_lex_symbol_node *build_symbol_tree(void)
{
	struct bshell_lex_symbol_node *root = malloc(sizeof *root);
	if (!root) {
		return NULL;
	}

	memset(root, 0x0, sizeof *root);
	root->s_def = NULL;

	enum bshell_status status = BSHELL_SUCCESS;
	for (size_t i = 0; i < nr_symbols; i++) {
		if (!symbols[i].name) {
			continue;
		}

		status = put_symbol(root, &symbols[i]);

		if (status != BSHELL_SUCCESS) {
			destroy_symbol_tree(root);
			return NULL;
		}
	}

	return root;
}

static void init_token_enabled_states(
	const struct bshell_lex_state_type *state_type)
{
	if (state_type->s_keywords) {
		for (size_t i = 0; state_type->s_keywords[i]; i++) {
			unsigned int id = state_type->s_keywords[i];
			keywords[id - __BSHELL_KW_INDEX_BASE].enabled_states
				|= state_type->s_id;
		}
	}

	if (state_type->s_operators) {
		for (size_t i = 0; state_type->s_operators[i]; i++) {
			unsigned int id = state_type->s_operators[i];
			operators[id - __BSHELL_TKOP_INDEX_BASE].enabled_states
				|= state_type->s_id;
		}
	}

	if (state_type->s_symbols) {
		for (size_t i = 0; state_type->s_symbols[i]; i++) {
			unsigned int id = state_type->s_symbols[i];
			symbols[id - __BSHELL_SYM_INDEX_BASE].enabled_states
				|= state_type->s_id;
		}
	}
}

enum bshell_status bshell_lex_ctx_init(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_flags flags,
	struct bshell_line_source *src)
{
	memset(ctx, 0x0, sizeof *ctx);

	ctx->lex_flags = flags;
	ctx->lex_status = BSHELL_SUCCESS;
	ctx->lex_buf = fx_stringstream_create();
	ctx->lex_sym_tree = build_symbol_tree();
	lex_state_push(ctx, BSHELL_LEX_STATE_STATEMENT, 0);
	ctx->lex_src = src;
	ctx->lex_ch = FX_WCHAR_INVALID;
	ctx->lex_cursor.c_row = ctx->lex_cursor.c_col = 1;

	init_token_enabled_states(&lex_statement_state);
	init_token_enabled_states(&lex_command_state);
	init_token_enabled_states(&lex_arithmetic_state);
	init_token_enabled_states(&lex_string_state);
	init_token_enabled_states(&lex_word_state);
	init_token_enabled_states(&lex_hashtable_state);

	return BSHELL_SUCCESS;
}

enum bshell_status bshell_lex_ctx_cleanup(struct bshell_lex_ctx *ctx)
{
	if (ctx->lex_sym_tree) {
		destroy_symbol_tree(ctx->lex_sym_tree);
	}

	if (ctx->lex_buf) {
		fx_stringstream_unref(ctx->lex_buf);
	}

	if (ctx->lex_tmp) {
		fx_string_unref(ctx->lex_tmp);
	}

	memset(ctx, 0x0, sizeof *ctx);
	return BSHELL_SUCCESS;
}

static enum bshell_status refill_buffer(struct bshell_lex_ctx *ctx)
{
	fx_stringstream_reset(ctx->lex_buf);
	ctx->lex_ch = FX_WCHAR_INVALID;

	return bshell_line_source_readline(ctx->lex_src, ctx->lex_buf);
}

static fx_wchar __peek_char(struct bshell_lex_ctx *ctx, bool noread)
{
	if (ctx->lex_status != BSHELL_SUCCESS) {
		return FX_WCHAR_INVALID;
	}

	if (ctx->lex_ch != FX_WCHAR_INVALID) {
		return ctx->lex_ch;
	}

	fx_status status = fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
	if (!FX_OK(status) && !noread) {
		enum bshell_status status2 = refill_buffer(ctx);
		if (status2 != BSHELL_SUCCESS) {
			ctx->lex_status = status2;
			ctx->lex_ch = FX_WCHAR_INVALID;
		} else {
			fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
		}
	}

	return ctx->lex_ch;
}

fx_wchar peek_char(struct bshell_lex_ctx *ctx)
{
	return __peek_char(ctx, false);
}

fx_wchar peek_char_noread(struct bshell_lex_ctx *ctx)
{
	return __peek_char(ctx, true);
}

static void __advance_char(struct bshell_lex_ctx *ctx, bool noread)
{
	if (ctx->lex_status != BSHELL_SUCCESS) {
		return;
	}

	ctx->lex_cursor.c_col++;
	if (ctx->lex_ch == '\n') {
		ctx->lex_cursor.c_col = 1;
		ctx->lex_cursor.c_row++;
	}

	if (ctx->lex_ch != FX_WCHAR_INVALID) {
		ctx->lex_ch = FX_WCHAR_INVALID;
		return;
	}

	fx_status status = fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
	if (!FX_OK(status) && !noread) {
		enum bshell_status status2 = refill_buffer(ctx);
		if (status2 != BSHELL_SUCCESS) {
			ctx->lex_status = status2;
			ctx->lex_ch = FX_WCHAR_INVALID;
		} else {
			fx_stream_read_char(ctx->lex_buf, &ctx->lex_ch);
		}
	}
}

void advance_char(struct bshell_lex_ctx *ctx)
{
	return __advance_char(ctx, false);
}

void advance_char_noread(struct bshell_lex_ctx *ctx)
{
	return __advance_char(ctx, true);
}

bool convert_word_to_keyword(struct bshell_lex_token *tok)
{
	if (!bshell_lex_token_has_string_value(tok)) {
		return false;
	}

	for (size_t i = 0; i < nr_keywords; i++) {
		const char *kw_str = keywords[i].name;
		if (!kw_str || strcmp(kw_str, tok->tok_str) != 0) {
			continue;
		}

		bshell_lex_token_change_type(tok, BSHELL_TOK_KEYWORD);
		tok->tok_keyword = keywords[i].id;
		return true;
	}

	return false;
}

bool convert_word_to_operator(
	struct bshell_lex_ctx *ctx,
	struct bshell_lex_token *tok)
{
	if (!bshell_lex_token_has_string_value(tok)) {
		return false;
	}

	enum bshell_lex_operator op
		= get_bshell_operator_with_string(ctx, tok->tok_str);
	if (op == BSHELL_TKOP_NONE) {
		return false;
	}

	bshell_lex_token_change_type(tok, BSHELL_TOK_OPERATOR);
	tok->tok_operator = op;
	return true;
}

static int get_int_base_by_prefix(const char **s)
{
#define CH(x) (tolower(value[x]))
	const char *value = *s;
	if (CH(0) != '0') {
		return 10;
	}

	switch (CH(1)) {
	case 'x':
		*s += 2;
		return 16;
	case 'b':
		*s += 2;
		return 2;
	default:
		*s += 1;
		return 8;
	}
#undef CH
}

static size_t get_int_multiplier_by_suffix(const char *suffix)
{
#define CH(x) (tolower(suffix[x]))
	if (CH(1) != 'b' || CH(2) != 0) {
		return 0;
	}

	switch (CH(0)) {
	case 'k':
		return 0x400;
	case 'm':
		return 0x100000;
	case 'g':
		return 0x40000000;
	case 't':
		return 0x10000000000;
	case 'b':
		return 0x4000000000000;
	default:
		return 0;
	}

#undef CH
	return 0;
}

static bool string_could_be_double(const char *s)
{
	if (strchr(s, '.')) {
		return true;
	}

	if (strchr(s, 'e') || strchr(s, 'E')) {
		return true;
	}

	return false;
}

bool string_is_valid_number(
	const char *s,
	fx_value *out,
	enum bshell_lex_token_type *out_type)
{
	if (s[0] == '\0') {
		return NULL;
	}

	char *ep = NULL;
	double dvalue = strtod(s, &ep);
	if (string_could_be_double(s) && *ep == '\0') {
		if (out) {
			*out = FX_DOUBLE(dvalue);
		}

		if (out_type) {
			*out_type = BSHELL_TOK_DOUBLE;
		}

		return true;
	}

	int base = get_int_base_by_prefix(&s);

	long long ivalue = strtoll(s, &ep, base);
	if (*ep == '\0') {
		if (out) {
			*out = FX_LONGLONG(ivalue);
		}

		if (out_type) {
			*out_type = BSHELL_TOK_INT;
		}

		return true;
	}

	size_t multiplier = get_int_multiplier_by_suffix(ep);
	if (multiplier != 0) {
		if (out) {
			*out = FX_LONGLONG(ivalue * multiplier);
		}

		if (out_type) {
			*out_type = BSHELL_TOK_INT;
		}

		return true;
	}

	return false;
}

bool convert_word_to_number(struct bshell_lex_token *tok)
{
	if (!bshell_lex_token_has_string_value(tok)) {
		return false;
	}

	const char *s = tok->tok_str;
	fx_value value;
	enum bshell_lex_token_type type;
	if (!string_is_valid_number(s, &value, &type)) {
		return false;
	}

	bshell_lex_token_change_type(tok, type);
	tok->tok_number = value;

	return true;
}

static struct bshell_lex_token *get_next_token(struct bshell_lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_first(&ctx->lex_tokens);
	return fx_unbox(struct bshell_lex_token, entry, tok_entry);
}

void enqueue_token(struct bshell_lex_ctx *ctx, struct bshell_lex_token *tok)
{
	enqueue_token_with_coordinates(
		ctx,
		tok,
		&ctx->lex_start,
		&ctx->lex_end);
}

extern void enqueue_token_with_coordinates(
	struct bshell_lex_ctx *ctx,
	struct bshell_lex_token *tok,
	const struct bshell_char_cell *start,
	const struct bshell_char_cell *end)
{
	if (tok->tok_type == BSHELL_TOK_LINEFEED
	    && ctx->lex_prev_token == BSHELL_TOK_LINEFEED) {
		bshell_lex_token_destroy(tok);
		return;
	}

	tok->tok_start = *start;
	tok->tok_end = *end;
	ctx->lex_prev_token = tok->tok_type;

	if (ctx->lex_token_scanned) {
		ctx->lex_token_scanned(ctx, tok);
	}

	fx_queue_push_back(&ctx->lex_tokens, &tok->tok_entry);
}

static struct bshell_lex_token *dequeue_next_token(struct bshell_lex_ctx *ctx)
{
	fx_queue_entry *entry = fx_queue_pop_front(&ctx->lex_tokens);
	return fx_unbox(struct bshell_lex_token, entry, tok_entry);
}

static fx_string *get_temp_string(struct bshell_lex_ctx *ctx)
{
	if (!ctx->lex_tmp) {
		ctx->lex_tmp = fx_string_create();
	}

	fx_string_clear(ctx->lex_tmp);
	return ctx->lex_tmp;
}

enum bshell_status push_symbol(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_symbol sym)
{
	struct bshell_lex_token *tok
		= bshell_lex_token_create(BSHELL_TOK_SYMBOL);
	if (!tok) {
		return BSHELL_ERR_NO_MEMORY;
	}

	tok->tok_symbol = sym;
	enqueue_token(ctx, tok);
	return BSHELL_SUCCESS;
}

enum bshell_status read_var(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_token_type type,
	struct bshell_lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);

	bool done = false;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		bool valid = fx_wchar_is_alnum(c) || (c == '_') || (c == ':');

		if (!valid) {
			break;
		}

		if (done) {
			break;
		}

		fx_string_append_wc(tmp, c);
		set_token_end(ctx);
		advance_char(ctx);
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		return ctx->lex_status;
	}

	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
		type,
		fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_braced_var(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_token_type type,
	struct bshell_lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);
	bool ok = false;

	while (1) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (c == '}') {
			ok = true;
			advance_char(ctx);
			break;
		}

		fx_string_append_wc(tmp, c);
		set_token_end(ctx);
		advance_char(ctx);
	}

	if (!ok) {
		return BSHELL_ERR_BAD_SYNTAX;
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		return ctx->lex_status;
	}

	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
		type,
		fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_literal_string(
	struct bshell_lex_ctx *ctx,
	struct bshell_lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);

	bool done = false;
	bool fail = true;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (c == '\'') {
			fail = false;
			done = true;
			set_token_end(ctx);
			advance_char(ctx);
			break;
		}

		fx_string_append_wc(tmp, c);
		advance_char(ctx);
	}

	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
		BSHELL_TOK_STRING,
		fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_line_comment(struct bshell_lex_ctx *lex)
{
	while (true) {
		fx_wchar c = peek_char(lex);

		if (c == FX_WCHAR_INVALID) {
			break;
		}

		advance_char(lex);

		if (c == '\n') {
			break;
		}
	}

	return BSHELL_SUCCESS;
}

enum bshell_status read_word(
	struct bshell_lex_ctx *ctx,
	enum read_flags flags,
	struct bshell_lex_token **out)
{
	fx_string *tmp = get_temp_string(ctx);
	bool word_is_number = false;

	if (!(flags & READ_NO_SET_TOKEN_START)) {
		set_token_start(ctx);
	}

	if (flags & READ_APPEND_HYPHEN) {
		fx_string_append_c(tmp, '-');
	}

	bool number_recog = !(flags & READ_NO_NUMBER_RECOGNITION);

	enum bshell_lex_operator op = BSHELL_TKOP_NONE;
	bool done = false;
	while (!done) {
		fx_wchar c = peek_char(ctx);
		if (c == FX_WCHAR_INVALID) {
			break;
		}

		if (fx_wchar_is_space(c)) {
			done = true;
			break;
		}

		const char *s = fx_string_get_cstr(tmp);
		if (number_recog && string_is_valid_number(s, NULL, NULL)) {
			if (c == '.') {
				fx_string_append_wc(tmp, c);
				set_token_end(ctx);
				advance_char(ctx);
				continue;
			}

			if (char_can_begin_symbol_in_state(
				    ctx,
				    c,
				    BSHELL_LEX_STATE_ARITHMETIC)) {
				done = true;
				break;
			}
		}

		if (char_can_begin_symbol(ctx, c)) {
			done = true;
			break;
		}

		if (!fx_wchar_is_alpha(c)) {
			op = get_bshell_operator_with_string(ctx, s);
			if (op != BSHELL_TKOP_NONE) {
				done = true;
				break;
			}
		}

		fx_string_append_wc(tmp, c);
		set_token_end(ctx);
		advance_char(ctx);
	}

	if (fx_string_get_size(tmp, FX_STRLEN_NORMAL) == 0) {
		if (ctx->lex_status == BSHELL_SUCCESS) {
			return BSHELL_ERR_BAD_SYNTAX;
		}

		return ctx->lex_status;
	}

	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
		BSHELL_TOK_WORD,
		fx_string_get_cstr(tmp));

	*out = tok;
	return BSHELL_SUCCESS;
}

enum bshell_status read_symbol(
	struct bshell_lex_ctx *ctx,
	const struct bshell_lex_token_definition **out)
{
	struct bshell_lex_state *state = lex_state_get(ctx);
	set_token_start(ctx);

	struct bshell_lex_symbol_node *node = ctx->lex_sym_tree;
	char prev = 0;

	while (true) {
		fx_wchar c = peek_char(ctx);
		if (c < 0) {
			break;
		}

		struct bshell_lex_symbol_node *next = get_symbol_node(node, c);
		if (!next
		    || (next->s_def
		        && !(next->s_def->enabled_states
		             & state->s_type->s_id))) {
			prev = c;
			break;
		}

		node = next;
		set_token_end(ctx);
		advance_char(ctx);
		prev = c;
	}

	if (!node || node->s_def == NULL) {
		return BSHELL_ERR_BAD_SYNTAX;
	}

	*out = node->s_def;
	return BSHELL_SUCCESS;
}

bool char_can_begin_symbol_in_state(
	struct bshell_lex_ctx *ctx,
	char c,
	enum bshell_lex_state_id state_type)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (!symbols[i].name) {
			continue;
		}

		if (symbols[i].name[0] != c) {
			continue;
		}

		if (symbols[i].enabled_states & state_type) {
			return true;
		}
	}

	return false;
}

bool char_can_begin_symbol(struct bshell_lex_ctx *ctx, char c)
{
	struct bshell_lex_state *state = lex_state_get(ctx);
	return char_can_begin_symbol_in_state(ctx, c, state->s_type->s_id);
}

bool char_has_flags(
	struct bshell_lex_ctx *ctx,
	char c,
	enum bshell_lex_token_flags flags)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (!symbols[i].name) {
			continue;
		}

		if (symbols[i].name[0] != c) {
			continue;
		}

		return (symbols[i].flags & flags) == flags;
	}

	return false;
}

bool keyword_has_flags(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_keyword kw,
	enum bshell_lex_token_flags flags)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (keywords[i].id == kw) {
			return (keywords[i].flags & flags) == flags;
		}
	}

	return false;
}

enum bshell_lex_token_flags keyword_get_flags(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_keyword kw)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (keywords[i].id == kw) {
			return keywords[i].flags;
		}
	}

	return false;
}

bool symbol_has_flags(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_symbol sym,
	enum bshell_lex_token_flags flags)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (symbols[i].id == sym) {
			return (symbols[i].flags & flags) == flags;
		}
	}

	return false;
}

enum bshell_lex_token_flags symbol_get_flags(
	struct bshell_lex_ctx *ctx,
	enum bshell_lex_symbol sym)
{
	for (size_t i = 0; i < nr_symbols; i++) {
		if (symbols[i].id == sym) {
			return symbols[i].flags;
		}
	}

	return false;
}

enum bshell_lex_operator get_bshell_operator_with_string(
	struct bshell_lex_ctx *ctx,
	const char *s)
{
	struct bshell_lex_state *state = lex_state_get(ctx);

	for (size_t i = 0; i < nr_operators; i++) {
		const char *op_str = operators[i].name;
		if (!op_str || strcmp(op_str, s) != 0) {
			continue;
		}

		if (!(operators[i].enabled_states & state->s_type->s_id)) {
			continue;
		}

		return operators[i].id;
	}

	return false;
}

int compare_token_types(unsigned int a, unsigned int b)
{
	if (a == b) {
		return 2;
	}

#define BETWEEN(v, lo, hi) ((v) >= (lo) && (v) <= (hi))
	enum bshell_lex_token_type a_type = BSHELL_TOK_NONE, b_type
							     = BSHELL_TOK_NONE;

	if (BETWEEN(a, __BSHELL_KW_INDEX_BASE, __BSHELL_KW_INDEX_LIMIT)) {
		a_type = BSHELL_TOK_KEYWORD;
	} else if (BETWEEN(a,
	                   __BSHELL_TKOP_INDEX_BASE,
	                   __BSHELL_TKOP_INDEX_LIMIT)) {
		a_type = BSHELL_TOK_OPERATOR;
	} else if (BETWEEN(a,
	                   __BSHELL_SYM_INDEX_BASE,
	                   __BSHELL_SYM_INDEX_LIMIT)) {
		a_type = BSHELL_TOK_SYMBOL;
	} else {
		a_type = a;
	}

	if (BETWEEN(b, __BSHELL_KW_INDEX_BASE, __BSHELL_KW_INDEX_LIMIT)) {
		b_type = BSHELL_TOK_KEYWORD;
	} else if (BETWEEN(b,
	                   __BSHELL_TKOP_INDEX_BASE,
	                   __BSHELL_TKOP_INDEX_LIMIT)) {
		b_type = BSHELL_TOK_OPERATOR;
	} else if (BETWEEN(b,
	                   __BSHELL_SYM_INDEX_BASE,
	                   __BSHELL_SYM_INDEX_LIMIT)) {
		b_type = BSHELL_TOK_SYMBOL;
	} else {
		b_type = b;
	}
#undef BETWEEN

	int result = 0;
	if (a_type == b_type) {
		if (a != a_type && b != b_type) {
			result = 0;
		} else {
			result = a == b ? 2 : 1;
		}
	}

	if (result < 0) {
		result = 0;
	}

	return result;
}

static bool do_lex_state_transition(
	struct bshell_lex_ctx *ctx,
	unsigned int token,
	bool recursive)
{
	struct bshell_lex_state *state = lex_state_get(ctx);
	enum link_flags required_flags = 0;
	if (recursive) {
		required_flags |= LINK_ALLOW_RECURSION;
	}

	if (!recursive) {
		for (unsigned int i = 0; i < state->s_nr_terminators; i++) {
			if (state->s_terminators[i] == token) {
				lex_state_pop(ctx);
				return true;
			}
		}
	}

	const struct bshell_lex_state_link *table = state->s_type->s_links;
	if (!table) {
		return false;
	}

#define MAX_MATCHES 8
	const struct bshell_lex_state_link *best_matches[MAX_MATCHES] = {0};
	unsigned int match_count = 0;
	int best_score = 0;

	for (unsigned int i = 0; table[i].l_token != BSHELL_TOK_NONE; i++) {
		int score = compare_token_types(table[i].l_token, token);
		if ((table[i].l_flags & required_flags) != required_flags) {
			score = 0;
		}

		if (score == 0) {
			continue;
		}

		assert(match_count < MAX_MATCHES
		       || "lex state has too many matches");
		if (score == best_score) {
			best_matches[match_count++] = &table[i];
		} else if (score > best_score) {
			match_count = 0;
			best_matches[match_count++] = &table[i];
			best_score = score;
		}
	}
#undef MAX_MATCHES

	if (!match_count) {
		return false;
	}

	bool result = false;
	for (unsigned int i = 0; i < match_count; i++) {
		const struct bshell_lex_state_link *link = best_matches[i];
		switch (link->l_type) {
		case BSHELL_LEX_STATE_LINK_POP:
			lex_state_pop(ctx);
			result = true;
			break;
		case BSHELL_LEX_STATE_LINK_PUSH: {
			struct bshell_lex_state *state = lex_state_push(
				ctx,
				link->l_target,
				link->l_target_flags);
			for (unsigned int i = 0; link->l_terminators[i]; i++) {
				lex_state_add_terminator(
					state,
					link->l_terminators[i]);
			}
			result = true;
			break;
		}

		case BSHELL_LEX_STATE_LINK_CHANGE:
			lex_state_change(ctx, link->l_target);
			result = true;
			break;
		default:
			break;
		}
	}

	return result;
}

void handle_lex_state_transition(struct bshell_lex_ctx *ctx, unsigned int token)
{
	bool cont = false;
	bool recursive = false;
	do {
		cont = do_lex_state_transition(ctx, token, recursive);
		recursive = true;
	} while (cont);
}

static enum bshell_status read_string_content(struct bshell_lex_ctx *ctx)
{
	fx_wchar c = FX_WCHAR_INVALID;
	fx_string *str = get_temp_string(ctx);
	struct bshell_lex_state *state = lex_state_get(ctx);

	if (!str) {
		return BSHELL_ERR_NO_MEMORY;
	}

	while (true) {
		c = peek_char(ctx);
		if (c < 0) {
			break;
		}

		if (char_can_begin_symbol(ctx, c)) {
			break;
		}

		fx_string_append_wc(str, c);
		// set_token_end(lex);
		advance_char(ctx);
	}

	if (fx_string_get_size(str, FX_STRLEN_NORMAL) == 0) {
		return BSHELL_SUCCESS;
	}

	struct bshell_lex_token *tok = bshell_lex_token_create_with_string(
		BSHELL_TOK_STRING,
		fx_string_get_cstr(str));
	enqueue_token(ctx, tok);

	return BSHELL_SUCCESS;
}

#if 0
static enum bshell_status do_pump_token_string(struct bshell_lex_ctx *ctx)
{
	fx_wchar c = peek_char(ctx);
	enum bshell_status status = BSHELL_SUCCESS;
	bool ok = false;

	if (char_can_begin_symbol(ctx, c)) {
		status = read_symbol(ctx);
		ok = true;
	}

	if (status != BSHELL_SUCCESS || !ok) {
		status = read_string_content(ctx);
	}

	return status;
}

static enum bshell_status do_pump_token_normal(struct bshell_lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;

	fx_wchar c = peek_char(ctx);
	bool newline = false;

	while (fx_wchar_is_space(c)) {
		if (c == '\n') {
			newline = true;
		}

		advance_char_noread(ctx);
		c = peek_char_noread(ctx);
	}

	if (newline) {
		struct bshell_lex_token *tok = bshell_lex_token_create(BSHELL_TOK_LINEFEED);
		enqueue_token(ctx, tok);
		return BSHELL_SUCCESS;
	}

	if (c == '-') {
		return read_flag(ctx);
	}

	if (char_can_begin_symbol(ctx, c)) {
		return read_symbol(ctx);
	}

	return read_word(ctx);
}
#endif

static enum bshell_status pump_tokens(struct bshell_lex_ctx *ctx)
{
	enum bshell_status status = BSHELL_SUCCESS;
	while (fx_queue_empty(&ctx->lex_tokens) && status == BSHELL_SUCCESS) {
		struct bshell_lex_state *state = lex_state_get(ctx);
		status = state->s_type->s_pump_token(ctx);
	}

	return status;
}

static void discard_all_tokens(struct bshell_lex_ctx *ctx)
{
	fx_queue_entry *cur = fx_queue_first(&ctx->lex_tokens);
	while (cur) {
		struct bshell_lex_token *tok
			= fx_unbox(struct bshell_lex_token, cur, tok_entry);
		if (tok->tok_type == BSHELL_TOK_LINEFEED) {
			break;
		}

		fx_queue_pop_front(&ctx->lex_tokens);
		bshell_lex_token_destroy(tok);
		cur = fx_queue_first(&ctx->lex_tokens);
	}
}

struct bshell_lex_token *bshell_lex_ctx_peek(struct bshell_lex_ctx *ctx)
{
	struct bshell_lex_token *tok = get_next_token(ctx);
	if (tok) {
		return tok;
	}

	pump_tokens(ctx);
	tok = get_next_token(ctx);

	return tok;
}

struct bshell_lex_token *bshell_lex_ctx_claim(struct bshell_lex_ctx *ctx)
{
	struct bshell_lex_token *tok = dequeue_next_token(ctx);
	if (tok) {
		return tok;
	}

	if (fx_queue_empty(&ctx->lex_tokens)) {
		pump_tokens(ctx);

		tok = get_next_token(ctx);
	}

	return dequeue_next_token(ctx);
}

void bshell_lex_ctx_discard(struct bshell_lex_ctx *ctx)
{
	struct bshell_lex_token *tok = dequeue_next_token(ctx);
	if (tok) {
		bshell_lex_token_destroy(tok);
		return;
	}

	/* request more tokens if necessary */
	if (fx_queue_empty(&ctx->lex_tokens)) {
		pump_tokens(ctx);
	}
}