parse: lots of improvements

bshell/parse/lex/arithmetic.c

@@ -19,18 +19,19 @@ static enum bshell_status arithmetic_hyphen(struct lex_ctx *ctx)
 	}
 
 	unsigned int token_type = TOK_WORD;
-	if (convert_word_to_int(tok)) {
-		token_type = TOK_INT;
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
 
 		/* because of APPEND_HYPHEN (which is needed to ensure operator
 		 * tokens are detected properly), the resulting number will be
 		 * negative.
 		 * this token will be preceded by a HYPHEN token, so the number
 		 * must be positive */
-		tok->tok_int *= -1;
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
 		push_symbol(ctx, SYM_HYPHEN);
 	} else if (convert_word_to_operator(ctx, tok)) {
-		token_type = TOK_OPERATOR;
+		token_type = tok->tok_operator;
 	}
 
 	enqueue_token(ctx, tok);
@@ -108,8 +109,8 @@ static enum bshell_status arithmetic_word(struct lex_ctx *ctx)
 	bool kw = false, number = false;
 	if (convert_word_to_keyword(word)) {
 		token_type = word->tok_keyword;
-	} else if (convert_word_to_int(word)) {
-		token_type = TOK_INT;
+	} else if (convert_word_to_number(word)) {
+		token_type = word->tok_type;
 	}
 
 	handle_lex_state_transition(ctx, token_type);

bshell/parse/lex/hashtable.c

@@ -19,14 +19,15 @@ static enum bshell_status hashtable_hyphen(struct lex_ctx *ctx)
 	}
 
 	unsigned int token_type = TOK_WORD;
-	if (convert_word_to_int(tok)) {
-		token_type = TOK_INT;
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
 		/* because of APPEND_HYPHEN (which is needed to ensure operator
 		 * tokens are detected properly), the resulting number will be
 		 * negative.
 		 * this token will be preceded by a HYPHEN token, so the number
 		 * must be positive */
-		tok->tok_int *= -1;
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
 		push_symbol(ctx, SYM_HYPHEN);
 	} else if (convert_word_to_operator(ctx, tok)) {
 		token_type = tok->tok_operator;
@@ -102,7 +103,7 @@ static enum bshell_status hashtable_word(struct lex_ctx *ctx)
 		return status;
 	}
 
-	convert_word_to_int(word);
+	convert_word_to_number(word);
 
 	handle_lex_state_transition(ctx, word->tok_type);
 	enqueue_token(ctx, word);

bshell/parse/lex/lex-internal.h

@@ -123,8 +123,11 @@ extern fx_wchar peek2_char_noread(struct lex_ctx *ctx);
 extern void advance_char(struct lex_ctx *ctx);
 extern void advance_char_noread(struct lex_ctx *ctx);
 
-extern bool string_is_valid_number(const char *s, long long *out);
-extern bool convert_word_to_int(struct lex_token *tok);
+extern bool string_is_valid_number(
+	const char *s,
+	fx_value *out,
+	enum token_type *out_type);
+extern bool convert_word_to_number(struct lex_token *tok);
 extern bool convert_word_to_keyword(struct lex_token *tok);
 extern bool convert_word_to_operator(
 	struct lex_ctx *ctx,

bshell/parse/lex/lex.c

@@ -599,44 +599,88 @@ static size_t get_int_multiplier_by_suffix(const char *suffix)
 	return 0;
 }
 
-bool string_is_valid_number(const char *s, long long *out)
+static bool string_could_be_double(const char *s)
 {
-	if (s[0] == '\0') {
-		return NULL;
-	}
-
-	int base = get_int_base_by_prefix(&s);
-
-	char *ep = NULL;
-	long long value = strtoll(s, &ep, base);
-	if (*ep == '\0') {
-		out && (*out = value);
+	if (strchr(s, '.')) {
 		return true;
 	}
 
-	size_t multiplier = get_int_multiplier_by_suffix(ep);
-	if (multiplier != 0) {
-		out && (*out = value * multiplier);
+	if (strchr(s, 'e') || strchr(s, 'E')) {
 		return true;
 	}
 
 	return false;
 }
 
-bool convert_word_to_int(struct lex_token *tok)
+bool string_is_valid_number(
+	const char *s,
+	fx_value *out,
+	enum token_type *out_type)
+{
+	if (s[0] == '\0') {
+		return NULL;
+	}
+
+	char *ep = NULL;
+	double dvalue = strtod(s, &ep);
+	if (string_could_be_double(s) && *ep == '\0') {
+		if (out) {
+			*out = FX_DOUBLE(dvalue);
+		}
+
+		if (out_type) {
+			*out_type = TOK_DOUBLE;
+		}
+
+		return true;
+	}
+
+	int base = get_int_base_by_prefix(&s);
+
+	long long ivalue = strtoll(s, &ep, base);
+	if (*ep == '\0') {
+		if (out) {
+			*out = FX_LONGLONG(ivalue);
+		}
+
+		if (out_type) {
+			*out_type = TOK_INT;
+		}
+
+		return true;
+	}
+
+	size_t multiplier = get_int_multiplier_by_suffix(ep);
+	if (multiplier != 0) {
+		if (out) {
+			*out = FX_LONGLONG(ivalue * multiplier);
+		}
+
+		if (out_type) {
+			*out_type = TOK_INT;
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+bool convert_word_to_number(struct lex_token *tok)
 {
 	if (!lex_token_has_string_value(tok)) {
 		return false;
 	}
 
 	const char *s = tok->tok_str;
-	long long value = 0;
-	if (!string_is_valid_number(s, &value)) {
+	fx_value value;
+	enum token_type type;
+	if (!string_is_valid_number(s, &value, &type)) {
 		return false;
 	}
 
-	lex_token_change_type(tok, TOK_INT);
-	tok->tok_int = value;
+	lex_token_change_type(tok, type);
+	tok->tok_number = value;
 
 	return true;
 }
@@ -873,13 +917,15 @@ enum bshell_status read_word(
 			break;
 		}
 
-		if (char_can_begin_symbol(ctx, c)) {
-			done = true;
-			break;
-		}
-
 		const char *s = fx_string_get_cstr(tmp);
-		if (number_recog && string_is_valid_number(s, NULL)) {
+		if (number_recog && string_is_valid_number(s, NULL, NULL)) {
+			if (c == '.') {
+				fx_string_append_wc(tmp, c);
+				set_token_end(ctx);
+				advance_char(ctx);
+				continue;
+			}
+
 			if (char_can_begin_symbol_in_state(
 				    ctx,
 				    c,
@@ -889,6 +935,11 @@ enum bshell_status read_word(
 			}
 		}
 
+		if (char_can_begin_symbol(ctx, c)) {
+			done = true;
+			break;
+		}
+
 		if (!fx_wchar_is_alpha(c)) {
 			op = get_operator_with_string(ctx, s);
 			if (op != TKOP_NONE) {

bshell/parse/lex/statement.c

@@ -20,18 +20,19 @@ static enum bshell_status statement_hyphen(struct lex_ctx *ctx)
 
 	unsigned int token_type = TOK_WORD;
 
-	if (convert_word_to_int(tok)) {
-		token_type = TOK_INT;
+	if (convert_word_to_number(tok)) {
+		token_type = tok->tok_type;
 
 		/* because of APPEND_HYPHEN (which is needed to ensure operator
 		 * tokens are detected properly), the resulting number will be
 		 * negative.
 		 * this token will be preceded by a HYPHEN token, so the number
 		 * must be positive */
-		tok->tok_int *= -1;
+		fx_value neg = FX_INT(-1);
+		fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number);
 		push_symbol(ctx, SYM_HYPHEN);
 	} else if (convert_word_to_operator(ctx, tok)) {
-		token_type = TOK_OPERATOR;
+		token_type = tok->tok_operator;
 	}
 
 	handle_lex_state_transition(ctx, token_type);
@@ -114,8 +115,8 @@ static enum bshell_status statement_word(struct lex_ctx *ctx)
 
 	if (enable_keywords && convert_word_to_keyword(word)) {
 		token = word->tok_keyword;
-	} else if (convert_word_to_int(word)) {
-		token = TOK_INT;
+	} else if (convert_word_to_number(word)) {
+		token = word->tok_type;
 	}
 
 	handle_lex_state_transition(ctx, token);
@@ -163,7 +164,9 @@ static const struct lex_state_link links[] = {
 	LINK_PUSH(SYM_DQUOTE, LEX_STATE_STRING, 0),
 	/* arithmetic tokens */
 	LINK_CHANGE(TOK_KEYWORD, LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(SYM_SQUOTE, LEX_STATE_ARITHMETIC),
 	LINK_CHANGE(TOK_INT, LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(TOK_DOUBLE, LEX_STATE_ARITHMETIC),
 	LINK_PUSH(SYM_DOLLAR, LEX_STATE_ARITHMETIC, 0),
 	LINK_PUSH_WITH_TERM(
 		SYM_DOLLAR_LEFT_BRACE,
@@ -171,6 +174,7 @@ static const struct lex_state_link links[] = {
 		0,
 		SYM_RIGHT_BRACE),
 	LINK_CHANGE(SYM_AT_LEFT_BRACE, LEX_STATE_ARITHMETIC),
+	LINK_CHANGE(SYM_AT_LEFT_PAREN, LEX_STATE_ARITHMETIC),
 	LINK_PUSH_WITH_TERM(
 		SYM_AT_LEFT_BRACE,
 		LEX_STATE_HASHTABLE,
@@ -226,6 +230,7 @@ static const unsigned int symbols[] = {
 	SYM_DQUOTE,
 	SYM_HASH,
 	SYM_AT,
+	SYM_AT_LEFT_PAREN,
 	SYM_AT_LEFT_BRACE,
 	SYM_PIPE,
 	SYM_COMMA,

bshell/parse/syntax/arith.c

@@ -47,8 +47,10 @@ static bool finalise_expr(
 			break;
 		}
 
-		struct op_ast_node *node
-			= fx_unbox(struct op_ast_node, entry, n_base.n_entry);
+		struct op_ast_node *node = fx_unbox(
+			struct op_ast_node,
+			entry,
+			n_base.n_entry);
 		if (!node) {
 			/* this should never happen */
 			return false;
@@ -72,8 +74,10 @@ static bool finalise_expr(
 	int i = 0;
 
 	while (entry) {
-		struct ast_node *item
-			= fx_unbox(struct ast_node, entry, n_entry);
+		struct ast_node *item = fx_unbox(
+			struct ast_node,
+			entry,
+			n_entry);
 		fx_queue_entry *next = fx_queue_next(entry);
 		fx_queue_delete(&ctx->expr_out_queue, entry);
 
@@ -115,12 +119,14 @@ static bool finalise_expr(
 
 		if (op->op_arity == OPA_BINARY) {
 			tmp = fx_queue_pop_back(&q);
-			op_node->n_left
-				= fx_unbox(struct ast_node, tmp, n_entry);
+			op_node->n_left = fx_unbox(
+				struct ast_node,
+				tmp,
+				n_entry);
 
 			if (op_node->n_left) {
-				op_node->n_left->n_parent
-					= (struct ast_node *)op_node;
+				op_node->n_left->n_parent = (struct ast_node *)
+					op_node;
 
 #if 0
 				ast_node_extend_bounds_recursive(
@@ -248,8 +254,8 @@ static bool parse_hashtable(struct parse_ctx *ctx, struct ast_node **out)
 
 	parse_linefeed(ctx);
 
-	struct hashtable_ast_node *table
-		= (struct hashtable_ast_node *)ast_node_create(AST_HASHTABLE);
+	struct hashtable_ast_node *table = (struct hashtable_ast_node *)
+		ast_node_create(AST_HASHTABLE);
 	if (!table) {
 		ctx->p_status = BSHELL_ERR_NO_MEMORY;
 		return false;
@@ -264,14 +270,14 @@ static bool parse_hashtable(struct parse_ctx *ctx, struct ast_node **out)
 
 		parse_linefeed(ctx);
 
-		struct hashtable_item_ast_node *item
-			= (struct hashtable_item_ast_node *)ast_node_create(
-				AST_HASHTABLE_ITEM);
+		struct hashtable_item_ast_node *item = (struct
+		                                        hashtable_item_ast_node
+		                                                *)
+			ast_node_create(AST_HASHTABLE_ITEM);
 		struct lex_token *tok = NULL;
 		if (parse_word(ctx, &tok)) {
-			struct string_ast_node *v
-				= (struct string_ast_node *)ast_node_create(
-					AST_STRING);
+			struct string_ast_node *v = (struct string_ast_node *)
+				ast_node_create(AST_STRING);
 			v->n_value = tok;
 			item->n_key = (struct ast_node *)v;
 		} else if (!parse_arith_value(ctx, &item->n_key)) {
@@ -328,8 +334,8 @@ static bool parse_array(struct parse_ctx *ctx, struct ast_node **out)
 		return false;
 	}
 
-	struct array_ast_node *array
-		= (struct array_ast_node *)ast_node_create(AST_ARRAY);
+	struct array_ast_node *array = (struct array_ast_node *)ast_node_create(
+		AST_ARRAY);
 	if (!array) {
 		ctx->p_status = BSHELL_ERR_NO_MEMORY;
 		return false;
@@ -377,8 +383,8 @@ bool parse_fstring(struct parse_ctx *ctx, struct ast_node **out)
 
 	discard_token(ctx);
 
-	struct fstring_ast_node *fstring
-		= (struct fstring_ast_node *)ast_node_create(AST_FSTRING);
+	struct fstring_ast_node *fstring = (struct fstring_ast_node *)
+		ast_node_create(AST_FSTRING);
 	if (!fstring) {
 		ctx->p_status = BSHELL_ERR_NO_MEMORY;
 		return false;
@@ -414,29 +420,36 @@ bool parse_arith_value(struct parse_ctx *ctx, struct ast_node **out)
 	struct lex_token *tok = peek_token(ctx);
 	switch (tok->tok_type) {
 	case TOK_INT: {
-		struct int_ast_node *v
-			= (struct int_ast_node *)ast_node_create(AST_INT);
+		struct int_ast_node *v = (struct int_ast_node *)ast_node_create(
+			AST_INT);
 		v->n_value = claim_token(ctx);
 		*out = (struct ast_node *)v;
 		return true;
 	}
 	case TOK_DOUBLE: {
-		struct double_ast_node *v
-			= (struct double_ast_node *)ast_node_create(AST_DOUBLE);
+		struct double_ast_node *v = (struct double_ast_node *)
+			ast_node_create(AST_DOUBLE);
 		v->n_value = claim_token(ctx);
 		*out = (struct ast_node *)v;
 		return true;
 	}
 	case TOK_STRING: {
-		struct string_ast_node *v
-			= (struct string_ast_node *)ast_node_create(AST_STRING);
+		struct string_ast_node *v = (struct string_ast_node *)
+			ast_node_create(AST_STRING);
+		v->n_value = claim_token(ctx);
+		*out = (struct ast_node *)v;
+		return true;
+	}
+	case TOK_WORD: {
+		struct word_ast_node *v = (struct word_ast_node *)
+			ast_node_create(AST_WORD);
 		v->n_value = claim_token(ctx);
 		*out = (struct ast_node *)v;
 		return true;
 	}
 	case TOK_VAR: {
-		struct var_ast_node *v
-			= (struct var_ast_node *)ast_node_create(AST_VAR);
+		struct var_ast_node *v = (struct var_ast_node *)ast_node_create(
+			AST_VAR);
 		v->n_ident = claim_token(ctx);
 		*out = (struct ast_node *)v;
 		return true;
@@ -492,20 +505,22 @@ void arith_push_operator(struct expr_parse_ctx *state, struct op_ast_node *node)
 	}
 
 	while (true) {
-		fx_queue_entry *top
-			= fx_queue_last(&state->expr_operator_stack);
+		fx_queue_entry *top = fx_queue_last(
+			&state->expr_operator_stack);
 		if (!top) {
 			break;
 		}
 
-		struct ast_node *top_node
-			= fx_unbox(struct ast_node, top, n_entry);
+		struct ast_node *top_node = fx_unbox(
+			struct ast_node,
+			top,
+			n_entry);
 		const struct operator_info *top_op = NULL;
 
 		switch (top_node->n_type) {
 		case AST_OP: {
-			struct op_ast_node *op_node
-				= (struct op_ast_node *)top_node;
+			struct op_ast_node *op_node = (struct op_ast_node *)
+				top_node;
 			top_op = op_node->n_op;
 			break;
 		}
@@ -575,8 +590,8 @@ static bool parse_unary_operator(
 
 	expr->expr_prev = EXPR_C_BINARY_OP;
 
-	struct op_ast_node *op_node
-		= (struct op_ast_node *)ast_node_create(AST_OP);
+	struct op_ast_node *op_node = (struct op_ast_node *)ast_node_create(
+		AST_OP);
 	if (!op_node) {
 		return false;
 	}
@@ -639,8 +654,8 @@ static bool parse_binary_operator(
 
 	expr->expr_prev = EXPR_C_BINARY_OP;
 
-	struct op_ast_node *op_node
-		= (struct op_ast_node *)ast_node_create(AST_OP);
+	struct op_ast_node *op_node = (struct op_ast_node *)ast_node_create(
+		AST_OP);
 	if (!op_node) {
 		return false;
 	}
@@ -669,8 +684,8 @@ static bool parse_comma(struct parse_ctx *ctx, struct expr_parse_ctx *expr)
 		return false;
 	}
 
-	struct array_ast_node *array
-		= (struct array_ast_node *)ast_node_create(AST_ARRAY);
+	struct array_ast_node *array = (struct array_ast_node *)ast_node_create(
+		AST_ARRAY);
 	if (!array) {
 		ctx->p_status = BSHELL_ERR_NO_MEMORY;
 		ast_node_destroy(item);
@@ -706,8 +721,10 @@ static void dump_expr_ctx(struct expr_parse_ctx *expr)
 	printf("op stack:\n");
 	fx_queue_entry *entry = fx_queue_first(&expr->expr_operator_stack);
 	while (entry) {
-		struct ast_node *node
-			= fx_unbox(struct ast_node, entry, n_entry);
+		struct ast_node *node = fx_unbox(
+			struct ast_node,
+			entry,
+			n_entry);
 		print_ast_node(node);
 		entry = fx_queue_next(entry);
 	}
@@ -715,8 +732,10 @@ static void dump_expr_ctx(struct expr_parse_ctx *expr)
 	printf("out queue:\n");
 	entry = fx_queue_first(&expr->expr_out_queue);
 	while (entry) {
-		struct ast_node *node
-			= fx_unbox(struct ast_node, entry, n_entry);
+		struct ast_node *node = fx_unbox(
+			struct ast_node,
+			entry,
+			n_entry);
 		print_ast_node(node);
 		entry = fx_queue_next(entry);
 	}
@@ -755,17 +774,16 @@ bool parse_arith_expr(
 
 		switch (tok->tok_type) {
 		case TOK_LINEFEED:
+			expr.expr_prev_symbol = tok->tok_type;
 			expr.expr_done = true;
 			break;
 		case TOK_WORD: {
 			if (!can_use_command(&expr)) {
-				report_error(
-					ctx,
-					"expected a value expression");
-				expr.expr_fail = true;
+				expr.expr_fail = !parse_operand(ctx, &expr);
 				break;
 			}
 
+			expr.expr_prev_symbol = tok->tok_type;
 			struct ast_node *value = NULL;
 			if (!parse_command(ctx, &value)) {
 				expr.expr_fail = true;
@@ -783,10 +801,12 @@ bool parse_arith_expr(
 		case TOK_DOUBLE:
 		case TOK_STRING:
 		case TOK_STR_START:
+			expr.expr_prev_symbol = tok->tok_type;
 			expr.expr_fail = !parse_operand(ctx, &expr);
 			expr.expr_prev_symbol = tok->tok_type;
 			break;
 		case TOK_OPERATOR:
+			expr.expr_prev_symbol = tok->tok_operator;
 			switch (tok->tok_operator) {
 			/* these two are special cases, as they are both
 			 * unary AND binary operators */
@@ -805,17 +825,21 @@ bool parse_arith_expr(
 				break;
 			case TKOP_BNOT:
 			case TKOP_NOT:
-				expr.expr_fail
-					= !parse_unary_operator(ctx, &expr);
+				expr.expr_fail = !parse_unary_operator(
+					ctx,
+					&expr);
 				break;
 			default:
-				expr.expr_fail
-					= !parse_binary_operator(ctx, &expr);
+				expr.expr_fail = !parse_binary_operator(
+					ctx,
+					&expr);
 				break;
 			}
 			expr.expr_prev_symbol = tok->tok_operator;
 			break;
 		case TOK_SYMBOL:
+			expr.expr_prev_symbol = tok->tok_symbol;
+
 			switch (tok->tok_symbol) {
 			case SYM_SEMICOLON:
 			case SYM_AMPERSAND:
@@ -868,7 +892,6 @@ bool parse_arith_expr(
 			}
 			}
 
-			expr.expr_prev_symbol = tok->tok_symbol;
 			break;
 		default:
 			report_error(

bshell/parse/syntax/generic.c

@@ -155,19 +155,3 @@ bool parse_var(struct parse_ctx *ctx, struct lex_token **out)
 	*out = claim_token(ctx);
 	return true;
 }
-
-bool parse_int(struct parse_ctx *ctx, long long *out)
-{
-	struct lex_token *tok = peek_token(ctx);
-	if (!tok) {
-		return false;
-	}
-
-	if (tok->tok_type != TOK_INT) {
-		return false;
-	}
-
-	*out = tok->tok_int;
-	discard_token(ctx);
-	return true;
-}

bshell/parse/token.h

@@ -2,6 +2,7 @@
 #define IVY_LANG_LEX_H_
 
 #include <fx/queue.h>
+#include <fx/value.h>
 #include <stdbool.h>
 
 struct char_cell {
@@ -129,8 +130,7 @@ struct lex_token {
 		enum token_keyword tok_keyword;
 		enum token_symbol tok_symbol;
 		enum token_operator tok_operator;
-		long long tok_int;
-		double tok_double;
+		fx_value tok_number;
 		char *tok_str;
 	};
 };