From fad13b5476f626e547f3b8631f40acb1718efde6 Mon Sep 17 00:00:00 2001 From: Max Wash Date: Sun, 24 May 2026 20:13:55 +0100 Subject: [PATCH] parse: lots of improvements --- bshell/parse/lex/arithmetic.c | 13 ++-- bshell/parse/lex/hashtable.c | 9 ++- bshell/parse/lex/lex-internal.h | 7 +- bshell/parse/lex/lex.c | 101 ++++++++++++++++++------- bshell/parse/lex/statement.c | 17 +++-- bshell/parse/syntax/arith.c | 129 +++++++++++++++++++------------- bshell/parse/syntax/generic.c | 16 ---- bshell/parse/token.h | 4 +- 8 files changed, 182 insertions(+), 114 deletions(-) diff --git a/bshell/parse/lex/arithmetic.c b/bshell/parse/lex/arithmetic.c index dbd94c6..ec8f6ef 100644 --- a/bshell/parse/lex/arithmetic.c +++ b/bshell/parse/lex/arithmetic.c @@ -19,18 +19,19 @@ static enum bshell_status arithmetic_hyphen(struct lex_ctx *ctx) } unsigned int token_type = TOK_WORD; - if (convert_word_to_int(tok)) { - token_type = TOK_INT; + if (convert_word_to_number(tok)) { + token_type = tok->tok_type; /* because of APPEND_HYPHEN (which is needed to ensure operator * tokens are detected properly), the resulting number will be * negative. * this token will be preceded by a HYPHEN token, so the number * must be positive */ - tok->tok_int *= -1; + fx_value neg = FX_INT(-1); + fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number); push_symbol(ctx, SYM_HYPHEN); } else if (convert_word_to_operator(ctx, tok)) { - token_type = TOK_OPERATOR; + token_type = tok->tok_operator; } enqueue_token(ctx, tok); @@ -108,8 +109,8 @@ static enum bshell_status arithmetic_word(struct lex_ctx *ctx) bool kw = false, number = false; if (convert_word_to_keyword(word)) { token_type = word->tok_keyword; - } else if (convert_word_to_int(word)) { - token_type = TOK_INT; + } else if (convert_word_to_number(word)) { + token_type = word->tok_type; } handle_lex_state_transition(ctx, token_type); diff --git a/bshell/parse/lex/hashtable.c b/bshell/parse/lex/hashtable.c index 1f62f94..5cbd78d 100644 --- a/bshell/parse/lex/hashtable.c +++ b/bshell/parse/lex/hashtable.c @@ -19,14 +19,15 @@ static enum bshell_status hashtable_hyphen(struct lex_ctx *ctx) } unsigned int token_type = TOK_WORD; - if (convert_word_to_int(tok)) { - token_type = TOK_INT; + if (convert_word_to_number(tok)) { + token_type = tok->tok_type; /* because of APPEND_HYPHEN (which is needed to ensure operator * tokens are detected properly), the resulting number will be * negative. * this token will be preceded by a HYPHEN token, so the number * must be positive */ - tok->tok_int *= -1; + fx_value neg = FX_INT(-1); + fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number); push_symbol(ctx, SYM_HYPHEN); } else if (convert_word_to_operator(ctx, tok)) { token_type = tok->tok_operator; @@ -102,7 +103,7 @@ static enum bshell_status hashtable_word(struct lex_ctx *ctx) return status; } - convert_word_to_int(word); + convert_word_to_number(word); handle_lex_state_transition(ctx, word->tok_type); enqueue_token(ctx, word); diff --git a/bshell/parse/lex/lex-internal.h b/bshell/parse/lex/lex-internal.h index d39c857..215ee32 100644 --- a/bshell/parse/lex/lex-internal.h +++ b/bshell/parse/lex/lex-internal.h @@ -123,8 +123,11 @@ extern fx_wchar peek2_char_noread(struct lex_ctx *ctx); extern void advance_char(struct lex_ctx *ctx); extern void advance_char_noread(struct lex_ctx *ctx); -extern bool string_is_valid_number(const char *s, long long *out); -extern bool convert_word_to_int(struct lex_token *tok); +extern bool string_is_valid_number( + const char *s, + fx_value *out, + enum token_type *out_type); +extern bool convert_word_to_number(struct lex_token *tok); extern bool convert_word_to_keyword(struct lex_token *tok); extern bool convert_word_to_operator( struct lex_ctx *ctx, diff --git a/bshell/parse/lex/lex.c b/bshell/parse/lex/lex.c index e1ea7bf..0d50706 100644 --- a/bshell/parse/lex/lex.c +++ b/bshell/parse/lex/lex.c @@ -599,44 +599,88 @@ static size_t get_int_multiplier_by_suffix(const char *suffix) return 0; } -bool string_is_valid_number(const char *s, long long *out) +static bool string_could_be_double(const char *s) { - if (s[0] == '\0') { - return NULL; - } - - int base = get_int_base_by_prefix(&s); - - char *ep = NULL; - long long value = strtoll(s, &ep, base); - if (*ep == '\0') { - out && (*out = value); + if (strchr(s, '.')) { return true; } - size_t multiplier = get_int_multiplier_by_suffix(ep); - if (multiplier != 0) { - out && (*out = value * multiplier); + if (strchr(s, 'e') || strchr(s, 'E')) { return true; } return false; } -bool convert_word_to_int(struct lex_token *tok) +bool string_is_valid_number( + const char *s, + fx_value *out, + enum token_type *out_type) +{ + if (s[0] == '\0') { + return NULL; + } + + char *ep = NULL; + double dvalue = strtod(s, &ep); + if (string_could_be_double(s) && *ep == '\0') { + if (out) { + *out = FX_DOUBLE(dvalue); + } + + if (out_type) { + *out_type = TOK_DOUBLE; + } + + return true; + } + + int base = get_int_base_by_prefix(&s); + + long long ivalue = strtoll(s, &ep, base); + if (*ep == '\0') { + if (out) { + *out = FX_LONGLONG(ivalue); + } + + if (out_type) { + *out_type = TOK_INT; + } + + return true; + } + + size_t multiplier = get_int_multiplier_by_suffix(ep); + if (multiplier != 0) { + if (out) { + *out = FX_LONGLONG(ivalue * multiplier); + } + + if (out_type) { + *out_type = TOK_INT; + } + + return true; + } + + return false; +} + +bool convert_word_to_number(struct lex_token *tok) { if (!lex_token_has_string_value(tok)) { return false; } const char *s = tok->tok_str; - long long value = 0; - if (!string_is_valid_number(s, &value)) { + fx_value value; + enum token_type type; + if (!string_is_valid_number(s, &value, &type)) { return false; } - lex_token_change_type(tok, TOK_INT); - tok->tok_int = value; + lex_token_change_type(tok, type); + tok->tok_number = value; return true; } @@ -873,13 +917,15 @@ enum bshell_status read_word( break; } - if (char_can_begin_symbol(ctx, c)) { - done = true; - break; - } - const char *s = fx_string_get_cstr(tmp); - if (number_recog && string_is_valid_number(s, NULL)) { + if (number_recog && string_is_valid_number(s, NULL, NULL)) { + if (c == '.') { + fx_string_append_wc(tmp, c); + set_token_end(ctx); + advance_char(ctx); + continue; + } + if (char_can_begin_symbol_in_state( ctx, c, @@ -889,6 +935,11 @@ enum bshell_status read_word( } } + if (char_can_begin_symbol(ctx, c)) { + done = true; + break; + } + if (!fx_wchar_is_alpha(c)) { op = get_operator_with_string(ctx, s); if (op != TKOP_NONE) { diff --git a/bshell/parse/lex/statement.c b/bshell/parse/lex/statement.c index 0c3f3d2..b43fbc4 100644 --- a/bshell/parse/lex/statement.c +++ b/bshell/parse/lex/statement.c @@ -20,18 +20,19 @@ static enum bshell_status statement_hyphen(struct lex_ctx *ctx) unsigned int token_type = TOK_WORD; - if (convert_word_to_int(tok)) { - token_type = TOK_INT; + if (convert_word_to_number(tok)) { + token_type = tok->tok_type; /* because of APPEND_HYPHEN (which is needed to ensure operator * tokens are detected properly), the resulting number will be * negative. * this token will be preceded by a HYPHEN token, so the number * must be positive */ - tok->tok_int *= -1; + fx_value neg = FX_INT(-1); + fx_value_multiply(&tok->tok_number, &neg, &tok->tok_number); push_symbol(ctx, SYM_HYPHEN); } else if (convert_word_to_operator(ctx, tok)) { - token_type = TOK_OPERATOR; + token_type = tok->tok_operator; } handle_lex_state_transition(ctx, token_type); @@ -114,8 +115,8 @@ static enum bshell_status statement_word(struct lex_ctx *ctx) if (enable_keywords && convert_word_to_keyword(word)) { token = word->tok_keyword; - } else if (convert_word_to_int(word)) { - token = TOK_INT; + } else if (convert_word_to_number(word)) { + token = word->tok_type; } handle_lex_state_transition(ctx, token); @@ -163,7 +164,9 @@ static const struct lex_state_link links[] = { LINK_PUSH(SYM_DQUOTE, LEX_STATE_STRING, 0), /* arithmetic tokens */ LINK_CHANGE(TOK_KEYWORD, LEX_STATE_ARITHMETIC), + LINK_CHANGE(SYM_SQUOTE, LEX_STATE_ARITHMETIC), LINK_CHANGE(TOK_INT, LEX_STATE_ARITHMETIC), + LINK_CHANGE(TOK_DOUBLE, LEX_STATE_ARITHMETIC), LINK_PUSH(SYM_DOLLAR, LEX_STATE_ARITHMETIC, 0), LINK_PUSH_WITH_TERM( SYM_DOLLAR_LEFT_BRACE, @@ -171,6 +174,7 @@ static const struct lex_state_link links[] = { 0, SYM_RIGHT_BRACE), LINK_CHANGE(SYM_AT_LEFT_BRACE, LEX_STATE_ARITHMETIC), + LINK_CHANGE(SYM_AT_LEFT_PAREN, LEX_STATE_ARITHMETIC), LINK_PUSH_WITH_TERM( SYM_AT_LEFT_BRACE, LEX_STATE_HASHTABLE, @@ -226,6 +230,7 @@ static const unsigned int symbols[] = { SYM_DQUOTE, SYM_HASH, SYM_AT, + SYM_AT_LEFT_PAREN, SYM_AT_LEFT_BRACE, SYM_PIPE, SYM_COMMA, diff --git a/bshell/parse/syntax/arith.c b/bshell/parse/syntax/arith.c index 91510ee..c84f487 100644 --- a/bshell/parse/syntax/arith.c +++ b/bshell/parse/syntax/arith.c @@ -47,8 +47,10 @@ static bool finalise_expr( break; } - struct op_ast_node *node - = fx_unbox(struct op_ast_node, entry, n_base.n_entry); + struct op_ast_node *node = fx_unbox( + struct op_ast_node, + entry, + n_base.n_entry); if (!node) { /* this should never happen */ return false; @@ -72,8 +74,10 @@ static bool finalise_expr( int i = 0; while (entry) { - struct ast_node *item - = fx_unbox(struct ast_node, entry, n_entry); + struct ast_node *item = fx_unbox( + struct ast_node, + entry, + n_entry); fx_queue_entry *next = fx_queue_next(entry); fx_queue_delete(&ctx->expr_out_queue, entry); @@ -115,12 +119,14 @@ static bool finalise_expr( if (op->op_arity == OPA_BINARY) { tmp = fx_queue_pop_back(&q); - op_node->n_left - = fx_unbox(struct ast_node, tmp, n_entry); + op_node->n_left = fx_unbox( + struct ast_node, + tmp, + n_entry); if (op_node->n_left) { - op_node->n_left->n_parent - = (struct ast_node *)op_node; + op_node->n_left->n_parent = (struct ast_node *) + op_node; #if 0 ast_node_extend_bounds_recursive( @@ -248,8 +254,8 @@ static bool parse_hashtable(struct parse_ctx *ctx, struct ast_node **out) parse_linefeed(ctx); - struct hashtable_ast_node *table - = (struct hashtable_ast_node *)ast_node_create(AST_HASHTABLE); + struct hashtable_ast_node *table = (struct hashtable_ast_node *) + ast_node_create(AST_HASHTABLE); if (!table) { ctx->p_status = BSHELL_ERR_NO_MEMORY; return false; @@ -264,14 +270,14 @@ static bool parse_hashtable(struct parse_ctx *ctx, struct ast_node **out) parse_linefeed(ctx); - struct hashtable_item_ast_node *item - = (struct hashtable_item_ast_node *)ast_node_create( - AST_HASHTABLE_ITEM); + struct hashtable_item_ast_node *item = (struct + hashtable_item_ast_node + *) + ast_node_create(AST_HASHTABLE_ITEM); struct lex_token *tok = NULL; if (parse_word(ctx, &tok)) { - struct string_ast_node *v - = (struct string_ast_node *)ast_node_create( - AST_STRING); + struct string_ast_node *v = (struct string_ast_node *) + ast_node_create(AST_STRING); v->n_value = tok; item->n_key = (struct ast_node *)v; } else if (!parse_arith_value(ctx, &item->n_key)) { @@ -328,8 +334,8 @@ static bool parse_array(struct parse_ctx *ctx, struct ast_node **out) return false; } - struct array_ast_node *array - = (struct array_ast_node *)ast_node_create(AST_ARRAY); + struct array_ast_node *array = (struct array_ast_node *)ast_node_create( + AST_ARRAY); if (!array) { ctx->p_status = BSHELL_ERR_NO_MEMORY; return false; @@ -377,8 +383,8 @@ bool parse_fstring(struct parse_ctx *ctx, struct ast_node **out) discard_token(ctx); - struct fstring_ast_node *fstring - = (struct fstring_ast_node *)ast_node_create(AST_FSTRING); + struct fstring_ast_node *fstring = (struct fstring_ast_node *) + ast_node_create(AST_FSTRING); if (!fstring) { ctx->p_status = BSHELL_ERR_NO_MEMORY; return false; @@ -414,29 +420,36 @@ bool parse_arith_value(struct parse_ctx *ctx, struct ast_node **out) struct lex_token *tok = peek_token(ctx); switch (tok->tok_type) { case TOK_INT: { - struct int_ast_node *v - = (struct int_ast_node *)ast_node_create(AST_INT); + struct int_ast_node *v = (struct int_ast_node *)ast_node_create( + AST_INT); v->n_value = claim_token(ctx); *out = (struct ast_node *)v; return true; } case TOK_DOUBLE: { - struct double_ast_node *v - = (struct double_ast_node *)ast_node_create(AST_DOUBLE); + struct double_ast_node *v = (struct double_ast_node *) + ast_node_create(AST_DOUBLE); v->n_value = claim_token(ctx); *out = (struct ast_node *)v; return true; } case TOK_STRING: { - struct string_ast_node *v - = (struct string_ast_node *)ast_node_create(AST_STRING); + struct string_ast_node *v = (struct string_ast_node *) + ast_node_create(AST_STRING); + v->n_value = claim_token(ctx); + *out = (struct ast_node *)v; + return true; + } + case TOK_WORD: { + struct word_ast_node *v = (struct word_ast_node *) + ast_node_create(AST_WORD); v->n_value = claim_token(ctx); *out = (struct ast_node *)v; return true; } case TOK_VAR: { - struct var_ast_node *v - = (struct var_ast_node *)ast_node_create(AST_VAR); + struct var_ast_node *v = (struct var_ast_node *)ast_node_create( + AST_VAR); v->n_ident = claim_token(ctx); *out = (struct ast_node *)v; return true; @@ -492,20 +505,22 @@ void arith_push_operator(struct expr_parse_ctx *state, struct op_ast_node *node) } while (true) { - fx_queue_entry *top - = fx_queue_last(&state->expr_operator_stack); + fx_queue_entry *top = fx_queue_last( + &state->expr_operator_stack); if (!top) { break; } - struct ast_node *top_node - = fx_unbox(struct ast_node, top, n_entry); + struct ast_node *top_node = fx_unbox( + struct ast_node, + top, + n_entry); const struct operator_info *top_op = NULL; switch (top_node->n_type) { case AST_OP: { - struct op_ast_node *op_node - = (struct op_ast_node *)top_node; + struct op_ast_node *op_node = (struct op_ast_node *) + top_node; top_op = op_node->n_op; break; } @@ -575,8 +590,8 @@ static bool parse_unary_operator( expr->expr_prev = EXPR_C_BINARY_OP; - struct op_ast_node *op_node - = (struct op_ast_node *)ast_node_create(AST_OP); + struct op_ast_node *op_node = (struct op_ast_node *)ast_node_create( + AST_OP); if (!op_node) { return false; } @@ -639,8 +654,8 @@ static bool parse_binary_operator( expr->expr_prev = EXPR_C_BINARY_OP; - struct op_ast_node *op_node - = (struct op_ast_node *)ast_node_create(AST_OP); + struct op_ast_node *op_node = (struct op_ast_node *)ast_node_create( + AST_OP); if (!op_node) { return false; } @@ -669,8 +684,8 @@ static bool parse_comma(struct parse_ctx *ctx, struct expr_parse_ctx *expr) return false; } - struct array_ast_node *array - = (struct array_ast_node *)ast_node_create(AST_ARRAY); + struct array_ast_node *array = (struct array_ast_node *)ast_node_create( + AST_ARRAY); if (!array) { ctx->p_status = BSHELL_ERR_NO_MEMORY; ast_node_destroy(item); @@ -706,8 +721,10 @@ static void dump_expr_ctx(struct expr_parse_ctx *expr) printf("op stack:\n"); fx_queue_entry *entry = fx_queue_first(&expr->expr_operator_stack); while (entry) { - struct ast_node *node - = fx_unbox(struct ast_node, entry, n_entry); + struct ast_node *node = fx_unbox( + struct ast_node, + entry, + n_entry); print_ast_node(node); entry = fx_queue_next(entry); } @@ -715,8 +732,10 @@ static void dump_expr_ctx(struct expr_parse_ctx *expr) printf("out queue:\n"); entry = fx_queue_first(&expr->expr_out_queue); while (entry) { - struct ast_node *node - = fx_unbox(struct ast_node, entry, n_entry); + struct ast_node *node = fx_unbox( + struct ast_node, + entry, + n_entry); print_ast_node(node); entry = fx_queue_next(entry); } @@ -755,17 +774,16 @@ bool parse_arith_expr( switch (tok->tok_type) { case TOK_LINEFEED: + expr.expr_prev_symbol = tok->tok_type; expr.expr_done = true; break; case TOK_WORD: { if (!can_use_command(&expr)) { - report_error( - ctx, - "expected a value expression"); - expr.expr_fail = true; + expr.expr_fail = !parse_operand(ctx, &expr); break; } + expr.expr_prev_symbol = tok->tok_type; struct ast_node *value = NULL; if (!parse_command(ctx, &value)) { expr.expr_fail = true; @@ -783,10 +801,12 @@ bool parse_arith_expr( case TOK_DOUBLE: case TOK_STRING: case TOK_STR_START: + expr.expr_prev_symbol = tok->tok_type; expr.expr_fail = !parse_operand(ctx, &expr); expr.expr_prev_symbol = tok->tok_type; break; case TOK_OPERATOR: + expr.expr_prev_symbol = tok->tok_operator; switch (tok->tok_operator) { /* these two are special cases, as they are both * unary AND binary operators */ @@ -805,17 +825,21 @@ bool parse_arith_expr( break; case TKOP_BNOT: case TKOP_NOT: - expr.expr_fail - = !parse_unary_operator(ctx, &expr); + expr.expr_fail = !parse_unary_operator( + ctx, + &expr); break; default: - expr.expr_fail - = !parse_binary_operator(ctx, &expr); + expr.expr_fail = !parse_binary_operator( + ctx, + &expr); break; } expr.expr_prev_symbol = tok->tok_operator; break; case TOK_SYMBOL: + expr.expr_prev_symbol = tok->tok_symbol; + switch (tok->tok_symbol) { case SYM_SEMICOLON: case SYM_AMPERSAND: @@ -868,7 +892,6 @@ bool parse_arith_expr( } } - expr.expr_prev_symbol = tok->tok_symbol; break; default: report_error( diff --git a/bshell/parse/syntax/generic.c b/bshell/parse/syntax/generic.c index 27125d2..0b0a848 100644 --- a/bshell/parse/syntax/generic.c +++ b/bshell/parse/syntax/generic.c @@ -155,19 +155,3 @@ bool parse_var(struct parse_ctx *ctx, struct lex_token **out) *out = claim_token(ctx); return true; } - -bool parse_int(struct parse_ctx *ctx, long long *out) -{ - struct lex_token *tok = peek_token(ctx); - if (!tok) { - return false; - } - - if (tok->tok_type != TOK_INT) { - return false; - } - - *out = tok->tok_int; - discard_token(ctx); - return true; -} diff --git a/bshell/parse/token.h b/bshell/parse/token.h index 3500b7f..1bf4f07 100644 --- a/bshell/parse/token.h +++ b/bshell/parse/token.h @@ -2,6 +2,7 @@ #define IVY_LANG_LEX_H_ #include +#include #include struct char_cell { @@ -129,8 +130,7 @@ struct lex_token { enum token_keyword tok_keyword; enum token_symbol tok_symbol; enum token_operator tok_operator; - long long tok_int; - double tok_double; + fx_value tok_number; char *tok_str; }; };