From cfaf53040b908ce9ac57ac3a44193cd7a19fd0a4 Mon Sep 17 00:00:00 2001 From: Max Wash Date: Sat, 9 May 2026 21:21:51 +0100 Subject: [PATCH] parse: improve command arg parsing --- bshell/parse/lex.c | 313 ++++++++++++++++++---------------- bshell/parse/lex.h | 25 +-- bshell/parse/syntax.h | 15 +- bshell/parse/syntax/arith.c | 18 +- bshell/parse/syntax/command.c | 40 +++-- bshell/parse/syntax/expr.c | 9 +- bshell/parse/syntax/generic.c | 38 +++-- 7 files changed, 255 insertions(+), 203 deletions(-) diff --git a/bshell/parse/lex.c b/bshell/parse/lex.c index 073c486..e9a3d12 100644 --- a/bshell/parse/lex.c +++ b/bshell/parse/lex.c @@ -48,29 +48,21 @@ static struct lex_token_def symbols[] = { }; static const size_t nr_symbols = sizeof symbols / sizeof symbols[0]; -typedef enum bshell_status (*pump_token_impl)(struct lex_ctx *, enum lex_flags); +typedef enum bshell_status (*pump_token_impl)(struct lex_ctx *); -static enum bshell_status do_pump_token_normal( - struct lex_ctx *, - enum lex_flags); -static enum bshell_status do_pump_token_string( - struct lex_ctx *ctx, - enum lex_flags); +static enum bshell_status do_pump_token_normal(struct lex_ctx *); +static enum bshell_status do_pump_token_string(struct lex_ctx *ctx); static const pump_token_impl token_pump_functions[] = { [LEX_STATE_NORMAL] = do_pump_token_normal, [LEX_STATE_STRING] = do_pump_token_string, [LEX_STATE_INTERPOLATION] = do_pump_token_normal, }; -static bool char_can_begin_symbol( - struct lex_ctx *ctx, - char c, - enum lex_flags flags); +static bool char_can_begin_symbol(struct lex_ctx *ctx, char c); static bool char_can_begin_symbol_in_context( struct lex_ctx *ctx, char c, - enum token_type context, - enum lex_flags flags); + enum token_type context); static struct lex_state *push_lex_state( struct lex_ctx *ctx, @@ -209,6 +201,7 @@ enum bshell_status lex_ctx_init( ctx->lex_status = BSHELL_SUCCESS; ctx->lex_buf = fx_stringstream_create(); ctx->lex_sym_tree = build_symbol_tree(); + ctx->lex_wordbuf = fx_string_create(); push_lex_state(ctx, LEX_STATE_NORMAL); ctx->lex_src = src; ctx->lex_ch = FX_WCHAR_INVALID; @@ -278,6 +271,10 @@ static fx_wchar peek_char_noread(struct lex_ctx *ctx) static void __advance_char(struct lex_ctx *ctx, bool noread) { + if (!fx_wchar_is_space(ctx->lex_ch)) { + fx_string_append_wc(ctx->lex_wordbuf, ctx->lex_ch); + } + if (ctx->lex_ch != FX_WCHAR_INVALID) { ctx->lex_ch = FX_WCHAR_INVALID; return; @@ -422,99 +419,37 @@ static bool convert_word_to_int( return ok; } -static struct lex_token *create_alt_token( - struct lex_ctx *ctx, - struct lex_token *tok, - enum lex_flags flags) -{ - if (ctx->lex_alt) { - return ctx->lex_alt; - } - - bool converted = false; - - struct lex_token alt = {0}; - - if (flags & LEX_ENABLE_KEYWORD) { - converted = convert_word_to_keyword(tok, &alt); - } - - if (!converted && (flags & LEX_ENABLE_INT)) { - converted = convert_word_to_int(tok, &alt); - } - - if (!converted) { - return NULL; - } - - ctx->lex_alt = lex_token_create(alt.tok_type); - if (!ctx->lex_alt) { - ctx->lex_status = BSHELL_ERR_NO_MEMORY; - return NULL; - } - - memcpy(ctx->lex_alt, &alt, sizeof alt); - - return ctx->lex_alt; -} - -static struct lex_token *get_next_token( - struct lex_ctx *ctx, - enum lex_flags flags) +static struct lex_token *get_next_token(struct lex_ctx *ctx) { fx_queue_entry *entry = fx_queue_first(&ctx->lex_tokens); - struct lex_token *tok = fx_unbox(struct lex_token, entry, tok_entry); + return fx_unbox(struct lex_token, entry, tok_entry); +} - if (!tok || !CONVERSION_REQUESTED(flags)) { - return tok; - } - - if (ctx->lex_alt) { - return ctx->lex_alt; - } - - ctx->lex_alt = create_alt_token(ctx, tok, flags); - - return ctx->lex_alt ? ctx->lex_alt : tok; +static struct lex_token *get_next_word(struct lex_ctx *ctx) +{ + fx_queue_entry *entry = fx_queue_first(&ctx->lex_words); + return fx_unbox(struct lex_token, entry, tok_entry); } static void enqueue_token(struct lex_ctx *ctx, struct lex_token *tok) { + if (tok && (ctx->lex_flags & LEX_PRINT_TOKENS)) { + print_lex_token(tok); + } + fx_queue_push_back(&ctx->lex_tokens, &tok->tok_entry); } -static struct lex_token *dequeue_next_token( - struct lex_ctx *ctx, - enum lex_flags flags) +static struct lex_token *dequeue_next_token(struct lex_ctx *ctx) { fx_queue_entry *entry = fx_queue_pop_front(&ctx->lex_tokens); - struct lex_token *tok = fx_unbox(struct lex_token, entry, tok_entry); + return fx_unbox(struct lex_token, entry, tok_entry); +} - if (!tok || !CONVERSION_REQUESTED(flags)) { - if (ctx->lex_alt) { - lex_token_destroy(ctx->lex_alt); - ctx->lex_alt = NULL; - } - - return tok; - } - - if (ctx->lex_alt) { - lex_token_destroy(tok); - tok = ctx->lex_alt; - ctx->lex_alt = NULL; - return tok; - } - - ctx->lex_alt = create_alt_token(ctx, tok, flags); - - if (ctx->lex_alt) { - lex_token_destroy(tok); - tok = ctx->lex_alt; - ctx->lex_alt = NULL; - } - - return tok; +static struct lex_token *dequeue_next_word(struct lex_ctx *ctx) +{ + fx_queue_entry *entry = fx_queue_pop_front(&ctx->lex_words); + return fx_unbox(struct lex_token, entry, tok_entry); } static fx_string *get_temp_string(struct lex_ctx *ctx) @@ -541,7 +476,7 @@ static enum bshell_status push_symbol( return BSHELL_SUCCESS; } -static enum bshell_status read_word(struct lex_ctx *ctx, enum lex_flags flags) +static enum bshell_status read_word(struct lex_ctx *ctx) { fx_string *tmp = get_temp_string(ctx); bool word_is_number = false; @@ -558,12 +493,12 @@ static enum bshell_status read_word(struct lex_ctx *ctx, enum lex_flags flags) break; } - if (word_is_number && char_can_begin_symbol(ctx, c, flags)) { + if (word_is_number && char_can_begin_symbol(ctx, c)) { done = true; break; } - if (char_can_begin_symbol_in_context(ctx, c, TOK_WORD, flags)) { + if (char_can_begin_symbol_in_context(ctx, c, TOK_WORD)) { done = true; break; } @@ -851,7 +786,7 @@ static enum bshell_status read_interpolation_marker(struct lex_ctx *ctx) return BSHELL_SUCCESS; } -static enum bshell_status read_symbol(struct lex_ctx *ctx, enum lex_flags flags) +static enum bshell_status read_symbol(struct lex_ctx *ctx) { struct lex_state *state = get_lex_state(ctx); enum lex_token_flags required_flags = 0; @@ -859,10 +794,6 @@ static enum bshell_status read_symbol(struct lex_ctx *ctx, enum lex_flags flags) required_flags |= LEX_TOKEN_ENABLE_IN_STRING; } - if (!(flags & LEX_ENABLE_SYMBOL)) { - required_flags |= LEX_TOKEN_ENABLE_IN_WORD; - } - struct lex_symbol_node *node = ctx->lex_sym_tree; char prev = 0; @@ -932,10 +863,7 @@ static enum bshell_status read_symbol(struct lex_ctx *ctx, enum lex_flags flags) return BSHELL_SUCCESS; } -static bool char_can_begin_symbol( - struct lex_ctx *ctx, - char c, - enum lex_flags flags) +static bool char_can_begin_symbol(struct lex_ctx *ctx, char c) { struct lex_state *state = get_lex_state(ctx); enum lex_token_flags required_flags = 0; @@ -943,10 +871,6 @@ static bool char_can_begin_symbol( required_flags |= LEX_TOKEN_ENABLE_IN_STRING; } - if (!(flags & LEX_ENABLE_SYMBOL)) { - required_flags |= LEX_TOKEN_ENABLE_IN_WORD; - } - for (size_t i = 0; i < nr_symbols; i++) { if (symbols[i].name[0] != c) { continue; @@ -965,8 +889,7 @@ static bool char_can_begin_symbol( static bool char_can_begin_symbol_in_context( struct lex_ctx *ctx, char c, - enum token_type context, - enum lex_flags flags) + enum token_type context) { enum lex_token_flags required_flags = 0; switch (context) { @@ -1011,7 +934,7 @@ static enum bshell_status read_string_content(struct lex_ctx *ctx) break; } - if (char_can_begin_symbol(ctx, c, 0)) { + if (char_can_begin_symbol(ctx, c)) { break; } @@ -1032,16 +955,14 @@ static enum bshell_status read_string_content(struct lex_ctx *ctx) return BSHELL_SUCCESS; } -static enum bshell_status do_pump_token_string( - struct lex_ctx *ctx, - enum lex_flags flags) +static enum bshell_status do_pump_token_string(struct lex_ctx *ctx) { fx_wchar c = peek_char(ctx); enum bshell_status status = BSHELL_SUCCESS; bool ok = false; - if (char_can_begin_symbol(ctx, c, flags)) { - status = read_symbol(ctx, flags); + if (char_can_begin_symbol(ctx, c)) { + status = read_symbol(ctx); ok = true; } @@ -1052,15 +973,32 @@ static enum bshell_status do_pump_token_string( return status; } -static enum bshell_status do_pump_token_normal( - struct lex_ctx *ctx, - enum lex_flags flags) +static void flush_wordbuf(struct lex_ctx *ctx) +{ + if (fx_string_get_size(ctx->lex_wordbuf, FX_STRLEN_NORMAL) == 0) { + return; + } + + struct lex_token *tok = lex_token_create_with_string( + TOK_WORD, + fx_string_get_cstr(ctx->lex_wordbuf)); + fx_queue_push_back(&ctx->lex_words, &tok->tok_entry); + fx_string_clear(ctx->lex_wordbuf); +} + +static enum bshell_status do_pump_token_normal(struct lex_ctx *ctx) { enum bshell_status status = BSHELL_SUCCESS; fx_wchar c = peek_char(ctx); + bool whitespace = false; bool newline = false; + if (fx_wchar_is_space(c)) { + flush_wordbuf(ctx); + whitespace = true; + } + while (fx_wchar_is_space(c)) { if (c == '\n') { newline = true; @@ -1073,6 +1011,9 @@ static enum bshell_status do_pump_token_normal( if (newline) { struct lex_token *tok = lex_token_create(TOK_LINEFEED); enqueue_token(ctx, tok); + } + + if (whitespace) { return BSHELL_SUCCESS; } @@ -1080,70 +1021,148 @@ static enum bshell_status do_pump_token_normal( return read_flag(ctx); } - if (char_can_begin_symbol(ctx, c, flags)) { - return read_symbol(ctx, flags); + if (char_can_begin_symbol(ctx, c)) { + return read_symbol(ctx); } - return read_word(ctx, flags); + return read_word(ctx); } -static enum bshell_status pump_tokens(struct lex_ctx *ctx, enum lex_flags flags) +static enum bshell_status pump_tokens(struct lex_ctx *ctx) { enum bshell_status status = BSHELL_SUCCESS; - while (fx_queue_empty(&ctx->lex_tokens) && status == BSHELL_SUCCESS) { + while (fx_queue_empty(&ctx->lex_words) && status == BSHELL_SUCCESS) { struct lex_state *state = get_lex_state(ctx); pump_token_impl impl = token_pump_functions[state->s_type]; - status = impl(ctx, flags); + status = impl(ctx); } return status; } -struct lex_token *lex_ctx_peek(struct lex_ctx *ctx, enum lex_flags flags) +static bool any_tokens_available(struct lex_ctx *ctx) { - struct lex_token *tok = get_next_token(ctx, flags); + return !fx_queue_empty(&ctx->lex_tokens) + || !fx_queue_empty(&ctx->lex_words); +} + +static void discard_all_tokens(struct lex_ctx *ctx) +{ + fx_queue_entry *cur = fx_queue_first(&ctx->lex_tokens); + while (cur) { + struct lex_token *tok + = fx_unbox(struct lex_token, cur, tok_entry); + if (tok->tok_type == TOK_LINEFEED) { + break; + } + + fx_queue_pop_front(&ctx->lex_tokens); + lex_token_destroy(tok); + cur = fx_queue_first(&ctx->lex_tokens); + } +} + +static void discard_all_words(struct lex_ctx *ctx) +{ + fx_queue_entry *cur = fx_queue_pop_front(&ctx->lex_words); + while (cur) { + struct lex_token *tok + = fx_unbox(struct lex_token, cur, tok_entry); + lex_token_destroy(tok); + cur = fx_queue_pop_front(&ctx->lex_words); + } +} + +struct lex_token *lex_ctx_peek(struct lex_ctx *ctx) +{ + struct lex_token *tok = get_next_token(ctx); if (tok) { return tok; } - pump_tokens(ctx, flags); - tok = get_next_token(ctx, flags); - if (tok && (ctx->lex_flags & LEX_PRINT_TOKENS)) { - print_lex_token(tok); - } + discard_all_words(ctx); + pump_tokens(ctx); + tok = get_next_token(ctx); return tok; } -struct lex_token *lex_ctx_claim(struct lex_ctx *ctx, enum lex_flags flags) +struct lex_token *lex_ctx_peek_word(struct lex_ctx *ctx) { - struct lex_token *tok = dequeue_next_token(ctx, flags); + struct lex_token *tok = get_next_word(ctx); if (tok) { return tok; } - if (fx_queue_empty(&ctx->lex_tokens)) { - pump_tokens(ctx, flags); + discard_all_tokens(ctx); + pump_tokens(ctx); + tok = get_next_token(ctx); - tok = get_next_token(ctx, flags); - if (tok && (ctx->lex_flags & LEX_PRINT_TOKENS)) { - print_lex_token(tok); - } - } - - return dequeue_next_token(ctx, flags); + return tok; } -void lex_ctx_discard(struct lex_ctx *ctx, enum lex_flags flags) +struct lex_token *lex_ctx_claim(struct lex_ctx *ctx) { - struct lex_token *tok = dequeue_next_token(ctx, 0); + struct lex_token *tok = dequeue_next_token(ctx); + if (tok) { + struct lex_token *tmp = get_next_token(ctx); + + if (tmp && tmp->tok_type == TOK_LINEFEED) { + tmp = dequeue_next_word(ctx); + lex_token_destroy(tmp); + } + + return tok; + } + + if (fx_queue_empty(&ctx->lex_tokens)) { + pump_tokens(ctx); + + tok = get_next_token(ctx); + } + + return dequeue_next_token(ctx); +} + +struct lex_token *lex_ctx_claim_word(struct lex_ctx *ctx) +{ + /* since we're claiming the whole word, discard any sub-tokens already + * generated up to the next linefeed */ + discard_all_tokens(ctx); + struct lex_token *tok = dequeue_next_word(ctx); + if (tok) { + return tok; + } + + if (fx_queue_empty(&ctx->lex_words)) { + pump_tokens(ctx); + + tok = get_next_token(ctx); + } + + return dequeue_next_word(ctx); +} + +void lex_ctx_discard(struct lex_ctx *ctx) +{ + struct lex_token *tok = dequeue_next_token(ctx); if (tok) { lex_token_destroy(tok); + tok = get_next_token(ctx); + + /* if the next token is a linefeed, we've reached the end + * of the current word, and should discard it */ + if (tok && tok->tok_type == TOK_LINEFEED) { + tok = dequeue_next_word(ctx); + lex_token_destroy(tok); + } + return; } + /* request more tokens if necessary */ if (fx_queue_empty(&ctx->lex_tokens)) { - pump_tokens(ctx, flags); + pump_tokens(ctx); } } diff --git a/bshell/parse/lex.h b/bshell/parse/lex.h index 97bc3cf..b974bdd 100644 --- a/bshell/parse/lex.h +++ b/bshell/parse/lex.h @@ -54,12 +54,19 @@ struct lex_state { struct lex_ctx { enum lex_flags lex_flags; - fx_queue lex_tokens; + /* lex_ctx maintains two queues of tokens. + * lex_words is a simple queue of all WORDS scanned by the lexer, + * without any further parsing applied to potentially convert the words + * into TOK_INT, TOK_SYMBOL, etc. + * lex_tokens represent all of the tokens generated by applying the + * aforementioned parsing. + * the two queues are kept in sync such that, as tokens are dequeued + * from one queue, the other queue is moved forward too. */ + fx_queue lex_tokens, lex_words; struct line_source *lex_src; fx_stringstream *lex_buf; - fx_string *lex_tmp; + fx_string *lex_tmp, *lex_wordbuf; fx_wchar lex_ch; - struct lex_token *lex_alt; fx_queue lex_state; struct lex_symbol_node *lex_sym_tree; enum bshell_status lex_status; @@ -71,12 +78,10 @@ extern enum bshell_status lex_ctx_init( struct line_source *src); extern enum bshell_status lex_ctx_cleanup(struct lex_ctx *ctx); -extern struct lex_token *lex_ctx_peek( - struct lex_ctx *ctx, - enum lex_flags flags); -extern struct lex_token *lex_ctx_claim( - struct lex_ctx *ctx, - enum lex_flags flags); -extern void lex_ctx_discard(struct lex_ctx *ctx, enum lex_flags flags); +extern struct lex_token *lex_ctx_peek(struct lex_ctx *ctx); +extern struct lex_token *lex_ctx_peek_word(struct lex_ctx *ctx); +extern struct lex_token *lex_ctx_claim(struct lex_ctx *ctx); +extern struct lex_token *lex_ctx_claim_word(struct lex_ctx *ctx); +extern void lex_ctx_discard(struct lex_ctx *ctx); #endif diff --git a/bshell/parse/syntax.h b/bshell/parse/syntax.h index 72388a5..6e40947 100644 --- a/bshell/parse/syntax.h +++ b/bshell/parse/syntax.h @@ -12,29 +12,26 @@ enum parse_operand_flags { OPERAND_BASIC = 0x01u, }; -extern struct lex_token *peek_token( - struct parse_ctx *ctx, - enum lex_flags flags); -extern enum token_type peek_token_type( - struct parse_ctx *ctx, - enum lex_flags flags); +extern struct lex_token *peek_token(struct parse_ctx *ctx); +extern enum token_type peek_token_type(struct parse_ctx *ctx); extern enum token_keyword peek_unknown_keyword(struct parse_ctx *ctx); extern enum token_symbol peek_unknown_symbol(struct parse_ctx *ctx); extern bool peek_int(struct parse_ctx *ctx); -extern struct lex_token *claim_token( - struct parse_ctx *ctx, - enum lex_flags flags); +extern struct lex_token *claim_token(struct parse_ctx *ctx); +extern struct lex_token *claim_glob_word(struct parse_ctx *ctx); extern void discard_token(struct parse_ctx *ctx); extern bool peek_linefeed(struct parse_ctx *ctx); extern bool peek_symbol(struct parse_ctx *ctx, enum token_symbol sym); +extern struct lex_token *peek_glob_word(struct parse_ctx *ctx); extern bool parse_linefeed(struct parse_ctx *ctx); extern bool parse_symbol(struct parse_ctx *ctx, enum token_symbol sym); extern bool parse_keyword(struct parse_ctx *ctx, enum token_keyword kw); extern bool parse_int(struct parse_ctx *ctx, long long *out); extern bool parse_flag(struct parse_ctx *ctx, struct lex_token **out); +extern struct lex_token *parse_glob_word(struct parse_ctx *ctx); extern bool peek_arith_expr(struct parse_ctx *ctx); extern bool parse_arith_expr(struct parse_ctx *ctx, struct ast_node **out); diff --git a/bshell/parse/syntax/arith.c b/bshell/parse/syntax/arith.c index 4758f4c..86a30b5 100644 --- a/bshell/parse/syntax/arith.c +++ b/bshell/parse/syntax/arith.c @@ -2,7 +2,23 @@ bool peek_arith_expr(struct parse_ctx *ctx) { - return false; + switch (peek_token_type(ctx)) { + case TOK_SYMBOL: + switch (peek_unknown_symbol(ctx)) { + case SYM_PLUS: + case SYM_HYPHEN: + return true; + default: + return false; + } + case TOK_INT: + case TOK_DOUBLE: + case TOK_STRING: + case TOK_STR_START: + return true; + default: + return false; + } } bool parse_arith_expr(struct parse_ctx *ctx, struct ast_node **out) diff --git a/bshell/parse/syntax/command.c b/bshell/parse/syntax/command.c index 2c8c345..b473871 100644 --- a/bshell/parse/syntax/command.c +++ b/bshell/parse/syntax/command.c @@ -4,7 +4,7 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) { - struct lex_token *tok = peek_token(ctx, 0); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; } @@ -20,11 +20,12 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) return false; } - n->n_value = claim_token(ctx, 0); + n->n_value = claim_token(ctx); *out = (struct ast_node *)n; return true; } +#if 0 case TOK_FLAG: { struct word_ast_node *n = (struct word_ast_node *)ast_node_create(AST_WORD); @@ -33,10 +34,11 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) return false; } - n->n_value = claim_token(ctx, 0); + n->n_value = claim_token(ctx); *out = (struct ast_node *)n; return true; } +#endif case TOK_VAR: { struct var_ast_node *n @@ -46,7 +48,7 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) return false; } - n->n_ident = claim_token(ctx, 0); + n->n_ident = claim_token(ctx); *out = (struct ast_node *)n; return true; } @@ -60,7 +62,7 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) return false; } - n->n_ident = claim_token(ctx, 0); + n->n_ident = claim_token(ctx); *out = (struct ast_node *)n; return true; } @@ -73,12 +75,24 @@ static bool parse_cmdcall_arg(struct parse_ctx *ctx, struct ast_node **out) return false; } - n->n_value = claim_token(ctx, 0); + n->n_value = claim_token(ctx); + *out = (struct ast_node *)n; + return true; + } + + default: { + struct word_ast_node *n + = (struct word_ast_node *)ast_node_create(AST_WORD); + if (!n) { + ctx->p_status = BSHELL_ERR_NO_MEMORY; + return false; + } + + n->n_value = claim_glob_word(ctx); *out = (struct ast_node *)n; return true; } - default: return false; } @@ -125,7 +139,7 @@ static bool parse_redirect_to_file_squashed( const char *str, struct ast_node **out) { - struct lex_token *tok = peek_token(ctx, 0); + struct lex_token *tok = peek_token(ctx); if (*str == '\0') { return false; } @@ -140,7 +154,7 @@ static bool parse_redirect_to_file_squashed( redirect->n_out_is_expr = false; redirect->n_out_path = str; - redirect->n_out_tok = claim_token(ctx, 0); + redirect->n_out_tok = claim_token(ctx); *out = (struct ast_node *)redirect; return true; @@ -174,7 +188,7 @@ static bool parse_redirect_to_file_separate( bool parse_redirect(struct parse_ctx *ctx, struct ast_node **out) { - struct lex_token *tok = peek_token(ctx, 0); + struct lex_token *tok = peek_token(ctx); if (!tok || tok->tok_type != TOK_WORD) { return false; } @@ -234,7 +248,7 @@ static bool peek_cmdcall_item(struct parse_ctx *ctx, bool unrestricted) * a command with this token as its name, the call operator must be * used. */ - switch (peek_token_type(ctx, LEX_ENABLE_INT | LEX_ENABLE_KEYWORD)) { + switch (peek_token_type(ctx)) { case TOK_KEYWORD: case TOK_INT: case TOK_DOUBLE: @@ -285,7 +299,7 @@ bool parse_cmdcall(struct parse_ctx *ctx, struct ast_node **out) return false; } - struct lex_token *tok = peek_token(ctx, 0); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; } @@ -301,7 +315,7 @@ bool parse_cmdcall(struct parse_ctx *ctx, struct ast_node **out) break; } - struct lex_token *tok = peek_token(ctx, 0); + struct lex_token *tok = peek_token(ctx); if (!tok) { break; } diff --git a/bshell/parse/syntax/expr.c b/bshell/parse/syntax/expr.c index 7c33b89..1d48dd6 100644 --- a/bshell/parse/syntax/expr.c +++ b/bshell/parse/syntax/expr.c @@ -3,13 +3,12 @@ bool parse_expr(struct parse_ctx *ctx, struct ast_node **out) { bool ok = false; - if (peek_command(ctx)) { - ok = parse_command(ctx, out); - } - - if (!ok && peek_arith_expr(ctx)) { + if (peek_arith_expr(ctx)) { ok = parse_arith_expr(ctx, out); } + if (!ok && peek_command(ctx)) { + ok = parse_command(ctx, out); + } return ok; } diff --git a/bshell/parse/syntax/generic.c b/bshell/parse/syntax/generic.c index d3eaaa6..67dc413 100644 --- a/bshell/parse/syntax/generic.c +++ b/bshell/parse/syntax/generic.c @@ -3,47 +3,49 @@ #include "../syntax.h" #include "../token.h" -#define DEFAULT_LEX_FLAGS \ - (LEX_ENABLE_INT | LEX_ENABLE_KEYWORD | LEX_ENABLE_SYMBOL) - -struct lex_token *claim_token(struct parse_ctx *ctx, enum lex_flags flags) +struct lex_token *claim_token(struct parse_ctx *ctx) { - return lex_ctx_claim(ctx->p_src, flags); + return lex_ctx_claim(ctx->p_src); +} + +struct lex_token *claim_glob_word(struct parse_ctx *ctx) +{ + return lex_ctx_claim_word(ctx->p_src); } void discard_token(struct parse_ctx *ctx) { - return lex_ctx_discard(ctx->p_src, DEFAULT_LEX_FLAGS); + return lex_ctx_discard(ctx->p_src); } -struct lex_token *peek_token(struct parse_ctx *ctx, enum lex_flags flags) +struct lex_token *peek_token(struct parse_ctx *ctx) { - return lex_ctx_peek(ctx->p_src, flags); + return lex_ctx_peek(ctx->p_src); } -enum token_type peek_token_type(struct parse_ctx *ctx, enum lex_flags flags) +enum token_type peek_token_type(struct parse_ctx *ctx) { - struct lex_token *tok = peek_token(ctx, flags); + struct lex_token *tok = peek_token(ctx); return tok ? tok->tok_type : TOK_NONE; } enum token_symbol peek_unknown_symbol(struct parse_ctx *ctx) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); return (tok && tok->tok_type == TOK_SYMBOL) ? tok->tok_symbol : SYM_NONE; } enum token_keyword peek_unknown_keyword(struct parse_ctx *ctx) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); return (tok && tok->tok_type == TOK_KEYWORD) ? tok->tok_keyword : KW_NONE; } bool peek_linefeed(struct parse_ctx *ctx) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (tok && tok->tok_type == TOK_LINEFEED) { return true; } @@ -53,7 +55,7 @@ bool peek_linefeed(struct parse_ctx *ctx) bool peek_symbol(struct parse_ctx *ctx, enum token_symbol sym) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; } @@ -71,7 +73,7 @@ bool peek_symbol(struct parse_ctx *ctx, enum token_symbol sym) bool parse_linefeed(struct parse_ctx *ctx) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (tok && tok->tok_type == TOK_LINEFEED) { discard_token(ctx); return true; @@ -82,7 +84,7 @@ bool parse_linefeed(struct parse_ctx *ctx) bool parse_symbol(struct parse_ctx *ctx, enum token_symbol sym) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; } @@ -101,7 +103,7 @@ bool parse_symbol(struct parse_ctx *ctx, enum token_symbol sym) bool parse_keyword(struct parse_ctx *ctx, enum token_keyword kw) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; } @@ -120,7 +122,7 @@ bool parse_keyword(struct parse_ctx *ctx, enum token_keyword kw) bool parse_int(struct parse_ctx *ctx, long long *out) { - struct lex_token *tok = peek_token(ctx, DEFAULT_LEX_FLAGS); + struct lex_token *tok = peek_token(ctx); if (!tok) { return false; }