parse: lex: fix scanning of sub-expressions and fstrings in statement mode

bshell/parse/lex/statement.c

@@ -53,6 +53,8 @@ static enum bshell_status statement_symbol(struct lex_ctx *ctx)
 
 	struct lex_token *tok = NULL;
 	switch (sym->id) {
+	case SYM_DQUOTE:
+		return BSHELL_SUCCESS;
 	case SYM_HYPHEN:
 		return statement_hyphen(ctx);
 	case SYM_SQUOTE:
@@ -141,6 +143,7 @@ static enum bshell_status statement_pump_token(struct lex_ctx *ctx)
 	if (newline) {
 		struct lex_token *tok = lex_token_create(TOK_LINEFEED);
 		enqueue_token(ctx, tok);
+		handle_lex_state_transition(ctx, TOK_LINEFEED);
 		return BSHELL_SUCCESS;
 	}
 
@@ -167,14 +170,20 @@ static const struct lex_state_link links[] = {
 	LINK_PUSH(SYM_AT_LEFT_BRACE, LEX_STATE_HASHTABLE, 0),
 	LINK_PUSH(SYM_AT, LEX_STATE_ARITHMETIC, 0),
 	LINK_CHANGE(SYM_LEFT_PAREN, LEX_STATE_ARITHMETIC),
-	LINK_PUSH(
+	LINK_CHANGE(SYM_BANG, LEX_STATE_ARITHMETIC),
+	LINK_PUSH_WITH_TERM(
 		SYM_LEFT_PAREN,
 		LEX_STATE_STATEMENT,
-		STATEMENT_F_DISABLE_KEYWORDS),
+		STATEMENT_F_DISABLE_KEYWORDS,
+		SYM_RIGHT_PAREN),
 
 	/* statement tokens */
 	LINK_PUSH(SYM_LEFT_BRACE, LEX_STATE_STATEMENT, 0),
-	LINK_PUSH(SYM_DOLLAR_LEFT_PAREN, LEX_STATE_STATEMENT, 0),
+	LINK_PUSH_WITH_TERM(
+		SYM_DOLLAR_LEFT_PAREN,
+		LEX_STATE_STATEMENT,
+		0,
+		SYM_RIGHT_PAREN),
 
 	/* command tokens */
 	LINK_CHANGE(KW_FUNC, LEX_STATE_COMMAND),
@@ -199,12 +208,10 @@ static const unsigned int operators[] = {
 
 static const unsigned int symbols[] = {
 	SYM_AMPERSAND,
+	SYM_BANG,
 	SYM_SQUOTE,
 	SYM_DQUOTE,
 	SYM_HASH,
-	SYM_DOLLAR,
-	SYM_DOLLAR_LEFT_PAREN,
-	SYM_DOLLAR_LEFT_BRACE,
 	SYM_AT,
 	SYM_AT_LEFT_BRACE,
 	SYM_PIPE,