Fix: key path token lookup in a more complex string

internal/lexer/lexer.go

@@ -4,6 +4,8 @@
 package lexer
 
 import (
+	"strings"
+
 	"github.com/DevLabFoundry/configmanager/v3/config"
 	"github.com/DevLabFoundry/configmanager/v3/internal/token"
 )
@@ -210,15 +212,15 @@ func (l *Lexer) setTextSeparatorToken() token.Token {
 // peekIsBeginOfToken attempts to identify the possible token
 func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPrefix, charsRead string) (bool, string, config.ImplementationPrefix) {
 	for _, pbt := range possibleBeginToken {
-		configToken := ""
+		var configToken strings.Builder
 		pbtWithTokenSep := string(pbt[len(charsRead):]) + l.config.TokenSeparator()
 		for i := 0; i < len(pbtWithTokenSep); i++ {
-			configToken += string(l.peekChar())
+			configToken.WriteString(string(l.peekChar()))
 			l.readChar()
 		}
 
-		if configToken == pbtWithTokenSep {
-			return true, charsRead + configToken, pbt
+		if configToken.String() == pbtWithTokenSep {
+			return true, charsRead + configToken.String(), pbt
 		}
 		l.resetAfterPeek(len(pbtWithTokenSep))
 	}

internal/lexer/lexer_test.go

@@ -90,6 +90,99 @@
 	}
 }
 
+func Test_Lexer_NextTokenEmbedded(t *testing.T) {
+	input := `AWSPARAMSTR:///path|key/something/else
+AWSSECRETS:///foo|another?param1=foo
+META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,version=1082313]/unused?param2=foo
+`
+	ttests := []struct {
+		expectedType    token.TokenType
+		expectedLiteral string
+	}{
+		{token.BEGIN_CONFIGMANAGER_TOKEN, "AWSPARAMSTR://"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "path"},
+		{token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"},
+		{token.TEXT, "key"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "something"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "else"},
+		{token.NEW_LINE, "\n"},
+		{token.BEGIN_CONFIGMANAGER_TOKEN, "AWSSECRETS://"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "foo"},
+		{token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"},
+		{token.TEXT, "another"},
+		{token.QUESTION_MARK, "?"},
+		{token.TEXT, "param1"},
+		{token.EQUALS, "="},
+		{token.TEXT, "foo"},
+		{token.NEW_LINE, "\n"},
+		{token.TEXT, "MET"},
+		{token.TEXT, "A"},
+		{token.TEXT, "_INCLUDED"},
+		// {token.TEXT, "U"},
+		// {token.TEXT, "DED"},
+		{token.EQUALS, "="},
+		{token.BEGIN_CONFIGMANAGER_TOKEN, "VAULT://"},
+		{token.TEXT, "baz"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "bar"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "123"},
+		{token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"},
+		{token.TEXT, "key1"},
+		{token.DOT, "."},
+		{token.TEXT, "prop2"},
+		{token.BEGIN_META_CONFIGMANAGER_TOKEN, "["},
+		{token.TEXT, "role"},
+		{token.EQUALS, "="},
+		{token.TEXT, "arn"},
+		{token.COLON, ":"},
+		{token.TEXT, "aws"},
+		{token.COLON, ":"},
+		{token.TEXT, "iam"},
+		{token.COLON, ":"},
+		{token.COLON, ":"},
+		{token.TEXT, "1111111"},
+		{token.COLON, ":"},
+		{token.TEXT, "role"},
+		{token.COMMA, ","},
+		{token.TEXT, "version"},
+		{token.EQUALS, "="},
+		{token.TEXT, "1082313"},
+		{token.END_META_CONFIGMANAGER_TOKEN, "]"},
+		{token.FORWARD_SLASH, "/"},
+		{token.TEXT, "unused"},
+		{token.QUESTION_MARK, "?"},
+		{token.TEXT, "param2"},
+		{token.EQUALS, "="},
+		{token.TEXT, "foo"},
+		{token.NEW_LINE, "\n"},
+		{token.EOF, ""},
+	}
+
+	l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig())
+
+	for i, tt := range ttests {
+
+		tok := l.NextToken()
+		if tok.Type != tt.expectedType {
+			t.Fatalf("tests[%d] - tokentype wrong. got=%q, expected=%q",
+				i, tok.Type, tt.expectedType)
+		}
+
+		if tok.Literal != tt.expectedLiteral {
+			t.Fatalf("tests[%d] - literal wrong. got=%q, expected=%q",
+				i, tok.Literal, tt.expectedLiteral)
+		}
+		if tok.Type == token.BEGIN_CONFIGMANAGER_TOKEN {
+
+		}
+	}
+}
+
 func Test_empty_file(t *testing.T) {
 	input := ``
 	l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig())

internal/parser/parser.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"errors"
 	"fmt"
+	"strings"
 
 	"os"
 
@@ -199,7 +200,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa
 func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenConfig) error {
 	// advance to next token i.e. post the path separator
 	p.nextToken()
-	keyPath := ""
+	keyPath := strings.Builder{}
 	if p.peekTokenIs(token.EOF) {
 		// if the next token EOF we set the path as current token and exit
 		// otherwise we would never hit the below loop
@@ -209,28 +210,28 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon
 	for !p.peekTokenIs(token.EOF) {
 		if p.peekTokenIs(token.BEGIN_META_CONFIGMANAGER_TOKEN) {
 			// add current token to the keysPath and move onto the metadata
-			keyPath += p.currentToken.Literal
+			keyPath.WriteString(p.currentToken.Literal)
 			p.nextToken()
 			if err := p.buildMetadata(configManagerToken); err != nil {
 				return err
 			}
 			break
 		}
 		// touching another token or end of token
-		if p.peekTokenIs(token.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() {
-			keyPath += p.currentToken.Literal
+		if p.peekTokenIs(token.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() || !p.peekTokenIs(token.TEXT) {
+			keyPath.WriteString(p.currentToken.Literal)
 			break
 		}
-		keyPath += p.currentToken.Literal
+		keyPath.WriteString(p.currentToken.Literal)
 		p.nextToken()
 		if p.peekTokenIs(token.EOF) {
 			// check if the next token is EOF once advanced
 			// if it is we want to consume current token else it will be skipped
-			keyPath += p.currentToken.Literal
+			keyPath.WriteString(p.currentToken.Literal)
 			break
 		}
 	}
-	configManagerToken.WithKeyPath(keyPath)
+	configManagerToken.WithKeyPath(keyPath.String())
 	return nil
 }
 

internal/parser/parser_test.go

@@ -77,6 +77,19 @@ func Test_ParserBlocks(t *testing.T) {
 				{string(config.ParamStorePrefix), "/config", "qp2"},
 			},
 		},
+		"mixed after key_path": {
+			`proto://AWSPARAMSTR:///config|endpoint/some/path
+foo=AWSSECRETS:///creds|password_one?param2=bar
+foo2=AWSSECRETS:///creds|password_two/not/partoftoken?param2=bar
+foo3=AWSSECRETS:///creds|password_three/?not/partoftoken?param2=bar
+`,
+			[][3]string{
+				{string(config.ParamStorePrefix), "/config", "endpoint"},
+				{string(config.SecretMgrPrefix), "/creds", "password_one"},
+				{string(config.SecretMgrPrefix), "/creds", "password_two"},
+				{string(config.SecretMgrPrefix), "/creds", "password_three"},
+			},
+		},
 	}
 
 	for name, tt := range ttests {