diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index ff352f3..eabacde 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -4,6 +4,8 @@ package lexer import ( + "strings" + "github.com/DevLabFoundry/configmanager/v3/config" "github.com/DevLabFoundry/configmanager/v3/internal/token" ) @@ -210,15 +212,15 @@ func (l *Lexer) setTextSeparatorToken() token.Token { // peekIsBeginOfToken attempts to identify the possible token func (l *Lexer) peekIsBeginOfToken(possibleBeginToken []config.ImplementationPrefix, charsRead string) (bool, string, config.ImplementationPrefix) { for _, pbt := range possibleBeginToken { - configToken := "" + var configToken strings.Builder pbtWithTokenSep := string(pbt[len(charsRead):]) + l.config.TokenSeparator() for i := 0; i < len(pbtWithTokenSep); i++ { - configToken += string(l.peekChar()) + configToken.WriteString(string(l.peekChar())) l.readChar() } - if configToken == pbtWithTokenSep { - return true, charsRead + configToken, pbt + if configToken.String() == pbtWithTokenSep { + return true, charsRead + configToken.String(), pbt } l.resetAfterPeek(len(pbtWithTokenSep)) } diff --git a/internal/lexer/lexer_test.go b/internal/lexer/lexer_test.go index ec90e35..d7d0b28 100644 --- a/internal/lexer/lexer_test.go +++ b/internal/lexer/lexer_test.go @@ -90,6 +90,99 @@ META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,vers } } +func Test_Lexer_NextTokenEmbedded(t *testing.T) { + input := `AWSPARAMSTR:///path|key/something/else +AWSSECRETS:///foo|another?param1=foo +META_INCLUDED=VAULT://baz/bar/123|key1.prop2[role=arn:aws:iam::1111111:role,version=1082313]/unused?param2=foo +` + ttests := []struct { + expectedType token.TokenType + expectedLiteral string + }{ + {token.BEGIN_CONFIGMANAGER_TOKEN, "AWSPARAMSTR://"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "path"}, + {token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {token.TEXT, "key"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "something"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "else"}, + {token.NEW_LINE, "\n"}, + {token.BEGIN_CONFIGMANAGER_TOKEN, "AWSSECRETS://"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "foo"}, + {token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {token.TEXT, "another"}, + {token.QUESTION_MARK, "?"}, + {token.TEXT, "param1"}, + {token.EQUALS, "="}, + {token.TEXT, "foo"}, + {token.NEW_LINE, "\n"}, + {token.TEXT, "MET"}, + {token.TEXT, "A"}, + {token.TEXT, "_INCLUDED"}, + // {token.TEXT, "U"}, + // {token.TEXT, "DED"}, + {token.EQUALS, "="}, + {token.BEGIN_CONFIGMANAGER_TOKEN, "VAULT://"}, + {token.TEXT, "baz"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "bar"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "123"}, + {token.CONFIGMANAGER_TOKEN_KEY_PATH_SEPARATOR, "|"}, + {token.TEXT, "key1"}, + {token.DOT, "."}, + {token.TEXT, "prop2"}, + {token.BEGIN_META_CONFIGMANAGER_TOKEN, "["}, + {token.TEXT, "role"}, + {token.EQUALS, "="}, + {token.TEXT, "arn"}, + {token.COLON, ":"}, + {token.TEXT, "aws"}, + {token.COLON, ":"}, + {token.TEXT, "iam"}, + {token.COLON, ":"}, + {token.COLON, ":"}, + {token.TEXT, "1111111"}, + {token.COLON, ":"}, + {token.TEXT, "role"}, + {token.COMMA, ","}, + {token.TEXT, "version"}, + {token.EQUALS, "="}, + {token.TEXT, "1082313"}, + {token.END_META_CONFIGMANAGER_TOKEN, "]"}, + {token.FORWARD_SLASH, "/"}, + {token.TEXT, "unused"}, + {token.QUESTION_MARK, "?"}, + {token.TEXT, "param2"}, + {token.EQUALS, "="}, + {token.TEXT, "foo"}, + {token.NEW_LINE, "\n"}, + {token.EOF, ""}, + } + + l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) + + for i, tt := range ttests { + + tok := l.NextToken() + if tok.Type != tt.expectedType { + t.Fatalf("tests[%d] - tokentype wrong. got=%q, expected=%q", + i, tok.Type, tt.expectedType) + } + + if tok.Literal != tt.expectedLiteral { + t.Fatalf("tests[%d] - literal wrong. got=%q, expected=%q", + i, tok.Literal, tt.expectedLiteral) + } + if tok.Type == token.BEGIN_CONFIGMANAGER_TOKEN { + + } + } +} + func Test_empty_file(t *testing.T) { input := `` l := lexer.New(lexer.Source{Input: input, FullPath: "/foo/bar", FileName: "bar"}, *config.NewConfig()) diff --git a/internal/parser/parser.go b/internal/parser/parser.go index 51ae8e4..8c6b39b 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -3,6 +3,7 @@ package parser import ( "errors" "fmt" + "strings" "os" @@ -199,7 +200,7 @@ func (p *Parser) buildConfigManagerTokenFromBlocks(configManagerToken *config.Pa func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenConfig) error { // advance to next token i.e. post the path separator p.nextToken() - keyPath := "" + keyPath := strings.Builder{} if p.peekTokenIs(token.EOF) { // if the next token EOF we set the path as current token and exit // otherwise we would never hit the below loop @@ -209,7 +210,7 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon for !p.peekTokenIs(token.EOF) { if p.peekTokenIs(token.BEGIN_META_CONFIGMANAGER_TOKEN) { // add current token to the keysPath and move onto the metadata - keyPath += p.currentToken.Literal + keyPath.WriteString(p.currentToken.Literal) p.nextToken() if err := p.buildMetadata(configManagerToken); err != nil { return err @@ -217,20 +218,20 @@ func (p *Parser) buildKeyPathSeparator(configManagerToken *config.ParsedTokenCon break } // touching another token or end of token - if p.peekTokenIs(token.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() { - keyPath += p.currentToken.Literal + if p.peekTokenIs(token.BEGIN_CONFIGMANAGER_TOKEN) || p.peekTokenIsEnd() || !p.peekTokenIs(token.TEXT) { + keyPath.WriteString(p.currentToken.Literal) break } - keyPath += p.currentToken.Literal + keyPath.WriteString(p.currentToken.Literal) p.nextToken() if p.peekTokenIs(token.EOF) { // check if the next token is EOF once advanced // if it is we want to consume current token else it will be skipped - keyPath += p.currentToken.Literal + keyPath.WriteString(p.currentToken.Literal) break } } - configManagerToken.WithKeyPath(keyPath) + configManagerToken.WithKeyPath(keyPath.String()) return nil } diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index b22b026..8c8963a 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -77,6 +77,19 @@ func Test_ParserBlocks(t *testing.T) { {string(config.ParamStorePrefix), "/config", "qp2"}, }, }, + "mixed after key_path": { + `proto://AWSPARAMSTR:///config|endpoint/some/path +foo=AWSSECRETS:///creds|password_one?param2=bar +foo2=AWSSECRETS:///creds|password_two/not/partoftoken?param2=bar +foo3=AWSSECRETS:///creds|password_three/?not/partoftoken?param2=bar +`, + [][3]string{ + {string(config.ParamStorePrefix), "/config", "endpoint"}, + {string(config.SecretMgrPrefix), "/creds", "password_one"}, + {string(config.SecretMgrPrefix), "/creds", "password_two"}, + {string(config.SecretMgrPrefix), "/creds", "password_three"}, + }, + }, } for name, tt := range ttests {