From a72488afd299b63bc16c4e6eca1bc1bede6687d3 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Wed, 28 Oct 2026 18:01:09 +0000 Subject: [PATCH] feat(runner): pin a target to an ip with a JSON input line A JSON input line pins its target to an ip, in the format of nuclei target lines, so one list feeds both tools: {"target": "https://app.example.com", "ip": "192.0.2.10"} Like curl --resolve, but on every port of the host, connections to the hostname go to the ip, redirects included, while the hostname stays in the Host header and SNI and is not resolved. Each target and ip pair is probed separately, and host_ip and the output show the ip. An input line is parsed once, where it enters probing, and the code after it uses the parsed target and ip. One value in the context of a request describes its host: the ip it is pinned to and the ip it is dialed to. -pa pins its ips the same way, instead of the fastdialer.IP value, which sent every dial of a request to the ip, redirects to other hosts and the proxy included. Options that do not connect through the ip stop the run when a pinned line is read, -pa is refused with -proxy, and a proxy from the environment fails the request, reported once with a hint to use NO_PROXY. The ips of a host are kept apart for -maxhr, -sr and -e cdn. An invalid line is reported and skipped. Closes #2667 --- README.md | 11 + common/httpx/httpx.go | 5 +- common/httpx/requesthost.go | 126 ++++++ common/httpx/requesthost_test.go | 114 +++++ common/httpx/tls.go | 2 +- runner/main_test.go | 23 + runner/runner.go | 223 ++++++---- runner/targetline.go | 220 +++++++++ runner/targetline_test.go | 739 +++++++++++++++++++++++++++++++ 9 files changed, 1370 insertions(+), 93 deletions(-) create mode 100644 common/httpx/requesthost.go create mode 100644 common/httpx/requesthost_test.go create mode 100644 runner/main_test.go create mode 100644 runner/targetline.go create mode 100644 runner/targetline_test.go diff --git a/README.md b/README.md index e84dd9805..bfab87245 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,17 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` +- A target can be pinned to an IP with a JSON input line, in the format of nuclei target lines, like `curl --resolve` but on every port of its host: + ``` + {"target": "https://app.example.com", "ip": "192.0.2.10"} + {"target": "https://app.example.com", "ip": "192.0.2.11"} + ``` + - Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. + - The keys nuclei uses to select templates, such as `tags`, are ignored, and any other key makes the line invalid. An invalid line is reported and skipped. The `input` of a result is the line in a canonical form. + - `-u` splits its value on commas, so pass such lines in a file (`-l`) or on stdin. + - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`. + - `-pa` probes each IP of a host the same way, so it cannot be used with `-proxy` either. The IPs of a host are kept apart: `-maxhr` counts their failures and `-sr` stores their responses separately. + - Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 91c86a49d..4e6e8b678 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) { gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation") } transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, + DialContext: recordDialedIP(httpx.Dialer.Dial), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) { // Clone before protocol-specific ALPN is applied. Impersonation keeps the // previous base config; utls does not take NextProtos from it. impersonateTLS := transport.TLSClientConfig.Clone() - transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS) + transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS)) switch httpx.Options.Protocol { case HTTP11: @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } + transport.Proxy = refusePinned(transport.Proxy) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/requesthost.go b/common/httpx/requesthost.go new file mode 100644 index 000000000..c8f00703b --- /dev/null +++ b/common/httpx/requesthost.go @@ -0,0 +1,126 @@ +package httpx + +import ( + "context" + "errors" + "net" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "golang.org/x/net/idna" +) + +// ErrPinnedThroughProxy is the error of a request to a pinned host that would +// go through a proxy, which resolves the host itself. +var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") + +// requestHost is the host a request is made to: the ip it is pinned to, if +// any, and the ip of the last connection dialed to it. +type requestHost struct { + name string + pinned string + mu sync.Mutex + dialed string +} + +// requestHostKey is the context key of the requestHost of WithHost. +type requestHostKey struct{} + +// WithHost returns a copy of ctx for requests to host. When ip is set, they +// connect to it, like curl --resolve but on every port of host, while the host +// keeps its name for the Host header and SNI. The ip they are dialed to is +// recorded for DialedIP, while the dialer only keeps the last ip dialed to a +// host by any request. Requests to other hosts, such as redirect targets, are +// resolved as usual and not recorded. +func WithHost(ctx context.Context, host, ip string) context.Context { + host = canonicalHost(host) + if ip != "" { + ctx = fastdialer.WithResolveOverride(ctx, host, ip) + } + return context.WithValue(ctx, requestHostKey{}, &requestHost{name: host, pinned: ip}) +} + +// PinnedContext returns a new context for another request derived from req, +// in which the host of req keeps the ip it is pinned to. The other values of +// the context of req, such as a trace, are not kept. +func PinnedContext(req *http.Request) context.Context { + return WithHost(context.Background(), req.URL.Hostname(), pinnedIP(req)) +} + +// DialedIP returns the ip of the last connection dialed to the host of a +// WithHost context, or "" if none was. +func DialedIP(ctx context.Context) string { + host, ok := ctx.Value(requestHostKey{}).(*requestHost) + if !ok { + return "" + } + host.mu.Lock() + defer host.mu.Unlock() + return host.dialed +} + +// hostOf returns the requestHost of ctx if name, however it is spelled, is its +// host, and nil otherwise. +func hostOf(ctx context.Context, name string) *requestHost { + host, ok := ctx.Value(requestHostKey{}).(*requestHost) + if !ok || host.name != canonicalHost(name) { + return nil + } + return host +} + +// pinnedIP returns the ip the host of req is pinned to in its context, if any. +func pinnedIP(req *http.Request) string { + if host := hostOf(req.Context(), req.URL.Hostname()); host != nil { + return host.pinned + } + return "" +} + +// canonicalHost folds the case, the IDNA form and a trailing dot of host, as +// fastdialer keys its resolve overrides, so that a host matches the requests +// and dials to it however it is spelled. +func canonicalHost(host string) string { + if ascii, err := idna.Lookup.ToASCII(host); err == nil { + host = ascii + } + return strings.TrimSuffix(strings.ToLower(host), ".") +} + +// refusePinned wraps the proxy of a transport so that a request to a pinned +// host fails instead of going through the proxy. +func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if proxyURL != nil && pinnedIP(req) != "" { + return nil, ErrPinnedThroughProxy + } + return proxyURL, err + } +} + +// dialFunc dials addr, as net.Dialer.DialContext does. +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// recordDialedIP wraps dial to record the ip of a connection to the host of a +// WithHost context. A connection to another host, such as a proxy, is not +// recorded. +func recordDialedIP(dial dialFunc) dialFunc { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + if err != nil { + return conn, err + } + name, _, _ := net.SplitHostPort(addr) + if host := hostOf(ctx, name); host != nil { + ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + host.mu.Lock() + host.dialed = ip + host.mu.Unlock() + } + return conn, nil + } +} diff --git a/common/httpx/requesthost_test.go b/common/httpx/requesthost_test.go new file mode 100644 index 000000000..3ee1b406f --- /dev/null +++ b/common/httpx/requesthost_test.go @@ -0,0 +1,114 @@ +package httpx + +import ( + "context" + "net" + "net/http" + "net/http/httptrace" + "net/url" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefusePinned(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") + require.NoError(t, err) + proxy := refusePinned(http.ProxyURL(proxyURL)) + request := func(ctx context.Context, rawURL string) *http.Request { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + require.NoError(t, err) + return req + } + + // a pinned host however it is spelled + for host, rawURL := range map[string]string{"Pinned.Test.": "https://PINNED.test.:8443/", "xn--bcher-kva.test": "https://bücher.test/"} { + _, err = proxy(request(WithHost(context.Background(), host, "192.0.2.10"), rawURL)) + require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL) + } + + pinned := WithHost(context.Background(), "pinned.test", "192.0.2.10") + got, err := proxy(request(pinned, "https://other.test/")) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a request to another host, as a redirect, keeps the proxy") + + got, err = proxy(request(WithHost(context.Background(), "plain.test", ""), "https://plain.test/")) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a host that is not pinned keeps the proxy") + + got, err = refusePinned(http.ProxyURL(nil))(request(pinned, "https://pinned.test/")) + require.NoError(t, err) + require.Nil(t, got, "a pinned request without a proxy") +} + +// A proxy set in the environment is refused for a pinned request as well. Go +// reads the proxies of the environment once, so the test runs in a new process. +func TestRefusePinned_environment(t *testing.T) { + if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" { + cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$") + // the lower case names first: on Windows, the last of two spellings wins + cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + return + } + + options := DefaultOptions + options.RetryMax = 0 + h, err := New(&options) + require.NoError(t, err) + req, err := h.NewRequestWithContext(WithHost(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/") + require.NoError(t, err) + _, err = h.Do(req, UnsafeOptions{}) + require.ErrorIs(t, err, ErrPinnedThroughProxy) +} + +func TestCanonicalHost(t *testing.T) { + for host, want := range map[string]string{ + "Pinned.Test.": "pinned.test", + "bücher.test": "xn--bcher-kva.test", + "BÜCHER.test": "xn--bcher-kva.test", + "bücher。test": "xn--bcher-kva.test", + "xn--bcher-kva.test": "xn--bcher-kva.test", + } { + require.Equal(t, want, canonicalHost(host), host) + } +} + +func TestPinnedContext(t *testing.T) { + ctx := httptrace.WithClientTrace(WithHost(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil) + require.NoError(t, err) + + derived := req.Clone(PinnedContext(req)) + require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept") + require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not") + + other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + require.Empty(t, pinnedIP(other.Clone(PinnedContext(other)))) +} + +func TestRecordDialedIP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = listener.Close() }() + // every address is dialed to the listener + dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String()) + }) + ctx := WithHost(context.Background(), "dialed.test", "") + + conn, err := dial(ctx, "tcp", "proxy.test:3128") + require.NoError(t, err) + _ = conn.Close() + require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") + + // net/http dials a host as it is spelled + conn, err = dial(ctx, "tcp", "DIALED.test.:80") + require.NoError(t, err) + _ = conn.Close() + require.Equal(t, "127.0.0.1", DialedIP(ctx)) +} diff --git a/common/httpx/tls.go b/common/httpx/tls.go index 87cee7444..77040ad7c 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -63,7 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response { } // canonical net concatenation host = net.JoinHostPort(hostname, fmt.Sprint(port)) - ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout) + ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout) defer cancel() tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host) if err != nil { diff --git a/runner/main_test.go b/runner/main_test.go new file mode 100644 index 000000000..3b74a2d2b --- /dev/null +++ b/runner/main_test.go @@ -0,0 +1,23 @@ +package runner + +import ( + "os" + "strings" + "testing" +) + +// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in +// the environment, which would resolve it itself, so that they run behind a +// proxy too. Go reads the proxies of the environment once per process, so +// t.Setenv cannot change them for a test. +func TestMain(m *testing.M) { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + noProxy = os.Getenv("no_proxy") + } + noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",") + for _, name := range []string{"NO_PROXY", "no_proxy"} { + _ = os.Setenv(name, noProxy) + } + os.Exit(m.Run()) +} diff --git a/runner/runner.go b/runner/runner.go index 97b9c7b88..87529648f 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -33,7 +33,6 @@ import ( "github.com/happyhackingspace/dit" "github.com/mfonda/simhash" asnmap "github.com/projectdiscovery/asnmap/libs" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -99,6 +98,7 @@ type Runner struct { httpApiEndpoint *Server authProvider authprovider.AuthProvider interruptCh chan struct{} + pinnedProxyWarning sync.Once } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -218,6 +218,10 @@ func New(options *Options) (*Runner, error) { options.Proxy = options.SocksProxy } httpxOptions.Proxy = options.Proxy + // a proxy resolves the host itself, so it cannot reach each ip of a host + if options.ProbeAllIPS && options.Proxy != "" { + return nil, errors.New("-pa/-probe-all-ips cannot be used with -proxy, which resolves the host itself") + } httpxOptions.Unsafe = options.Unsafe httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN @@ -556,6 +560,10 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { + target, ok := r.readInputLine(target) + if !ok { + continue + } expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { numHosts += expandedTarget @@ -738,6 +746,18 @@ func (r *Runner) streamInput() (chan string, error) { return true } } + // sendLine normalizes, deduplicates and sends an input line, returning + // false if interrupted. + sendLine := func(item string) bool { + item, ok := r.readInputLine(item) + if !ok { + return true + } + if r.options.SkipDedupe || r.testAndSet(item) { + return trySend(item) + } + return true + } if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling @@ -760,14 +780,9 @@ func (r *Runner) streamInput() (chan string, error) { } } else { for item, err := range fileutil.Lines(r.options.InputFile) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } else if r.options.InputFile != "" { @@ -777,27 +792,17 @@ func (r *Runner) streamInput() (chan string, error) { } for _, file := range files { for item, err := range fileutil.Lines(file) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } if fileutil.HasStdin() { for item, err := range fileutil.LinesReader(os.Stdin) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } }() @@ -807,7 +812,10 @@ func (r *Runner) streamInput() (chan string, error) { func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { - target := strings.TrimSpace(scanner.Text()) + target, ok := r.readInputLine(scanner.Text()) + if !ok { + continue + } // Used just to get the exact number of targets expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { @@ -1538,9 +1546,13 @@ func (r *Runner) RunEnumeration() { } } + line, ok := r.inputOf(k) + if !ok { + return nil + } protocol := r.options.protocol // attempt to parse url as is - if u, err := r.parseURL(k); err == nil { + if u, err := r.parseURL(line.Target); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -1552,10 +1564,10 @@ func (r *Runner) RunEnumeration() { for _, p := range r.options.requestURIs { scanopts := r.scanopts.Clone() scanopts.RequestURI = p - r.process(k, wg, r.hp, protocol, scanopts, output) + r.process(line, wg, r.hp, protocol, scanopts, output) } } else { - r.process(k, wg, r.hp, protocol, &r.scanopts, output) + r.process(line, wg, r.hp, protocol, &r.scanopts, output) } } } @@ -1679,11 +1691,14 @@ func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } +// Process probes t, read as an input line is. func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { - r.process(t, wg, r.hp, protocol, scanopts, output) + if line, ok := r.inputOf(t); ok { + r.process(line, wg, r.hp, protocol, scanopts, output) + } } -func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { +func (r *Runner) process(line targetLine, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { // attempts to set the workpool size to the number of threads if r.options.Threads > 0 && wg.Size != r.options.Threads { if err := wg.Resize(context.Background(), r.options.Threads); err != nil { @@ -1696,7 +1711,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT protocols = []string{httpx.HTTPS, httpx.HTTP} } - for target := range r.targets(hp, t) { + for target := range r.targetsOf(hp, line) { // if no custom ports specified then test the default ones if len(customport.Ports) == 0 { for _, method := range scanopts.Methods { @@ -1706,18 +1721,10 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT wg.Add() go func(target httpx.Target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, method, t, scanopts) + result := r.analyze(hp, protocol, target, method, line, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } if scanopts.CSPProbe && result.CSPData != nil { scanopts.CSPProbe = false @@ -1727,7 +1734,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT if !r.testAndSet(tt) { continue } - r.process(tt, wg, hp, protocol, scanopts, output) + r.process(targetLine{Target: tt}, wg, hp, protocol, scanopts, output) } } }(target, method, prot) @@ -1760,18 +1767,10 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT urlx.UpdatePort(fmt.Sprint(port)) target.Host = urlx.String() } - result := r.analyze(hp, protocol, target, method, t, scanopts) + result := r.analyze(hp, protocol, target, method, line, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } }(port, target, method, wantedProtocol) } @@ -1783,6 +1782,33 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT } } +// probeTLSNames probes the names of the certificate of result, for -tls-probe. +func (r *Runner) probeTLSNames(result Result, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { + names := append(slices.Clone(result.TLSData.SubjectAN), result.TLSData.SubjectCN) + for _, name := range names { + if r.testAndSet(name) { + r.process(targetLine{Target: name}, wg, hp, protocol, scanopts, output) + } + } +} + +// targetsOf returns the targets of an input: the target of a line pinned to an +// ip, or the targets of a plain input. +func (r *Runner) targetsOf(hp *httpx.HTTPX, line targetLine) chan httpx.Target { + if line.IP == "" { + return r.targets(hp, line.Target) + } + results := make(chan httpx.Target, 1) + results <- httpx.Target{Host: line.Target, CustomIP: line.IP} + close(results) + return results +} + +// isVHostInput reports whether target is a vhost,ip input. +func isVHostInput(target string) bool { + return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") +} + // returns all the targets within a cidr range or the single target func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) @@ -1831,7 +1857,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { for _, ip := range ips { results <- httpx.Target{Host: target, CustomIP: ip} } - case !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ","): + case isVHostInput(target): idxComma := strings.Index(target, ",") results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} default: @@ -1841,7 +1867,18 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result { +// pinnedKey returns key for a target that is not pinned, and key and ip +// joined by a NUL, which neither holds, for one pinned to ip, from a target +// line or -pa, as the ips of a host fail and respond independently. +func pinnedKey(key, ip string) string { + if ip == "" { + return key + } + return key + "\x00" + ip +} + +func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method string, input targetLine, scanopts *ScanOptions) Result { + origInput := input.key() origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = determineMostLikelySchemeOrder(target.Host) @@ -1867,7 +1904,7 @@ retry: } // check if we have to skip the host:port as a result of a previous failure - hostPort := net.JoinHostPort(URL.Host, URL.Port()) + hostPort := pinnedKey(net.JoinHostPort(URL.Host, URL.Port()), target.CustomIP) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors >= r.options.HostMaxErrors { @@ -1892,21 +1929,8 @@ retry: if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil { gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } - var ( - req *retryablehttp.Request - ctx context.Context - ) - if target.CustomIP != "" { - var requestIP string - if iputil.IsIPv6(target.CustomIP) { - requestIP = fmt.Sprintf("[%s]", target.CustomIP) - } else { - requestIP = target.CustomIP - } - ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP) - } else { - ctx = context.Background() - } + var req *retryablehttp.Request + ctx := httpx.WithHost(context.Background(), URL.Hostname(), target.CustomIP) req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { return Result{URL: URL.String(), Input: origInput, Err: err} @@ -2049,6 +2073,11 @@ retry: builder.WriteRune(']') } if err != nil { + if errors.Is(err, httpx.ErrPinnedThroughProxy) { + r.pinnedProxyWarning.Do(func() { + gologger.Error().Msgf("Targets pinned to an ip, by a target line or -pa, cannot be probed through the proxy of HTTP_PROXY or HTTPS_PROXY, which resolves the host itself: add their hosts to NO_PROXY\n") + }) + } errString := "" errString = err.Error() splitErr := strings.Split(errString, ":") @@ -2086,7 +2115,11 @@ retry: } if r.options.Probe { - return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()} + // the pinned ip tells apart the failures of the ips of a host + if target.CustomIP != "" { + _, _ = fmt.Fprintf(builder, " [%s]", target.CustomIP) + } + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, HostIP: target.CustomIP, str: builder.String()} } else { return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err} } @@ -2294,8 +2327,13 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // hp.Dialer.GetDialedIP would return only the last dialed one - ip = hp.Dialer.GetDialedIP(URL.Host) + // the ip this request was dialed to or, if it was not dialed to + // its host, as through a proxy, the last ip the dialer dialed to + // the host, which may be another request's + ip = httpx.DialedIP(req.Context()) + if ip == "" { + ip = hp.Dialer.GetDialedIP(URL.Host) + } if ip == "" { ip = hp.Dialer.GetDialedIP(onlyHost) } @@ -2327,7 +2365,7 @@ retry: } } - if scanopts.OutputIP || scanopts.ProbeAllIPS { + if scanopts.OutputIP || scanopts.ProbeAllIPS || target.CustomIP != "" { _, _ = fmt.Fprintf(builder, " [%s]", ip) } @@ -2336,9 +2374,14 @@ retry: if err != nil { onlyHost = URL.Host } - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost) - if err != nil { - allIps = append(allIps, ip) + var allIps, cnames, resolvers []string + // like curl --resolve, a target line gives the ip of its host, which is + // not resolved + if input.IP == "" { + allIps, cnames, resolvers, err = getDNSData(hp, onlyHost) + if err != nil { + allIps = append(allIps, ip) + } } var ips4, ips6 []string @@ -2505,7 +2548,7 @@ retry: } // store responses or chain in directory - domainFile := method + ":" + URL.EscapedString() + domainFile := pinnedKey(method+":"+URL.EscapedString(), target.CustomIP) hash := hashes.Sha1([]byte(domainFile)) domainResponseFile := fmt.Sprintf("%s.txt", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") @@ -2767,7 +2810,7 @@ retry: } func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) { - if r.skipCDNPort(URL.Hostname(), URL.Port()) { + if r.skipCDNPort(URL.Hostname(), target.CustomIP, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} } @@ -2827,8 +2870,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } } - // Clone original request (reuse headers/cookies) - clone := req.Clone(context.Background()) + // Clone original request (reuse headers/cookies and the pinned ip) + clone := req.Clone(httpx.PinnedContext(req.Request)) var ( faviconMMH3 string @@ -3036,26 +3079,26 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint return res } -func (r *Runner) skipCDNPort(host string, port string) bool { +// skipCDNPort reports whether a target is skipped as its ip is a CDN ip and +// port is not allowed for one. The ip is the one the target is pinned to, or +// else the first ip of host. +func (r *Runner) skipCDNPort(host, ip, port string) bool { // if the option is not enabled we don't skip if !r.scanopts.ExcludeCDN { return false } - // uses the dealer to pre-resolve the target - dnsData, err := r.hp.Dialer.GetDNSData(host) - // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } - - if len(dnsData.A) == 0 { - return false + if ip == "" { + // uses the dealer to pre-resolve the target + dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly + if err != nil || len(dnsData.A) == 0 { + return false + } + // pick the first ip as target + ip = dnsData.A[0] } - // pick the first ip as target - hostIP := dnsData.A[0] - - isCdnIP, _, _, err := r.hp.CdnCheck(hostIP) + isCdnIP, _, _, err := r.hp.CdnCheck(ip) if err != nil { return false } diff --git a/runner/targetline.go b/runner/targetline.go new file mode 100644 index 000000000..37e510447 --- /dev/null +++ b/runner/targetline.go @@ -0,0 +1,220 @@ +package runner + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "slices" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/mapcidr/asn" + iputil "github.com/projectdiscovery/utils/ip" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +// errTargetLineConflict is the error of a target line with an ip and options +// that do not keep to the ip. +var errTargetLineConflict = errors.New("options that do not keep to its ip") + +// nucleiKeys are the keys a nuclei target line may hold besides target and ip, +// which select its templates. They are ignored, so that one list feeds both +// tools. +var nucleiKeys = []string{ + "profile", "templates", "tags", "exclude-tags", "include-tags", "author", "template-id", + "exclude-id", "severity", "exclude-severity", "type", "exclude-type", "tech", +} + +// targetLine is an input line: a target, pinned to an ip when IP is set. A +// line that pins it is a JSON object, in the format of nuclei target lines: +// +// {"target": "https://app.example", "ip": "192.0.2.10"} +// +// Like curl --resolve, but on every port of its host, connections to the +// hostname go to the ip, while the hostname is kept for the Host header and +// SNI and is not resolved. +type targetLine struct { + Target string `json:"target"` + IP string `json:"ip,omitempty"` +} + +// readInputLine returns the form an input line is stored and deduplicated in, +// and false if the line is skipped. +func (r *Runner) readInputLine(line string) (string, bool) { + parsed, ok := r.inputOf(line) + return parsed.key(), ok +} + +// inputOf parses an input line: an invalid line is reported and skipped, while +// a line that conflicts with the options stops the run, as every such line +// would. +func (r *Runner) inputOf(line string) (targetLine, bool) { + parsed, err := r.parseInput(line) + if errors.Is(err, errTargetLineConflict) { + gologger.Fatal().Msgf("Invalid input: %s\n", err) + } + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return targetLine{}, false + } + return parsed, true +} + +// parseInput parses an input line, and refuses a target line with an ip when +// an option does not keep to the ip. +func (r *Runner) parseInput(line string) (targetLine, error) { + line = strings.TrimSpace(line) + // a target line is a JSON object + if !strings.HasPrefix(line, "{") { + return targetLine{Target: line}, nil + } + parsed, err := parseTargetLine(line) + if err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + if parsed.IP != "" { + if flags := r.options.targetLineConflicts(); len(flags) > 0 { + return targetLine{}, fmt.Errorf("target line %q cannot be probed with %w: %s", line, errTargetLineConflict, strings.Join(flags, ", ")) + } + } + return parsed, nil +} + +// key returns the line as it is stored and deduplicated: the target alone +// when it is not pinned, and a canonical target line otherwise. +func (t targetLine) key() string { + if t.IP == "" { + return t.Target + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(t) + return strings.TrimSpace(buf.String()) +} + +// parseTargetLine decodes a target line, or returns why it is invalid. Unknown +// keys and an ip that is null or empty are rejected, so that a typo cannot +// silently probe the host through DNS. +func parseTargetLine(line string) (targetLine, error) { + // the values are kept raw to tell an ip that is null or empty from no ip + var keys map[string]json.RawMessage + decoder := json.NewDecoder(strings.NewReader(line)) + if err := decoder.Decode(&keys); err != nil { + return targetLine{}, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return targetLine{}, errors.New("unexpected data after the object") + } + if err := checkKeys(keys); err != nil { + return targetLine{}, err + } + + var target string + if raw, ok := keys["target"]; ok { + if err := json.Unmarshal(raw, &target); err != nil { + return targetLine{}, err + } + } + target = strings.TrimSpace(target) + if target == "" { + return targetLine{}, errors.New("no target") + } + rawIP, ok := keys["ip"] + if !ok { + return targetLine{Target: target}, nil + } + if err := checkPinnable(target); err != nil { + return targetLine{}, err + } + ip, err := decodeIP(rawIP) + if err != nil { + return targetLine{}, err + } + return targetLine{Target: target, IP: ip}, nil +} + +// checkKeys returns an error naming the keys of a target line that are neither +// target, ip nor keys of nuclei target lines. +func checkKeys(keys map[string]json.RawMessage) error { + var unknown []string + for key := range keys { + if key != "target" && key != "ip" && !slices.Contains(nucleiKeys, key) { + unknown = append(unknown, key) + } + } + if len(unknown) == 0 { + return nil + } + slices.Sort(unknown) + return fmt.Errorf("unknown keys %s (supported: target, ip and the keys of nuclei target lines)", strings.Join(unknown, ", ")) +} + +// decodeIP decodes and validates the ip of a target line, and returns it in +// canonical form. +func decodeIP(raw json.RawMessage) (string, error) { + var ip string + if err := json.Unmarshal(raw, &ip); err != nil { + return "", err + } + addr, err := netip.ParseAddr(strings.TrimSpace(ip)) + if err != nil || addr.Zone() != "" || addr.IsUnspecified() { + return "", fmt.Errorf("invalid ip %q", ip) + } + return addr.Unmap().String(), nil +} + +// checkPinnable returns why target cannot be pinned to an ip: only a single +// target with a hostname can. +func checkPinnable(target string) error { + switch { + case iputil.IsCIDR(target), asn.IsASN(target): + return errors.New("a CIDR or ASN target cannot be pinned to an ip") + case stringsutil.HasPrefixAny(target, "*", "."): + return errors.New("a wildcard target cannot be pinned to an ip") + case isVHostInput(target): + return errors.New("a vhost,ip target cannot be pinned to an ip") + } + parsed, err := urlutil.Parse(target) + if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) { + return errors.New("a target pinned to an ip must have a hostname") + } + return nil +} + +// targetLineConflicts returns the options a target line with an ip cannot be +// probed with: their probes do not connect through the ip, or share a +// connection between the ips of a host, as HTTP/2 does. +func (options *Options) targetLineConflicts() []string { + var flags []string + if options.Unsafe { + flags = append(flags, "-unsafe") + } + if options.Pipeline { + flags = append(flags, "-pipeline") + } + if options.HTTP2Probe { + flags = append(flags, "-http2") + } + if strings.EqualFold(options.Protocol, string(httpx.HTTP2)) { + flags = append(flags, "-pr http2") + } + if options.Jarm { + flags = append(flags, "-jarm") + } + if options.Screenshot { + flags = append(flags, "-screenshot") + } + if options.VHostInput { + flags = append(flags, "-vhost-input") + } + if options.Proxy != "" { + flags = append(flags, "-proxy") + } + return flags +} diff --git a/runner/targetline_test.go b/runner/targetline_test.go new file mode 100644 index 000000000..d0f91f383 --- /dev/null +++ b/runner/targetline_test.go @@ -0,0 +1,739 @@ +package runner + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/gologger/writer" + "github.com/projectdiscovery/httpx/common/httpx" + syncutil "github.com/projectdiscovery/utils/sync" + "github.com/stretchr/testify/require" + "golang.org/x/net/dns/dnsmessage" +) + +func TestParseTargetLine(t *testing.T) { + valid := map[string]struct { + line string + want targetLine + }{ + "ipv4": {`{"target": "https://a.example", "ip": "192.0.2.10"}`, targetLine{Target: "https://a.example", IP: "192.0.2.10"}}, + "ipv6": {`{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`, targetLine{Target: "a.example:8443", IP: "2001:db8::1"}}, + "ipv4-mapped": {`{"target": "a.example", "ip": "::ffff:192.0.2.10"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "spaced ip": {`{"target": "a.example", "ip": " 192.0.2.10 "}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "no ip": {`{"target": "a.example"}`, targetLine{Target: "a.example"}}, + "escaped target": {`{"target": "https://a.example/?a=1&b=2"}`, targetLine{Target: "https://a.example/?a=1&b=2"}}, + "nuclei keys": {`{"target": "a.example", "ip": "192.0.2.10", "tags": ["cve"], "severity": "high"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + } + for name, tc := range valid { + t.Run(name, func(t *testing.T) { + got, err := parseTargetLine(tc.line) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + invalid := map[string]struct{ line, reason string }{ + "malformed json": {`{"target": "a.example"`, "unexpected EOF"}, + "trailing data": {`{"target": "a.example"} x`, "unexpected data after the object"}, + "unknown key": {`{"target": "a.example", "ips": "192.0.2.10"}`, "unknown keys ips"}, + "key case": {`{"target": "a.example", "IP": "192.0.2.10"}`, "unknown keys IP"}, + "missing target": {`{"ip": "192.0.2.10"}`, "no target"}, + "invalid ip": {`{"target": "a.example", "ip": "192.0.2"}`, `invalid ip "192.0.2"`}, + "null ip": {`{"target": "a.example", "ip": null}`, `invalid ip ""`}, + "empty ip": {`{"target": "a.example", "ip": ""}`, `invalid ip ""`}, + "number ip": {`{"target": "a.example", "ip": 1}`, "cannot unmarshal number"}, + "ip with zone": {`{"target": "a.example", "ip": "fe80::1%eth0"}`, "invalid ip"}, + "unspecified ip": {`{"target": "a.example", "ip": "0.0.0.0"}`, "invalid ip"}, + "ip target": {`{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, "must have a hostname"}, + "cidr target": {`{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "asn target": {`{"target": "AS13335", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "wildcard": {`{"target": "*.a.example", "ip": "192.0.2.10"}`, "wildcard"}, + "vhost form": {`{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, "vhost,ip"}, + } + for name, tc := range invalid { + t.Run(name, func(t *testing.T) { + _, err := parseTargetLine(tc.line) + require.ErrorContains(t, err, tc.reason) + }) + } +} + +func TestRunner_parseInput(t *testing.T) { + r := &Runner{options: &Options{}} + + t.Run("equivalent lines are stored once", func(t *testing.T) { + for _, line := range []string{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`, + ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, + } { + parsed, err := r.parseInput(line) + require.NoError(t, err) + require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, parsed.key()) + } + }) + t.Run("a line without ip is the plain target", func(t *testing.T) { + parsed, err := r.parseInput(`{"target": "https://a.example"}`) + require.NoError(t, err) + require.Equal(t, targetLine{Target: "https://a.example"}, parsed) + require.Equal(t, "https://a.example", parsed.key()) + }) + t.Run("an invalid line is named", func(t *testing.T) { + _, err := r.parseInput(`{"target": "https://a.example", "ip": "192.0.2"}`) + require.ErrorContains(t, err, `invalid target line "{\"target\"`) + }) +} + +func TestRunner_targetLineConflicts(t *testing.T) { + conflicts := map[string]Options{ + "-unsafe": {Unsafe: true}, + "-pipeline": {Pipeline: true}, + "-http2": {HTTP2Probe: true}, + "-pr http2": {Protocol: "HTTP2"}, + "-jarm": {Jarm: true}, + "-screenshot": {Screenshot: true}, + "-vhost-input": {VHostInput: true}, + "-proxy": {Proxy: "http://127.0.0.1:8080"}, + } + for flag, options := range conflicts { + t.Run(flag, func(t *testing.T) { + r := &Runner{options: &options} + _, err := r.parseInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorIs(t, err, errTargetLineConflict) + require.ErrorContains(t, err, "keep to its ip: "+flag) + _, err = r.parseInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only a line with an ip conflicts") + }) + } +} + +func TestRunner_New_probeAllIPsWithProxy(t *testing.T) { + for _, options := range []*Options{ + {ProbeAllIPS: true, Proxy: "http://127.0.0.1:8080"}, + {ProbeAllIPS: true, HTTPProxy: "http://127.0.0.1:8080"}, + } { + _, err := New(options) + require.ErrorContains(t, err, "-pa/-probe-all-ips cannot be used with -proxy") + } +} + +// A line given to Process is read as an input line is. +func TestRunner_Process_readsTheLine(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1") + process := func(t *testing.T, options *Options, line string) []Result { + options.Timeout = 5 + options.Methods = http.MethodGet + options.Resolvers = []string{resolver} + require.NoError(t, options.ValidateOptions()) + r, err := New(options) + require.NoError(t, err) + defer r.Close() + wg, err := syncutil.New(syncutil.WithSize(1)) + require.NoError(t, err) + + output := make(chan Result, 1) + r.Process(line, wg, httpx.HTTP, &r.scanopts, output) + wg.Wait() + close(output) + var results []Result + for result := range output { + results = append(results, result) + } + return results + } + + t.Run("a line without ip is the plain target", func(t *testing.T) { + results := process(t, &Options{}, `{"target": "http://pinned.test:`+port+`"}`) + require.Len(t, results, 1) + require.Equal(t, "http://pinned.test:"+port, results[0].Input) + require.Equal(t, []string{"127.0.0.1"}, results[0].A, "its host is resolved") + }) + t.Run("an invalid line is reported and skipped", func(t *testing.T) { + logged := recordErrors(t) + require.Empty(t, process(t, &Options{}, `{"target": "http://pinned.test:`+port+`", "ip": null}`)) + require.Len(t, logged.containing("Skipping input"), 1) + }) +} + +// An invalid line, such as a line of another tool's JSON output, is reported +// and skipped, and the other lines are read, before probing or, with -stream, +// while probing. +func TestRunner_skipsInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "a.example\n" + + `{"host": "b.example"}` + "\n" + + `{"target": "c.example", "ip": null}` + "\n" + + `{"target": "d.example", "ip": "192.0.2.10"}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + want := []string{"a.example", `{"target":"d.example","ip":"192.0.2.10"}`} + + t.Run("-l", func(t *testing.T) { + logged := recordErrors(t) + r, err := New(&Options{}) + require.NoError(t, err) + defer r.Close() + input, err := os.Open(file) + require.NoError(t, err) + + count, err := r.loadAndCloseFile(input) + require.NoError(t, err) + require.Equal(t, len(want), count) + for _, key := range want { + _, ok := r.hm.Get(key) + require.True(t, ok, key) + } + require.Len(t, logged.containing("Skipping input"), 2, "each skipped line is reported") + }) + t.Run("-stream", func(t *testing.T) { + r, err := New(&Options{Stream: true, InputFile: file}) + require.NoError(t, err) + defer r.Close() + out, err := r.streamInput() + require.NoError(t, err) + + var got []string + for item := range out { + got = append(got, item) + } + require.Equal(t, want, got) + }) +} + +func TestRunner_skip_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn"}}) + require.NoError(t, err) + defer r.Close() + URL, err := r.parseURL("http://pinned.test:8443") + require.NoError(t, err) + + // pinned.test does not resolve: only the pinned ip can be checked + skip, result := r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "151.101.1.1"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "cdn target") + + skip, _ = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.11"}, "") + require.False(t, skip) +} + +// A proxy resolves the host itself, so a probe of a pinned target through it +// fails, which is reported once. Only the proxy of the environment can be +// used with pinned targets, but it is read once per process: the proxy of the +// options stands for it here, and common/httpx tests it. +func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { + logged := recordErrors(t) + r, err := New(&Options{Proxy: "http://127.0.0.1:1", HostMaxErrors: -1}) + require.NoError(t, err) + defer r.Close() + + for range 2 { + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) + require.ErrorIs(t, result.Err, httpx.ErrPinnedThroughProxy) + } + require.Len(t, logged.containing("NO_PROXY"), 1) +} + +// A name of a certificate shaped like a target line is probed with -tls-probe +// as a plain name: the server being probed cannot pin a target to an ip. +func TestRunner_probeTLSNames_cannotPin(t *testing.T) { + var requests atomic.Int32 + // the names are probed over the scheme of the target they come from + pinned := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + }), &tls.Config{}) + certificate := selfSignedCertificate(t, pinnedLine("https://pinned.test:"+portOf(t, pinned.Listener), "127.0.0.1")) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{Certificates: []tls.Certificate{certificate}}) + + results := probe(t, &Options{TLSProbe: true, TLSGrab: true, Threads: 5}, server.URL) + require.NotEmpty(t, results) + require.Zero(t, requests.Load()) +} + +// -maxhr counts the failures of each ip of a host apart, and -probe shows the +// ip that failed. +func TestRunner_analyze_hostErrorsPerIP(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + r, err := New(&Options{HostMaxErrors: 1, Timeout: 2, Probe: true}) + require.NoError(t, err) + defer r.Close() + target := func(ip string) httpx.Target { + return httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} + } + + failed := analyzeHTTP(r, target("127.0.0.3")) + require.Error(t, failed.Err, "nothing listens there") + require.Contains(t, failed.str, "[127.0.0.3]") + require.Equal(t, "127.0.0.3", failed.HostIP) + require.ErrorContains(t, analyzeHTTP(r, target("127.0.0.3")).Err, "previously unresponsive") + require.NoError(t, analyzeHTTP(r, target("127.0.0.1")).Err, "the other ip of the host is not skipped") +} + +// The key of a pinned target differs from the key of any target that is not, +// whose url may hold an "@". +func TestPinnedKey(t *testing.T) { + require.Equal(t, "GET:http://a.example/x", pinnedKey("GET:http://a.example/x", "")) + require.NotEqual(t, pinnedKey("GET:http://a.example/x@192.0.2.1", ""), pinnedKey("GET:http://a.example/x", "192.0.2.1")) +} + +// Lines pinning a host to two ips reach the server on each, once for +// equivalent lines, with the ip in the output and the canonical line as input. +func TestTargetLinePinsTheConnection(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + target := "http://pinned.test:" + port + lines := append(pinnedLines(port, "127.0.0.1", "127.0.0.2"), fmt.Sprintf(` {"ip": "::ffff:127.0.0.1", "target": %q}`, target)) + results := probe(t, &Options{}, lines...) + + var got []string + for _, result := range results { + got = append(got, result.HostIP+" "+result.Title) + require.Contains(t, result.str, "["+result.HostIP+"]", "the output shows the ip") + require.Equal(t, fmt.Sprintf(`{"target":%q,"ip":%q}`, target, result.HostIP), result.Input) + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +// The scheme of the target of a line is kept, as for a url given as is. +func TestTargetLineKeepsTheScheme(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + require.Len(t, probe(t, &Options{}, pinnedLine("http://pinned.test:"+port, "127.0.0.1")), 1) + require.Empty(t, probe(t, &Options{}, pinnedLine("https://pinned.test:"+port, "127.0.0.1")), "a plain http server does not answer https") +} + +// A redirect to the pinned host goes to the ip, on any port, and a redirect to +// another host does not. +func TestTargetLineRedirects(t *testing.T) { + other := servePair(t, titleHandler("one"), titleHandler("two")) + for _, tc := range []struct{ name, location, want string }{ + {"to the pinned host", "http://pinned.test:" + other + "/", "two pinned.test:" + other}, + {"to another host", "http://127.0.0.1:" + other + "/", "one 127.0.0.1:" + other}, + } { + t.Run(tc.name, func(t *testing.T) { + port := servePair(t, titleHandler("one"), http.RedirectHandler(tc.location, http.StatusFound)) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + + require.Len(t, results, 1) + require.Equal(t, tc.want, results[0].Title) + }) + } +} + +// An ip excluded with -e is not connected to, as the dialer keeps to the +// network policy. +func TestTargetLineKeepsToTheExclusions(t *testing.T) { + var requests atomic.Int32 + port := serveLocal(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + })) + require.Empty(t, probe(t, &Options{Exclude: []string{"127.0.0.1"}}, pinnedLines(port, "127.0.0.1")...)) + require.Zero(t, requests.Load()) +} + +// With a matcher, responses are stored after matching, under a name that +// must differ for each ip of a url. +func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + options := &Options{StoreResponse: true, StoreResponseDir: t.TempDir(), OutputMatchStatusCode: "200"} + results := probe(t, options, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + require.Len(t, results, 2) + for _, result := range results { + data, err := os.ReadFile(result.StoredResponsePath) + require.NoError(t, err) + require.Contains(t, string(data), ""+result.Title+"", "the response of %s", result.HostIP) + } +} + +// Like curl --resolve, the host of a pinned target is not resolved, with -pa +// too, so its result has no A records. +func TestTargetLineIsNotResolved(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + for _, probeAllIPs := range []bool{false, true} { + t.Run(fmt.Sprint("-pa ", probeAllIPs), func(t *testing.T) { + resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: probeAllIPs}, pinnedLines(port, "127.0.0.1")...) + + require.Len(t, results, 1) + require.Equal(t, "one pinned.test:"+port, results[0].Title) + require.Empty(t, results[0].A) + require.Zero(t, queries.Load()) + }) + } +} + +// -pa resolves the host, and each of its results keeps all of its addresses. +func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: true}, "http://pinned.test:"+port) + + require.Len(t, results, 2) + for _, result := range results { + require.ElementsMatch(t, []string{"127.0.0.1", "127.0.0.2"}, result.A, "the result of %s", result.HostIP) + } +} + +// A plain target reports the ip it was dialed to, not the one a pinned target +// of the same host was dialed to meanwhile. +func TestPlainTargetReportsItsOwnIP(t *testing.T) { + pinnedDialed := make(chan struct{}) + var once sync.Once + one := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + once.Do(func() { close(pinnedDialed) }) + titleHandler("one").ServeHTTP(w, r) + }) + // the plain target is answered once the pinned target is dialed + two := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-pinnedDialed: + titleHandler("two").ServeHTTP(w, r) + case <-r.Context().Done(): + } + }) + port := servePair(t, one, two) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.2") + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "http://pinned.test:" + port + "\n" + pinnedLines(port, "127.0.0.1")[0] + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + + // the delay dials the plain target first + options := &Options{Resolvers: []string{resolver}, Stream: true, InputFile: file, Threads: 2, Delay: 100 * time.Millisecond} + var got []string + for _, result := range probe(t, options) { + got = append(got, result.HostIP+" "+result.Title) + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +// An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. +func TestTargetLinePinsIPv6(t *testing.T) { + listener, err := net.Listen("tcp", "[::1]:0") + if err != nil { + t.Skipf("IPv6 is not available: %v", err) + } + port := portOf(t, serve(t, listener, titleHandler("six"), nil).Listener) + results := probe(t, &Options{}, pinnedLines(port, "::1")...) + + require.Len(t, results, 1) + require.Equal(t, "six pinned.test:"+port, results[0].Title) +} + +func TestTargetLineKeepsSNI(t *testing.T) { + var ( + mu sync.Mutex + names []string + ) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + mu.Lock() + names = append(names, hello.ServerName) + mu.Unlock() + return nil, nil + }, + }) + results := probe(t, &Options{}, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) + + require.Len(t, results, 1) + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, names) + for _, name := range names { + require.Equal(t, "pinned.test", name) + } +} + +// The favicon request and the ztls handshake of -tls-grab connect through the +// pinned ip as well. +func TestTargetLinePinsTheFaviconAndTLSGrab(t *testing.T) { + server := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("tls")), &tls.Config{}) + options := &Options{Favicon: true, ZTLS: true, TLSGrab: true} + results := probe(t, options, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) + + require.Len(t, results, 1) + require.NotEmpty(t, results[0].FavIconMMH3) + require.NotNil(t, results[0].TLSData) +} + +// The favicon request does not take the trace of the probe, whose -trace +// result would then describe the favicon request. +func TestRunner_HandleFaviconHash_keepsTheTraceOfTheProbe(t *testing.T) { + favicon := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("favicon")), nil) + page := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, ``, favicon.URL) + }), nil) + r, err := New(&Options{Timeout: 5, Methods: http.MethodGet, Trace: true, Favicon: true, MaxResponseBodySizeToRead: int(httpx.DefaultMaxResponseBodySize)}) + require.NoError(t, err) + defer r.Close() + + result := analyzeHTTP(r, httpx.Target{Host: page.URL}) + require.NoError(t, result.Err) + require.NotEmpty(t, result.FavIconMMH3) + require.Equal(t, page.Listener.Addr().String(), fmt.Sprint(result.Trace.GetConn.Info)) +} + +// errorRecorder is a gologger writer that keeps the errors, which are shown +// by default. +type errorRecorder struct { + mu sync.Mutex + errors []string +} + +func (w *errorRecorder) Write(data []byte, level levels.Level) { + if level == levels.LevelError { + w.mu.Lock() + w.errors = append(w.errors, string(data)) + w.mu.Unlock() + } +} + +// recordErrors records the errors logged until the end of the test. +func recordErrors(t *testing.T) *errorRecorder { + t.Helper() + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + return logged +} + +// containing returns the errors that contain text. +func (w *errorRecorder) containing(text string) []string { + w.mu.Lock() + defer w.mu.Unlock() + var lines []string + for _, line := range w.errors { + if strings.Contains(line, text) { + lines = append(lines, line) + } + } + return lines +} + +// selfSignedCertificate returns a certificate whose common name is name. +func selfSignedCertificate(t *testing.T, name string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +// titleHandler serves a page whose title is name and the Host header. +func titleHandler(name string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) + }) +} + +// withFavicon serves a PNG image as /favicon.ico, and the rest with handler. +func withFavicon(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/favicon.ico" { + _, _ = w.Write([]byte("\x89PNG\r\n\x1a\n")) + return + } + handler.ServeHTTP(w, r) + }) +} + +func listen(t *testing.T, addr string) net.Listener { + t.Helper() + listener, err := net.Listen("tcp", addr) + require.NoError(t, err) + return listener +} + +// serve serves handler on listener, over TLS if tlsConfig is set. +func serve(t *testing.T, listener net.Listener, handler http.Handler, tlsConfig *tls.Config) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(handler) + _ = server.Listener.Close() + server.Listener = listener + if tlsConfig != nil { + server.TLS = tlsConfig + server.StartTLS() + } else { + server.Start() + } + t.Cleanup(server.Close) + return server +} + +// serveLocal serves handler on 127.0.0.1, and returns the port. +func serveLocal(t *testing.T, handler http.Handler) string { + t.Helper() + return portOf(t, serve(t, listen(t, "127.0.0.1:0"), handler, nil).Listener) +} + +func portOf(t *testing.T, listener net.Listener) string { + t.Helper() + _, port, err := net.SplitHostPort(listener.Addr().String()) + require.NoError(t, err) + return port +} + +// servePair serves one on 127.0.0.1 and two on 127.0.0.2 with the same port, +// and returns the port. It skips the test where 127.0.0.2 cannot be bound, as +// on macOS. +func servePair(t *testing.T, one, two http.Handler) string { + t.Helper() + var err error + for range 5 { + first := listen(t, "127.0.0.1:0") + port := portOf(t, first) + var second net.Listener + if second, err = net.Listen("tcp", net.JoinHostPort("127.0.0.2", port)); err == nil { + serve(t, first, one, nil) + serve(t, second, two, nil) + return port + } + // the port may be taken on 127.0.0.2: try another one + _ = first.Close() + } + t.Skipf("cannot listen on 127.0.0.2: %v", err) + return "" +} + +func pinnedLine(target, ip string) string { + return fmt.Sprintf(`{"target": %q, "ip": %q}`, target, ip) +} + +// pinnedLines returns the lines pinning http://pinned.test:port to each ip. +func pinnedLines(port string, ips ...string) []string { + var lines []string + for _, ip := range ips { + lines = append(lines, pinnedLine("http://pinned.test:"+port, ip)) + } + return lines +} + +// serveDNS runs a resolver that answers the A queries for host with ips, and +// returns its address and the number of queries for host it got. +func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) { + t.Helper() + var records []dnsmessage.AResource + for _, ip := range ips { + addr := netip.MustParseAddr(ip) + require.True(t, addr.Is4(), ip) + records = append(records, dnsmessage.AResource{A: addr.As4()}) + } + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + var queries atomic.Int32 + go func() { + buf := make([]byte, 512) + for { + n, addr, err := conn.ReadFrom(buf) + if err != nil { + return + } + var parser dnsmessage.Parser + header, err := parser.Start(buf[:n]) + if err != nil { + continue + } + question, err := parser.Question() + if err != nil { + continue + } + header.Response = true + builder := dnsmessage.NewBuilder(nil, header) + _ = builder.StartQuestions() + _ = builder.Question(question) + _ = builder.StartAnswers() + if strings.EqualFold(strings.TrimSuffix(question.Name.String(), "."), host) { + queries.Add(1) + if question.Type == dnsmessage.TypeA { + answer := dnsmessage.ResourceHeader{Name: question.Name, Class: dnsmessage.ClassINET, TTL: 60} + for _, record := range records { + _ = builder.AResource(answer, record) + } + } + } + if msg, err := builder.Finish(); err == nil { + _, _ = conn.WriteTo(msg, addr) + } + } + }() + return conn.LocalAddr().String(), &queries +} + +// analyzeHTTP probes target over http with r. +func analyzeHTTP(r *Runner, target httpx.Target) Result { + return r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, targetLine{Target: target.Host, IP: target.CustomIP}, &r.scanopts) +} + +// probe runs a runner on inputs with options, completed with test defaults, +// and returns its results. The results that failed are logged. +func probe(t *testing.T, options *Options, inputs ...string) []Result { + t.Helper() + if options.Threads == 0 { + options.Threads = 1 + } + if options.Timeout == 0 { + options.Timeout = 5 + } + if options.MaxResponseBodySizeToRead == 0 { + options.MaxResponseBodySizeToRead = int(httpx.DefaultMaxResponseBodySize) + options.MaxResponseBodySizeToSave = int(httpx.DefaultMaxResponseBodySize) + } + options.Methods = http.MethodGet + options.InputTargetHost = inputs + require.NoError(t, options.ValidateOptions()) + + var ( + mu sync.Mutex + results []Result + ) + options.OnResult = func(result Result) { + if result.Err != nil || result.URL == "" { + t.Logf("%s: %v", result.Input, result.Err) + return + } + mu.Lock() + results = append(results, result) + mu.Unlock() + } + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + return results +}