diff --git a/README.md b/README.md index e84dd980..bfab8724 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,17 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` +- A target can be pinned to an IP with a JSON input line, in the format of nuclei target lines, like `curl --resolve` but on every port of its host: + ``` + {"target": "https://app.example.com", "ip": "192.0.2.10"} + {"target": "https://app.example.com", "ip": "192.0.2.11"} + ``` + - Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. + - The keys nuclei uses to select templates, such as `tags`, are ignored, and any other key makes the line invalid. An invalid line is reported and skipped. The `input` of a result is the line in a canonical form. + - `-u` splits its value on commas, so pass such lines in a file (`-l`) or on stdin. + - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`. + - `-pa` probes each IP of a host the same way, so it cannot be used with `-proxy` either. The IPs of a host are kept apart: `-maxhr` counts their failures and `-sr` stores their responses separately. + - Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 91c86a49..4e6e8b67 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) { gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation") } transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, + DialContext: recordDialedIP(httpx.Dialer.Dial), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) { // Clone before protocol-specific ALPN is applied. Impersonation keeps the // previous base config; utls does not take NextProtos from it. impersonateTLS := transport.TLSClientConfig.Clone() - transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS) + transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS)) switch httpx.Options.Protocol { case HTTP11: @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } + transport.Proxy = refusePinned(transport.Proxy) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/requesthost.go b/common/httpx/requesthost.go new file mode 100644 index 00000000..c8f00703 --- /dev/null +++ b/common/httpx/requesthost.go @@ -0,0 +1,126 @@ +package httpx + +import ( + "context" + "errors" + "net" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "golang.org/x/net/idna" +) + +// ErrPinnedThroughProxy is the error of a request to a pinned host that would +// go through a proxy, which resolves the host itself. +var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") + +// requestHost is the host a request is made to: the ip it is pinned to, if +// any, and the ip of the last connection dialed to it. +type requestHost struct { + name string + pinned string + mu sync.Mutex + dialed string +} + +// requestHostKey is the context key of the requestHost of WithHost. +type requestHostKey struct{} + +// WithHost returns a copy of ctx for requests to host. When ip is set, they +// connect to it, like curl --resolve but on every port of host, while the host +// keeps its name for the Host header and SNI. The ip they are dialed to is +// recorded for DialedIP, while the dialer only keeps the last ip dialed to a +// host by any request. Requests to other hosts, such as redirect targets, are +// resolved as usual and not recorded. +func WithHost(ctx context.Context, host, ip string) context.Context { + host = canonicalHost(host) + if ip != "" { + ctx = fastdialer.WithResolveOverride(ctx, host, ip) + } + return context.WithValue(ctx, requestHostKey{}, &requestHost{name: host, pinned: ip}) +} + +// PinnedContext returns a new context for another request derived from req, +// in which the host of req keeps the ip it is pinned to. The other values of +// the context of req, such as a trace, are not kept. +func PinnedContext(req *http.Request) context.Context { + return WithHost(context.Background(), req.URL.Hostname(), pinnedIP(req)) +} + +// DialedIP returns the ip of the last connection dialed to the host of a +// WithHost context, or "" if none was. +func DialedIP(ctx context.Context) string { + host, ok := ctx.Value(requestHostKey{}).(*requestHost) + if !ok { + return "" + } + host.mu.Lock() + defer host.mu.Unlock() + return host.dialed +} + +// hostOf returns the requestHost of ctx if name, however it is spelled, is its +// host, and nil otherwise. +func hostOf(ctx context.Context, name string) *requestHost { + host, ok := ctx.Value(requestHostKey{}).(*requestHost) + if !ok || host.name != canonicalHost(name) { + return nil + } + return host +} + +// pinnedIP returns the ip the host of req is pinned to in its context, if any. +func pinnedIP(req *http.Request) string { + if host := hostOf(req.Context(), req.URL.Hostname()); host != nil { + return host.pinned + } + return "" +} + +// canonicalHost folds the case, the IDNA form and a trailing dot of host, as +// fastdialer keys its resolve overrides, so that a host matches the requests +// and dials to it however it is spelled. +func canonicalHost(host string) string { + if ascii, err := idna.Lookup.ToASCII(host); err == nil { + host = ascii + } + return strings.TrimSuffix(strings.ToLower(host), ".") +} + +// refusePinned wraps the proxy of a transport so that a request to a pinned +// host fails instead of going through the proxy. +func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if proxyURL != nil && pinnedIP(req) != "" { + return nil, ErrPinnedThroughProxy + } + return proxyURL, err + } +} + +// dialFunc dials addr, as net.Dialer.DialContext does. +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// recordDialedIP wraps dial to record the ip of a connection to the host of a +// WithHost context. A connection to another host, such as a proxy, is not +// recorded. +func recordDialedIP(dial dialFunc) dialFunc { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + if err != nil { + return conn, err + } + name, _, _ := net.SplitHostPort(addr) + if host := hostOf(ctx, name); host != nil { + ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + host.mu.Lock() + host.dialed = ip + host.mu.Unlock() + } + return conn, nil + } +} diff --git a/common/httpx/requesthost_test.go b/common/httpx/requesthost_test.go new file mode 100644 index 00000000..3ee1b406 --- /dev/null +++ b/common/httpx/requesthost_test.go @@ -0,0 +1,114 @@ +package httpx + +import ( + "context" + "net" + "net/http" + "net/http/httptrace" + "net/url" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefusePinned(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") + require.NoError(t, err) + proxy := refusePinned(http.ProxyURL(proxyURL)) + request := func(ctx context.Context, rawURL string) *http.Request { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + require.NoError(t, err) + return req + } + + // a pinned host however it is spelled + for host, rawURL := range map[string]string{"Pinned.Test.": "https://PINNED.test.:8443/", "xn--bcher-kva.test": "https://bücher.test/"} { + _, err = proxy(request(WithHost(context.Background(), host, "192.0.2.10"), rawURL)) + require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL) + } + + pinned := WithHost(context.Background(), "pinned.test", "192.0.2.10") + got, err := proxy(request(pinned, "https://other.test/")) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a request to another host, as a redirect, keeps the proxy") + + got, err = proxy(request(WithHost(context.Background(), "plain.test", ""), "https://plain.test/")) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a host that is not pinned keeps the proxy") + + got, err = refusePinned(http.ProxyURL(nil))(request(pinned, "https://pinned.test/")) + require.NoError(t, err) + require.Nil(t, got, "a pinned request without a proxy") +} + +// A proxy set in the environment is refused for a pinned request as well. Go +// reads the proxies of the environment once, so the test runs in a new process. +func TestRefusePinned_environment(t *testing.T) { + if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" { + cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$") + // the lower case names first: on Windows, the last of two spellings wins + cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + return + } + + options := DefaultOptions + options.RetryMax = 0 + h, err := New(&options) + require.NoError(t, err) + req, err := h.NewRequestWithContext(WithHost(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/") + require.NoError(t, err) + _, err = h.Do(req, UnsafeOptions{}) + require.ErrorIs(t, err, ErrPinnedThroughProxy) +} + +func TestCanonicalHost(t *testing.T) { + for host, want := range map[string]string{ + "Pinned.Test.": "pinned.test", + "bücher.test": "xn--bcher-kva.test", + "BÜCHER.test": "xn--bcher-kva.test", + "bücher。test": "xn--bcher-kva.test", + "xn--bcher-kva.test": "xn--bcher-kva.test", + } { + require.Equal(t, want, canonicalHost(host), host) + } +} + +func TestPinnedContext(t *testing.T) { + ctx := httptrace.WithClientTrace(WithHost(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil) + require.NoError(t, err) + + derived := req.Clone(PinnedContext(req)) + require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept") + require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not") + + other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + require.Empty(t, pinnedIP(other.Clone(PinnedContext(other)))) +} + +func TestRecordDialedIP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = listener.Close() }() + // every address is dialed to the listener + dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String()) + }) + ctx := WithHost(context.Background(), "dialed.test", "") + + conn, err := dial(ctx, "tcp", "proxy.test:3128") + require.NoError(t, err) + _ = conn.Close() + require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") + + // net/http dials a host as it is spelled + conn, err = dial(ctx, "tcp", "DIALED.test.:80") + require.NoError(t, err) + _ = conn.Close() + require.Equal(t, "127.0.0.1", DialedIP(ctx)) +} diff --git a/common/httpx/tls.go b/common/httpx/tls.go index 87cee744..77040ad7 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -63,7 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response { } // canonical net concatenation host = net.JoinHostPort(hostname, fmt.Sprint(port)) - ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout) + ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout) defer cancel() tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host) if err != nil { diff --git a/runner/main_test.go b/runner/main_test.go new file mode 100644 index 00000000..3b74a2d2 --- /dev/null +++ b/runner/main_test.go @@ -0,0 +1,23 @@ +package runner + +import ( + "os" + "strings" + "testing" +) + +// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in +// the environment, which would resolve it itself, so that they run behind a +// proxy too. Go reads the proxies of the environment once per process, so +// t.Setenv cannot change them for a test. +func TestMain(m *testing.M) { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + noProxy = os.Getenv("no_proxy") + } + noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",") + for _, name := range []string{"NO_PROXY", "no_proxy"} { + _ = os.Setenv(name, noProxy) + } + os.Exit(m.Run()) +} diff --git a/runner/runner.go b/runner/runner.go index 97b9c7b8..87529648 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -33,7 +33,6 @@ import ( "github.com/happyhackingspace/dit" "github.com/mfonda/simhash" asnmap "github.com/projectdiscovery/asnmap/libs" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -99,6 +98,7 @@ type Runner struct { httpApiEndpoint *Server authProvider authprovider.AuthProvider interruptCh chan struct{} + pinnedProxyWarning sync.Once } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -218,6 +218,10 @@ func New(options *Options) (*Runner, error) { options.Proxy = options.SocksProxy } httpxOptions.Proxy = options.Proxy + // a proxy resolves the host itself, so it cannot reach each ip of a host + if options.ProbeAllIPS && options.Proxy != "" { + return nil, errors.New("-pa/-probe-all-ips cannot be used with -proxy, which resolves the host itself") + } httpxOptions.Unsafe = options.Unsafe httpxOptions.UnsafeURI = options.RequestURI httpxOptions.CdnCheck = options.OutputCDN @@ -556,6 +560,10 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { + target, ok := r.readInputLine(target) + if !ok { + continue + } expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { numHosts += expandedTarget @@ -738,6 +746,18 @@ func (r *Runner) streamInput() (chan string, error) { return true } } + // sendLine normalizes, deduplicates and sends an input line, returning + // false if interrupted. + sendLine := func(item string) bool { + item, ok := r.readInputLine(item) + if !ok { + return true + } + if r.options.SkipDedupe || r.testAndSet(item) { + return trySend(item) + } + return true + } if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling @@ -760,14 +780,9 @@ func (r *Runner) streamInput() (chan string, error) { } } else { for item, err := range fileutil.Lines(r.options.InputFile) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } else if r.options.InputFile != "" { @@ -777,27 +792,17 @@ func (r *Runner) streamInput() (chan string, error) { } for _, file := range files { for item, err := range fileutil.Lines(file) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } if fileutil.HasStdin() { for item, err := range fileutil.LinesReader(os.Stdin) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } }() @@ -807,7 +812,10 @@ func (r *Runner) streamInput() (chan string, error) { func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { - target := strings.TrimSpace(scanner.Text()) + target, ok := r.readInputLine(scanner.Text()) + if !ok { + continue + } // Used just to get the exact number of targets expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { @@ -1538,9 +1546,13 @@ func (r *Runner) RunEnumeration() { } } + line, ok := r.inputOf(k) + if !ok { + return nil + } protocol := r.options.protocol // attempt to parse url as is - if u, err := r.parseURL(k); err == nil { + if u, err := r.parseURL(line.Target); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -1552,10 +1564,10 @@ func (r *Runner) RunEnumeration() { for _, p := range r.options.requestURIs { scanopts := r.scanopts.Clone() scanopts.RequestURI = p - r.process(k, wg, r.hp, protocol, scanopts, output) + r.process(line, wg, r.hp, protocol, scanopts, output) } } else { - r.process(k, wg, r.hp, protocol, &r.scanopts, output) + r.process(line, wg, r.hp, protocol, &r.scanopts, output) } } } @@ -1679,11 +1691,14 @@ func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } +// Process probes t, read as an input line is. func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { - r.process(t, wg, r.hp, protocol, scanopts, output) + if line, ok := r.inputOf(t); ok { + r.process(line, wg, r.hp, protocol, scanopts, output) + } } -func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { +func (r *Runner) process(line targetLine, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { // attempts to set the workpool size to the number of threads if r.options.Threads > 0 && wg.Size != r.options.Threads { if err := wg.Resize(context.Background(), r.options.Threads); err != nil { @@ -1696,7 +1711,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT protocols = []string{httpx.HTTPS, httpx.HTTP} } - for target := range r.targets(hp, t) { + for target := range r.targetsOf(hp, line) { // if no custom ports specified then test the default ones if len(customport.Ports) == 0 { for _, method := range scanopts.Methods { @@ -1706,18 +1721,10 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT wg.Add() go func(target httpx.Target, method, protocol string) { defer wg.Done() - result := r.analyze(hp, protocol, target, method, t, scanopts) + result := r.analyze(hp, protocol, target, method, line, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } if scanopts.CSPProbe && result.CSPData != nil { scanopts.CSPProbe = false @@ -1727,7 +1734,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT if !r.testAndSet(tt) { continue } - r.process(tt, wg, hp, protocol, scanopts, output) + r.process(targetLine{Target: tt}, wg, hp, protocol, scanopts, output) } } }(target, method, prot) @@ -1760,18 +1767,10 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT urlx.UpdatePort(fmt.Sprint(port)) target.Host = urlx.String() } - result := r.analyze(hp, protocol, target, method, t, scanopts) + result := r.analyze(hp, protocol, target, method, line, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } }(port, target, method, wantedProtocol) } @@ -1783,6 +1782,33 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT } } +// probeTLSNames probes the names of the certificate of result, for -tls-probe. +func (r *Runner) probeTLSNames(result Result, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { + names := append(slices.Clone(result.TLSData.SubjectAN), result.TLSData.SubjectCN) + for _, name := range names { + if r.testAndSet(name) { + r.process(targetLine{Target: name}, wg, hp, protocol, scanopts, output) + } + } +} + +// targetsOf returns the targets of an input: the target of a line pinned to an +// ip, or the targets of a plain input. +func (r *Runner) targetsOf(hp *httpx.HTTPX, line targetLine) chan httpx.Target { + if line.IP == "" { + return r.targets(hp, line.Target) + } + results := make(chan httpx.Target, 1) + results <- httpx.Target{Host: line.Target, CustomIP: line.IP} + close(results) + return results +} + +// isVHostInput reports whether target is a vhost,ip input. +func isVHostInput(target string) bool { + return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") +} + // returns all the targets within a cidr range or the single target func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) @@ -1831,7 +1857,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { for _, ip := range ips { results <- httpx.Target{Host: target, CustomIP: ip} } - case !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ","): + case isVHostInput(target): idxComma := strings.Index(target, ",") results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} default: @@ -1841,7 +1867,18 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result { +// pinnedKey returns key for a target that is not pinned, and key and ip +// joined by a NUL, which neither holds, for one pinned to ip, from a target +// line or -pa, as the ips of a host fail and respond independently. +func pinnedKey(key, ip string) string { + if ip == "" { + return key + } + return key + "\x00" + ip +} + +func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method string, input targetLine, scanopts *ScanOptions) Result { + origInput := input.key() origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = determineMostLikelySchemeOrder(target.Host) @@ -1867,7 +1904,7 @@ retry: } // check if we have to skip the host:port as a result of a previous failure - hostPort := net.JoinHostPort(URL.Host, URL.Port()) + hostPort := pinnedKey(net.JoinHostPort(URL.Host, URL.Port()), target.CustomIP) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors >= r.options.HostMaxErrors { @@ -1892,21 +1929,8 @@ retry: if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil { gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } - var ( - req *retryablehttp.Request - ctx context.Context - ) - if target.CustomIP != "" { - var requestIP string - if iputil.IsIPv6(target.CustomIP) { - requestIP = fmt.Sprintf("[%s]", target.CustomIP) - } else { - requestIP = target.CustomIP - } - ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP) - } else { - ctx = context.Background() - } + var req *retryablehttp.Request + ctx := httpx.WithHost(context.Background(), URL.Hostname(), target.CustomIP) req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { return Result{URL: URL.String(), Input: origInput, Err: err} @@ -2049,6 +2073,11 @@ retry: builder.WriteRune(']') } if err != nil { + if errors.Is(err, httpx.ErrPinnedThroughProxy) { + r.pinnedProxyWarning.Do(func() { + gologger.Error().Msgf("Targets pinned to an ip, by a target line or -pa, cannot be probed through the proxy of HTTP_PROXY or HTTPS_PROXY, which resolves the host itself: add their hosts to NO_PROXY\n") + }) + } errString := "" errString = err.Error() splitErr := strings.Split(errString, ":") @@ -2086,7 +2115,11 @@ retry: } if r.options.Probe { - return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()} + // the pinned ip tells apart the failures of the ips of a host + if target.CustomIP != "" { + _, _ = fmt.Fprintf(builder, " [%s]", target.CustomIP) + } + return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, HostIP: target.CustomIP, str: builder.String()} } else { return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err} } @@ -2294,8 +2327,13 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // hp.Dialer.GetDialedIP would return only the last dialed one - ip = hp.Dialer.GetDialedIP(URL.Host) + // the ip this request was dialed to or, if it was not dialed to + // its host, as through a proxy, the last ip the dialer dialed to + // the host, which may be another request's + ip = httpx.DialedIP(req.Context()) + if ip == "" { + ip = hp.Dialer.GetDialedIP(URL.Host) + } if ip == "" { ip = hp.Dialer.GetDialedIP(onlyHost) } @@ -2327,7 +2365,7 @@ retry: } } - if scanopts.OutputIP || scanopts.ProbeAllIPS { + if scanopts.OutputIP || scanopts.ProbeAllIPS || target.CustomIP != "" { _, _ = fmt.Fprintf(builder, " [%s]", ip) } @@ -2336,9 +2374,14 @@ retry: if err != nil { onlyHost = URL.Host } - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost) - if err != nil { - allIps = append(allIps, ip) + var allIps, cnames, resolvers []string + // like curl --resolve, a target line gives the ip of its host, which is + // not resolved + if input.IP == "" { + allIps, cnames, resolvers, err = getDNSData(hp, onlyHost) + if err != nil { + allIps = append(allIps, ip) + } } var ips4, ips6 []string @@ -2505,7 +2548,7 @@ retry: } // store responses or chain in directory - domainFile := method + ":" + URL.EscapedString() + domainFile := pinnedKey(method+":"+URL.EscapedString(), target.CustomIP) hash := hashes.Sha1([]byte(domainFile)) domainResponseFile := fmt.Sprintf("%s.txt", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") @@ -2767,7 +2810,7 @@ retry: } func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) { - if r.skipCDNPort(URL.Hostname(), URL.Port()) { + if r.skipCDNPort(URL.Hostname(), target.CustomIP, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} } @@ -2827,8 +2870,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } } - // Clone original request (reuse headers/cookies) - clone := req.Clone(context.Background()) + // Clone original request (reuse headers/cookies and the pinned ip) + clone := req.Clone(httpx.PinnedContext(req.Request)) var ( faviconMMH3 string @@ -3036,26 +3079,26 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint return res } -func (r *Runner) skipCDNPort(host string, port string) bool { +// skipCDNPort reports whether a target is skipped as its ip is a CDN ip and +// port is not allowed for one. The ip is the one the target is pinned to, or +// else the first ip of host. +func (r *Runner) skipCDNPort(host, ip, port string) bool { // if the option is not enabled we don't skip if !r.scanopts.ExcludeCDN { return false } - // uses the dealer to pre-resolve the target - dnsData, err := r.hp.Dialer.GetDNSData(host) - // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } - - if len(dnsData.A) == 0 { - return false + if ip == "" { + // uses the dealer to pre-resolve the target + dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly + if err != nil || len(dnsData.A) == 0 { + return false + } + // pick the first ip as target + ip = dnsData.A[0] } - // pick the first ip as target - hostIP := dnsData.A[0] - - isCdnIP, _, _, err := r.hp.CdnCheck(hostIP) + isCdnIP, _, _, err := r.hp.CdnCheck(ip) if err != nil { return false } diff --git a/runner/targetline.go b/runner/targetline.go new file mode 100644 index 00000000..37e51044 --- /dev/null +++ b/runner/targetline.go @@ -0,0 +1,220 @@ +package runner + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "slices" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/mapcidr/asn" + iputil "github.com/projectdiscovery/utils/ip" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +// errTargetLineConflict is the error of a target line with an ip and options +// that do not keep to the ip. +var errTargetLineConflict = errors.New("options that do not keep to its ip") + +// nucleiKeys are the keys a nuclei target line may hold besides target and ip, +// which select its templates. They are ignored, so that one list feeds both +// tools. +var nucleiKeys = []string{ + "profile", "templates", "tags", "exclude-tags", "include-tags", "author", "template-id", + "exclude-id", "severity", "exclude-severity", "type", "exclude-type", "tech", +} + +// targetLine is an input line: a target, pinned to an ip when IP is set. A +// line that pins it is a JSON object, in the format of nuclei target lines: +// +// {"target": "https://app.example", "ip": "192.0.2.10"} +// +// Like curl --resolve, but on every port of its host, connections to the +// hostname go to the ip, while the hostname is kept for the Host header and +// SNI and is not resolved. +type targetLine struct { + Target string `json:"target"` + IP string `json:"ip,omitempty"` +} + +// readInputLine returns the form an input line is stored and deduplicated in, +// and false if the line is skipped. +func (r *Runner) readInputLine(line string) (string, bool) { + parsed, ok := r.inputOf(line) + return parsed.key(), ok +} + +// inputOf parses an input line: an invalid line is reported and skipped, while +// a line that conflicts with the options stops the run, as every such line +// would. +func (r *Runner) inputOf(line string) (targetLine, bool) { + parsed, err := r.parseInput(line) + if errors.Is(err, errTargetLineConflict) { + gologger.Fatal().Msgf("Invalid input: %s\n", err) + } + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return targetLine{}, false + } + return parsed, true +} + +// parseInput parses an input line, and refuses a target line with an ip when +// an option does not keep to the ip. +func (r *Runner) parseInput(line string) (targetLine, error) { + line = strings.TrimSpace(line) + // a target line is a JSON object + if !strings.HasPrefix(line, "{") { + return targetLine{Target: line}, nil + } + parsed, err := parseTargetLine(line) + if err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + if parsed.IP != "" { + if flags := r.options.targetLineConflicts(); len(flags) > 0 { + return targetLine{}, fmt.Errorf("target line %q cannot be probed with %w: %s", line, errTargetLineConflict, strings.Join(flags, ", ")) + } + } + return parsed, nil +} + +// key returns the line as it is stored and deduplicated: the target alone +// when it is not pinned, and a canonical target line otherwise. +func (t targetLine) key() string { + if t.IP == "" { + return t.Target + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(t) + return strings.TrimSpace(buf.String()) +} + +// parseTargetLine decodes a target line, or returns why it is invalid. Unknown +// keys and an ip that is null or empty are rejected, so that a typo cannot +// silently probe the host through DNS. +func parseTargetLine(line string) (targetLine, error) { + // the values are kept raw to tell an ip that is null or empty from no ip + var keys map[string]json.RawMessage + decoder := json.NewDecoder(strings.NewReader(line)) + if err := decoder.Decode(&keys); err != nil { + return targetLine{}, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return targetLine{}, errors.New("unexpected data after the object") + } + if err := checkKeys(keys); err != nil { + return targetLine{}, err + } + + var target string + if raw, ok := keys["target"]; ok { + if err := json.Unmarshal(raw, &target); err != nil { + return targetLine{}, err + } + } + target = strings.TrimSpace(target) + if target == "" { + return targetLine{}, errors.New("no target") + } + rawIP, ok := keys["ip"] + if !ok { + return targetLine{Target: target}, nil + } + if err := checkPinnable(target); err != nil { + return targetLine{}, err + } + ip, err := decodeIP(rawIP) + if err != nil { + return targetLine{}, err + } + return targetLine{Target: target, IP: ip}, nil +} + +// checkKeys returns an error naming the keys of a target line that are neither +// target, ip nor keys of nuclei target lines. +func checkKeys(keys map[string]json.RawMessage) error { + var unknown []string + for key := range keys { + if key != "target" && key != "ip" && !slices.Contains(nucleiKeys, key) { + unknown = append(unknown, key) + } + } + if len(unknown) == 0 { + return nil + } + slices.Sort(unknown) + return fmt.Errorf("unknown keys %s (supported: target, ip and the keys of nuclei target lines)", strings.Join(unknown, ", ")) +} + +// decodeIP decodes and validates the ip of a target line, and returns it in +// canonical form. +func decodeIP(raw json.RawMessage) (string, error) { + var ip string + if err := json.Unmarshal(raw, &ip); err != nil { + return "", err + } + addr, err := netip.ParseAddr(strings.TrimSpace(ip)) + if err != nil || addr.Zone() != "" || addr.IsUnspecified() { + return "", fmt.Errorf("invalid ip %q", ip) + } + return addr.Unmap().String(), nil +} + +// checkPinnable returns why target cannot be pinned to an ip: only a single +// target with a hostname can. +func checkPinnable(target string) error { + switch { + case iputil.IsCIDR(target), asn.IsASN(target): + return errors.New("a CIDR or ASN target cannot be pinned to an ip") + case stringsutil.HasPrefixAny(target, "*", "."): + return errors.New("a wildcard target cannot be pinned to an ip") + case isVHostInput(target): + return errors.New("a vhost,ip target cannot be pinned to an ip") + } + parsed, err := urlutil.Parse(target) + if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) { + return errors.New("a target pinned to an ip must have a hostname") + } + return nil +} + +// targetLineConflicts returns the options a target line with an ip cannot be +// probed with: their probes do not connect through the ip, or share a +// connection between the ips of a host, as HTTP/2 does. +func (options *Options) targetLineConflicts() []string { + var flags []string + if options.Unsafe { + flags = append(flags, "-unsafe") + } + if options.Pipeline { + flags = append(flags, "-pipeline") + } + if options.HTTP2Probe { + flags = append(flags, "-http2") + } + if strings.EqualFold(options.Protocol, string(httpx.HTTP2)) { + flags = append(flags, "-pr http2") + } + if options.Jarm { + flags = append(flags, "-jarm") + } + if options.Screenshot { + flags = append(flags, "-screenshot") + } + if options.VHostInput { + flags = append(flags, "-vhost-input") + } + if options.Proxy != "" { + flags = append(flags, "-proxy") + } + return flags +} diff --git a/runner/targetline_test.go b/runner/targetline_test.go new file mode 100644 index 00000000..d0f91f38 --- /dev/null +++ b/runner/targetline_test.go @@ -0,0 +1,739 @@ +package runner + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/gologger/writer" + "github.com/projectdiscovery/httpx/common/httpx" + syncutil "github.com/projectdiscovery/utils/sync" + "github.com/stretchr/testify/require" + "golang.org/x/net/dns/dnsmessage" +) + +func TestParseTargetLine(t *testing.T) { + valid := map[string]struct { + line string + want targetLine + }{ + "ipv4": {`{"target": "https://a.example", "ip": "192.0.2.10"}`, targetLine{Target: "https://a.example", IP: "192.0.2.10"}}, + "ipv6": {`{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`, targetLine{Target: "a.example:8443", IP: "2001:db8::1"}}, + "ipv4-mapped": {`{"target": "a.example", "ip": "::ffff:192.0.2.10"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "spaced ip": {`{"target": "a.example", "ip": " 192.0.2.10 "}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "no ip": {`{"target": "a.example"}`, targetLine{Target: "a.example"}}, + "escaped target": {`{"target": "https://a.example/?a=1&b=2"}`, targetLine{Target: "https://a.example/?a=1&b=2"}}, + "nuclei keys": {`{"target": "a.example", "ip": "192.0.2.10", "tags": ["cve"], "severity": "high"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + } + for name, tc := range valid { + t.Run(name, func(t *testing.T) { + got, err := parseTargetLine(tc.line) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + invalid := map[string]struct{ line, reason string }{ + "malformed json": {`{"target": "a.example"`, "unexpected EOF"}, + "trailing data": {`{"target": "a.example"} x`, "unexpected data after the object"}, + "unknown key": {`{"target": "a.example", "ips": "192.0.2.10"}`, "unknown keys ips"}, + "key case": {`{"target": "a.example", "IP": "192.0.2.10"}`, "unknown keys IP"}, + "missing target": {`{"ip": "192.0.2.10"}`, "no target"}, + "invalid ip": {`{"target": "a.example", "ip": "192.0.2"}`, `invalid ip "192.0.2"`}, + "null ip": {`{"target": "a.example", "ip": null}`, `invalid ip ""`}, + "empty ip": {`{"target": "a.example", "ip": ""}`, `invalid ip ""`}, + "number ip": {`{"target": "a.example", "ip": 1}`, "cannot unmarshal number"}, + "ip with zone": {`{"target": "a.example", "ip": "fe80::1%eth0"}`, "invalid ip"}, + "unspecified ip": {`{"target": "a.example", "ip": "0.0.0.0"}`, "invalid ip"}, + "ip target": {`{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, "must have a hostname"}, + "cidr target": {`{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "asn target": {`{"target": "AS13335", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "wildcard": {`{"target": "*.a.example", "ip": "192.0.2.10"}`, "wildcard"}, + "vhost form": {`{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, "vhost,ip"}, + } + for name, tc := range invalid { + t.Run(name, func(t *testing.T) { + _, err := parseTargetLine(tc.line) + require.ErrorContains(t, err, tc.reason) + }) + } +} + +func TestRunner_parseInput(t *testing.T) { + r := &Runner{options: &Options{}} + + t.Run("equivalent lines are stored once", func(t *testing.T) { + for _, line := range []string{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`, + ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, + } { + parsed, err := r.parseInput(line) + require.NoError(t, err) + require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, parsed.key()) + } + }) + t.Run("a line without ip is the plain target", func(t *testing.T) { + parsed, err := r.parseInput(`{"target": "https://a.example"}`) + require.NoError(t, err) + require.Equal(t, targetLine{Target: "https://a.example"}, parsed) + require.Equal(t, "https://a.example", parsed.key()) + }) + t.Run("an invalid line is named", func(t *testing.T) { + _, err := r.parseInput(`{"target": "https://a.example", "ip": "192.0.2"}`) + require.ErrorContains(t, err, `invalid target line "{\"target\"`) + }) +} + +func TestRunner_targetLineConflicts(t *testing.T) { + conflicts := map[string]Options{ + "-unsafe": {Unsafe: true}, + "-pipeline": {Pipeline: true}, + "-http2": {HTTP2Probe: true}, + "-pr http2": {Protocol: "HTTP2"}, + "-jarm": {Jarm: true}, + "-screenshot": {Screenshot: true}, + "-vhost-input": {VHostInput: true}, + "-proxy": {Proxy: "http://127.0.0.1:8080"}, + } + for flag, options := range conflicts { + t.Run(flag, func(t *testing.T) { + r := &Runner{options: &options} + _, err := r.parseInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorIs(t, err, errTargetLineConflict) + require.ErrorContains(t, err, "keep to its ip: "+flag) + _, err = r.parseInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only a line with an ip conflicts") + }) + } +} + +func TestRunner_New_probeAllIPsWithProxy(t *testing.T) { + for _, options := range []*Options{ + {ProbeAllIPS: true, Proxy: "http://127.0.0.1:8080"}, + {ProbeAllIPS: true, HTTPProxy: "http://127.0.0.1:8080"}, + } { + _, err := New(options) + require.ErrorContains(t, err, "-pa/-probe-all-ips cannot be used with -proxy") + } +} + +// A line given to Process is read as an input line is. +func TestRunner_Process_readsTheLine(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1") + process := func(t *testing.T, options *Options, line string) []Result { + options.Timeout = 5 + options.Methods = http.MethodGet + options.Resolvers = []string{resolver} + require.NoError(t, options.ValidateOptions()) + r, err := New(options) + require.NoError(t, err) + defer r.Close() + wg, err := syncutil.New(syncutil.WithSize(1)) + require.NoError(t, err) + + output := make(chan Result, 1) + r.Process(line, wg, httpx.HTTP, &r.scanopts, output) + wg.Wait() + close(output) + var results []Result + for result := range output { + results = append(results, result) + } + return results + } + + t.Run("a line without ip is the plain target", func(t *testing.T) { + results := process(t, &Options{}, `{"target": "http://pinned.test:`+port+`"}`) + require.Len(t, results, 1) + require.Equal(t, "http://pinned.test:"+port, results[0].Input) + require.Equal(t, []string{"127.0.0.1"}, results[0].A, "its host is resolved") + }) + t.Run("an invalid line is reported and skipped", func(t *testing.T) { + logged := recordErrors(t) + require.Empty(t, process(t, &Options{}, `{"target": "http://pinned.test:`+port+`", "ip": null}`)) + require.Len(t, logged.containing("Skipping input"), 1) + }) +} + +// An invalid line, such as a line of another tool's JSON output, is reported +// and skipped, and the other lines are read, before probing or, with -stream, +// while probing. +func TestRunner_skipsInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "a.example\n" + + `{"host": "b.example"}` + "\n" + + `{"target": "c.example", "ip": null}` + "\n" + + `{"target": "d.example", "ip": "192.0.2.10"}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + want := []string{"a.example", `{"target":"d.example","ip":"192.0.2.10"}`} + + t.Run("-l", func(t *testing.T) { + logged := recordErrors(t) + r, err := New(&Options{}) + require.NoError(t, err) + defer r.Close() + input, err := os.Open(file) + require.NoError(t, err) + + count, err := r.loadAndCloseFile(input) + require.NoError(t, err) + require.Equal(t, len(want), count) + for _, key := range want { + _, ok := r.hm.Get(key) + require.True(t, ok, key) + } + require.Len(t, logged.containing("Skipping input"), 2, "each skipped line is reported") + }) + t.Run("-stream", func(t *testing.T) { + r, err := New(&Options{Stream: true, InputFile: file}) + require.NoError(t, err) + defer r.Close() + out, err := r.streamInput() + require.NoError(t, err) + + var got []string + for item := range out { + got = append(got, item) + } + require.Equal(t, want, got) + }) +} + +func TestRunner_skip_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn"}}) + require.NoError(t, err) + defer r.Close() + URL, err := r.parseURL("http://pinned.test:8443") + require.NoError(t, err) + + // pinned.test does not resolve: only the pinned ip can be checked + skip, result := r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "151.101.1.1"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "cdn target") + + skip, _ = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.11"}, "") + require.False(t, skip) +} + +// A proxy resolves the host itself, so a probe of a pinned target through it +// fails, which is reported once. Only the proxy of the environment can be +// used with pinned targets, but it is read once per process: the proxy of the +// options stands for it here, and common/httpx tests it. +func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { + logged := recordErrors(t) + r, err := New(&Options{Proxy: "http://127.0.0.1:1", HostMaxErrors: -1}) + require.NoError(t, err) + defer r.Close() + + for range 2 { + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) + require.ErrorIs(t, result.Err, httpx.ErrPinnedThroughProxy) + } + require.Len(t, logged.containing("NO_PROXY"), 1) +} + +// A name of a certificate shaped like a target line is probed with -tls-probe +// as a plain name: the server being probed cannot pin a target to an ip. +func TestRunner_probeTLSNames_cannotPin(t *testing.T) { + var requests atomic.Int32 + // the names are probed over the scheme of the target they come from + pinned := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + }), &tls.Config{}) + certificate := selfSignedCertificate(t, pinnedLine("https://pinned.test:"+portOf(t, pinned.Listener), "127.0.0.1")) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{Certificates: []tls.Certificate{certificate}}) + + results := probe(t, &Options{TLSProbe: true, TLSGrab: true, Threads: 5}, server.URL) + require.NotEmpty(t, results) + require.Zero(t, requests.Load()) +} + +// -maxhr counts the failures of each ip of a host apart, and -probe shows the +// ip that failed. +func TestRunner_analyze_hostErrorsPerIP(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + r, err := New(&Options{HostMaxErrors: 1, Timeout: 2, Probe: true}) + require.NoError(t, err) + defer r.Close() + target := func(ip string) httpx.Target { + return httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} + } + + failed := analyzeHTTP(r, target("127.0.0.3")) + require.Error(t, failed.Err, "nothing listens there") + require.Contains(t, failed.str, "[127.0.0.3]") + require.Equal(t, "127.0.0.3", failed.HostIP) + require.ErrorContains(t, analyzeHTTP(r, target("127.0.0.3")).Err, "previously unresponsive") + require.NoError(t, analyzeHTTP(r, target("127.0.0.1")).Err, "the other ip of the host is not skipped") +} + +// The key of a pinned target differs from the key of any target that is not, +// whose url may hold an "@". +func TestPinnedKey(t *testing.T) { + require.Equal(t, "GET:http://a.example/x", pinnedKey("GET:http://a.example/x", "")) + require.NotEqual(t, pinnedKey("GET:http://a.example/x@192.0.2.1", ""), pinnedKey("GET:http://a.example/x", "192.0.2.1")) +} + +// Lines pinning a host to two ips reach the server on each, once for +// equivalent lines, with the ip in the output and the canonical line as input. +func TestTargetLinePinsTheConnection(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + target := "http://pinned.test:" + port + lines := append(pinnedLines(port, "127.0.0.1", "127.0.0.2"), fmt.Sprintf(` {"ip": "::ffff:127.0.0.1", "target": %q}`, target)) + results := probe(t, &Options{}, lines...) + + var got []string + for _, result := range results { + got = append(got, result.HostIP+" "+result.Title) + require.Contains(t, result.str, "["+result.HostIP+"]", "the output shows the ip") + require.Equal(t, fmt.Sprintf(`{"target":%q,"ip":%q}`, target, result.HostIP), result.Input) + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +// The scheme of the target of a line is kept, as for a url given as is. +func TestTargetLineKeepsTheScheme(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + require.Len(t, probe(t, &Options{}, pinnedLine("http://pinned.test:"+port, "127.0.0.1")), 1) + require.Empty(t, probe(t, &Options{}, pinnedLine("https://pinned.test:"+port, "127.0.0.1")), "a plain http server does not answer https") +} + +// A redirect to the pinned host goes to the ip, on any port, and a redirect to +// another host does not. +func TestTargetLineRedirects(t *testing.T) { + other := servePair(t, titleHandler("one"), titleHandler("two")) + for _, tc := range []struct{ name, location, want string }{ + {"to the pinned host", "http://pinned.test:" + other + "/", "two pinned.test:" + other}, + {"to another host", "http://127.0.0.1:" + other + "/", "one 127.0.0.1:" + other}, + } { + t.Run(tc.name, func(t *testing.T) { + port := servePair(t, titleHandler("one"), http.RedirectHandler(tc.location, http.StatusFound)) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + + require.Len(t, results, 1) + require.Equal(t, tc.want, results[0].Title) + }) + } +} + +// An ip excluded with -e is not connected to, as the dialer keeps to the +// network policy. +func TestTargetLineKeepsToTheExclusions(t *testing.T) { + var requests atomic.Int32 + port := serveLocal(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + })) + require.Empty(t, probe(t, &Options{Exclude: []string{"127.0.0.1"}}, pinnedLines(port, "127.0.0.1")...)) + require.Zero(t, requests.Load()) +} + +// With a matcher, responses are stored after matching, under a name that +// must differ for each ip of a url. +func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + options := &Options{StoreResponse: true, StoreResponseDir: t.TempDir(), OutputMatchStatusCode: "200"} + results := probe(t, options, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + require.Len(t, results, 2) + for _, result := range results { + data, err := os.ReadFile(result.StoredResponsePath) + require.NoError(t, err) + require.Contains(t, string(data), ""+result.Title+"", "the response of %s", result.HostIP) + } +} + +// Like curl --resolve, the host of a pinned target is not resolved, with -pa +// too, so its result has no A records. +func TestTargetLineIsNotResolved(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + for _, probeAllIPs := range []bool{false, true} { + t.Run(fmt.Sprint("-pa ", probeAllIPs), func(t *testing.T) { + resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: probeAllIPs}, pinnedLines(port, "127.0.0.1")...) + + require.Len(t, results, 1) + require.Equal(t, "one pinned.test:"+port, results[0].Title) + require.Empty(t, results[0].A) + require.Zero(t, queries.Load()) + }) + } +} + +// -pa resolves the host, and each of its results keeps all of its addresses. +func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: true}, "http://pinned.test:"+port) + + require.Len(t, results, 2) + for _, result := range results { + require.ElementsMatch(t, []string{"127.0.0.1", "127.0.0.2"}, result.A, "the result of %s", result.HostIP) + } +} + +// A plain target reports the ip it was dialed to, not the one a pinned target +// of the same host was dialed to meanwhile. +func TestPlainTargetReportsItsOwnIP(t *testing.T) { + pinnedDialed := make(chan struct{}) + var once sync.Once + one := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + once.Do(func() { close(pinnedDialed) }) + titleHandler("one").ServeHTTP(w, r) + }) + // the plain target is answered once the pinned target is dialed + two := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-pinnedDialed: + titleHandler("two").ServeHTTP(w, r) + case <-r.Context().Done(): + } + }) + port := servePair(t, one, two) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.2") + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "http://pinned.test:" + port + "\n" + pinnedLines(port, "127.0.0.1")[0] + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + + // the delay dials the plain target first + options := &Options{Resolvers: []string{resolver}, Stream: true, InputFile: file, Threads: 2, Delay: 100 * time.Millisecond} + var got []string + for _, result := range probe(t, options) { + got = append(got, result.HostIP+" "+result.Title) + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +// An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. +func TestTargetLinePinsIPv6(t *testing.T) { + listener, err := net.Listen("tcp", "[::1]:0") + if err != nil { + t.Skipf("IPv6 is not available: %v", err) + } + port := portOf(t, serve(t, listener, titleHandler("six"), nil).Listener) + results := probe(t, &Options{}, pinnedLines(port, "::1")...) + + require.Len(t, results, 1) + require.Equal(t, "six pinned.test:"+port, results[0].Title) +} + +func TestTargetLineKeepsSNI(t *testing.T) { + var ( + mu sync.Mutex + names []string + ) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + mu.Lock() + names = append(names, hello.ServerName) + mu.Unlock() + return nil, nil + }, + }) + results := probe(t, &Options{}, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) + + require.Len(t, results, 1) + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, names) + for _, name := range names { + require.Equal(t, "pinned.test", name) + } +} + +// The favicon request and the ztls handshake of -tls-grab connect through the +// pinned ip as well. +func TestTargetLinePinsTheFaviconAndTLSGrab(t *testing.T) { + server := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("tls")), &tls.Config{}) + options := &Options{Favicon: true, ZTLS: true, TLSGrab: true} + results := probe(t, options, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) + + require.Len(t, results, 1) + require.NotEmpty(t, results[0].FavIconMMH3) + require.NotNil(t, results[0].TLSData) +} + +// The favicon request does not take the trace of the probe, whose -trace +// result would then describe the favicon request. +func TestRunner_HandleFaviconHash_keepsTheTraceOfTheProbe(t *testing.T) { + favicon := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("favicon")), nil) + page := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, ``, favicon.URL) + }), nil) + r, err := New(&Options{Timeout: 5, Methods: http.MethodGet, Trace: true, Favicon: true, MaxResponseBodySizeToRead: int(httpx.DefaultMaxResponseBodySize)}) + require.NoError(t, err) + defer r.Close() + + result := analyzeHTTP(r, httpx.Target{Host: page.URL}) + require.NoError(t, result.Err) + require.NotEmpty(t, result.FavIconMMH3) + require.Equal(t, page.Listener.Addr().String(), fmt.Sprint(result.Trace.GetConn.Info)) +} + +// errorRecorder is a gologger writer that keeps the errors, which are shown +// by default. +type errorRecorder struct { + mu sync.Mutex + errors []string +} + +func (w *errorRecorder) Write(data []byte, level levels.Level) { + if level == levels.LevelError { + w.mu.Lock() + w.errors = append(w.errors, string(data)) + w.mu.Unlock() + } +} + +// recordErrors records the errors logged until the end of the test. +func recordErrors(t *testing.T) *errorRecorder { + t.Helper() + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + return logged +} + +// containing returns the errors that contain text. +func (w *errorRecorder) containing(text string) []string { + w.mu.Lock() + defer w.mu.Unlock() + var lines []string + for _, line := range w.errors { + if strings.Contains(line, text) { + lines = append(lines, line) + } + } + return lines +} + +// selfSignedCertificate returns a certificate whose common name is name. +func selfSignedCertificate(t *testing.T, name string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +// titleHandler serves a page whose title is name and the Host header. +func titleHandler(name string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) + }) +} + +// withFavicon serves a PNG image as /favicon.ico, and the rest with handler. +func withFavicon(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/favicon.ico" { + _, _ = w.Write([]byte("\x89PNG\r\n\x1a\n")) + return + } + handler.ServeHTTP(w, r) + }) +} + +func listen(t *testing.T, addr string) net.Listener { + t.Helper() + listener, err := net.Listen("tcp", addr) + require.NoError(t, err) + return listener +} + +// serve serves handler on listener, over TLS if tlsConfig is set. +func serve(t *testing.T, listener net.Listener, handler http.Handler, tlsConfig *tls.Config) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(handler) + _ = server.Listener.Close() + server.Listener = listener + if tlsConfig != nil { + server.TLS = tlsConfig + server.StartTLS() + } else { + server.Start() + } + t.Cleanup(server.Close) + return server +} + +// serveLocal serves handler on 127.0.0.1, and returns the port. +func serveLocal(t *testing.T, handler http.Handler) string { + t.Helper() + return portOf(t, serve(t, listen(t, "127.0.0.1:0"), handler, nil).Listener) +} + +func portOf(t *testing.T, listener net.Listener) string { + t.Helper() + _, port, err := net.SplitHostPort(listener.Addr().String()) + require.NoError(t, err) + return port +} + +// servePair serves one on 127.0.0.1 and two on 127.0.0.2 with the same port, +// and returns the port. It skips the test where 127.0.0.2 cannot be bound, as +// on macOS. +func servePair(t *testing.T, one, two http.Handler) string { + t.Helper() + var err error + for range 5 { + first := listen(t, "127.0.0.1:0") + port := portOf(t, first) + var second net.Listener + if second, err = net.Listen("tcp", net.JoinHostPort("127.0.0.2", port)); err == nil { + serve(t, first, one, nil) + serve(t, second, two, nil) + return port + } + // the port may be taken on 127.0.0.2: try another one + _ = first.Close() + } + t.Skipf("cannot listen on 127.0.0.2: %v", err) + return "" +} + +func pinnedLine(target, ip string) string { + return fmt.Sprintf(`{"target": %q, "ip": %q}`, target, ip) +} + +// pinnedLines returns the lines pinning http://pinned.test:port to each ip. +func pinnedLines(port string, ips ...string) []string { + var lines []string + for _, ip := range ips { + lines = append(lines, pinnedLine("http://pinned.test:"+port, ip)) + } + return lines +} + +// serveDNS runs a resolver that answers the A queries for host with ips, and +// returns its address and the number of queries for host it got. +func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) { + t.Helper() + var records []dnsmessage.AResource + for _, ip := range ips { + addr := netip.MustParseAddr(ip) + require.True(t, addr.Is4(), ip) + records = append(records, dnsmessage.AResource{A: addr.As4()}) + } + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + var queries atomic.Int32 + go func() { + buf := make([]byte, 512) + for { + n, addr, err := conn.ReadFrom(buf) + if err != nil { + return + } + var parser dnsmessage.Parser + header, err := parser.Start(buf[:n]) + if err != nil { + continue + } + question, err := parser.Question() + if err != nil { + continue + } + header.Response = true + builder := dnsmessage.NewBuilder(nil, header) + _ = builder.StartQuestions() + _ = builder.Question(question) + _ = builder.StartAnswers() + if strings.EqualFold(strings.TrimSuffix(question.Name.String(), "."), host) { + queries.Add(1) + if question.Type == dnsmessage.TypeA { + answer := dnsmessage.ResourceHeader{Name: question.Name, Class: dnsmessage.ClassINET, TTL: 60} + for _, record := range records { + _ = builder.AResource(answer, record) + } + } + } + if msg, err := builder.Finish(); err == nil { + _, _ = conn.WriteTo(msg, addr) + } + } + }() + return conn.LocalAddr().String(), &queries +} + +// analyzeHTTP probes target over http with r. +func analyzeHTTP(r *Runner, target httpx.Target) Result { + return r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, targetLine{Target: target.Host, IP: target.CustomIP}, &r.scanopts) +} + +// probe runs a runner on inputs with options, completed with test defaults, +// and returns its results. The results that failed are logged. +func probe(t *testing.T, options *Options, inputs ...string) []Result { + t.Helper() + if options.Threads == 0 { + options.Threads = 1 + } + if options.Timeout == 0 { + options.Timeout = 5 + } + if options.MaxResponseBodySizeToRead == 0 { + options.MaxResponseBodySizeToRead = int(httpx.DefaultMaxResponseBodySize) + options.MaxResponseBodySizeToSave = int(httpx.DefaultMaxResponseBodySize) + } + options.Methods = http.MethodGet + options.InputTargetHost = inputs + require.NoError(t, options.ValidateOptions()) + + var ( + mu sync.Mutex + results []Result + ) + options.OnResult = func(result Result) { + if result.Err != nil || result.URL == "" { + t.Logf("%s: %v", result.Input, result.Err) + return + } + mu.Lock() + results = append(results, result) + mu.Unlock() + } + r, err := New(options) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + return results +}