From 500a560e212d0bdee1755425a695c1d4c5cfec87 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Sat, 10 Oct 2026 12:06:31 +0000 Subject: [PATCH 01/16] feat(runner): probe a target through the ip of a JSON input line Probing a vhost on a given server, or an origin behind a CDN, needs the hostname in the Host header and SNI while the connection goes to another ip, and the same host may need probing on several ips. The vhost,ip form builds the URL from the ip, so it sends no SNI and reports the ip as the URL host, and -pa probes the ips DNS returns. An input line can now be a JSON object with the target and the ip, like curl --resolve: {"target": "https://app.example.com", "ip": "192.0.2.10"} The target is probed with the ip as its CustomIP, and host_ip reports it. Lines are read from -l and stdin; -u splits its value on commas, so it cannot carry them. The ip is validated and canonicalized, and the target must have a hostname. An invalid line stops the run, or is skipped with a warning in stream mode. Equivalent lines are stored once, and -pa does not resolve a pinned target. -proxy, -unsafe, -pipeline, -http2, -jarm, -screenshot and -vhost-input do not connect through the ip, so they are refused with pinned targets. The ip of a pinned line, and each ip of -pa, now pins only the probed host, with fastdialer.WithResolveOverride. The fastdialer.IP value used before pinned every dial of the request, so a redirect to another host followed with -fr went to the same ip. IPv6 ips were passed in brackets, which fastdialer does not accept as an ip, so those -pa probes failed. Favicon and ztls requests keep the ip, -maxhr counts errors per ip, and the ip is checked against the network policy as well as the host. go.mod points fastdialer at a fork commit with WithResolveOverride until a fastdialer release includes it. --- README.md | 6 + common/httpx/tls.go | 3 +- go.mod | 2 + go.sum | 4 +- runner/runner.go | 80 ++++++++----- runner/targetline.go | 153 +++++++++++++++++++++++++ runner/targetline_test.go | 233 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 447 insertions(+), 34 deletions(-) create mode 100644 runner/targetline.go create mode 100644 runner/targetline_test.go diff --git a/README.md b/README.md index 298c87333..db3553c40 100644 --- a/README.md +++ b/README.md @@ -365,6 +365,12 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` +- A target can be probed through a given IP with a JSON input line, like `curl --resolve`. The hostname is still used for the Host header and SNI, and the connection goes to the IP, which `host_ip` reports. The same host can be listed with several IPs: + ``` + {"target": "https://app.example.com", "ip": "192.0.2.10"} + {"target": "https://app.example.com", "ip": "192.0.2.11"} + ``` + Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with a warning with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP; a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/tls.go b/common/httpx/tls.go index 87cee7444..9ca5e1e9e 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -63,7 +63,8 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response { } // canonical net concatenation host = net.JoinHostPort(hostname, fmt.Sprint(port)) - ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout) + // keeps the ip the request was pinned to, without its cancellation + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Request.Context()), h.Options.Timeout) defer cancel() tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host) if err != nil { diff --git a/go.mod b/go.mod index 0845e201d..2966877da 100644 --- a/go.mod +++ b/go.mod @@ -169,3 +169,5 @@ require ( golang.org/x/time v0.15.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) + +replace github.com/projectdiscovery/fastdialer => github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f diff --git a/go.sum b/go.sum index ce13bc3e2..d60ac295c 100644 --- a/go.sum +++ b/go.sum @@ -205,6 +205,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f h1:w2j/CqFpNDk4KL6A0pwI7nivKwOqIs/N2NaOjkqrl8w= +github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f/go.mod h1:GybngvpSevg/48Kk/NgPw2HQOB95FngbOS93f1pTJ84= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -256,8 +258,6 @@ github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9Mch github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8= github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk= github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ= -github.com/projectdiscovery/fastdialer v0.5.23 h1:uL0ILvwrOv52BK4XXFM7CyakR+0d3tmc0m1R+i8NelI= -github.com/projectdiscovery/fastdialer v0.5.23/go.mod h1:psWXmdbZhGxmVLNP2o1YI93R8d2YtOwK5XSNhlWxXJ8= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= diff --git a/runner/runner.go b/runner/runner.go index c903054dc..c75dcf336 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -546,6 +546,10 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { + target, err := r.normalizeInput(target) + if err != nil { + gologger.Fatal().Msgf("Could not read input target: %s\n", err) + } expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { numHosts += expandedTarget @@ -722,6 +726,20 @@ func (r *Runner) streamInput() (chan string, error) { return true } } + // sendLine normalizes, deduplicates and sends an input line, returning + // false if interrupted. The scan has already started, so an invalid + // line is skipped with a warning. + sendLine := func(item string) bool { + item, err := r.normalizeInput(item) + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return true + } + if r.options.SkipDedupe || r.testAndSet(item) { + return trySend(item) + } + return true + } if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling @@ -744,14 +762,9 @@ func (r *Runner) streamInput() (chan string, error) { } } else { for item, err := range fileutil.Lines(r.options.InputFile) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } else if r.options.InputFile != "" { @@ -761,27 +774,17 @@ func (r *Runner) streamInput() (chan string, error) { } for _, file := range files { for item, err := range fileutil.Lines(file) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } if fileutil.HasStdin() { for item, err := range fileutil.LinesReader(os.Stdin) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } }() @@ -791,7 +794,11 @@ func (r *Runner) streamInput() (chan string, error) { func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { - target := strings.TrimSpace(scanner.Text()) + target, err := r.normalizeInput(scanner.Text()) + if err != nil { + _ = finput.Close() + return 0, err + } // Used just to get the exact number of targets expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { @@ -1524,7 +1531,7 @@ func (r *Runner) RunEnumeration() { protocol := r.options.protocol // attempt to parse url as is - if u, err := r.parseURL(k); err == nil { + if u, err := r.parseURL(targetOf(k)); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -1776,6 +1783,13 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { target = strings.TrimSpace(target) switch { + case isTargetLine(target): + line, err := parseTargetLine(target) + if err != nil { + gologger.Warning().Msgf("%s\n", err) + return + } + results <- httpx.Target{Host: line.Target, CustomIP: line.IP} case stringsutil.HasPrefixAny(target, "*", "."): // A valid target does not contain: // trim * and/or . (prefix) from the target to return the domain instead of wildcard @@ -1852,6 +1866,10 @@ retry: // check if we have to skip the host:port as a result of a previous failure hostPort := net.JoinHostPort(URL.Host, URL.Port()) + if target.CustomIP != "" { + // the ips of a host fail independently + hostPort += "@" + target.CustomIP + } if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors >= r.options.HostMaxErrors { @@ -1880,16 +1898,11 @@ retry: req *retryablehttp.Request ctx context.Context ) + ctx = context.Background() if target.CustomIP != "" { - var requestIP string - if iputil.IsIPv6(target.CustomIP) { - requestIP = fmt.Sprintf("[%s]", target.CustomIP) - } else { - requestIP = target.CustomIP - } - ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP) - } else { - ctx = context.Background() + // only dials to the target's host go to the ip, not those to a + // redirect target on another host + ctx = fastdialer.WithResolveOverride(ctx, URL.Hostname(), target.CustomIP) } req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { @@ -2761,6 +2774,11 @@ func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) ( return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target host is not allowed by network policy")} } + if target.CustomIP != "" && !r.hp.NetworkPolicy.Validate(target.CustomIP) { + gologger.Debug().Msgf("Skipping target due to network policy: %s (%s)\n", URL.Hostname(), target.CustomIP) + return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target ip is not allowed by network policy")} + } + return false, Result{} } @@ -2811,8 +2829,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } } - // Clone original request (reuse headers/cookies) - clone := req.Clone(context.Background()) + // Clone original request (reuse headers/cookies, and the pinned ip) + clone := req.Clone(req.Context()) var ( faviconMMH3 string diff --git a/runner/targetline.go b/runner/targetline.go new file mode 100644 index 000000000..d1ae21a22 --- /dev/null +++ b/runner/targetline.go @@ -0,0 +1,153 @@ +package runner + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "strings" + + "github.com/projectdiscovery/mapcidr/asn" + iputil "github.com/projectdiscovery/utils/ip" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +// targetLine is an input line that pins a target to an ip, like +// curl --resolve: +// +// {"target": "https://app.example", "ip": "192.0.2.10"} +// +// The target keeps its hostname for the Host header and SNI, while the +// connection goes to the ip. +type targetLine struct { + Target string `json:"target"` + IP string `json:"ip,omitempty"` +} + +// isTargetLine reports whether an input line is a target line. +func isTargetLine(line string) bool { + return strings.HasPrefix(strings.TrimSpace(line), "{") +} + +// parseTargetLine decodes a target line. Unknown keys are rejected so that a +// typo cannot silently probe the host through DNS. +func parseTargetLine(line string) (targetLine, error) { + decoder := json.NewDecoder(strings.NewReader(line)) + decoder.DisallowUnknownFields() + var parsed targetLine + if err := decoder.Decode(&parsed); err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return targetLine{}, fmt.Errorf("invalid target line %q: unexpected data after the object", line) + } + + parsed.Target = strings.TrimSpace(parsed.Target) + if parsed.Target == "" { + return targetLine{}, fmt.Errorf("target line %q has no target", line) + } + if parsed.IP == "" { + return parsed, nil + } + ip, err := pinnedIP(parsed.Target, parsed.IP) + if err != nil { + return targetLine{}, fmt.Errorf("target %q: %w", parsed.Target, err) + } + parsed.IP = ip + return parsed, nil +} + +// pinnedIP validates the ip a target is pinned to and returns it in canonical +// form. Only a target with a hostname can be pinned. +func pinnedIP(target, value string) (string, error) { + addr, err := netip.ParseAddr(strings.TrimSpace(value)) + if err != nil || addr.Zone() != "" || addr.IsUnspecified() { + return "", fmt.Errorf("invalid ip %q", value) + } + if iputil.IsCIDR(target) || asn.IsASN(target) { + return "", errors.New("a CIDR or ASN target cannot have an ip") + } + if stringsutil.HasPrefixAny(target, "*", ".") || (!stringsutil.HasPrefixAny(target, "http://", "https://") && strings.Contains(target, ",")) { + return "", errors.New("a wildcard or vhost,ip target cannot have an ip") + } + parsed, err := urlutil.Parse(target) + if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) { + return "", errors.New("a target with an ip must have a hostname") + } + return addr.Unmap().String(), nil +} + +// key returns the line as it is stored and deduplicated: the target alone +// when it is not pinned, and a canonical target line otherwise. +func (t targetLine) key() string { + if t.IP == "" { + return t.Target + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(t) + return strings.TrimSpace(buf.String()) +} + +// targetOf returns the target of an input line: the line itself, or the +// target of a target line. +func targetOf(item string) string { + if !isTargetLine(item) { + return item + } + if line, err := parseTargetLine(item); err == nil { + return line.Target + } + return item +} + +// normalizeInput validates an input line and returns the form it is stored +// and deduplicated in. +func (r *Runner) normalizeInput(item string) (string, error) { + item = strings.TrimSpace(item) + if !isTargetLine(item) { + return item, nil + } + line, err := parseTargetLine(item) + if err != nil { + return "", err + } + if line.IP != "" { + if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 { + return "", fmt.Errorf("target %q: a target with an ip cannot be probed with %s", line.Target, strings.Join(flags, ", ")) + } + } + return line.key(), nil +} + +// pinnedTargetConflicts returns the flags whose probes do not connect through +// the ip a target is pinned to. +func (options *Options) pinnedTargetConflicts() []string { + var flags []string + if options.Unsafe { + flags = append(flags, "-unsafe") + } + if options.Pipeline { + flags = append(flags, "-pipeline") + } + if options.HTTP2Probe { + flags = append(flags, "-http2") + } + if options.Jarm { + flags = append(flags, "-jarm") + } + if options.Screenshot { + flags = append(flags, "-screenshot") + } + if options.VHostInput { + flags = append(flags, "-vhost-input") + } + if options.Proxy != "" || options.HTTPProxy != "" || options.SocksProxy != "" { + flags = append(flags, "-proxy") + } + return flags +} diff --git a/runner/targetline_test.go b/runner/targetline_test.go new file mode 100644 index 000000000..9b505abc9 --- /dev/null +++ b/runner/targetline_test.go @@ -0,0 +1,233 @@ +package runner + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sort" + "sync" + "testing" + + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/stretchr/testify/require" +) + +func TestParseTargetLine(t *testing.T) { + valid := map[string]targetLine{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`: {Target: "https://a.example", IP: "192.0.2.10"}, + `{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`: {Target: "a.example:8443", IP: "2001:db8::1"}, + `{"target": "a.example", "ip": "::ffff:192.0.2.10"}`: {Target: "a.example", IP: "192.0.2.10"}, + `{"target": "a.example"}`: {Target: "a.example"}, + } + for line, want := range valid { + t.Run(line, func(t *testing.T) { + got, err := parseTargetLine(line) + require.NoError(t, err) + require.Equal(t, want, got) + }) + } + + invalid := map[string]string{ + "malformed json": `{"target": "a.example"`, + "trailing data": `{"target": "a.example"} x`, + "unknown key": `{"target": "a.example", "ips": "192.0.2.10"}`, + "missing target": `{"ip": "192.0.2.10"}`, + "invalid ip": `{"target": "a.example", "ip": "192.0.2"}`, + "ip with zone": `{"target": "a.example", "ip": "fe80::1%eth0"}`, + "unspecified ip": `{"target": "a.example", "ip": "0.0.0.0"}`, + "ip target": `{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, + "cidr target": `{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, + "asn target": `{"target": "AS13335", "ip": "192.0.2.10"}`, + "wildcard": `{"target": "*.a.example", "ip": "192.0.2.10"}`, + "vhost form": `{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, + } + for name, line := range invalid { + t.Run(name, func(t *testing.T) { + _, err := parseTargetLine(line) + require.Error(t, err) + }) + } +} + +func TestRunner_normalizeInput(t *testing.T) { + r := &Runner{options: &Options{}} + for _, line := range []string{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`, + ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, + } { + key, err := r.normalizeInput(line) + require.NoError(t, err) + require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, key, "equivalent lines are stored once") + } + + key, err := r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err) + require.Equal(t, "https://a.example", key, "a line without ip is the plain target") + + _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2"}`) + require.Error(t, err) + + r = &Runner{options: &Options{Unsafe: true, Jarm: true}} + _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorContains(t, err, "-unsafe, -jarm") + _, err = r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only pinned targets conflict") +} + +func TestRunner_targetLine_targets(t *testing.T) { + // -pa must not resolve a pinned target + r, err := New(&Options{ProbeAllIPS: true}) + require.NoError(t, err) + defer r.Close() + + key, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.NoError(t, err) + var got []httpx.Target + for target := range r.targets(r.hp, key) { + got = append(got, target) + } + require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, got) +} + +func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "a.example\n" + + `{"target": "b.example"` + "\n" + + `{"target": "c.example", "ip": "192.0.2.10"}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + + r, err := New(&Options{Stream: true, InputFile: file}) + require.NoError(t, err) + defer r.Close() + out, err := r.streamInput() + require.NoError(t, err) + + var got []string + for item := range out { + got = append(got, item) + } + require.Equal(t, []string{"a.example", `{"target":"c.example","ip":"192.0.2.10"}`}, got, "the malformed line is skipped") +} + +// skipIfProxied skips a test whose requests to target go through a proxy set +// in the environment, as the proxy resolves the host itself. +func skipIfProxied(t *testing.T, target string) { + t.Helper() + u, err := url.Parse(target) + require.NoError(t, err) + if proxy, _ := http.ProxyFromEnvironment(&http.Request{URL: u}); proxy != nil { + t.Skipf("%s goes through the proxy %s", target, proxy.Redacted()) + } +} + +// runPinned probes the target lines with a runner and returns the results. +func runPinned(t *testing.T, lines ...string) []Result { + t.Helper() + + var ( + mu sync.Mutex + results []Result + ) + r, err := New(&Options{ + Threads: 1, + RateLimit: 10, + Timeout: 5, + Methods: http.MethodGet, + Delay: -1, + OnResult: func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + }, + InputTargetHost: lines, + }) + require.NoError(t, err) + defer r.Close() + r.RunEnumeration() + + mu.Lock() + defer mu.Unlock() + return results +} + +func TestTargetLinePinsTheConnection(t *testing.T) { + serve := func(addr, name string) *httptest.Server { + listener, err := net.Listen("tcp", addr) + if err != nil { + t.Skipf("cannot listen on %s: %v", addr, err) + } + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) + })) + _ = ts.Listener.Close() + ts.Listener = listener + ts.Start() + t.Cleanup(ts.Close) + return ts + } + + one := serve("127.0.0.1:0", "one") + _, port, err := net.SplitHostPort(one.Listener.Addr().String()) + require.NoError(t, err) + serve(net.JoinHostPort("127.0.0.2", port), "two") + + target := "http://pinned.test:" + port + skipIfProxied(t, target) + results := runPinned(t, + fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target), + fmt.Sprintf(`{"target": %q, "ip": "127.0.0.2"}`, target), + ) + + var got []string + for _, result := range results { + got = append(got, result.HostIP+" "+result.Title) + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +func TestTargetLineKeepsSNI(t *testing.T) { + var ( + mu sync.Mutex + names []string + ) + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, "tls") + })) + ts.TLS = &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + mu.Lock() + names = append(names, hello.ServerName) + mu.Unlock() + return nil, nil + }, + } + ts.StartTLS() + defer ts.Close() + _, port, err := net.SplitHostPort(ts.Listener.Addr().String()) + require.NoError(t, err) + + target := "https://pinned.test:" + port + skipIfProxied(t, target) + results := runPinned(t, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) + require.Len(t, results, 1) + require.Equal(t, "127.0.0.1", results[0].HostIP) + + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, names) + for _, name := range names { + require.Equal(t, "pinned.test", name) + } +} From 58ba3c878452d0fde9c9dba7310a1b7167b44ed7 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 13 Oct 2026 20:21:20 +0000 Subject: [PATCH 02/16] chore(deps): update the fastdialer replace to v0.5.25 The branch of projectdiscovery/fastdialer#580 now includes v0.5.25. Through MVS this raises networkpolicy to v0.1.54 and utils to v0.11.9, which that fastdialer commit requires. --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 2966877da..8539e42db 100644 --- a/go.mod +++ b/go.mod @@ -29,13 +29,13 @@ require ( github.com/projectdiscovery/gologger v1.1.73 github.com/projectdiscovery/hmap v0.0.102 github.com/projectdiscovery/mapcidr v1.1.97 - github.com/projectdiscovery/networkpolicy v0.1.53 + github.com/projectdiscovery/networkpolicy v0.1.54 github.com/projectdiscovery/ratelimit v0.0.92 github.com/projectdiscovery/rawhttp v0.1.92 github.com/projectdiscovery/retryablehttp-go v1.3.29 github.com/projectdiscovery/tlsx v1.4.0 github.com/projectdiscovery/useragent v0.0.109 - github.com/projectdiscovery/utils v0.11.8 + github.com/projectdiscovery/utils v0.11.9 github.com/projectdiscovery/wappalyzergo v0.3.4 github.com/rs/xid v1.6.0 github.com/spaolacci/murmur3 v1.1.0 @@ -170,4 +170,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/projectdiscovery/fastdialer => github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f +replace github.com/projectdiscovery/fastdialer => github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c diff --git a/go.sum b/go.sum index d60ac295c..a0da96382 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f h1:w2j/CqFpNDk4KL6A0pwI7nivKwOqIs/N2NaOjkqrl8w= -github.com/mazzz3r/fastdialer v0.0.0-20261010115852-eced62301c6f/go.mod h1:GybngvpSevg/48Kk/NgPw2HQOB95FngbOS93f1pTJ84= +github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c h1:Bf6RjxuR/g7zniyJX7pJfh6y/FYx+KCPrxRtCVgaxQs= +github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -280,8 +280,8 @@ github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0 github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.97 h1:7FkxNNVXp+m1rIu5Nv/2SrF9k4+LwP8QuWs2puwy+2w= github.com/projectdiscovery/mapcidr v1.1.97/go.mod h1:9dgTJh1SP02gYZdpzMjm6vtYFkEHQHoTyaVNvaeJ7lA= -github.com/projectdiscovery/networkpolicy v0.1.53 h1:Oduo0GXFm0aSWqkbkyVXRjwT+GitqnqEEV0KiEtR2aM= -github.com/projectdiscovery/networkpolicy v0.1.53/go.mod h1:+4ef+If67QABeDgSEoOxOgs5ZiM7ygaEeYQPkaZkLDQ= +github.com/projectdiscovery/networkpolicy v0.1.54 h1:+v7yyuRBu2S7dzeIJToDENy1i+HjRhNzqnQrDlm00z8= +github.com/projectdiscovery/networkpolicy v0.1.54/go.mod h1:tN+YAFVlJjHv/znPEhkcSAjmBMkMhslC9XpWKTrKEHw= github.com/projectdiscovery/ratelimit v0.0.92 h1:2h2/AU81Lbn/xevb4KtZCMhaQej+u7W3uLXFUD163iU= github.com/projectdiscovery/ratelimit v0.0.92/go.mod h1:Tqpx1QKdFI63L01T2gpSvV5Uro9CnZkZWr80ciSXSBY= github.com/projectdiscovery/rawhttp v0.1.92 h1:38GRsVLNllES/dZcEcA6s6P+rYBp34H9mDphMJMDvvg= @@ -296,8 +296,8 @@ github.com/projectdiscovery/tlsx v1.4.0 h1:OU2mNK6TOB/YiFX3jhmgwOepFWxPoj5rn8npW github.com/projectdiscovery/tlsx v1.4.0/go.mod h1:WCrUXAHjdKbDNjsu5EbVBS7qvzap3fxi/E85v/Flyx8= github.com/projectdiscovery/useragent v0.0.109 h1:b86BPdZvmgbJousa04jAugDRY4Y+rAvEamE0TFCPmeE= github.com/projectdiscovery/useragent v0.0.109/go.mod h1:sRyBqDAcD31RKYrO8fyXGowLd36fNwY1g0MNW9mp7zk= -github.com/projectdiscovery/utils v0.11.8 h1:oBP3eRG/S/Pdid6CkL58IVyq+KZdWdXohDtEKIKvWag= -github.com/projectdiscovery/utils v0.11.8/go.mod h1:AcQ1n3ETcYj4cHiPXsSaAyFnvN4Bn8UgN5tCmh+7KWM= +github.com/projectdiscovery/utils v0.11.9 h1:aKLje3zPcZuOGmopN51PasWxPX/CypPwzP2ke/Qa2bA= +github.com/projectdiscovery/utils v0.11.9/go.mod h1:d3BcbsHhuxW9T3jpvrgzwshvqhaL9qPSkoi+3h/DY8E= github.com/projectdiscovery/wappalyzergo v0.3.4 h1:yO+xel+C0d/vOFWcUKVZRpIDAfcEmsjpVDjQ0jifAUw= github.com/projectdiscovery/wappalyzergo v0.3.4/go.mod h1:6xTRCUQhX8m+L1eqJDQIALJlRv2Bnhrmku80YU9bBXE= github.com/refraction-networking/utls v1.8.2 h1:j4Q1gJj0xngdeH+Ox/qND11aEfhpgoEvV+S9iJ2IdQo= From d29d531ac9146eb5731915c0f5fb0009f8a74ddd Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Thu, 22 Oct 2026 12:00:39 +0000 Subject: [PATCH 03/16] chore(deps): point the fastdialer replace at the upstream commit fastdialer#580 is merged into main, so the fork branch it was on no longer exists. The replace resolves the same commit through the upstream module path github.com/projectdiscovery/fastdialer. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 769b15197..b5e647dba 100644 --- a/go.mod +++ b/go.mod @@ -170,4 +170,4 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect ) -replace github.com/projectdiscovery/fastdialer => github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c +replace github.com/projectdiscovery/fastdialer => github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c diff --git a/go.sum b/go.sum index aac3f4040..b6128f934 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c h1:Bf6RjxuR/g7zniyJX7pJfh6y/FYx+KCPrxRtCVgaxQs= -github.com/mazzz3r/fastdialer v0.0.0-20261013195407-11ed52b6df6c/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 h1:bjfMeqxWEJ6IRUvGkiTkSwx0a6UdQJsbirRSoXogteY= github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6/go.mod h1:WVJJvUw/pIOcwu2O8ZzHEhmigq2jzwRNfJVRMJB7bR8= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= @@ -258,6 +256,8 @@ github.com/projectdiscovery/clistats v0.1.7 h1:NXf4lUzx1QcbJd6QycNwwYHTv2FsW9Mch github.com/projectdiscovery/clistats v0.1.7/go.mod h1:RxRGxhMzkT344VTTvZWIrojurUXdQQRN1yfwgg/88I8= github.com/projectdiscovery/dsl v0.8.24 h1:rN2w6LSIjA16qKGRjyj70oxdwXGaspe6sAjEkP7F2fk= github.com/projectdiscovery/dsl v0.8.24/go.mod h1:V+EludvS5PS19UNMlM3CuBwZ+sFnMaQhCFGxysDUanQ= +github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c h1:nWPzF0xe4LzMfSHllHbVzBxdAhwtusRisN091e049KI= +github.com/projectdiscovery/fastdialer v0.5.26-0.20261013195407-11ed52b6df6c/go.mod h1:CZZ0PwdTw2KCI/PrZTW9wyBY88MIjpT8WhsKWSw3bkw= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= From 2bb9dae9a022c7c6e913d786518bcc1ca9d78102 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 18:40:22 +0000 Subject: [PATCH 04/16] test(runner): probe a target line pinned to an IPv6 ip The test serves on [::1] and is skipped where IPv6 is not available. The server helper of the pinned test is shared with it. --- runner/targetline_test.go | 49 ++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 9b505abc9..c11eaed37 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -158,26 +158,29 @@ func runPinned(t *testing.T, lines ...string) []Result { return results } -func TestTargetLinePinsTheConnection(t *testing.T) { - serve := func(addr, name string) *httptest.Server { - listener, err := net.Listen("tcp", addr) - if err != nil { - t.Skipf("cannot listen on %s: %v", addr, err) - } - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) - })) - _ = ts.Listener.Close() - ts.Listener = listener - ts.Start() - t.Cleanup(ts.Close) - return ts +// serveTitle serves a page whose title is name and the Host header, and skips +// the test when addr cannot be bound. +func serveTitle(t *testing.T, addr, name string) *httptest.Server { + t.Helper() + listener, err := net.Listen("tcp", addr) + if err != nil { + t.Skipf("cannot listen on %s: %v", addr, err) } + ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) + })) + _ = ts.Listener.Close() + ts.Listener = listener + ts.Start() + t.Cleanup(ts.Close) + return ts +} - one := serve("127.0.0.1:0", "one") +func TestTargetLinePinsTheConnection(t *testing.T) { + one := serveTitle(t, "127.0.0.1:0", "one") _, port, err := net.SplitHostPort(one.Listener.Addr().String()) require.NoError(t, err) - serve(net.JoinHostPort("127.0.0.2", port), "two") + serveTitle(t, net.JoinHostPort("127.0.0.2", port), "two") target := "http://pinned.test:" + port skipIfProxied(t, target) @@ -197,6 +200,20 @@ func TestTargetLinePinsTheConnection(t *testing.T) { }, got) } +// An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. +func TestTargetLinePinsIPv6(t *testing.T) { + six := serveTitle(t, "[::1]:0", "six") + _, port, err := net.SplitHostPort(six.Listener.Addr().String()) + require.NoError(t, err) + + target := "http://pinned.test:" + port + skipIfProxied(t, target) + results := runPinned(t, fmt.Sprintf(`{"target": %q, "ip": "::1"}`, target)) + require.Len(t, results, 1) + require.Equal(t, "::1", results[0].HostIP) + require.Equal(t, "six pinned.test:"+port, results[0].Title) +} + func TestTargetLineKeepsSNI(t *testing.T) { var ( mu sync.Mutex From 39f1b437ca4c29d91e55dd471a030e9d103ef56d Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 19:22:59 +0000 Subject: [PATCH 05/16] fix(runner): store the response of each pinned ip apart With a matcher or filter, -sr writes a response after matching under a name hashed from the method and url, so the responses of one url on several ips, from target lines or -pa, overwrote each other. The name now includes the pinned ip, as the -maxhr key does. --- runner/runner.go | 8 ++--- runner/targetline.go | 9 +++++ runner/targetline_test.go | 73 +++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 7286be94a..630ae3f1b 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1881,11 +1881,7 @@ retry: } // check if we have to skip the host:port as a result of a previous failure - hostPort := net.JoinHostPort(URL.Host, URL.Port()) - if target.CustomIP != "" { - // the ips of a host fail independently - hostPort += "@" + target.CustomIP - } + hostPort := pinnedKey(net.JoinHostPort(URL.Host, URL.Port()), target.CustomIP) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors >= r.options.HostMaxErrors { @@ -2518,7 +2514,7 @@ retry: } // store responses or chain in directory - domainFile := method + ":" + URL.EscapedString() + domainFile := pinnedKey(method+":"+URL.EscapedString(), target.CustomIP) hash := hashes.Sha1([]byte(domainFile)) domainResponseFile := fmt.Sprintf("%s.txt", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") diff --git a/runner/targetline.go b/runner/targetline.go index d1ae21a22..a6648512d 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -93,6 +93,15 @@ func (t targetLine) key() string { return strings.TrimSpace(buf.String()) } +// pinnedKey returns key for a target that is not pinned, and key@ip for one +// pinned to ip, as the ips of a host fail and respond independently. +func pinnedKey(key, ip string) string { + if ip == "" { + return key + } + return key + "@" + ip +} + // targetOf returns the target of an input line: the line itself, or the // target of a target line. func targetOf(item string) string { diff --git a/runner/targetline_test.go b/runner/targetline_test.go index c11eaed37..adbff4e19 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -3,6 +3,7 @@ package runner import ( "crypto/tls" "fmt" + "math" "net" "net/http" "net/http/httptest" @@ -128,27 +129,32 @@ func skipIfProxied(t *testing.T, target string) { // runPinned probes the target lines with a runner and returns the results. func runPinned(t *testing.T, lines ...string) []Result { t.Helper() + return runPinnedWith(t, &Options{}, lines...) +} + +// runPinnedWith is runPinned with options holding the flags under test. +func runPinnedWith(t *testing.T, options *Options, lines ...string) []Result { + t.Helper() var ( mu sync.Mutex results []Result ) - r, err := New(&Options{ - Threads: 1, - RateLimit: 10, - Timeout: 5, - Methods: http.MethodGet, - Delay: -1, - OnResult: func(r Result) { - if r.Err != nil || r.URL == "" { - return - } - mu.Lock() - results = append(results, r) - mu.Unlock() - }, - InputTargetHost: lines, - }) + options.Threads = 1 + options.RateLimit = 10 + options.Timeout = 5 + options.Methods = http.MethodGet + options.Delay = -1 + options.OnResult = func(r Result) { + if r.Err != nil || r.URL == "" { + return + } + mu.Lock() + results = append(results, r) + mu.Unlock() + } + options.InputTargetHost = lines + r, err := New(options) require.NoError(t, err) defer r.Close() r.RunEnumeration() @@ -176,7 +182,10 @@ func serveTitle(t *testing.T, addr, name string) *httptest.Server { return ts } -func TestTargetLinePinsTheConnection(t *testing.T) { +// servePair serves "one" on 127.0.0.1 and "two" on 127.0.0.2 with the same +// port, and returns the port and the lines pinning pinned.test to each ip. +func servePair(t *testing.T) (string, []string) { + t.Helper() one := serveTitle(t, "127.0.0.1:0", "one") _, port, err := net.SplitHostPort(one.Listener.Addr().String()) require.NoError(t, err) @@ -184,10 +193,15 @@ func TestTargetLinePinsTheConnection(t *testing.T) { target := "http://pinned.test:" + port skipIfProxied(t, target) - results := runPinned(t, + return port, []string{ fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target), fmt.Sprintf(`{"target": %q, "ip": "127.0.0.2"}`, target), - ) + } +} + +func TestTargetLinePinsTheConnection(t *testing.T) { + port, lines := servePair(t) + results := runPinned(t, lines...) var got []string for _, result := range results { @@ -200,6 +214,27 @@ func TestTargetLinePinsTheConnection(t *testing.T) { }, got) } +// With a matcher, responses are stored after matching, under a name that +// must differ for each ip of a url. +func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { + _, lines := servePair(t) + options := &Options{ + StoreResponse: true, + StoreResponseDir: t.TempDir(), + MaxResponseBodySizeToSave: math.MaxInt32, + OutputMatchStatusCode: "200", + } + require.NoError(t, options.ValidateOptions()) + results := runPinnedWith(t, options, lines...) + + require.Len(t, results, 2) + for _, result := range results { + data, err := os.ReadFile(result.StoredResponsePath) + require.NoError(t, err) + require.Contains(t, string(data), ""+result.Title+"", "the response of %s", result.HostIP) + } +} + // An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. func TestTargetLinePinsIPv6(t *testing.T) { six := serveTitle(t, "[::1]:0", "six") From 299c575f164db29bce6b6547475534446ceec77f Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 19:35:35 +0000 Subject: [PATCH 06/16] fix(runner): check the pinned ip for -e cdn and refuse env-proxied lines -e cdn classified the first resolved ip of a host, so a target pinned to an origin behind a CDN was skipped on ports other than 80 and 443; the pinned ip is classified instead. A pinned line whose request goes through a proxy set in HTTP_PROXY or HTTPS_PROXY is refused like one with -proxy, as the proxy resolves the host itself. --- README.md | 2 +- runner/runner.go | 31 +++++++++++++++----------- runner/targetline.go | 28 +++++++++++++++++++++++ runner/targetline_test.go | 47 +++++++++++++++++++++++++++++++++++---- 4 files changed, 90 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 285bb3d7a..f61df6516 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with a warning with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP; a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well. + Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP, nor probed through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY`, which resolves the host itself. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/runner/runner.go b/runner/runner.go index 630ae3f1b..5821ac108 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2776,7 +2776,7 @@ retry: } func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) { - if r.skipCDNPort(URL.Hostname(), URL.Port()) { + if r.skipCDNPort(URL.Hostname(), target.CustomIP, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} } @@ -3050,24 +3050,29 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint return res } -func (r *Runner) skipCDNPort(host string, port string) bool { +// skipCDNPort checks the ip a target is pinned to, or the first ip of host +// when it is not pinned. +func (r *Runner) skipCDNPort(host, ip, port string) bool { // if the option is not enabled we don't skip if !r.scanopts.ExcludeCDN { return false } - // uses the dealer to pre-resolve the target - dnsData, err := r.hp.Dialer.GetDNSData(host) - // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } + hostIP := ip + if hostIP == "" { + // uses the dealer to pre-resolve the target + dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly + if err != nil { + return false + } - if len(dnsData.A) == 0 { - return false - } + if len(dnsData.A) == 0 { + return false + } - // pick the first ip as target - hostIP := dnsData.A[0] + // pick the first ip as target + hostIP = dnsData.A[0] + } isCdnIP, _, _, err := r.hp.CdnCheck(hostIP) if err != nil { diff --git a/runner/targetline.go b/runner/targetline.go index a6648512d..dcad3dd52 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "io" + "net/http" "net/netip" + "net/url" "strings" "github.com/projectdiscovery/mapcidr/asn" @@ -129,10 +131,36 @@ func (r *Runner) normalizeInput(item string) (string, error) { if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 { return "", fmt.Errorf("target %q: a target with an ip cannot be probed with %s", line.Target, strings.Join(flags, ", ")) } + if envProxied(line.Target) { + return "", fmt.Errorf("target %q: a target with an ip cannot be probed through the proxy set in HTTP_PROXY or HTTPS_PROXY", line.Target) + } } return line.key(), nil } +// proxyFromEnvironment is the proxy of the HTTP client when -proxy is not set. +var proxyFromEnvironment = http.ProxyFromEnvironment + +// envProxied reports whether a request to target goes through a proxy set in +// the environment, which resolves the host itself. A target without a scheme +// may be probed over both. +func envProxied(target string) bool { + urls := []string{target} + if !strings.Contains(target, "://") { + urls = []string{"https://" + target, "http://" + target} + } + for _, rawURL := range urls { + u, err := url.Parse(rawURL) + if err != nil { + continue + } + if proxy, _ := proxyFromEnvironment(&http.Request{URL: u}); proxy != nil { + return true + } + } + return false +} + // pinnedTargetConflicts returns the flags whose probes do not connect through // the ip a target is pinned to. func (options *Options) pinnedTargetConflicts() []string { diff --git a/runner/targetline_test.go b/runner/targetline_test.go index adbff4e19..5848b0223 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -56,6 +56,7 @@ func TestParseTargetLine(t *testing.T) { } func TestRunner_normalizeInput(t *testing.T) { + stubEnvProxy(t, http.ProxyURL(nil)) r := &Runner{options: &Options{}} for _, line := range []string{ `{"target": "https://a.example", "ip": "192.0.2.10"}`, @@ -80,7 +81,46 @@ func TestRunner_normalizeInput(t *testing.T) { require.NoError(t, err, "only pinned targets conflict") } +// stubEnvProxy replaces the proxy set in the environment for the test. +func stubEnvProxy(t *testing.T, proxy func(*http.Request) (*url.URL, error)) { + t.Helper() + proxied := proxyFromEnvironment + t.Cleanup(func() { proxyFromEnvironment = proxied }) + proxyFromEnvironment = proxy +} + +func TestRunner_normalizeInput_envProxy(t *testing.T) { + stubEnvProxy(t, func(req *http.Request) (*url.URL, error) { + if req.URL.Scheme == "https" { + return url.Parse("http://127.0.0.1:3128") + } + return nil, nil + }) + + r := &Runner{options: &Options{}} + _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorContains(t, err, "HTTPS_PROXY") + _, err = r.normalizeInput(`{"target": "a.example", "ip": "192.0.2.10"}`) + require.Error(t, err, "a target without a scheme may be probed over https") + _, err = r.normalizeInput(`{"target": "http://a.example", "ip": "192.0.2.10"}`) + require.NoError(t, err, "http is not proxied") + _, err = r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only pinned targets conflict") +} + +func TestRunner_skipCDNPort_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn"}}) + require.NoError(t, err) + defer r.Close() + + // pinned.test does not resolve: only the pinned ip can be classified + require.True(t, r.skipCDNPort("pinned.test", "151.101.1.1", "8443"), "a CDN ip on another port") + require.False(t, r.skipCDNPort("pinned.test", "151.101.1.1", "443"), "a CDN ip on 443") + require.False(t, r.skipCDNPort("pinned.test", "192.0.2.10", "8443"), "an origin ip") +} + func TestRunner_targetLine_targets(t *testing.T) { + stubEnvProxy(t, http.ProxyURL(nil)) // -pa must not resolve a pinned target r, err := New(&Options{ProbeAllIPS: true}) require.NoError(t, err) @@ -96,6 +136,7 @@ func TestRunner_targetLine_targets(t *testing.T) { } func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { + stubEnvProxy(t, http.ProxyURL(nil)) file := filepath.Join(t.TempDir(), "targets.txt") lines := "a.example\n" + `{"target": "b.example"` + "\n" + @@ -119,10 +160,8 @@ func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { // in the environment, as the proxy resolves the host itself. func skipIfProxied(t *testing.T, target string) { t.Helper() - u, err := url.Parse(target) - require.NoError(t, err) - if proxy, _ := http.ProxyFromEnvironment(&http.Request{URL: u}); proxy != nil { - t.Skipf("%s goes through the proxy %s", target, proxy.Redacted()) + if envProxied(target) { + t.Skipf("%s goes through a proxy set in the environment", target) } } From fce694a6d6b49f424171b88f633f383b23d55744 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 19:53:10 +0000 Subject: [PATCH 07/16] fix(httpx): fail a pinned request that would go through a proxy A proxy resolves the host itself, so a request to a pinned host through a proxy set with -proxy or in HTTP_PROXY/HTTPS_PROXY did not reach the ip. Pinning moves to httpx.WithPinnedIP, and the transport fails such a request. The check sees the URL actually requested, so -ports, a scheme fallback and NO_PROXY apply as for any request, and a redirect to another host still goes through the proxy. It replaces the check of target lines against the environment. --- README.md | 2 +- common/httpx/httpx.go | 1 + common/httpx/pinned.go | 53 +++++++++++++++++++++++++++++++++++++ common/httpx/pinned_test.go | 37 ++++++++++++++++++++++++++ runner/runner.go | 7 ++--- runner/targetline.go | 28 -------------------- runner/targetline_test.go | 42 +++++++++-------------------- 7 files changed, 107 insertions(+), 63 deletions(-) create mode 100644 common/httpx/pinned.go create mode 100644 common/httpx/pinned_test.go diff --git a/README.md b/README.md index f61df6516..664048cf9 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP, nor probed through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY`, which resolves the host itself. + Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP. A probe that a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` applies to fails as well, as the proxy resolves the host itself. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 91c86a49d..b5318b13d 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } + transport.Proxy = refusePinned(transport.Proxy) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go new file mode 100644 index 000000000..c766aa882 --- /dev/null +++ b/common/httpx/pinned.go @@ -0,0 +1,53 @@ +package httpx + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "golang.org/x/net/idna" +) + +// errPinnedThroughProxy is the error of a request to a pinned host that would +// go through a proxy, which resolves the host itself. +var errPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") + +// pinnedHostKey is the context key of the host pinned with WithPinnedIP. +type pinnedHostKey struct{} + +// WithPinnedIP returns a copy of ctx in which requests to host connect to ip, +// like curl --resolve. The host keeps its name for the Host header and SNI, +// and requests to other hosts, such as redirect targets, are resolved as usual. +func WithPinnedIP(ctx context.Context, host, ip string) context.Context { + ctx = fastdialer.WithResolveOverride(ctx, host, ip) + return context.WithValue(ctx, pinnedHostKey{}, canonicalHost(host)) +} + +// isPinned reports whether req goes to the host pinned in its context. +func isPinned(req *http.Request) bool { + host, ok := req.Context().Value(pinnedHostKey{}).(string) + return ok && host == canonicalHost(req.URL.Hostname()) +} + +// canonicalHost returns host in the form fastdialer compares hosts in. +func canonicalHost(host string) string { + if ascii, err := idna.ToASCII(host); err == nil { + host = ascii + } + return strings.TrimSuffix(strings.ToLower(host), ".") +} + +// refusePinned wraps the proxy of a transport so that a request to a pinned +// host fails instead of going through the proxy. +func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if proxyURL != nil && isPinned(req) { + return nil, errPinnedThroughProxy + } + return proxyURL, err + } +} diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go new file mode 100644 index 000000000..2056a78bd --- /dev/null +++ b/common/httpx/pinned_test.go @@ -0,0 +1,37 @@ +package httpx + +import ( + "context" + "net/http" + "net/url" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefusePinned(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") + require.NoError(t, err) + proxy := refusePinned(http.ProxyURL(proxyURL)) + ctx := WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10") + + pinned, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test:8443/", nil) + require.NoError(t, err) + _, err = proxy(pinned) + require.ErrorIs(t, err, errPinnedThroughProxy) + + idn, err := http.NewRequestWithContext(WithPinnedIP(context.Background(), "bücher.test", "192.0.2.10"), http.MethodGet, "https://xn--bcher-kva.test/", nil) + require.NoError(t, err) + _, err = proxy(idn) + require.ErrorIs(t, err, errPinnedThroughProxy, "an IDN host in ASCII form") + + redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + got, err := proxy(redirect) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a request to another host keeps the proxy") + + got, err = refusePinned(http.ProxyURL(nil))(pinned) + require.NoError(t, err) + require.Nil(t, got, "a pinned request without a proxy") +} diff --git a/runner/runner.go b/runner/runner.go index 5821ac108..23e80004c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -33,7 +33,6 @@ import ( "github.com/happyhackingspace/dit" "github.com/mfonda/simhash" asnmap "github.com/projectdiscovery/asnmap/libs" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -744,7 +743,7 @@ func (r *Runner) streamInput() (chan string, error) { } // sendLine normalizes, deduplicates and sends an input line, returning // false if interrupted. The scan has already started, so an invalid - // line is skipped with a warning. + // line is skipped with an error. sendLine := func(item string) bool { item, err := r.normalizeInput(item) if err != nil { @@ -1912,9 +1911,7 @@ retry: ) ctx = context.Background() if target.CustomIP != "" { - // only dials to the target's host go to the ip, not those to a - // redirect target on another host - ctx = fastdialer.WithResolveOverride(ctx, URL.Hostname(), target.CustomIP) + ctx = httpx.WithPinnedIP(ctx, URL.Hostname(), target.CustomIP) } req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { diff --git a/runner/targetline.go b/runner/targetline.go index dcad3dd52..a6648512d 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -6,9 +6,7 @@ import ( "errors" "fmt" "io" - "net/http" "net/netip" - "net/url" "strings" "github.com/projectdiscovery/mapcidr/asn" @@ -131,36 +129,10 @@ func (r *Runner) normalizeInput(item string) (string, error) { if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 { return "", fmt.Errorf("target %q: a target with an ip cannot be probed with %s", line.Target, strings.Join(flags, ", ")) } - if envProxied(line.Target) { - return "", fmt.Errorf("target %q: a target with an ip cannot be probed through the proxy set in HTTP_PROXY or HTTPS_PROXY", line.Target) - } } return line.key(), nil } -// proxyFromEnvironment is the proxy of the HTTP client when -proxy is not set. -var proxyFromEnvironment = http.ProxyFromEnvironment - -// envProxied reports whether a request to target goes through a proxy set in -// the environment, which resolves the host itself. A target without a scheme -// may be probed over both. -func envProxied(target string) bool { - urls := []string{target} - if !strings.Contains(target, "://") { - urls = []string{"https://" + target, "http://" + target} - } - for _, rawURL := range urls { - u, err := url.Parse(rawURL) - if err != nil { - continue - } - if proxy, _ := proxyFromEnvironment(&http.Request{URL: u}); proxy != nil { - return true - } - } - return false -} - // pinnedTargetConflicts returns the flags whose probes do not connect through // the ip a target is pinned to. func (options *Options) pinnedTargetConflicts() []string { diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 5848b0223..e944c44ad 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -56,7 +56,6 @@ func TestParseTargetLine(t *testing.T) { } func TestRunner_normalizeInput(t *testing.T) { - stubEnvProxy(t, http.ProxyURL(nil)) r := &Runner{options: &Options{}} for _, line := range []string{ `{"target": "https://a.example", "ip": "192.0.2.10"}`, @@ -81,31 +80,16 @@ func TestRunner_normalizeInput(t *testing.T) { require.NoError(t, err, "only pinned targets conflict") } -// stubEnvProxy replaces the proxy set in the environment for the test. -func stubEnvProxy(t *testing.T, proxy func(*http.Request) (*url.URL, error)) { - t.Helper() - proxied := proxyFromEnvironment - t.Cleanup(func() { proxyFromEnvironment = proxied }) - proxyFromEnvironment = proxy -} - -func TestRunner_normalizeInput_envProxy(t *testing.T) { - stubEnvProxy(t, func(req *http.Request) (*url.URL, error) { - if req.URL.Scheme == "https" { - return url.Parse("http://127.0.0.1:3128") - } - return nil, nil - }) +// A proxy, set with -proxy for -pa or in the environment, resolves the host +// itself, so a probe of a pinned target through it fails. +func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { + r, err := New(&Options{HTTPProxy: "http://127.0.0.1:1"}) + require.NoError(t, err) + defer r.Close() - r := &Runner{options: &Options{}} - _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) - require.ErrorContains(t, err, "HTTPS_PROXY") - _, err = r.normalizeInput(`{"target": "a.example", "ip": "192.0.2.10"}`) - require.Error(t, err, "a target without a scheme may be probed over https") - _, err = r.normalizeInput(`{"target": "http://a.example", "ip": "192.0.2.10"}`) - require.NoError(t, err, "http is not proxied") - _, err = r.normalizeInput(`{"target": "https://a.example"}`) - require.NoError(t, err, "only pinned targets conflict") + target := httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"} + result := r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) + require.ErrorContains(t, result.Err, "through a proxy") } func TestRunner_skipCDNPort_pinned(t *testing.T) { @@ -120,7 +104,6 @@ func TestRunner_skipCDNPort_pinned(t *testing.T) { } func TestRunner_targetLine_targets(t *testing.T) { - stubEnvProxy(t, http.ProxyURL(nil)) // -pa must not resolve a pinned target r, err := New(&Options{ProbeAllIPS: true}) require.NoError(t, err) @@ -136,7 +119,6 @@ func TestRunner_targetLine_targets(t *testing.T) { } func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { - stubEnvProxy(t, http.ProxyURL(nil)) file := filepath.Join(t.TempDir(), "targets.txt") lines := "a.example\n" + `{"target": "b.example"` + "\n" + @@ -160,8 +142,10 @@ func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { // in the environment, as the proxy resolves the host itself. func skipIfProxied(t *testing.T, target string) { t.Helper() - if envProxied(target) { - t.Skipf("%s goes through a proxy set in the environment", target) + u, err := url.Parse(target) + require.NoError(t, err) + if proxy, _ := http.ProxyFromEnvironment(&http.Request{URL: u}); proxy != nil { + t.Skipf("%s goes through the proxy %s", target, proxy.Redacted()) } } From 06d914a5d25edfe517a121a64112ff811f870304 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 21:08:33 +0000 Subject: [PATCH 08/16] refactor(httpx): key each pinned host in the context WithPinnedIP keeps a context key per host, as fastdialer does for its overrides, so the proxy is refused for every host pinned in a context, not only the last one. The README sentence on proxies set in the environment is simplified, and the probe tests follow the input tests. --- README.md | 2 +- common/httpx/pinned.go | 11 ++++----- common/httpx/pinned_test.go | 5 ++++ runner/targetline_test.go | 46 ++++++++++++++++++------------------- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 664048cf9..a1eccd92c 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP. A probe that a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` applies to fails as well, as the proxy resolves the host itself. + Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP. A probe that would go through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` fails as well, as the proxy resolves the host itself. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go index c766aa882..76578e81a 100644 --- a/common/httpx/pinned.go +++ b/common/httpx/pinned.go @@ -15,21 +15,20 @@ import ( // go through a proxy, which resolves the host itself. var errPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") -// pinnedHostKey is the context key of the host pinned with WithPinnedIP. -type pinnedHostKey struct{} +// pinnedHostKey is the context key of a host pinned with WithPinnedIP. +type pinnedHostKey string // WithPinnedIP returns a copy of ctx in which requests to host connect to ip, // like curl --resolve. The host keeps its name for the Host header and SNI, // and requests to other hosts, such as redirect targets, are resolved as usual. func WithPinnedIP(ctx context.Context, host, ip string) context.Context { ctx = fastdialer.WithResolveOverride(ctx, host, ip) - return context.WithValue(ctx, pinnedHostKey{}, canonicalHost(host)) + return context.WithValue(ctx, pinnedHostKey(canonicalHost(host)), ip) } -// isPinned reports whether req goes to the host pinned in its context. +// isPinned reports whether req goes to a host pinned in its context. func isPinned(req *http.Request) bool { - host, ok := req.Context().Value(pinnedHostKey{}).(string) - return ok && host == canonicalHost(req.URL.Hostname()) + return req.Context().Value(pinnedHostKey(canonicalHost(req.URL.Hostname()))) != nil } // canonicalHost returns host in the form fastdialer compares hosts in. diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go index 2056a78bd..78378bfb2 100644 --- a/common/httpx/pinned_test.go +++ b/common/httpx/pinned_test.go @@ -25,6 +25,11 @@ func TestRefusePinned(t *testing.T) { _, err = proxy(idn) require.ErrorIs(t, err, errPinnedThroughProxy, "an IDN host in ASCII form") + second, err := http.NewRequestWithContext(WithPinnedIP(ctx, "second.test", "192.0.2.11"), http.MethodGet, "https://second.test/", nil) + require.NoError(t, err) + _, err = proxy(second) + require.ErrorIs(t, err, errPinnedThroughProxy, "each host pinned in a context") + redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) require.NoError(t, err) got, err := proxy(redirect) diff --git a/runner/targetline_test.go b/runner/targetline_test.go index e944c44ad..c0b1741ca 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -80,29 +80,6 @@ func TestRunner_normalizeInput(t *testing.T) { require.NoError(t, err, "only pinned targets conflict") } -// A proxy, set with -proxy for -pa or in the environment, resolves the host -// itself, so a probe of a pinned target through it fails. -func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { - r, err := New(&Options{HTTPProxy: "http://127.0.0.1:1"}) - require.NoError(t, err) - defer r.Close() - - target := httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"} - result := r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) - require.ErrorContains(t, result.Err, "through a proxy") -} - -func TestRunner_skipCDNPort_pinned(t *testing.T) { - r, err := New(&Options{Exclude: []string{"cdn"}}) - require.NoError(t, err) - defer r.Close() - - // pinned.test does not resolve: only the pinned ip can be classified - require.True(t, r.skipCDNPort("pinned.test", "151.101.1.1", "8443"), "a CDN ip on another port") - require.False(t, r.skipCDNPort("pinned.test", "151.101.1.1", "443"), "a CDN ip on 443") - require.False(t, r.skipCDNPort("pinned.test", "192.0.2.10", "8443"), "an origin ip") -} - func TestRunner_targetLine_targets(t *testing.T) { // -pa must not resolve a pinned target r, err := New(&Options{ProbeAllIPS: true}) @@ -138,6 +115,29 @@ func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { require.Equal(t, []string{"a.example", `{"target":"c.example","ip":"192.0.2.10"}`}, got, "the malformed line is skipped") } +// A proxy, set with -proxy for -pa or in the environment, resolves the host +// itself, so a probe of a pinned target through it fails. +func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { + r, err := New(&Options{HTTPProxy: "http://127.0.0.1:1"}) + require.NoError(t, err) + defer r.Close() + + target := httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"} + result := r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) + require.ErrorContains(t, result.Err, "through a proxy") +} + +func TestRunner_skipCDNPort_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn"}}) + require.NoError(t, err) + defer r.Close() + + // pinned.test does not resolve: only the pinned ip can be classified + require.True(t, r.skipCDNPort("pinned.test", "151.101.1.1", "8443"), "a CDN ip on another port") + require.False(t, r.skipCDNPort("pinned.test", "151.101.1.1", "443"), "a CDN ip on 443") + require.False(t, r.skipCDNPort("pinned.test", "192.0.2.10", "8443"), "an origin ip") +} + // skipIfProxied skips a test whose requests to target go through a proxy set // in the environment, as the proxy resolves the host itself. func skipIfProxied(t *testing.T, target string) { From 86c00f85fbb2c5cc9f274eaae7920cf31bcbacec Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 21:20:35 +0000 Subject: [PATCH 09/16] fix(runner): do not resolve the host of a pinned target Like curl --resolve, a pinned target is probed without resolving its host: a, aaaa and cname come from the dialer cache only, so the pinned ip is reported as its address, while -pa, which resolved the host for its ips, keeps all of them. Before, every probe of a pinned target looked the host up, which sent its name to the resolvers and reported their addresses. --- README.md | 2 +- runner/runner.go | 16 ++++++--- runner/targetline_test.go | 74 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a1eccd92c..cdfb06a12 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,7 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` -- A target can be probed through a given IP with a JSON input line, like `curl --resolve`. The hostname is still used for the Host header and SNI, and the connection goes to the IP, which `host_ip` reports. The same host can be listed with several IPs: +- A target can be probed through a given IP with a JSON input line, like `curl --resolve`. The hostname is still used for the Host header and SNI, but it is not resolved: the connection goes to the IP, which `host_ip` reports. The same host can be listed with several IPs: ``` {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} diff --git a/runner/runner.go b/runner/runner.go index 23e80004c..01ade01b6 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1836,7 +1836,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results <- httpx.Target{Host: target} return } - ips, _, _, err := getDNSData(hp, URL.Hostname()) + ips, _, _, err := getDNSData(hp, URL.Hostname(), false) if err != nil || len(ips) == 0 { results <- httpx.Target{Host: target} return @@ -2342,7 +2342,9 @@ retry: if err != nil { onlyHost = URL.Host } - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost) + // the host of a pinned target is not resolved: only dns data already + // cached, such as that of -pa, is reported + allIps, cnames, resolvers, err := getDNSData(hp, onlyHost, target.CustomIP != "") if err != nil { allIps = append(allIps, ip) } @@ -3096,8 +3098,14 @@ func (r *Runner) parseURL(url string) (*urlutil.URL, error) { return urlx, err } -func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames, resolvers []string, err error) { - dnsData, err := hp.Dialer.GetDNSData(hostname) +// getDNSData returns the dns data of hostname, from the cache of the dialer +// only if cachedOnly is set. +func getDNSData(hp *httpx.HTTPX, hostname string, cachedOnly bool) (ips, cnames, resolvers []string, err error) { + lookup := hp.Dialer.GetDNSData + if cachedOnly { + lookup = hp.Dialer.GetDNSDataFromCache + } + dnsData, err := lookup(hostname) if err != nil { return nil, nil, nil, err } diff --git a/runner/targetline_test.go b/runner/targetline_test.go index c0b1741ca..8d30da272 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -11,11 +11,14 @@ import ( "os" "path/filepath" "sort" + "strings" "sync" + "sync/atomic" "testing" "github.com/projectdiscovery/httpx/common/httpx" "github.com/stretchr/testify/require" + "golang.org/x/net/dns/dnsmessage" ) func TestParseTargetLine(t *testing.T) { @@ -222,6 +225,53 @@ func servePair(t *testing.T) (string, []string) { } } +// serveDNS runs a resolver that answers the A queries for host with ips, and +// returns its address and the number of queries for host it got. +func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) { + t.Helper() + conn, err := net.ListenPacket("udp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + var queries atomic.Int32 + go func() { + buf := make([]byte, 512) + for { + n, addr, err := conn.ReadFrom(buf) + if err != nil { + return + } + var parser dnsmessage.Parser + header, err := parser.Start(buf[:n]) + if err != nil { + continue + } + question, err := parser.Question() + if err != nil { + continue + } + header.Response = true + builder := dnsmessage.NewBuilder(nil, header) + _ = builder.StartQuestions() + _ = builder.Question(question) + _ = builder.StartAnswers() + if strings.EqualFold(strings.TrimSuffix(question.Name.String(), "."), host) { + queries.Add(1) + for _, ip := range ips { + if question.Type == dnsmessage.TypeA { + answer := dnsmessage.ResourceHeader{Name: question.Name, Class: dnsmessage.ClassINET, TTL: 60} + _ = builder.AResource(answer, dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())}) + } + } + } + if msg, err := builder.Finish(); err == nil { + _, _ = conn.WriteTo(msg, addr) + } + } + }() + return conn.LocalAddr().String(), &queries +} + func TestTargetLinePinsTheConnection(t *testing.T) { port, lines := servePair(t) results := runPinned(t, lines...) @@ -258,6 +308,30 @@ func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { } } +// Like curl --resolve, the host of a pinned target is not resolved, and the +// pinned ip is reported as its address. +func TestTargetLineIsNotResolved(t *testing.T) { + _, lines := servePair(t) + resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") + results := runPinnedWith(t, &Options{Resolvers: []string{resolver}}, lines[0]) + + require.Len(t, results, 1) + require.Equal(t, []string{"127.0.0.1"}, results[0].A) + require.Zero(t, queries.Load()) +} + +// -pa resolves the host, and each of its results keeps all of its addresses. +func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { + port, _ := servePair(t) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1", "127.0.0.2") + results := runPinnedWith(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: true}, "http://pinned.test:"+port) + + require.Len(t, results, 2) + for _, result := range results { + require.ElementsMatch(t, []string{"127.0.0.1", "127.0.0.2"}, result.A, "the result of %s", result.HostIP) + } +} + // An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. func TestTargetLinePinsIPv6(t *testing.T) { six := serveTitle(t, "[::1]:0", "six") From 93291f54c14c83e5c2405ded2c0d83ac891f7a7f Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 21:59:54 +0000 Subject: [PATCH 10/16] fix(runner): refuse pinned lines with -pr http2 An HTTP/2 connection serves every request to its host, whatever ip it was dialed to, so concurrent probes of one host pinned to different ips could get each other's responses with -pr http2. A pinned line is refused with it, as with -http2. --- README.md | 2 +- runner/targetline.go | 7 ++++++- runner/targetline_test.go | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cdfb06a12..204946274 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP. A probe that would go through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` fails as well, as the proxy resolves the host itself. + Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP, nor with `-pr http2`, which shares a connection between the IPs of a host. A probe that would go through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` fails as well, as the proxy resolves the host itself. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/runner/targetline.go b/runner/targetline.go index a6648512d..8ff6d465a 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -9,6 +9,7 @@ import ( "net/netip" "strings" + "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" iputil "github.com/projectdiscovery/utils/ip" stringsutil "github.com/projectdiscovery/utils/strings" @@ -134,7 +135,8 @@ func (r *Runner) normalizeInput(item string) (string, error) { } // pinnedTargetConflicts returns the flags whose probes do not connect through -// the ip a target is pinned to. +// the ip a target is pinned to, or share a connection between the ips of a +// host, as HTTP/2 does. func (options *Options) pinnedTargetConflicts() []string { var flags []string if options.Unsafe { @@ -146,6 +148,9 @@ func (options *Options) pinnedTargetConflicts() []string { if options.HTTP2Probe { flags = append(flags, "-http2") } + if strings.EqualFold(options.Protocol, string(httpx.HTTP2)) { + flags = append(flags, "-pr http2") + } if options.Jarm { flags = append(flags, "-jarm") } diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 8d30da272..85b33fd5b 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -76,9 +76,9 @@ func TestRunner_normalizeInput(t *testing.T) { _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2"}`) require.Error(t, err) - r = &Runner{options: &Options{Unsafe: true, Jarm: true}} + r = &Runner{options: &Options{Unsafe: true, Jarm: true, Protocol: "HTTP2"}} _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) - require.ErrorContains(t, err, "-unsafe, -jarm") + require.ErrorContains(t, err, "-unsafe, -pr http2, -jarm") _, err = r.normalizeInput(`{"target": "https://a.example"}`) require.NoError(t, err, "only pinned targets conflict") } From 150e7875d3cc050205fd7c08ac7ad5de6a4485d7 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 22:01:21 +0000 Subject: [PATCH 11/16] fix(runner): keep the pinned ip out of a and aaaa a and aaaa hold dns data. When the host of a pinned target is not cached, there is none, and the pinned ip stays in host_ip instead of being reported as a record of the host. --- runner/runner.go | 7 ++++--- runner/targetline_test.go | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 01ade01b6..de7640ebe 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2343,9 +2343,10 @@ retry: onlyHost = URL.Host } // the host of a pinned target is not resolved: only dns data already - // cached, such as that of -pa, is reported - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost, target.CustomIP != "") - if err != nil { + // cached, such as that of -pa, is reported, and its ip is in host_ip + pinned := target.CustomIP != "" + allIps, cnames, resolvers, err := getDNSData(hp, onlyHost, pinned) + if err != nil && !pinned { allIps = append(allIps, ip) } diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 85b33fd5b..89b1f7f29 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -308,15 +308,16 @@ func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { } } -// Like curl --resolve, the host of a pinned target is not resolved, and the -// pinned ip is reported as its address. +// Like curl --resolve, the host of a pinned target is not resolved: the +// pinned ip is reported in host_ip, and a holds dns data only. func TestTargetLineIsNotResolved(t *testing.T) { _, lines := servePair(t) resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") results := runPinnedWith(t, &Options{Resolvers: []string{resolver}}, lines[0]) require.Len(t, results, 1) - require.Equal(t, []string{"127.0.0.1"}, results[0].A) + require.Equal(t, "127.0.0.1", results[0].HostIP) + require.Empty(t, results[0].A) require.Zero(t, queries.Load()) } From b5908a313622df4bf669013fbadcf4d1dffdd73e Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 22:09:58 +0000 Subject: [PATCH 12/16] fix(runner): reject a target line whose ip is null or empty encoding/json leaves a string empty for null, so a line with "ip": null or "ip": "" was taken as a line without ip and probed through DNS. The ip is decoded raw, and a present ip goes to the ip validation, which rejects both. --- runner/targetline.go | 23 ++++++++++++++++------- runner/targetline_test.go | 3 +++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/runner/targetline.go b/runner/targetline.go index 8ff6d465a..ed9936163 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -33,27 +33,36 @@ func isTargetLine(line string) bool { return strings.HasPrefix(strings.TrimSpace(line), "{") } -// parseTargetLine decodes a target line. Unknown keys are rejected so that a -// typo cannot silently probe the host through DNS. +// parseTargetLine decodes a target line. Unknown keys and an ip that is null +// or empty are rejected so that a typo cannot silently probe the host through +// DNS. func parseTargetLine(line string) (targetLine, error) { + // the ip is kept raw to tell an ip that is null or empty from no ip + var decoded struct { + Target string `json:"target"` + IP json.RawMessage `json:"ip"` + } decoder := json.NewDecoder(strings.NewReader(line)) decoder.DisallowUnknownFields() - var parsed targetLine - if err := decoder.Decode(&parsed); err != nil { + if err := decoder.Decode(&decoded); err != nil { return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { return targetLine{}, fmt.Errorf("invalid target line %q: unexpected data after the object", line) } - parsed.Target = strings.TrimSpace(parsed.Target) + parsed := targetLine{Target: strings.TrimSpace(decoded.Target)} if parsed.Target == "" { return targetLine{}, fmt.Errorf("target line %q has no target", line) } - if parsed.IP == "" { + if decoded.IP == nil { return parsed, nil } - ip, err := pinnedIP(parsed.Target, parsed.IP) + var value string + if err := json.Unmarshal(decoded.IP, &value); err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + ip, err := pinnedIP(parsed.Target, value) if err != nil { return targetLine{}, fmt.Errorf("target %q: %w", parsed.Target, err) } diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 89b1f7f29..2714eeda5 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -42,6 +42,9 @@ func TestParseTargetLine(t *testing.T) { "unknown key": `{"target": "a.example", "ips": "192.0.2.10"}`, "missing target": `{"ip": "192.0.2.10"}`, "invalid ip": `{"target": "a.example", "ip": "192.0.2"}`, + "null ip": `{"target": "a.example", "ip": null}`, + "empty ip": `{"target": "a.example", "ip": ""}`, + "number ip": `{"target": "a.example", "ip": 1}`, "ip with zone": `{"target": "a.example", "ip": "fe80::1%eth0"}`, "unspecified ip": `{"target": "a.example", "ip": "0.0.0.0"}`, "ip target": `{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, From 3ec868822139fad87adf1e588859dc07ee2b61ef Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Tue, 27 Oct 2026 23:12:42 +0000 Subject: [PATCH 13/16] fix(runner): address the review of pinned targets A pinned host is matched as net/http dials it, so an IDN host spelled in upper case or with an ideographic full stop keeps its pin instead of being resolved. A request that is not pinned records the ip it is dialed to, for its host_ip: the dialer keeps only the last ip dialed to a host, which could be the one a pinned target of the same host was dialed to meanwhile. The favicon request and the ztls grab take only the pin of the probe, not its trace, which the favicon request overwrote with -trace. Only the host of a target line goes unresolved; -pa results resolve it as before, as a cache miss emptied their a, aaaa and cname. The ip of a pinned target is shown as with -pa, every target line is checked against the conflicting options, whatever its source, and invalid lines share one error format. The tests cover each of these and run behind a proxy set in the environment. --- README.md | 4 +- common/httpx/dialedip.go | 57 ++++ common/httpx/httpx.go | 4 +- common/httpx/pinned.go | 28 +- common/httpx/pinned_test.go | 79 ++++- common/httpx/tls.go | 3 +- runner/runner.go | 74 ++--- runner/targetline.go | 100 +++--- runner/targetline_test.go | 613 +++++++++++++++++++++++------------- 9 files changed, 638 insertions(+), 324 deletions(-) create mode 100644 common/httpx/dialedip.go diff --git a/README.md b/README.md index 204946274..99ed453ed 100644 --- a/README.md +++ b/README.md @@ -380,12 +380,12 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` -- A target can be probed through a given IP with a JSON input line, like `curl --resolve`. The hostname is still used for the Host header and SNI, but it is not resolved: the connection goes to the IP, which `host_ip` reports. The same host can be listed with several IPs: +- A target can be pinned to an IP with a JSON input line, like `curl --resolve`: the connection goes to the IP, which `host_ip` reports, and the hostname is kept for the Host header and SNI without being resolved. A host can be listed with several IPs, and a line without `ip` is the plain target: ``` {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Only the `target` and `ip` keys are accepted. An invalid line stops the run, or is skipped with an error with `-stream`. Read such lines from `-l` or stdin, as `-u` splits its value on commas. They cannot be combined with `-proxy`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` or `-vhost-input`, which do not connect through the IP, nor with `-pr http2`, which shares a connection between the IPs of a host. A probe that would go through a proxy set with `HTTP_PROXY`/`HTTPS_PROXY` fails as well, as the proxy resolves the host itself. + Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line stops the run, or is skipped with an error with `-stream`. Pinned lines are refused with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, which do not keep to the IP. A proxy resolves the host itself, so a probe of a pinned target, `-pa` ones included, fails if it would go through one, such as one set with `HTTP_PROXY`/`HTTPS_PROXY`. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/dialedip.go b/common/httpx/dialedip.go new file mode 100644 index 000000000..ffdad541f --- /dev/null +++ b/common/httpx/dialedip.go @@ -0,0 +1,57 @@ +package httpx + +import ( + "context" + "net" + "sync" +) + +// dialedIPKey is the context key of the dialedIP of WithDialedIP. +type dialedIPKey struct{} + +// dialedIP is the ip of the last connection dialed to a host. +type dialedIP struct { + host string + mu sync.Mutex + ip string +} + +// WithDialedIP returns a copy of ctx that records the ip of the connections +// dialed to host, for DialedIP. The dialer only keeps the last ip dialed to a +// host by any request, which may be another request's. +func WithDialedIP(ctx context.Context, host string) context.Context { + return context.WithValue(ctx, dialedIPKey{}, &dialedIP{host: canonicalHost(host)}) +} + +// DialedIP returns the ip of the last connection dialed to the host of a +// WithDialedIP context, or "" if none was. +func DialedIP(ctx context.Context) string { + dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) + if !ok { + return "" + } + dialed.mu.Lock() + defer dialed.mu.Unlock() + return dialed.ip +} + +// recordDialedIP wraps dial to record the ip of a connection to the host of a +// WithDialedIP context. A connection to another host, such as a proxy, is not +// recorded. +func recordDialedIP(dial func(ctx context.Context, network, addr string) (net.Conn, error)) func(ctx context.Context, network, addr string) (net.Conn, error) { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) + if err != nil || !ok { + return conn, err + } + host, _, _ := net.SplitHostPort(addr) + ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + if canonicalHost(host) == dialed.host && ip != "" { + dialed.mu.Lock() + dialed.ip = ip + dialed.mu.Unlock() + } + return conn, err + } +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index b5318b13d..4e6e8b678 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) { gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation") } transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, + DialContext: recordDialedIP(httpx.Dialer.Dial), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) { // Clone before protocol-specific ALPN is applied. Impersonation keeps the // previous base config; utls does not take NextProtos from it. impersonateTLS := transport.TLSClientConfig.Clone() - transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS) + transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS)) switch httpx.Options.Protocol { case HTTP11: diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go index 76578e81a..7f0152c1b 100644 --- a/common/httpx/pinned.go +++ b/common/httpx/pinned.go @@ -22,18 +22,32 @@ type pinnedHostKey string // like curl --resolve. The host keeps its name for the Host header and SNI, // and requests to other hosts, such as redirect targets, are resolved as usual. func WithPinnedIP(ctx context.Context, host, ip string) context.Context { + host = canonicalHost(host) ctx = fastdialer.WithResolveOverride(ctx, host, ip) - return context.WithValue(ctx, pinnedHostKey(canonicalHost(host)), ip) + return context.WithValue(ctx, pinnedHostKey(host), ip) } -// isPinned reports whether req goes to a host pinned in its context. -func isPinned(req *http.Request) bool { - return req.Context().Value(pinnedHostKey(canonicalHost(req.URL.Hostname()))) != nil +// PinnedContext returns a new context in which the host of req keeps the ip it +// is pinned to, for another request derived from req. The other values of the +// context of req, such as a trace, are not kept. +func PinnedContext(req *http.Request) context.Context { + ctx := context.Background() + if ip := pinnedIP(req); ip != "" { + ctx = WithPinnedIP(ctx, req.URL.Hostname(), ip) + } + return ctx +} + +// pinnedIP returns the ip the host of req is pinned to in its context, if any. +func pinnedIP(req *http.Request) string { + ip, _ := req.Context().Value(pinnedHostKey(canonicalHost(req.URL.Hostname()))).(string) + return ip } -// canonicalHost returns host in the form fastdialer compares hosts in. +// canonicalHost returns host as net/http dials it, so that a pin matches the +// dials to its host however the host is spelled. func canonicalHost(host string) string { - if ascii, err := idna.ToASCII(host); err == nil { + if ascii, err := idna.Lookup.ToASCII(host); err == nil { host = ascii } return strings.TrimSuffix(strings.ToLower(host), ".") @@ -44,7 +58,7 @@ func canonicalHost(host string) string { func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { return func(req *http.Request) (*url.URL, error) { proxyURL, err := proxy(req) - if proxyURL != nil && isPinned(req) { + if proxyURL != nil && pinnedIP(req) != "" { return nil, errPinnedThroughProxy } return proxyURL, err diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go index 78378bfb2..e5c3d7139 100644 --- a/common/httpx/pinned_test.go +++ b/common/httpx/pinned_test.go @@ -2,7 +2,9 @@ package httpx import ( "context" + "net" "net/http" + "net/http/httptrace" "net/url" "testing" @@ -13,22 +15,14 @@ func TestRefusePinned(t *testing.T) { proxyURL, err := url.Parse("http://127.0.0.1:3128") require.NoError(t, err) proxy := refusePinned(http.ProxyURL(proxyURL)) - ctx := WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10") + ctx := WithPinnedIP(WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10"), "second.test", "192.0.2.11") - pinned, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test:8443/", nil) - require.NoError(t, err) - _, err = proxy(pinned) - require.ErrorIs(t, err, errPinnedThroughProxy) - - idn, err := http.NewRequestWithContext(WithPinnedIP(context.Background(), "bücher.test", "192.0.2.10"), http.MethodGet, "https://xn--bcher-kva.test/", nil) - require.NoError(t, err) - _, err = proxy(idn) - require.ErrorIs(t, err, errPinnedThroughProxy, "an IDN host in ASCII form") - - second, err := http.NewRequestWithContext(WithPinnedIP(ctx, "second.test", "192.0.2.11"), http.MethodGet, "https://second.test/", nil) - require.NoError(t, err) - _, err = proxy(second) - require.ErrorIs(t, err, errPinnedThroughProxy, "each host pinned in a context") + for _, rawURL := range []string{"https://pinned.test:8443/", "https://second.test/"} { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + require.NoError(t, err) + _, err = proxy(req) + require.ErrorIs(t, err, errPinnedThroughProxy, "each host pinned in a context") + } redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) require.NoError(t, err) @@ -36,7 +30,62 @@ func TestRefusePinned(t *testing.T) { require.NoError(t, err) require.Equal(t, proxyURL, got, "a request to another host keeps the proxy") + pinned, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/", nil) + require.NoError(t, err) got, err = refusePinned(http.ProxyURL(nil))(pinned) require.NoError(t, err) require.Nil(t, got, "a pinned request without a proxy") } + +// A pin matches the host as net/http dials it, however the host is spelled. +func TestCanonicalHost(t *testing.T) { + for host, want := range map[string]string{ + "Pinned.Test.": "pinned.test", + "bücher.test": "xn--bcher-kva.test", + "BÜCHER.test": "xn--bcher-kva.test", + "bücher。test": "xn--bcher-kva.test", + "xn--bcher-kva.test": "xn--bcher-kva.test", + } { + require.Equal(t, want, canonicalHost(host), host) + } +} + +func TestPinnedContext(t *testing.T) { + ctx := httptrace.WithClientTrace(WithPinnedIP(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil) + require.NoError(t, err) + + derived := req.Clone(PinnedContext(req)) + require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept") + require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not") + + other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + require.Empty(t, pinnedIP(other.Clone(PinnedContext(other)))) +} + +func TestRecordDialedIP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = listener.Close() }() + // every address is dialed to the listener + dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String()) + }) + ctx := WithDialedIP(context.Background(), "Dialed.Test") + + conn, err := dial(ctx, "tcp", "proxy.test:3128") + require.NoError(t, err) + _ = conn.Close() + require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") + + conn, err = dial(ctx, "tcp", "dialed.test:80") + require.NoError(t, err) + _ = conn.Close() + require.Equal(t, "127.0.0.1", DialedIP(ctx)) + + conn, err = dial(context.Background(), "tcp", "dialed.test:80") + require.NoError(t, err) + _ = conn.Close() + require.Empty(t, DialedIP(context.Background()), "a context without WithDialedIP") +} diff --git a/common/httpx/tls.go b/common/httpx/tls.go index 9ca5e1e9e..77040ad7c 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -63,8 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response { } // canonical net concatenation host = net.JoinHostPort(hostname, fmt.Sprint(port)) - // keeps the ip the request was pinned to, without its cancellation - ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Request.Context()), h.Options.Timeout) + ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout) defer cancel() tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host) if err != nil { diff --git a/runner/runner.go b/runner/runner.go index de7640ebe..7c7f007ce 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1799,9 +1799,9 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { switch { case isTargetLine(target): - line, err := parseTargetLine(target) + line, err := r.parseInputLine(target) if err != nil { - gologger.Warning().Msgf("%s\n", err) + gologger.Warning().Msgf("Skipping input: %s\n", err) return } results <- httpx.Target{Host: line.Target, CustomIP: line.IP} @@ -1836,7 +1836,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results <- httpx.Target{Host: target} return } - ips, _, _, err := getDNSData(hp, URL.Hostname(), false) + ips, _, _, err := getDNSData(hp, URL.Hostname()) if err != nil || len(ips) == 0 { results <- httpx.Target{Host: target} return @@ -1844,7 +1844,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { for _, ip := range ips { results <- httpx.Target{Host: target, CustomIP: ip} } - case !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ","): + case isVHostInput(target): idxComma := strings.Index(target, ",") results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} default: @@ -1905,13 +1905,12 @@ retry: if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil { gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } - var ( - req *retryablehttp.Request - ctx context.Context - ) - ctx = context.Background() + var req *retryablehttp.Request + ctx := context.Background() if target.CustomIP != "" { ctx = httpx.WithPinnedIP(ctx, URL.Hostname(), target.CustomIP) + } else { + ctx = httpx.WithDialedIP(ctx, URL.Hostname()) } req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { @@ -2300,8 +2299,12 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // hp.Dialer.GetDialedIP would return only the last dialed one - ip = hp.Dialer.GetDialedIP(URL.Host) + // the ip this request was dialed to: the dialer only keeps the + // last ip dialed to a host, which may be another request's + ip = httpx.DialedIP(req.Context()) + if ip == "" { + ip = hp.Dialer.GetDialedIP(URL.Host) + } if ip == "" { ip = hp.Dialer.GetDialedIP(onlyHost) } @@ -2333,7 +2336,7 @@ retry: } } - if scanopts.OutputIP || scanopts.ProbeAllIPS { + if scanopts.OutputIP || scanopts.ProbeAllIPS || target.CustomIP != "" { _, _ = fmt.Fprintf(builder, " [%s]", ip) } @@ -2342,12 +2345,14 @@ retry: if err != nil { onlyHost = URL.Host } - // the host of a pinned target is not resolved: only dns data already - // cached, such as that of -pa, is reported, and its ip is in host_ip - pinned := target.CustomIP != "" - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost, pinned) - if err != nil && !pinned { - allIps = append(allIps, ip) + var allIps, cnames, resolvers []string + // like curl --resolve, a target line gives the ip of its host, which is + // not resolved + if !isTargetLine(origInput) { + allIps, cnames, resolvers, err = getDNSData(hp, onlyHost) + if err != nil { + allIps = append(allIps, ip) + } } var ips4, ips6 []string @@ -2841,8 +2846,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } } - // Clone original request (reuse headers/cookies, and the pinned ip) - clone := req.Clone(req.Context()) + // Clone original request (reuse headers/cookies and the pinned ip) + clone := req.Clone(httpx.PinnedContext(req.Request)) var ( faviconMMH3 string @@ -3050,31 +3055,26 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint return res } -// skipCDNPort checks the ip a target is pinned to, or the first ip of host -// when it is not pinned. +// skipCDNPort reports whether a target is skipped as its ip is a CDN ip and +// port is not allowed for one. The ip is the one the target is pinned to, or +// else the first ip of host. func (r *Runner) skipCDNPort(host, ip, port string) bool { // if the option is not enabled we don't skip if !r.scanopts.ExcludeCDN { return false } - hostIP := ip - if hostIP == "" { + if ip == "" { // uses the dealer to pre-resolve the target dnsData, err := r.hp.Dialer.GetDNSData(host) // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } - - if len(dnsData.A) == 0 { + if err != nil || len(dnsData.A) == 0 { return false } - // pick the first ip as target - hostIP = dnsData.A[0] + ip = dnsData.A[0] } - isCdnIP, _, _, err := r.hp.CdnCheck(hostIP) + isCdnIP, _, _, err := r.hp.CdnCheck(ip) if err != nil { return false } @@ -3099,14 +3099,8 @@ func (r *Runner) parseURL(url string) (*urlutil.URL, error) { return urlx, err } -// getDNSData returns the dns data of hostname, from the cache of the dialer -// only if cachedOnly is set. -func getDNSData(hp *httpx.HTTPX, hostname string, cachedOnly bool) (ips, cnames, resolvers []string, err error) { - lookup := hp.Dialer.GetDNSData - if cachedOnly { - lookup = hp.Dialer.GetDNSDataFromCache - } - dnsData, err := lookup(hostname) +func getDNSData(hp *httpx.HTTPX, hostname string) (ips, cnames, resolvers []string, err error) { + dnsData, err := hp.Dialer.GetDNSData(hostname) if err != nil { return nil, nil, nil, err } diff --git a/runner/targetline.go b/runner/targetline.go index ed9936163..151e87d0e 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -22,20 +22,21 @@ import ( // {"target": "https://app.example", "ip": "192.0.2.10"} // // The target keeps its hostname for the Host header and SNI, while the -// connection goes to the ip. +// connection goes to the ip. A line without ip is the plain target. type targetLine struct { Target string `json:"target"` IP string `json:"ip,omitempty"` } -// isTargetLine reports whether an input line is a target line. +// isTargetLine reports whether an input line is a JSON object, the form of a +// target line. func isTargetLine(line string) bool { return strings.HasPrefix(strings.TrimSpace(line), "{") } -// parseTargetLine decodes a target line. Unknown keys and an ip that is null -// or empty are rejected so that a typo cannot silently probe the host through -// DNS. +// parseTargetLine decodes and validates a target line. Unknown keys and an ip +// that is null or empty are rejected, so that a typo cannot silently probe the +// host through DNS. func parseTargetLine(line string) (targetLine, error) { // the ip is kept raw to tell an ip that is null or empty from no ip var decoded struct { @@ -45,49 +46,58 @@ func parseTargetLine(line string) (targetLine, error) { decoder := json.NewDecoder(strings.NewReader(line)) decoder.DisallowUnknownFields() if err := decoder.Decode(&decoded); err != nil { - return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + return targetLine{}, invalidTargetLine(line, err) } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - return targetLine{}, fmt.Errorf("invalid target line %q: unexpected data after the object", line) + return targetLine{}, invalidTargetLine(line, errors.New("unexpected data after the object")) } - parsed := targetLine{Target: strings.TrimSpace(decoded.Target)} - if parsed.Target == "" { - return targetLine{}, fmt.Errorf("target line %q has no target", line) + target := strings.TrimSpace(decoded.Target) + if target == "" { + return targetLine{}, invalidTargetLine(line, errors.New("no target")) } if decoded.IP == nil { - return parsed, nil + return targetLine{Target: target}, nil } - var value string - if err := json.Unmarshal(decoded.IP, &value); err != nil { - return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + var ip string + if err := json.Unmarshal(decoded.IP, &ip); err != nil { + return targetLine{}, invalidTargetLine(line, err) } - ip, err := pinnedIP(parsed.Target, value) - if err != nil { - return targetLine{}, fmt.Errorf("target %q: %w", parsed.Target, err) + if err := checkPinnable(target); err != nil { + return targetLine{}, invalidTargetLine(line, err) } - parsed.IP = ip - return parsed, nil -} - -// pinnedIP validates the ip a target is pinned to and returns it in canonical -// form. Only a target with a hostname can be pinned. -func pinnedIP(target, value string) (string, error) { - addr, err := netip.ParseAddr(strings.TrimSpace(value)) + addr, err := netip.ParseAddr(strings.TrimSpace(ip)) if err != nil || addr.Zone() != "" || addr.IsUnspecified() { - return "", fmt.Errorf("invalid ip %q", value) + return targetLine{}, invalidTargetLine(line, fmt.Errorf("invalid ip %q", ip)) } - if iputil.IsCIDR(target) || asn.IsASN(target) { - return "", errors.New("a CIDR or ASN target cannot have an ip") - } - if stringsutil.HasPrefixAny(target, "*", ".") || (!stringsutil.HasPrefixAny(target, "http://", "https://") && strings.Contains(target, ",")) { - return "", errors.New("a wildcard or vhost,ip target cannot have an ip") + return targetLine{Target: target, IP: addr.Unmap().String()}, nil +} + +func invalidTargetLine(line string, reason error) error { + return fmt.Errorf("invalid target line %q: %w", line, reason) +} + +// checkPinnable returns why target cannot be pinned to an ip: only a single +// target with a hostname can. +func checkPinnable(target string) error { + switch { + case iputil.IsCIDR(target), asn.IsASN(target): + return errors.New("a CIDR or ASN target cannot be pinned to an ip") + case stringsutil.HasPrefixAny(target, "*", "."): + return errors.New("a wildcard target cannot be pinned to an ip") + case isVHostInput(target): + return errors.New("a vhost,ip target cannot be pinned to an ip") } parsed, err := urlutil.Parse(target) if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) { - return "", errors.New("a target with an ip must have a hostname") + return errors.New("a target pinned to an ip must have a hostname") } - return addr.Unmap().String(), nil + return nil +} + +// isVHostInput reports whether target is a vhost,ip input. +func isVHostInput(target string) bool { + return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") } // key returns the line as it is stored and deduplicated: the target alone @@ -131,21 +141,31 @@ func (r *Runner) normalizeInput(item string) (string, error) { if !isTargetLine(item) { return item, nil } - line, err := parseTargetLine(item) + line, err := r.parseInputLine(item) if err != nil { return "", err } + return line.key(), nil +} + +// parseInputLine parses a target line, and refuses one pinned to an ip with +// the options that do not keep to it. +func (r *Runner) parseInputLine(item string) (targetLine, error) { + line, err := parseTargetLine(item) + if err != nil { + return targetLine{}, err + } if line.IP != "" { if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 { - return "", fmt.Errorf("target %q: a target with an ip cannot be probed with %s", line.Target, strings.Join(flags, ", ")) + return targetLine{}, invalidTargetLine(item, fmt.Errorf("a target pinned to an ip cannot be probed with %s", strings.Join(flags, ", "))) } } - return line.key(), nil + return line, nil } -// pinnedTargetConflicts returns the flags whose probes do not connect through -// the ip a target is pinned to, or share a connection between the ips of a -// host, as HTTP/2 does. +// pinnedTargetConflicts returns the options whose probes do not connect +// through the ip a target is pinned to, or share a connection between the ips +// of a host, as HTTP/2 does. func (options *Options) pinnedTargetConflicts() []string { var flags []string if options.Unsafe { @@ -169,7 +189,7 @@ func (options *Options) pinnedTargetConflicts() []string { if options.VHostInput { flags = append(flags, "-vhost-input") } - if options.Proxy != "" || options.HTTPProxy != "" || options.SocksProxy != "" { + if options.Proxy != "" { flags = append(flags, "-proxy") } return flags diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 2714eeda5..91e839277 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -3,11 +3,10 @@ package runner import ( "crypto/tls" "fmt" - "math" "net" "net/http" "net/http/httptest" - "net/url" + "net/netip" "os" "path/filepath" "sort" @@ -15,12 +14,22 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/projectdiscovery/httpx/common/httpx" "github.com/stretchr/testify/require" "golang.org/x/net/dns/dnsmessage" ) +// TestMain keeps pinned.test out of a proxy set in the environment, which +// would resolve the host itself, so that the probes below run behind one too. +func TestMain(m *testing.M) { + for _, name := range []string{"NO_PROXY", "no_proxy"} { + _ = os.Setenv(name, strings.TrimPrefix(os.Getenv(name)+",pinned.test", ",")) + } + os.Exit(m.Run()) +} + func TestParseTargetLine(t *testing.T) { valid := map[string]targetLine{ `{"target": "https://a.example", "ip": "192.0.2.10"}`: {Target: "https://a.example", IP: "192.0.2.10"}, @@ -36,69 +45,102 @@ func TestParseTargetLine(t *testing.T) { }) } - invalid := map[string]string{ - "malformed json": `{"target": "a.example"`, - "trailing data": `{"target": "a.example"} x`, - "unknown key": `{"target": "a.example", "ips": "192.0.2.10"}`, - "missing target": `{"ip": "192.0.2.10"}`, - "invalid ip": `{"target": "a.example", "ip": "192.0.2"}`, - "null ip": `{"target": "a.example", "ip": null}`, - "empty ip": `{"target": "a.example", "ip": ""}`, - "number ip": `{"target": "a.example", "ip": 1}`, - "ip with zone": `{"target": "a.example", "ip": "fe80::1%eth0"}`, - "unspecified ip": `{"target": "a.example", "ip": "0.0.0.0"}`, - "ip target": `{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, - "cidr target": `{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, - "asn target": `{"target": "AS13335", "ip": "192.0.2.10"}`, - "wildcard": `{"target": "*.a.example", "ip": "192.0.2.10"}`, - "vhost form": `{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, + invalid := map[string]struct{ line, reason string }{ + "malformed json": {`{"target": "a.example"`, "unexpected EOF"}, + "trailing data": {`{"target": "a.example"} x`, "unexpected data after the object"}, + "unknown key": {`{"target": "a.example", "ips": "192.0.2.10"}`, `unknown field "ips"`}, + "missing target": {`{"ip": "192.0.2.10"}`, "no target"}, + "invalid ip": {`{"target": "a.example", "ip": "192.0.2"}`, `invalid ip "192.0.2"`}, + "null ip": {`{"target": "a.example", "ip": null}`, `invalid ip ""`}, + "empty ip": {`{"target": "a.example", "ip": ""}`, `invalid ip ""`}, + "number ip": {`{"target": "a.example", "ip": 1}`, "cannot unmarshal number"}, + "ip with zone": {`{"target": "a.example", "ip": "fe80::1%eth0"}`, "invalid ip"}, + "unspecified ip": {`{"target": "a.example", "ip": "0.0.0.0"}`, "invalid ip"}, + "ip target": {`{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, "must have a hostname"}, + "cidr target": {`{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "asn target": {`{"target": "AS13335", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "wildcard": {`{"target": "*.a.example", "ip": "192.0.2.10"}`, "wildcard"}, + "vhost form": {`{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, "vhost,ip"}, } - for name, line := range invalid { + for name, tc := range invalid { t.Run(name, func(t *testing.T) { - _, err := parseTargetLine(line) - require.Error(t, err) + _, err := parseTargetLine(tc.line) + require.ErrorContains(t, err, "invalid target line") + require.ErrorContains(t, err, tc.reason) }) } } func TestRunner_normalizeInput(t *testing.T) { r := &Runner{options: &Options{}} - for _, line := range []string{ - `{"target": "https://a.example", "ip": "192.0.2.10"}`, - ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, - } { - key, err := r.normalizeInput(line) - require.NoError(t, err) - require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, key, "equivalent lines are stored once") - } - - key, err := r.normalizeInput(`{"target": "https://a.example"}`) - require.NoError(t, err) - require.Equal(t, "https://a.example", key, "a line without ip is the plain target") - _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2"}`) - require.Error(t, err) + t.Run("equivalent lines are stored once", func(t *testing.T) { + for _, line := range []string{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`, + ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, + } { + key, err := r.normalizeInput(line) + require.NoError(t, err) + require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, key) + } + }) + t.Run("a line without ip is the plain target", func(t *testing.T) { + key, err := r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err) + require.Equal(t, "https://a.example", key) + }) + t.Run("an invalid line", func(t *testing.T) { + _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2"}`) + require.ErrorContains(t, err, "invalid ip") + }) +} - r = &Runner{options: &Options{Unsafe: true, Jarm: true, Protocol: "HTTP2"}} - _, err = r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) - require.ErrorContains(t, err, "-unsafe, -pr http2, -jarm") - _, err = r.normalizeInput(`{"target": "https://a.example"}`) - require.NoError(t, err, "only pinned targets conflict") +func TestRunner_pinnedTargetConflicts(t *testing.T) { + conflicts := map[string]Options{ + "-unsafe": {Unsafe: true}, + "-pipeline": {Pipeline: true}, + "-http2": {HTTP2Probe: true}, + "-pr http2": {Protocol: "HTTP2"}, + "-jarm": {Jarm: true}, + "-screenshot": {Screenshot: true}, + "-vhost-input": {VHostInput: true}, + "-proxy": {Proxy: "http://127.0.0.1:8080"}, + } + for flag, options := range conflicts { + t.Run(flag, func(t *testing.T) { + r := &Runner{options: &options} + _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorContains(t, err, "cannot be probed with "+flag) + _, err = r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only pinned targets conflict") + }) + } } func TestRunner_targetLine_targets(t *testing.T) { - // -pa must not resolve a pinned target - r, err := New(&Options{ProbeAllIPS: true}) - require.NoError(t, err) - defer r.Close() - - key, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) - require.NoError(t, err) - var got []httpx.Target - for target := range r.targets(r.hp, key) { - got = append(got, target) + line := `{"target":"https://a.example","ip":"192.0.2.10"}` + targetsOf := func(r *Runner) []httpx.Target { + var got []httpx.Target + for target := range r.targets(r.hp, line) { + got = append(got, target) + } + return got } - require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, got) + + t.Run("-pa does not resolve a pinned target", func(t *testing.T) { + r, err := New(&Options{ProbeAllIPS: true}) + require.NoError(t, err) + defer r.Close() + require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, targetsOf(r)) + }) + // a line from another source than the input, such as a certificate name + // with -tls-probe, is refused with an option that does not keep to the ip + t.Run("conflicts are refused for every target line", func(t *testing.T) { + r, err := New(&Options{Unsafe: true}) + require.NoError(t, err) + defer r.Close() + require.Empty(t, targetsOf(r)) + }) } func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { @@ -121,10 +163,43 @@ func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { require.Equal(t, []string{"a.example", `{"target":"c.example","ip":"192.0.2.10"}`}, got, "the malformed line is skipped") } +func TestRunner_loadAndCloseFile_refusesInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + require.NoError(t, os.WriteFile(file, []byte("a.example\n"+`{"target": "b.example", "ip": null}`+"\n"), 0o600)) + input, err := os.Open(file) + require.NoError(t, err) + + r, err := New(&Options{}) + require.NoError(t, err) + defer r.Close() + _, err = r.loadAndCloseFile(input) + require.ErrorContains(t, err, "invalid ip") +} + +func TestRunner_skip_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn", "192.0.2.10"}}) + require.NoError(t, err) + defer r.Close() + URL, err := r.parseURL("http://pinned.test:8443") + require.NoError(t, err) + + // pinned.test does not resolve: only the pinned ip can be checked + skip, result := r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "151.101.1.1"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "cdn target") + + skip, result = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.10"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "not allowed by network policy") + + skip, _ = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.11"}, "") + require.False(t, skip) +} + // A proxy, set with -proxy for -pa or in the environment, resolves the host // itself, so a probe of a pinned target through it fails. func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { - r, err := New(&Options{HTTPProxy: "http://127.0.0.1:1"}) + r, err := New(&Options{Proxy: "http://127.0.0.1:1"}) require.NoError(t, err) defer r.Close() @@ -133,105 +208,280 @@ func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { require.ErrorContains(t, result.Err, "through a proxy") } -func TestRunner_skipCDNPort_pinned(t *testing.T) { - r, err := New(&Options{Exclude: []string{"cdn"}}) +// -maxhr counts the failures of each ip of a host apart. +func TestRunner_analyze_hostErrorsPerIP(t *testing.T) { + port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) + r, err := New(&Options{HostMaxErrors: 1, Timeout: 2}) require.NoError(t, err) defer r.Close() - // pinned.test does not resolve: only the pinned ip can be classified - require.True(t, r.skipCDNPort("pinned.test", "151.101.1.1", "8443"), "a CDN ip on another port") - require.False(t, r.skipCDNPort("pinned.test", "151.101.1.1", "443"), "a CDN ip on 443") - require.False(t, r.skipCDNPort("pinned.test", "192.0.2.10", "8443"), "an origin ip") + analyze := func(ip string) Result { + target := httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} + return r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) + } + require.Error(t, analyze("127.0.0.3").Err, "nothing listens there") + require.NoError(t, analyze("127.0.0.1").Err, "the other ip of the host is not skipped") } -// skipIfProxied skips a test whose requests to target go through a proxy set -// in the environment, as the proxy resolves the host itself. -func skipIfProxied(t *testing.T, target string) { - t.Helper() - u, err := url.Parse(target) - require.NoError(t, err) - if proxy, _ := http.ProxyFromEnvironment(&http.Request{URL: u}); proxy != nil { - t.Skipf("%s goes through the proxy %s", target, proxy.Redacted()) +func TestTargetLinePinsTheConnection(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + results := probe(t, &Options{}, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + var got []string + for _, result := range results { + got = append(got, result.HostIP+" "+result.Title) } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) } -// runPinned probes the target lines with a runner and returns the results. -func runPinned(t *testing.T, lines ...string) []Result { - t.Helper() - return runPinnedWith(t, &Options{}, lines...) +func TestTargetLineDeduplicatesEquivalentLines(t *testing.T) { + port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) + target := "http://pinned.test:" + port + results := probe(t, &Options{}, + fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target), + fmt.Sprintf(` {"ip": "::ffff:127.0.0.1", "target": %q}`, target), + ) + require.Len(t, results, 1) } -// runPinnedWith is runPinned with options holding the flags under test. -func runPinnedWith(t *testing.T, options *Options, lines ...string) []Result { - t.Helper() +// Only the pinned host goes to the ip: a redirect to another host does not. +func TestTargetLineKeepsOtherHostsUnpinned(t *testing.T) { + redirect := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, port, _ := net.SplitHostPort(r.Host) + http.Redirect(w, r, "http://127.0.0.1:"+port+"/", http.StatusFound) + }) + port := servePair(t, titleHandler("one"), redirect) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + require.Len(t, results, 1) + require.Equal(t, "one 127.0.0.1:"+port, results[0].Title) +} + +// With a matcher, responses are stored after matching, under a name that +// must differ for each ip of a url. +func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + options := &Options{StoreResponse: true, StoreResponseDir: t.TempDir(), OutputMatchStatusCode: "200"} + results := probe(t, options, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + require.Len(t, results, 2) + for _, result := range results { + data, err := os.ReadFile(result.StoredResponsePath) + require.NoError(t, err) + require.Contains(t, string(data), ""+result.Title+"", "the response of %s", result.HostIP) + } +} + +// Like curl --resolve, the host of a pinned target is not resolved, and a +// holds dns data only. +func TestTargetLineIsNotResolved(t *testing.T) { + port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) + resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}}, pinnedLines(port, "127.0.0.1")...) + + require.Len(t, results, 1) + require.Equal(t, "one pinned.test:"+port, results[0].Title) + require.Empty(t, results[0].A) + require.Zero(t, queries.Load()) +} + +// -pa resolves the host, and each of its results keeps all of its addresses. +func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1", "127.0.0.2") + results := probe(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: true}, "http://pinned.test:"+port) + + require.Len(t, results, 2) + for _, result := range results { + require.ElementsMatch(t, []string{"127.0.0.1", "127.0.0.2"}, result.A, "the result of %s", result.HostIP) + } +} + +// A plain target reports the ip it was dialed to, not the one a pinned target +// of the same host was dialed to meanwhile. +func TestPlainTargetReportsItsOwnIP(t *testing.T) { + slow := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(500 * time.Millisecond) + titleHandler("two").ServeHTTP(w, r) + }) + port := servePair(t, titleHandler("one"), slow) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.2") + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "http://pinned.test:" + port + "\n" + pinnedLines(port, "127.0.0.1")[0] + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + + options := &Options{Resolvers: []string{resolver}, Stream: true, InputFile: file, Threads: 2, Delay: 100 * time.Millisecond} + results := probe(t, options) + require.Len(t, results, 2) + for _, result := range results { + ip := "127.0.0.1" + if strings.HasPrefix(result.Title, "two") { + ip = "127.0.0.2" + } + require.Equal(t, ip, result.HostIP, result.Title) + } +} + +// An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. +func TestTargetLinePinsIPv6(t *testing.T) { + port := portOf(t, serve(t, listen(t, "[::1]:0"), titleHandler("six"), nil).Listener) + results := probe(t, &Options{}, pinnedLines(port, "::1")...) + + require.Len(t, results, 1) + require.Equal(t, "six pinned.test:"+port, results[0].Title) +} + +func TestTargetLineKeepsSNI(t *testing.T) { var ( - mu sync.Mutex - results []Result + mu sync.Mutex + names []string ) - options.Threads = 1 - options.RateLimit = 10 - options.Timeout = 5 - options.Methods = http.MethodGet - options.Delay = -1 - options.OnResult = func(r Result) { - if r.Err != nil || r.URL == "" { - return - } - mu.Lock() - results = append(results, r) - mu.Unlock() + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + mu.Lock() + names = append(names, hello.ServerName) + mu.Unlock() + return nil, nil + }, + }) + target := "https://pinned.test:" + portOf(t, server.Listener) + results := probe(t, &Options{}, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) + + require.Len(t, results, 1) + mu.Lock() + defer mu.Unlock() + require.NotEmpty(t, names) + for _, name := range names { + require.Equal(t, "pinned.test", name) } - options.InputTargetHost = lines - r, err := New(options) +} + +// The favicon request and the ztls handshake of -tls-grab connect through the +// pinned ip as well. +func TestTargetLinePinsTheFaviconAndTLSGrab(t *testing.T) { + server := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("tls")), &tls.Config{}) + target := "https://pinned.test:" + portOf(t, server.Listener) + options := &Options{Favicon: true, ZTLS: true, TLSGrab: true} + results := probe(t, options, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) + + require.Len(t, results, 1) + require.NotEmpty(t, results[0].FavIconMMH3) + require.NotNil(t, results[0].TLSData) +} + +// The favicon request does not take the trace of the probe, whose -trace +// result would then describe the favicon request. +func TestRunner_HandleFaviconHash_keepsTheTraceOfTheProbe(t *testing.T) { + favicon := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("favicon")), nil) + page := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, ``, favicon.URL) + }), nil) + r, err := New(&Options{Timeout: 5, Methods: http.MethodGet, Trace: true, Favicon: true, MaxResponseBodySizeToRead: int(httpx.DefaultMaxResponseBodySize)}) require.NoError(t, err) defer r.Close() - r.RunEnumeration() - mu.Lock() - defer mu.Unlock() - return results + result := r.analyze(r.hp, httpx.HTTP, httpx.Target{Host: page.URL}, http.MethodGet, page.URL, &r.scanopts) + require.NoError(t, result.Err) + require.NotEmpty(t, result.FavIconMMH3) + require.Equal(t, page.Listener.Addr().String(), fmt.Sprint(result.Trace.GetConn.Info)) +} + +// titleHandler serves a page whose title is name and the Host header. +func titleHandler(name string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) + }) } -// serveTitle serves a page whose title is name and the Host header, and skips -// the test when addr cannot be bound. -func serveTitle(t *testing.T, addr, name string) *httptest.Server { +// withFavicon serves a PNG image as /favicon.ico, and the rest with handler. +func withFavicon(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/favicon.ico" { + _, _ = w.Write([]byte("\x89PNG\r\n\x1a\n")) + return + } + handler.ServeHTTP(w, r) + }) +} + +// listen listens on addr, and skips the test where addr cannot be bound. +func listen(t *testing.T, addr string) net.Listener { t.Helper() listener, err := net.Listen("tcp", addr) if err != nil { t.Skipf("cannot listen on %s: %v", addr, err) } - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprintf(w, "%s %s", name, r.Host) - })) - _ = ts.Listener.Close() - ts.Listener = listener - ts.Start() - t.Cleanup(ts.Close) - return ts + return listener +} + +// serve serves handler on listener, over TLS if tlsConfig is set. +func serve(t *testing.T, listener net.Listener, handler http.Handler, tlsConfig *tls.Config) *httptest.Server { + t.Helper() + server := httptest.NewUnstartedServer(handler) + _ = server.Listener.Close() + server.Listener = listener + if tlsConfig != nil { + server.TLS = tlsConfig + server.StartTLS() + } else { + server.Start() + } + t.Cleanup(server.Close) + return server } -// servePair serves "one" on 127.0.0.1 and "two" on 127.0.0.2 with the same -// port, and returns the port and the lines pinning pinned.test to each ip. -func servePair(t *testing.T) (string, []string) { +func portOf(t *testing.T, listener net.Listener) string { t.Helper() - one := serveTitle(t, "127.0.0.1:0", "one") - _, port, err := net.SplitHostPort(one.Listener.Addr().String()) + _, port, err := net.SplitHostPort(listener.Addr().String()) require.NoError(t, err) - serveTitle(t, net.JoinHostPort("127.0.0.2", port), "two") + return port +} - target := "http://pinned.test:" + port - skipIfProxied(t, target) - return port, []string{ - fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target), - fmt.Sprintf(`{"target": %q, "ip": "127.0.0.2"}`, target), +// servePair serves one on 127.0.0.1 and two on 127.0.0.2 with the same port, +// and returns the port. It skips the test where 127.0.0.2 cannot be bound, as +// on macOS. +func servePair(t *testing.T, one, two http.Handler) string { + t.Helper() + var err error + for range 5 { + first := listen(t, "127.0.0.1:0") + port := portOf(t, first) + var second net.Listener + if second, err = net.Listen("tcp", net.JoinHostPort("127.0.0.2", port)); err == nil { + serve(t, first, one, nil) + serve(t, second, two, nil) + return port + } + // the port may be taken on 127.0.0.2: try another one + _ = first.Close() } + t.Skipf("cannot listen on 127.0.0.2: %v", err) + return "" +} + +// pinnedLines returns the lines pinning http://pinned.test:port to each ip. +func pinnedLines(port string, ips ...string) []string { + var lines []string + for _, ip := range ips { + lines = append(lines, fmt.Sprintf(`{"target": "http://pinned.test:%s", "ip": %q}`, port, ip)) + } + return lines } // serveDNS runs a resolver that answers the A queries for host with ips, and // returns its address and the number of queries for host it got. func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) { t.Helper() + var records []dnsmessage.AResource + for _, ip := range ips { + addr := netip.MustParseAddr(ip) + require.True(t, addr.Is4(), ip) + records = append(records, dnsmessage.AResource{A: addr.As4()}) + } conn, err := net.ListenPacket("udp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { _ = conn.Close() }) @@ -260,10 +510,10 @@ func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) _ = builder.StartAnswers() if strings.EqualFold(strings.TrimSuffix(question.Name.String(), "."), host) { queries.Add(1) - for _, ip := range ips { - if question.Type == dnsmessage.TypeA { - answer := dnsmessage.ResourceHeader{Name: question.Name, Class: dnsmessage.ClassINET, TTL: 60} - _ = builder.AResource(answer, dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())}) + if question.Type == dnsmessage.TypeA { + answer := dnsmessage.ResourceHeader{Name: question.Name, Class: dnsmessage.ClassINET, TTL: 60} + for _, record := range records { + _ = builder.AResource(answer, record) } } } @@ -275,112 +525,43 @@ func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) return conn.LocalAddr().String(), &queries } -func TestTargetLinePinsTheConnection(t *testing.T) { - port, lines := servePair(t) - results := runPinned(t, lines...) - - var got []string - for _, result := range results { - got = append(got, result.HostIP+" "+result.Title) +// probe runs a runner on inputs with options, completed with test defaults, +// and returns its results. The results that failed are logged. +func probe(t *testing.T, options *Options, inputs ...string) []Result { + t.Helper() + if options.Threads == 0 { + options.Threads = 1 } - sort.Strings(got) - require.Equal(t, []string{ - "127.0.0.1 one pinned.test:" + port, - "127.0.0.2 two pinned.test:" + port, - }, got) -} - -// With a matcher, responses are stored after matching, under a name that -// must differ for each ip of a url. -func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { - _, lines := servePair(t) - options := &Options{ - StoreResponse: true, - StoreResponseDir: t.TempDir(), - MaxResponseBodySizeToSave: math.MaxInt32, - OutputMatchStatusCode: "200", + if options.Timeout == 0 { + options.Timeout = 5 } - require.NoError(t, options.ValidateOptions()) - results := runPinnedWith(t, options, lines...) - - require.Len(t, results, 2) - for _, result := range results { - data, err := os.ReadFile(result.StoredResponsePath) - require.NoError(t, err) - require.Contains(t, string(data), ""+result.Title+"", "the response of %s", result.HostIP) - } -} - -// Like curl --resolve, the host of a pinned target is not resolved: the -// pinned ip is reported in host_ip, and a holds dns data only. -func TestTargetLineIsNotResolved(t *testing.T) { - _, lines := servePair(t) - resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") - results := runPinnedWith(t, &Options{Resolvers: []string{resolver}}, lines[0]) - - require.Len(t, results, 1) - require.Equal(t, "127.0.0.1", results[0].HostIP) - require.Empty(t, results[0].A) - require.Zero(t, queries.Load()) -} - -// -pa resolves the host, and each of its results keeps all of its addresses. -func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { - port, _ := servePair(t) - resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1", "127.0.0.2") - results := runPinnedWith(t, &Options{Resolvers: []string{resolver}, ProbeAllIPS: true}, "http://pinned.test:"+port) - - require.Len(t, results, 2) - for _, result := range results { - require.ElementsMatch(t, []string{"127.0.0.1", "127.0.0.2"}, result.A, "the result of %s", result.HostIP) + if options.MaxResponseBodySizeToRead == 0 { + options.MaxResponseBodySizeToRead = int(httpx.DefaultMaxResponseBodySize) + options.MaxResponseBodySizeToSave = int(httpx.DefaultMaxResponseBodySize) } -} - -// An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. -func TestTargetLinePinsIPv6(t *testing.T) { - six := serveTitle(t, "[::1]:0", "six") - _, port, err := net.SplitHostPort(six.Listener.Addr().String()) - require.NoError(t, err) - - target := "http://pinned.test:" + port - skipIfProxied(t, target) - results := runPinned(t, fmt.Sprintf(`{"target": %q, "ip": "::1"}`, target)) - require.Len(t, results, 1) - require.Equal(t, "::1", results[0].HostIP) - require.Equal(t, "six pinned.test:"+port, results[0].Title) -} + options.Methods = http.MethodGet + options.InputTargetHost = inputs + require.NoError(t, options.ValidateOptions()) -func TestTargetLineKeepsSNI(t *testing.T) { var ( - mu sync.Mutex - names []string + mu sync.Mutex + results []Result ) - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - _, _ = fmt.Fprint(w, "tls") - })) - ts.TLS = &tls.Config{ - GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { - mu.Lock() - names = append(names, hello.ServerName) - mu.Unlock() - return nil, nil - }, + options.OnResult = func(result Result) { + if result.Err != nil || result.URL == "" { + t.Logf("%s: %v", result.Input, result.Err) + return + } + mu.Lock() + results = append(results, result) + mu.Unlock() } - ts.StartTLS() - defer ts.Close() - _, port, err := net.SplitHostPort(ts.Listener.Addr().String()) + r, err := New(options) require.NoError(t, err) - - target := "https://pinned.test:" + port - skipIfProxied(t, target) - results := runPinned(t, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) - require.Len(t, results, 1) - require.Equal(t, "127.0.0.1", results[0].HostIP) + defer r.Close() + r.RunEnumeration() mu.Lock() defer mu.Unlock() - require.NotEmpty(t, names) - for _, name := range names { - require.Equal(t, "pinned.test", name) - } + return results } From 29454e4d4a7f2ad4fe7245a489b4e7bda257e8d2 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Wed, 28 Oct 2026 00:00:37 +0000 Subject: [PATCH 14/16] fix(runner): address the second review of pinned targets A request that goes through a proxy records it, so its host_ip stays empty instead of taking the last ip the dialer dialed to its host, which could be a pinned target's. A failed probe of a pinned target shows its ip. Option conflicts are named after target lines, the only inputs they apply to, and are no longer reported as invalid lines. A skipped line is a warning in every mode. The README describes what changes for -pa and what -tls-probe does with a pinned host. The tests cover a proxy set in the environment end to end, host spellings on the request side, the scheme of a line, the ip in the output and the skip of a failing ip. The TestMain of the package moves to main_test.go. --- README.md | 7 +- common/httpx/dialedip.go | 40 +++++++--- common/httpx/httpx.go | 2 +- common/httpx/pinned.go | 3 +- common/httpx/pinned_test.go | 59 +++++++++++--- runner/main_test.go | 23 ++++++ runner/runner.go | 39 +++++++--- runner/targetline.go | 99 +++++++++++------------- runner/targetline_test.go | 149 ++++++++++++++++++++++-------------- 9 files changed, 278 insertions(+), 143 deletions(-) create mode 100644 runner/main_test.go diff --git a/README.md b/README.md index 99ed453ed..522bbb9cb 100644 --- a/README.md +++ b/README.md @@ -380,12 +380,15 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` -- A target can be pinned to an IP with a JSON input line, like `curl --resolve`: the connection goes to the IP, which `host_ip` reports, and the hostname is kept for the Host header and SNI without being resolved. A host can be listed with several IPs, and a line without `ip` is the plain target: +- A target can be pinned to an IP with a JSON input line, like `curl --resolve`: ``` {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line stops the run, or is skipped with an error with `-stream`. Pinned lines are refused with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, which do not keep to the IP. A proxy resolves the host itself, so a probe of a pinned target, `-pa` ones included, fails if it would go through one, such as one set with `HTTP_PROXY`/`HTTPS_PROXY`. + - The connection goes to the IP, which `host_ip` reports and the output shows, and the hostname is kept for the Host header and SNI without being resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. + - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line stops the run, or is skipped with a warning with `-stream`. + - A line with an IP is refused with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well, so a probe that would go through one fails, `-pa` ones included. + - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. The names `-tls-probe` and `-csp-probe` find, the pinned host included, are resolved as new targets. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/dialedip.go b/common/httpx/dialedip.go index ffdad541f..5cc92a871 100644 --- a/common/httpx/dialedip.go +++ b/common/httpx/dialedip.go @@ -3,17 +3,24 @@ package httpx import ( "context" "net" + "net/http" + "net/url" "sync" ) +// dialFunc dials addr, as net.Dialer.DialContext does. +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + // dialedIPKey is the context key of the dialedIP of WithDialedIP. type dialedIPKey struct{} -// dialedIP is the ip of the last connection dialed to a host. +// dialedIP is the ip of the last connection dialed to a host, and whether a +// request went through a proxy instead. type dialedIP struct { - host string - mu sync.Mutex - ip string + host string + mu sync.Mutex + ip string + proxied bool } // WithDialedIP returns a copy of ctx that records the ip of the connections @@ -24,21 +31,22 @@ func WithDialedIP(ctx context.Context, host string) context.Context { } // DialedIP returns the ip of the last connection dialed to the host of a -// WithDialedIP context, or "" if none was. -func DialedIP(ctx context.Context) string { +// WithDialedIP context, and whether a request with it went through a proxy, +// which dials the host itself. +func DialedIP(ctx context.Context) (ip string, proxied bool) { dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) if !ok { - return "" + return "", false } dialed.mu.Lock() defer dialed.mu.Unlock() - return dialed.ip + return dialed.ip, dialed.proxied } // recordDialedIP wraps dial to record the ip of a connection to the host of a // WithDialedIP context. A connection to another host, such as a proxy, is not // recorded. -func recordDialedIP(dial func(ctx context.Context, network, addr string) (net.Conn, error)) func(ctx context.Context, network, addr string) (net.Conn, error) { +func recordDialedIP(dial dialFunc) dialFunc { return func(ctx context.Context, network, addr string) (net.Conn, error) { conn, err := dial(ctx, network, addr) dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) @@ -55,3 +63,17 @@ func recordDialedIP(dial func(ctx context.Context, network, addr string) (net.Co return conn, err } } + +// recordProxied wraps the proxy of a transport to record that a request with +// a WithDialedIP context goes through a proxy. +func recordProxied(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if dialed, ok := req.Context().Value(dialedIPKey{}).(*dialedIP); ok && proxyURL != nil { + dialed.mu.Lock() + dialed.proxied = true + dialed.mu.Unlock() + } + return proxyURL, err + } +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 4e6e8b678..233d0fdf0 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -176,7 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } - transport.Proxy = refusePinned(transport.Proxy) + transport.Proxy = recordProxied(refusePinned(transport.Proxy)) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go index 7f0152c1b..01d685457 100644 --- a/common/httpx/pinned.go +++ b/common/httpx/pinned.go @@ -44,7 +44,8 @@ func pinnedIP(req *http.Request) string { return ip } -// canonicalHost returns host as net/http dials it, so that a pin matches the +// canonicalHost folds the case, the IDNA form and a trailing dot of host, as +// fastdialer keys its resolve overrides, so that a pin matches the requests and // dials to its host however the host is spelled. func canonicalHost(host string) string { if ascii, err := idna.Lookup.ToASCII(host); err == nil { diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go index e5c3d7139..ef936dc4b 100644 --- a/common/httpx/pinned_test.go +++ b/common/httpx/pinned_test.go @@ -6,6 +6,8 @@ import ( "net/http" "net/http/httptrace" "net/url" + "os" + "os/exec" "testing" "github.com/stretchr/testify/require" @@ -15,13 +17,14 @@ func TestRefusePinned(t *testing.T) { proxyURL, err := url.Parse("http://127.0.0.1:3128") require.NoError(t, err) proxy := refusePinned(http.ProxyURL(proxyURL)) - ctx := WithPinnedIP(WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10"), "second.test", "192.0.2.11") + ctx := WithPinnedIP(WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10"), "xn--bcher-kva.test", "192.0.2.11") - for _, rawURL := range []string{"https://pinned.test:8443/", "https://second.test/"} { + // each host pinned in a context, however it is spelled + for _, rawURL := range []string{"https://PINNED.test.:8443/", "https://bücher.test/"} { req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) require.NoError(t, err) _, err = proxy(req) - require.ErrorIs(t, err, errPinnedThroughProxy, "each host pinned in a context") + require.ErrorIs(t, err, errPinnedThroughProxy, rawURL) } redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) @@ -37,7 +40,28 @@ func TestRefusePinned(t *testing.T) { require.Nil(t, got, "a pinned request without a proxy") } -// A pin matches the host as net/http dials it, however the host is spelled. +// A proxy set in the environment is refused for a pinned request as well. Go +// reads the proxies of the environment once, so the test runs in a new process. +func TestRefusePinned_environment(t *testing.T) { + if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" { + cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$") + // the lower case names first: on Windows, the last of two spellings wins + cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + return + } + + options := DefaultOptions + options.RetryMax = 0 + h, err := New(&options) + require.NoError(t, err) + req, err := h.NewRequestWithContext(WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/") + require.NoError(t, err) + _, err = h.Do(req, UnsafeOptions{}) + require.ErrorContains(t, err, errPinnedThroughProxy.Error()) +} + func TestCanonicalHost(t *testing.T) { for host, want := range map[string]string{ "Pinned.Test.": "pinned.test", @@ -72,20 +96,33 @@ func TestRecordDialedIP(t *testing.T) { dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String()) }) - ctx := WithDialedIP(context.Background(), "Dialed.Test") + ctx := WithDialedIP(context.Background(), "dialed.test") conn, err := dial(ctx, "tcp", "proxy.test:3128") require.NoError(t, err) _ = conn.Close() - require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") + ip, _ := DialedIP(ctx) + require.Empty(t, ip, "a dial to another host, such as a proxy") - conn, err = dial(ctx, "tcp", "dialed.test:80") + // net/http dials a host as it is spelled + conn, err = dial(ctx, "tcp", "DIALED.test.:80") require.NoError(t, err) _ = conn.Close() - require.Equal(t, "127.0.0.1", DialedIP(ctx)) + ip, _ = DialedIP(ctx) + require.Equal(t, "127.0.0.1", ip) +} - conn, err = dial(context.Background(), "tcp", "dialed.test:80") +func TestRecordProxied(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") require.NoError(t, err) - _ = conn.Close() - require.Empty(t, DialedIP(context.Background()), "a context without WithDialedIP") + + for proxy, want := range map[*url.URL]bool{proxyURL: true, nil: false} { + ctx := WithDialedIP(context.Background(), "dialed.test") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://dialed.test/", nil) + require.NoError(t, err) + _, err = recordProxied(http.ProxyURL(proxy))(req) + require.NoError(t, err) + _, proxied := DialedIP(ctx) + require.Equal(t, want, proxied) + } } diff --git a/runner/main_test.go b/runner/main_test.go new file mode 100644 index 000000000..3b74a2d2b --- /dev/null +++ b/runner/main_test.go @@ -0,0 +1,23 @@ +package runner + +import ( + "os" + "strings" + "testing" +) + +// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in +// the environment, which would resolve it itself, so that they run behind a +// proxy too. Go reads the proxies of the environment once per process, so +// t.Setenv cannot change them for a test. +func TestMain(m *testing.M) { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + noProxy = os.Getenv("no_proxy") + } + noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",") + for _, name := range []string{"NO_PROXY", "no_proxy"} { + _ = os.Setenv(name, noProxy) + } + os.Exit(m.Run()) +} diff --git a/runner/runner.go b/runner/runner.go index 7c7f007ce..022aa19af 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -743,11 +743,11 @@ func (r *Runner) streamInput() (chan string, error) { } // sendLine normalizes, deduplicates and sends an input line, returning // false if interrupted. The scan has already started, so an invalid - // line is skipped with an error. + // line is skipped with a warning. sendLine := func(item string) bool { item, err := r.normalizeInput(item) if err != nil { - gologger.Error().Msgf("Skipping input: %s\n", err) + gologger.Warning().Msgf("Skipping input: %s\n", err) return true } if r.options.SkipDedupe || r.testAndSet(item) { @@ -1789,6 +1789,11 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT } } +// isVHostInput reports whether target is a vhost,ip input. +func isVHostInput(target string) bool { + return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") +} + // returns all the targets within a cidr range or the single target func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) @@ -1799,6 +1804,8 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { switch { case isTargetLine(target): + // input lines were checked when read, but the names -tls-probe + // and -csp-probe find come here directly line, err := r.parseInputLine(target) if err != nil { gologger.Warning().Msgf("Skipping input: %s\n", err) @@ -1854,6 +1861,16 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { return results } +// pinnedKey returns key for a target that is not pinned, and key@ip for one +// pinned to ip, from a target line or -pa, as the ips of a host fail and +// respond independently. +func pinnedKey(key, ip string) string { + if ip == "" { + return key + } + return key + "@" + ip +} + func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { @@ -2091,6 +2108,9 @@ retry: } if r.options.Probe { + if target.CustomIP != "" { + _, _ = fmt.Fprintf(builder, " [%s]", target.CustomIP) + } return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err} @@ -2299,14 +2319,15 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // the ip this request was dialed to: the dialer only keeps the - // last ip dialed to a host, which may be another request's - ip = httpx.DialedIP(req.Context()) - if ip == "" { + var proxied bool + ip, proxied = httpx.DialedIP(req.Context()) + if ip == "" && !proxied { + // no dial was recorded, as for a reused connection or -unsafe: + // hp.Dialer.GetDialedIP would return only the last dialed one ip = hp.Dialer.GetDialedIP(URL.Host) - } - if ip == "" { - ip = hp.Dialer.GetDialedIP(onlyHost) + if ip == "" { + ip = hp.Dialer.GetDialedIP(onlyHost) + } } } } diff --git a/runner/targetline.go b/runner/targetline.go index 151e87d0e..be89e1a30 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -31,13 +31,22 @@ type targetLine struct { // isTargetLine reports whether an input line is a JSON object, the form of a // target line. func isTargetLine(line string) bool { - return strings.HasPrefix(strings.TrimSpace(line), "{") + return strings.HasPrefix(line, "{") } -// parseTargetLine decodes and validates a target line. Unknown keys and an ip -// that is null or empty are rejected, so that a typo cannot silently probe the -// host through DNS. +// parseTargetLine decodes and validates a target line. func parseTargetLine(line string) (targetLine, error) { + parsed, err := decodeTargetLine(line) + if err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + return parsed, nil +} + +// decodeTargetLine decodes a target line, or returns why it is invalid. +// Unknown keys and an ip that is null or empty are rejected, so that a typo +// cannot silently probe the host through DNS. +func decodeTargetLine(line string) (targetLine, error) { // the ip is kept raw to tell an ip that is null or empty from no ip var decoded struct { Target string `json:"target"` @@ -46,37 +55,33 @@ func parseTargetLine(line string) (targetLine, error) { decoder := json.NewDecoder(strings.NewReader(line)) decoder.DisallowUnknownFields() if err := decoder.Decode(&decoded); err != nil { - return targetLine{}, invalidTargetLine(line, err) + return targetLine{}, err } if _, err := decoder.Token(); !errors.Is(err, io.EOF) { - return targetLine{}, invalidTargetLine(line, errors.New("unexpected data after the object")) + return targetLine{}, errors.New("unexpected data after the object") } target := strings.TrimSpace(decoded.Target) if target == "" { - return targetLine{}, invalidTargetLine(line, errors.New("no target")) + return targetLine{}, errors.New("no target") } if decoded.IP == nil { return targetLine{Target: target}, nil } + if err := checkPinnable(target); err != nil { + return targetLine{}, err + } var ip string if err := json.Unmarshal(decoded.IP, &ip); err != nil { - return targetLine{}, invalidTargetLine(line, err) - } - if err := checkPinnable(target); err != nil { - return targetLine{}, invalidTargetLine(line, err) + return targetLine{}, err } addr, err := netip.ParseAddr(strings.TrimSpace(ip)) if err != nil || addr.Zone() != "" || addr.IsUnspecified() { - return targetLine{}, invalidTargetLine(line, fmt.Errorf("invalid ip %q", ip)) + return targetLine{}, fmt.Errorf("invalid ip %q", ip) } return targetLine{Target: target, IP: addr.Unmap().String()}, nil } -func invalidTargetLine(line string, reason error) error { - return fmt.Errorf("invalid target line %q: %w", line, reason) -} - // checkPinnable returns why target cannot be pinned to an ip: only a single // target with a hostname can. func checkPinnable(target string) error { @@ -95,11 +100,6 @@ func checkPinnable(target string) error { return nil } -// isVHostInput reports whether target is a vhost,ip input. -func isVHostInput(target string) bool { - return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") -} - // key returns the line as it is stored and deduplicated: the target alone // when it is not pinned, and a canonical target line otherwise. func (t targetLine) key() string { @@ -113,60 +113,51 @@ func (t targetLine) key() string { return strings.TrimSpace(buf.String()) } -// pinnedKey returns key for a target that is not pinned, and key@ip for one -// pinned to ip, as the ips of a host fail and respond independently. -func pinnedKey(key, ip string) string { - if ip == "" { - return key - } - return key + "@" + ip -} - // targetOf returns the target of an input line: the line itself, or the // target of a target line. -func targetOf(item string) string { - if !isTargetLine(item) { - return item +func targetOf(line string) string { + if !isTargetLine(line) { + return line } - if line, err := parseTargetLine(item); err == nil { - return line.Target + if parsed, err := parseTargetLine(line); err == nil { + return parsed.Target } - return item + return line } // normalizeInput validates an input line and returns the form it is stored // and deduplicated in. -func (r *Runner) normalizeInput(item string) (string, error) { - item = strings.TrimSpace(item) - if !isTargetLine(item) { - return item, nil +func (r *Runner) normalizeInput(line string) (string, error) { + line = strings.TrimSpace(line) + if !isTargetLine(line) { + return line, nil } - line, err := r.parseInputLine(item) + parsed, err := r.parseInputLine(line) if err != nil { return "", err } - return line.key(), nil + return parsed.key(), nil } -// parseInputLine parses a target line, and refuses one pinned to an ip with -// the options that do not keep to it. -func (r *Runner) parseInputLine(item string) (targetLine, error) { - line, err := parseTargetLine(item) +// parseInputLine parses a target line, and refuses one with an ip when an +// option does not keep to the ip. +func (r *Runner) parseInputLine(line string) (targetLine, error) { + parsed, err := parseTargetLine(line) if err != nil { return targetLine{}, err } - if line.IP != "" { - if flags := r.options.pinnedTargetConflicts(); len(flags) > 0 { - return targetLine{}, invalidTargetLine(item, fmt.Errorf("a target pinned to an ip cannot be probed with %s", strings.Join(flags, ", "))) + if parsed.IP != "" { + if flags := r.options.targetLineConflicts(); len(flags) > 0 { + return targetLine{}, fmt.Errorf("target line %q: a target pinned to an ip cannot be probed with %s", line, strings.Join(flags, ", ")) } } - return line, nil + return parsed, nil } -// pinnedTargetConflicts returns the options whose probes do not connect -// through the ip a target is pinned to, or share a connection between the ips -// of a host, as HTTP/2 does. -func (options *Options) pinnedTargetConflicts() []string { +// targetLineConflicts returns the options a target line with an ip cannot be +// probed with: their probes do not connect through the ip, or share a +// connection between the ips of a host, as HTTP/2 does. +func (options *Options) targetLineConflicts() []string { var flags []string if options.Unsafe { flags = append(flags, "-unsafe") diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 91e839277..3ebaec0ef 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -1,6 +1,7 @@ package runner import ( + "context" "crypto/tls" "fmt" "net" @@ -21,27 +22,23 @@ import ( "golang.org/x/net/dns/dnsmessage" ) -// TestMain keeps pinned.test out of a proxy set in the environment, which -// would resolve the host itself, so that the probes below run behind one too. -func TestMain(m *testing.M) { - for _, name := range []string{"NO_PROXY", "no_proxy"} { - _ = os.Setenv(name, strings.TrimPrefix(os.Getenv(name)+",pinned.test", ",")) - } - os.Exit(m.Run()) -} - func TestParseTargetLine(t *testing.T) { - valid := map[string]targetLine{ - `{"target": "https://a.example", "ip": "192.0.2.10"}`: {Target: "https://a.example", IP: "192.0.2.10"}, - `{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`: {Target: "a.example:8443", IP: "2001:db8::1"}, - `{"target": "a.example", "ip": "::ffff:192.0.2.10"}`: {Target: "a.example", IP: "192.0.2.10"}, - `{"target": "a.example"}`: {Target: "a.example"}, + valid := map[string]struct { + line string + want targetLine + }{ + "ipv4": {`{"target": "https://a.example", "ip": "192.0.2.10"}`, targetLine{Target: "https://a.example", IP: "192.0.2.10"}}, + "ipv6": {`{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`, targetLine{Target: "a.example:8443", IP: "2001:db8::1"}}, + "ipv4-mapped": {`{"target": "a.example", "ip": "::ffff:192.0.2.10"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "spaced ip": {`{"target": "a.example", "ip": " 192.0.2.10 "}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "no ip": {`{"target": "a.example"}`, targetLine{Target: "a.example"}}, + "escaped target": {`{"target": "https://a.example/?a=1&b=2"}`, targetLine{Target: "https://a.example/?a=1&b=2"}}, } - for line, want := range valid { - t.Run(line, func(t *testing.T) { - got, err := parseTargetLine(line) + for name, tc := range valid { + t.Run(name, func(t *testing.T) { + got, err := parseTargetLine(tc.line) require.NoError(t, err) - require.Equal(t, want, got) + require.Equal(t, tc.want, got) }) } @@ -95,7 +92,7 @@ func TestRunner_normalizeInput(t *testing.T) { }) } -func TestRunner_pinnedTargetConflicts(t *testing.T) { +func TestRunner_targetLineConflicts(t *testing.T) { conflicts := map[string]Options{ "-unsafe": {Unsafe: true}, "-pipeline": {Pipeline: true}, @@ -112,7 +109,7 @@ func TestRunner_pinnedTargetConflicts(t *testing.T) { _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) require.ErrorContains(t, err, "cannot be probed with "+flag) _, err = r.normalizeInput(`{"target": "https://a.example"}`) - require.NoError(t, err, "only pinned targets conflict") + require.NoError(t, err, "only a line with an ip conflicts") }) } } @@ -196,31 +193,48 @@ func TestRunner_skip_pinned(t *testing.T) { require.False(t, skip) } -// A proxy, set with -proxy for -pa or in the environment, resolves the host -// itself, so a probe of a pinned target through it fails. +// A proxy, which -pa can be run with, resolves the host itself, so a probe of +// a pinned target through it fails. A proxy set in the environment is tested +// in common/httpx. func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { r, err := New(&Options{Proxy: "http://127.0.0.1:1"}) require.NoError(t, err) defer r.Close() - target := httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"} - result := r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) require.ErrorContains(t, result.Err, "through a proxy") } +// A target probed through a proxy has no ip of its own, and does not report +// the last ip the dialer dialed to its host, here for a pinned target. +func TestRunner_analyze_proxiedTargetHasNoDialedIP(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + r, err := New(&Options{Proxy: "http://127.0.0.1:" + serveLocal(t, titleHandler("proxy"))}) + require.NoError(t, err) + defer r.Close() + conn, err := r.hp.Dialer.Dial(httpx.WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), "tcp", "pinned.test:"+port) + require.NoError(t, err) + _ = conn.Close() + + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test:" + port}) + require.NoError(t, result.Err) + require.Equal(t, "proxy pinned.test:"+port, result.Title) + require.Empty(t, result.HostIP) +} + // -maxhr counts the failures of each ip of a host apart. func TestRunner_analyze_hostErrorsPerIP(t *testing.T) { - port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) + port := serveLocal(t, titleHandler("one")) r, err := New(&Options{HostMaxErrors: 1, Timeout: 2}) require.NoError(t, err) defer r.Close() - - analyze := func(ip string) Result { - target := httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} - return r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) + target := func(ip string) httpx.Target { + return httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} } - require.Error(t, analyze("127.0.0.3").Err, "nothing listens there") - require.NoError(t, analyze("127.0.0.1").Err, "the other ip of the host is not skipped") + + require.Error(t, analyzeHTTP(r, target("127.0.0.3")).Err, "nothing listens there") + require.ErrorContains(t, analyzeHTTP(r, target("127.0.0.3")).Err, "previously unresponsive") + require.NoError(t, analyzeHTTP(r, target("127.0.0.1")).Err, "the other ip of the host is not skipped") } func TestTargetLinePinsTheConnection(t *testing.T) { @@ -230,6 +244,7 @@ func TestTargetLinePinsTheConnection(t *testing.T) { var got []string for _, result := range results { got = append(got, result.HostIP+" "+result.Title) + require.Contains(t, result.str, "["+result.HostIP+"]", "the output shows the ip") } sort.Strings(got) require.Equal(t, []string{ @@ -238,11 +253,17 @@ func TestTargetLinePinsTheConnection(t *testing.T) { }, got) } +// The scheme of the target of a line is kept, as for a url given as is. +func TestTargetLineKeepsTheScheme(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + results := probe(t, &Options{}, pinnedLine("https://pinned.test:"+port, "127.0.0.1")) + require.Empty(t, results, "a plain http server does not answer https") +} + func TestTargetLineDeduplicatesEquivalentLines(t *testing.T) { - port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) - target := "http://pinned.test:" + port + target := "http://pinned.test:" + serveLocal(t, titleHandler("one")) results := probe(t, &Options{}, - fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target), + pinnedLine(target, "127.0.0.1"), fmt.Sprintf(` {"ip": "::ffff:127.0.0.1", "target": %q}`, target), ) require.Len(t, results, 1) @@ -276,10 +297,10 @@ func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { } } -// Like curl --resolve, the host of a pinned target is not resolved, and a -// holds dns data only. +// Like curl --resolve, the host of a pinned target is not resolved, so its +// result has no A records. func TestTargetLineIsNotResolved(t *testing.T) { - port := portOf(t, serve(t, listen(t, "127.0.0.1:0"), titleHandler("one"), nil).Listener) + port := serveLocal(t, titleHandler("one")) resolver, queries := serveDNS(t, "pinned.test", "127.0.0.2") results := probe(t, &Options{Resolvers: []string{resolver}}, pinnedLines(port, "127.0.0.1")...) @@ -314,21 +335,27 @@ func TestPlainTargetReportsItsOwnIP(t *testing.T) { lines := "http://pinned.test:" + port + "\n" + pinnedLines(port, "127.0.0.1")[0] + "\n" require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + // the delay dials the plain target first, and its slow server keeps it + // open while the pinned target of the same host is dialed options := &Options{Resolvers: []string{resolver}, Stream: true, InputFile: file, Threads: 2, Delay: 100 * time.Millisecond} - results := probe(t, options) - require.Len(t, results, 2) - for _, result := range results { - ip := "127.0.0.1" - if strings.HasPrefix(result.Title, "two") { - ip = "127.0.0.2" - } - require.Equal(t, ip, result.HostIP, result.Title) + var got []string + for _, result := range probe(t, options) { + got = append(got, result.HostIP+" "+result.Title) } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) } // An IPv6 ip is dialed as well, which -pa relies on for the IPv6 ips of a host. func TestTargetLinePinsIPv6(t *testing.T) { - port := portOf(t, serve(t, listen(t, "[::1]:0"), titleHandler("six"), nil).Listener) + listener, err := net.Listen("tcp", "[::1]:0") + if err != nil { + t.Skipf("IPv6 is not available: %v", err) + } + port := portOf(t, serve(t, listener, titleHandler("six"), nil).Listener) results := probe(t, &Options{}, pinnedLines(port, "::1")...) require.Len(t, results, 1) @@ -348,8 +375,7 @@ func TestTargetLineKeepsSNI(t *testing.T) { return nil, nil }, }) - target := "https://pinned.test:" + portOf(t, server.Listener) - results := probe(t, &Options{}, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) + results := probe(t, &Options{}, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) require.Len(t, results, 1) mu.Lock() @@ -364,9 +390,8 @@ func TestTargetLineKeepsSNI(t *testing.T) { // pinned ip as well. func TestTargetLinePinsTheFaviconAndTLSGrab(t *testing.T) { server := serve(t, listen(t, "127.0.0.1:0"), withFavicon(titleHandler("tls")), &tls.Config{}) - target := "https://pinned.test:" + portOf(t, server.Listener) options := &Options{Favicon: true, ZTLS: true, TLSGrab: true} - results := probe(t, options, fmt.Sprintf(`{"target": %q, "ip": "127.0.0.1"}`, target)) + results := probe(t, options, pinnedLine("https://pinned.test:"+portOf(t, server.Listener), "127.0.0.1")) require.Len(t, results, 1) require.NotEmpty(t, results[0].FavIconMMH3) @@ -384,7 +409,7 @@ func TestRunner_HandleFaviconHash_keepsTheTraceOfTheProbe(t *testing.T) { require.NoError(t, err) defer r.Close() - result := r.analyze(r.hp, httpx.HTTP, httpx.Target{Host: page.URL}, http.MethodGet, page.URL, &r.scanopts) + result := analyzeHTTP(r, httpx.Target{Host: page.URL}) require.NoError(t, result.Err) require.NotEmpty(t, result.FavIconMMH3) require.Equal(t, page.Listener.Addr().String(), fmt.Sprint(result.Trace.GetConn.Info)) @@ -408,13 +433,10 @@ func withFavicon(handler http.Handler) http.Handler { }) } -// listen listens on addr, and skips the test where addr cannot be bound. func listen(t *testing.T, addr string) net.Listener { t.Helper() listener, err := net.Listen("tcp", addr) - if err != nil { - t.Skipf("cannot listen on %s: %v", addr, err) - } + require.NoError(t, err) return listener } @@ -434,6 +456,12 @@ func serve(t *testing.T, listener net.Listener, handler http.Handler, tlsConfig return server } +// serveLocal serves handler on 127.0.0.1, and returns the port. +func serveLocal(t *testing.T, handler http.Handler) string { + t.Helper() + return portOf(t, serve(t, listen(t, "127.0.0.1:0"), handler, nil).Listener) +} + func portOf(t *testing.T, listener net.Listener) string { t.Helper() _, port, err := net.SplitHostPort(listener.Addr().String()) @@ -463,11 +491,15 @@ func servePair(t *testing.T, one, two http.Handler) string { return "" } +func pinnedLine(target, ip string) string { + return fmt.Sprintf(`{"target": %q, "ip": %q}`, target, ip) +} + // pinnedLines returns the lines pinning http://pinned.test:port to each ip. func pinnedLines(port string, ips ...string) []string { var lines []string for _, ip := range ips { - lines = append(lines, fmt.Sprintf(`{"target": "http://pinned.test:%s", "ip": %q}`, port, ip)) + lines = append(lines, pinnedLine("http://pinned.test:"+port, ip)) } return lines } @@ -525,6 +557,11 @@ func serveDNS(t *testing.T, host string, ips ...string) (string, *atomic.Int32) return conn.LocalAddr().String(), &queries } +// analyzeHTTP probes target over http with r. +func analyzeHTTP(r *Runner, target httpx.Target) Result { + return r.analyze(r.hp, httpx.HTTP, target, http.MethodGet, target.Host, &r.scanopts) +} + // probe runs a runner on inputs with options, completed with test defaults, // and returns its results. The results that failed are logged. func probe(t *testing.T, options *Options, inputs ...string) []Result { From 8bfab430913c7f5cdecb0fbfa46f8bbc3b850239 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Wed, 28 Oct 2026 00:41:45 +0000 Subject: [PATCH 15/16] fix(runner): skip an invalid target line instead of stopping the run An invalid line is reported and skipped, as in -stream, so that one bad line in a large list does not stop the run. A line that conflicts with the options still stops it, as every such line would. Names of a certificate shaped like target lines are not probed with -tls-probe: only an input pins a target to an ip. The refusal of a proxy for a pinned target is reported once, naming NO_PROXY, and host_ip falls back to the dialer history for a proxied request again, as on dev. --- README.md | 6 +- common/httpx/dialedip.go | 35 +++-------- common/httpx/httpx.go | 2 +- common/httpx/pinned.go | 6 +- common/httpx/pinned_test.go | 25 ++------ runner/runner.go | 81 +++++++++++++------------- runner/targetline.go | 25 +++++++- runner/targetline_test.go | 113 ++++++++++++++++++++++++++++-------- 8 files changed, 171 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 522bbb9cb..a33bb547f 100644 --- a/README.md +++ b/README.md @@ -386,9 +386,9 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - The connection goes to the IP, which `host_ip` reports and the output shows, and the hostname is kept for the Host header and SNI without being resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. - - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line stops the run, or is skipped with a warning with `-stream`. - - A line with an IP is refused with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well, so a probe that would go through one fails, `-pa` ones included. - - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. The names `-tls-probe` and `-csp-probe` find, the pinned host included, are resolved as new targets. + - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line is reported and skipped. + - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well, so a probe that would go through one fails, `-pa` ones included: add the pinned hosts to `NO_PROXY` to probe them directly. + - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. The names `-tls-probe` and `-csp-probe` find, the pinned host included, are resolved as new targets and cannot pin one to an IP. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/dialedip.go b/common/httpx/dialedip.go index 5cc92a871..7ec6fb5a5 100644 --- a/common/httpx/dialedip.go +++ b/common/httpx/dialedip.go @@ -3,8 +3,6 @@ package httpx import ( "context" "net" - "net/http" - "net/url" "sync" ) @@ -14,13 +12,11 @@ type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) // dialedIPKey is the context key of the dialedIP of WithDialedIP. type dialedIPKey struct{} -// dialedIP is the ip of the last connection dialed to a host, and whether a -// request went through a proxy instead. +// dialedIP is the ip of the last connection dialed to a host. type dialedIP struct { - host string - mu sync.Mutex - ip string - proxied bool + host string + mu sync.Mutex + ip string } // WithDialedIP returns a copy of ctx that records the ip of the connections @@ -31,16 +27,15 @@ func WithDialedIP(ctx context.Context, host string) context.Context { } // DialedIP returns the ip of the last connection dialed to the host of a -// WithDialedIP context, and whether a request with it went through a proxy, -// which dials the host itself. -func DialedIP(ctx context.Context) (ip string, proxied bool) { +// WithDialedIP context, or "" if none was. +func DialedIP(ctx context.Context) string { dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) if !ok { - return "", false + return "" } dialed.mu.Lock() defer dialed.mu.Unlock() - return dialed.ip, dialed.proxied + return dialed.ip } // recordDialedIP wraps dial to record the ip of a connection to the host of a @@ -63,17 +58,3 @@ func recordDialedIP(dial dialFunc) dialFunc { return conn, err } } - -// recordProxied wraps the proxy of a transport to record that a request with -// a WithDialedIP context goes through a proxy. -func recordProxied(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { - return func(req *http.Request) (*url.URL, error) { - proxyURL, err := proxy(req) - if dialed, ok := req.Context().Value(dialedIPKey{}).(*dialedIP); ok && proxyURL != nil { - dialed.mu.Lock() - dialed.proxied = true - dialed.mu.Unlock() - } - return proxyURL, err - } -} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 233d0fdf0..4e6e8b678 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -176,7 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } - transport.Proxy = recordProxied(refusePinned(transport.Proxy)) + transport.Proxy = refusePinned(transport.Proxy) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go index 01d685457..f5e61e8af 100644 --- a/common/httpx/pinned.go +++ b/common/httpx/pinned.go @@ -11,9 +11,9 @@ import ( "golang.org/x/net/idna" ) -// errPinnedThroughProxy is the error of a request to a pinned host that would +// ErrPinnedThroughProxy is the error of a request to a pinned host that would // go through a proxy, which resolves the host itself. -var errPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") +var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") // pinnedHostKey is the context key of a host pinned with WithPinnedIP. type pinnedHostKey string @@ -60,7 +60,7 @@ func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Reques return func(req *http.Request) (*url.URL, error) { proxyURL, err := proxy(req) if proxyURL != nil && pinnedIP(req) != "" { - return nil, errPinnedThroughProxy + return nil, ErrPinnedThroughProxy } return proxyURL, err } diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go index ef936dc4b..650dd7688 100644 --- a/common/httpx/pinned_test.go +++ b/common/httpx/pinned_test.go @@ -24,7 +24,7 @@ func TestRefusePinned(t *testing.T) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) require.NoError(t, err) _, err = proxy(req) - require.ErrorIs(t, err, errPinnedThroughProxy, rawURL) + require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL) } redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) @@ -59,7 +59,7 @@ func TestRefusePinned_environment(t *testing.T) { req, err := h.NewRequestWithContext(WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/") require.NoError(t, err) _, err = h.Do(req, UnsafeOptions{}) - require.ErrorContains(t, err, errPinnedThroughProxy.Error()) + require.ErrorIs(t, err, ErrPinnedThroughProxy) } func TestCanonicalHost(t *testing.T) { @@ -101,28 +101,11 @@ func TestRecordDialedIP(t *testing.T) { conn, err := dial(ctx, "tcp", "proxy.test:3128") require.NoError(t, err) _ = conn.Close() - ip, _ := DialedIP(ctx) - require.Empty(t, ip, "a dial to another host, such as a proxy") + require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") // net/http dials a host as it is spelled conn, err = dial(ctx, "tcp", "DIALED.test.:80") require.NoError(t, err) _ = conn.Close() - ip, _ = DialedIP(ctx) - require.Equal(t, "127.0.0.1", ip) -} - -func TestRecordProxied(t *testing.T) { - proxyURL, err := url.Parse("http://127.0.0.1:3128") - require.NoError(t, err) - - for proxy, want := range map[*url.URL]bool{proxyURL: true, nil: false} { - ctx := WithDialedIP(context.Background(), "dialed.test") - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://dialed.test/", nil) - require.NoError(t, err) - _, err = recordProxied(http.ProxyURL(proxy))(req) - require.NoError(t, err) - _, proxied := DialedIP(ctx) - require.Equal(t, want, proxied) - } + require.Equal(t, "127.0.0.1", DialedIP(ctx)) } diff --git a/runner/runner.go b/runner/runner.go index 022aa19af..a8a654a26 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -98,6 +98,7 @@ type Runner struct { httpApiEndpoint *Server authProvider authprovider.AuthProvider interruptCh chan struct{} + pinnedProxyWarning sync.Once } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -555,9 +556,9 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { - target, err := r.normalizeInput(target) - if err != nil { - gologger.Fatal().Msgf("Could not read input target: %s\n", err) + target, ok := r.readInputLine(target) + if !ok { + continue } expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { @@ -742,12 +743,10 @@ func (r *Runner) streamInput() (chan string, error) { } } // sendLine normalizes, deduplicates and sends an input line, returning - // false if interrupted. The scan has already started, so an invalid - // line is skipped with a warning. + // false if interrupted. sendLine := func(item string) bool { - item, err := r.normalizeInput(item) - if err != nil { - gologger.Warning().Msgf("Skipping input: %s\n", err) + item, ok := r.readInputLine(item) + if !ok { return true } if r.options.SkipDedupe || r.testAndSet(item) { @@ -809,10 +808,9 @@ func (r *Runner) streamInput() (chan string, error) { func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { - target, err := r.normalizeInput(scanner.Text()) - if err != nil { - _ = finput.Close() - return 0, err + target, ok := r.readInputLine(scanner.Text()) + if !ok { + continue } // Used just to get the exact number of targets expandedTarget, err := r.countTargetFromRawTarget(target) @@ -1715,15 +1713,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } if scanopts.CSPProbe && result.CSPData != nil { scanopts.CSPProbe = false @@ -1769,15 +1759,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } }(port, target, method, wantedProtocol) } @@ -1789,6 +1771,18 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT } } +// probeTLSNames probes the names of the certificate of result, for -tls-probe. +// A name shaped like a target line is skipped: only an input pins a target to +// an ip, not the server being probed. +func (r *Runner) probeTLSNames(result Result, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { + names := append(slices.Clone(result.TLSData.SubjectAN), result.TLSData.SubjectCN) + for _, name := range names { + if !isTargetLine(name) && r.testAndSet(name) { + r.process(name, wg, hp, protocol, scanopts, output) + } + } +} + // isVHostInput reports whether target is a vhost,ip input. func isVHostInput(target string) bool { return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") @@ -1804,11 +1798,11 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { switch { case isTargetLine(target): - // input lines were checked when read, but the names -tls-probe - // and -csp-probe find come here directly + // input lines were checked when read, but not a line given to + // Process line, err := r.parseInputLine(target) if err != nil { - gologger.Warning().Msgf("Skipping input: %s\n", err) + gologger.Error().Msgf("Skipping input: %s\n", err) return } results <- httpx.Target{Host: line.Target, CustomIP: line.IP} @@ -2071,6 +2065,11 @@ retry: builder.WriteRune(']') } if err != nil { + if errors.Is(err, httpx.ErrPinnedThroughProxy) { + r.pinnedProxyWarning.Do(func() { + gologger.Error().Msgf("Targets pinned to an ip cannot be probed through a proxy, which resolves the host itself: with HTTP_PROXY or HTTPS_PROXY set, add their hosts to NO_PROXY\n") + }) + } errString := "" errString = err.Error() splitErr := strings.Split(errString, ":") @@ -2319,15 +2318,15 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - var proxied bool - ip, proxied = httpx.DialedIP(req.Context()) - if ip == "" && !proxied { - // no dial was recorded, as for a reused connection or -unsafe: - // hp.Dialer.GetDialedIP would return only the last dialed one + // the ip this request was dialed to, or else, as for a reused + // connection, -unsafe or a proxy, the last ip the dialer dialed + // to the host, which may be another request's + ip = httpx.DialedIP(req.Context()) + if ip == "" { ip = hp.Dialer.GetDialedIP(URL.Host) - if ip == "" { - ip = hp.Dialer.GetDialedIP(onlyHost) - } + } + if ip == "" { + ip = hp.Dialer.GetDialedIP(onlyHost) } } } diff --git a/runner/targetline.go b/runner/targetline.go index be89e1a30..57d5d837c 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -9,6 +9,7 @@ import ( "net/netip" "strings" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" iputil "github.com/projectdiscovery/utils/ip" @@ -16,6 +17,10 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) +// errTargetLineConflict is the error of a target line with an ip and an option +// that does not keep to the ip. +var errTargetLineConflict = errors.New("a target pinned to an ip cannot be probed with") + // targetLine is an input line that pins a target to an ip, like // curl --resolve: // @@ -31,7 +36,7 @@ type targetLine struct { // isTargetLine reports whether an input line is a JSON object, the form of a // target line. func isTargetLine(line string) bool { - return strings.HasPrefix(line, "{") + return strings.HasPrefix(strings.TrimSpace(line), "{") } // parseTargetLine decodes and validates a target line. @@ -125,6 +130,22 @@ func targetOf(line string) string { return line } +// readInputLine returns the form an input line is stored and deduplicated in, +// and false if the line is skipped: an invalid line is reported and skipped, +// while a target line that conflicts with the options stops the run, as every +// such line would. +func (r *Runner) readInputLine(line string) (string, bool) { + key, err := r.normalizeInput(line) + if errors.Is(err, errTargetLineConflict) { + gologger.Fatal().Msgf("Could not read input: %s\n", err) + } + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return "", false + } + return key, true +} + // normalizeInput validates an input line and returns the form it is stored // and deduplicated in. func (r *Runner) normalizeInput(line string) (string, error) { @@ -148,7 +169,7 @@ func (r *Runner) parseInputLine(line string) (targetLine, error) { } if parsed.IP != "" { if flags := r.options.targetLineConflicts(); len(flags) > 0 { - return targetLine{}, fmt.Errorf("target line %q: a target pinned to an ip cannot be probed with %s", line, strings.Join(flags, ", ")) + return targetLine{}, fmt.Errorf("target line %q: %w %s", line, errTargetLineConflict, strings.Join(flags, ", ")) } } return parsed, nil diff --git a/runner/targetline_test.go b/runner/targetline_test.go index 3ebaec0ef..adda85257 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -1,15 +1,21 @@ package runner import ( - "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "fmt" + "math/big" "net" "net/http" "net/http/httptest" "net/netip" "os" "path/filepath" + "slices" "sort" "strings" "sync" @@ -17,6 +23,9 @@ import ( "testing" "time" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/gologger/writer" "github.com/projectdiscovery/httpx/common/httpx" "github.com/stretchr/testify/require" "golang.org/x/net/dns/dnsmessage" @@ -107,6 +116,7 @@ func TestRunner_targetLineConflicts(t *testing.T) { t.Run(flag, func(t *testing.T) { r := &Runner{options: &options} _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorIs(t, err, errTargetLineConflict) require.ErrorContains(t, err, "cannot be probed with "+flag) _, err = r.normalizeInput(`{"target": "https://a.example"}`) require.NoError(t, err, "only a line with an ip conflicts") @@ -160,17 +170,28 @@ func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { require.Equal(t, []string{"a.example", `{"target":"c.example","ip":"192.0.2.10"}`}, got, "the malformed line is skipped") } -func TestRunner_loadAndCloseFile_refusesInvalidLines(t *testing.T) { +// An invalid line, such as a line of another tool's JSON output, is skipped +// and the other lines are read. +func TestRunner_loadAndCloseFile_skipsInvalidLines(t *testing.T) { file := filepath.Join(t.TempDir(), "targets.txt") - require.NoError(t, os.WriteFile(file, []byte("a.example\n"+`{"target": "b.example", "ip": null}`+"\n"), 0o600)) + lines := "a.example\n" + `{"host": "b.example"}` + "\n" + `{"target": "c.example", "ip": null}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) input, err := os.Open(file) require.NoError(t, err) + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + r, err := New(&Options{}) require.NoError(t, err) defer r.Close() - _, err = r.loadAndCloseFile(input) - require.ErrorContains(t, err, "invalid ip") + count, err := r.loadAndCloseFile(input) + require.NoError(t, err) + require.Equal(t, 1, count) + _, ok := r.hm.Get("a.example") + require.True(t, ok) + require.Len(t, logged.lines(), 2, "each skipped line is reported") } func TestRunner_skip_pinned(t *testing.T) { @@ -194,32 +215,39 @@ func TestRunner_skip_pinned(t *testing.T) { } // A proxy, which -pa can be run with, resolves the host itself, so a probe of -// a pinned target through it fails. A proxy set in the environment is tested -// in common/httpx. +// a pinned target through it fails, which is reported once. A proxy set in +// the environment is tested in common/httpx. func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { - r, err := New(&Options{Proxy: "http://127.0.0.1:1"}) + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + r, err := New(&Options{Proxy: "http://127.0.0.1:1", HostMaxErrors: -1}) require.NoError(t, err) defer r.Close() - result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) - require.ErrorContains(t, result.Err, "through a proxy") + for range 2 { + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) + require.ErrorIs(t, result.Err, httpx.ErrPinnedThroughProxy) + } + require.Len(t, logged.lines(), 1) + require.Contains(t, logged.lines()[0], "NO_PROXY") } -// A target probed through a proxy has no ip of its own, and does not report -// the last ip the dialer dialed to its host, here for a pinned target. -func TestRunner_analyze_proxiedTargetHasNoDialedIP(t *testing.T) { - port := serveLocal(t, titleHandler("one")) - r, err := New(&Options{Proxy: "http://127.0.0.1:" + serveLocal(t, titleHandler("proxy"))}) - require.NoError(t, err) - defer r.Close() - conn, err := r.hp.Dialer.Dial(httpx.WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), "tcp", "pinned.test:"+port) - require.NoError(t, err) - _ = conn.Close() +// The names of a certificate shaped like a target line are not probed with +// -tls-probe: the server being probed cannot pin a target to an ip. +func TestRunner_probeTLSNames_skipsTargetLines(t *testing.T) { + var requests atomic.Int32 + // the names are probed over the scheme of the target they come from + pinned := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + }), &tls.Config{}) + // the leading space is trimmed when the name is read as an input + certificate := selfSignedCertificate(t, " "+pinnedLine("https://pinned.test:"+portOf(t, pinned.Listener), "127.0.0.1")) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{Certificates: []tls.Certificate{certificate}}) - result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test:" + port}) - require.NoError(t, result.Err) - require.Equal(t, "proxy pinned.test:"+port, result.Title) - require.Empty(t, result.HostIP) + results := probe(t, &Options{TLSProbe: true, TLSGrab: true, Threads: 5}, server.URL) + require.NotEmpty(t, results) + require.Zero(t, requests.Load()) } // -maxhr counts the failures of each ip of a host apart. @@ -415,6 +443,43 @@ func TestRunner_HandleFaviconHash_keepsTheTraceOfTheProbe(t *testing.T) { require.Equal(t, page.Listener.Addr().String(), fmt.Sprint(result.Trace.GetConn.Info)) } +// errorRecorder is a gologger writer that keeps the errors, which are shown +// by default. +type errorRecorder struct { + mu sync.Mutex + errors []string +} + +func (w *errorRecorder) Write(data []byte, level levels.Level) { + if level == levels.LevelError { + w.mu.Lock() + w.errors = append(w.errors, string(data)) + w.mu.Unlock() + } +} + +func (w *errorRecorder) lines() []string { + w.mu.Lock() + defer w.mu.Unlock() + return slices.Clone(w.errors) +} + +// selfSignedCertificate returns a certificate whose common name is name. +func selfSignedCertificate(t *testing.T, name string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: name}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + // titleHandler serves a page whose title is name and the Host header. func titleHandler(name string) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From f7de334c084d8d30ff78be57319dd82e266fdf93 Mon Sep 17 00:00:00 2001 From: Georgii Butakov Date: Wed, 28 Oct 2026 01:12:48 +0000 Subject: [PATCH 16/16] fix(runner): read a line given to Process as an input line A line given to Process is normalized as the lines of the input are, so a line without ip is its plain target, expanded and resolved as one, and the options a pinned line conflicts with are checked once, when a line is read. -pa is refused with -proxy, which resolves the host itself: it used to dial the port of the proxy on each ip of the host. The README and the doc comments say that a host is pinned on every port. --- README.md | 9 +-- common/httpx/pinned.go | 5 +- runner/options.go | 4 ++ runner/runner.go | 21 ++++--- runner/targetline.go | 21 ++----- runner/targetline_test.go | 125 ++++++++++++++++++++++++++++---------- 6 files changed, 125 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index a33bb547f..6bce0ca3c 100644 --- a/README.md +++ b/README.md @@ -380,15 +380,16 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` -- A target can be pinned to an IP with a JSON input line, like `curl --resolve`: +- A target can be pinned to an IP with a JSON input line, like `curl --resolve` but on every port of its host: ``` {"target": "https://app.example.com", "ip": "192.0.2.10"} {"target": "https://app.example.com", "ip": "192.0.2.11"} ``` - - The connection goes to the IP, which `host_ip` reports and the output shows, and the hostname is kept for the Host header and SNI without being resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. + - Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line is reported and skipped. - - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself as well, so a probe that would go through one fails, `-pa` ones included: add the pinned hosts to `NO_PROXY` to probe them directly. - - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. The names `-tls-probe` and `-csp-probe` find, the pinned host included, are resolved as new targets and cannot pin one to an IP. + - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host, and `-pa` cannot be used with `-proxy`. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`. + - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. + - Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go index f5e61e8af..1d10220fd 100644 --- a/common/httpx/pinned.go +++ b/common/httpx/pinned.go @@ -19,8 +19,9 @@ var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached type pinnedHostKey string // WithPinnedIP returns a copy of ctx in which requests to host connect to ip, -// like curl --resolve. The host keeps its name for the Host header and SNI, -// and requests to other hosts, such as redirect targets, are resolved as usual. +// like curl --resolve, but on every port of host. The host keeps its name for +// the Host header and SNI, and requests to other hosts, such as redirect +// targets, are resolved as usual. func WithPinnedIP(ctx context.Context, host, ip string) context.Context { host = canonicalHost(host) ctx = fastdialer.WithResolveOverride(ctx, host, ip) diff --git a/runner/options.go b/runner/options.go index 4f89b2c8d..ca2001265 100644 --- a/runner/options.go +++ b/runner/options.go @@ -768,6 +768,10 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("invalid metrics port %d, must be between 0 and 65535", options.MetricsPort) } + if options.ProbeAllIPS && options.Proxy != "" { + return errors.New("-pa/-probe-all-ips cannot be used with -proxy, which resolves the host itself") + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index a8a654a26..6bbf18ba5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1683,7 +1683,14 @@ func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } +// Process probes t, read as an input line is: an invalid line is reported and +// skipped. func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { + t, err := r.normalizeInput(t) + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return + } r.process(t, wg, r.hp, protocol, scanopts, output) } @@ -1798,9 +1805,8 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { switch { case isTargetLine(target): - // input lines were checked when read, but not a line given to - // Process - line, err := r.parseInputLine(target) + // a target line, normalized when read, pins its target to an ip + line, err := parseTargetLine(target) if err != nil { gologger.Error().Msgf("Skipping input: %s\n", err) return @@ -2067,7 +2073,7 @@ retry: if err != nil { if errors.Is(err, httpx.ErrPinnedThroughProxy) { r.pinnedProxyWarning.Do(func() { - gologger.Error().Msgf("Targets pinned to an ip cannot be probed through a proxy, which resolves the host itself: with HTTP_PROXY or HTTPS_PROXY set, add their hosts to NO_PROXY\n") + gologger.Error().Msgf("Targets pinned to an ip, by a target line or -pa, cannot be probed through the proxy of HTTP_PROXY or HTTPS_PROXY, which resolves the host itself: add their hosts to NO_PROXY\n") }) } errString := "" @@ -2107,6 +2113,7 @@ retry: } if r.options.Probe { + // the pinned ip tells apart the failures of the ips of a host if target.CustomIP != "" { _, _ = fmt.Fprintf(builder, " [%s]", target.CustomIP) } @@ -2318,9 +2325,9 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // the ip this request was dialed to, or else, as for a reused - // connection, -unsafe or a proxy, the last ip the dialer dialed - // to the host, which may be another request's + // the ip this request was dialed to or, if it was not dialed to + // its host, as through a proxy, the last ip the dialer dialed to + // the host, which may be another request's ip = httpx.DialedIP(req.Context()) if ip == "" { ip = hp.Dialer.GetDialedIP(URL.Host) diff --git a/runner/targetline.go b/runner/targetline.go index 57d5d837c..29631be73 100644 --- a/runner/targetline.go +++ b/runner/targetline.go @@ -22,7 +22,7 @@ import ( var errTargetLineConflict = errors.New("a target pinned to an ip cannot be probed with") // targetLine is an input line that pins a target to an ip, like -// curl --resolve: +// curl --resolve, but on every port of its host: // // {"target": "https://app.example", "ip": "192.0.2.10"} // @@ -147,32 +147,23 @@ func (r *Runner) readInputLine(line string) (string, bool) { } // normalizeInput validates an input line and returns the form it is stored -// and deduplicated in. +// and deduplicated in. A target line with an ip is refused when an option +// does not keep to the ip. func (r *Runner) normalizeInput(line string) (string, error) { line = strings.TrimSpace(line) if !isTargetLine(line) { return line, nil } - parsed, err := r.parseInputLine(line) - if err != nil { - return "", err - } - return parsed.key(), nil -} - -// parseInputLine parses a target line, and refuses one with an ip when an -// option does not keep to the ip. -func (r *Runner) parseInputLine(line string) (targetLine, error) { parsed, err := parseTargetLine(line) if err != nil { - return targetLine{}, err + return "", err } if parsed.IP != "" { if flags := r.options.targetLineConflicts(); len(flags) > 0 { - return targetLine{}, fmt.Errorf("target line %q: %w %s", line, errTargetLineConflict, strings.Join(flags, ", ")) + return "", fmt.Errorf("target line %q: %w %s", line, errTargetLineConflict, strings.Join(flags, ", ")) } } - return parsed, nil + return parsed.key(), nil } // targetLineConflicts returns the options a target line with an ip cannot be diff --git a/runner/targetline_test.go b/runner/targetline_test.go index adda85257..3a225ae5e 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -15,7 +15,6 @@ import ( "net/netip" "os" "path/filepath" - "slices" "sort" "strings" "sync" @@ -27,6 +26,7 @@ import ( "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/gologger/writer" "github.com/projectdiscovery/httpx/common/httpx" + syncutil "github.com/projectdiscovery/utils/sync" "github.com/stretchr/testify/require" "golang.org/x/net/dns/dnsmessage" ) @@ -124,29 +124,63 @@ func TestRunner_targetLineConflicts(t *testing.T) { } } +func TestOptions_ValidateOptions_probeAllIPsWithProxy(t *testing.T) { + options := &Options{ProbeAllIPS: true, Proxy: "http://127.0.0.1:8080"} + require.ErrorContains(t, options.ValidateOptions(), "-pa/-probe-all-ips cannot be used with -proxy") +} + +// -pa does not resolve a pinned target. func TestRunner_targetLine_targets(t *testing.T) { - line := `{"target":"https://a.example","ip":"192.0.2.10"}` - targetsOf := func(r *Runner) []httpx.Target { - var got []httpx.Target - for target := range r.targets(r.hp, line) { - got = append(got, target) - } - return got + r, err := New(&Options{ProbeAllIPS: true}) + require.NoError(t, err) + defer r.Close() + + var got []httpx.Target + for target := range r.targets(r.hp, `{"target":"https://a.example","ip":"192.0.2.10"}`) { + got = append(got, target) } + require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, got) +} - t.Run("-pa does not resolve a pinned target", func(t *testing.T) { - r, err := New(&Options{ProbeAllIPS: true}) +// A line given to Process is read as an input line is. +func TestRunner_Process_readsTheLine(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1") + process := func(t *testing.T, options *Options, line string) []Result { + options.Timeout = 5 + options.Methods = http.MethodGet + options.Resolvers = []string{resolver} + require.NoError(t, options.ValidateOptions()) + r, err := New(options) require.NoError(t, err) defer r.Close() - require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, targetsOf(r)) - }) - // a line from another source than the input, such as a certificate name - // with -tls-probe, is refused with an option that does not keep to the ip - t.Run("conflicts are refused for every target line", func(t *testing.T) { - r, err := New(&Options{Unsafe: true}) + wg, err := syncutil.New(syncutil.WithSize(1)) require.NoError(t, err) - defer r.Close() - require.Empty(t, targetsOf(r)) + + output := make(chan Result, 1) + r.Process(line, wg, httpx.HTTP, &r.scanopts, output) + wg.Wait() + close(output) + var results []Result + for result := range output { + results = append(results, result) + } + return results + } + + t.Run("a line without ip is the plain target", func(t *testing.T) { + results := process(t, &Options{}, `{"target": "http://pinned.test:`+port+`"}`) + require.Len(t, results, 1) + require.Equal(t, "http://pinned.test:"+port, results[0].Input) + require.Equal(t, []string{"127.0.0.1"}, results[0].A, "its host is resolved") + }) + t.Run("a line that conflicts with an option is skipped", func(t *testing.T) { + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + + require.Empty(t, process(t, &Options{Unsafe: true}, pinnedLine("http://pinned.test:"+port, "127.0.0.1"))) + require.Len(t, logged.containing("Skipping input"), 1) }) } @@ -191,7 +225,7 @@ func TestRunner_loadAndCloseFile_skipsInvalidLines(t *testing.T) { require.Equal(t, 1, count) _, ok := r.hm.Get("a.example") require.True(t, ok) - require.Len(t, logged.lines(), 2, "each skipped line is reported") + require.Len(t, logged.containing("Skipping input"), 2, "each skipped line is reported") } func TestRunner_skip_pinned(t *testing.T) { @@ -214,9 +248,10 @@ func TestRunner_skip_pinned(t *testing.T) { require.False(t, skip) } -// A proxy, which -pa can be run with, resolves the host itself, so a probe of -// a pinned target through it fails, which is reported once. A proxy set in -// the environment is tested in common/httpx. +// A proxy resolves the host itself, so a probe of a pinned target through it +// fails, which is reported once. Only the proxy of the environment can be +// used with pinned targets, but it is read once per process: the proxy of the +// options stands for it here, and common/httpx tests it. func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { logged := &errorRecorder{} gologger.DefaultLogger.SetWriter(logged) @@ -229,8 +264,7 @@ func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) require.ErrorIs(t, result.Err, httpx.ErrPinnedThroughProxy) } - require.Len(t, logged.lines(), 1) - require.Contains(t, logged.lines()[0], "NO_PROXY") + require.Len(t, logged.containing("NO_PROXY"), 1) } // The names of a certificate shaped like a target line are not probed with @@ -310,6 +344,17 @@ func TestTargetLineKeepsOtherHostsUnpinned(t *testing.T) { require.Equal(t, "one 127.0.0.1:"+port, results[0].Title) } +// The pinned host goes to the ip on every port, as through a redirect. +func TestTargetLinePinsEveryPortOfTheHost(t *testing.T) { + other := servePair(t, titleHandler("one"), titleHandler("two")) + redirect := http.RedirectHandler("http://pinned.test:"+other+"/", http.StatusFound) + port := servePair(t, titleHandler("one"), redirect) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + + require.Len(t, results, 1) + require.Equal(t, "two pinned.test:"+other, results[0].Title) +} + // With a matcher, responses are stored after matching, under a name that // must differ for each ip of a url. func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { @@ -353,18 +398,27 @@ func TestProbeAllIPsKeepsTheAddressesOfTheHost(t *testing.T) { // A plain target reports the ip it was dialed to, not the one a pinned target // of the same host was dialed to meanwhile. func TestPlainTargetReportsItsOwnIP(t *testing.T) { - slow := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(500 * time.Millisecond) - titleHandler("two").ServeHTTP(w, r) + pinnedDialed := make(chan struct{}) + var once sync.Once + one := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + once.Do(func() { close(pinnedDialed) }) + titleHandler("one").ServeHTTP(w, r) + }) + // the plain target is answered once the pinned target is dialed + two := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-pinnedDialed: + titleHandler("two").ServeHTTP(w, r) + case <-r.Context().Done(): + } }) - port := servePair(t, titleHandler("one"), slow) + port := servePair(t, one, two) resolver, _ := serveDNS(t, "pinned.test", "127.0.0.2") file := filepath.Join(t.TempDir(), "targets.txt") lines := "http://pinned.test:" + port + "\n" + pinnedLines(port, "127.0.0.1")[0] + "\n" require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) - // the delay dials the plain target first, and its slow server keeps it - // open while the pinned target of the same host is dialed + // the delay dials the plain target first options := &Options{Resolvers: []string{resolver}, Stream: true, InputFile: file, Threads: 2, Delay: 100 * time.Millisecond} var got []string for _, result := range probe(t, options) { @@ -458,10 +512,17 @@ func (w *errorRecorder) Write(data []byte, level levels.Level) { } } -func (w *errorRecorder) lines() []string { +// containing returns the errors that contain text. +func (w *errorRecorder) containing(text string) []string { w.mu.Lock() defer w.mu.Unlock() - return slices.Clone(w.errors) + var lines []string + for _, line := range w.errors { + if strings.Contains(line, text) { + lines = append(lines, line) + } + } + return lines } // selfSignedCertificate returns a certificate whose common name is name.