diff --git a/README.md b/README.md index e84dd980..6bce0ca3 100644 --- a/README.md +++ b/README.md @@ -380,6 +380,16 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas - As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable. - Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp` +- A target can be pinned to an IP with a JSON input line, like `curl --resolve` but on every port of its host: + ``` + {"target": "https://app.example.com", "ip": "192.0.2.10"} + {"target": "https://app.example.com", "ip": "192.0.2.11"} + ``` + - Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target. + - Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line is reported and skipped. + - A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host, and `-pa` cannot be used with `-proxy`. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`. + - The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately. + - Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved. - The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result. - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443` - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`) diff --git a/common/httpx/dialedip.go b/common/httpx/dialedip.go new file mode 100644 index 00000000..7ec6fb5a --- /dev/null +++ b/common/httpx/dialedip.go @@ -0,0 +1,60 @@ +package httpx + +import ( + "context" + "net" + "sync" +) + +// dialFunc dials addr, as net.Dialer.DialContext does. +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// dialedIPKey is the context key of the dialedIP of WithDialedIP. +type dialedIPKey struct{} + +// dialedIP is the ip of the last connection dialed to a host. +type dialedIP struct { + host string + mu sync.Mutex + ip string +} + +// WithDialedIP returns a copy of ctx that records the ip of the connections +// dialed to host, for DialedIP. The dialer only keeps the last ip dialed to a +// host by any request, which may be another request's. +func WithDialedIP(ctx context.Context, host string) context.Context { + return context.WithValue(ctx, dialedIPKey{}, &dialedIP{host: canonicalHost(host)}) +} + +// DialedIP returns the ip of the last connection dialed to the host of a +// WithDialedIP context, or "" if none was. +func DialedIP(ctx context.Context) string { + dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) + if !ok { + return "" + } + dialed.mu.Lock() + defer dialed.mu.Unlock() + return dialed.ip +} + +// recordDialedIP wraps dial to record the ip of a connection to the host of a +// WithDialedIP context. A connection to another host, such as a proxy, is not +// recorded. +func recordDialedIP(dial dialFunc) dialFunc { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := dial(ctx, network, addr) + dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP) + if err != nil || !ok { + return conn, err + } + host, _, _ := net.SplitHostPort(addr) + ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String()) + if canonicalHost(host) == dialed.host && ip != "" { + dialed.mu.Lock() + dialed.ip = ip + dialed.mu.Unlock() + } + return conn, err + } +} diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 91c86a49..4e6e8b67 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) { gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation") } transport := &http.Transport{ - DialContext: httpx.Dialer.Dial, + DialContext: recordDialedIP(httpx.Dialer.Dial), MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) { // Clone before protocol-specific ALPN is applied. Impersonation keeps the // previous base config; utls does not take NextProtos from it. impersonateTLS := transport.TLSClientConfig.Clone() - transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS) + transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS)) switch httpx.Options.Protocol { case HTTP11: @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) { } else { transport.Proxy = http.ProxyFromEnvironment } + transport.Proxy = refusePinned(transport.Proxy) httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{ Transport: transport, diff --git a/common/httpx/pinned.go b/common/httpx/pinned.go new file mode 100644 index 00000000..1d10220f --- /dev/null +++ b/common/httpx/pinned.go @@ -0,0 +1,68 @@ +package httpx + +import ( + "context" + "errors" + "net/http" + "net/url" + "strings" + + "github.com/projectdiscovery/fastdialer/fastdialer" + "golang.org/x/net/idna" +) + +// ErrPinnedThroughProxy is the error of a request to a pinned host that would +// go through a proxy, which resolves the host itself. +var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy") + +// pinnedHostKey is the context key of a host pinned with WithPinnedIP. +type pinnedHostKey string + +// WithPinnedIP returns a copy of ctx in which requests to host connect to ip, +// like curl --resolve, but on every port of host. The host keeps its name for +// the Host header and SNI, and requests to other hosts, such as redirect +// targets, are resolved as usual. +func WithPinnedIP(ctx context.Context, host, ip string) context.Context { + host = canonicalHost(host) + ctx = fastdialer.WithResolveOverride(ctx, host, ip) + return context.WithValue(ctx, pinnedHostKey(host), ip) +} + +// PinnedContext returns a new context in which the host of req keeps the ip it +// is pinned to, for another request derived from req. The other values of the +// context of req, such as a trace, are not kept. +func PinnedContext(req *http.Request) context.Context { + ctx := context.Background() + if ip := pinnedIP(req); ip != "" { + ctx = WithPinnedIP(ctx, req.URL.Hostname(), ip) + } + return ctx +} + +// pinnedIP returns the ip the host of req is pinned to in its context, if any. +func pinnedIP(req *http.Request) string { + ip, _ := req.Context().Value(pinnedHostKey(canonicalHost(req.URL.Hostname()))).(string) + return ip +} + +// canonicalHost folds the case, the IDNA form and a trailing dot of host, as +// fastdialer keys its resolve overrides, so that a pin matches the requests and +// dials to its host however the host is spelled. +func canonicalHost(host string) string { + if ascii, err := idna.Lookup.ToASCII(host); err == nil { + host = ascii + } + return strings.TrimSuffix(strings.ToLower(host), ".") +} + +// refusePinned wraps the proxy of a transport so that a request to a pinned +// host fails instead of going through the proxy. +func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + proxyURL, err := proxy(req) + if proxyURL != nil && pinnedIP(req) != "" { + return nil, ErrPinnedThroughProxy + } + return proxyURL, err + } +} diff --git a/common/httpx/pinned_test.go b/common/httpx/pinned_test.go new file mode 100644 index 00000000..650dd768 --- /dev/null +++ b/common/httpx/pinned_test.go @@ -0,0 +1,111 @@ +package httpx + +import ( + "context" + "net" + "net/http" + "net/http/httptrace" + "net/url" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRefusePinned(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:3128") + require.NoError(t, err) + proxy := refusePinned(http.ProxyURL(proxyURL)) + ctx := WithPinnedIP(WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10"), "xn--bcher-kva.test", "192.0.2.11") + + // each host pinned in a context, however it is spelled + for _, rawURL := range []string{"https://PINNED.test.:8443/", "https://bücher.test/"} { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + require.NoError(t, err) + _, err = proxy(req) + require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL) + } + + redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + got, err := proxy(redirect) + require.NoError(t, err) + require.Equal(t, proxyURL, got, "a request to another host keeps the proxy") + + pinned, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/", nil) + require.NoError(t, err) + got, err = refusePinned(http.ProxyURL(nil))(pinned) + require.NoError(t, err) + require.Nil(t, got, "a pinned request without a proxy") +} + +// A proxy set in the environment is refused for a pinned request as well. Go +// reads the proxies of the environment once, so the test runs in a new process. +func TestRefusePinned_environment(t *testing.T) { + if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" { + cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$") + // the lower case names first: on Windows, the last of two spellings wins + cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1") + output, err := cmd.CombinedOutput() + require.NoError(t, err, string(output)) + return + } + + options := DefaultOptions + options.RetryMax = 0 + h, err := New(&options) + require.NoError(t, err) + req, err := h.NewRequestWithContext(WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/") + require.NoError(t, err) + _, err = h.Do(req, UnsafeOptions{}) + require.ErrorIs(t, err, ErrPinnedThroughProxy) +} + +func TestCanonicalHost(t *testing.T) { + for host, want := range map[string]string{ + "Pinned.Test.": "pinned.test", + "bücher.test": "xn--bcher-kva.test", + "BÜCHER.test": "xn--bcher-kva.test", + "bücher。test": "xn--bcher-kva.test", + "xn--bcher-kva.test": "xn--bcher-kva.test", + } { + require.Equal(t, want, canonicalHost(host), host) + } +} + +func TestPinnedContext(t *testing.T) { + ctx := httptrace.WithClientTrace(WithPinnedIP(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{}) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil) + require.NoError(t, err) + + derived := req.Clone(PinnedContext(req)) + require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept") + require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not") + + other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil) + require.NoError(t, err) + require.Empty(t, pinnedIP(other.Clone(PinnedContext(other)))) +} + +func TestRecordDialedIP(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer func() { _ = listener.Close() }() + // every address is dialed to the listener + dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String()) + }) + ctx := WithDialedIP(context.Background(), "dialed.test") + + conn, err := dial(ctx, "tcp", "proxy.test:3128") + require.NoError(t, err) + _ = conn.Close() + require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy") + + // net/http dials a host as it is spelled + conn, err = dial(ctx, "tcp", "DIALED.test.:80") + require.NoError(t, err) + _ = conn.Close() + require.Equal(t, "127.0.0.1", DialedIP(ctx)) +} diff --git a/common/httpx/tls.go b/common/httpx/tls.go index 87cee744..77040ad7 100644 --- a/common/httpx/tls.go +++ b/common/httpx/tls.go @@ -63,7 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response { } // canonical net concatenation host = net.JoinHostPort(hostname, fmt.Sprint(port)) - ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout) + ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout) defer cancel() tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host) if err != nil { diff --git a/runner/main_test.go b/runner/main_test.go new file mode 100644 index 00000000..3b74a2d2 --- /dev/null +++ b/runner/main_test.go @@ -0,0 +1,23 @@ +package runner + +import ( + "os" + "strings" + "testing" +) + +// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in +// the environment, which would resolve it itself, so that they run behind a +// proxy too. Go reads the proxies of the environment once per process, so +// t.Setenv cannot change them for a test. +func TestMain(m *testing.M) { + noProxy := os.Getenv("NO_PROXY") + if noProxy == "" { + noProxy = os.Getenv("no_proxy") + } + noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",") + for _, name := range []string{"NO_PROXY", "no_proxy"} { + _ = os.Setenv(name, noProxy) + } + os.Exit(m.Run()) +} diff --git a/runner/options.go b/runner/options.go index 4f89b2c8..ca200126 100644 --- a/runner/options.go +++ b/runner/options.go @@ -768,6 +768,10 @@ func (options *Options) ValidateOptions() error { return fmt.Errorf("invalid metrics port %d, must be between 0 and 65535", options.MetricsPort) } + if options.ProbeAllIPS && options.Proxy != "" { + return errors.New("-pa/-probe-all-ips cannot be used with -proxy, which resolves the host itself") + } + if options.Silent { incompatibleFlagsList := flagsIncompatibleWithSilent(options) if len(incompatibleFlagsList) > 0 { diff --git a/runner/runner.go b/runner/runner.go index 97b9c7b8..6bbf18ba 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -33,7 +33,6 @@ import ( "github.com/happyhackingspace/dit" "github.com/mfonda/simhash" asnmap "github.com/projectdiscovery/asnmap/libs" - "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/httpx/common/authprovider" "github.com/projectdiscovery/httpx/common/customextract" "github.com/projectdiscovery/httpx/common/hashes/jarm" @@ -99,6 +98,7 @@ type Runner struct { httpApiEndpoint *Server authProvider authprovider.AuthProvider interruptCh chan struct{} + pinnedProxyWarning sync.Once } func (r *Runner) HTTPX() *httpx.HTTPX { @@ -556,6 +556,10 @@ func (r *Runner) prepareInput() { // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { + target, ok := r.readInputLine(target) + if !ok { + continue + } expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { numHosts += expandedTarget @@ -738,6 +742,18 @@ func (r *Runner) streamInput() (chan string, error) { return true } } + // sendLine normalizes, deduplicates and sends an input line, returning + // false if interrupted. + sendLine := func(item string) bool { + item, ok := r.readInputLine(item) + if !ok { + return true + } + if r.options.SkipDedupe || r.testAndSet(item) { + return trySend(item) + } + return true + } if fileutil.FileExists(r.options.InputFile) { // check if input mode is specified for special format handling @@ -760,14 +776,9 @@ func (r *Runner) streamInput() (chan string, error) { } } else { for item, err := range fileutil.Lines(r.options.InputFile) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } else if r.options.InputFile != "" { @@ -777,27 +788,17 @@ func (r *Runner) streamInput() (chan string, error) { } for _, file := range files { for item, err := range fileutil.Lines(file) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } } if fileutil.HasStdin() { for item, err := range fileutil.LinesReader(os.Stdin) { - if err != nil { + if err != nil || !sendLine(item) { return } - if r.options.SkipDedupe || r.testAndSet(item) { - if !trySend(item) { - return - } - } } } }() @@ -807,7 +808,10 @@ func (r *Runner) streamInput() (chan string, error) { func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { scanner := bufio.NewScanner(finput) for scanner.Scan() { - target := strings.TrimSpace(scanner.Text()) + target, ok := r.readInputLine(scanner.Text()) + if !ok { + continue + } // Used just to get the exact number of targets expandedTarget, err := r.countTargetFromRawTarget(target) if err == nil && expandedTarget > 0 { @@ -1540,7 +1544,7 @@ func (r *Runner) RunEnumeration() { protocol := r.options.protocol // attempt to parse url as is - if u, err := r.parseURL(k); err == nil { + if u, err := r.parseURL(targetOf(k)); err == nil { if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS { protocol = u.Scheme } @@ -1679,7 +1683,14 @@ func (r *Runner) GetScanOpts() ScanOptions { return r.scanopts } +// Process probes t, read as an input line is: an invalid line is reported and +// skipped. func (r *Runner) Process(t string, wg *syncutil.AdaptiveWaitGroup, protocol string, scanopts *ScanOptions, output chan Result) { + t, err := r.normalizeInput(t) + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return + } r.process(t, wg, r.hp, protocol, scanopts, output) } @@ -1709,15 +1720,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } if scanopts.CSPProbe && result.CSPData != nil { scanopts.CSPProbe = false @@ -1763,15 +1766,7 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { - for _, tt := range result.TLSData.SubjectAN { - if !r.testAndSet(tt) { - continue - } - r.process(tt, wg, hp, protocol, scanopts, output) - } - if r.testAndSet(result.TLSData.SubjectCN) { - r.process(result.TLSData.SubjectCN, wg, hp, protocol, scanopts, output) - } + r.probeTLSNames(result, wg, hp, protocol, scanopts, output) } }(port, target, method, wantedProtocol) } @@ -1783,6 +1778,23 @@ func (r *Runner) process(t string, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTT } } +// probeTLSNames probes the names of the certificate of result, for -tls-probe. +// A name shaped like a target line is skipped: only an input pins a target to +// an ip, not the server being probed. +func (r *Runner) probeTLSNames(result Result, wg *syncutil.AdaptiveWaitGroup, hp *httpx.HTTPX, protocol string, scanopts *ScanOptions, output chan Result) { + names := append(slices.Clone(result.TLSData.SubjectAN), result.TLSData.SubjectCN) + for _, name := range names { + if !isTargetLine(name) && r.testAndSet(name) { + r.process(name, wg, hp, protocol, scanopts, output) + } + } +} + +// isVHostInput reports whether target is a vhost,ip input. +func isVHostInput(target string) bool { + return !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ",") +} + // returns all the targets within a cidr range or the single target func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { results := make(chan httpx.Target) @@ -1792,6 +1804,14 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { target = strings.TrimSpace(target) switch { + case isTargetLine(target): + // a target line, normalized when read, pins its target to an ip + line, err := parseTargetLine(target) + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return + } + results <- httpx.Target{Host: line.Target, CustomIP: line.IP} case stringsutil.HasPrefixAny(target, "*", "."): // A valid target does not contain: // trim * and/or . (prefix) from the target to return the domain instead of wildcard @@ -1831,7 +1851,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { for _, ip := range ips { results <- httpx.Target{Host: target, CustomIP: ip} } - case !stringsutil.HasPrefixAny(target, "http://", "https://") && stringsutil.ContainsAny(target, ","): + case isVHostInput(target): idxComma := strings.Index(target, ",") results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} default: @@ -1841,6 +1861,16 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { return results } +// pinnedKey returns key for a target that is not pinned, and key@ip for one +// pinned to ip, from a target line or -pa, as the ips of a host fail and +// respond independently. +func pinnedKey(key, ip string) string { + if ip == "" { + return key + } + return key + "@" + ip +} + func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *ScanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { @@ -1867,7 +1897,7 @@ retry: } // check if we have to skip the host:port as a result of a previous failure - hostPort := net.JoinHostPort(URL.Host, URL.Port()) + hostPort := pinnedKey(net.JoinHostPort(URL.Host, URL.Port()), target.CustomIP) if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors >= r.options.HostMaxErrors { @@ -1892,20 +1922,12 @@ retry: if err := URL.MergePath(scanopts.RequestURI, scanopts.Unsafe); err != nil { gologger.Debug().Msgf("failed to merge paths of url %v and %v", URL.String(), scanopts.RequestURI) } - var ( - req *retryablehttp.Request - ctx context.Context - ) + var req *retryablehttp.Request + ctx := context.Background() if target.CustomIP != "" { - var requestIP string - if iputil.IsIPv6(target.CustomIP) { - requestIP = fmt.Sprintf("[%s]", target.CustomIP) - } else { - requestIP = target.CustomIP - } - ctx = context.WithValue(context.Background(), fastdialer.IP, requestIP) + ctx = httpx.WithPinnedIP(ctx, URL.Hostname(), target.CustomIP) } else { - ctx = context.Background() + ctx = httpx.WithDialedIP(ctx, URL.Hostname()) } req, err = hp.NewRequestWithContext(ctx, method, URL.String()) if err != nil { @@ -2049,6 +2071,11 @@ retry: builder.WriteRune(']') } if err != nil { + if errors.Is(err, httpx.ErrPinnedThroughProxy) { + r.pinnedProxyWarning.Do(func() { + gologger.Error().Msgf("Targets pinned to an ip, by a target line or -pa, cannot be probed through the proxy of HTTP_PROXY or HTTPS_PROXY, which resolves the host itself: add their hosts to NO_PROXY\n") + }) + } errString := "" errString = err.Error() splitErr := strings.Split(errString, ":") @@ -2086,6 +2113,10 @@ retry: } if r.options.Probe { + // the pinned ip tells apart the failures of the ips of a host + if target.CustomIP != "" { + _, _ = fmt.Fprintf(builder, " [%s]", target.CustomIP) + } return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err, Failed: err != nil, Error: errString, str: builder.String()} } else { return Result{URL: URL.String(), Input: origInput, Timestamp: time.Now(), Err: err} @@ -2294,8 +2325,13 @@ retry: if onlyHost, _, err := net.SplitHostPort(URL.Host); err == nil && iputil.IsIP(onlyHost) { ip = onlyHost } else { - // hp.Dialer.GetDialedIP would return only the last dialed one - ip = hp.Dialer.GetDialedIP(URL.Host) + // the ip this request was dialed to or, if it was not dialed to + // its host, as through a proxy, the last ip the dialer dialed to + // the host, which may be another request's + ip = httpx.DialedIP(req.Context()) + if ip == "" { + ip = hp.Dialer.GetDialedIP(URL.Host) + } if ip == "" { ip = hp.Dialer.GetDialedIP(onlyHost) } @@ -2327,7 +2363,7 @@ retry: } } - if scanopts.OutputIP || scanopts.ProbeAllIPS { + if scanopts.OutputIP || scanopts.ProbeAllIPS || target.CustomIP != "" { _, _ = fmt.Fprintf(builder, " [%s]", ip) } @@ -2336,9 +2372,14 @@ retry: if err != nil { onlyHost = URL.Host } - allIps, cnames, resolvers, err := getDNSData(hp, onlyHost) - if err != nil { - allIps = append(allIps, ip) + var allIps, cnames, resolvers []string + // like curl --resolve, a target line gives the ip of its host, which is + // not resolved + if !isTargetLine(origInput) { + allIps, cnames, resolvers, err = getDNSData(hp, onlyHost) + if err != nil { + allIps = append(allIps, ip) + } } var ips4, ips6 []string @@ -2505,7 +2546,7 @@ retry: } // store responses or chain in directory - domainFile := method + ":" + URL.EscapedString() + domainFile := pinnedKey(method+":"+URL.EscapedString(), target.CustomIP) hash := hashes.Sha1([]byte(domainFile)) domainResponseFile := fmt.Sprintf("%s.txt", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") @@ -2767,7 +2808,7 @@ retry: } func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) (bool, Result) { - if r.skipCDNPort(URL.Hostname(), URL.Port()) { + if r.skipCDNPort(URL.Hostname(), target.CustomIP, URL.Port()) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port()) return true, Result{URL: target.Host, Input: origInput, Err: errors.New("cdn target only allows ports 80 and 443")} } @@ -2777,6 +2818,11 @@ func (r *Runner) skip(URL *urlutil.URL, target httpx.Target, origInput string) ( return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target host is not allowed by network policy")} } + if target.CustomIP != "" && !r.hp.NetworkPolicy.Validate(target.CustomIP) { + gologger.Debug().Msgf("Skipping target due to network policy: %s (%s)\n", URL.Hostname(), target.CustomIP) + return true, Result{URL: target.Host, Input: origInput, Err: errors.New("target ip is not allowed by network policy")} + } + return false, Result{} } @@ -2827,8 +2873,8 @@ func (r *Runner) HandleFaviconHash(hp *httpx.HTTPX, req *retryablehttp.Request, } } - // Clone original request (reuse headers/cookies) - clone := req.Clone(context.Background()) + // Clone original request (reuse headers/cookies and the pinned ip) + clone := req.Clone(httpx.PinnedContext(req.Request)) var ( faviconMMH3 string @@ -3036,26 +3082,26 @@ func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint return res } -func (r *Runner) skipCDNPort(host string, port string) bool { +// skipCDNPort reports whether a target is skipped as its ip is a CDN ip and +// port is not allowed for one. The ip is the one the target is pinned to, or +// else the first ip of host. +func (r *Runner) skipCDNPort(host, ip, port string) bool { // if the option is not enabled we don't skip if !r.scanopts.ExcludeCDN { return false } - // uses the dealer to pre-resolve the target - dnsData, err := r.hp.Dialer.GetDNSData(host) - // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly - if err != nil { - return false - } - - if len(dnsData.A) == 0 { - return false + if ip == "" { + // uses the dealer to pre-resolve the target + dnsData, err := r.hp.Dialer.GetDNSData(host) + // if we get an error the target cannot be resolved, so we return false so that the program logic continues as usual and handles the errors accordingly + if err != nil || len(dnsData.A) == 0 { + return false + } + // pick the first ip as target + ip = dnsData.A[0] } - // pick the first ip as target - hostIP := dnsData.A[0] - - isCdnIP, _, _, err := r.hp.CdnCheck(hostIP) + isCdnIP, _, _, err := r.hp.CdnCheck(ip) if err != nil { return false } diff --git a/runner/targetline.go b/runner/targetline.go new file mode 100644 index 00000000..29631be7 --- /dev/null +++ b/runner/targetline.go @@ -0,0 +1,199 @@ +package runner + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/httpx/common/httpx" + "github.com/projectdiscovery/mapcidr/asn" + iputil "github.com/projectdiscovery/utils/ip" + stringsutil "github.com/projectdiscovery/utils/strings" + urlutil "github.com/projectdiscovery/utils/url" +) + +// errTargetLineConflict is the error of a target line with an ip and an option +// that does not keep to the ip. +var errTargetLineConflict = errors.New("a target pinned to an ip cannot be probed with") + +// targetLine is an input line that pins a target to an ip, like +// curl --resolve, but on every port of its host: +// +// {"target": "https://app.example", "ip": "192.0.2.10"} +// +// The target keeps its hostname for the Host header and SNI, while the +// connection goes to the ip. A line without ip is the plain target. +type targetLine struct { + Target string `json:"target"` + IP string `json:"ip,omitempty"` +} + +// isTargetLine reports whether an input line is a JSON object, the form of a +// target line. +func isTargetLine(line string) bool { + return strings.HasPrefix(strings.TrimSpace(line), "{") +} + +// parseTargetLine decodes and validates a target line. +func parseTargetLine(line string) (targetLine, error) { + parsed, err := decodeTargetLine(line) + if err != nil { + return targetLine{}, fmt.Errorf("invalid target line %q: %w", line, err) + } + return parsed, nil +} + +// decodeTargetLine decodes a target line, or returns why it is invalid. +// Unknown keys and an ip that is null or empty are rejected, so that a typo +// cannot silently probe the host through DNS. +func decodeTargetLine(line string) (targetLine, error) { + // the ip is kept raw to tell an ip that is null or empty from no ip + var decoded struct { + Target string `json:"target"` + IP json.RawMessage `json:"ip"` + } + decoder := json.NewDecoder(strings.NewReader(line)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&decoded); err != nil { + return targetLine{}, err + } + if _, err := decoder.Token(); !errors.Is(err, io.EOF) { + return targetLine{}, errors.New("unexpected data after the object") + } + + target := strings.TrimSpace(decoded.Target) + if target == "" { + return targetLine{}, errors.New("no target") + } + if decoded.IP == nil { + return targetLine{Target: target}, nil + } + if err := checkPinnable(target); err != nil { + return targetLine{}, err + } + var ip string + if err := json.Unmarshal(decoded.IP, &ip); err != nil { + return targetLine{}, err + } + addr, err := netip.ParseAddr(strings.TrimSpace(ip)) + if err != nil || addr.Zone() != "" || addr.IsUnspecified() { + return targetLine{}, fmt.Errorf("invalid ip %q", ip) + } + return targetLine{Target: target, IP: addr.Unmap().String()}, nil +} + +// checkPinnable returns why target cannot be pinned to an ip: only a single +// target with a hostname can. +func checkPinnable(target string) error { + switch { + case iputil.IsCIDR(target), asn.IsASN(target): + return errors.New("a CIDR or ASN target cannot be pinned to an ip") + case stringsutil.HasPrefixAny(target, "*", "."): + return errors.New("a wildcard target cannot be pinned to an ip") + case isVHostInput(target): + return errors.New("a vhost,ip target cannot be pinned to an ip") + } + parsed, err := urlutil.Parse(target) + if err != nil || parsed.Hostname() == "" || iputil.IsIP(parsed.Hostname()) { + return errors.New("a target pinned to an ip must have a hostname") + } + return nil +} + +// key returns the line as it is stored and deduplicated: the target alone +// when it is not pinned, and a canonical target line otherwise. +func (t targetLine) key() string { + if t.IP == "" { + return t.Target + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(t) + return strings.TrimSpace(buf.String()) +} + +// targetOf returns the target of an input line: the line itself, or the +// target of a target line. +func targetOf(line string) string { + if !isTargetLine(line) { + return line + } + if parsed, err := parseTargetLine(line); err == nil { + return parsed.Target + } + return line +} + +// readInputLine returns the form an input line is stored and deduplicated in, +// and false if the line is skipped: an invalid line is reported and skipped, +// while a target line that conflicts with the options stops the run, as every +// such line would. +func (r *Runner) readInputLine(line string) (string, bool) { + key, err := r.normalizeInput(line) + if errors.Is(err, errTargetLineConflict) { + gologger.Fatal().Msgf("Could not read input: %s\n", err) + } + if err != nil { + gologger.Error().Msgf("Skipping input: %s\n", err) + return "", false + } + return key, true +} + +// normalizeInput validates an input line and returns the form it is stored +// and deduplicated in. A target line with an ip is refused when an option +// does not keep to the ip. +func (r *Runner) normalizeInput(line string) (string, error) { + line = strings.TrimSpace(line) + if !isTargetLine(line) { + return line, nil + } + parsed, err := parseTargetLine(line) + if err != nil { + return "", err + } + if parsed.IP != "" { + if flags := r.options.targetLineConflicts(); len(flags) > 0 { + return "", fmt.Errorf("target line %q: %w %s", line, errTargetLineConflict, strings.Join(flags, ", ")) + } + } + return parsed.key(), nil +} + +// targetLineConflicts returns the options a target line with an ip cannot be +// probed with: their probes do not connect through the ip, or share a +// connection between the ips of a host, as HTTP/2 does. +func (options *Options) targetLineConflicts() []string { + var flags []string + if options.Unsafe { + flags = append(flags, "-unsafe") + } + if options.Pipeline { + flags = append(flags, "-pipeline") + } + if options.HTTP2Probe { + flags = append(flags, "-http2") + } + if strings.EqualFold(options.Protocol, string(httpx.HTTP2)) { + flags = append(flags, "-pr http2") + } + if options.Jarm { + flags = append(flags, "-jarm") + } + if options.Screenshot { + flags = append(flags, "-screenshot") + } + if options.VHostInput { + flags = append(flags, "-vhost-input") + } + if options.Proxy != "" { + flags = append(flags, "-proxy") + } + return flags +} diff --git a/runner/targetline_test.go b/runner/targetline_test.go new file mode 100644 index 00000000..3a225ae5 --- /dev/null +++ b/runner/targetline_test.go @@ -0,0 +1,730 @@ +package runner + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/gologger/writer" + "github.com/projectdiscovery/httpx/common/httpx" + syncutil "github.com/projectdiscovery/utils/sync" + "github.com/stretchr/testify/require" + "golang.org/x/net/dns/dnsmessage" +) + +func TestParseTargetLine(t *testing.T) { + valid := map[string]struct { + line string + want targetLine + }{ + "ipv4": {`{"target": "https://a.example", "ip": "192.0.2.10"}`, targetLine{Target: "https://a.example", IP: "192.0.2.10"}}, + "ipv6": {`{"target": " a.example:8443 ", "ip": "2001:DB8::1"}`, targetLine{Target: "a.example:8443", IP: "2001:db8::1"}}, + "ipv4-mapped": {`{"target": "a.example", "ip": "::ffff:192.0.2.10"}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "spaced ip": {`{"target": "a.example", "ip": " 192.0.2.10 "}`, targetLine{Target: "a.example", IP: "192.0.2.10"}}, + "no ip": {`{"target": "a.example"}`, targetLine{Target: "a.example"}}, + "escaped target": {`{"target": "https://a.example/?a=1&b=2"}`, targetLine{Target: "https://a.example/?a=1&b=2"}}, + } + for name, tc := range valid { + t.Run(name, func(t *testing.T) { + got, err := parseTargetLine(tc.line) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + + invalid := map[string]struct{ line, reason string }{ + "malformed json": {`{"target": "a.example"`, "unexpected EOF"}, + "trailing data": {`{"target": "a.example"} x`, "unexpected data after the object"}, + "unknown key": {`{"target": "a.example", "ips": "192.0.2.10"}`, `unknown field "ips"`}, + "missing target": {`{"ip": "192.0.2.10"}`, "no target"}, + "invalid ip": {`{"target": "a.example", "ip": "192.0.2"}`, `invalid ip "192.0.2"`}, + "null ip": {`{"target": "a.example", "ip": null}`, `invalid ip ""`}, + "empty ip": {`{"target": "a.example", "ip": ""}`, `invalid ip ""`}, + "number ip": {`{"target": "a.example", "ip": 1}`, "cannot unmarshal number"}, + "ip with zone": {`{"target": "a.example", "ip": "fe80::1%eth0"}`, "invalid ip"}, + "unspecified ip": {`{"target": "a.example", "ip": "0.0.0.0"}`, "invalid ip"}, + "ip target": {`{"target": "https://192.0.2.1", "ip": "192.0.2.10"}`, "must have a hostname"}, + "cidr target": {`{"target": "192.0.2.0/24", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "asn target": {`{"target": "AS13335", "ip": "192.0.2.10"}`, "CIDR or ASN"}, + "wildcard": {`{"target": "*.a.example", "ip": "192.0.2.10"}`, "wildcard"}, + "vhost form": {`{"target": "a.example,192.0.2.1", "ip": "192.0.2.10"}`, "vhost,ip"}, + } + for name, tc := range invalid { + t.Run(name, func(t *testing.T) { + _, err := parseTargetLine(tc.line) + require.ErrorContains(t, err, "invalid target line") + require.ErrorContains(t, err, tc.reason) + }) + } +} + +func TestRunner_normalizeInput(t *testing.T) { + r := &Runner{options: &Options{}} + + t.Run("equivalent lines are stored once", func(t *testing.T) { + for _, line := range []string{ + `{"target": "https://a.example", "ip": "192.0.2.10"}`, + ` {"ip":"::ffff:192.0.2.10","target":"https://a.example"}`, + } { + key, err := r.normalizeInput(line) + require.NoError(t, err) + require.Equal(t, `{"target":"https://a.example","ip":"192.0.2.10"}`, key) + } + }) + t.Run("a line without ip is the plain target", func(t *testing.T) { + key, err := r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err) + require.Equal(t, "https://a.example", key) + }) + t.Run("an invalid line", func(t *testing.T) { + _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2"}`) + require.ErrorContains(t, err, "invalid ip") + }) +} + +func TestRunner_targetLineConflicts(t *testing.T) { + conflicts := map[string]Options{ + "-unsafe": {Unsafe: true}, + "-pipeline": {Pipeline: true}, + "-http2": {HTTP2Probe: true}, + "-pr http2": {Protocol: "HTTP2"}, + "-jarm": {Jarm: true}, + "-screenshot": {Screenshot: true}, + "-vhost-input": {VHostInput: true}, + "-proxy": {Proxy: "http://127.0.0.1:8080"}, + } + for flag, options := range conflicts { + t.Run(flag, func(t *testing.T) { + r := &Runner{options: &options} + _, err := r.normalizeInput(`{"target": "https://a.example", "ip": "192.0.2.10"}`) + require.ErrorIs(t, err, errTargetLineConflict) + require.ErrorContains(t, err, "cannot be probed with "+flag) + _, err = r.normalizeInput(`{"target": "https://a.example"}`) + require.NoError(t, err, "only a line with an ip conflicts") + }) + } +} + +func TestOptions_ValidateOptions_probeAllIPsWithProxy(t *testing.T) { + options := &Options{ProbeAllIPS: true, Proxy: "http://127.0.0.1:8080"} + require.ErrorContains(t, options.ValidateOptions(), "-pa/-probe-all-ips cannot be used with -proxy") +} + +// -pa does not resolve a pinned target. +func TestRunner_targetLine_targets(t *testing.T) { + r, err := New(&Options{ProbeAllIPS: true}) + require.NoError(t, err) + defer r.Close() + + var got []httpx.Target + for target := range r.targets(r.hp, `{"target":"https://a.example","ip":"192.0.2.10"}`) { + got = append(got, target) + } + require.Equal(t, []httpx.Target{{Host: "https://a.example", CustomIP: "192.0.2.10"}}, got) +} + +// A line given to Process is read as an input line is. +func TestRunner_Process_readsTheLine(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + resolver, _ := serveDNS(t, "pinned.test", "127.0.0.1") + process := func(t *testing.T, options *Options, line string) []Result { + options.Timeout = 5 + options.Methods = http.MethodGet + options.Resolvers = []string{resolver} + require.NoError(t, options.ValidateOptions()) + r, err := New(options) + require.NoError(t, err) + defer r.Close() + wg, err := syncutil.New(syncutil.WithSize(1)) + require.NoError(t, err) + + output := make(chan Result, 1) + r.Process(line, wg, httpx.HTTP, &r.scanopts, output) + wg.Wait() + close(output) + var results []Result + for result := range output { + results = append(results, result) + } + return results + } + + t.Run("a line without ip is the plain target", func(t *testing.T) { + results := process(t, &Options{}, `{"target": "http://pinned.test:`+port+`"}`) + require.Len(t, results, 1) + require.Equal(t, "http://pinned.test:"+port, results[0].Input) + require.Equal(t, []string{"127.0.0.1"}, results[0].A, "its host is resolved") + }) + t.Run("a line that conflicts with an option is skipped", func(t *testing.T) { + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + + require.Empty(t, process(t, &Options{Unsafe: true}, pinnedLine("http://pinned.test:"+port, "127.0.0.1"))) + require.Len(t, logged.containing("Skipping input"), 1) + }) +} + +func TestRunner_streamInput_skipsInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "a.example\n" + + `{"target": "b.example"` + "\n" + + `{"target": "c.example", "ip": "192.0.2.10"}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + + r, err := New(&Options{Stream: true, InputFile: file}) + require.NoError(t, err) + defer r.Close() + out, err := r.streamInput() + require.NoError(t, err) + + var got []string + for item := range out { + got = append(got, item) + } + require.Equal(t, []string{"a.example", `{"target":"c.example","ip":"192.0.2.10"}`}, got, "the malformed line is skipped") +} + +// An invalid line, such as a line of another tool's JSON output, is skipped +// and the other lines are read. +func TestRunner_loadAndCloseFile_skipsInvalidLines(t *testing.T) { + file := filepath.Join(t.TempDir(), "targets.txt") + lines := "a.example\n" + `{"host": "b.example"}` + "\n" + `{"target": "c.example", "ip": null}` + "\n" + require.NoError(t, os.WriteFile(file, []byte(lines), 0o600)) + input, err := os.Open(file) + require.NoError(t, err) + + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + + r, err := New(&Options{}) + require.NoError(t, err) + defer r.Close() + count, err := r.loadAndCloseFile(input) + require.NoError(t, err) + require.Equal(t, 1, count) + _, ok := r.hm.Get("a.example") + require.True(t, ok) + require.Len(t, logged.containing("Skipping input"), 2, "each skipped line is reported") +} + +func TestRunner_skip_pinned(t *testing.T) { + r, err := New(&Options{Exclude: []string{"cdn", "192.0.2.10"}}) + require.NoError(t, err) + defer r.Close() + URL, err := r.parseURL("http://pinned.test:8443") + require.NoError(t, err) + + // pinned.test does not resolve: only the pinned ip can be checked + skip, result := r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "151.101.1.1"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "cdn target") + + skip, result = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.10"}, "") + require.True(t, skip) + require.ErrorContains(t, result.Err, "not allowed by network policy") + + skip, _ = r.skip(URL, httpx.Target{Host: "http://pinned.test:8443", CustomIP: "192.0.2.11"}, "") + require.False(t, skip) +} + +// A proxy resolves the host itself, so a probe of a pinned target through it +// fails, which is reported once. Only the proxy of the environment can be +// used with pinned targets, but it is read once per process: the proxy of the +// options stands for it here, and common/httpx tests it. +func TestRunner_analyze_pinnedThroughProxy(t *testing.T) { + logged := &errorRecorder{} + gologger.DefaultLogger.SetWriter(logged) + t.Cleanup(func() { gologger.DefaultLogger.SetWriter(writer.NewCLI()) }) + r, err := New(&Options{Proxy: "http://127.0.0.1:1", HostMaxErrors: -1}) + require.NoError(t, err) + defer r.Close() + + for range 2 { + result := analyzeHTTP(r, httpx.Target{Host: "http://pinned.test", CustomIP: "127.0.0.1"}) + require.ErrorIs(t, result.Err, httpx.ErrPinnedThroughProxy) + } + require.Len(t, logged.containing("NO_PROXY"), 1) +} + +// The names of a certificate shaped like a target line are not probed with +// -tls-probe: the server being probed cannot pin a target to an ip. +func TestRunner_probeTLSNames_skipsTargetLines(t *testing.T) { + var requests atomic.Int32 + // the names are probed over the scheme of the target they come from + pinned := serve(t, listen(t, "127.0.0.1:0"), http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + }), &tls.Config{}) + // the leading space is trimmed when the name is read as an input + certificate := selfSignedCertificate(t, " "+pinnedLine("https://pinned.test:"+portOf(t, pinned.Listener), "127.0.0.1")) + server := serve(t, listen(t, "127.0.0.1:0"), titleHandler("tls"), &tls.Config{Certificates: []tls.Certificate{certificate}}) + + results := probe(t, &Options{TLSProbe: true, TLSGrab: true, Threads: 5}, server.URL) + require.NotEmpty(t, results) + require.Zero(t, requests.Load()) +} + +// -maxhr counts the failures of each ip of a host apart. +func TestRunner_analyze_hostErrorsPerIP(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + r, err := New(&Options{HostMaxErrors: 1, Timeout: 2}) + require.NoError(t, err) + defer r.Close() + target := func(ip string) httpx.Target { + return httpx.Target{Host: "http://pinned.test:" + port, CustomIP: ip} + } + + require.Error(t, analyzeHTTP(r, target("127.0.0.3")).Err, "nothing listens there") + require.ErrorContains(t, analyzeHTTP(r, target("127.0.0.3")).Err, "previously unresponsive") + require.NoError(t, analyzeHTTP(r, target("127.0.0.1")).Err, "the other ip of the host is not skipped") +} + +func TestTargetLinePinsTheConnection(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + results := probe(t, &Options{}, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + var got []string + for _, result := range results { + got = append(got, result.HostIP+" "+result.Title) + require.Contains(t, result.str, "["+result.HostIP+"]", "the output shows the ip") + } + sort.Strings(got) + require.Equal(t, []string{ + "127.0.0.1 one pinned.test:" + port, + "127.0.0.2 two pinned.test:" + port, + }, got) +} + +// The scheme of the target of a line is kept, as for a url given as is. +func TestTargetLineKeepsTheScheme(t *testing.T) { + port := serveLocal(t, titleHandler("one")) + results := probe(t, &Options{}, pinnedLine("https://pinned.test:"+port, "127.0.0.1")) + require.Empty(t, results, "a plain http server does not answer https") +} + +func TestTargetLineDeduplicatesEquivalentLines(t *testing.T) { + target := "http://pinned.test:" + serveLocal(t, titleHandler("one")) + results := probe(t, &Options{}, + pinnedLine(target, "127.0.0.1"), + fmt.Sprintf(` {"ip": "::ffff:127.0.0.1", "target": %q}`, target), + ) + require.Len(t, results, 1) +} + +// Only the pinned host goes to the ip: a redirect to another host does not. +func TestTargetLineKeepsOtherHostsUnpinned(t *testing.T) { + redirect := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, port, _ := net.SplitHostPort(r.Host) + http.Redirect(w, r, "http://127.0.0.1:"+port+"/", http.StatusFound) + }) + port := servePair(t, titleHandler("one"), redirect) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + + require.Len(t, results, 1) + require.Equal(t, "one 127.0.0.1:"+port, results[0].Title) +} + +// The pinned host goes to the ip on every port, as through a redirect. +func TestTargetLinePinsEveryPortOfTheHost(t *testing.T) { + other := servePair(t, titleHandler("one"), titleHandler("two")) + redirect := http.RedirectHandler("http://pinned.test:"+other+"/", http.StatusFound) + port := servePair(t, titleHandler("one"), redirect) + results := probe(t, &Options{FollowRedirects: true, MaxRedirects: 10}, pinnedLines(port, "127.0.0.2")...) + + require.Len(t, results, 1) + require.Equal(t, "two pinned.test:"+other, results[0].Title) +} + +// With a matcher, responses are stored after matching, under a name that +// must differ for each ip of a url. +func TestTargetLineStoresTheResponseOfEachIP(t *testing.T) { + port := servePair(t, titleHandler("one"), titleHandler("two")) + options := &Options{StoreResponse: true, StoreResponseDir: t.TempDir(), OutputMatchStatusCode: "200"} + results := probe(t, options, pinnedLines(port, "127.0.0.1", "127.0.0.2")...) + + require.Len(t, results, 2) + for _, result := range results { + data, err := os.ReadFile(result.StoredResponsePath) + require.NoError(t, err) + require.Contains(t, string(data), "