Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
500a560
feat(runner): probe a target through the ip of a JSON input line
mazzz3r Oct 10, 2026
58ba3c8
chore(deps): update the fastdialer replace to v0.5.25
mazzz3r Oct 13, 2026
b18dc8a
Merge branch 'dev' into feat/target-line-ip
mazzz3r Oct 19, 2026
d29d531
chore(deps): point the fastdialer replace at the upstream commit
mazzz3r Oct 22, 2026
2489982
Merge branch 'dev' into feat/target-line-ip
mazzz3r Oct 27, 2026
2bb9dae
test(runner): probe a target line pinned to an IPv6 ip
mazzz3r Oct 27, 2026
39f1b43
fix(runner): store the response of each pinned ip apart
mazzz3r Oct 27, 2026
299c575
fix(runner): check the pinned ip for -e cdn and refuse env-proxied lines
mazzz3r Oct 27, 2026
fce694a
fix(httpx): fail a pinned request that would go through a proxy
mazzz3r Oct 27, 2026
06d914a
refactor(httpx): key each pinned host in the context
mazzz3r Oct 27, 2026
86c00f8
fix(runner): do not resolve the host of a pinned target
mazzz3r Oct 27, 2026
93291f5
fix(runner): refuse pinned lines with -pr http2
mazzz3r Oct 27, 2026
150e787
fix(runner): keep the pinned ip out of a and aaaa
mazzz3r Oct 27, 2026
b5908a3
fix(runner): reject a target line whose ip is null or empty
mazzz3r Oct 27, 2026
3ec8688
fix(runner): address the review of pinned targets
mazzz3r Oct 27, 2026
29454e4
fix(runner): address the second review of pinned targets
mazzz3r Oct 28, 2026
8bfab43
fix(runner): skip an invalid target line instead of stopping the run
mazzz3r Oct 28, 2026
f7de334
fix(runner): read a line given to Process as an input line
mazzz3r Oct 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,16 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas

- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
- A target can be pinned to an IP with a JSON input line, like `curl --resolve` but on every port of its host:
```
{"target": "https://app.example.com", "ip": "192.0.2.10"}
{"target": "https://app.example.com", "ip": "192.0.2.11"}
```
- Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target.
- Such lines must come from `-l` or stdin, as `-u` splits its value on commas. An invalid line is reported and skipped.
- A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host, and `-pa` cannot be used with `-proxy`. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`.
- The IPs of a host are kept apart, with `-pa` too: `-maxhr` counts their failures and `-sr` stores their responses separately.
- Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved.
- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
Expand Down
60 changes: 60 additions & 0 deletions common/httpx/dialedip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package httpx

import (
"context"
"net"
"sync"
)

// dialFunc dials addr, as net.Dialer.DialContext does.
type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)

// dialedIPKey is the context key of the dialedIP of WithDialedIP.
type dialedIPKey struct{}

// dialedIP is the ip of the last connection dialed to a host.
type dialedIP struct {
host string
mu sync.Mutex
ip string
}

// WithDialedIP returns a copy of ctx that records the ip of the connections
// dialed to host, for DialedIP. The dialer only keeps the last ip dialed to a
// host by any request, which may be another request's.
func WithDialedIP(ctx context.Context, host string) context.Context {
return context.WithValue(ctx, dialedIPKey{}, &dialedIP{host: canonicalHost(host)})
}

// DialedIP returns the ip of the last connection dialed to the host of a
// WithDialedIP context, or "" if none was.
func DialedIP(ctx context.Context) string {
dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP)
if !ok {
return ""
}
dialed.mu.Lock()
defer dialed.mu.Unlock()
return dialed.ip
}

// recordDialedIP wraps dial to record the ip of a connection to the host of a
// WithDialedIP context. A connection to another host, such as a proxy, is not
// recorded.
func recordDialedIP(dial dialFunc) dialFunc {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dial(ctx, network, addr)
dialed, ok := ctx.Value(dialedIPKey{}).(*dialedIP)
if err != nil || !ok {
return conn, err
}
host, _, _ := net.SplitHostPort(addr)
ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
if canonicalHost(host) == dialed.host && ip != "" {
dialed.mu.Lock()
dialed.ip = ip
dialed.mu.Unlock()
}
return conn, err
}
}
5 changes: 3 additions & 2 deletions common/httpx/httpx.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) {
gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation")
}
transport := &http.Transport{
DialContext: httpx.Dialer.Dial,
DialContext: recordDialedIP(httpx.Dialer.Dial),
MaxIdleConnsPerHost: -1,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
Expand All @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) {
// Clone before protocol-specific ALPN is applied. Impersonation keeps the
// previous base config; utls does not take NextProtos from it.
impersonateTLS := transport.TLSClientConfig.Clone()
transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS)
transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS))

switch httpx.Options.Protocol {
case HTTP11:
Expand Down Expand Up @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) {
} else {
transport.Proxy = http.ProxyFromEnvironment
}
transport.Proxy = refusePinned(transport.Proxy)

httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{
Transport: transport,
Expand Down
68 changes: 68 additions & 0 deletions common/httpx/pinned.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package httpx

import (
"context"
"errors"
"net/http"
"net/url"
"strings"

"github.com/projectdiscovery/fastdialer/fastdialer"
"golang.org/x/net/idna"
)

// ErrPinnedThroughProxy is the error of a request to a pinned host that would
// go through a proxy, which resolves the host itself.
var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy")

// pinnedHostKey is the context key of a host pinned with WithPinnedIP.
type pinnedHostKey string

// WithPinnedIP returns a copy of ctx in which requests to host connect to ip,
// like curl --resolve, but on every port of host. The host keeps its name for
// the Host header and SNI, and requests to other hosts, such as redirect
// targets, are resolved as usual.
func WithPinnedIP(ctx context.Context, host, ip string) context.Context {
host = canonicalHost(host)
ctx = fastdialer.WithResolveOverride(ctx, host, ip)
return context.WithValue(ctx, pinnedHostKey(host), ip)
}

// PinnedContext returns a new context in which the host of req keeps the ip it
// is pinned to, for another request derived from req. The other values of the
// context of req, such as a trace, are not kept.
func PinnedContext(req *http.Request) context.Context {
ctx := context.Background()
if ip := pinnedIP(req); ip != "" {
ctx = WithPinnedIP(ctx, req.URL.Hostname(), ip)
}
return ctx
}

// pinnedIP returns the ip the host of req is pinned to in its context, if any.
func pinnedIP(req *http.Request) string {
ip, _ := req.Context().Value(pinnedHostKey(canonicalHost(req.URL.Hostname()))).(string)
return ip
}

// canonicalHost folds the case, the IDNA form and a trailing dot of host, as
// fastdialer keys its resolve overrides, so that a pin matches the requests and
// dials to its host however the host is spelled.
func canonicalHost(host string) string {
if ascii, err := idna.Lookup.ToASCII(host); err == nil {
host = ascii
}
return strings.TrimSuffix(strings.ToLower(host), ".")
}

// refusePinned wraps the proxy of a transport so that a request to a pinned
// host fails instead of going through the proxy.
func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) {
return func(req *http.Request) (*url.URL, error) {
proxyURL, err := proxy(req)
if proxyURL != nil && pinnedIP(req) != "" {
return nil, ErrPinnedThroughProxy
}
return proxyURL, err
}
}
111 changes: 111 additions & 0 deletions common/httpx/pinned_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package httpx

import (
"context"
"net"
"net/http"
"net/http/httptrace"
"net/url"
"os"
"os/exec"
"testing"

"github.com/stretchr/testify/require"
)

func TestRefusePinned(t *testing.T) {
proxyURL, err := url.Parse("http://127.0.0.1:3128")
require.NoError(t, err)
proxy := refusePinned(http.ProxyURL(proxyURL))
ctx := WithPinnedIP(WithPinnedIP(context.Background(), "Pinned.Test.", "192.0.2.10"), "xn--bcher-kva.test", "192.0.2.11")

// each host pinned in a context, however it is spelled
for _, rawURL := range []string{"https://PINNED.test.:8443/", "https://bücher.test/"} {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
require.NoError(t, err)
_, err = proxy(req)
require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL)
}

redirect, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://other.test/", nil)
require.NoError(t, err)
got, err := proxy(redirect)
require.NoError(t, err)
require.Equal(t, proxyURL, got, "a request to another host keeps the proxy")

pinned, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/", nil)
require.NoError(t, err)
got, err = refusePinned(http.ProxyURL(nil))(pinned)
require.NoError(t, err)
require.Nil(t, got, "a pinned request without a proxy")
}

// A proxy set in the environment is refused for a pinned request as well. Go
// reads the proxies of the environment once, so the test runs in a new process.
func TestRefusePinned_environment(t *testing.T) {
if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" {
cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$")
// the lower case names first: on Windows, the last of two spellings wins
cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1")
output, err := cmd.CombinedOutput()
require.NoError(t, err, string(output))
return
}

options := DefaultOptions
options.RetryMax = 0
h, err := New(&options)
require.NoError(t, err)
req, err := h.NewRequestWithContext(WithPinnedIP(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/")
require.NoError(t, err)
_, err = h.Do(req, UnsafeOptions{})
require.ErrorIs(t, err, ErrPinnedThroughProxy)
}

func TestCanonicalHost(t *testing.T) {
for host, want := range map[string]string{
"Pinned.Test.": "pinned.test",
"bücher.test": "xn--bcher-kva.test",
"BÜCHER.test": "xn--bcher-kva.test",
"bücher。test": "xn--bcher-kva.test",
"xn--bcher-kva.test": "xn--bcher-kva.test",
} {
require.Equal(t, want, canonicalHost(host), host)
}
}

func TestPinnedContext(t *testing.T) {
ctx := httptrace.WithClientTrace(WithPinnedIP(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{})
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil)
require.NoError(t, err)

derived := req.Clone(PinnedContext(req))
require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept")
require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not")

other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil)
require.NoError(t, err)
require.Empty(t, pinnedIP(other.Clone(PinnedContext(other))))
}

func TestRecordDialedIP(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = listener.Close() }()
// every address is dialed to the listener
dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String())
})
ctx := WithDialedIP(context.Background(), "dialed.test")

conn, err := dial(ctx, "tcp", "proxy.test:3128")
require.NoError(t, err)
_ = conn.Close()
require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy")

// net/http dials a host as it is spelled
conn, err = dial(ctx, "tcp", "DIALED.test.:80")
require.NoError(t, err)
_ = conn.Close()
require.Equal(t, "127.0.0.1", DialedIP(ctx))
}
2 changes: 1 addition & 1 deletion common/httpx/tls.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response {
}
// canonical net concatenation
host = net.JoinHostPort(hostname, fmt.Sprint(port))
ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout)
ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout)
defer cancel()
tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host)
if err != nil {
Expand Down
23 changes: 23 additions & 0 deletions runner/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package runner

import (
"os"
"strings"
"testing"
)

// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in
// the environment, which would resolve it itself, so that they run behind a
// proxy too. Go reads the proxies of the environment once per process, so
// t.Setenv cannot change them for a test.
func TestMain(m *testing.M) {
noProxy := os.Getenv("NO_PROXY")
if noProxy == "" {
noProxy = os.Getenv("no_proxy")
}
noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",")
for _, name := range []string{"NO_PROXY", "no_proxy"} {
_ = os.Setenv(name, noProxy)
}
os.Exit(m.Run())
}
4 changes: 4 additions & 0 deletions runner/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,10 @@ func (options *Options) ValidateOptions() error {
return fmt.Errorf("invalid metrics port %d, must be between 0 and 65535", options.MetricsPort)
}

if options.ProbeAllIPS && options.Proxy != "" {
return errors.New("-pa/-probe-all-ips cannot be used with -proxy, which resolves the host itself")
}

if options.Silent {
incompatibleFlagsList := flagsIncompatibleWithSilent(options)
if len(incompatibleFlagsList) > 0 {
Expand Down
Loading
Loading