Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,17 @@ echo target.com | httpx -path '/.well-known/security.txt,/.well-known/change-pas

- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
- Burp Suite XML exports can be used as input with `-l burp-export.xml -im burp`
- A target can be pinned to an IP with a JSON input line, in the format of nuclei target lines, like `curl --resolve` but on every port of its host:
```
{"target": "https://app.example.com", "ip": "192.0.2.10"}
{"target": "https://app.example.com", "ip": "192.0.2.11"}
```
- Every connection to the hostname, redirects included, goes to the IP, which `host_ip` reports and the output shows. The hostname is kept for the Host header and SNI and is not resolved. A host can be listed with several IPs, and a line without `ip` is the plain target.
- The keys nuclei uses to select templates, such as `tags`, are ignored, and any other key makes the line invalid. An invalid line is reported and skipped. The `input` of a result is the line in a canonical form.
- `-u` splits its value on commas, so pass such lines in a file (`-l`) or on stdin.
- A line with an IP stops the run with `-proxy`, `-pr http2`, `-unsafe`, `-pipeline`, `-http2`, `-jarm`, `-screenshot` and `-vhost-input`, as they do not connect through the IP or, for HTTP/2, share a connection between the IPs of a host. A proxy set with `HTTP_PROXY`/`HTTPS_PROXY` resolves the host itself too, so a probe through it fails: add the pinned hosts to `NO_PROXY`.
- `-pa` probes each IP of a host the same way, so it cannot be used with `-proxy` either. The IPs of a host are kept apart: `-maxhr` counts their failures and `-sr` stores their responses separately.
- Hostnames found by `-tls-probe` and `-csp-probe`, the pinned one included, are probed as new targets and resolved.
- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
- Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
Expand Down
5 changes: 3 additions & 2 deletions common/httpx/httpx.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func New(options *Options) (*HTTPX, error) {
gologger.Warning().Msg("HTTP/3 is experimental and currently falls back to default TLS negotiation")
}
transport := &http.Transport{
DialContext: httpx.Dialer.Dial,
DialContext: recordDialedIP(httpx.Dialer.Dial),
MaxIdleConnsPerHost: -1,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
Expand All @@ -140,7 +140,7 @@ func New(options *Options) (*HTTPX, error) {
// Clone before protocol-specific ALPN is applied. Impersonation keeps the
// previous base config; utls does not take NextProtos from it.
impersonateTLS := transport.TLSClientConfig.Clone()
transport.DialTLSContext = httpx.buildTLSDialer(options, impersonateTLS)
transport.DialTLSContext = recordDialedIP(httpx.buildTLSDialer(options, impersonateTLS))

switch httpx.Options.Protocol {
case HTTP11:
Expand Down Expand Up @@ -176,6 +176,7 @@ func New(options *Options) (*HTTPX, error) {
} else {
transport.Proxy = http.ProxyFromEnvironment
}
transport.Proxy = refusePinned(transport.Proxy)

httpx.client = retryablehttp.NewWithHTTPClient(&http.Client{
Transport: transport,
Expand Down
126 changes: 126 additions & 0 deletions common/httpx/requesthost.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package httpx

import (
"context"
"errors"
"net"
"net/http"
"net/url"
"strings"
"sync"

"github.com/projectdiscovery/fastdialer/fastdialer"
"golang.org/x/net/idna"
)

// ErrPinnedThroughProxy is the error of a request to a pinned host that would
// go through a proxy, which resolves the host itself.
var ErrPinnedThroughProxy = errors.New("a host pinned to an ip cannot be reached through a proxy")

// requestHost is the host a request is made to: the ip it is pinned to, if
// any, and the ip of the last connection dialed to it.
type requestHost struct {
name string
pinned string
mu sync.Mutex
dialed string
}

// requestHostKey is the context key of the requestHost of WithHost.
type requestHostKey struct{}

// WithHost returns a copy of ctx for requests to host. When ip is set, they
// connect to it, like curl --resolve but on every port of host, while the host
// keeps its name for the Host header and SNI. The ip they are dialed to is
// recorded for DialedIP, while the dialer only keeps the last ip dialed to a
// host by any request. Requests to other hosts, such as redirect targets, are
// resolved as usual and not recorded.
func WithHost(ctx context.Context, host, ip string) context.Context {
host = canonicalHost(host)
if ip != "" {
ctx = fastdialer.WithResolveOverride(ctx, host, ip)
}
return context.WithValue(ctx, requestHostKey{}, &requestHost{name: host, pinned: ip})
}

// PinnedContext returns a new context for another request derived from req,
// in which the host of req keeps the ip it is pinned to. The other values of
// the context of req, such as a trace, are not kept.
func PinnedContext(req *http.Request) context.Context {
return WithHost(context.Background(), req.URL.Hostname(), pinnedIP(req))
}

// DialedIP returns the ip of the last connection dialed to the host of a
// WithHost context, or "" if none was.
func DialedIP(ctx context.Context) string {
host, ok := ctx.Value(requestHostKey{}).(*requestHost)
if !ok {
return ""
}
host.mu.Lock()
defer host.mu.Unlock()
return host.dialed
}

// hostOf returns the requestHost of ctx if name, however it is spelled, is its
// host, and nil otherwise.
func hostOf(ctx context.Context, name string) *requestHost {
host, ok := ctx.Value(requestHostKey{}).(*requestHost)
if !ok || host.name != canonicalHost(name) {
return nil
}
return host
}

// pinnedIP returns the ip the host of req is pinned to in its context, if any.
func pinnedIP(req *http.Request) string {
if host := hostOf(req.Context(), req.URL.Hostname()); host != nil {
return host.pinned
}
return ""
}

// canonicalHost folds the case, the IDNA form and a trailing dot of host, as
// fastdialer keys its resolve overrides, so that a host matches the requests
// and dials to it however it is spelled.
func canonicalHost(host string) string {
if ascii, err := idna.Lookup.ToASCII(host); err == nil {
host = ascii
}
return strings.TrimSuffix(strings.ToLower(host), ".")
}

// refusePinned wraps the proxy of a transport so that a request to a pinned
// host fails instead of going through the proxy.
func refusePinned(proxy func(*http.Request) (*url.URL, error)) func(*http.Request) (*url.URL, error) {
return func(req *http.Request) (*url.URL, error) {
proxyURL, err := proxy(req)
if proxyURL != nil && pinnedIP(req) != "" {
return nil, ErrPinnedThroughProxy
}
return proxyURL, err
}
}

// dialFunc dials addr, as net.Dialer.DialContext does.
type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error)

// recordDialedIP wraps dial to record the ip of a connection to the host of a
// WithHost context. A connection to another host, such as a proxy, is not
// recorded.
func recordDialedIP(dial dialFunc) dialFunc {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
conn, err := dial(ctx, network, addr)
if err != nil {
return conn, err
}
name, _, _ := net.SplitHostPort(addr)
if host := hostOf(ctx, name); host != nil {
ip, _, _ := net.SplitHostPort(conn.RemoteAddr().String())
host.mu.Lock()
host.dialed = ip
host.mu.Unlock()
}
return conn, nil
}
}
114 changes: 114 additions & 0 deletions common/httpx/requesthost_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package httpx

import (
"context"
"net"
"net/http"
"net/http/httptrace"
"net/url"
"os"
"os/exec"
"testing"

"github.com/stretchr/testify/require"
)

func TestRefusePinned(t *testing.T) {
proxyURL, err := url.Parse("http://127.0.0.1:3128")
require.NoError(t, err)
proxy := refusePinned(http.ProxyURL(proxyURL))
request := func(ctx context.Context, rawURL string) *http.Request {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
require.NoError(t, err)
return req
}

// a pinned host however it is spelled
for host, rawURL := range map[string]string{"Pinned.Test.": "https://PINNED.test.:8443/", "xn--bcher-kva.test": "https://bücher.test/"} {
_, err = proxy(request(WithHost(context.Background(), host, "192.0.2.10"), rawURL))
require.ErrorIs(t, err, ErrPinnedThroughProxy, rawURL)
}

pinned := WithHost(context.Background(), "pinned.test", "192.0.2.10")
got, err := proxy(request(pinned, "https://other.test/"))
require.NoError(t, err)
require.Equal(t, proxyURL, got, "a request to another host, as a redirect, keeps the proxy")

got, err = proxy(request(WithHost(context.Background(), "plain.test", ""), "https://plain.test/"))
require.NoError(t, err)
require.Equal(t, proxyURL, got, "a host that is not pinned keeps the proxy")

got, err = refusePinned(http.ProxyURL(nil))(request(pinned, "https://pinned.test/"))
require.NoError(t, err)
require.Nil(t, got, "a pinned request without a proxy")
}

// A proxy set in the environment is refused for a pinned request as well. Go
// reads the proxies of the environment once, so the test runs in a new process.
func TestRefusePinned_environment(t *testing.T) {
if os.Getenv("HTTPX_TEST_ENV_PROXY") == "" {
cmd := exec.Command(os.Args[0], "-test.run=^TestRefusePinned_environment$")
// the lower case names first: on Windows, the last of two spellings wins
cmd.Env = append(os.Environ(), "HTTPX_TEST_ENV_PROXY=1", "http_proxy=", "no_proxy=", "NO_PROXY=", "HTTP_PROXY=http://127.0.0.1:1")
output, err := cmd.CombinedOutput()
require.NoError(t, err, string(output))
return
}

options := DefaultOptions
options.RetryMax = 0
h, err := New(&options)
require.NoError(t, err)
req, err := h.NewRequestWithContext(WithHost(context.Background(), "pinned.test", "127.0.0.1"), http.MethodGet, "http://pinned.test/")
require.NoError(t, err)
_, err = h.Do(req, UnsafeOptions{})
require.ErrorIs(t, err, ErrPinnedThroughProxy)
}

func TestCanonicalHost(t *testing.T) {
for host, want := range map[string]string{
"Pinned.Test.": "pinned.test",
"bücher.test": "xn--bcher-kva.test",
"BÜCHER.test": "xn--bcher-kva.test",
"bücher。test": "xn--bcher-kva.test",
"xn--bcher-kva.test": "xn--bcher-kva.test",
} {
require.Equal(t, want, canonicalHost(host), host)
}
}

func TestPinnedContext(t *testing.T) {
ctx := httptrace.WithClientTrace(WithHost(context.Background(), "pinned.test", "192.0.2.10"), &httptrace.ClientTrace{})
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://pinned.test/favicon.ico", nil)
require.NoError(t, err)

derived := req.Clone(PinnedContext(req))
require.Equal(t, "192.0.2.10", pinnedIP(derived), "the pin is kept")
require.Nil(t, httptrace.ContextClientTrace(derived.Context()), "the trace is not")

other, err := http.NewRequest(http.MethodGet, "https://other.test/", nil)
require.NoError(t, err)
require.Empty(t, pinnedIP(other.Clone(PinnedContext(other))))
}

func TestRecordDialedIP(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer func() { _ = listener.Close() }()
// every address is dialed to the listener
dial := recordDialedIP(func(ctx context.Context, network, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, network, listener.Addr().String())
})
ctx := WithHost(context.Background(), "dialed.test", "")

conn, err := dial(ctx, "tcp", "proxy.test:3128")
require.NoError(t, err)
_ = conn.Close()
require.Empty(t, DialedIP(ctx), "a dial to another host, such as a proxy")

// net/http dials a host as it is spelled
conn, err = dial(ctx, "tcp", "DIALED.test.:80")
require.NoError(t, err)
_ = conn.Close()
require.Equal(t, "127.0.0.1", DialedIP(ctx))
}
2 changes: 1 addition & 1 deletion common/httpx/tls.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ func (h *HTTPX) ZTLSGrab(r *http.Response) *clients.Response {
}
// canonical net concatenation
host = net.JoinHostPort(hostname, fmt.Sprint(port))
ctx, cancel := context.WithTimeout(context.Background(), h.Options.Timeout)
ctx, cancel := context.WithTimeout(PinnedContext(r.Request), h.Options.Timeout)
defer cancel()
tlsConn, err := h.Dialer.DialTLS(ctx, "tcp", host)
if err != nil {
Expand Down
23 changes: 23 additions & 0 deletions runner/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package runner

import (
"os"
"strings"
"testing"
)

// TestMain keeps pinned.test, the host the tests pin, out of a proxy set in
// the environment, which would resolve it itself, so that they run behind a
// proxy too. Go reads the proxies of the environment once per process, so
// t.Setenv cannot change them for a test.
func TestMain(m *testing.M) {
noProxy := os.Getenv("NO_PROXY")
if noProxy == "" {
noProxy = os.Getenv("no_proxy")
}
noProxy = strings.TrimPrefix(noProxy+",pinned.test", ",")
for _, name := range []string{"NO_PROXY", "no_proxy"} {
_ = os.Setenv(name, noProxy)
}
os.Exit(m.Run())
}
Loading
Loading