diff --git a/runner/runner.go b/runner/runner.go index d20b0524..22912cf9 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -10,6 +10,7 @@ import ( "html/template" "image" "io" + "iter" "net" "net/http" "net/http/httputil" @@ -26,6 +27,8 @@ import ( "time" "golang.org/x/exp/maps" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" "github.com/PuerkitoBio/goquery" "github.com/corona10/goimagehash" @@ -779,7 +782,7 @@ func (r *Runner) streamInput() (chan string, error) { return } } else { - for item, err := range fileutil.Lines(r.options.InputFile) { + for item, err := range inputFileLines(r.options.InputFile) { if err != nil || !sendLine(item) { return } @@ -791,7 +794,7 @@ func (r *Runner) streamInput() (chan string, error) { gologger.Fatal().Msgf("No input provided: %s", err) } for _, file := range files { - for item, err := range fileutil.Lines(file) { + for item, err := range inputFileLines(file) { if err != nil || !sendLine(item) { return } @@ -799,7 +802,7 @@ func (r *Runner) streamInput() (chan string, error) { } } if fileutil.HasStdin() { - for item, err := range fileutil.LinesReader(os.Stdin) { + for item, err := range fileutil.LinesReader(decodeInput(os.Stdin)) { if err != nil || !sendLine(item) { return } @@ -810,7 +813,7 @@ func (r *Runner) streamInput() (chan string, error) { } func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { - scanner := bufio.NewScanner(finput) + scanner := bufio.NewScanner(decodeInput(finput)) for scanner.Scan() { target, ok := r.readInputLine(scanner.Text()) if !ok { @@ -833,6 +836,30 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { return numTargets, err } +// decodeInput converts input starting with a UTF-16 byte order mark to UTF-8 +// (Windows PowerShell writes UTF-16) and strips a UTF-8 one. Input without a +// byte order mark is passed through. +func decodeInput(r io.Reader) io.Reader { + return transform.NewReader(r, unicode.BOMOverride(transform.Nop)) +} + +// inputFileLines streams the lines of an input file, decoded as decodeInput does. +func inputFileLines(name string) iter.Seq2[string, error] { + return func(yield func(string, error) bool) { + f, err := os.Open(name) + if err != nil { + yield("", err) + return + } + defer f.Close() //nolint:errcheck + for line, err := range fileutil.LinesReader(decodeInput(f)) { + if !yield(line, err) { + return + } + } + } +} + func (r *Runner) loadFromFormat(filePath string, format inputformats.Format) (numTargets int, err error) { finput, err := os.Open(filePath) if err != nil { diff --git a/runner/targetline_test.go b/runner/targetline_test.go index d0f91f38..051fe739 100644 --- a/runner/targetline_test.go +++ b/runner/targetline_test.go @@ -219,6 +219,50 @@ func TestRunner_skipsInvalidLines(t *testing.T) { }) } +// A file saved with a byte order mark, such as UTF-16 written by Windows +// PowerShell, is decoded before its lines are read. +func TestRunner_readsInputWithBOM(t *testing.T) { + want := []string{"a.example", "b.example"} + encodings := map[string][]byte{ + "utf-16le": {0xff, 0xfe, 'a', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\r', 0, '\n', 0, 'b', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\r', 0, '\n', 0}, + "utf-16be": {0xfe, 0xff, 0, 'a', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\n', 0, 'b', 0, '.', 0, 'e', 0, 'x', 0, 'a', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, '\n'}, + "utf-8": append([]byte{0xef, 0xbb, 0xbf}, "a.example\nb.example\n"...), + } + for name, content := range encodings { + file := filepath.Join(t.TempDir(), "targets.txt") + require.NoError(t, os.WriteFile(file, content, 0o600)) + + t.Run(name+" -l", func(t *testing.T) { + r, err := New(&Options{}) + require.NoError(t, err) + defer r.Close() + input, err := os.Open(file) + require.NoError(t, err) + + count, err := r.loadAndCloseFile(input) + require.NoError(t, err) + require.Equal(t, len(want), count) + for _, key := range want { + _, ok := r.hm.Get(key) + require.True(t, ok, key) + } + }) + t.Run(name+" -stream", func(t *testing.T) { + r, err := New(&Options{Stream: true, InputFile: file}) + require.NoError(t, err) + defer r.Close() + out, err := r.streamInput() + require.NoError(t, err) + + var got []string + for item := range out { + got = append(got, item) + } + require.Equal(t, want, got) + }) + } +} + func TestRunner_skip_pinned(t *testing.T) { r, err := New(&Options{Exclude: []string{"cdn"}}) require.NoError(t, err)