From ca6e781d731a3ae53bcf7aa0e048ec55b735fb01 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Sun, 15 Nov 2026 01:34:31 +0000 Subject: [PATCH] Decode UTF-16 files with a byte order mark A file starting with a UTF-16 LE or BE byte order mark was read as UTF-8 after the mark was dropped, so its names and values held NUL bytes. It is now decoded as UTF-16 before parsing (#66). --- docs/migrate-v1-to-v2.md | 1 + lexer.go | 29 ++++++++++++++++++++++++++++- lexer_test.go | 31 ++++++++++++++++++++++++------- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/docs/migrate-v1-to-v2.md b/docs/migrate-v1-to-v2.md index f3a58a8..a41c7af 100644 --- a/docs/migrate-v1-to-v2.md +++ b/docs/migrate-v1-to-v2.md @@ -10,6 +10,7 @@ v2 needs Go 1.26 or later. v1 and v2 can be used in the same program, so you can - A value in single or double quotes keeps the `#` and `;` inside them, so `k = "a#b"` is `a#b`. v1 read `"a`. Text other than a comment after the closing quote still cuts the value as before. - Comment lines after the last key of a file are kept and written back after the last section. Sections added later, or loaded from a later source, are written after them. v1 dropped them. +- A file that starts with a UTF-16 LE or BE byte order mark, as Windows tools write, is decoded as UTF-16. v1 dropped the mark and read the rest as UTF-8, so names and values held NUL bytes. `Write` writes UTF-8. - A byte order mark repeated at the start of a file is skipped. v1 read the second one as part of the first key name, or failed on a section header. - A Python-style multiline value longer than 4 KiB is read in full. v1 cut it at its read buffer and failed. diff --git a/lexer.go b/lexer.go index cc14a00..8f214b9 100644 --- a/lexer.go +++ b/lexer.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "unicode" + "unicode/utf16" "unicode/utf8" ) @@ -373,6 +374,32 @@ func (l *lexer) readPythonMultilines(line string) string { } } +// decodeUTF16 converts src to UTF-8 when it starts with a UTF-16 LE or BE +// byte order mark, as files saved as "Unicode" by Windows tools do, and +// returns other input unchanged. A trailing odd byte becomes U+FFFD. +func decodeUTF16(src string) string { + var little bool + switch { + case strings.HasPrefix(src, "\xFF\xFE"): + little = true + case strings.HasPrefix(src, "\xFE\xFF"): + default: + return src + } + units := make([]uint16, 0, len(src)/2) + for i := 2; i+1 < len(src); i += 2 { + if little { + units = append(units, uint16(src[i])|uint16(src[i+1])<<8) + } else { + units = append(units, uint16(src[i])<<8|uint16(src[i+1])) + } + } + if len(src)%2 == 1 { + units = append(units, utf8.RuneError) + } + return string(utf16.Decode(units)) +} + // skipBOM strips the UTF-8, UTF-16 LE and UTF-16 BE byte order marks at // the start of src. Some tools add a second one to a file that has one. // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding @@ -396,7 +423,7 @@ func (f *File) parse(reader io.Reader) error { if _, err := io.Copy(&b, reader); err != nil { return err } - l := &lexer{src: skipBOM(b.String()), opts: &f.options} + l := &lexer{src: skipBOM(decodeUTF16(b.String())), opts: &f.options} // Ignore error because default section name is never empty string. name := f.options.DefaultSection diff --git a/lexer_test.go b/lexer_test.go index c62f137..4a98894 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" "testing" + "unicode/utf16" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,13 +20,29 @@ func TestBOM(t *testing.T) { assert.Equal(t, "example@email.com", f.Section("author").Key("E-MAIL").String()) }) - t.Run("UTF-16-LE-BOM", func(t *testing.T) { - f, err := Load("testdata/UTF-16-LE-BOM.ini") - require.NoError(t, err) - require.NotNil(t, f) - }) - - t.Run("UTF-16-BE-BOM", func(t *testing.T) { + for _, name := range []string{"UTF-16-LE-BOM", "UTF-16-BE-BOM"} { + t.Run(name, func(t *testing.T) { + f, err := Load("testdata/" + name + ".ini") + require.NoError(t, err) + assert.Equal(t, []string{DefaultSection, "author"}, f.SectionNames()) + assert.Equal(t, "example@email.com", f.Section("author").Key("E-MAIL").String()) + }) + } + + t.Run("UTF-16 non-ASCII", func(t *testing.T) { + // A surrogate pair, CRLF line ends, and an odd trailing byte. + units := utf16.Encode([]rune("\ufeff[節]\r\n名前 = 😀\r\nk = v")) + le, be := []byte{}, []byte{} + for _, u := range units { + le = append(le, byte(u), byte(u>>8)) + be = append(be, byte(u>>8), byte(u)) + } + for _, data := range [][]byte{le, be} { + f, err := Load(append(data, 'x')) + require.NoError(t, err) + assert.Equal(t, "😀", f.Section("節").Key("名前").String()) + assert.Equal(t, "v\ufffd", f.Section("節").Key("k").String()) + } }) t.Run("two BOMs", func(t *testing.T) {