Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/migrate-v1-to-v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ v2 needs Go 1.26 or later. v1 and v2 can be used in the same program, so you can

- A value in single or double quotes keeps the `#` and `;` inside them, so `k = "a#b"` is `a#b`. v1 read `"a`. Text other than a comment after the closing quote still cuts the value as before.
- Comment lines after the last key of a file are kept and written back after the last section. Sections added later, or loaded from a later source, are written after them. v1 dropped them.
- A file that starts with a UTF-16 LE or BE byte order mark, as Windows tools write, is decoded as UTF-16. v1 dropped the mark and read the rest as UTF-8, so names and values held NUL bytes. `Write` writes UTF-8.
- A byte order mark repeated at the start of a file is skipped. v1 read the second one as part of the first key name, or failed on a section header.
- A Python-style multiline value longer than 4 KiB is read in full. v1 cut it at its read buffer and failed.

Expand Down
29 changes: 28 additions & 1 deletion lexer.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"strconv"
"strings"
"unicode"
"unicode/utf16"
"unicode/utf8"
)

Expand Down Expand Up @@ -373,6 +374,32 @@ func (l *lexer) readPythonMultilines(line string) string {
}
}

// decodeUTF16 converts src to UTF-8 when it starts with a UTF-16 LE or BE
// byte order mark, as files saved as "Unicode" by Windows tools do, and
// returns other input unchanged. A trailing odd byte becomes U+FFFD.
func decodeUTF16(src string) string {
var little bool
switch {
case strings.HasPrefix(src, "\xFF\xFE"):
little = true
case strings.HasPrefix(src, "\xFE\xFF"):
default:
return src
}
units := make([]uint16, 0, len(src)/2)
for i := 2; i+1 < len(src); i += 2 {
if little {
units = append(units, uint16(src[i])|uint16(src[i+1])<<8)
} else {
units = append(units, uint16(src[i])<<8|uint16(src[i+1]))
}
}
if len(src)%2 == 1 {
units = append(units, utf8.RuneError)
}
return string(utf16.Decode(units))
}

// skipBOM strips the UTF-8, UTF-16 LE and UTF-16 BE byte order marks at
// the start of src. Some tools add a second one to a file that has one.
// http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
Expand All @@ -396,7 +423,7 @@ func (f *File) parse(reader io.Reader) error {
if _, err := io.Copy(&b, reader); err != nil {
return err
}
l := &lexer{src: skipBOM(b.String()), opts: &f.options}
l := &lexer{src: skipBOM(decodeUTF16(b.String())), opts: &f.options}

// Ignore error because default section name is never empty string.
name := f.options.DefaultSection
Expand Down
31 changes: 24 additions & 7 deletions lexer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"testing"
"unicode/utf16"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand All @@ -19,13 +20,29 @@ func TestBOM(t *testing.T) {
assert.Equal(t, "example@email.com", f.Section("author").Key("E-MAIL").String())
})

t.Run("UTF-16-LE-BOM", func(t *testing.T) {
f, err := Load("testdata/UTF-16-LE-BOM.ini")
require.NoError(t, err)
require.NotNil(t, f)
})

t.Run("UTF-16-BE-BOM", func(t *testing.T) {
for _, name := range []string{"UTF-16-LE-BOM", "UTF-16-BE-BOM"} {
t.Run(name, func(t *testing.T) {
f, err := Load("testdata/" + name + ".ini")
require.NoError(t, err)
assert.Equal(t, []string{DefaultSection, "author"}, f.SectionNames())
assert.Equal(t, "example@email.com", f.Section("author").Key("E-MAIL").String())
})
}

t.Run("UTF-16 non-ASCII", func(t *testing.T) {
// A surrogate pair, CRLF line ends, and an odd trailing byte.
units := utf16.Encode([]rune("\ufeff[節]\r\n名前 = 😀\r\nk = v"))
le, be := []byte{}, []byte{}
for _, u := range units {
le = append(le, byte(u), byte(u>>8))
be = append(be, byte(u>>8), byte(u))
}
for _, data := range [][]byte{le, be} {
f, err := Load(append(data, 'x'))
require.NoError(t, err)
assert.Equal(t, "😀", f.Section("節").Key("名前").String())
assert.Equal(t, "v\ufffd", f.Section("節").Key("k").String())
}
})

t.Run("two BOMs", func(t *testing.T) {
Expand Down
Loading