Go中JSON解碼非UTF-8二進制值的問題 ; 論野生技術&二次元

我們在一個項目中創（luan）新（xie）地用JSON來編碼msgpack編碼後的結果（即encoded = json_encode(msgpack_encode(txt))），結果發現Golang側無法解碼。

首先我們可以確定msgpack沒有問題，因為輸入給msgpack解碼數據就與輸入值不一致。

我們使用lua-cjson來編碼一個JSON，因為結果不是printable的，所以在外面加一層base64.encode

print(ngx.encode_base64(require("cjson").encode({a = "\134123"})))

1	print(ngx.encode_base64(require("cjson").encode({a = "\134123"})))

結果是eyJhIjoihjEyMyJ9。

在Python里解碼它：

json.loads(base64.b64decode('eyJhIjoihjEyMyJ9').decode('raw_unicode_escape'))

1	json.loads(base64.b64decode('eyJhIjoihjEyMyJ9').decode('raw_unicode_escape'))

結果是{‘a’: ‘\x86123’}。沒有問題，和輸入一致。

在Go里解碼它：

type A struct {
	Data string `json:"a"`
}

b64 := "eyJhIjoihjEyMyJ9"
jsonEncoded, _ := base64.StdEncoding.DecodeString(b64)
var aa A
err = json.Unmarshal(jsonEncoded, &aa)
fmt.Println([]byte(aa.Data))

type A struct {

Data string `json:"a"`

}

b64 := "eyJhIjoihjEyMyJ9"

jsonEncoded, _ := base64.StdEncoding.DecodeString(b64)

var aa A

err = json.Unmarshal(jsonEncoded, &aa)

fmt.Println([]byte(aa.Data))

結果是[239 191 189 49 50 51]，可以看到\x86被解碼成了\239 \191 \189即\xefbfbd，表示無效的UTF8字符。

這是因為Go默認採用UTF-8解碼，如果field被標記為string，則json.Unmarshal會使用utf8.DecodeRune來嘗試解碼輸入https://github.com/golang/go/blob/master/src/encoding/json/decode.go#L1304。但在我們的場景中，\x86是一個單字節的非UTF-8字符，所以utf8.DecodeRune返回了utf8.RuneError並把它放到了結果里。

那麼到底是哪裡出了問題呢？

首先，JSON的RFC指出，其中的字符串必須以UTF-8編碼（https://datatracker.ietf.org/doc/html/rfc8259#section-8.1），但是同時也提到，除了幾個特殊的字符外，其中的字符可以被escape也可以不escape（https://datatracker.ietf.org/doc/html/rfc8259#section-7）。所以lua-cjson的這種編碼方式似乎也是合法的？

解決辦法是寫一個自己的Unmarshal方法。首先把結構體中的field標記為自定義類型：

type Payload struct {
	A RawBytes `json:"a"`
}

type rawBytes []byte

func (r *rawBytes) UnmarshalJSON(b []byte) error {
	unqoutedBytes, ok := unqouteBytes(b)
	if !ok {
		return fmt.Errorf("failed to uncode msgpacked data")
	}
	*r = unqoutedBytes
	return nil
}

type Payload struct {

A RawBytes `json:"a"`

}

type rawBytes []byte

func (r *rawBytes) UnmarshalJSON(b []byte) error {

unqoutedBytes, ok := unqouteBytes(b)

if !ok {

return fmt.Errorf("failed to uncode msgpacked data")

}

*r = unqoutedBytes

return nil

}

然後我們魔改unquote方法，在原來的基礎上加上對解碼結果是否為utf8.RuneError的判斷

// getu4 has been copied verbatim from
// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1167-L1188
func getu4(s []byte) rune {
	if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
		return -1
	}
	var r rune
	for _, c := range s[2:6] {
		switch {
		case '0' <= c && c <= '9':
			c = c - '0'
		case 'a' <= c && c <= 'f':
			c = c - 'a' + 10
		case 'A' <= c && c <= 'F':
			c = c - 'A' + 10
		default:
			return -1
		}
		r = r*16 + rune(c)
	}
	return r
}

// unqouteBytes converts a quoted literal []byte s.
// The rules are different than for Go, so cannot use strconv.Unqoute().
// This function is copied verbatim from
// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1198-L1310
// Please read the comment beginning with "PATCHED" below.
func unqouteBytes(s []byte) (t []byte, ok bool) {
	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
		return
	}
	s = s[1 : len(s)-1]

	// Check for unusual characters. If there are none,
	// then no unquoting is needed, so return a slice of the
	// original bytes.
	r := 0
	for r < len(s) {
		c := s[r]
		if c == '\\' || c == '"' || c < ' ' {
			break
		}
		if c < utf8.RuneSelf {
			r++
			continue
		}
		rr, size := utf8.DecodeRune(s[r:])
		if rr == utf8.RuneError && size == 1 {
			break
		}
		r += size
	}
	if r == len(s) {
		return s, true
	}

	b := make([]byte, len(s)+2*utf8.UTFMax)
	w := copy(b, s[0:r])
	for r < len(s) {
		// Out of room?  Can only happen if s is full of
		// malformed UTF-8 and we're replacing each
		// byte with RuneError.
		if w >= len(b)-2*utf8.UTFMax {
			nb := make([]byte, (len(b)+utf8.UTFMax)*2)
			copy(nb, b[0:w])
			b = nb
		}
		switch c := s[r]; {
		case c == '\\':
			r++
			if r >= len(s) {
				return
			}
			switch s[r] {
			default:
				return
			case '"', '\\', '/', '\'':
				b[w] = s[r]
				r++
				w++
			case 'b':
				b[w] = '\b'
				r++
				w++
			case 'f':
				b[w] = '\f'
				r++
				w++
			case 'n':
				b[w] = '\n'
				r++
				w++
			case 'r':
				b[w] = '\r'
				r++
				w++
			case 't':
				b[w] = '\t'
				r++
				w++
			case 'u':
				r--
				rr := getu4(s[r:])
				if rr < 0 {
					return
				}
				r += 6
				if utf16.IsSurrogate(rr) {
					rr1 := getu4(s[r:])
					if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
						// A valid pair; consume.
						r += 6
						w += utf8.EncodeRune(b[w:], dec)
						break
					}
					// Invalid surrogate; fall back to replacement rune.
					rr = unicode.ReplacementChar
				}
				w += utf8.EncodeRune(b[w:], rr)
			}

		// Quote, control characters are invalid.
		case c == '"', c < ' ':
			return

		// ASCII
		case c < utf8.RuneSelf:
			b[w] = c
			r++
			w++

		// Coerce to well-formed UTF-8.
		default:
			// PATCHED: on RuneError, copy verbatim
			rr, size := utf8.DecodeRune(s[r:])
			if rr != utf8.RuneError {
				r += size
				w += utf8.EncodeRune(b[w:], rr)
			} else {
				b[w] = s[r]
				r++
				w++
			}
		}
	}
	return b[0:w], true
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

// getu4 has been copied verbatim from

// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1167-L1188

func getu4(s []byte) rune {

if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {

return -1

}

var r rune

for _, c := range s[2:6] {

switch {

case '0' <= c && c <= '9':

c = c - '0'

case 'a' <= c && c <= 'f':

c = c - 'a' + 10

case 'A' <= c && c <= 'F':

c = c - 'A' + 10

default:

return -1

}

r = r*16 + rune(c)

}

return r

}

// unqouteBytes converts a quoted literal []byte s.

// The rules are different than for Go, so cannot use strconv.Unqoute().

// This function is copied verbatim from

// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1198-L1310

// Please read the comment beginning with "PATCHED" below.

func unqouteBytes(s []byte) (t []byte, ok bool) {

if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {

return

}

s = s[1 : len(s)-1]

// Check for unusual characters. If there are none,

// then no unquoting is needed, so return a slice of the

// original bytes.

r := 0

for r < len(s) {

c := s[r]

if c == '\\' || c == '"' || c < ' ' {

break

}

if c < utf8.RuneSelf {

r++

continue

}

rr, size := utf8.DecodeRune(s[r:])

if rr == utf8.RuneError && size == 1 {

break

}

r += size

}

if r == len(s) {

return s, true

}

b := make([]byte, len(s)+2*utf8.UTFMax)

w := copy(b, s[0:r])

for r < len(s) {

// Out of room? Can only happen if s is full of

// malformed UTF-8 and we're replacing each

// byte with RuneError.

if w >= len(b)-2*utf8.UTFMax {

nb := make([]byte, (len(b)+utf8.UTFMax)*2)

copy(nb, b[0:w])

b = nb

}

switch c := s[r]; {

case c == '\\':

r++

if r >= len(s) {

return

}

switch s[r] {

default:

return

case '"', '\\', '/', '\'':

b[w] = s[r]

r++

w++

case 'b':

b[w] = '\b'

r++

w++

case 'f':

b[w] = '\f'

r++

w++

case 'n':

b[w] = '\n'

r++

w++

case 'r':

b[w] = '\r'

r++

w++

case 't':

b[w] = '\t'

r++

w++

case 'u':

r--

rr := getu4(s[r:])

if rr < 0 {

return

}

r += 6

if utf16.IsSurrogate(rr) {

rr1 := getu4(s[r:])

if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {

// A valid pair; consume.

r += 6

w += utf8.EncodeRune(b[w:], dec)

break

}

// Invalid surrogate; fall back to replacement rune.

rr = unicode.ReplacementChar

}

w += utf8.EncodeRune(b[w:], rr)

}

// Quote, control characters are invalid.

case c == '"', c < ' ':

return

// ASCII

case c < utf8.RuneSelf:

b[w] = c

r++

w++

// Coerce to well-formed UTF-8.

default:

// PATCHED: on RuneError, copy verbatim

rr, size := utf8.DecodeRune(s[r:])

if rr != utf8.RuneError {

r += size

w += utf8.EncodeRune(b[w:], rr)

} else {

b[w] = s[r]

r++

w++

}

return b[0:w], true

}

還需要注意的是，Go的json包默認對[]byte類型的field進行base64編解碼：

b, _ := json.Marshal(map[string][]byte{
	"x": []byte{147},
})
fmt.Println("> ", b, string(b))

b, _ := json.Marshal(map[string][]byte{

"x": []byte{147},

})

fmt.Println("> ", b, string(b))

結果是> [123 34 120 34 58 34 107 119 61 61 34 125] {“x”:”kw==”}；同理Unmarshal時也會需要輸入為base64編碼結果。

因此在上面這個解決方法中，我們用rawBytes這個新類型來alias到[]byte，而並不直接使用[]byte類型再在之後自己解碼。

發了一個issue：https://github.com/golang/go/issues/51094。

另外的JSON庫沒有這個問題，測試了https://github.com/json-iterator/go 和 https://github.com/bytedance/sonic 。