Go中JSON解码非UTF-8二进制值的问题 ; 论野生技术&二次元

我们在一个项目中创（luan）新（xie）地用JSON来编码msgpack编码后的结果（即encoded = json_encode(msgpack_encode(txt))），结果发现Golang侧无法解码。

首先我们可以确定msgpack没有问题，因为输入给msgpack解码数据就与输入值不一致。

我们使用lua-cjson来编码一个JSON，因为结果不是printable的，所以在外面加一层base64.encode

print(ngx.encode_base64(require("cjson").encode({a = "\134123"})))

1	print(ngx.encode_base64(require("cjson").encode({a = "\134123"})))

结果是eyJhIjoihjEyMyJ9。

在Python里解码它：

json.loads(base64.b64decode('eyJhIjoihjEyMyJ9').decode('raw_unicode_escape'))

1	json.loads(base64.b64decode('eyJhIjoihjEyMyJ9').decode('raw_unicode_escape'))

结果是{‘a’: ‘\x86123’}。没有问题，和输入一致。

在Go里解码它：

type A struct {
	Data string `json:"a"`
}

b64 := "eyJhIjoihjEyMyJ9"
jsonEncoded, _ := base64.StdEncoding.DecodeString(b64)
var aa A
err = json.Unmarshal(jsonEncoded, &aa)
fmt.Println([]byte(aa.Data))

type A struct {

Data string `json:"a"`

}

b64 := "eyJhIjoihjEyMyJ9"

jsonEncoded, _ := base64.StdEncoding.DecodeString(b64)

var aa A

err = json.Unmarshal(jsonEncoded, &aa)

fmt.Println([]byte(aa.Data))

结果是[239 191 189 49 50 51]，可以看到\x86被解码成了\239 \191 \189即\xefbfbd，表示无效的UTF8字符。

这是因为Go默认采用UTF-8解码，如果field被标记为string，则json.Unmarshal会使用utf8.DecodeRune来尝试解码输入https://github.com/golang/go/blob/master/src/encoding/json/decode.go#L1304。但在我们的场景中，\x86是一个单字节的非UTF-8字符，所以utf8.DecodeRune返回了utf8.RuneError并把它放到了结果里。

那么到底是哪里出了问题呢？

首先，JSON的RFC指出，其中的字符串必须以UTF-8编码（https://datatracker.ietf.org/doc/html/rfc8259#section-8.1），但是同时也提到，除了几个特殊的字符外，其中的字符可以被escape也可以不escape（https://datatracker.ietf.org/doc/html/rfc8259#section-7）。所以lua-cjson的这种编码方式似乎也是合法的？

解决办法是写一个自己的Unmarshal方法。首先把结构体中的field标记为自定义类型：

type Payload struct {
	A RawBytes `json:"a"`
}

type rawBytes []byte

func (r *rawBytes) UnmarshalJSON(b []byte) error {
	unqoutedBytes, ok := unqouteBytes(b)
	if !ok {
		return fmt.Errorf("failed to uncode msgpacked data")
	}
	*r = unqoutedBytes
	return nil
}

type Payload struct {

A RawBytes `json:"a"`

}

type rawBytes []byte

func (r *rawBytes) UnmarshalJSON(b []byte) error {

unqoutedBytes, ok := unqouteBytes(b)

if !ok {

return fmt.Errorf("failed to uncode msgpacked data")

}

*r = unqoutedBytes

return nil

}

然后我们魔改unquote方法，在原来的基础上加上对解码结果是否为utf8.RuneError的判断

// getu4 has been copied verbatim from
// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1167-L1188
func getu4(s []byte) rune {
	if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {
		return -1
	}
	var r rune
	for _, c := range s[2:6] {
		switch {
		case '0' <= c && c <= '9':
			c = c - '0'
		case 'a' <= c && c <= 'f':
			c = c - 'a' + 10
		case 'A' <= c && c <= 'F':
			c = c - 'A' + 10
		default:
			return -1
		}
		r = r*16 + rune(c)
	}
	return r
}

// unqouteBytes converts a quoted literal []byte s.
// The rules are different than for Go, so cannot use strconv.Unqoute().
// This function is copied verbatim from
// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1198-L1310
// Please read the comment beginning with "PATCHED" below.
func unqouteBytes(s []byte) (t []byte, ok bool) {
	if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {
		return
	}
	s = s[1 : len(s)-1]

	// Check for unusual characters. If there are none,
	// then no unquoting is needed, so return a slice of the
	// original bytes.
	r := 0
	for r < len(s) {
		c := s[r]
		if c == '\\' || c == '"' || c < ' ' {
			break
		}
		if c < utf8.RuneSelf {
			r++
			continue
		}
		rr, size := utf8.DecodeRune(s[r:])
		if rr == utf8.RuneError && size == 1 {
			break
		}
		r += size
	}
	if r == len(s) {
		return s, true
	}

	b := make([]byte, len(s)+2*utf8.UTFMax)
	w := copy(b, s[0:r])
	for r < len(s) {
		// Out of room?  Can only happen if s is full of
		// malformed UTF-8 and we're replacing each
		// byte with RuneError.
		if w >= len(b)-2*utf8.UTFMax {
			nb := make([]byte, (len(b)+utf8.UTFMax)*2)
			copy(nb, b[0:w])
			b = nb
		}
		switch c := s[r]; {
		case c == '\\':
			r++
			if r >= len(s) {
				return
			}
			switch s[r] {
			default:
				return
			case '"', '\\', '/', '\'':
				b[w] = s[r]
				r++
				w++
			case 'b':
				b[w] = '\b'
				r++
				w++
			case 'f':
				b[w] = '\f'
				r++
				w++
			case 'n':
				b[w] = '\n'
				r++
				w++
			case 'r':
				b[w] = '\r'
				r++
				w++
			case 't':
				b[w] = '\t'
				r++
				w++
			case 'u':
				r--
				rr := getu4(s[r:])
				if rr < 0 {
					return
				}
				r += 6
				if utf16.IsSurrogate(rr) {
					rr1 := getu4(s[r:])
					if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {
						// A valid pair; consume.
						r += 6
						w += utf8.EncodeRune(b[w:], dec)
						break
					}
					// Invalid surrogate; fall back to replacement rune.
					rr = unicode.ReplacementChar
				}
				w += utf8.EncodeRune(b[w:], rr)
			}

		// Quote, control characters are invalid.
		case c == '"', c < ' ':
			return

		// ASCII
		case c < utf8.RuneSelf:
			b[w] = c
			r++
			w++

		// Coerce to well-formed UTF-8.
		default:
			// PATCHED: on RuneError, copy verbatim
			rr, size := utf8.DecodeRune(s[r:])
			if rr != utf8.RuneError {
				r += size
				w += utf8.EncodeRune(b[w:], rr)
			} else {
				b[w] = s[r]
				r++
				w++
			}
		}
	}
	return b[0:w], true
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

// getu4 has been copied verbatim from

// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1167-L1188

func getu4(s []byte) rune {

if len(s) < 6 || s[0] != '\\' || s[1] != 'u' {

return -1

}

var r rune

for _, c := range s[2:6] {

switch {

case '0' <= c && c <= '9':

c = c - '0'

case 'a' <= c && c <= 'f':

c = c - 'a' + 10

case 'A' <= c && c <= 'F':

c = c - 'A' + 10

default:

return -1

}

r = r*16 + rune(c)

}

return r

}

// unqouteBytes converts a quoted literal []byte s.

// The rules are different than for Go, so cannot use strconv.Unqoute().

// This function is copied verbatim from

// https://github.com/golang/go/blob/2580d0e08d5e9f979b943758d3c49877fb2324cb/src/encoding/json/decode.go#L1198-L1310

// Please read the comment beginning with "PATCHED" below.

func unqouteBytes(s []byte) (t []byte, ok bool) {

if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' {

return

}

s = s[1 : len(s)-1]

// Check for unusual characters. If there are none,

// then no unquoting is needed, so return a slice of the

// original bytes.

r := 0

for r < len(s) {

c := s[r]

if c == '\\' || c == '"' || c < ' ' {

break

}

if c < utf8.RuneSelf {

r++

continue

}

rr, size := utf8.DecodeRune(s[r:])

if rr == utf8.RuneError && size == 1 {

break

}

r += size

}

if r == len(s) {

return s, true

}

b := make([]byte, len(s)+2*utf8.UTFMax)

w := copy(b, s[0:r])

for r < len(s) {

// Out of room? Can only happen if s is full of

// malformed UTF-8 and we're replacing each

// byte with RuneError.

if w >= len(b)-2*utf8.UTFMax {

nb := make([]byte, (len(b)+utf8.UTFMax)*2)

copy(nb, b[0:w])

b = nb

}

switch c := s[r]; {

case c == '\\':

r++

if r >= len(s) {

return

}

switch s[r] {

default:

return

case '"', '\\', '/', '\'':

b[w] = s[r]

r++

w++

case 'b':

b[w] = '\b'

r++

w++

case 'f':

b[w] = '\f'

r++

w++

case 'n':

b[w] = '\n'

r++

w++

case 'r':

b[w] = '\r'

r++

w++

case 't':

b[w] = '\t'

r++

w++

case 'u':

r--

rr := getu4(s[r:])

if rr < 0 {

return

}

r += 6

if utf16.IsSurrogate(rr) {

rr1 := getu4(s[r:])

if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar {

// A valid pair; consume.

r += 6

w += utf8.EncodeRune(b[w:], dec)

break

}

// Invalid surrogate; fall back to replacement rune.

rr = unicode.ReplacementChar

}

w += utf8.EncodeRune(b[w:], rr)

}

// Quote, control characters are invalid.

case c == '"', c < ' ':

return

// ASCII

case c < utf8.RuneSelf:

b[w] = c

r++

w++

// Coerce to well-formed UTF-8.

default:

// PATCHED: on RuneError, copy verbatim

rr, size := utf8.DecodeRune(s[r:])

if rr != utf8.RuneError {

r += size

w += utf8.EncodeRune(b[w:], rr)

} else {

b[w] = s[r]

r++

w++

}

return b[0:w], true

}

还需要注意的是，Go的json包默认对[]byte类型的field进行base64编解码：

b, _ := json.Marshal(map[string][]byte{
	"x": []byte{147},
})
fmt.Println("> ", b, string(b))

b, _ := json.Marshal(map[string][]byte{

"x": []byte{147},

})

fmt.Println("> ", b, string(b))

结果是> [123 34 120 34 58 34 107 119 61 61 34 125] {“x”:”kw==”}；同理Unmarshal时也会需要输入为base64编码结果。

因此在上面这个解决方法中，我们用rawBytes这个新类型来alias到[]byte，而并不直接使用[]byte类型再在之后自己解码。

发了一个issue：https://github.com/golang/go/issues/51094。

另外的JSON库没有这个问题，测试了https://github.com/json-iterator/go 和 https://github.com/bytedance/sonic 。