mirror of https://github.com/go-gitea/gitea.git
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
247 lines
9.4 KiB
247 lines
9.4 KiB
// Copyright 2019 The Gitea Authors. All rights reserved. |
|
// SPDX-License-Identifier: MIT |
|
|
|
package charset |
|
|
|
import ( |
|
"io" |
|
"os" |
|
"strings" |
|
"testing" |
|
|
|
"code.gitea.io/gitea/modules/setting" |
|
"code.gitea.io/gitea/modules/test" |
|
|
|
"github.com/stretchr/testify/assert" |
|
) |
|
|
|
func TestMain(m *testing.M) { |
|
setting.Repository.DetectedCharsetScore = map[string]int{} |
|
for i, charset := range setting.Repository.DetectedCharsetsOrder { |
|
setting.Repository.DetectedCharsetScore[strings.ToLower(charset)] = i |
|
} |
|
os.Exit(m.Run()) |
|
} |
|
|
|
func TestMaybeRemoveBOM(t *testing.T) { |
|
res := maybeRemoveBOM([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
res = maybeRemoveBOM([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
} |
|
|
|
func TestToUTF8(t *testing.T) { |
|
// Note: golang compiler seems so behave differently depending on the current |
|
// locale, so some conversions might behave differently. For that reason, we don't |
|
// depend on particular conversions but in expected behaviors. |
|
|
|
res := ToUTF8([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) |
|
assert.Equal(t, "ABC", string(res)) |
|
|
|
// "áéíóú" |
|
res = ToUTF8([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
// "áéíóú" |
|
res = ToUTF8([]byte{ |
|
0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, |
|
0xc3, 0xba, |
|
}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
res = ToUTF8([]byte{ |
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, |
|
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, |
|
}, ConvertOpts{}) |
|
stringMustStartWith(t, "Hola,", res) |
|
stringMustEndWith(t, "AAA.", res) |
|
|
|
res = ToUTF8([]byte{ |
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, |
|
0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, |
|
}, ConvertOpts{}) |
|
stringMustStartWith(t, "Hola,", res) |
|
stringMustEndWith(t, "AAA.", res) |
|
|
|
res = ToUTF8([]byte{ |
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, |
|
0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73, 0x41, 0x41, 0x41, 0x2e, |
|
}, ConvertOpts{}) |
|
stringMustStartWith(t, "Hola,", res) |
|
stringMustEndWith(t, "AAA.", res) |
|
|
|
// Japanese (Shift-JIS) |
|
// 日属秘ぞしちゅ。 |
|
res = ToUTF8([]byte{ |
|
0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, |
|
0xBF, 0x82, 0xE3, 0x81, 0x42, |
|
}, ConvertOpts{}) |
|
assert.Equal(t, []byte{ |
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, |
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, |
|
}, res) |
|
|
|
res = ToUTF8([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) |
|
} |
|
|
|
func TestToUTF8WithFallback(t *testing.T) { |
|
// "ABC" |
|
res := ToUTF8WithFallback([]byte{0x41, 0x42, 0x43}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) |
|
|
|
// "áéíóú" |
|
res = ToUTF8WithFallback([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
// UTF8 BOM + "áéíóú" |
|
res = ToUTF8WithFallback([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
// "Hola, así cómo ños" |
|
res = ToUTF8WithFallback([]byte{ |
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, |
|
0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73, |
|
}, ConvertOpts{}) |
|
assert.Equal(t, []byte{ |
|
0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, |
|
0xC3, 0xB3, 0x6D, 0x6F, 0x20, 0xC3, 0xB1, 0x6F, 0x73, |
|
}, res) |
|
|
|
// "Hola, así cómo " |
|
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20} |
|
|
|
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}, ConvertOpts{}) |
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those |
|
assert.Equal(t, minmatch, res[0:len(minmatch)]) |
|
|
|
res = ToUTF8WithFallback([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}, ConvertOpts{}) |
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those |
|
assert.Equal(t, minmatch, res[0:len(minmatch)]) |
|
|
|
// Japanese (Shift-JIS) |
|
// "日属秘ぞしちゅ。" |
|
res = ToUTF8WithFallback([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}, ConvertOpts{}) |
|
assert.Equal(t, []byte{ |
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, |
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, |
|
}, res) |
|
|
|
res = ToUTF8WithFallback([]byte{0x00, 0x00, 0x00, 0x00}, ConvertOpts{}) |
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) |
|
} |
|
|
|
func TestToUTF8DropErrors(t *testing.T) { |
|
// "ABC" |
|
res := ToUTF8DropErrors([]byte{0x41, 0x42, 0x43}) |
|
assert.Equal(t, []byte{0x41, 0x42, 0x43}, res) |
|
|
|
// "áéíóú" |
|
res = ToUTF8DropErrors([]byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
// UTF8 BOM + "áéíóú" |
|
res = ToUTF8DropErrors([]byte{0xef, 0xbb, 0xbf, 0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}) |
|
assert.Equal(t, []byte{0xc3, 0xa1, 0xc3, 0xa9, 0xc3, 0xad, 0xc3, 0xb3, 0xc3, 0xba}, res) |
|
|
|
// "Hola, así cómo ños" |
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0xF1, 0x6F, 0x73}) |
|
assert.Equal(t, []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73}, res[:8]) |
|
assert.Equal(t, []byte{0x73}, res[len(res)-1:]) |
|
|
|
// "Hola, así cómo " |
|
minmatch := []byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xC3, 0xAD, 0x20, 0x63, 0xC3, 0xB3, 0x6D, 0x6F, 0x20} |
|
|
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x07, 0xA4, 0x6F, 0x73}) |
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those |
|
assert.Equal(t, minmatch, res[0:len(minmatch)]) |
|
|
|
res = ToUTF8DropErrors([]byte{0x48, 0x6F, 0x6C, 0x61, 0x2C, 0x20, 0x61, 0x73, 0xED, 0x20, 0x63, 0xF3, 0x6D, 0x6F, 0x20, 0x81, 0xA4, 0x6F, 0x73}) |
|
// Do not fail for differences in invalid cases, as the library might change the conversion criteria for those |
|
assert.Equal(t, minmatch, res[0:len(minmatch)]) |
|
|
|
// Japanese (Shift-JIS) |
|
// "日属秘ぞしちゅ。" |
|
res = ToUTF8DropErrors([]byte{0x93, 0xFA, 0x91, 0xAE, 0x94, 0xE9, 0x82, 0xBC, 0x82, 0xB5, 0x82, 0xBF, 0x82, 0xE3, 0x81, 0x42}) |
|
assert.Equal(t, []byte{ |
|
0xE6, 0x97, 0xA5, 0xE5, 0xB1, 0x9E, 0xE7, 0xA7, 0x98, 0xE3, |
|
0x81, 0x9E, 0xE3, 0x81, 0x97, 0xE3, 0x81, 0xA1, 0xE3, 0x82, 0x85, 0xE3, 0x80, 0x82, |
|
}, res) |
|
|
|
res = ToUTF8DropErrors([]byte{0x00, 0x00, 0x00, 0x00}) |
|
assert.Equal(t, []byte{0x00, 0x00, 0x00, 0x00}, res) |
|
} |
|
|
|
func TestDetectEncoding(t *testing.T) { |
|
testSuccess := func(b []byte, expected string) { |
|
encoding, err := DetectEncoding(b) |
|
assert.NoError(t, err) |
|
assert.Equal(t, expected, encoding) |
|
} |
|
|
|
// invalid bytes |
|
encoding, err := DetectEncoding([]byte{0xfa}) |
|
assert.Error(t, err) |
|
assert.Equal(t, "UTF-8", encoding) |
|
|
|
// utf-8 |
|
b := []byte("just some ascii") |
|
testSuccess(b, "UTF-8") |
|
|
|
// utf-8-sig: "hey" (with BOM) |
|
b = []byte{0xef, 0xbb, 0xbf, 0x68, 0x65, 0x79} |
|
testSuccess(b, "UTF-8") |
|
|
|
// utf-16: "hey<accented G>" |
|
b = []byte{0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x79, 0x00, 0xf4, 0x01} |
|
testSuccess(b, "UTF-16LE") |
|
|
|
// iso-8859-1: d<accented e>cor<newline> |
|
b = []byte{0x44, 0xe9, 0x63, 0x6f, 0x72, 0x0a} |
|
encoding, err = DetectEncoding(b) |
|
assert.NoError(t, err) |
|
assert.Contains(t, encoding, "ISO-8859-1") |
|
|
|
defer test.MockVariableValue(&setting.Repository.AnsiCharset, "MyEncoding")() |
|
testSuccess(b, "MyEncoding") |
|
} |
|
|
|
func stringMustStartWith(t *testing.T, expected string, value []byte) { |
|
assert.Equal(t, expected, string(value[:len(expected)])) |
|
} |
|
|
|
func stringMustEndWith(t *testing.T, expected string, value []byte) { |
|
assert.Equal(t, expected, string(value[len(value)-len(expected):])) |
|
} |
|
|
|
func TestToUTF8WithFallbackReader(t *testing.T) { |
|
test.MockVariableValue(&ToUTF8WithFallbackReaderPrefetchSize) |
|
|
|
block := "aá啊🤔" |
|
runes := []rune(block) |
|
assert.Len(t, string(runes[0]), 1) |
|
assert.Len(t, string(runes[1]), 2) |
|
assert.Len(t, string(runes[2]), 3) |
|
assert.Len(t, string(runes[3]), 4) |
|
|
|
content := strings.Repeat(block, 2) |
|
for i := 1; i < len(content); i++ { |
|
encoding, err := DetectEncoding([]byte(content[:i])) |
|
assert.NoError(t, err) |
|
assert.Equal(t, "UTF-8", encoding) |
|
|
|
ToUTF8WithFallbackReaderPrefetchSize = i |
|
rd := ToUTF8WithFallbackReader(strings.NewReader(content), ConvertOpts{}) |
|
r, _ := io.ReadAll(rd) |
|
assert.Equal(t, content, string(r)) |
|
} |
|
for _, r := range runes { |
|
content = "abc abc " + string(r) + string(r) + string(r) |
|
for i := 0; i < len(content); i++ { |
|
encoding, err := DetectEncoding([]byte(content[:i])) |
|
assert.NoError(t, err) |
|
assert.Equal(t, "UTF-8", encoding) |
|
} |
|
} |
|
}
|
|
|