Convert non-unicode input when reading w/ Jackson

This commit makes sure that Jackson-based message converters and
decoders can deal with non-unicode input. It does so by reading
non-unicode input messages with a InputStreamReader.

This commit also adds additional tests forthe canRead/canWrite methods
on both codecs and message converters.

Closes: gh-25247

spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Encoder.java

@@ -69,10 +69,18 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
 
 	private static final Map<MediaType, byte[]> STREAM_SEPARATORS;
 
+	private static final Map<Charset, JsonEncoding> ENCODINGS;
+
 	static {
 		STREAM_SEPARATORS = new HashMap<>(4);
 		STREAM_SEPARATORS.put(MediaType.APPLICATION_STREAM_JSON, NEWLINE_SEPARATOR);
 		STREAM_SEPARATORS.put(MediaType.parseMediaType("application/stream+x-jackson-smile"), new byte[0]);
+
+		ENCODINGS = new HashMap<>(JsonEncoding.values().length);
+		for (JsonEncoding encoding : JsonEncoding.values()) {
+			Charset charset = Charset.forName(encoding.getJavaName());
+			ENCODINGS.put(charset, encoding);
+		}
 	}
 
 
@@ -103,7 +111,16 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
 	@Override
 	public boolean canEncode(ResolvableType elementType, @Nullable MimeType mimeType) {
 		Class<?> clazz = elementType.toClass();
-		return supportsMimeType(mimeType) && (Object.class == clazz ||
+		if (!supportsMimeType(mimeType)) {
+			return false;
+		}
+		if (mimeType != null && mimeType.getCharset() != null) {
+			Charset charset = mimeType.getCharset();
+			if (!ENCODINGS.containsKey(charset)) {
+				return false;
+			}
+		}
+		return (Object.class == clazz ||
 				(!String.class.isAssignableFrom(elementType.resolve(clazz)) && getObjectMapper().canSerialize(clazz)));
 	}
 
@@ -269,10 +286,9 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
 	protected JsonEncoding getJsonEncoding(@Nullable MimeType mimeType) {
 		if (mimeType != null && mimeType.getCharset() != null) {
 			Charset charset = mimeType.getCharset();
-			for (JsonEncoding encoding : JsonEncoding.values()) {
-				if (charset.name().equals(encoding.getJavaName())) {
-					return encoding;
-				}
+			JsonEncoding result = ENCODINGS.get(charset);
+			if (result != null) {
+				return result;
 			}
 		}
 		return JsonEncoding.UTF8;

spring-web/src/main/java/org/springframework/http/codec/json/Jackson2CodecSupport.java

@@ -18,18 +18,13 @@ package org.springframework.http.codec.json;
 
 import java.lang.annotation.Annotation;
 import java.lang.reflect.Type;
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import com.fasterxml.jackson.annotation.JsonView;
-import com.fasterxml.jackson.core.JsonEncoding;
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.type.TypeFactory;
@@ -69,9 +64,6 @@ public abstract class Jackson2CodecSupport {
 					new MimeType("application", "json", StandardCharsets.UTF_8),
 					new MimeType("application", "*+json", StandardCharsets.UTF_8)));
 
-	private static final Map<String, JsonEncoding> ENCODINGS = jsonEncodings();
-
-
 
 	protected final Log logger = HttpLogging.forLogName(getClass());
 
@@ -104,17 +96,7 @@ public abstract class Jackson2CodecSupport {
 
 
 	protected boolean supportsMimeType(@Nullable MimeType mimeType) {
-		if (mimeType == null) {
-			return true;
-		}
-		else if (this.mimeTypes.stream().noneMatch(m -> m.isCompatibleWith(mimeType))) {
-			return false;
-		}
-		else if (mimeType.getCharset() != null) {
-			Charset charset = mimeType.getCharset();
-			return ENCODINGS.containsKey(charset.name());
-		}
-		return true;
+		return (mimeType == null || this.mimeTypes.stream().anyMatch(m -> m.isCompatibleWith(mimeType)));
 	}
 
 	protected JavaType getJavaType(Type type, @Nullable Class<?> contextClass) {
@@ -143,10 +125,4 @@ public abstract class Jackson2CodecSupport {
 	@Nullable
 	protected abstract <A extends Annotation> A getAnnotation(MethodParameter parameter, Class<A> annotType);
 
-	private static Map<String, JsonEncoding> jsonEncodings() {
-		return EnumSet.allOf(JsonEncoding.class).stream()
-				.collect(Collectors.toMap(JsonEncoding::getJavaName, Function.identity()));
-	}
-
-
 }

spring-web/src/main/java/org/springframework/http/converter/json/AbstractJackson2HttpMessageConverter.java

@@ -17,6 +17,8 @@
 package org.springframework.http.converter.json;
 
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.lang.reflect.Type;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
@@ -37,6 +39,7 @@ import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import com.fasterxml.jackson.databind.SerializationConfig;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -73,7 +76,7 @@ import org.springframework.util.TypeUtils;
  */
 public abstract class AbstractJackson2HttpMessageConverter extends AbstractGenericHttpMessageConverter<Object> {
 
-	private static final Map<String, JsonEncoding> ENCODINGS = jsonEncodings();
+	private static final Map<Charset, JsonEncoding> ENCODINGS = jsonEncodings();
 
 	/**
 	 * The default charset used by the converter.
@@ -173,19 +176,17 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 		return false;
 	}
 
-	@Override
-	protected boolean canRead(@Nullable MediaType mediaType) {
-		if (!super.canRead(mediaType)) {
-			return false;
-		}
-		return checkEncoding(mediaType);
-	}
-
 	@Override
 	public boolean canWrite(Class<?> clazz, @Nullable MediaType mediaType) {
 		if (!canWrite(mediaType)) {
 			return false;
 		}
+		if (mediaType != null && mediaType.getCharset() != null) {
+			Charset charset = mediaType.getCharset();
+			if (!ENCODINGS.containsKey(charset)) {
+				return false;
+			}
+		}
 		AtomicReference<Throwable> causeRef = new AtomicReference<>();
 		if (this.objectMapper.canSerialize(clazz, causeRef)) {
 			return true;
@@ -194,14 +195,6 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 		return false;
 	}
 
-	@Override
-	protected boolean canWrite(@Nullable MediaType mediaType) {
-		if (!super.canWrite(mediaType)) {
-			return false;
-		}
-		return checkEncoding(mediaType);
-	}
-
 	/**
 	 * Determine whether to log the given exception coming from a
 	 * {@link ObjectMapper#canDeserialize} / {@link ObjectMapper#canSerialize} check.
@@ -233,14 +226,6 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 		}
 	}
 
-	private boolean checkEncoding(@Nullable MediaType mediaType) {
-		if (mediaType != null && mediaType.getCharset() != null) {
-			Charset charset = mediaType.getCharset();
-			return ENCODINGS.containsKey(charset.name());
-		}
-		return true;
-	}
-
 	@Override
 	protected Object readInternal(Class<?> clazz, HttpInputMessage inputMessage)
 			throws IOException, HttpMessageNotReadableException {
@@ -258,15 +243,31 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 	}
 
 	private Object readJavaType(JavaType javaType, HttpInputMessage inputMessage) throws IOException {
+		MediaType contentType = inputMessage.getHeaders().getContentType();
+		Charset charset = getCharset(contentType);
+
+		boolean isUnicode = ENCODINGS.containsKey(charset);
 		try {
 			if (inputMessage instanceof MappingJacksonInputMessage) {
 				Class<?> deserializationView = ((MappingJacksonInputMessage) inputMessage).getDeserializationView();
 				if (deserializationView != null) {
-					return this.objectMapper.readerWithView(deserializationView).forType(javaType).
-							readValue(inputMessage.getBody());
+					ObjectReader objectReader = this.objectMapper.readerWithView(deserializationView).forType(javaType);
+					if (isUnicode) {
+						return objectReader.readValue(inputMessage.getBody());
+					}
+					else {
+						Reader reader = new InputStreamReader(inputMessage.getBody(), charset);
+						return objectReader.readValue(reader);
+					}
 				}
 			}
-			return this.objectMapper.readValue(inputMessage.getBody(), javaType);
+			if (isUnicode) {
+				return this.objectMapper.readValue(inputMessage.getBody(), javaType);
+			}
+			else {
+				Reader reader = new InputStreamReader(inputMessage.getBody(), charset);
+				return this.objectMapper.readValue(reader, javaType);
+			}
 		}
 		catch (InvalidDefinitionException ex) {
 			throw new HttpMessageConversionException("Type definition error: " + ex.getType(), ex);
@@ -276,6 +277,15 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 		}
 	}
 
+	private static Charset getCharset(@Nullable MediaType contentType) {
+		if (contentType != null && contentType.getCharset() != null) {
+			return contentType.getCharset();
+		}
+		else {
+			return StandardCharsets.UTF_8;
+		}
+	}
+
 	@Override
 	protected void writeInternal(Object object, @Nullable Type type, HttpOutputMessage outputMessage)
 			throws IOException, HttpMessageNotWritableException {
@@ -363,7 +373,7 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 	protected JsonEncoding getJsonEncoding(@Nullable MediaType contentType) {
 		if (contentType != null && contentType.getCharset() != null) {
 			Charset charset = contentType.getCharset();
-			JsonEncoding encoding = ENCODINGS.get(charset.name());
+			JsonEncoding encoding = ENCODINGS.get(charset);
 			if (encoding != null) {
 				return encoding;
 			}
@@ -388,9 +398,9 @@ public abstract class AbstractJackson2HttpMessageConverter extends AbstractGener
 		return super.getContentLength(object, contentType);
 	}
 
-	private static Map<String, JsonEncoding> jsonEncodings() {
+	private static Map<Charset, JsonEncoding> jsonEncodings() {
 		return EnumSet.allOf(JsonEncoding.class).stream()
-				.collect(Collectors.toMap(JsonEncoding::getJavaName, Function.identity()));
+				.collect(Collectors.toMap(encoding -> Charset.forName(encoding.getJavaName()), Function.identity()));
 	}
 
 }

spring-web/src/test/java/org/springframework/http/codec/json/Jackson2JsonDecoderTests.java

@@ -91,7 +91,7 @@ public class Jackson2JsonDecoderTests extends AbstractDecoderTestCase<Jackson2Js
 		assertFalse(decoder.canDecode(forClass(Pojo.class), APPLICATION_XML));
 		assertTrue(this.decoder.canDecode(forClass(Pojo.class),
 				new MediaType("application", "json", StandardCharsets.UTF_8)));
-		assertFalse(this.decoder.canDecode(forClass(Pojo.class),
+		assertTrue(this.decoder.canDecode(forClass(Pojo.class),
 				new MediaType("application", "json", StandardCharsets.ISO_8859_1)));
 
 	}
@@ -239,6 +239,26 @@ public class Jackson2JsonDecoderTests extends AbstractDecoderTestCase<Jackson2Js
 				null);
 	}
 
+	@Test
+	@SuppressWarnings("unchecked")
+	public void decodeNonUnicode() {
+		Flux<DataBuffer> input = Flux.concat(
+				stringBuffer("{\"føø\":\"bår\"}", StandardCharsets.ISO_8859_1)
+		);
+
+		testDecode(input, ResolvableType.forType(new ParameterizedTypeReference<Map<String, String>>() {
+				}),
+				step -> step.assertNext(o -> {
+					assertTrue(o instanceof Map);
+					Map<String, String> map = (Map<String, String>) o;
+					assertEquals(1, map.size());
+					assertEquals("bår", map.get("føø"));
+				})
+						.verifyComplete(),
+				MediaType.parseMediaType("application/json; charset=iso-8859-1"),
+				null);
+	}
+
 	@Test
 	public void decodeMonoNonUtf8Encoding() {
 		Mono<DataBuffer> input = stringBuffer("{\"foo\":\"bar\"}", StandardCharsets.UTF_16);

spring-web/src/test/java/org/springframework/http/codec/json/Jackson2SmileDecoderTests.java

@@ -16,7 +16,6 @@
 
 package org.springframework.http.codec.json;
 
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
@@ -65,11 +64,6 @@ public class Jackson2SmileDecoderTests extends AbstractDecoderTestCase<Jackson2S
 
 		assertFalse(decoder.canDecode(forClass(String.class), null));
 		assertFalse(decoder.canDecode(forClass(Pojo.class), APPLICATION_JSON));
-
-		assertTrue(this.decoder.canDecode(ResolvableType.forClass(Pojo.class),
-				new MimeType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
-		assertFalse(this.decoder.canDecode(ResolvableType.forClass(Pojo.class),
-				new MimeType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));
 	}
 
 

spring-web/src/test/java/org/springframework/http/codec/json/Jackson2SmileEncoderTests.java

@@ -18,7 +18,6 @@ package org.springframework.http.codec.json;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.List;
 
@@ -71,11 +70,6 @@ public class Jackson2SmileEncoderTests extends AbstractEncoderTestCase<Jackson2S
 		assertTrue(this.encoder.canEncode(pojoType, STREAM_SMILE_MIME_TYPE));
 		assertTrue(this.encoder.canEncode(pojoType, null));
 
-		assertTrue(this.encoder.canEncode(ResolvableType.forClass(Pojo.class),
-				new MimeType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
-		assertFalse(this.encoder.canEncode(ResolvableType.forClass(Pojo.class),
-				new MimeType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));
-
 		// SPR-15464
 		assertTrue(this.encoder.canEncode(ResolvableType.NONE, null));
 	}

spring-web/src/test/java/org/springframework/http/converter/json/MappingJackson2HttpMessageConverterTests.java

@@ -18,6 +18,7 @@ package org.springframework.http.converter.json;
 
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -64,7 +65,7 @@ public class MappingJackson2HttpMessageConverterTests {
 		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json")));
 		assertTrue(converter.canRead(Map.class, new MediaType("application", "json")));
 		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.UTF_8)));
-		assertFalse(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.ISO_8859_1)));
+		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "json", StandardCharsets.ISO_8859_1)));
 	}
 
 	@Test
@@ -439,7 +440,7 @@ public class MappingJackson2HttpMessageConverterTests {
 	@Test
 	public void readWithNoDefaultConstructor() throws Exception {
 		String body = "{\"property1\":\"foo\",\"property2\":\"bar\"}";
-		MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes("UTF-8"));
+		MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes(StandardCharsets.UTF_8));
 		inputMessage.getHeaders().setContentType(new MediaType("application", "json"));
 		try {
 			converter.read(BeanWithNoDefaultConstructor.class, inputMessage);
@@ -451,6 +452,19 @@ public class MappingJackson2HttpMessageConverterTests {
 		fail();
 	}
 
+	@Test
+	@SuppressWarnings("unchecked")
+	public void readNonUnicode() throws Exception {
+		String body = "{\"føø\":\"bår\"}";
+		Charset charset = StandardCharsets.ISO_8859_1;
+		MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes(charset));
+		inputMessage.getHeaders().setContentType(new MediaType("application", "json", charset));
+		HashMap<String, Object> result = (HashMap<String, Object>) this.converter.read(HashMap.class, inputMessage);
+
+		assertEquals(1, result.size());
+		assertEquals("bår", result.get("føø"));
+	}
+
 
 	interface MyInterface {
 

spring-web/src/test/java/org/springframework/http/converter/smile/MappingJackson2SmileHttpMessageConverterTests.java

@@ -49,8 +49,6 @@ public class MappingJackson2SmileHttpMessageConverterTests {
 		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "x-jackson-smile")));
 		assertFalse(converter.canRead(MyBean.class, new MediaType("application", "json")));
 		assertFalse(converter.canRead(MyBean.class, new MediaType("application", "xml")));
-		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
-		assertFalse(converter.canRead(MyBean.class, new MediaType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));
 	}
 
 	@Test
@@ -58,8 +56,6 @@ public class MappingJackson2SmileHttpMessageConverterTests {
 		assertTrue(converter.canWrite(MyBean.class, new MediaType("application", "x-jackson-smile")));
 		assertFalse(converter.canWrite(MyBean.class, new MediaType("application", "json")));
 		assertFalse(converter.canWrite(MyBean.class, new MediaType("application", "xml")));
-		assertTrue(converter.canWrite(MyBean.class, new MediaType("application", "x-jackson-smile", StandardCharsets.UTF_8)));
-		assertFalse(converter.canWrite(MyBean.class, new MediaType("application", "x-jackson-smile", StandardCharsets.ISO_8859_1)));
 	}
 
 	@Test

spring-web/src/test/java/org/springframework/http/converter/xml/MappingJackson2XmlHttpMessageConverterTests.java

@@ -17,6 +17,7 @@
 package org.springframework.http.converter.xml;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
 import com.fasterxml.jackson.annotation.JsonView;
@@ -55,7 +56,7 @@ public class MappingJackson2XmlHttpMessageConverterTests {
 		assertTrue(converter.canRead(MyBean.class, new MediaType("text", "xml")));
 		assertTrue(converter.canRead(MyBean.class, new MediaType("application", "soap+xml")));
 		assertTrue(converter.canRead(MyBean.class, new MediaType("text", "xml", StandardCharsets.UTF_8)));
-		assertFalse(converter.canRead(MyBean.class, new MediaType("text", "xml", StandardCharsets.ISO_8859_1)));
+		assertTrue(converter.canRead(MyBean.class, new MediaType("text", "xml", StandardCharsets.ISO_8859_1)));
 	}
 
 	@Test
@@ -194,6 +195,21 @@ public class MappingJackson2XmlHttpMessageConverterTests {
 		this.converter.read(MyBean.class, inputMessage);
 	}
 
+	@Test
+	@SuppressWarnings("unchecked")
+	public void readNonUnicode() throws Exception {
+		String body = "<MyBean>" +
+				"<string>føø bår</string>" +
+				"</MyBean>";
+
+		Charset charset = StandardCharsets.ISO_8859_1;
+		MockHttpInputMessage inputMessage = new MockHttpInputMessage(body.getBytes(charset));
+		inputMessage.getHeaders().setContentType(new MediaType("application", "xml", charset));
+		MyBean result = (MyBean) converter.read(MyBean.class, inputMessage);
+		assertEquals("føø bår", result.getString());
+	}
+
+
 
 	public static class MyBean {