diff --git a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/MappingMongoConverter.java b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/MappingMongoConverter.java
index 88d3e72a1..601555ca0 100644
--- a/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/MappingMongoConverter.java
+++ b/spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/convert/MappingMongoConverter.java
@@ -1040,7 +1040,7 @@ public class MappingMongoConverter extends AbstractMongoConverter implements App
 	@SuppressWarnings({ "rawtypes", "unchecked" })
 	private Object getPotentiallyConvertedSimpleRead(Object value, @Nullable Class<?> target) {
 
-		if (target == null || ClassUtils.isAssignableValue(target, value)) {
+		if (target == null) {
 			return value;
 		}
 
@@ -1048,6 +1048,10 @@ public class MappingMongoConverter extends AbstractMongoConverter implements App
 			return doConvert(value, target);
 		}
 
+		if (ClassUtils.isAssignableValue(target, value)) {
+			return value;
+		}
+
 		if (Enum.class.isAssignableFrom(target)) {
 			return Enum.valueOf((Class<Enum>) target, value.toString());
 		}
diff --git a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/convert/MappingMongoConverterUnitTests.java b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/convert/MappingMongoConverterUnitTests.java
index 494439594..911c0a2bc 100644
--- a/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/convert/MappingMongoConverterUnitTests.java
+++ b/spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/convert/MappingMongoConverterUnitTests.java
@@ -30,6 +30,7 @@ import java.time.LocalDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
 
+import org.bson.types.Binary;
 import org.bson.types.Code;
 import org.bson.types.Decimal128;
 import org.bson.types.ObjectId;
@@ -2555,6 +2556,21 @@ class MappingMongoConverterUnitTests {
 				.containsEntry("item3", "i3");
 	}
 
+	@Test // GH-3670
+	void appliesCustomConverterEvenToSimpleTypes() {
+
+		converter = new MappingMongoConverter(resolver, mappingContext);
+		converter.setCustomConversions(MongoCustomConversions.create(it -> {
+			it.registerConverter(new MongoSimpleTypeConverter());
+		}));
+		converter.afterPropertiesSet();
+
+		org.bson.Document source = new org.bson.Document("content", new Binary(new byte[] {0x00, 0x42}));
+
+		GenericType<Object> target = converter.read(GenericType.class, source);
+		assertThat(target.content).isInstanceOf(byte[].class);
+	}
+
 	static class GenericType<T> {
 		T content;
 	}
@@ -3123,6 +3139,15 @@ class MappingMongoConverterUnitTests {
 		}
 	}
 
+	@ReadingConverter
+	public static class MongoSimpleTypeConverter implements Converter<Binary, Object> {
+
+		@Override
+		public byte[] convert(Binary source) {
+			return source.getData();
+		}
+	}
+
 	static class TypeWrappingTypeImplementingMap {
 
 		String id;