Fix keyset backwards scrolling.

We now correctly scroll backwards by reversing sort order to apply the correct limit and reverse the results again to restore the actual sort order.

Closes #4332

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/MongoTemplate.java

@@ -66,7 +66,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext;
 import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext;
 import org.springframework.data.mongodb.core.QueryOperations.QueryContext;
 import org.springframework.data.mongodb.core.QueryOperations.UpdateContext;
-import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery;
+import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery;
 import org.springframework.data.mongodb.core.aggregation.Aggregation;
 import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext;
 import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
@@ -876,14 +876,14 @@ public class MongoTemplate
 
 		if (query.hasKeyset()) {
 
-			KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
+			KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
 					operations.getIdPropertyName(sourceClass));
 
 			List<T> result = doFind(collectionName, createDelegate(query), keysetPaginationQuery.query(),
 					keysetPaginationQuery.fields(), sourceClass,
 					new QueryCursorPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback);
 
-			return ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), result, sourceClass, operations);
+			return ScrollUtils.createWindow(query, result, sourceClass, operations);
 		}
 
 		List<T> result = doFind(collectionName, createDelegate(query), query.getQueryObject(), query.getFieldsObject(),

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ReactiveMongoTemplate.java

@@ -80,7 +80,7 @@ import org.springframework.data.mongodb.core.QueryOperations.DeleteContext;
 import org.springframework.data.mongodb.core.QueryOperations.DistinctQueryContext;
 import org.springframework.data.mongodb.core.QueryOperations.QueryContext;
 import org.springframework.data.mongodb.core.QueryOperations.UpdateContext;
-import org.springframework.data.mongodb.core.ScrollUtils.KeySetScrollQuery;
+import org.springframework.data.mongodb.core.ScrollUtils.KeysetScrollQuery;
 import org.springframework.data.mongodb.core.aggregation.Aggregation;
 import org.springframework.data.mongodb.core.aggregation.AggregationOperationContext;
 import org.springframework.data.mongodb.core.aggregation.AggregationOptions;
@@ -855,14 +855,14 @@ public class ReactiveMongoTemplate implements ReactiveMongoOperations, Applicati
 
 		if (query.hasKeyset()) {
 
-			KeySetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
+			KeysetScrollQuery keysetPaginationQuery = ScrollUtils.createKeysetPaginationQuery(query,
 					operations.getIdPropertyName(sourceClass));
 
 			Mono<List<T>> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query),
 					keysetPaginationQuery.query(), keysetPaginationQuery.fields(), sourceClass,
 					new QueryFindPublisherPreparer(query, keysetPaginationQuery.sort(), limit, 0, sourceClass), callback).collectList();
 
-			return result.map(it -> ScrollUtils.createWindow(query.getSortObject(), query.getLimit(), it, sourceClass, operations));
+			return result.map(it -> ScrollUtils.createWindow(query, it, sourceClass, operations));
 		}
 
 		Mono<List<T>> result = doFind(collectionName, ReactiveCollectionPreparerDelegate.of(query), query.getQueryObject(),

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/ScrollUtils.java

@@ -16,6 +16,7 @@
 package org.springframework.data.mongodb.core;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.function.IntFunction;
@@ -45,32 +46,110 @@ class ScrollUtils {
 	 * @param idPropertyName
 	 * @return
 	 */
-	static KeySetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) {
+	static KeysetScrollQuery createKeysetPaginationQuery(Query query, String idPropertyName) {
 
 		KeysetScrollPosition keyset = query.getKeyset();
-		Map<String, Object> keysetValues = keyset.getKeys();
-		Document queryObject = query.getQueryObject();
+		KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection());
+		Document sortObject = director.getSortObject(idPropertyName, query);
+		Document fieldsObject = director.getFieldsObject(query.getFieldsObject(), sortObject);
+		Document queryObject = director.createQuery(keyset, query.getQueryObject(), sortObject);
 
-		Document sortObject = query.isSorted() ? query.getSortObject() : new Document();
-		sortObject.put(idPropertyName, 1);
+		return new KeysetScrollQuery(queryObject, fieldsObject, sortObject);
+	}
 
-		// make sure we can extract the keyset
-		Document fieldsObject = query.getFieldsObject();
-		if (!fieldsObject.isEmpty()) {
-			for (String field : sortObject.keySet()) {
-				fieldsObject.put(field, 1);
-			}
+	static <T> Window<T> createWindow(Query query, List<T> result, Class<?> sourceType, EntityOperations operations) {
+
+		Document sortObject = query.getSortObject();
+		KeysetScrollPosition keyset = query.getKeyset();
+		KeysetScrollDirector director = KeysetScrollDirector.of(keyset.getDirection());
+
+		director.postPostProcessResults(result);
+
+		IntFunction<KeysetScrollPosition> positionFunction = value -> {
+
+			T last = result.get(value);
+			Entity<T> entity = operations.forEntity(last);
+
+			Map<String, Object> keys = entity.extractKeys(sortObject, sourceType);
+			return KeysetScrollPosition.of(keys);
+		};
+
+		return createWindow(result, query.getLimit(), positionFunction);
+	}
+
+	static <T> Window<T> createWindow(List<T> result, int limit, IntFunction<? extends ScrollPosition> positionFunction) {
+		return Window.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit));
+	}
+
+	static boolean hasMoreElements(List<?> result, int limit) {
+		return !result.isEmpty() && result.size() > limit;
+	}
+
+	static <T> List<T> getSubList(List<T> result, int limit) {
+
+		if (limit > 0 && result.size() > limit) {
+			return result.subList(0, limit);
 		}
 
-		List<Document> or = (List<Document>) queryObject.getOrDefault("$or", new ArrayList<>());
-		List<String> sortKeys = new ArrayList<>(sortObject.keySet());
+		return result;
+	}
+
+	record KeysetScrollQuery(Document query, Document fields, Document sort) {
 
-		if (!keysetValues.isEmpty() && !keysetValues.keySet().containsAll(sortKeys)) {
-			throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values");
+	}
+
+	/**
+	 * Director for keyset scrolling.
+	 */
+	static class KeysetScrollDirector {
+
+		private static final KeysetScrollDirector forward = new KeysetScrollDirector();
+		private static final KeysetScrollDirector reverse = new ReverseKeysetScrollDirector();
+
+		/**
+		 * Factory method to obtain the right {@link KeysetScrollDirector}.
+		 *
+		 * @param direction
+		 * @return
+		 */
+		public static KeysetScrollDirector of(KeysetScrollPosition.Direction direction) {
+			return direction == Direction.Forward ? forward : reverse;
+		}
+
+		public Document getSortObject(String idPropertyName, Query query) {
+
+			Document sortObject = query.isSorted() ? query.getSortObject() : new Document();
+			sortObject.put(idPropertyName, 1);
+
+			return sortObject;
 		}
 
-		// first query doesn't come with a keyset
-		if (!keysetValues.isEmpty()) {
+		public Document getFieldsObject(Document fieldsObject, Document sortObject) {
+
+			// make sure we can extract the keyset
+			if (!fieldsObject.isEmpty()) {
+				for (String field : sortObject.keySet()) {
+					fieldsObject.put(field, 1);
+				}
+			}
+
+			return fieldsObject;
+		}
+
+		public Document createQuery(KeysetScrollPosition keyset, Document queryObject, Document sortObject) {
+
+			Map<String, Object> keysetValues = keyset.getKeys();
+			List<Document> or = (List<Document>) queryObject.getOrDefault("$or", new ArrayList<>());
+			List<String> sortKeys = new ArrayList<>(sortObject.keySet());
+
+			// first query doesn't come with a keyset
+			if (keysetValues.isEmpty()) {
+				return queryObject;
+			}
+
+			if (!keysetValues.keySet().containsAll(sortKeys)) {
+				throw new IllegalStateException("KeysetScrollPosition does not contain all keyset values");
+			}
 
 			// build matrix query for keyset paging that contains sort^2 queries
 			// reflecting a query that follows sort order semantics starting from the last returned keyset
@@ -89,7 +168,7 @@ class ScrollUtils {
 							throw new IllegalStateException(
 									"Cannot resume from KeysetScrollPosition. Offending key: '%s' is 'null'".formatted(sortSegment));
 						}
-						sortConstraint.put(sortSegment, new Document(getComparator(sortOrder, keyset.getDirection()), o));
+						sortConstraint.put(sortSegment, new Document(getComparator(sortOrder), o));
 						break;
 					}
 
@@ -100,61 +179,60 @@ class ScrollUtils {
 					or.add(sortConstraint);
 				}
 			}
-		}
 
-		if (!or.isEmpty()) {
-			queryObject.put("$or", or);
-		}
+			if (!or.isEmpty()) {
+				queryObject.put("$or", or);
+			}
 
-		return new KeySetScrollQuery(queryObject, fieldsObject, sortObject);
-	}
+			return queryObject;
+		}
 
-	private static String getComparator(int sortOrder, Direction direction) {
+		public <T> void postPostProcessResults(List<T> result) {
 
-		// use gte/lte to include the object at the cursor/keyset so that
-		// we can include it in the result to check whether there is a next object.
-		// It needs to be filtered out later on.
-		if (direction == Direction.Backward) {
-			return sortOrder == 0 ? "$gte" : "$lte";
 		}
 
-		return sortOrder == 1 ? "$gt" : "$lt";
+		protected String getComparator(int sortOrder) {
+			return sortOrder == 1 ? "$gt" : "$lt";
+		}
 	}
 
-	static <T> Window<T> createWindow(Document sortObject, int limit, List<T> result, Class<?> sourceType,
-			EntityOperations operations) {
-
-		IntFunction<KeysetScrollPosition> positionFunction = value -> {
-
-			T last = result.get(value);
-			Entity<T> entity = operations.forEntity(last);
-
-			Map<String, Object> keys = entity.extractKeys(sortObject, sourceType);
-			return KeysetScrollPosition.of(keys);
-		};
+	/**
+	 * Reverse scrolling director variant applying {@link KeysetScrollPosition.Direction#Backward}. In reverse scrolling,
+	 * we need to flip directions for the actual query so that we do not get everything from the top position and apply
+	 * the limit but rather flip the sort direction, apply the limit and then reverse the result to restore the actual
+	 * sort order.
+	 */
+	private static class ReverseKeysetScrollDirector extends KeysetScrollDirector {
 
-		return createWindow(result, limit, positionFunction);
-	}
+		@Override
+		public Document getSortObject(String idPropertyName, Query query) {
 
-	static <T> Window<T> createWindow(List<T> result, int limit, IntFunction<? extends ScrollPosition> positionFunction) {
-		return Window.from(getSubList(result, limit), positionFunction, hasMoreElements(result, limit));
-	}
+			Document sortObject = super.getSortObject(idPropertyName, query);
 
-	static boolean hasMoreElements(List<?> result, int limit) {
-		return !result.isEmpty() && result.size() > limit;
-	}
+			// flip sort direction for backward scrolling
 
-	static <T> List<T> getSubList(List<T> result, int limit) {
+			for (String field : sortObject.keySet()) {
+				sortObject.put(field, sortObject.getInteger(field) == 1 ? -1 : 1);
+			}
 
-		if (limit > 0 && result.size() > limit) {
-			return result.subList(0, limit);
+			return sortObject;
 		}
 
-		return result;
-	}
+		@Override
+		protected String getComparator(int sortOrder) {
 
-	record KeySetScrollQuery(Document query, Document fields, Document sort) {
+			// use gte/lte to include the object at the cursor/keyset so that
+			// we can include it in the result to check whether there is a next object.
+			// It needs to be filtered out later on.
+			return sortOrder == 1 ? "$gte" : "$lte";
+		}
 
+		@Override
+		public <T> void postPostProcessResults(List<T> result) {
+			// flip direction of the result list as we need to accomodate for the flipped sort order for proper offset
+			// querying.
+			Collections.reverse(result);
+		}
 	}
 
 }

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/MongoTemplateScrollTests.java

@@ -167,42 +167,32 @@ class MongoTemplateScrollTests {
 	@Test // GH-4308
 	void shouldAllowReverseSort() {
 
-		WithNestedDocument john20 = new WithNestedDocument(null, "John", 120, new WithNestedDocument("John", 20),
-				new Document("name", "bar"));
-		WithNestedDocument john40 = new WithNestedDocument(null, "John", 140, new WithNestedDocument("John", 40),
-				new Document("name", "baz"));
-		WithNestedDocument john41 = new WithNestedDocument(null, "John", 141, new WithNestedDocument("John", 41),
-				new Document("name", "foo"));
-
-		template.insertAll(Arrays.asList(john20, john40, john41));
-
-		Query q = new Query(where("name").regex("J.*")).with(Sort.by("nested.name", "nested.age", "document.name"))
-				.limit(2);
-		q.with(KeysetScrollPosition.initial());
-
-		Window<WithNestedDocument> window = template.scroll(q, WithNestedDocument.class);
+		Person jane_20 = new Person("Jane", 20);
+		Person jane_40 = new Person("Jane", 40);
+		Person jane_42 = new Person("Jane", 42);
+		Person john20 = new Person("John", 20);
+		Person john40_1 = new Person("John", 40);
+		Person john40_2 = new Person("John", 40);
 
-		assertThat(window.hasNext()).isTrue();
-		assertThat(window.isLast()).isFalse();
-		assertThat(window).hasSize(2);
-		assertThat(window).containsOnly(john20, john40);
+		template.insertAll(Arrays.asList(john20, john40_1, john40_2, jane_20, jane_40, jane_42));
+		Query q = new Query(where("firstName").regex("J.*")).with(Sort.by("firstName", "age"));
+		q.with(KeysetScrollPosition.initial()).limit(6);
 
-		window = template.scroll(q.with(window.positionAt(window.size() - 1)), WithNestedDocument.class);
+		Window<Person> window = template.scroll(q, Person.class);
 
 		assertThat(window.hasNext()).isFalse();
 		assertThat(window.isLast()).isTrue();
-		assertThat(window).hasSize(1);
-		assertThat(window).containsOnly(john41);
+		assertThat(window).hasSize(6);
 
-		KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(0);
+		KeysetScrollPosition scrollPosition = (KeysetScrollPosition) window.positionAt(window.size() - 1);
 		KeysetScrollPosition reversePosition = KeysetScrollPosition.of(scrollPosition.getKeys(), Direction.Backward);
 
-		window = template.scroll(q.with(reversePosition), WithNestedDocument.class);
+		window = template.scroll(q.with(reversePosition).limit(2), Person.class);
 
+		assertThat(window).hasSize(2);
+		assertThat(window).containsOnly(john20, john40_1);
 		assertThat(window.hasNext()).isTrue();
 		assertThat(window.isLast()).isFalse();
-		assertThat(window).hasSize(2);
-		assertThat(window).containsOnly(john20, john40);
 	}
 
 	@ParameterizedTest // GH-4308