DATAMONGO-1585 - Expose synthetic fields in $project aggregation stage.

Field projections now expose their fields as synthetic simple fields. Projection aggregation stage redefines the available field set available for later aggregation stages entirely so projected fields are considered synthetic. A simple synthetic field has no target field which causes later aggregation stages to not pick up the underlying target but the exposed field name when rendering aggregation operations to Mongo documents.

The change is motivated by a bug where previously an aggregation consisting of projection of an aliased field and sort caused the sort projection stage to render with the original field name instead of the aliased field. The sort did not apply any sorting since projection redefines the available field set entirely and the original field is no longer accessible.

Original Pull Request: #433

spring-data-mongodb/src/main/java/org/springframework/data/mongodb/core/aggregation/ProjectionOperation.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2015 the original author or authors.
+ * Copyright 2013-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+
 import org.springframework.data.mongodb.core.aggregation.ExposedFields.ExposedField;
 import org.springframework.data.mongodb.core.aggregation.Fields.AggregationField;
 import org.springframework.data.mongodb.core.aggregation.ProjectionOperation.ProjectionOperationBuilder.FieldProjection;
@@ -552,7 +553,7 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 		/**
 		 * Generates an {@code $mod} expression that divides the value of the given field by the previously mentioned field
 		 * and returns the remainder.
-		 * 
+		 *
 		 * @param fieldReference
 		 * @return
 		 */
@@ -566,7 +567,7 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 			return project("size");
 		}
 
-		/* 
+		/*
 		 * (non-Javadoc)
 		 * @see org.springframework.data.mongodb.core.aggregation.AggregationOperation#toDBObject(org.springframework.data.mongodb.core.aggregation.AggregationOperationContext)
 		 */
@@ -622,6 +623,7 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 		 * 
 		 * @author Oliver Gierke
 		 * @author Thomas Darimont
+		 * @author Mark Paluch
 		 */
 		static class FieldProjection extends Projection {
 
@@ -640,7 +642,7 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 
 			private FieldProjection(Field field, Object value) {
 
-				super(field);
+				super(new ExposedField(field.getName(), true));
 
 				this.field = field;
 				this.value = value;
@@ -732,7 +734,7 @@ public class ProjectionOperation implements FieldsExposingAggregationOperation {
 				this.values = Arrays.asList(values);
 			}
 
-			/* 
+			/*
 			 * (non-Javadoc)
 			 * @see org.springframework.data.mongodb.core.aggregation.ProjectionOperation.Projection#toDBObject(org.springframework.data.mongodb.core.aggregation.AggregationOperationContext)
 			 */

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/aggregation/AggregationUnitTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2015 the original author or authors.
+ * Copyright 2013-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import static org.junit.Assert.*;
 import static org.springframework.data.mongodb.core.DBObjectTestUtils.*;
 import static org.springframework.data.mongodb.core.aggregation.Aggregation.*;
 import static org.springframework.data.mongodb.core.query.Criteria.*;
+import static org.springframework.data.mongodb.test.util.IsBsonObject.*;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -32,6 +33,7 @@ import org.springframework.data.domain.Sort.Direction;
 import com.mongodb.BasicDBObject;
 import com.mongodb.BasicDBObjectBuilder;
 import com.mongodb.DBObject;
+import com.mongodb.util.JSON;
 
 /**
  * Unit tests for {@link Aggregation}.
@@ -204,6 +206,47 @@ public class AggregationUnitTests {
 		assertThat(id.get("ruleType"), is((Object) "$rules.ruleType"));
 	}
 
+	/**
+	 * @see DATAMONGO-1585
+	 */
+	@Test
+	public void shouldSupportSortingBySyntheticAndExposedGroupFields() {
+
+		DBObject agg = newAggregation( //
+				group("cmsParameterId").addToSet("title").as("titles"), //
+				sort(Direction.ASC, "cmsParameterId", "titles") //
+		).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
+
+		assertThat(agg, is(notNullValue()));
+
+		DBObject sort = ((List<DBObject>) agg.get("pipeline")).get(1);
+
+		assertThat(getAsDBObject(sort, "$sort"), is(JSON.parse("{ \"_id.cmsParameterId\" : 1 , \"titles\" : 1}")));
+	}
+
+	/**
+	 * @see DATAMONGO-1585
+	 */
+	@Test
+	public void shouldSupportSortingByProjectedFields() {
+
+		DBObject agg = newAggregation( //
+				project("cmsParameterId") //
+						.and(SystemVariable.CURRENT + ".titles").as("titles") //
+						.and("field").as("alias"), //
+				sort(Direction.ASC, "cmsParameterId", "titles", "alias") //
+		).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
+
+		assertThat(agg, is(notNullValue()));
+
+		DBObject sort = ((List<DBObject>) agg.get("pipeline")).get(1);
+
+		assertThat(getAsDBObject(sort, "$sort"),
+				isBsonObject().containing("cmsParameterId", 1) //
+						.containing("titles", 1) //
+						.containing("alias", 1));
+	}
+
 	/**
 	 * @see DATAMONGO-924
 	 */
@@ -248,19 +291,20 @@ public class AggregationUnitTests {
 		DBObject agg = newAggregation( //
 				project().and("a").as("aa") //
 		) //
-		.withOptions(aggregationOptions) //
+				.withOptions(aggregationOptions) //
 				.toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
 
-		assertThat(agg.toString(), is("{ \"aggregate\" : \"foo\" , " //
-				+ "\"pipeline\" : [ { \"$project\" : { \"aa\" : \"$a\"}}] , " //
-				+ "\"allowDiskUse\" : true , " //
-				+ "\"explain\" : true , " //
-				+ "\"cursor\" : { \"foo\" : 1}}" //
-		));
+		assertThat(agg.toString(),
+				is("{ \"aggregate\" : \"foo\" , " //
+						+ "\"pipeline\" : [ { \"$project\" : { \"aa\" : \"$a\"}}] , " //
+						+ "\"allowDiskUse\" : true , " //
+						+ "\"explain\" : true , " //
+						+ "\"cursor\" : { \"foo\" : 1}}" //
+				));
 	}
 
 	/**
-	 * @see DATAMONGO-954
+	 * @see DATAMONGO-954, DATAMONGO-1585
 	 */
 	@Test
 	public void shouldSupportReferencingSystemVariables() {
@@ -269,16 +313,16 @@ public class AggregationUnitTests {
 				project("someKey") //
 						.and("a").as("a1") //
 						.and(Aggregation.CURRENT + ".a").as("a2") //
-				, sort(Direction.DESC, "a") //
+				, sort(Direction.DESC, "a1") //
 				, group("someKey").first(Aggregation.ROOT).as("doc") //
 		).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
 
 		DBObject projection0 = extractPipelineElement(agg, 0, "$project");
-		assertThat(projection0, is((DBObject) new BasicDBObject("someKey", 1).append("a1", "$a")
-				.append("a2", "$$CURRENT.a")));
+		assertThat(projection0,
+				is((DBObject) new BasicDBObject("someKey", 1).append("a1", "$a").append("a2", "$$CURRENT.a")));
 
 		DBObject sort = extractPipelineElement(agg, 1, "$sort");
-		assertThat(sort, is((DBObject) new BasicDBObject("a", -1)));
+		assertThat(sort, is((DBObject) new BasicDBObject("a1", -1)));
 
 		DBObject group = extractPipelineElement(agg, 2, "$group");
 		assertThat(group,
@@ -296,7 +340,7 @@ public class AggregationUnitTests {
 						.and("tags").minus(10).as("tags_count")//
 				, group("date")//
 						.sum("tags_count").as("count")//
-				).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
+		).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
 
 		DBObject group = extractPipelineElement(agg, 1, "$group");
 		assertThat(getAsDBObject(group, "count"), is(new BasicDBObjectBuilder().add("$sum", "$tags_count").get()));
@@ -313,7 +357,7 @@ public class AggregationUnitTests {
 						.andExpression("tags-10")//
 				, group("date")//
 						.sum("tags_count").as("count")//
-				).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
+		).toDbObject("foo", Aggregation.DEFAULT_CONTEXT);
 
 		DBObject group = extractPipelineElement(agg, 1, "$group");
 		assertThat(getAsDBObject(group, "count"), is(new BasicDBObjectBuilder().add("$sum", "$tags_count").get()));

spring-data-mongodb/src/test/java/org/springframework/data/mongodb/core/aggregation/TypeBasedAggregationOperationContextUnitTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2016 the original author or authors.
+ * Copyright 2013-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -171,6 +171,23 @@ public class TypeBasedAggregationOperationContextUnitTests {
 		assertThat(dbo.get("cursor"), is((Object) new BasicDBObject("foo", 1)));
 	}
 
+	/**
+	 * @see DATAMONGO-1585
+	 */
+	@Test
+	public void rendersSortOfProjectedFieldCorrectly() {
+
+		TypeBasedAggregationOperationContext context = getContext(MeterData.class);
+		TypedAggregation<MeterData> agg = newAggregation(MeterData.class, project().and("counterName").as("counter"), //
+				sort(Direction.ASC, "counter"));
+
+		DBObject dbo = agg.toDbObject("meterData", context);
+		DBObject sort = getPipelineElementFromAggregationAt(dbo, 1);
+
+		DBObject definition = (DBObject) sort.get("$sort");
+		assertThat(definition.get("counter"), is(equalTo((Object) 1)));
+	}
+
 	/**
 	 * @see DATAMONGO-1133
 	 */
@@ -190,14 +207,15 @@ public class TypeBasedAggregationOperationContextUnitTests {
 	}
 
 	/**
-	 * @see DATAMONGO-1326
+	 * @see DATAMONGO-1326, DATAMONGO-1585
 	 */
 	@Test
 	public void lookupShouldInheritFieldsFromInheritingAggregationOperation() {
 
 		TypeBasedAggregationOperationContext context = getContext(MeterData.class);
 		TypedAggregation<MeterData> agg = newAggregation(MeterData.class,
-				lookup("OtherCollection", "resourceId", "otherId", "lookup"), sort(Direction.ASC, "resourceId"));
+				lookup("OtherCollection", "resourceId", "otherId", "lookup"), //
+				sort(Direction.ASC, "resourceId", "counterName"));
 
 		DBObject dbo = agg.toDbObject("meterData", context);
 		DBObject sort = getPipelineElementFromAggregationAt(dbo, 1);
@@ -205,6 +223,7 @@ public class TypeBasedAggregationOperationContextUnitTests {
 		DBObject definition = (DBObject) sort.get("$sort");
 
 		assertThat(definition.get("resourceId"), is(equalTo((Object) 1)));
+		assertThat(definition.get("counter_name"), is(equalTo((Object) 1)));
 	}
 
 	/**

src/main/asciidoc/reference/mongodb.adoc

@@ -1678,8 +1678,8 @@ Note that the aggregation operations not listed here are currently not supported
 [[mongo.aggregation.projection]]
 === Projection Expressions
 
-Projection expressions are used to define the fields that are the outcome of a particular aggregation step. Projection expressions can be defined via the `project` method of the `Aggregate` class either by passing a list of `String` 's or an aggregation framework `Fields` object. The projection can be extended with additional fields through a fluent API via the `and(String)` method and aliased via the `as(String)` method.
-Note that one can also define fields with aliases via the static factory method `Fields.field` of the aggregation framework that can then be used to construct a new `Fields` instance.
+Projection expressions are used to define the fields that are the outcome of a particular aggregation step. Projection expressions can be defined via the `project` method of the `Aggregate` class either by passing a list of ``String``'s or an aggregation framework `Fields` object. The projection can be extended with additional fields through a fluent API via the `and(String)` method and aliased via the `as(String)` method.
+Note that one can also define fields with aliases via the static factory method `Fields.field` of the aggregation framework that can then be used to construct a new `Fields` instance. References to projected fields in later aggregation stages are only valid by using the field name of included fields or their alias of aliased or newly defined fields. Fields not included in the projection cannot be referenced in later aggregation stages.
 
 .Projection expression examples
 ====
@@ -1691,9 +1691,19 @@ project("a","b").and("foo").as("bar") // will generate {$project: {a: 1, b: 1, b
 ----
 ====
 
-Note that more examples for project operations can be found in the `AggregationTests` class.
+.Multi-Stage Aggregation using Projection and Sorting
+====
+[source,java]
+----
+project("name", "netPrice"), sort(ASC, "name") // will generate {$project: {name: 1, netPrice: 1}}, {$sort: {name: 1}}
+
+project().and("foo").as("bar"), sort(ASC, "bar") // will generate {$project: {bar: $foo}}, {$sort: {bar: 1}}
+
+project().and("foo").as("bar"), sort(ASC, "foo") // this will not work
+----
+====
 
-Note that further details regarding the projection expressions can be found in the http://docs.mongodb.org/manual/reference/operator/aggregation/project/#pipe._S_project[corresponding section] of the MongoDB Aggregation Framework reference documentation.
+More examples for project operations can be found in the `AggregationTests` class. Note that further details regarding the projection expressions can be found in the http://docs.mongodb.org/manual/reference/operator/aggregation/project/#pipe._S_project[corresponding section] of the MongoDB Aggregation Framework reference documentation.
 
 [[mongo.aggregation.projection.expressions]]
 ==== Spring Expression Support in Projection Expressions
@@ -1823,7 +1833,7 @@ ZipInfoStats firstZipInfoStats = result.getMappedResults().get(0);
 
 * The class `ZipInfo` maps the structure of the given input-collection. The class `ZipInfoStats` defines the structure in the desired output format.
 * As a first step we use the `group` operation to define a group from the input-collection. The grouping criteria is the combination of the fields `"state"` and `"city"` which forms the id structure of the group. We aggregate the value of the `"population"` property from the grouped elements with by using the `sum` operator saving the result in the field `"pop"`.
-* In a second step we use the `sort` operation to sort the intermediate-result by the fields `"pop"`, `"state"` and `"city"` in ascending order, such that the smallest city is at the top and the biggest city is at the bottom of the result. Note that the sorting on "state" and `"city"` is implicitly performed against the group id fields which Spring Data MongoDB took care of.
+* In a second step we use the `sort` operation to sort the intermediate-result by the fields `"pop"`, `"state"` and `"city"` in ascending order, such that the smallest city is at the top and the biggest city is at the bottom of the result. Note that the sorting on `"state"` and `"city"` is implicitly performed against the group id fields which Spring Data MongoDB took care of.
 * In the third step we use a `group` operation again to group the intermediate result by `"state"`. Note that `"state"` again implicitly references an group-id field. We select the name and the population count of the biggest and smallest city with calls to the `last(…)` and `first(...)` operator respectively via the `project` operation.
 * As the forth step we select the `"state"` field from the previous `group` operation. Note that `"state"` again implicitly references an group-id field. As we do not want an implicitly generated id to appear, we exclude the id from the previous operation via `and(previousOperation()).exclude()`. As we want to populate the nested `City` structures in our output-class accordingly we have to emit appropriate sub-documents with the nested method.
 * Finally as the fifth step we sort the resulting list of `StateStats` by their state name in ascending order via the `sort` operation.