Postpone extParam/allParam calculations until the very end of planning.

Until now we computed these Param ID sets at the end of subquery_planner,
but that approach depends on subquery_planner returning a concrete Plan
tree.  We would like to switch over to returning one or more Paths for a
subquery, and in that representation the necessary details aren't fully
fleshed out (not to mention that we don't really want to do this work for
Paths that end up getting discarded).  Hence, refactor so that we can
compute the param ID sets at the end of planning, just before
set_plan_references is run.

The main change necessary to make this work is that we need to capture
the set of outer-level Param IDs available to the current query level
before exiting subquery_planner, since the outer levels' plan_params lists
are transient.  (That's not going to pose a problem for returning Paths,
since all the work involved in producing that data is part of expression
preprocessing, which will continue to happen before Paths are produced.)
On the plus side, this change gets rid of several existing kluges.

Eventually I'd like to get rid of SS_finalize_plan altogether in favor of
doing this work during set_plan_references, but that will require some
complex rejiggering because SS_finalize_plan needs to visit subplans and
initplans before the main plan.  So leave that idea for another day.

src/backend/nodes/outfuncs.c

@@ -1799,6 +1799,7 @@ _outPlannerInfo(StringInfo str, const PlannerInfo *node)
 	WRITE_NODE_FIELD(glob);
 	WRITE_UINT_FIELD(query_level);
 	WRITE_NODE_FIELD(plan_params);
+	WRITE_BITMAPSET_FIELD(outer_params);
 	WRITE_BITMAPSET_FIELD(all_baserels);
 	WRITE_BITMAPSET_FIELD(nullable_baserels);
 	WRITE_NODE_FIELD(join_rel_list);

src/backend/optimizer/plan/createplan.c

@@ -4473,11 +4473,7 @@ make_material(Plan *lefttree)
  * materialize_finished_plan: stick a Material node atop a completed plan
  *
  * There are a couple of places where we want to attach a Material node
- * after completion of subquery_planner().  This currently requires hackery.
- * Since subquery_planner has already run SS_finalize_plan on the subplan
- * tree, we have to kluge up parameter lists for the Material node.
- * Possibly this could be fixed by postponing SS_finalize_plan processing
- * until setrefs.c is run?
+ * after completion of subquery_planner(), without any MaterialPath path.
  */
 Plan *
 materialize_finished_plan(Plan *subplan)
@@ -4498,10 +4494,6 @@ materialize_finished_plan(Plan *subplan)
 	matplan->plan_rows = subplan->plan_rows;
 	matplan->plan_width = subplan->plan_width;
 
-	/* parameter kluge --- see comments above */
-	matplan->extParam = bms_copy(subplan->extParam);
-	matplan->allParam = bms_copy(subplan->allParam);
-
 	return matplan;
 }
 

src/backend/optimizer/plan/planagg.c

@@ -416,13 +416,23 @@ build_minmax_path(PlannerInfo *root, MinMaxAggInfo *mminfo,
 	 *		 WHERE col IS NOT NULL AND existing-quals
 	 *		 ORDER BY col ASC/DESC
 	 *		 LIMIT 1)
+	 *
+	 * We cheat a bit here by building what is effectively a subplan query
+	 * level without taking the trouble to increment varlevelsup of outer
+	 * references.  Therefore we don't increment the subroot's query_level nor
+	 * repoint its parent_root to the parent level.  We can get away with that
+	 * because the plan will be an initplan and therefore cannot need any
+	 * parameters from the parent level.  But see hackery in make_agg_subplan;
+	 * we might someday need to do this the hard way.
 	 *----------
 	 */
 	subroot = (PlannerInfo *) palloc(sizeof(PlannerInfo));
 	memcpy(subroot, root, sizeof(PlannerInfo));
 	subroot->parse = parse = (Query *) copyObject(root->parse);
-	/* make sure subroot planning won't change root->init_plans contents */
-	subroot->init_plans = list_copy(root->init_plans);
+	/* reset subplan-related stuff */
+	subroot->plan_params = NIL;
+	subroot->outer_params = NULL;
+	subroot->init_plans = NIL;
 	/* There shouldn't be any OJ or LATERAL info to translate, as yet */
 	Assert(subroot->join_info_list == NIL);
 	Assert(subroot->lateral_info_list == NIL);
@@ -577,24 +587,31 @@ make_agg_subplan(PlannerInfo *root, MinMaxAggInfo *mminfo)
 							   subparse->limitCount,
 							   0, 1);
 
+	/*
+	 * We have to do some of the same cleanup that subquery_planner() would
+	 * do, namely cope with params and initplans used within this plan tree.
+	 *
+	 * This is a little bit messy because although we initially created the
+	 * subroot by cloning the outer root, it really is a subplan and needs to
+	 * consider initplans belonging to the outer root as providing available
+	 * parameters.  So temporarily change its parent_root pointer.
+	 * (Fortunately, SS_identify_outer_params doesn't care whether the depth
+	 * of parent_root nesting matches query_level.)
+	 */
+	subroot->parent_root = root;
+	SS_identify_outer_params(subroot);
+	subroot->parent_root = root->parent_root;
+
+	SS_attach_initplans(subroot, plan);
+
 	/*
 	 * Convert the plan into an InitPlan, and make a Param for its result.
 	 */
 	mminfo->param =
-		SS_make_initplan_from_plan(subroot, plan,
+		SS_make_initplan_from_plan(root, subroot, plan,
 								   exprType((Node *) mminfo->target),
 								   -1,
 								   exprCollation((Node *) mminfo->target));
-
-	/*
-	 * Make sure the initplan gets into the outer PlannerInfo, along with any
-	 * other initplans generated by the sub-planning run.  We had to include
-	 * the outer PlannerInfo's pre-existing initplans into the inner one's
-	 * init_plans list earlier, so make sure we don't put back any duplicate
-	 * entries.
-	 */
-	root->init_plans = list_concat_unique_ptr(root->init_plans,
-											  subroot->init_plans);
 }
 
 /*

src/backend/optimizer/plan/planner.c

@@ -239,6 +239,25 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams)
 			top_plan = materialize_finished_plan(top_plan);
 	}
 
+	/*
+	 * If any Params were generated, run through the plan tree and compute
+	 * each plan node's extParam/allParam sets.  Ideally we'd merge this into
+	 * set_plan_references' tree traversal, but for now it has to be separate
+	 * because we need to visit subplans before not after main plan.
+	 */
+	if (glob->nParamExec > 0)
+	{
+		Assert(list_length(glob->subplans) == list_length(glob->subroots));
+		forboth(lp, glob->subplans, lr, glob->subroots)
+		{
+			Plan	   *subplan = (Plan *) lfirst(lp);
+			PlannerInfo *subroot = (PlannerInfo *) lfirst(lr);
+
+			SS_finalize_plan(subroot, subplan);
+		}
+		SS_finalize_plan(root, top_plan);
+	}
+
 	/* final cleanup of the plan */
 	Assert(glob->finalrtable == NIL);
 	Assert(glob->finalrowmarks == NIL);
@@ -312,7 +331,6 @@ subquery_planner(PlannerGlobal *glob, Query *parse,
 				 bool hasRecursion, double tuple_fraction,
 				 PlannerInfo **subroot)
 {
-	int			num_old_subplans = list_length(glob->subplans);
 	PlannerInfo *root;
 	Plan	   *plan;
 	List	   *newWithCheckOptions;
@@ -327,6 +345,7 @@ subquery_planner(PlannerGlobal *glob, Query *parse,
 	root->query_level = parent_root ? parent_root->query_level + 1 : 1;
 	root->parent_root = parent_root;
 	root->plan_params = NIL;
+	root->outer_params = NULL;
 	root->planner_cxt = CurrentMemoryContext;
 	root->init_plans = NIL;
 	root->cte_plan_ids = NIL;
@@ -656,13 +675,17 @@ subquery_planner(PlannerGlobal *glob, Query *parse,
 	}
 
 	/*
-	 * If any subplans were generated, or if there are any parameters to worry
-	 * about, build initPlan list and extParam/allParam sets for plan nodes,
-	 * and attach the initPlans to the top plan node.
+	 * Capture the set of outer-level param IDs we have access to, for use in
+	 * extParam/allParam calculations later.
+	 */
+	SS_identify_outer_params(root);
+
+	/*
+	 * If any initPlans were created in this query level, attach them to the
+	 * topmost plan node for the level, and increment that node's cost to
+	 * account for them.
 	 */
-	if (list_length(glob->subplans) != num_old_subplans ||
-		root->glob->nParamExec > 0)
-		SS_finalize_plan(root, plan, true);
+	SS_attach_initplans(root, plan);
 
 	/* Return internal info if caller wants it */
 	if (subroot)

src/backend/optimizer/prep/prepjointree.c

@@ -899,6 +899,7 @@ pull_up_simple_subquery(PlannerInfo *root, Node *jtnode, RangeTblEntry *rte,
 	subroot->query_level = root->query_level;
 	subroot->parent_root = root->parent_root;
 	subroot->plan_params = NIL;
+	subroot->outer_params = NULL;
 	subroot->planner_cxt = CurrentMemoryContext;
 	subroot->init_plans = NIL;
 	subroot->cte_plan_ids = NIL;

src/include/nodes/relation.h

@@ -131,7 +131,14 @@ typedef struct PlannerInfo
 
 	struct PlannerInfo *parent_root;	/* NULL at outermost Query */
 
+	/*
+	 * plan_params contains the expressions that this query level needs to
+	 * make available to a lower query level that is currently being planned.
+	 * outer_params contains the paramIds of PARAM_EXEC Params that outer
+	 * query levels will make available to this query level.
+	 */
 	List	   *plan_params;	/* list of PlannerParamItems, see below */
+	Bitmapset  *outer_params;
 
 	/*
 	 * simple_rel_array holds pointers to "base rels" and "other rels" (see

src/include/optimizer/subselect.h

@@ -25,9 +25,11 @@ extern JoinExpr *convert_EXISTS_sublink_to_join(PlannerInfo *root,
 							   Relids available_rels);
 extern Node *SS_replace_correlation_vars(PlannerInfo *root, Node *expr);
 extern Node *SS_process_sublinks(PlannerInfo *root, Node *expr, bool isQual);
-extern void SS_finalize_plan(PlannerInfo *root, Plan *plan,
-				 bool attach_initplans);
-extern Param *SS_make_initplan_from_plan(PlannerInfo *root, Plan *plan,
+extern void SS_identify_outer_params(PlannerInfo *root);
+extern void SS_attach_initplans(PlannerInfo *root, Plan *plan);
+extern void SS_finalize_plan(PlannerInfo *root, Plan *plan);
+extern Param *SS_make_initplan_from_plan(PlannerInfo *root,
+						   PlannerInfo *subroot, Plan *plan,
 					Oid resulttype, int32 resulttypmod, Oid resultcollation);
 extern Param *assign_nestloop_param_var(PlannerInfo *root, Var *var);
 extern Param *assign_nestloop_param_placeholdervar(PlannerInfo *root,

src/test/regress/expected/join.out

@@ -4889,6 +4889,63 @@ select * from
   0 | 9998 |  0
 (1 row)
 
+-- check proper extParam/allParam handling (this isn't exactly a LATERAL issue,
+-- but we can make the test case much more compact with LATERAL)
+explain (verbose, costs off)
+select * from (values (0), (1)) v(id),
+lateral (select * from int8_tbl t1,
+         lateral (select * from
+                    (select * from int8_tbl t2
+                     where q1 = any (select q2 from int8_tbl t3
+                                     where q2 = (select greatest(t1.q1,t2.q2))
+                                       and (select v.id=0)) offset 0) ss2) ss
+         where t1.q1 = ss.q2) ss0;
+                           QUERY PLAN                            
+-----------------------------------------------------------------
+ Nested Loop
+   Output: "*VALUES*".column1, t1.q1, t1.q2, ss2.q1, ss2.q2
+   ->  Seq Scan on public.int8_tbl t1
+         Output: t1.q1, t1.q2
+   ->  Nested Loop
+         Output: "*VALUES*".column1, ss2.q1, ss2.q2
+         ->  Values Scan on "*VALUES*"
+               Output: "*VALUES*".column1
+         ->  Subquery Scan on ss2
+               Output: ss2.q1, ss2.q2
+               Filter: (t1.q1 = ss2.q2)
+               ->  Seq Scan on public.int8_tbl t2
+                     Output: t2.q1, t2.q2
+                     Filter: (SubPlan 3)
+                     SubPlan 3
+                       ->  Result
+                             Output: t3.q2
+                             One-Time Filter: $4
+                             InitPlan 1 (returns $2)
+                               ->  Result
+                                     Output: GREATEST($0, t2.q2)
+                             InitPlan 2 (returns $4)
+                               ->  Result
+                                     Output: ($3 = 0)
+                             ->  Seq Scan on public.int8_tbl t3
+                                   Output: t3.q1, t3.q2
+                                   Filter: (t3.q2 = $2)
+(27 rows)
+
+select * from (values (0), (1)) v(id),
+lateral (select * from int8_tbl t1,
+         lateral (select * from
+                    (select * from int8_tbl t2
+                     where q1 = any (select q2 from int8_tbl t3
+                                     where q2 = (select greatest(t1.q1,t2.q2))
+                                       and (select v.id=0)) offset 0) ss2) ss
+         where t1.q1 = ss.q2) ss0;
+ id |        q1        |        q2         |        q1        |        q2        
+----+------------------+-------------------+------------------+------------------
+  0 | 4567890123456789 |               123 | 4567890123456789 | 4567890123456789
+  0 | 4567890123456789 |  4567890123456789 | 4567890123456789 | 4567890123456789
+  0 | 4567890123456789 | -4567890123456789 | 4567890123456789 | 4567890123456789
+(3 rows)
+
 -- test some error cases where LATERAL should have been used but wasn't
 select f1,g from int4_tbl a, (select f1 as g) ss;
 ERROR:  column "f1" does not exist

src/test/regress/sql/join.sql

@@ -1542,6 +1542,27 @@ select * from
            where f1 = any (select unique1 from tenk1
                            where unique2 = v.x offset 0)) ss;
 
+-- check proper extParam/allParam handling (this isn't exactly a LATERAL issue,
+-- but we can make the test case much more compact with LATERAL)
+explain (verbose, costs off)
+select * from (values (0), (1)) v(id),
+lateral (select * from int8_tbl t1,
+         lateral (select * from
+                    (select * from int8_tbl t2
+                     where q1 = any (select q2 from int8_tbl t3
+                                     where q2 = (select greatest(t1.q1,t2.q2))
+                                       and (select v.id=0)) offset 0) ss2) ss
+         where t1.q1 = ss.q2) ss0;
+
+select * from (values (0), (1)) v(id),
+lateral (select * from int8_tbl t1,
+         lateral (select * from
+                    (select * from int8_tbl t2
+                     where q1 = any (select q2 from int8_tbl t3
+                                     where q2 = (select greatest(t1.q1,t2.q2))
+                                       and (select v.id=0)) offset 0) ss2) ss
+         where t1.q1 = ss.q2) ss0;
+
 -- test some error cases where LATERAL should have been used but wasn't
 select f1,g from int4_tbl a, (select f1 as g) ss;
 select f1,g from int4_tbl a, (select a.f1 as g) ss;