Instead of rechecking lossy index operators by putting them into the

regular qpqual ('filter condition'), add special-purpose code to
nodeIndexscan.c to recheck them.  This ends being almost no net addition
of code, because the removal of planner code balances out the extra
executor code, but it is significantly more efficient when a lossy
operator is involved in an OR indexscan.  The old implementation had
to recheck the entire indexqual in such cases.

src/backend/executor/nodeIndexscan.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/executor/nodeIndexscan.c,v 1.88 2003/12/30 20:05:05 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/executor/nodeIndexscan.c,v 1.89 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -99,7 +99,9 @@ IndexNext(IndexScanState *node)
 	ExprContext *econtext;
 	ScanDirection direction;
 	IndexScanDescPtr scanDescs;
+	List	  **lossyQuals;
 	IndexScanDesc scandesc;
+	List	   *lossyQual;
 	Index		scanrelid;
 	HeapTuple	tuple;
 	TupleTableSlot *slot;
@@ -120,6 +122,7 @@ IndexNext(IndexScanState *node)
 			direction = ForwardScanDirection;
 	}
 	scanDescs = node->iss_ScanDescs;
+	lossyQuals = node->iss_LossyQuals;
 	numIndices = node->iss_NumIndices;
 	econtext = node->ss.ps.ps_ExprContext;
 	slot = node->ss.ss_ScanTupleSlot;
@@ -188,6 +191,8 @@ IndexNext(IndexScanState *node)
 	while (indexNumber < numIndices)
 	{
 		scandesc = scanDescs[node->iss_IndexPtr];
+		lossyQual = lossyQuals[node->iss_IndexPtr];
+
 		while ((tuple = index_getnext(scandesc, direction)) != NULL)
 		{
 			/*
@@ -201,6 +206,22 @@ IndexNext(IndexScanState *node)
 						   scandesc->xs_cbuf,	/* buffer containing tuple */
 						   false);		/* don't pfree */
 
+			/*
+			 * If any of the index operators involved in this scan are lossy,
+			 * recheck them by evaluating the original operator clauses.
+			 */
+			if (lossyQual)
+			{
+				econtext->ecxt_scantuple = slot;
+				ResetExprContext(econtext);
+				if (!ExecQual(lossyQual, econtext, false))
+				{
+					/* Fails lossy op, so drop it and loop back for another */
+					ExecClearTuple(slot);
+					continue;
+				}
+			}
+
 			/*
 			 * If it's a multiple-index scan, make sure not to double-report
 			 * a tuple matched by more than one index.  (See notes above.)
@@ -615,6 +636,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 	List	   *indxqual;
 	List	   *indxstrategy;
 	List	   *indxsubtype;
+	List	   *indxlossy;
 	List	   *indxid;
 	int			i;
 	int			numIndices;
@@ -623,6 +645,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 	int		   *numScanKeys;
 	RelationPtr indexDescs;
 	IndexScanDescPtr scanDescs;
+	List	  **lossyQuals;
 	ExprState ***runtimeKeyInfo;
 	bool		have_runtime_keys;
 	RangeTblEntry *rtentry;
@@ -680,6 +703,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 	indexstate->iss_RuntimeKeysReady = false;
 	indexstate->iss_RelationDescs = NULL;
 	indexstate->iss_ScanDescs = NULL;
+	indexstate->iss_LossyQuals = NULL;
 
 	/*
 	 * get the index node information
@@ -699,6 +723,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 	scanKeys = (ScanKey *) palloc(numIndices * sizeof(ScanKey));
 	indexDescs = (RelationPtr) palloc(numIndices * sizeof(Relation));
 	scanDescs = (IndexScanDescPtr) palloc(numIndices * sizeof(IndexScanDesc));
+	lossyQuals = (List **) palloc0(numIndices * sizeof(List *));
 
 	/*
 	 * initialize space for runtime key info (may not be needed)
@@ -712,11 +737,13 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 	indxqual = node->indxqual;
 	indxstrategy = node->indxstrategy;
 	indxsubtype = node->indxsubtype;
+	indxlossy = node->indxlossy;
 	for (i = 0; i < numIndices; i++)
 	{
 		List	   *quals;
 		List	   *strategies;
 		List	   *subtypes;
+		List	   *lossyflags;
 		int			n_keys;
 		ScanKey		scan_keys;
 		ExprState **run_keys;
@@ -728,6 +755,8 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 		indxstrategy = lnext(indxstrategy);
 		subtypes = lfirst(indxsubtype);
 		indxsubtype = lnext(indxsubtype);
+		lossyflags = lfirst(indxlossy);
+		indxlossy = lnext(indxlossy);
 		n_keys = length(quals);
 		scan_keys = (n_keys <= 0) ? (ScanKey) NULL :
 			(ScanKey) palloc(n_keys * sizeof(ScanKeyData));
@@ -747,6 +776,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 			AttrNumber	varattno;		/* att number used in scan */
 			StrategyNumber strategy;	/* op's strategy number */
 			Oid			subtype;		/* op's strategy subtype */
+			int			lossy;			/* op's recheck flag */
 			RegProcedure opfuncid;		/* operator proc id used in scan */
 			Datum		scanvalue;		/* value used in scan (if const) */
 
@@ -759,6 +789,8 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 			strategies = lnext(strategies);
 			subtype = lfirsto(subtypes);
 			subtypes = lnext(subtypes);
+			lossy = lfirsti(lossyflags);
+			lossyflags = lnext(lossyflags);
 
 			if (!IsA(clause, OpExpr))
 				elog(ERROR, "indxqual is not an OpExpr");
@@ -839,6 +871,20 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 								   subtype,		/* strategy subtype */
 								   opfuncid,	/* reg proc to use */
 								   scanvalue);	/* constant */
+
+			/*
+			 * If this operator is lossy, add its indxqualorig expression
+			 * to the list of quals to recheck.  The nth() calls here could
+			 * be avoided by chasing the lists in parallel to all the other
+			 * lists, but since lossy operators are very uncommon, it's
+			 * probably a waste of time to do so.
+			 */
+			if (lossy)
+			{
+				lossyQuals[i] = lappend(lossyQuals[i],
+										nth(j,
+											(List *) nth(i, indexstate->indxqualorig)));
+			}
 		}
 
 		/*
@@ -928,6 +974,7 @@ ExecInitIndexScan(IndexScan *node, EState *estate)
 
 	indexstate->iss_RelationDescs = indexDescs;
 	indexstate->iss_ScanDescs = scanDescs;
+	indexstate->iss_LossyQuals = lossyQuals;
 
 	/*
 	 * Initialize result tuple type and projection info.

src/backend/nodes/copyfuncs.c

@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/nodes/copyfuncs.c,v 1.273 2004/01/05 05:07:35 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/nodes/copyfuncs.c,v 1.274 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -284,6 +284,17 @@ _copyIndexScan(IndexScan *from)
 		}
 		newnode->indxsubtype = newsubtype;
 	}
+	/* this can become COPY_NODE_FIELD when intlists are normal objects: */
+	{
+		List	*newstrat = NIL;
+		List    *tmp;
+
+		foreach(tmp, from->indxlossy)
+		{
+			newstrat = lappend(newstrat, listCopy(lfirst(tmp)));
+		}
+		newnode->indxlossy = newstrat;
+	}
 	COPY_SCALAR_FIELD(indxorderdir);
 
 	return newnode;

src/backend/nodes/outfuncs.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/nodes/outfuncs.c,v 1.228 2004/01/05 23:39:53 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/nodes/outfuncs.c,v 1.229 2004/01/06 04:31:01 tgl Exp $
  *
  * NOTES
  *	  Every node type that can appear in stored rules' parsetrees *must*
@@ -356,6 +356,16 @@ _outIndexScan(StringInfo str, IndexScan *node)
 			_outOidList(str, lfirst(tmp));
 		}
 	}
+	/* this can become WRITE_NODE_FIELD when intlists are normal objects: */
+	{
+		List    *tmp;
+
+		appendStringInfo(str, " :indxlossy ");
+		foreach(tmp, node->indxlossy)
+		{
+			_outIntList(str, lfirst(tmp));
+		}
+	}
 	WRITE_ENUM_FIELD(indxorderdir, ScanDirection);
 }
 

src/backend/optimizer/path/costsize.c

@@ -49,7 +49,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/optimizer/path/costsize.c,v 1.121 2004/01/05 23:39:54 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/optimizer/path/costsize.c,v 1.122 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -384,10 +384,6 @@ cost_index(Path *path, Query *root,
 	 * some of the indexquals are join clauses and shouldn't be
 	 * subtracted. Rather than work out exactly how much to subtract, we
 	 * don't subtract anything.
-	 *
-	 * XXX For a lossy index, not all the quals will be removed and so we
-	 * really shouldn't subtract their costs; but detecting that seems
-	 * more expensive than it's worth.
 	 */
 	startup_cost += baserel->baserestrictcost.startup;
 	cpu_per_tuple = cpu_tuple_cost + baserel->baserestrictcost.per_tuple;
@@ -397,6 +393,7 @@ cost_index(Path *path, Query *root,
 		QualCost	index_qual_cost;
 
 		cost_qual_eval(&index_qual_cost, indexQuals);
+		/* any startup cost still has to be paid ... */
 		cpu_per_tuple -= index_qual_cost.per_tuple;
 	}
 

src/backend/optimizer/plan/setrefs.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/optimizer/plan/setrefs.c,v 1.99 2003/11/29 19:51:50 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/optimizer/plan/setrefs.c,v 1.100 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -362,8 +362,8 @@ set_join_references(Join *join, List *rtable)
 
 				/*
 				 * We must fix the inner qpqual too, if it has join
-				 * clauses (this could happen if the index is lossy: some
-				 * indxquals may get rechecked as qpquals).
+				 * clauses (this could happen if special operators are
+				 * involved: some indxquals may get rechecked as qpquals).
 				 */
 				if (NumRelids((Node *) inner_plan->qual) > 1)
 					inner_plan->qual = join_references(inner_plan->qual,

src/include/nodes/execnodes.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/nodes/execnodes.h,v 1.109 2003/12/18 22:23:42 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/nodes/execnodes.h,v 1.110 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -776,6 +776,7 @@ typedef ScanState SeqScanState;
  *		RuntimeKeysReady   true if runtime Skeys have been computed
  *		RelationDescs	   ptr to array of relation descriptors
  *		ScanDescs		   ptr to array of scan descriptors
+ *		LossyQuals		   ptr to array of qual lists for lossy operators
  *		DupHash			   hashtable for recognizing dups in multiple scan
  *		MaxHash			   max # entries we will allow in hashtable
  * ----------------
@@ -795,6 +796,7 @@ typedef struct IndexScanState
 	bool		iss_RuntimeKeysReady;
 	RelationPtr iss_RelationDescs;
 	IndexScanDescPtr iss_ScanDescs;
+	List	  **iss_LossyQuals;
 	HTAB	   *iss_DupHash;
 	long		iss_MaxHash;
 } IndexScanState;

src/include/nodes/plannodes.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/nodes/plannodes.h,v 1.72 2003/11/29 22:41:06 pgsql Exp $
+ * $PostgreSQL: pgsql/src/include/nodes/plannodes.h,v 1.73 2004/01/06 04:31:01 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -166,6 +166,7 @@ typedef struct IndexScan
 	List	   *indxqualorig;	/* the same in original form */
 	List	   *indxstrategy;	/* list of sublists of strategy numbers */
 	List	   *indxsubtype;	/* list of sublists of strategy subtypes */
+	List	   *indxlossy;		/* list of sublists of lossy flags (ints) */
 	ScanDirection indxorderdir;	/* forward or backward or don't care */
 } IndexScan;