Re: pgsql_fdw, FDW for PostgreSQL server - Mailing list pgsql-hackers
| From | Tom Lane |
|---|---|
| Subject | Re: pgsql_fdw, FDW for PostgreSQL server |
| Date | |
| Msg-id | 20139.1331269201@sss.pgh.pa.us Whole thread Raw |
| In response to | Re: pgsql_fdw, FDW for PostgreSQL server (Tom Lane <tgl@sss.pgh.pa.us>) |
| Responses |
Re: pgsql_fdw, FDW for PostgreSQL server
Re: pgsql_fdw, FDW for PostgreSQL server |
| List | pgsql-hackers |
I wrote:
> There are a couple of other points that make me think we need to revisit
> the PlanForeignScan API definition some more, too. ...
> So we need to break down what PlanForeignScan currently does into three
> separate steps. The first idea that comes to mind is to call them
> GetForeignRelSize, GetForeignPaths, GetForeignPlan; but maybe somebody
> has a better idea for names?
Attached is a draft patch for that. While I was working on this
I realized that we were very far short of allowing FDWs to set up
expressions of their choice for execution; there was nothing for that in
setrefs.c, nor some other places that need to post-process expressions.
I had originally supposed that fdw_private could just contain some
expression trees, but that wasn't going to work without post-processing.
So this patch attempts to cover that too, by breaking what had been
fdw_private into a "private" part and an "fdw_exprs" list that will be
subject to expression post-processing. (The alternative to this would
be to do post-processing on all of fdw_private, but that would
considerably restrict what can be in fdw_private, so it seemed better
to decree two separate fields.)
Working on this also helped me identify some other things that had been
subliminally bothering me about pgsql_fdw's qual pushdown code. That
patch is set up with the idea of pushing entire quals (boolean
RestrictInfo expressions) across to the remote side, but I think that
is probably the wrong granularity, or at least not the only mechanism
we should have. IMO it is more important to provide a structure similar
to index quals; that is, what you want to identify is RestrictInfo
expressions of the form
remote_variable operator local_expression
where the operator has to be one that the remote can execute with the
same semantics as we think it has, but the only real restriction on the
local_expression is that it be stable, because we'll execute it locally
and send only its result value across to the remote. (The SQL sent to
the remote looks like "remote_variable operator $1", or some such.)
Thus, to take an example that's said to be unsafe in the existing code
comments, there's no problem at all with
remote_timestamp_col = now()
as long as we execute now() locally.
There might be some value in pushing entire quals across too, for
clauses like "remote_variable_1 = remote_variable_2", but I believe
that these are not nearly as important as "variable = constant" and
"variable = join_variable" cases. Consider that when dealing with a
local table, only the latter two cases can be accelerated by indexes.
regards, tom lane
diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c
index 29f203c6f10eeeed194670aa37956a6190e8e7a1..e8907709bd90a6342384dfb6f10b00e55018d65d 100644
*** a/contrib/file_fdw/file_fdw.c
--- b/contrib/file_fdw/file_fdw.c
***************
*** 26,31 ****
--- 26,33 ----
#include "nodes/makefuncs.h"
#include "optimizer/cost.h"
#include "optimizer/pathnode.h"
+ #include "optimizer/planmain.h"
+ #include "optimizer/restrictinfo.h"
#include "utils/rel.h"
PG_MODULE_MAGIC;
*************** struct FileFdwOption
*** 48,54 ****
* Note: If you are adding new option for user mapping, you need to modify
* fileGetOptions(), which currently doesn't bother to look at user mappings.
*/
! static struct FileFdwOption valid_options[] = {
/* File options */
{"filename", ForeignTableRelationId},
--- 50,56 ----
* Note: If you are adding new option for user mapping, you need to modify
* fileGetOptions(), which currently doesn't bother to look at user mappings.
*/
! static const struct FileFdwOption valid_options[] = {
/* File options */
{"filename", ForeignTableRelationId},
*************** static struct FileFdwOption valid_option
*** 72,77 ****
--- 74,90 ----
};
/*
+ * FDW-specific information for RelOptInfo.fdw_private.
+ */
+ typedef struct FileFdwPlanState
+ {
+ char *filename; /* file to read */
+ List *options; /* merged COPY options, excluding filename */
+ BlockNumber pages; /* estimate of file's physical size */
+ double ntuples; /* estimate of number of rows in file */
+ } FileFdwPlanState;
+
+ /*
* FDW-specific information for ForeignScanState.fdw_state.
*/
typedef struct FileFdwExecutionState
*************** PG_FUNCTION_INFO_V1(file_fdw_validator);
*** 93,101 ****
/*
* FDW callback routines
*/
! static void filePlanForeignScan(Oid foreigntableid,
! PlannerInfo *root,
! RelOptInfo *baserel);
static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es);
static void fileBeginForeignScan(ForeignScanState *node, int eflags);
static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node);
--- 106,123 ----
/*
* FDW callback routines
*/
! static void fileGetForeignRelSize(PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
! static void fileGetForeignPaths(PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
! static ForeignScan *fileGetForeignPlan(PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid,
! ForeignPath *best_path,
! List *tlist,
! List *scan_clauses);
static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es);
static void fileBeginForeignScan(ForeignScanState *node, int eflags);
static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node);
*************** static bool is_valid_option(const char *
*** 109,116 ****
static void fileGetOptions(Oid foreigntableid,
char **filename, List **other_options);
static List *get_file_fdw_attribute_options(Oid relid);
static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
! const char *filename,
Cost *startup_cost, Cost *total_cost);
--- 131,140 ----
static void fileGetOptions(Oid foreigntableid,
char **filename, List **other_options);
static List *get_file_fdw_attribute_options(Oid relid);
+ static void estimate_size(PlannerInfo *root, RelOptInfo *baserel,
+ FileFdwPlanState *fdw_private);
static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
! FileFdwPlanState *fdw_private,
Cost *startup_cost, Cost *total_cost);
*************** file_fdw_handler(PG_FUNCTION_ARGS)
*** 123,129 ****
{
FdwRoutine *fdwroutine = makeNode(FdwRoutine);
! fdwroutine->PlanForeignScan = filePlanForeignScan;
fdwroutine->ExplainForeignScan = fileExplainForeignScan;
fdwroutine->BeginForeignScan = fileBeginForeignScan;
fdwroutine->IterateForeignScan = fileIterateForeignScan;
--- 147,155 ----
{
FdwRoutine *fdwroutine = makeNode(FdwRoutine);
! fdwroutine->GetForeignRelSize = fileGetForeignRelSize;
! fdwroutine->GetForeignPaths = fileGetForeignPaths;
! fdwroutine->GetForeignPlan = fileGetForeignPlan;
fdwroutine->ExplainForeignScan = fileExplainForeignScan;
fdwroutine->BeginForeignScan = fileBeginForeignScan;
fdwroutine->IterateForeignScan = fileIterateForeignScan;
*************** file_fdw_validator(PG_FUNCTION_ARGS)
*** 177,183 ****
if (!is_valid_option(def->defname, catalog))
{
! struct FileFdwOption *opt;
StringInfoData buf;
/*
--- 203,209 ----
if (!is_valid_option(def->defname, catalog))
{
! const struct FileFdwOption *opt;
StringInfoData buf;
/*
*************** file_fdw_validator(PG_FUNCTION_ARGS)
*** 249,255 ****
static bool
is_valid_option(const char *option, Oid context)
{
! struct FileFdwOption *opt;
for (opt = valid_options; opt->optname; opt++)
{
--- 275,281 ----
static bool
is_valid_option(const char *option, Oid context)
{
! const struct FileFdwOption *opt;
for (opt = valid_options; opt->optname; opt++)
{
*************** get_file_fdw_attribute_options(Oid relid
*** 381,387 ****
}
/*
! * filePlanForeignScan
* Create possible access paths for a scan on the foreign table
*
* Currently we don't support any push-down feature, so there is only one
--- 407,437 ----
}
/*
! * fileGetForeignRelSize
! * Obtain relation size estimates for a foreign table
! */
! static void
! fileGetForeignRelSize(PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid)
! {
! FileFdwPlanState *fdw_private;
!
! /*
! * Fetch options. We only need filename at this point, but we might
! * as well get everything and not need to re-fetch it later in planning.
! */
! fdw_private = (FileFdwPlanState *) palloc(sizeof(FileFdwPlanState));
! fileGetOptions(foreigntableid,
! &fdw_private->filename, &fdw_private->options);
! baserel->fdw_private = (void *) fdw_private;
!
! /* Estimate relation size */
! estimate_size(root, baserel, fdw_private);
! }
!
! /*
! * fileGetForeignPaths
* Create possible access paths for a scan on the foreign table
*
* Currently we don't support any push-down feature, so there is only one
*************** get_file_fdw_attribute_options(Oid relid
*** 389,408 ****
* the data file.
*/
static void
! filePlanForeignScan(Oid foreigntableid,
! PlannerInfo *root,
! RelOptInfo *baserel)
{
! char *filename;
! List *options;
Cost startup_cost;
Cost total_cost;
! /* Fetch options --- we only need filename at this point */
! fileGetOptions(foreigntableid, &filename, &options);
!
! /* Estimate costs and update baserel->rows */
! estimate_costs(root, baserel, filename,
&startup_cost, &total_cost);
/* Create a ForeignPath node and add it as only possible path */
--- 439,454 ----
* the data file.
*/
static void
! fileGetForeignPaths(PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid)
{
! FileFdwPlanState *fdw_private = (FileFdwPlanState *) baserel->fdw_private;
Cost startup_cost;
Cost total_cost;
! /* Estimate costs */
! estimate_costs(root, baserel, fdw_private,
&startup_cost, &total_cost);
/* Create a ForeignPath node and add it as only possible path */
*************** filePlanForeignScan(Oid foreigntableid,
*** 423,428 ****
--- 469,505 ----
}
/*
+ * fileGetForeignPlan
+ * Create a ForeignScan plan node for scanning the foreign table
+ */
+ static ForeignScan *
+ fileGetForeignPlan(PlannerInfo *root,
+ RelOptInfo *baserel,
+ Oid foreigntableid,
+ ForeignPath *best_path,
+ List *tlist,
+ List *scan_clauses)
+ {
+ Index scan_relid = baserel->relid;
+
+ /*
+ * We have no native ability to evaluate restriction clauses, so we just
+ * put all the scan_clauses into the plan node's qual list for the
+ * executor to check. So all we have to do here is strip RestrictInfo
+ * nodes from the clauses and ignore pseudoconstants (which will be
+ * handled elsewhere).
+ */
+ scan_clauses = extract_actual_clauses(scan_clauses, false);
+
+ /* Create the ForeignScan node */
+ return make_foreignscan(tlist,
+ scan_clauses,
+ scan_relid,
+ NIL, /* no expressions to evaluate */
+ NIL); /* no private state either */
+ }
+
+ /*
* fileExplainForeignScan
* Produce extra output for EXPLAIN
*/
*************** fileReScanForeignScan(ForeignScanState *
*** 568,605 ****
}
/*
! * Estimate costs of scanning a foreign table.
*
! * In addition to setting *startup_cost and *total_cost, this should
! * update baserel->rows.
*/
static void
! estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
! const char *filename,
! Cost *startup_cost, Cost *total_cost)
{
struct stat stat_buf;
BlockNumber pages;
int tuple_width;
double ntuples;
double nrows;
- Cost run_cost = 0;
- Cost cpu_per_tuple;
/*
* Get size of the file. It might not be there at plan time, though, in
* which case we have to use a default estimate.
*/
! if (stat(filename, &stat_buf) < 0)
stat_buf.st_size = 10 * BLCKSZ;
/*
! * Convert size to pages for use in I/O cost estimate below.
*/
pages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ;
if (pages < 1)
pages = 1;
/*
* Estimate the number of tuples in the file. We back into this estimate
* using the planner's idea of the relation width; which is bogus if not
--- 645,682 ----
}
/*
! * Estimate size of a foreign table.
*
! * The main result is returned in baserel->rows. We also set
! * fdw_private->pages and fdw_private->ntuples for later use in the cost
! * calculation.
*/
static void
! estimate_size(PlannerInfo *root, RelOptInfo *baserel,
! FileFdwPlanState *fdw_private)
{
struct stat stat_buf;
BlockNumber pages;
int tuple_width;
double ntuples;
double nrows;
/*
* Get size of the file. It might not be there at plan time, though, in
* which case we have to use a default estimate.
*/
! if (stat(fdw_private->filename, &stat_buf) < 0)
stat_buf.st_size = 10 * BLCKSZ;
/*
! * Convert size to pages for use in I/O cost estimate later.
*/
pages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ;
if (pages < 1)
pages = 1;
+ fdw_private->pages = pages;
+
/*
* Estimate the number of tuples in the file. We back into this estimate
* using the planner's idea of the relation width; which is bogus if not
*************** estimate_costs(PlannerInfo *root, RelOpt
*** 611,616 ****
--- 688,695 ----
ntuples = clamp_row_est((double) stat_buf.st_size / (double) tuple_width);
+ fdw_private->ntuples = ntuples;
+
/*
* Now estimate the number of rows returned by the scan after applying the
* baserestrictinfo quals. This is pretty bogus too, since the planner
*************** estimate_costs(PlannerInfo *root, RelOpt
*** 627,638 ****
/* Save the output-rows estimate for the planner */
baserel->rows = nrows;
/*
! * Now estimate costs. We estimate costs almost the same way as
! * cost_seqscan(), thus assuming that I/O costs are equivalent to a
! * regular table file of the same size. However, we take per-tuple CPU
! * costs as 10x of a seqscan, to account for the cost of parsing records.
*/
run_cost += seq_page_cost * pages;
--- 706,733 ----
/* Save the output-rows estimate for the planner */
baserel->rows = nrows;
+ }
+
+ /*
+ * Estimate costs of scanning a foreign table.
+ *
+ * Results are returned in *startup_cost and *total_cost.
+ */
+ static void
+ estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
+ FileFdwPlanState *fdw_private,
+ Cost *startup_cost, Cost *total_cost)
+ {
+ BlockNumber pages = fdw_private->pages;
+ double ntuples = fdw_private->ntuples;
+ Cost run_cost = 0;
+ Cost cpu_per_tuple;
/*
! * We estimate costs almost the same way as cost_seqscan(), thus assuming
! * that I/O costs are equivalent to a regular table file of the same size.
! * However, we take per-tuple CPU costs as 10x of a seqscan, to account
! * for the cost of parsing records.
*/
run_cost += seq_page_cost * pages;
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index dbfcbbc2b36dd49b0f0a8ffc9893da0b2a25a891..330953ee785d861a48c26166446eeec220e7a032 100644
*** a/doc/src/sgml/fdwhandler.sgml
--- b/doc/src/sgml/fdwhandler.sgml
***************
*** 89,140 ****
<para>
<programlisting>
void
! PlanForeignScan (Oid foreigntableid,
! PlannerInfo *root,
! RelOptInfo *baserel);
</programlisting>
! Create possible access paths for a scan on a foreign table. This is
! called when a query is planned.
<literal>foreigntableid</> is the <structname>pg_class</> OID of the
! foreign table. <literal>root</> is the planner's global information
! about the query, and <literal>baserel</> is the planner's information
! about this table.
! </para>
!
! <para>
! The function must generate at least one access path (ForeignPath node)
! for a scan on the foreign table and must call <function>add_path</> to
! add the path to <literal>baserel->pathlist</>. It's recommended to
! use <function>create_foreignscan_path</> to build the ForeignPath node.
! The function may generate multiple access paths, e.g., a path which has
! valid <literal>pathkeys</> to represent a pre-sorted result. Each access
! path must contain cost estimates, and can contain any FDW-private
! information that is needed to execute the foreign scan at a later time.
! (Note that the private information must be represented in a form that
! <function>copyObject</> knows how to copy.)
</para>
<para>
The information in <literal>root</> and <literal>baserel</> can be used
to reduce the amount of information that has to be fetched from the
! foreign table (and therefore reduce the cost estimate).
<literal>baserel->baserestrictinfo</> is particularly interesting, as
! it contains restriction quals (<literal>WHERE</> clauses) that can be
used to filter the rows to be fetched. (The FDW is not required to
! enforce these quals, as the finished plan will recheck them anyway.)
<literal>baserel->reltargetlist</> can be used to determine which
columns need to be fetched.
</para>
<para>
! In addition to returning cost estimates, the function should update
! <literal>baserel->rows</> to be the expected number of rows returned
! by the scan, after accounting for the filtering done by the restriction
! quals. The initial value of <literal>baserel->rows</> is just a
! constant default estimate, which should be replaced if at all possible.
! The function may also choose to update <literal>baserel->width</> if
! it can compute a better estimate of the average result row width.
</para>
<para>
--- 89,227 ----
<para>
<programlisting>
void
! GetForeignRelSize (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
</programlisting>
! Obtain relation size estimates for a foreign table. This is called
! at the beginning of planning for a query involving a foreign table.
! <literal>root</> is the planner's global information about the query;
! <literal>baserel</> is the planner's information about this table; and
<literal>foreigntableid</> is the <structname>pg_class</> OID of the
! foreign table. (<literal>foreigntableid</> could be obtained from the
! planner data structures, but it's passed explicitly to save effort.)
</para>
<para>
The information in <literal>root</> and <literal>baserel</> can be used
to reduce the amount of information that has to be fetched from the
! foreign table (and therefore reduce the cost).
<literal>baserel->baserestrictinfo</> is particularly interesting, as
! it contains restriction quals (<literal>WHERE</> clauses) that should be
used to filter the rows to be fetched. (The FDW is not required to
! enforce these quals, as the executor can check them instead.)
<literal>baserel->reltargetlist</> can be used to determine which
columns need to be fetched.
</para>
<para>
! This function should update <literal>baserel->rows</> to be the
! expected number of rows returned by the table scan, after accounting for
! the filtering done by the restriction quals. The initial value of
! <literal>baserel->rows</> is just a constant default estimate, which
! should be replaced if at all possible. The function may also choose to
! update <literal>baserel->width</> if it can compute a better estimate
! of the average result row width.
! </para>
!
! <para>
! <literal>baserel->fdw_private</> is a <type>void</> pointer that is
! available for use by FDW planning functions. It can be used to pass
! information forward from <function>GetForeignRelSize</> to
! <function>GetForeignPaths</> and/or <function>GetForeignPaths</> to
! <function>GetForeignPlan</>, thereby avoiding recalculation.
! </para>
!
! <para>
! <programlisting>
! void
! GetForeignPaths (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
! </programlisting>
!
! Create possible access paths for a scan on a foreign table.
! This is called during query planning.
! The parameters are the same as for <function>GetForeignRelSize</>,
! which has already been called.
! </para>
!
! <para>
! This function must generate at least one access path
! (<structname>ForeignPath</> node) for a scan on the foreign table and
! must call <function>add_path</> to add each such path to
! <literal>baserel->pathlist</>. It's recommended to use
! <function>create_foreignscan_path</> to build the ForeignPath nodes. The
! function may generate multiple access paths, e.g., a path which has valid
! <literal>pathkeys</> to represent a pre-sorted result. Each access path
! must contain cost estimates, and can contain any FDW-private information
! that is needed to identify the specific scan method intended.
! </para>
!
! <para>
! This private information is stored in the <structfield>fdw_private</>
! field of <structname>ForeignPath</> nodes. <structfield>fdw_private</>
! is declared as a <type>List</>, but could actually contain anything since
! the core planner does not touch it. However, best practice is to use a
! representation that's dumpable by <function>nodeToString</>, for use with
! debugging support available in the backend.
! </para>
!
! <para>
! <programlisting>
! ForeignScan *
! GetForeignPlan (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid,
! ForeignPath *best_path,
! List *tlist,
! List *scan_clauses);
! </programlisting>
!
! Create a <structname>ForeignScan</> plan node from the selected foreign
! access path. This is called at the end of query planning.
! The parameters are as for <function>GetForeignRelSize</>, plus
! the selected <structname>ForeignPath</> (previously produced by
! <function>GetForeignPaths</>), the target list to be emitted by the
! plan node, and the restriction clauses to be enforced by the plan node.
! </para>
!
! <para>
! This function must create and return a <structname>ForeignScan</> plan
! node; it's recommended to use <function>make_foreignscan</> to build the
! <structname>ForeignScan</> node. Generally the targetlist can be copied
! into the plan as-is. The passed scan_clauses list contains the same
! clauses as <literal>baserel->baserestrictinfo</>, but may be
! re-ordered for better execution efficiency. In simple cases the FDW can
! just strip RestrictInfo nodes from the scan_clauses list (using
! <function>extract_actual_clauses</>) and put all the clauses into the
! plan node's qual list, which means that all the clauses will be checked
! by the executor at runtime. More complex FDWs may be able to check some
! of the clauses internally, in which case those clauses can be removed
! from the list given to the executor.
! </para>
!
! <para>
! In addition, the FDW can generate <structfield>fdw_exprs</> and
! <structfield>fdw_private</> lists to be placed in the plan node, where
! they will be available at execution time. Both of these lists must be
! represented in a form that <function>copyObject</> knows how to copy.
! The <structfield>fdw_private</> list has no other restrictions and is
! not interpreted by the core backend in any way. The
! <structfield>fdw_exprs</> list, if not NIL, is expected to contain
! expression trees that are intended to be executed at runtime. These
! trees will undergo post-processing by the planner to make them fully
! executable. As an example, the FDW might identify some restriction
! clauses of the form <replaceable>foreign_variable</> <literal>=</>
! <replaceable>sub_expression</>, which it determines can be executed on
! the remote server given the locally-evaluated value of the
! <replaceable>sub_expression</>. It would remove such a clause from
! scan_clauses, but add the <replaceable>sub_expression</> to
! <structfield>fdw_exprs</> to ensure that it gets massaged into
! executable form. It would probably also put some control information
! into <structfield>fdw_private</> to tell it what to do with the
! expression value at runtime.
</para>
<para>
*************** BeginForeignScan (ForeignScanState *node
*** 170,176 ****
the table to scan is accessible through the
<structname>ForeignScanState</> node (in particular, from the underlying
<structname>ForeignScan</> plan node, which contains any FDW-private
! information provided by <function>PlanForeignScan</>).
</para>
<para>
--- 257,263 ----
the table to scan is accessible through the
<structname>ForeignScanState</> node (in particular, from the underlying
<structname>ForeignScan</> plan node, which contains any FDW-private
! information provided by <function>GetForeignPlan</>).
</para>
<para>
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index 868fb7130a8b28cf3e074d7d3903e58366c0c914..5cde22543f5b7d4d607224acfa22a604a419ed63 100644
*** a/src/backend/nodes/copyfuncs.c
--- b/src/backend/nodes/copyfuncs.c
*************** _copyForeignScan(const ForeignScan *from
*** 591,598 ****
/*
* copy remainder of node
*/
! COPY_SCALAR_FIELD(fsSystemCol);
COPY_NODE_FIELD(fdw_private);
return newnode;
}
--- 591,599 ----
/*
* copy remainder of node
*/
! COPY_NODE_FIELD(fdw_exprs);
COPY_NODE_FIELD(fdw_private);
+ COPY_SCALAR_FIELD(fsSystemCol);
return newnode;
}
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 9daeb3e7b43e911aeab25b7521d41191928499bd..51181a9a7438e8609ab922340d1c2d20ba73726d 100644
*** a/src/backend/nodes/outfuncs.c
--- b/src/backend/nodes/outfuncs.c
*************** _outForeignScan(StringInfo str, const Fo
*** 559,566 ****
_outScanInfo(str, (const Scan *) node);
! WRITE_BOOL_FIELD(fsSystemCol);
WRITE_NODE_FIELD(fdw_private);
}
static void
--- 559,567 ----
_outScanInfo(str, (const Scan *) node);
! WRITE_NODE_FIELD(fdw_exprs);
WRITE_NODE_FIELD(fdw_private);
+ WRITE_BOOL_FIELD(fsSystemCol);
}
static void
*************** _outRelOptInfo(StringInfo str, const Rel
*** 1741,1746 ****
--- 1742,1748 ----
WRITE_FLOAT_FIELD(allvisfrac, "%.6f");
WRITE_NODE_FIELD(subplan);
WRITE_NODE_FIELD(subroot);
+ /* we don't try to print fdwroutine or fdw_private */
WRITE_NODE_FIELD(baserestrictinfo);
WRITE_NODE_FIELD(joininfo);
WRITE_BOOL_FIELD(has_eclass_joins);
diff --git a/src/backend/optimizer/path/allpaths.c b/src/backend/optimizer/path/allpaths.c
index 6e81ce0fc26496f73e89d5048b0fd8a19da33b74..03c604a03d6f37d9d6d975fd0809429c56d61b38 100644
*** a/src/backend/optimizer/path/allpaths.c
--- b/src/backend/optimizer/path/allpaths.c
*************** set_foreign_size(PlannerInfo *root, RelO
*** 396,401 ****
--- 396,407 ----
{
/* Mark rel with estimated output rows, width, etc */
set_foreign_size_estimates(root, rel);
+
+ /* Get FDW routine pointers for the rel */
+ rel->fdwroutine = GetFdwRoutineByRelId(rte->relid);
+
+ /* Let FDW adjust the size estimates, if it can */
+ rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid);
}
/*
*************** set_foreign_size(PlannerInfo *root, RelO
*** 405,415 ****
static void
set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
{
! FdwRoutine *fdwroutine;
!
! /* Call the FDW's PlanForeignScan function to generate path(s) */
! fdwroutine = GetFdwRoutineByRelId(rte->relid);
! fdwroutine->PlanForeignScan(rte->relid, root, rel);
/* Select cheapest path */
set_cheapest(rel);
--- 411,418 ----
static void
set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
{
! /* Call the FDW's GetForeignPaths function to generate path(s) */
! rel->fdwroutine->GetForeignPaths(root, rel, rte->relid);
/* Select cheapest path */
set_cheapest(rel);
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 885d8558c319fd283df351c2c8e062a449b72d3c..24c853d47ef1aabb95156f90be4a8f3ea3d4995e 100644
*** a/src/backend/optimizer/path/costsize.c
--- b/src/backend/optimizer/path/costsize.c
*************** set_cte_size_estimates(PlannerInfo *root
*** 3745,3751 ****
* using what will be purely datatype-driven estimates from the targetlist.
* There is no way to do anything sane with the rows value, so we just put
* a default estimate and hope that the wrapper can improve on it. The
! * wrapper's PlanForeignScan function will be called momentarily.
*
* The rel's targetlist and restrictinfo list must have been constructed
* already.
--- 3745,3751 ----
* using what will be purely datatype-driven estimates from the targetlist.
* There is no way to do anything sane with the rows value, so we just put
* a default estimate and hope that the wrapper can improve on it. The
! * wrapper's GetForeignRelSize function will be called momentarily.
*
* The rel's targetlist and restrictinfo list must have been constructed
* already.
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index b1df56cafd25abfda40657555a1c832aa6db797a..94140d304f754236955452e26b80de8276ca729b 100644
*** a/src/backend/optimizer/plan/createplan.c
--- b/src/backend/optimizer/plan/createplan.c
***************
*** 20,25 ****
--- 20,26 ----
#include <math.h>
#include "access/skey.h"
+ #include "foreign/fdwapi.h"
#include "miscadmin.h"
#include "nodes/makefuncs.h"
#include "nodes/nodeFuncs.h"
*************** static CteScan *make_ctescan(List *qptli
*** 119,126 ****
Index scanrelid, int ctePlanId, int cteParam);
static WorkTableScan *make_worktablescan(List *qptlist, List *qpqual,
Index scanrelid, int wtParam);
- static ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
- Index scanrelid, bool fsSystemCol, List *fdw_private);
static BitmapAnd *make_bitmap_and(List *bitmapplans);
static BitmapOr *make_bitmap_or(List *bitmapplans);
static NestLoop *make_nestloop(List *tlist,
--- 120,125 ----
*************** create_foreignscan_plan(PlannerInfo *roo
*** 1816,1822 ****
RelOptInfo *rel = best_path->path.parent;
Index scan_relid = rel->relid;
RangeTblEntry *rte;
- bool fsSystemCol;
int i;
/* it should be a base rel... */
--- 1815,1820 ----
*************** create_foreignscan_plan(PlannerInfo *roo
*** 1825,1855 ****
rte = planner_rt_fetch(scan_relid, root);
Assert(rte->rtekind == RTE_RELATION);
! /* Sort clauses into best execution order */
scan_clauses = order_qual_clauses(root, scan_clauses);
! /* Reduce RestrictInfo list to bare expressions; ignore pseudoconstants */
! scan_clauses = extract_actual_clauses(scan_clauses, false);
! /* Detect whether any system columns are requested from rel */
! fsSystemCol = false;
for (i = rel->min_attr; i < 0; i++)
{
if (!bms_is_empty(rel->attr_needed[i - rel->min_attr]))
{
! fsSystemCol = true;
break;
}
}
- scan_plan = make_foreignscan(tlist,
- scan_clauses,
- scan_relid,
- fsSystemCol,
- best_path->fdw_private);
-
- copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
-
return scan_plan;
}
--- 1823,1878 ----
rte = planner_rt_fetch(scan_relid, root);
Assert(rte->rtekind == RTE_RELATION);
! /*
! * Sort clauses into best execution order. We do this first since the
! * FDW might have more info than we do and wish to adjust the ordering.
! */
scan_clauses = order_qual_clauses(root, scan_clauses);
! /*
! * Let the FDW perform its processing on the restriction clauses and
! * generate the plan node. Note that the FDW might remove restriction
! * clauses that it intends to execute remotely, or even add more (if it
! * has selected some join clauses for remote use but also wants them
! * rechecked locally).
! */
! scan_plan = rel->fdwroutine->GetForeignPlan(root, rel, rte->relid,
! best_path,
! tlist, scan_clauses);
! /* Copy cost data from Path to Plan; no need to make FDW do this */
! copy_path_costsize(&scan_plan->scan.plan, &best_path->path);
!
! /*
! * Replace any outer-relation variables with nestloop params in the qual
! * and fdw_exprs expressions. We do this last so that the FDW doesn't
! * have to be involved. (Note that parts of fdw_exprs could have come
! * from join clauses, so doing this beforehand on the scan_clauses
! * wouldn't work.)
! */
! if (best_path->path.required_outer)
! {
! scan_plan->scan.plan.qual = (List *)
! replace_nestloop_params(root, (Node *) scan_plan->scan.plan.qual);
! scan_plan->fdw_exprs = (List *)
! replace_nestloop_params(root, (Node *) scan_plan->fdw_exprs);
! }
!
! /*
! * Detect whether any system columns are requested from rel. This is a
! * bit of a kluge and might go away someday, so we intentionally leave it
! * out of the API presented to FDWs.
! */
! scan_plan->fsSystemCol = false;
for (i = rel->min_attr; i < 0; i++)
{
if (!bms_is_empty(rel->attr_needed[i - rel->min_attr]))
{
! scan_plan->fsSystemCol = true;
break;
}
}
return scan_plan;
}
*************** make_worktablescan(List *qptlist,
*** 3183,3206 ****
return node;
}
! static ForeignScan *
make_foreignscan(List *qptlist,
List *qpqual,
Index scanrelid,
! bool fsSystemCol,
List *fdw_private)
{
ForeignScan *node = makeNode(ForeignScan);
Plan *plan = &node->scan.plan;
! /* cost should be inserted by caller */
plan->targetlist = qptlist;
plan->qual = qpqual;
plan->lefttree = NULL;
plan->righttree = NULL;
node->scan.scanrelid = scanrelid;
! node->fsSystemCol = fsSystemCol;
node->fdw_private = fdw_private;
return node;
}
--- 3206,3231 ----
return node;
}
! ForeignScan *
make_foreignscan(List *qptlist,
List *qpqual,
Index scanrelid,
! List *fdw_exprs,
List *fdw_private)
{
ForeignScan *node = makeNode(ForeignScan);
Plan *plan = &node->scan.plan;
! /* cost will be filled in by create_foreignscan_plan */
plan->targetlist = qptlist;
plan->qual = qpqual;
plan->lefttree = NULL;
plan->righttree = NULL;
node->scan.scanrelid = scanrelid;
! node->fdw_exprs = fdw_exprs;
node->fdw_private = fdw_private;
+ /* fsSystemCol will be filled in by create_foreignscan_plan */
+ node->fsSystemCol = false;
return node;
}
diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c
index e1b48fb4f53061f5b06653c67275dc0b323bffd0..69396694aaa9df0edbc361ede98b3f77db6724dc 100644
*** a/src/backend/optimizer/plan/setrefs.c
--- b/src/backend/optimizer/plan/setrefs.c
*************** set_plan_refs(PlannerInfo *root, Plan *p
*** 428,433 ****
--- 428,435 ----
fix_scan_list(root, splan->scan.plan.targetlist, rtoffset);
splan->scan.plan.qual =
fix_scan_list(root, splan->scan.plan.qual, rtoffset);
+ splan->fdw_exprs =
+ fix_scan_list(root, splan->fdw_exprs, rtoffset);
}
break;
diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c
index 40a420a3546f12de37459881ed587d89ae6954c3..b64db1e1c0659ea9b6af25327f689b083de845e6 100644
*** a/src/backend/optimizer/plan/subselect.c
--- b/src/backend/optimizer/plan/subselect.c
*************** finalize_plan(PlannerInfo *root, Plan *p
*** 2137,2142 ****
--- 2137,2144 ----
break;
case T_ForeignScan:
+ finalize_primnode((Node *) ((ForeignScan *) plan)->fdw_exprs,
+ &context);
context.paramids = bms_add_members(context.paramids, scan_params);
break;
diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c
index 6d1545476df7b054d48cae8a85669935d406c879..a2fc75a659e50ca7d5726ed9d024622af9e7f191 100644
*** a/src/backend/optimizer/util/pathnode.c
--- b/src/backend/optimizer/util/pathnode.c
*************** create_worktablescan_path(PlannerInfo *r
*** 1767,1773 ****
* returning the pathnode.
*
* This function is never called from core Postgres; rather, it's expected
! * to be called by the PlanForeignScan function of a foreign data wrapper.
* We make the FDW supply all fields of the path, since we do not have any
* way to calculate them in core.
*/
--- 1767,1773 ----
* returning the pathnode.
*
* This function is never called from core Postgres; rather, it's expected
! * to be called by the GetForeignPaths function of a foreign data wrapper.
* We make the FDW supply all fields of the path, since we do not have any
* way to calculate them in core.
*/
diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c
index 0cdf638c1ddb1c614eb8ef3cd2ddea4571248ff6..cee092a8810102fdf48023e180739cd8dbc3d81c 100644
*** a/src/backend/optimizer/util/relnode.c
--- b/src/backend/optimizer/util/relnode.c
*************** build_simple_rel(PlannerInfo *root, int
*** 113,118 ****
--- 113,120 ----
rel->allvisfrac = 0;
rel->subplan = NULL;
rel->subroot = NULL;
+ rel->fdwroutine = NULL;
+ rel->fdw_private = NULL;
rel->baserestrictinfo = NIL;
rel->baserestrictcost.startup = 0;
rel->baserestrictcost.per_tuple = 0;
*************** build_join_rel(PlannerInfo *root,
*** 366,371 ****
--- 368,375 ----
joinrel->allvisfrac = 0;
joinrel->subplan = NULL;
joinrel->subroot = NULL;
+ joinrel->fdwroutine = NULL;
+ joinrel->fdw_private = NULL;
joinrel->baserestrictinfo = NIL;
joinrel->baserestrictcost.startup = 0;
joinrel->baserestrictcost.per_tuple = 0;
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 9e135c62069fdc200e4f3cf58fa9725847d279fb..854f17755c4543ec85a66c6a6a3376fbcffd1cda 100644
*** a/src/include/foreign/fdwapi.h
--- b/src/include/foreign/fdwapi.h
*************** struct ExplainState;
*** 23,31 ****
* Callback function signatures --- see fdwhandler.sgml for more info.
*/
! typedef void (*PlanForeignScan_function) (Oid foreigntableid,
! PlannerInfo *root,
! RelOptInfo *baserel);
typedef void (*ExplainForeignScan_function) (ForeignScanState *node,
struct ExplainState *es);
--- 23,42 ----
* Callback function signatures --- see fdwhandler.sgml for more info.
*/
! typedef void (*GetForeignRelSize_function) (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
!
! typedef void (*GetForeignPaths_function) (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid);
!
! typedef ForeignScan *(*GetForeignPlan_function) (PlannerInfo *root,
! RelOptInfo *baserel,
! Oid foreigntableid,
! ForeignPath *best_path,
! List *tlist,
! List *scan_clauses);
typedef void (*ExplainForeignScan_function) (ForeignScanState *node,
struct ExplainState *es);
*************** typedef struct FdwRoutine
*** 53,59 ****
{
NodeTag type;
! PlanForeignScan_function PlanForeignScan;
ExplainForeignScan_function ExplainForeignScan;
BeginForeignScan_function BeginForeignScan;
IterateForeignScan_function IterateForeignScan;
--- 64,72 ----
{
NodeTag type;
! GetForeignRelSize_function GetForeignRelSize;
! GetForeignPaths_function GetForeignPaths;
! GetForeignPlan_function GetForeignPlan;
ExplainForeignScan_function ExplainForeignScan;
BeginForeignScan_function BeginForeignScan;
IterateForeignScan_function IterateForeignScan;
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 3962792d3d89a6d5077d1d682e2c3d112a6c568f..e6bb3239f4214c26aa1d70d4ca4ac50f63148ad5 100644
*** a/src/include/nodes/plannodes.h
--- b/src/include/nodes/plannodes.h
*************** typedef struct WorkTableScan
*** 462,474 ****
/* ----------------
* ForeignScan node
* ----------------
*/
typedef struct ForeignScan
{
Scan scan;
! bool fsSystemCol; /* true if any "system column" is needed */
List *fdw_private; /* private data for FDW */
} ForeignScan;
--- 462,483 ----
/* ----------------
* ForeignScan node
+ *
+ * fdw_exprs and fdw_private are both under the control of the foreign-data
+ * wrapper, but fdw_exprs is presumed to contain expression trees and will
+ * be post-processed accordingly by the planner; fdw_private won't be.
+ * Note that everything in both lists must be copiable by copyObject().
+ * One way to store an arbitrary blob of bytes is to represent it as a bytea
+ * Const. Usually, though, you'll be better off choosing a representation
+ * that can be dumped usefully by nodeToString().
* ----------------
*/
typedef struct ForeignScan
{
Scan scan;
! List *fdw_exprs; /* expressions that FDW may evaluate */
List *fdw_private; /* private data for FDW */
+ bool fsSystemCol; /* true if any "system column" is needed */
} ForeignScan;
diff --git a/src/include/nodes/relation.h b/src/include/nodes/relation.h
index 2a686080059f3ffd26313798324323c6f2f6b56d..8616223f24a8cbe5424b89599b331538db05dd76 100644
*** a/src/include/nodes/relation.h
--- b/src/include/nodes/relation.h
*************** typedef struct PlannerInfo
*** 334,343 ****
* allvisfrac - fraction of disk pages that are marked all-visible
* subplan - plan for subquery (NULL if it's not a subquery)
* subroot - PlannerInfo for subquery (NULL if it's not a subquery)
*
* Note: for a subquery, tuples, subplan, subroot are not set immediately
* upon creation of the RelOptInfo object; they are filled in when
! * set_base_rel_pathlist processes the object.
*
* For otherrels that are appendrel members, these fields are filled
* in just as for a baserel.
--- 334,346 ----
* allvisfrac - fraction of disk pages that are marked all-visible
* subplan - plan for subquery (NULL if it's not a subquery)
* subroot - PlannerInfo for subquery (NULL if it's not a subquery)
+ * fdwroutine - function hooks for FDW, if foreign table (else NULL)
+ * fdw_private - private state for FDW, if foreign table (else NULL)
*
* Note: for a subquery, tuples, subplan, subroot are not set immediately
* upon creation of the RelOptInfo object; they are filled in when
! * set_subquery_pathlist processes the object. Likewise, fdwroutine
! * and fdw_private are filled during initial path creation.
*
* For otherrels that are appendrel members, these fields are filled
* in just as for a baserel.
*************** typedef struct RelOptInfo
*** 414,421 ****
--- 417,428 ----
BlockNumber pages; /* size estimates derived from pg_class */
double tuples;
double allvisfrac;
+ /* use "struct Plan" to avoid including plannodes.h here */
struct Plan *subplan; /* if subquery */
PlannerInfo *subroot; /* if subquery */
+ /* use "struct FdwRoutine" to avoid including fdwapi.h here */
+ struct FdwRoutine *fdwroutine; /* if foreign table */
+ void *fdw_private; /* if foreign table */
/* used by various scans and joins: */
List *baserestrictinfo; /* RestrictInfo structures (if base
*************** typedef struct TidPath
*** 793,806 ****
} TidPath;
/*
! * ForeignPath represents a scan of a foreign table
*
! * fdw_private contains FDW private data about the scan, which will be copied
! * to the final ForeignScan plan node so that it is available at execution
! * time. Note that everything in this list must be copiable by copyObject().
! * One way to store an arbitrary blob of bytes is to represent it as a bytea
! * Const. Usually, though, you'll be better off choosing a representation
! * that can be dumped usefully by nodeToString().
*/
typedef struct ForeignPath
{
--- 800,812 ----
} TidPath;
/*
! * ForeignPath represents a potential scan of a foreign table
*
! * fdw_private stores FDW private data about the scan. While fdw_private is
! * not actually touched by the core code during normal operations, it's
! * generally a good idea to use a representation that can be dumped by
! * nodeToString(), so that you can examine the structure during debugging
! * with tools like pprint().
*/
typedef struct ForeignPath
{
diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h
index 8bd603124b3ac8ab87af160646301c7bbf8dbf16..47cc39cf1d9c3646fb10b76e6c4a97e166e65e08 100644
*** a/src/include/optimizer/planmain.h
--- b/src/include/optimizer/planmain.h
*************** extern Plan *optimize_minmax_aggregates(
*** 42,47 ****
--- 42,49 ----
extern Plan *create_plan(PlannerInfo *root, Path *best_path);
extern SubqueryScan *make_subqueryscan(List *qptlist, List *qpqual,
Index scanrelid, Plan *subplan);
+ extern ForeignScan *make_foreignscan(List *qptlist, List *qpqual,
+ Index scanrelid, List *fdw_exprs, List *fdw_private);
extern Append *make_append(List *appendplans, List *tlist);
extern RecursiveUnion *make_recursive_union(List *tlist,
Plan *lefttree, Plan *righttree, int wtParam,
pgsql-hackers by date: