From afc6064e9a93fb0f00eefefe706c59d6325d2727 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryzbyj@telsasoft.com>
Date: Sun, 8 Mar 2020 17:15:02 -0500
Subject: [PATCH v13 3/8] Add pg_ls_dir_metadata to list a dir with file
 metadata..

Generalize pg_ls_dir_files and retire pg_ls_dir

Change to use lstat() to allow pg_ls_dir_recurse to avoid infinite recursion.
That means:
 - links to dirs are shown with isdir=false;
 - timestamps shown are those of the link;
 - changed pg_stat_file for consistency;

Need catversion bumped?
---
 doc/src/sgml/func.sgml               |  19 ++-
 src/backend/catalog/system_views.sql |   1 +
 src/backend/utils/adt/genfile.c      | 218 +++++++++++++++------------
 src/include/catalog/pg_proc.dat      |   6 +
 4 files changed, 150 insertions(+), 94 deletions(-)

diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 2c6142a0e0..68c7327e1d 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21342,6 +21342,15 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
         List the contents of a directory.  Restricted to superusers by default, but other users can be granted EXECUTE to run the function.
        </entry>
       </row>
+      <row>
+       <entry>
+        <literal><function>pg_ls_dir_metadata(<parameter>dirname</parameter> <type>text</type> [, <parameter>missing_ok</parameter> <type>boolean</type>, <parameter>include_dot_dirs</parameter> <type>boolean</type>])</function></literal>
+       </entry>
+       <entry><type>setof text</type></entry>
+       <entry>
+        For each file in a directory, list the file and its metadata.  Restricted to superusers by default, but other users can be granted EXECUTE to run the function.
+       </entry>
+      </row>
       <row>
        <entry>
         <literal><function>pg_ls_logdir()</function></literal>
@@ -21442,6 +21451,14 @@ postgres=# SELECT * FROM pg_walfile_name_offset(pg_stop_backup());
     empty directory from an non-existent directory.
    </para>
 
+   <indexterm>
+    <primary>pg_ls_dir_metadata</primary>
+   </indexterm>
+   <para>
+    <function>pg_ls_dir_metadata</function> lists the files in the specified
+    directory along with the file's metadata.
+   </para>
+
    <indexterm>
     <primary>pg_ls_logdir</primary>
    </indexterm>
@@ -21528,7 +21545,7 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8');
     size, last accessed time stamp, last modified time stamp,
     last file status change time stamp (Unix platforms only),
     file creation time stamp (Windows only), and a <type>boolean</type>
-    indicating if it is a directory (or a symbolic link to a directory).
+    indicating if it is a directory (and not a symbolic link to a directory).
     Typical usages include:
 <programlisting>
 SELECT * FROM pg_stat_file('filename');
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index b8a3f46912..05a644a7c9 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -1435,6 +1435,7 @@ REVOKE EXECUTE ON FUNCTION pg_stat_file(text,boolean) FROM public;
 
 REVOKE EXECUTE ON FUNCTION pg_ls_dir(text) FROM public;
 REVOKE EXECUTE ON FUNCTION pg_ls_dir(text,boolean,boolean) FROM public;
+REVOKE EXECUTE ON FUNCTION pg_ls_dir_metadata(text,boolean,boolean) FROM public;
 
 --
 -- We also set up some things as accessible to standard roles.
diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c
index 01185f218b..4b70a00a35 100644
--- a/src/backend/utils/adt/genfile.c
+++ b/src/backend/utils/adt/genfile.c
@@ -36,6 +36,21 @@
 #include "utils/syscache.h"
 #include "utils/timestamp.h"
 
+static Datum pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags);
+
+#define	LS_DIR_ISDIR				(1<<0) /* Show column: isdir */
+#define	LS_DIR_METADATA				(1<<1) /* Show columns: mtime, size */
+#define	LS_DIR_MISSING_OK			(1<<2) /* Ignore ENOENT if the toplevel dir is missing */
+#define	LS_DIR_SKIP_DOT_DIRS		(1<<3) /* Do not show . or .. */
+#define	LS_DIR_SKIP_HIDDEN			(1<<4) /* Do not show anything begining with . */
+#define	LS_DIR_SKIP_DIRS			(1<<5) /* Do not show directories */
+#define	LS_DIR_SKIP_SPECIAL			(1<<6) /* Do not show special file types */
+
+/*
+ * Shortcut for the historic behavior of the pg_ls_* functions (not including
+ * pg_ls_dir, which skips different files and doesn't show metadata.
+ */
+#define LS_DIR_HISTORIC				(LS_DIR_SKIP_DIRS|LS_DIR_SKIP_HIDDEN|LS_DIR_SKIP_SPECIAL|LS_DIR_METADATA)
 
 /*
  * Convert a "text" filename argument to C string, and check it's allowable.
@@ -370,7 +385,7 @@ pg_stat_file(PG_FUNCTION_ARGS)
 
 	filename = convert_and_check_filename(filename_t);
 
-	if (stat(filename, &fst) < 0)
+	if (lstat(filename, &fst) < 0)
 	{
 		if (missing_ok && errno == ENOENT)
 			PG_RETURN_NULL();
@@ -413,6 +428,10 @@ pg_stat_file(PG_FUNCTION_ARGS)
 	values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime));
 #endif
 	values[5] = BoolGetDatum(S_ISDIR(fst.st_mode));
+#ifdef WIN32
+	if (pgwin32_is_junction(path)) /* Links are not directories */
+		values[5] = BoolGetDatum(false);
+#endif
 
 	tuple = heap_form_tuple(tupdesc, values, isnull);
 
@@ -440,79 +459,9 @@ pg_stat_file_1arg(PG_FUNCTION_ARGS)
 Datum
 pg_ls_dir(PG_FUNCTION_ARGS)
 {
-	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
-	char	   *location;
-	bool		missing_ok = false;
-	bool		include_dot_dirs = false;
-	bool		randomAccess;
-	TupleDesc	tupdesc;
-	Tuplestorestate *tupstore;
-	DIR		   *dirdesc;
-	struct dirent *de;
-	MemoryContext oldcontext;
-
-	location = convert_and_check_filename(PG_GETARG_TEXT_PP(0));
-
-	/* check the optional arguments */
-	if (PG_NARGS() == 3)
-	{
-		if (!PG_ARGISNULL(1))
-			missing_ok = PG_GETARG_BOOL(1);
-		if (!PG_ARGISNULL(2))
-			include_dot_dirs = PG_GETARG_BOOL(2);
-	}
-
-	/* check to see if caller supports us returning a tuplestore */
-	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
-		ereport(ERROR,
-				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("set-valued function called in context that cannot accept a set")));
-	if (!(rsinfo->allowedModes & SFRM_Materialize))
-		ereport(ERROR,
-				(errcode(ERRCODE_SYNTAX_ERROR),
-				 errmsg("materialize mode required, but it is not allowed in this context")));
-
-	/* The tupdesc and tuplestore must be created in ecxt_per_query_memory */
-	oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory);
-
-	tupdesc = CreateTemplateTupleDesc(1);
-	TupleDescInitEntry(tupdesc, (AttrNumber) 1, "pg_ls_dir", TEXTOID, -1, 0);
-
-	randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0;
-	tupstore = tuplestore_begin_heap(randomAccess, false, work_mem);
-	rsinfo->returnMode = SFRM_Materialize;
-	rsinfo->setResult = tupstore;
-	rsinfo->setDesc = tupdesc;
-
-	MemoryContextSwitchTo(oldcontext);
-
-	dirdesc = AllocateDir(location);
-	if (!dirdesc)
-	{
-		/* Return empty tuplestore if appropriate */
-		if (missing_ok && errno == ENOENT)
-			return (Datum) 0;
-		/* Otherwise, we can let ReadDir() throw the error */
-	}
-
-	while ((de = ReadDir(dirdesc, location)) != NULL)
-	{
-		Datum		values[1];
-		bool		nulls[1];
-
-		if (!include_dot_dirs &&
-			(strcmp(de->d_name, ".") == 0 ||
-			 strcmp(de->d_name, "..") == 0))
-			continue;
-
-		values[0] = CStringGetTextDatum(de->d_name);
-		nulls[0] = false;
-
-		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
-	}
-
-	FreeDir(dirdesc);
-	return (Datum) 0;
+	text	*filename_t = PG_GETARG_TEXT_PP(0);
+	char	*filename = convert_and_check_filename(filename_t);
+	return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS);
 }
 
 /*
@@ -525,7 +474,9 @@ pg_ls_dir(PG_FUNCTION_ARGS)
 Datum
 pg_ls_dir_1arg(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir(fcinfo);
+	text	*filename_t = PG_GETARG_TEXT_PP(0);
+	char	*filename = convert_and_check_filename(filename_t);
+	return pg_ls_dir_files(fcinfo, filename, LS_DIR_SKIP_DOT_DIRS);
 }
 
 /*
@@ -535,7 +486,7 @@ pg_ls_dir_1arg(PG_FUNCTION_ARGS)
  * Other unreadable-directory cases throw an error.
  */
 static Datum
-pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
+pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, int flags)
 {
 	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
 	bool		randomAccess;
@@ -544,6 +495,32 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 	DIR		   *dirdesc;
 	struct dirent *de;
 	MemoryContext oldcontext;
+	TypeFuncClass	tuptype ;
+
+	/* isdir depends on metadata */
+	Assert(!(flags&LS_DIR_ISDIR) || (flags&LS_DIR_METADATA));
+	/* Unreasonable to show isdir and skip dirs */
+	Assert(!(flags&LS_DIR_ISDIR) || !(flags&LS_DIR_SKIP_DIRS));
+
+	/* check the optional arguments */
+	if (PG_NARGS() == 3)
+	{
+		if (!PG_ARGISNULL(1))
+		{
+			if (PG_GETARG_BOOL(1))
+				flags |= LS_DIR_MISSING_OK;
+			else
+				flags &= ~LS_DIR_MISSING_OK;
+		}
+
+		if (!PG_ARGISNULL(2))
+		{
+			if (PG_GETARG_BOOL(2))
+				flags &= ~LS_DIR_SKIP_DOT_DIRS;
+			else
+				flags |= LS_DIR_SKIP_DOT_DIRS;
+		}
+	}
 
 	/* check to see if caller supports us returning a tuplestore */
 	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
@@ -558,8 +535,18 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 	/* The tupdesc and tuplestore must be created in ecxt_per_query_memory */
 	oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory);
 
-	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
-		elog(ERROR, "return type must be a row type");
+	tuptype = get_call_result_type(fcinfo, NULL, &tupdesc);
+	if (flags & LS_DIR_METADATA)
+	{
+		if (tuptype != TYPEFUNC_COMPOSITE)
+			elog(ERROR, "return type must be a row type");
+	} else {
+		/* pg_ls_dir returns a simple scalar */
+		if (tuptype != TYPEFUNC_SCALAR)
+			elog(ERROR, "return type must be a scalar type");
+		tupdesc = CreateTemplateTupleDesc(1);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "column", TEXTOID, -1, 0);
+	}
 
 	randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0;
 	tupstore = tuplestore_begin_heap(randomAccess, false, work_mem);
@@ -578,36 +565,67 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 	if (!dirdesc)
 	{
 		/* Return empty tuplestore if appropriate */
-		if (missing_ok && errno == ENOENT)
+		if (flags&LS_DIR_MISSING_OK && errno == ENOENT)
+		{
+			tuplestore_donestoring(tupstore);
 			return (Datum) 0;
+		}
 		/* Otherwise, we can let ReadDir() throw the error */
 	}
 
 	while ((de = ReadDir(dirdesc, dir)) != NULL)
 	{
-		Datum		values[3];
-		bool		nulls[3];
+		Datum		values[4];
+		bool		nulls[4];
 		char		path[MAXPGPATH * 2];
 		struct stat attrib;
 
-		/* Skip hidden files */
-		if (de->d_name[0] == '.')
+		/* Skip dot dirs? */
+		if (flags & LS_DIR_SKIP_DOT_DIRS &&
+			(strcmp(de->d_name, ".") == 0 ||
+			 strcmp(de->d_name, "..") == 0))
+			continue;
+
+		/* Skip hidden files? */
+		if (flags & LS_DIR_SKIP_HIDDEN &&
+			de->d_name[0] == '.')
 			continue;
 
 		/* Get the file info */
 		snprintf(path, sizeof(path), "%s/%s", dir, de->d_name);
-		if (stat(path, &attrib) < 0)
+		if (lstat(path, &attrib) < 0)
 			ereport(ERROR,
 					(errcode_for_file_access(),
 					 errmsg("could not stat file \"%s\": %m", path)));
 
-		/* Ignore anything but regular files */
-		if (!S_ISREG(attrib.st_mode))
-			continue;
+		/* Skip dirs or special files? */
+		if (S_ISDIR(attrib.st_mode))
+		{
+			if (flags & LS_DIR_SKIP_DIRS)
+				continue;
+		}
+		else if (!S_ISREG(attrib.st_mode))
+		{
+			if (flags & LS_DIR_SKIP_SPECIAL)
+				continue;
+		}
 
 		values[0] = CStringGetTextDatum(de->d_name);
-		values[1] = Int64GetDatum((int64) attrib.st_size);
-		values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime));
+		if (flags & LS_DIR_METADATA)
+		{
+			values[1] = Int64GetDatum((int64) attrib.st_size);
+			values[2] = TimestampTzGetDatum(time_t_to_timestamptz(attrib.st_mtime));
+			if (flags & LS_DIR_ISDIR)
+			{
+				values[3] = BoolGetDatum(S_ISDIR(attrib.st_mode));
+#ifdef WIN32
+				/* Links are not directories */
+				if (pgwin32_is_junction(path))
+					values[3] = BoolGetDatum(false);
+#endif
+			}
+		}
+
 		memset(nulls, 0, sizeof(nulls));
 
 		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
@@ -621,14 +639,14 @@ pg_ls_dir_files(FunctionCallInfo fcinfo, const char *dir, bool missing_ok)
 Datum
 pg_ls_logdir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, Log_directory, false);
+	return pg_ls_dir_files(fcinfo, Log_directory, LS_DIR_HISTORIC);
 }
 
 /* Function to return the list of files in the WAL directory */
 Datum
 pg_ls_waldir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, XLOGDIR, false);
+	return pg_ls_dir_files(fcinfo, XLOGDIR, LS_DIR_HISTORIC);
 }
 
 /*
@@ -646,7 +664,8 @@ pg_ls_tmpdir(FunctionCallInfo fcinfo, Oid tblspc)
 						tblspc)));
 
 	TempTablespacePath(path, tblspc);
-	return pg_ls_dir_files(fcinfo, path, true);
+	return pg_ls_dir_files(fcinfo, path,
+			LS_DIR_HISTORIC|LS_DIR_MISSING_OK);
 }
 
 /*
@@ -675,5 +694,18 @@ pg_ls_tmpdir_1arg(PG_FUNCTION_ARGS)
 Datum
 pg_ls_archive_statusdir(PG_FUNCTION_ARGS)
 {
-	return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status", true);
+	return pg_ls_dir_files(fcinfo, XLOGDIR "/archive_status",
+			LS_DIR_HISTORIC|LS_DIR_MISSING_OK);
+}
+
+/*
+ * Function to return the list of files and metadata in an arbitrary directory.
+ */
+Datum
+pg_ls_dir_metadata(PG_FUNCTION_ARGS)
+{
+	char	*dirname = convert_and_check_filename(PG_GETARG_TEXT_PP(0));
+
+	return pg_ls_dir_files(fcinfo, dirname,
+			LS_DIR_METADATA|LS_DIR_SKIP_SPECIAL|LS_DIR_ISDIR);
 }
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 7fb574f9dc..0a1859f709 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -10741,6 +10741,12 @@
   proallargtypes => '{oid,text,int8,timestamptz}', proargmodes => '{i,o,o,o}',
   proargnames => '{tablespace,name,size,modification}',
   prosrc => 'pg_ls_tmpdir_1arg' },
+{ oid => '5032', descr => 'list directory with metadata',
+  proname => 'pg_ls_dir_metadata', procost => '10', prorows => '20', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => 'text bool bool bool',
+  proallargtypes => '{text,bool,bool,bool,text,int8,timestamptz,bool}', proargmodes => '{i,i,i,i,o,o,o,o}',
+  proargnames => '{dirname,missing_ok,include_dot_dirs,dir_ok,name,size,modification,isdir}',
+  prosrc => 'pg_ls_dir_metadata' },
 
 # hash partitioning constraint function
 { oid => '5028', descr => 'hash partition CHECK constraint',
-- 
2.17.0

