PostgreSQL 18 beta1 - Segmentation fault on custom type casting - Mailing list pgsql-bugs

# PostgreSQL 18 Beta - Custom Type Casting Crash

## **Summary**
PostgreSQL 18 beta crashes with a segmentation fault when casting strings to custom types. The crash occurs specifically in PostgreSQL's type-casting system, not in extension code.

## **Environment**
- **PostgreSQL Version**: 18beta1 on aarch64-apple-darwin24.5.0
- **Platform**: macOS (Apple Silicon)
- **Compiler**: Apple clang version 17.0.0 (clang-1700.0.13.5)

## **Minimal Reproduction**

### **Extension Code**
```c
// Simple custom type - just holds a string
typedef struct {
    char *value;
} SimpleType;

// Input function
Datum simple_type_in(PG_FUNCTION_ARGS) {
    char *str = PG_GETARG_CSTRING(0);
    SimpleType *result = palloc0(sizeof(SimpleType));
    result->value = pstrdup(str);
    PG_RETURN_POINTER(result);
}

// Output function
Datum simple_type_out(PG_FUNCTION_ARGS) {
    SimpleType *simple = (SimpleType *) PG_GETARG_POINTER(0);
    if (!simple || !simple->value)
        PG_RETURN_CSTRING(pstrdup(""));
    PG_RETURN_CSTRING(pstrdup(simple->value));
}
```

### **Type Registration**
```sql
CREATE TYPE simple_type (
    INPUT = simple_type_in,
    OUTPUT = simple_type_out,
    STORAGE = EXTENDED
);
```

### **Crash Reproduction**

**Build and install extension:**
```bash
make && sudo make install
psql postgres -c "CREATE EXTENSION pg18_crash_repro;"
```

**Test results:**
```sql
-- This works fine
SELECT create_simple_type('test');
-- Returns: test

-- This crashes the server
SELECT 'hello'::simple_type;
-- Result: server closed the connection unexpectedly
```

## **Analysis**

The crash occurs specifically in PostgreSQL's **type casting system**, not in direct function calls:

- ✅ **Direct function calls work**: `create_simple_type('test')` executes successfully
- ❌ **Type casting crashes**: `'hello'::simple_type` causes segmentation fault

This indicates the bug is in PostgreSQL's internal type-casting mechanism when handling custom types returned via `PG_RETURN_POINTER()`.

## **Impact**
This affects all custom type extensions that use type casting (`::custom_type` syntax), making them incompatible with PostgreSQL 18 beta.

## **Complete Test Case**

### **Makefile**
```makefile
EXTENSION = pg18_crash_repro
MODULE_big = pg18_crash_repro
OBJS = pg18_crash_repro.o

DATA = pg18_crash_repro--1.0.sql
PGFILEDESC = "pg18_crash_repro - minimal reproduction of PG18 beta custom type crash"

PG_CONFIG = pg_config
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
```

### **pg18_crash_repro.control**
```
# pg18_crash_repro extension
comment = 'Minimal reproduction of PostgreSQL 18 beta custom type crash'
default_version = '1.0'
module_pathname = '$libdir/pg18_crash_repro'
relocatable = true
```

### **pg18_crash_repro.c**
```c
/*
 * pg18_crash_repro.c
 *
 * Minimal reproduction case for PostgreSQL 18 beta custom type crash
 * Demonstrates segmentation fault when casting to custom types
 */

#include "postgres.h"
#include "fmgr.h"
#include "utils/builtins.h"
#include "libpq/pqformat.h"
#include <string.h>

PG_MODULE_MAGIC;

/* Simple custom type - just holds a string */
typedef struct {
    char *value;
} SimpleType;

/* Function declarations */
PG_FUNCTION_INFO_V1(simple_type_in);
PG_FUNCTION_INFO_V1(simple_type_out);
PG_FUNCTION_INFO_V1(simple_type_send);
PG_FUNCTION_INFO_V1(simple_type_recv);
PG_FUNCTION_INFO_V1(create_simple_type);

/* Input function */
Datum
simple_type_in(PG_FUNCTION_ARGS)
{
    char *str = PG_GETARG_CSTRING(0);
    SimpleType *result;
   
    result = (SimpleType *) palloc0(sizeof(SimpleType));
    result->value = pstrdup(str);
   
    PG_RETURN_POINTER(result);
}

/* Output function */
Datum
simple_type_out(PG_FUNCTION_ARGS)
{
    SimpleType *simple = (SimpleType *) PG_GETARG_POINTER(0);
   
    if (!simple || !simple->value)
        PG_RETURN_CSTRING(pstrdup(""));
   
    PG_RETURN_CSTRING(pstrdup(simple->value));
}

/* Send function (binary output) */
Datum
simple_type_send(PG_FUNCTION_ARGS)
{
    SimpleType *simple = (SimpleType *) PG_GETARG_POINTER(0);
    StringInfoData buf;
   
    pq_begintypsend(&buf);
   
    if (simple && simple->value) {
        pq_sendint32(&buf, strlen(simple->value));
        pq_sendbytes(&buf, simple->value, strlen(simple->value));
    } else {
        pq_sendint32(&buf, -1);
    }
   
    PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
}

/* Receive function (binary input) */
Datum
simple_type_recv(PG_FUNCTION_ARGS)
{
    StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
    SimpleType *result;
    int32 len;
   
    result = (SimpleType *) palloc0(sizeof(SimpleType));
   
    len = pq_getmsgint(buf, 4);
    if (len == -1) {
        result->value = NULL;
    } else {
        const char *data = pq_getmsgbytes(buf, len);
        result->value = (char *) palloc(len + 1);
        memcpy(result->value, data, len);
        result->value[len] = '\0';
    }
   
    PG_RETURN_POINTER(result);
}

/*
 * This function works fine in PostgreSQL 18 beta
 * Direct function calls don't trigger the crash
 */
Datum
create_simple_type(PG_FUNCTION_ARGS)
{
    text *input = PG_GETARG_TEXT_P(0);
    char *str = text_to_cstring(input);
    SimpleType *result;
   
    elog(NOTICE, "create_simple_type: Creating SimpleType with value '%s'", str);
   
    result = (SimpleType *) palloc0(sizeof(SimpleType));
    result->value = pstrdup(str);
   
    elog(NOTICE, "create_simple_type: About to return pointer %p with value '%s'",
         result, result->value);
   
    /* This works fine - direct function calls don't crash */
    PG_RETURN_POINTER(result);
}
```

### **pg18_crash_repro--1.0.sql**
```sql
/* pg18_crash_repro--1.0.sql */

-- Create shell type first
CREATE TYPE simple_type;

-- Input/Output functions
CREATE OR REPLACE FUNCTION simple_type_in(cstring)
    RETURNS simple_type
    AS '$libdir/pg18_crash_repro', 'simple_type_in'
    LANGUAGE C IMMUTABLE STRICT;

CREATE OR REPLACE FUNCTION simple_type_out(simple_type)
    RETURNS cstring
    AS '$libdir/pg18_crash_repro', 'simple_type_out'
    LANGUAGE C IMMUTABLE STRICT;

-- Send/Receive functions
CREATE OR REPLACE FUNCTION simple_type_send(simple_type)
    RETURNS bytea
    AS '$libdir/pg18_crash_repro', 'simple_type_send'
    LANGUAGE C IMMUTABLE STRICT;

CREATE OR REPLACE FUNCTION simple_type_recv(internal)
    RETURNS simple_type
    AS '$libdir/pg18_crash_repro', 'simple_type_recv'
    LANGUAGE C IMMUTABLE STRICT;

-- Now create the full type definition
CREATE TYPE simple_type (
    INPUT = simple_type_in,
    OUTPUT = simple_type_out,
    SEND = simple_type_send,
    RECEIVE = simple_type_recv,
    STORAGE = EXTENDED
);

-- This function works fine (doesn't crash)
CREATE OR REPLACE FUNCTION create_simple_type(text)
    RETURNS simple_type
    AS '$libdir/pg18_crash_repro', 'create_simple_type'
    LANGUAGE C IMMUTABLE STRICT;
```

### **Build and Test Instructions**
```bash
# Build and install extension
make && sudo make install

# Install extension in database
psql postgres -c "CREATE EXTENSION pg18_crash_repro;"

# Test that works (direct function call)
psql postgres -c "SELECT create_simple_type('test');"
# Returns: test

# Test that crashes (type casting)
psql postgres -c "SELECT 'hello'::simple_type;"
# Result: server closed the connection unexpectedly
```