From 9a547efe90e3f92f11b740f4e5fb278dc1de92d4 Mon Sep 17 00:00:00 2001
From: Zsolt Parragi <zsolt.parragi@percona.com>
Date: Wed, 8 Jul 2026 22:45:09 +0000
Subject: [PATCH 5/8] Add JSON5 number extensions to the JSON lexer

When json5 is set, accept hex integers, leading and trailing decimal
points, explicit +/- sign, and Infinity/-Infinity/+Infinity/NaN, in
normal and incremental modes.
---
 src/common/jsonapi.c                          | 270 +++++++++++++++---
 src/include/common/jsonapi.h                  |   7 +-
 .../test_json_parser/json5_numbers.json5      |  19 ++
 .../test_json_parser/json5_numbers.out        |  19 ++
 .../test_json_parser/t/005_test_json5.pl      |  87 +++++-
 5 files changed, 366 insertions(+), 36 deletions(-)
 create mode 100644 src/test/modules/test_json_parser/json5_numbers.json5
 create mode 100644 src/test/modules/test_json_parser/json5_numbers.out

diff --git a/src/common/jsonapi.c b/src/common/jsonapi.c
index f7728ffcf1f..5920a3e1c7c 100644
--- a/src/common/jsonapi.c
+++ b/src/common/jsonapi.c
@@ -661,12 +661,17 @@ have_prediction(JsonParserStack *pstack)
 	return pstack->pred_index > 0;
 }
 
-/* reserved words that json5 accepts as unquoted object keys */
+/*
+ * Keywords that json5 accepts as unquoted object keys: the reserved words
+ * true/false/null, plus Infinity/NaN, which lex as numbers but are ordinary
+ * identifiers too.
+ */
 static inline bool
 json5_keyword_key(const JsonLexContext *lex, JsonTokenType tok)
 {
 	return tok == JSON_TOKEN_TRUE || tok == JSON_TOKEN_FALSE ||
-		tok == JSON_TOKEN_NULL;
+		tok == JSON_TOKEN_NULL ||
+		(tok == JSON_TOKEN_NUMBER && lex->token_is_identifier);
 }
 
 /*
@@ -1562,7 +1567,8 @@ parse_object(JsonLexContext *lex, const JsonSemAction *sem)
 		case JSON_TOKEN_TRUE:
 		case JSON_TOKEN_FALSE:
 		case JSON_TOKEN_NULL:
-			if (!lex->json5)
+		case JSON_TOKEN_NUMBER:
+			if (!lex->json5 || !json5_keyword_key(lex, tok))
 			{
 				result = report_parse_error(JSON_PARSE_OBJECT_START, lex);
 				break;
@@ -1713,6 +1719,41 @@ parse_array(JsonLexContext *lex, const JsonSemAction *sem)
 	return JSON_SUCCESS;
 }
 
+/*
+ * Lex a json5 signed Infinity/NaN.  s points at the '+' or '-' and a letter
+ * follows.  The whole identifier word after the sign is scanned, so trailing
+ * identifier characters make the whole word one invalid token instead of
+ * leaving a prefix match behind, and a word that might continue in the next
+ * chunk is saved as a partial token.  Sets token_type on success.
+ */
+static JsonParseErrorType
+json5_lex_signed_word(JsonLexContext *lex, const char *s)
+{
+	const char *const end = lex->input + lex->input_length;
+	const char *p;
+
+	for (p = s + 1; p < end && (JSON_ALPHANUMERIC_CHAR(*p) || *p == '$'); p++)
+		 /* skip */ ;
+
+	if (p == end && lex->incremental && !lex->inc_state->is_last_chunk)
+	{
+		jsonapi_appendBinaryStringInfo(&(lex->inc_state->partial_token),
+									   s, end - s);
+		return JSON_INCOMPLETE;
+	}
+
+	lex->prev_token_terminator = lex->token_terminator;
+	lex->token_terminator = p;
+
+	if ((p - s == 9 && memcmp(s + 1, "Infinity", 8) == 0) ||
+		(p - s == 4 && memcmp(s + 1, "NaN", 3) == 0))
+	{
+		lex->token_type = JSON_TOKEN_NUMBER;
+		return JSON_SUCCESS;
+	}
+	return JSON_INVALID_TOKEN;
+}
+
 /*
  * Lex one token from the input stream.
  *
@@ -1815,31 +1856,50 @@ json_lex(JsonLexContext *lex)
 			/* not a string */
 			char		c = ptok->data[0];
 
-			if (c == '-' || (c >= '0' && c <= '9'))
+			if (c == '-' || (c >= '0' && c <= '9') ||
+				(lex->json5 && (c == '+' || c == '.')))
 			{
 				/*
 				 * Accumulate numeric continuations, respecting JSON number
-				 * grammar: -? int [frac] [exp]
+				 * grammar: -? int [frac] [exp], plus the json5 extensions
+				 * (leading '+', leading/trailing '.', 0x hex).
 				 *
 				 * We must track what parts of the number we've already seen
 				 * so we don't over-consume.  '.' is valid only once and not
 				 * after 'e'/'E'; 'e'/'E' is valid only once; '+'/'-' are
-				 * valid only immediately after 'e'/'E'.
+				 * valid only immediately after 'e'/'E'.  Once we know we're
+				 * looking at a json5 hex number, digits switch to the hex
+				 * alphabet and '.'/'e'/'E' just become hex digits too.
 				 */
 				bool		numend = false;
 				bool		seen_dot = false;
 				bool		seen_exp = false;
+				bool		seen_hex = false;
 				char		prev;
+				int			scan_start = (lex->json5 && (c == '+' || c == '-')) ? 1 : 0;
 
-				/* Scan existing partial token for state */
-				for (int j = 0; j < ptok->len; j++)
+				/*
+				 * hex detected in an earlier chunk, recovered from the
+				 * buffered prefix
+				 */
+				if (lex->json5 && ptok->len - scan_start >= 2 &&
+					ptok->data[scan_start] == '0' &&
+					(ptok->data[scan_start + 1] == 'x' ||
+					 ptok->data[scan_start + 1] == 'X'))
+					seen_hex = true;
+
+				if (!seen_hex)
 				{
-					char		pc = ptok->data[j];
+					/* Scan existing partial token for state */
+					for (int j = 0; j < ptok->len; j++)
+					{
+						char		pc = ptok->data[j];
 
-					if (pc == '.')
-						seen_dot = true;
-					else if (pc == 'e' || pc == 'E')
-						seen_exp = true;
+						if (pc == '.')
+							seen_dot = true;
+						else if (pc == 'e' || pc == 'E')
+							seen_exp = true;
+					}
 				}
 				prev = ptok->data[ptok->len - 1];
 
@@ -1847,6 +1907,22 @@ json_lex(JsonLexContext *lex)
 				{
 					char		cc = lex->input[i];
 
+					if (seen_hex)
+					{
+						if ((cc >= '0' && cc <= '9') ||
+							(cc >= 'a' && cc <= 'f') ||
+							(cc >= 'A' && cc <= 'F'))
+						{
+							jsonapi_appendStringInfoCharMacro(ptok, cc);
+							added++;
+						}
+						else
+							numend = true;
+						if (!numend)
+							prev = cc;
+						continue;
+					}
+
 					switch (cc)
 					{
 						case '+':
@@ -1893,6 +1969,24 @@ json_lex(JsonLexContext *lex)
 							jsonapi_appendStringInfoCharMacro(ptok, cc);
 							added++;
 							break;
+						case 'x':
+						case 'X':
+
+							/*
+							 * hex 'x' just arrived this chunk, right after a
+							 * lone '0'
+							 */
+							if (lex->json5 && !seen_dot && !seen_exp &&
+								ptok->len - scan_start == 1 &&
+								ptok->data[scan_start] == '0')
+							{
+								seen_hex = true;
+								jsonapi_appendStringInfoCharMacro(ptok, cc);
+								added++;
+							}
+							else
+								numend = true;
+							break;
 						default:
 							numend = true;
 					}
@@ -2204,12 +2298,60 @@ json_lex(JsonLexContext *lex)
 				lex->token_type = JSON_TOKEN_STRING;
 				break;
 			case '-':
+				if (lex->json5 && s + 1 < end &&
+					(*(s + 1) == 'I' || *(s + 1) == 'N'))
+				{
+					/* Signed Infinity/NaN. */
+					result = json5_lex_signed_word(lex, s);
+					if (result != JSON_SUCCESS)
+						return result;
+					break;
+				}
 				/* Negative number. */
 				result = json_lex_number(lex, s + 1, NULL, NULL);
 				if (result != JSON_SUCCESS)
 					return result;
 				lex->token_type = JSON_TOKEN_NUMBER;
 				break;
+			case '+':
+				if (lex->json5)
+				{
+					if (s + 1 < end && (*(s + 1) == 'I' || *(s + 1) == 'N'))
+					{
+						/* Signed Infinity/NaN. */
+						result = json5_lex_signed_word(lex, s);
+						if (result != JSON_SUCCESS)
+							return result;
+						break;
+					}
+					/* Explicit positive sign. */
+					result = json_lex_number(lex, s + 1, NULL, NULL);
+					if (result != JSON_SUCCESS)
+						return result;
+					lex->token_type = JSON_TOKEN_NUMBER;
+					break;
+				}
+				lex->prev_token_terminator = lex->token_terminator;
+				lex->token_terminator = s + 1;
+				return JSON_INVALID_TOKEN;
+			case '.':
+				if (lex->json5)
+				{
+					/*
+					 * Leading decimal point, e.g. .5.  json_lex_number's own
+					 * incremental handling takes care of a lone '.' sitting
+					 * at the end of a chunk (it buffers just the '.' and asks
+					 * for more input).
+					 */
+					result = json_lex_number(lex, s, NULL, NULL);
+					if (result != JSON_SUCCESS)
+						return result;
+					lex->token_type = JSON_TOKEN_NUMBER;
+					break;
+				}
+				lex->prev_token_terminator = lex->token_terminator;
+				lex->token_terminator = s + 1;
+				return JSON_INVALID_TOKEN;
 			case '0':
 			case '1':
 			case '2':
@@ -2282,9 +2424,19 @@ json_lex(JsonLexContext *lex)
 					}
 					else if (p - s == 5 && memcmp(s, "false", 5) == 0)
 						lex->token_type = JSON_TOKEN_FALSE;
+					else if (lex->json5 && p - s == 8 && memcmp(s, "Infinity", 8) == 0)
+					{
+						/* also an identifier, so usable as an unquoted key */
+						lex->token_type = JSON_TOKEN_NUMBER;
+						lex->token_is_identifier = true;
+					}
+					else if (lex->json5 && p - s == 3 && memcmp(s, "NaN", 3) == 0)
+					{
+						lex->token_type = JSON_TOKEN_NUMBER;
+						lex->token_is_identifier = true;
+					}
 					else if (lex->json5)
 					{
-						/* Infinity/NaN handled with number extensions */
 json5_identifier:
 						if ((*s >= 'a' && *s <= 'z') ||
 							(*s >= 'A' && *s <= 'Z') ||
@@ -2627,34 +2779,43 @@ json_lex_number(JsonLexContext *lex, const char *s,
 				bool *num_err, size_t *total_len)
 {
 	bool		error = false;
+	bool		is_hex = false;
 	int			len = s - lex->input;
 
 	/* Part (1): leading sign indicator. */
 	/* Caller already did this for us; so do nothing. */
 
-	/* Part (2): parse main digit string. */
-	if (len < lex->input_length && *s == '0')
+	if (lex->json5 && len < lex->input_length && *s == '0' &&
+		len + 1 < lex->input_length &&
+		(*(s + 1) == 'x' || *(s + 1) == 'X'))
 	{
-		s++;
-		len++;
-	}
-	else if (len < lex->input_length && *s >= '1' && *s <= '9')
-	{
-		do
+		/* json5 hex integer, e.g. 0x1F -- no fraction or exponent part. */
+		is_hex = true;
+		s += 2;
+		len += 2;
+		if (len >= lex->input_length ||
+			!((*s >= '0' && *s <= '9') ||
+			  (*s >= 'a' && *s <= 'f') ||
+			  (*s >= 'A' && *s <= 'F')))
+			error = true;
+		else
 		{
-			s++;
-			len++;
-		} while (len < lex->input_length && *s >= '0' && *s <= '9');
+			do
+			{
+				s++;
+				len++;
+			} while (len < lex->input_length &&
+					 ((*s >= '0' && *s <= '9') ||
+					  (*s >= 'a' && *s <= 'f') ||
+					  (*s >= 'A' && *s <= 'F')));
+		}
 	}
-	else
-		error = true;
-
-	/* Part (3): parse optional decimal portion. */
-	if (len < lex->input_length && *s == '.')
+	else if (lex->json5 && len < lex->input_length && *s == '.')
 	{
+		/* Part (2)/(3): json5 leading decimal point, e.g. .5 */
 		s++;
 		len++;
-		if (len == lex->input_length || *s < '0' || *s > '9')
+		if (len >= lex->input_length || *s < '0' || *s > '9')
 			error = true;
 		else
 		{
@@ -2665,9 +2826,52 @@ json_lex_number(JsonLexContext *lex, const char *s,
 			} while (len < lex->input_length && *s >= '0' && *s <= '9');
 		}
 	}
+	else
+	{
+		/* Part (2): parse main digit string. */
+		if (len < lex->input_length && *s == '0')
+		{
+			s++;
+			len++;
+		}
+		else if (len < lex->input_length && *s >= '1' && *s <= '9')
+		{
+			do
+			{
+				s++;
+				len++;
+			} while (len < lex->input_length && *s >= '0' && *s <= '9');
+		}
+		else
+			error = true;
 
-	/* Part (4): parse optional exponent. */
-	if (len < lex->input_length && (*s == 'e' || *s == 'E'))
+		/* Part (3): parse optional decimal portion. */
+		if (len < lex->input_length && *s == '.')
+		{
+			s++;
+			len++;
+			if (len == lex->input_length || *s < '0' || *s > '9')
+			{
+				/* json5 allows a bare trailing decimal point, e.g. 5. */
+				if (!lex->json5)
+					error = true;
+			}
+			else
+			{
+				do
+				{
+					s++;
+					len++;
+				} while (len < lex->input_length && *s >= '0' && *s <= '9');
+			}
+		}
+	}
+
+	/*
+	 * Part (4): parse optional exponent.  Shared by the leading-dot and
+	 * standard paths; hex integers have no exponent part.
+	 */
+	if (!is_hex && len < lex->input_length && (*s == 'e' || *s == 'E'))
 	{
 		s++;
 		len++;
diff --git a/src/include/common/jsonapi.h b/src/include/common/jsonapi.h
index d19f03f8556..19d970996ee 100644
--- a/src/include/common/jsonapi.h
+++ b/src/include/common/jsonapi.h
@@ -109,7 +109,12 @@ typedef struct JsonLexContext
 	bool		incremental;
 	bool		json5;
 	JsonTokenType token_type;
-	/* current token is a json5 unquoted identifier, only legal as a key */
+
+	/*
+	 * The current token is a json5 identifier: an unquoted string, legal
+	 * only as an object key, or an Infinity/NaN number, which doubles as an
+	 * unquoted key.
+	 */
 	bool		token_is_identifier;
 	int			lex_level;
 	uint32		flags;
diff --git a/src/test/modules/test_json_parser/json5_numbers.json5 b/src/test/modules/test_json_parser/json5_numbers.json5
new file mode 100644
index 00000000000..0c27b445f08
--- /dev/null
+++ b/src/test/modules/test_json_parser/json5_numbers.json5
@@ -0,0 +1,19 @@
+{
+  hexUpper: 0xDEADBEEF,
+  hexLower: 0xdeadbeef,
+  hexZero: 0x0,
+  hexSimple: 0x1F,
+  leadingDot: .5,
+  trailingDot: 5.,
+  plusInt: +42,
+  minusInt: -42,
+  exp: 6.022e23,
+  posInfinity: +Infinity,
+  negInfinity: -Infinity,
+  infinity: Infinity,
+  notANumber: NaN,
+  plain: 123,
+  negFloat: -4.5,
+  expPlain: 1e10,
+  zero: 0
+}
diff --git a/src/test/modules/test_json_parser/json5_numbers.out b/src/test/modules/test_json_parser/json5_numbers.out
new file mode 100644
index 00000000000..9842692a2f0
--- /dev/null
+++ b/src/test/modules/test_json_parser/json5_numbers.out
@@ -0,0 +1,19 @@
+{
+"hexUpper": 0xDEADBEEF,
+"hexLower": 0xdeadbeef,
+"hexZero": 0x0,
+"hexSimple": 0x1F,
+"leadingDot": .5,
+"trailingDot": 5.,
+"plusInt": +42,
+"minusInt": -42,
+"exp": 6.022e23,
+"posInfinity": +Infinity,
+"negInfinity": -Infinity,
+"infinity": Infinity,
+"notANumber": NaN,
+"plain": 123,
+"negFloat": -4.5,
+"expPlain": 1e10,
+"zero": 0
+}
diff --git a/src/test/modules/test_json_parser/t/005_test_json5.pl b/src/test/modules/test_json_parser/t/005_test_json5.pl
index 3302c756b50..62572f51c45 100644
--- a/src/test/modules/test_json_parser/t/005_test_json5.pl
+++ b/src/test/modules/test_json_parser/t/005_test_json5.pl
@@ -33,7 +33,8 @@ my @features = (
 	},
 	{ name => 'trailing commas', file => 'json5_trailing_commas' },
 	{ name => 'unquoted keys', file => 'json5_keys' },
-	{ name => 'single-quoted strings', file => 'json5_strings' },);
+	{ name => 'single-quoted strings', file => 'json5_strings' },
+	{ name => 'numbers', file => 'json5_numbers' },);
 
 # Inputs that stay invalid even in json5 mode.
 my @json5_invalid = (
@@ -43,7 +44,45 @@ my @json5_invalid = (
 	[ 'identifier value in array', '[a]' ],
 	[ 'bare identifier', 'undefined' ],
 	[ 'dollar after number', '2$' ],
-	[ 'dollar after number in array', '[25$]' ],);
+	[ 'dollar after number in array', '[25$]' ],
+	[ 'bare 0x', '0x' ],
+	[ 'bare dot', '.' ],
+	[ 'missing value', '{"a":}' ],
+	[ 'missing colon and value', '{"a"}' ],
+	[ 'missing value after unquoted key', '{a:}' ],
+	[ 'reserved word key without value', '{true}' ],
+	[ 'missing colon and value after second key', '{"a":1,"b"}' ],
+	[ 'unterminated block comment after value', '1 /*' ],
+	[ 'unterminated block comment in array', '[1 /*' ],
+	[ 'unterminated block comment with content', '1 /*/ 2' ],
+	[ 'garbage after cr-terminated line comment', "1 // c\rx" ],
+	[ 'signed infinity key', '{-Infinity: 1}' ],
+	[ 'signed infinity with trailing junk', '-Infinityz' ],
+	[ 'signed infinity with trailing dollar', '-Infinity$' ],
+	[ 'signed nan with trailing junk', '+NaN5' ],);
+
+# Valid json5 whose parse must not depend on where chunk boundaries
+# fall (each caught a real bug: comment resume state, '$' in the
+# partial-token continuation, identifier handling of Infinity/NaN).
+my @json5_chunk_valid = (
+	[ 'block comment containing /*-like text', '/*/ */ 1' ],
+	[ 'dollar inside unquoted key', '{ab$c: 1}' ],
+	[ 'Infinity as unquoted key', '{Infinity: 1}' ],
+	[ 'NaN as unquoted key', '{NaN: 1}' ],
+	[ 'negative NaN', '-NaN' ],
+	[ 'positive NaN', '+NaN' ],
+	[ 'cr-terminated line comment', "[1, // c\r2\n]" ],);
+
+# Number extensions, each individually rejected without --json5.
+my @json_invalid_numbers = (
+	[ 'hex', '0x1F' ],
+	[ 'leading dot', '.5' ],
+	[ 'trailing dot', '5.' ],
+	[ 'plus sign', '+42' ],
+	[ 'infinity', 'Infinity' ],
+	[ 'negative infinity', '-Infinity' ],
+	[ 'positive infinity', '+Infinity' ],
+	[ 'nan', 'NaN' ],);
 
 # Write $content to a temp file and return the file name.
 sub inline_file
@@ -130,6 +169,50 @@ foreach my $exe (@exes)
 		}
 	}
 
+	foreach my $v (@json5_chunk_valid)
+	{
+		my ($label, $content) = @$v;
+		my $fname = inline_file($content);
+
+		my ($whole, $whole_err) =
+		  run_command([ @$exe, "-s", "--json5", $fname ]);
+
+		is($whole_err, "", "json5 mode, whole input: $label: no error");
+
+		for my $size (3, 1)
+		{
+			my ($stdout, $stderr) =
+			  run_command([ @$exe, "-s", "--json5", "-c", $size, $fname ]);
+
+			is($stderr, "",
+				"json5 mode, chunk size $size: $label: no error");
+			is($stdout, $whole,
+				"json5 mode, chunk size $size: $label: matches whole input");
+		}
+	}
+
+	# Specifically exercise a hex number split in the middle of its
+	# digit run (0x1|F) and a leading-dot number split right after the
+	# dot (.|5), at the chunk size that forces exactly that split.
+	my ($stdout, $stderr) =
+	  run_command([ @$exe, "-s", "--json5", "-c", 3, inline_file("0x1F") ]);
+
+	is($stdout, "0x1F", "json5 mode: 0x1|F split reassembles");
+	is($stderr, "", "json5 mode: 0x1|F split: no error output");
+
+	($stdout, $stderr) =
+	  run_command([ @$exe, "-s", "--json5", "-c", 1, inline_file(".5") ]);
+
+	is($stdout, ".5", "json5 mode: .|5 split reassembles");
+	is($stderr, "", "json5 mode: .|5 split: no error output");
+
+	foreach my $inv (@json_invalid_numbers)
+	{
+		my ($label, $content) = @$inv;
+
+		check_rejected($exe, inline_file($content),
+			"non-json5 mode: $label form");
+	}
 }
 
 done_testing();
-- 
2.43.0

