From 8ce9323ae113341b98eca215d74ca4392d1f7bc3 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 02:52:25 +0530 Subject: [PATCH] Core: Load and serialize nanosecond timestamp defaults JSON single-value conversion had no handler for timestamp_ns and timestamptz_ns, so loading a V3 metadata file with a nanosecond default raised TypeError. Python datetime cannot hold nanoseconds, so parse these values as nanoseconds since epoch and format them back with nine fractional digits. --- pyiceberg/conversions.py | 48 +++++++++++++++++++++++++++++++++++++ pyiceberg/utils/datetime.py | 14 +++++++++++ tests/test_conversions.py | 16 +++++++++++++ tests/test_types.py | 17 +++++++++++++ 4 files changed, 95 insertions(+) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index eee8da504b..2662084b86 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -76,11 +76,15 @@ time_str_to_micros, time_to_micros, timestamp_to_micros, + timestamp_to_nanos, timestamptz_to_micros, + timestamptz_to_nanos, to_human_day, to_human_time, to_human_timestamp, + to_human_timestamp_ns, to_human_timestamptz, + to_human_timestamptz_ns, ) from pyiceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal @@ -487,6 +491,22 @@ def _(_: TimestamptzType, val: int | datetime) -> str: return to_human_timestamptz(val) +@to_json.register(TimestampNanoType) +def _(_: TimestampNanoType, val: int | datetime) -> str: + """Python datetime (without timezone) or nanoseconds since epoch serializes into an ISO8601 timestamp.""" + if isinstance(val, datetime): + val = datetime_to_nanos(val) + return to_human_timestamp_ns(val) + + +@to_json.register(TimestamptzNanoType) +def _(_: TimestamptzNanoType, val: int | datetime) -> str: + """Python datetime (with timezone) or nanoseconds since epoch serializes into an ISO8601 timestamp.""" + if isinstance(val, datetime): + val = datetime_to_nanos(val) + return to_human_timestamptz_ns(val) + + @to_json.register(FloatType) @to_json.register(DoubleType) def _(_: FloatType | DoubleType, val: float) -> float: @@ -641,6 +661,34 @@ def _(_: TimestamptzType, val: str | int | datetime) -> datetime: return val +@from_json.register(TimestampNanoType) +def _(_: TimestampNanoType, val: str | int | datetime) -> int: + """JSON ISO8601 string into nanoseconds since epoch. + + Python datetime cannot hold nanoseconds, so the value stays an int. + """ + if isinstance(val, str): + return timestamp_to_nanos(val) + elif isinstance(val, datetime): + return datetime_to_nanos(val) + else: + return val + + +@from_json.register(TimestamptzNanoType) +def _(_: TimestamptzNanoType, val: str | int | datetime) -> int: + """JSON ISO8601 string into nanoseconds since epoch. + + Python datetime cannot hold nanoseconds, so the value stays an int. + """ + if isinstance(val, str): + return timestamptz_to_nanos(val) + elif isinstance(val, datetime): + return datetime_to_nanos(val) + else: + return val + + @from_json.register(FloatType) @from_json.register(DoubleType) def _(_: FloatType | DoubleType, val: float) -> float: diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 70ffb1f303..776513fdf7 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -224,6 +224,20 @@ def to_human_timestamp(timestamp_micros: int) -> str: return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat() +def to_human_timestamp_ns(timestamp_nanos: int) -> str: + """Convert a TimestampNanoType value to human string.""" + seconds, nanos = divmod(timestamp_nanos, 1_000_000_000) + timestamp = EPOCH_TIMESTAMP + timedelta(seconds=seconds) + return f"{timestamp.isoformat(timespec='seconds')}.{nanos:09d}" + + +def to_human_timestamptz_ns(timestamp_nanos: int) -> str: + """Convert a TimestamptzNanoType value to human string.""" + seconds, nanos = divmod(timestamp_nanos, 1_000_000_000) + timestamp = EPOCH_TIMESTAMPTZ + timedelta(seconds=seconds) + return f"{timestamp.replace(tzinfo=None).isoformat(timespec='seconds')}.{nanos:09d}+00:00" + + def micros_to_hours(micros: int) -> int: """Convert a timestamp in microseconds to hours from 1970-01-01T00:00.""" return micros // 3_600_000_000 diff --git a/tests/test_conversions.py b/tests/test_conversions.py index fbab096fa5..eae58b9e89 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -610,6 +610,14 @@ def test_datetime_obj_to_bytes(primitive_type: PrimitiveType, value: datetime | (TimeType(), time(22, 31, 8, 123456), "22:31:08.123456"), (TimestampType(), datetime(2017, 11, 16, 22, 31, 8, 123456), "2017-11-16T22:31:08.123456"), (TimestamptzType(), datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC), "2017-11-16T22:31:08.123456+00:00"), + (TimestampNanoType(), 1510871468123456789, "2017-11-16T22:31:08.123456789"), + (TimestamptzNanoType(), 1510871468123456789, "2017-11-16T22:31:08.123456789+00:00"), + (TimestampNanoType(), datetime(2017, 11, 16, 22, 31, 8, 123456), "2017-11-16T22:31:08.123456000"), + ( + TimestamptzNanoType(), + datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC), + "2017-11-16T22:31:08.123456000+00:00", + ), (StringType(), "iceberg", "iceberg"), (BinaryType(), b"\x01\x02\x03\xff", "010203ff"), (FixedType(4), b"\x01\x02\x03\xff", "010203ff"), @@ -635,6 +643,8 @@ def test_json_single_serialization(primitive_type: PrimitiveType, value: Any, ex (TimeType(), time(22, 31, 8, 123456)), (TimestampType(), datetime(2017, 11, 16, 22, 31, 8, 123456)), (TimestamptzType(), datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC)), + (TimestampNanoType(), 1510871468123456789), + (TimestamptzNanoType(), 1510871468123456789), (StringType(), "iceberg"), (BinaryType(), b"\x01\x02\x03\xff"), (FixedType(4), b"\x01\x02\x03\xff"), @@ -653,6 +663,12 @@ def test_json_serialize_roundtrip(primitive_type: PrimitiveType, value: Any) -> "2017-11-16T22:31:08.123456789+00:00", datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC), ), + (TimestampNanoType(), "2017-11-16T22:31:08.123456789", 1510871468123456789), + ( + TimestamptzNanoType(), + "2017-11-16T22:31:08.123456789+00:00", + 1510871468123456789, + ), ], ) def test_json_single_deserialization_with_nano_precision( diff --git a/tests/test_types.py b/tests/test_types.py index eb8ae2ea52..7def05e4fd 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -43,7 +43,9 @@ PrimitiveType, StringType, StructType, + TimestampNanoType, TimestampType, + TimestamptzNanoType, TimestamptzType, TimeType, UUIDType, @@ -947,3 +949,18 @@ def test_decimal_scale_validation() -> None: with pytest.raises(ValidationError, match="Decimal scale must be between 0 and the precision"): DecimalType(5, 10) + + +@pytest.mark.parametrize( + "field_type, expected_json", + [ + (TimestampNanoType(), "2017-11-16T22:31:08.123456789"), + (TimestamptzNanoType(), "2017-11-16T22:31:08.123456789+00:00"), + ], +) +def test_nested_field_nanosecond_defaults(field_type: PrimitiveType, expected_json: str) -> None: + """Nanosecond defaults load as nanoseconds since epoch and serialize back without losing precision.""" + field = NestedField(1, "ts", field_type, initial_default=expected_json) + + assert field.initial_default == 1510871468123456789 + assert field.model_dump()["initial-default"] == expected_json