Skip to content

fix: support MonthTransform for partitioning #1176

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kevinjqliu merged 4 commits into from
Sep 16, 2024
Merged

fix: support MonthTransform for partitioning #1176

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d964a25
fix: support MonthTransform for partitioning
felixscherz Sep 13, 2024
1cce3a2
test: add tests for generating default names for other transforms
felixscherz Sep 14, 2024
32bf153
fix: delete duplicate test case
felixscherz Sep 14, 2024
e30d54e
chore: run formatting
felixscherz Sep 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions pyiceberg/partitioning.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
DayTransform,
HourTransform,
IdentityTransform,
MonthTransform,
Transform,
TruncateTransform,
UnknownTransform,
Expand Down Expand Up @@ -359,6 +360,8 @@ def _visit_partition_field(schema: Schema, field: PartitionField, visitor: Parti
return visitor.day(field.field_id, source_name, field.source_id)
elif isinstance(transform, HourTransform):
return visitor.hour(field.field_id, source_name, field.source_id)
elif isinstance(transform, MonthTransform):
return visitor.month(field.field_id, source_name, field.source_id)
elif isinstance(transform, YearTransform):
return visitor.year(field.field_id, source_name, field.source_id)
elif isinstance(transform, VoidTransform):
Expand Down
79 changes: 79 additions & 0 deletions tests/integration/test_partition_evolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ def test_add_year(catalog: Catalog) -> None:
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, YearTransform(), "year_transform"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_year_generates_default_name(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", YearTransform()).commit()
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, YearTransform(), "event_ts_year"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_month(catalog: Catalog) -> None:
Expand All @@ -100,6 +108,13 @@ def test_add_month(catalog: Catalog) -> None:
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, MonthTransform(), "month_transform"))


@pytest.mark.integration
Copy link
Copy Markdown
Contributor

@kevinjqliu kevinjqliu Sep 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: it seems like there's no test for the "default name" for any of the transforms. do you mind adding those as well?

https://github.com/apache/iceberg-python/pull/1176/files#diff-428471774c19f5aef70af04a0b2dcbbc263c2c8b372dbc3993356e2c851539ecR310-R336

Copy link
Copy Markdown
Contributor Author

@felixscherz felixscherz Sep 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes of course, I'll add them:)

@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_month_generates_default_name(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", MonthTransform()).commit()
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, MonthTransform(), "event_ts_month"))

@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_day(catalog: Catalog) -> None:
Expand All @@ -108,6 +123,14 @@ def test_add_day(catalog: Catalog) -> None:
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, DayTransform(), "day_transform"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_day_generates_default_name(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", DayTransform()).commit()
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, DayTransform(), "event_ts_day"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_hour(catalog: Catalog) -> None:
Expand All @@ -116,6 +139,14 @@ def test_add_hour(catalog: Catalog) -> None:
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, HourTransform(), "hour_transform"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_hour_generates_default_name(catalog: Catalog) -> None:
table = _table(catalog)
table.update_spec().add_field("event_ts", HourTransform()).commit()
_validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, HourTransform(), "event_ts_hour"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_bucket(catalog: Catalog, table_schema_simple: Schema) -> None:
Expand All @@ -124,6 +155,14 @@ def test_add_bucket(catalog: Catalog, table_schema_simple: Schema) -> None:
_validate_new_partition_fields(simple_table, 1000, 1, 1000, PartitionField(1, 1000, BucketTransform(12), "bucket_transform"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_bucket_generates_default_name(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", BucketTransform(12)).commit()
_validate_new_partition_fields(simple_table, 1000, 1, 1000, PartitionField(1, 1000, BucketTransform(12), "foo_bucket_12"))


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_truncate(catalog: Catalog, table_schema_simple: Schema) -> None:
Expand All @@ -134,6 +173,16 @@ def test_add_truncate(catalog: Catalog, table_schema_simple: Schema) -> None:
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_truncate_generates_default_name(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", TruncateTransform(1)).commit()
_validate_new_partition_fields(
simple_table, 1000, 1, 1000, PartitionField(1, 1000, TruncateTransform(1), "foo_trunc_1")
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_multiple_adds(catalog: Catalog) -> None:
Expand All @@ -152,6 +201,36 @@ def test_multiple_adds(catalog: Catalog) -> None:
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_void(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", VoidTransform(), "void_transform").commit()
_validate_new_partition_fields(
simple_table, 1000, 1, 1000, PartitionField(1, 1000, VoidTransform(), "void_transform")
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_void_generates_default_name(catalog: Catalog, table_schema_simple: Schema) -> None:
simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", VoidTransform()).commit()
_validate_new_partition_fields(
simple_table, 1000, 1, 1000, PartitionField(1, 1000, VoidTransform(), "foo_null")
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_truncate_generates_default_name(catalog: Catalog, table_schema_simple: Schema) -> None:
Copy link
Copy Markdown
Contributor

@kevinjqliu kevinjqliu Sep 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this is defined twice, remove this one

Copy link
Copy Markdown
Contributor Author

@felixscherz felixscherz Sep 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for catching that, removed it!

simple_table = _create_table_with_schema(catalog, table_schema_simple, "1")
simple_table.update_spec().add_field("foo", TruncateTransform(1)).commit()
_validate_new_partition_fields(
simple_table, 1000, 1, 1000, PartitionField(1, 1000, TruncateTransform(1), "foo_trunc_1")
)


@pytest.mark.integration
@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
def test_add_hour_to_day(catalog: Catalog) -> None:
Expand Down