You can connect RisingWave to an AWS Glue Data Catalog to manage metadata for your Iceberg tables. This allows you to use Glue as a central metastore when creating an Iceberg SOURCE, SINK, or CONNECTION (for internal Iceberg tables).
Example
Examples for using an AWS Glue catalog with internal Iceberg tables, sinks, and sources.
Use Glue as the catalog for RisingWave-managed (internal) Iceberg tables by creating an Iceberg CONNECTION, then creating tables with ENGINE = iceberg.CREATE CONNECTION glue_conn WITH (
type = 'iceberg',
warehouse.path = 's3://my-bucket/warehouse/',
catalog.type = 'glue',
catalog.name = 'my_glue_catalog',
glue.region = 'us-west-2',
s3.region = 'us-west-2',
s3.access.key = '...',
s3.secret.key = '...',
enable_config_load = false
);
SET iceberg_engine_connection = 'public.glue_conn';
-- `commit_checkpoint_interval` controls Iceberg commit frequency. Default: about every 60 seconds; set to 1 for faster commits and visibility.
CREATE TABLE my_internal_iceberg_table (
id INT PRIMARY KEY,
name VARCHAR
)
WITH (commit_checkpoint_interval = 1)
ENGINE = iceberg;
Deliver data from RisingWave to a Glue-managed Iceberg table using CREATE SINK.CREATE SINK my_glue_sink FROM my_mv WITH (
connector = 'iceberg',
type = 'upsert',
primary_key = 'id',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'my_db',
create_table_if_not_exists = 'true',
table.name = 'my_table',
catalog.type = 'glue',
catalog.name = 'my_glue_catalog',
glue.region = 'us-west-2',
s3.region = 'us-west-2',
s3.access.key = '...',
s3.secret.key = '...',
enable_config_load = false
);
Ingest data from a Glue-managed Iceberg table into RisingWave using CREATE SOURCE.CREATE SOURCE my_glue_source WITH (
connector = 'iceberg',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'my_db',
table.name = 'my_table',
catalog.type = 'glue',
catalog.name = 'my_glue_catalog',
glue.region = 'us-west-2',
s3.region = 'us-west-2',
s3.access.key = '...',
s3.secret.key = '...',
enable_config_load = false
);
Optional: Use separate permissions for S3 and Glue
Support for glue.* IAM role parameters was added in v2.7.0.
-- Option 1 (separate AK/SK): enable_config_load = false
CREATE CONNECTION glue_conn WITH (
type = 'iceberg',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
-- S3 credentials
s3.region = 'ap-southeast-2',
s3.access.key = '...',
s3.secret.key = '...',
-- Glue credentials (optional; otherwise falls back to S3)
glue.region = 'ap-southeast-2',
glue.access.key = '...',
glue.secret.key = '...',
enable_config_load = false
);
-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE CONNECTION glue_conn WITH (
type = 'iceberg',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
-- S3 role (optional if already in env/config)
s3.region = 'ap-southeast-2',
s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',
-- Glue role (optional; otherwise falls back to S3 role)
glue.region = 'ap-southeast-2',
glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',
enable_config_load = true
);
-- Option 1 (separate AK/SK): enable_config_load = false
CREATE SINK sink_t FROM t WITH (
connector = 'iceberg',
type = 'append-only',
force_append_only = 'true',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'demo',
create_table_if_not_exists = 'true',
table.name = 't',
-- S3 credentials
s3.region = 'ap-southeast-2',
s3.access.key = '...',
s3.secret.key = '...',
-- Glue credentials (optional; otherwise falls back to S3)
glue.region = 'ap-southeast-2',
glue.access.key = '...',
glue.secret.key = '...',
enable_config_load = false
);
-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE SINK sink_t FROM t WITH (
connector = 'iceberg',
type = 'append-only',
force_append_only = 'true',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'demo',
create_table_if_not_exists = 'true',
table.name = 't',
-- S3 role (optional if already in env/config)
s3.region = 'ap-southeast-2',
s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',
-- Glue role (optional; otherwise falls back to S3 role)
glue.region = 'ap-southeast-2',
glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',
enable_config_load = true
);
-- Option 1 (separate AK/SK): enable_config_load = false
CREATE SOURCE source_t WITH (
connector = 'iceberg',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'demo',
table.name = 't',
-- S3 credentials
s3.region = 'ap-southeast-2',
s3.access.key = '...',
s3.secret.key = '...',
-- Glue credentials (optional; otherwise falls back to S3)
glue.region = 'ap-southeast-2',
glue.access.key = '...',
glue.secret.key = '...',
enable_config_load = false
);
-- Option 2 (separate IAM roles): enable_config_load = true
-- Prereq: grant Glue and S3 permissions to your IAM roles, then make sure the machine running this SQL can assume the roles (via env/config),
-- or specify the roles explicitly with s3.iam_role_arn and glue.iam_role_arn.
CREATE SOURCE source_t WITH (
connector = 'iceberg',
catalog.type = 'glue',
catalog.name = 'demo',
warehouse.path = 's3://my-bucket/warehouse/',
database.name = 'demo',
table.name = 't',
-- S3 role (optional if already in env/config)
s3.region = 'ap-southeast-2',
s3.iam_role_arn = 'arn:aws:iam::123456789012:role/MyS3Role',
-- Glue role (optional; otherwise falls back to S3 role)
glue.region = 'ap-southeast-2',
glue.iam_role_arn = 'arn:aws:iam::123456789012:role/MyGlueRole',
enable_config_load = true
);
