Ceph RGW Hammer->Jewel upgrade: adding realms, periods etc
Some quick notes on upgrading a Hammer-era Ceph RGW setup to Jewel, because the upstream notes don't cover it well. The multisite docs are the closest there is, but here's what I put together instead.
- The Zone concept has remained the same.
- A Region is now a Zonegroup.
- The top-level RegionMap is moved inside the content of a Period
- Only one Period can be live at a time, and changes are made to a non-live Period
- The Realm describes which Period is live.
- Additionally, there can be a default Zonegroup and Zone inside the period, as well as a default Zone inside a Zonegroup.
# radosgw-admin region list { "default_info": { "default_region": "default" }, "regions": [ "default" ] } # radosgw-admin region-map get { "master_region": "default", "bucket_quota": { "max_objects": -1, "enabled": false, "max_size_kb": -1 }, "user_quota": { "max_objects": -1, "enabled": false, "max_size_kb": -1 }, "regions": [ { "val": { "zones": [ { "name": "default", "log_meta": "false", "endpoints": [ ], "bucket_index_max_shards": 31, "log_data": "false" } ], "name": "default", "endpoints": [ "https://CENSORED-1.EXAMPLE.COM", "https://CENSORED-2.EXAMPLE.COM" ], "api_name": "CENSORED", "default_placement": "default-placement", "is_master": "true", "hostnames": [ "CENSORED-1.EXAMPLE.COM", "CENSORED-2.EXAMPLE.COM" ], "placement_targets": [ { "name": "default-placement", "tags": [ ] } ], "master_zone": "" }, "key": "default" } ] } # radosgw-admin region get --rgw-region=default { "zones": [ { "log_meta": "false", "name": "default", "bucket_index_max_shards": 31, "endpoints": [ ], "log_data": "false" } ], "master_zone": "", "is_master": "true", "placement_targets": [ { "name": "default-placement", "tags": [ ] } ], "default_placement": "default-placement", "name": "default", "hostnames": [ "CENSORED-1.EXAMPLE.COM", "CENSORED-2.EXAMPLE.COM" ], "endpoints": [ "https://CENSORED-1.EXAMPLE.COM", "https://CENSORED-2.EXAMPLE.COM" ], "api_name": "CENSORED" } # radosgw-admin zone get --rgw-region=default --rgw-zone=default { "log_pool": ".log", "user_swift_pool": ".users.swift", "placement_pools": [ { "val": { "data_pool": ".rgw.buckets", "data_extra_pool": ".rgw.buckets.extra", "index_pool": ".rgw.buckets.index" }, "key": "default-placement" } ], "user_keys_pool": ".users", "control_pool": ".rgw.control", "domain_root": ".rgw", "usage_log_pool": ".usage", "gc_pool": ".rgw.gc", "system_key": { "access_key": "", "secret_key": "" }, "intent_log_pool": ".intent-log", "user_uid_pool": ".users.uid", "user_email_pool": ".users.email" } Initial state, if you were to look on Jewel:
# radosgw-admin zone list { "default_info": "", "zones": [ "default" ] } # radosgw-admin zonegroup list { "default_info": "", "zonegroups": [ "default" ] } # TODO: fill the rest of this up. # Now changing stuff up:
# export SYSTEM_ACCESS_KEY=... SYSTEM_SECRET_KEY=... # radosgw-admin user create \ --system --uid=zone.user \ --display-name="Zone User" \ --access-key=$SYSTEM_ACCESS_KEY \ --secret=$SYSTEM_SECRET_KEY { "user_id": "zone.user", "display_name": "Zone User", "email": "", "suspended": 0, "max_buckets": 1000, "auid": 0, "subusers": [], "keys": [ { "user": "zone.user", "access_key": "...", "secret_key": "..." } ], "swift_keys": [], "caps": [], "op_mask": "read, write, delete", "system": "true", "default_placement": "", "placement_tags": [], "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "temp_url_keys": [] } # radosgw-admin realm create --rgw-realm gold { "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "name": "gold", "current_period": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "epoch": 1 } # radosgw-admin realm list { "default_info": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realms": [ "gold" ] } # radosgw-admin realm get { "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "name": "gold", "current_period": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "epoch": 1 } # radosgw-admin period list { "periods": [ "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb" ] } # radosgw-admin period get { "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "epoch": 1, "predecessor_uuid": "", "sync_status": [], "period_map": { "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "zonegroups": [], "short_zone_ids": [] }, "master_zonegroup": "", "master_zone": "", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realm_name": "gold", "realm_epoch": 1 } # radosgw-admin period update --master-zone=default --master-zonegroup=default { "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103:staging", "epoch": 1, "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "sync_status": [], "period_map": { "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "zonegroups": [], "short_zone_ids": [] }, "master_zonegroup": "", "master_zone": "", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realm_name": "gold", "realm_epoch": 2 } # radosgw-admin period prepare { "id": "8fb1cfbc-ad63-4d92-886a-d939cc52862b", "epoch": 1, "predecessor_uuid": "", "sync_status": [], "period_map": { "id": "8fb1cfbc-ad63-4d92-886a-d939cc52862b", "zonegroups": [], "short_zone_ids": [] }, "master_zonegroup": "", "master_zone": "", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realm_name": "gold", "realm_epoch": 1 } # radosgw-admin zone get --rgw-zonegroup=default --rgw-zone=default >zone.json # radosgw-admin zonegroup get --rgw-zonegroup=default --rgw-zone=default >zonegroup.json # $EDITOR zonegroup.json zone.json ## Add the following data: ## both files: Set realm_id ## zone.json: Set system_user.access_key, Set system_user.secret_key ## zonegroup.json: Set master_zone to "default", Set is_master to "true". # radosgw-admin zone set --rgw-zone=default --rgw-zonegroup=default \ --realm-id=1ac4fd8d-9e77-4fd2-ad54-b591f1734103 \ --infile zone.json \ --master --default # radosgw-admin zonegroup set --rgw-zonegroup=default \ --realm-id=1ac4fd8d-9e77-4fd2-ad54-b591f1734103 \ --infile zonegroup.json \ --master --default # radosgw-admin period update { "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103:staging", "epoch": 1, "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "sync_status": [], "period_map": { "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "zonegroups": [ { "id": "default", "name": "default", "api_name": "CENSORED", "is_master": "true", "endpoints": [ "https:\/\/CENSORED-1.EXAMPLE.COM", "https:\/\/CENSORED-2.EXAMPLE.COM" ], "hostnames": [ "CENSORED-1.EXAMPLE.COM", "CENSORED-2.EXAMPLE.COM" ], "hostnames_s3website": [], "master_zone": "default", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "true", "log_data": "false", "bucket_index_max_shards": 31, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103" } ], "short_zone_ids": [ { "key": "default", "val": 2610307010 } ] }, "master_zonegroup": "default", "master_zone": "default", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realm_name": "gold", "realm_epoch": 2 } # radosgw-admin period commit 2016-08-16 17:51:22.324368 7f8562da6900 0 error read_lastest_epoch .rgw.root:periods.8d0d4955-592c-48b5-93d1-3fa1cec17579.latest_epoch 2016-08-16 17:51:22.347375 7f8562da6900 1 Set the period's master zonegroup default as the default { "id": "8d0d4955-592c-48b5-93d1-3fa1cec17579", "epoch": 1, "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb", "sync_status": [ "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ], "period_map": { "id": "8d0d4955-592c-48b5-93d1-3fa1cec17579", "zonegroups": [ { "id": "default", "name": "default", "api_name": "CENSORED", "is_master": "true", "endpoints": [ "https:\/\/CENSORED-1.EXAMPLE.COM", "https:\/\/CENSORED-2.EXAMPLE.COM" ], "hostnames": [ "CENSORED-1.EXAMPLE.COM", "CENSORED-2.EXAMPLE.COM" ], "hostnames_s3website": [], "master_zone": "default", "zones": [ { "id": "default", "name": "default", "endpoints": [], "log_meta": "true", "log_data": "false", "bucket_index_max_shards": 31, "read_only": "false" } ], "placement_targets": [ { "name": "default-placement", "tags": [] } ], "default_placement": "default-placement", "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103" } ], "short_zone_ids": [ { "key": "default", "val": 2610307010 } ] }, "master_zonegroup": "default", "master_zone": "default", "period_config": { "bucket_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 }, "user_quota": { "enabled": false, "max_size_kb": -1, "max_objects": -1 } }, "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103", "realm_name": "gold", "realm_epoch": 2 }