new-cluster localities=us-east-1,us-east-1,us-east-1,us-west-1,us-west-1,us-west-1,us-central-1,eu-central-1,eu-west-1
----

exec-sql idx=0
CREATE DATABASE db PRIMARY REGION "us-east-1" REGIONS  "us-central-1", "eu-central-1";
----

exec-sql idx=0
CREATE TABLE db.rbt(k INT PRIMARY KEY, v INT) LOCALITY REGIONAL BY TABLE IN "us-east-1";
----

exec-sql idx=0
INSERT INTO db.rbt VALUES (1, 1), (2,2)
----

wait-for-zone-config-changes db-name=db table-name=rbt num-voters=3 num-non-voters=2 leaseholder=0 voter=1,2 non-voter=6,7 not-present=3,4,5,8
----

trace-sql idx=0
SELECT * FROM db.rbt WHERE k = 1
----
served locally: true
served via follower read: false

refresh-range-descriptor-cache idx=1 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

trace-sql idx=1
SELECT * FROM db.rbt WHERE k = 1
----
served locally: false

refresh-range-descriptor-cache idx=2 table-name=rbt
SELECT * FROM db.rbt WHERE k = 2
----
LAG_BY_CLUSTER_SETTING

trace-sql idx=2
SELECT * FROM db.rbt WHERE k = 1
----
served locally: false

# Sleep to ensure we can get follower reads.
sleep-for-follower-read
----

refresh-range-descriptor-cache idx=3 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

# us-west-1 should be able to serve neither follower reads nor live reads.
trace-sql idx=3
SELECT * FROM db.rbt WHERE k = 1
----
served locally: false

trace-sql idx=3
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: false

refresh-range-descriptor-cache idx=7 table-name=rbt
SELECT * FROM db.rbt WHERE k = 2
----
LAG_BY_CLUSTER_SETTING

# eu-central-1 is a non-voter, so we should only be able to get follower reads.
trace-sql idx=7
SELECT * FROM db.rbt WHERE k = 1
----
served locally: false

trace-sql idx=7
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: true
served via follower read: true

# Add back us-west-1 for rehoming.
exec-sql idx=0
ALTER DATABASE db ADD REGION "us-west-1";
----

# Now rehome the table to us-west-1.
exec-sql idx=0
ALTER TABLE db.rbt SET LOCALITY REGIONAL BY TABLE IN "us-west-1";
----

wait-for-zone-config-changes db-name=db table-name=rbt num-voters=3 num-non-voters=3 leaseholder=3 voter=4,5
----

refresh-range-descriptor-cache idx=3 table-name=rbt
SELECT * FROM db.rbt WHERE k = 2
----
LAG_BY_CLUSTER_SETTING

# Reads from us-west-1 now should be local since we're homed in us-west-1.
trace-sql idx=3
SELECT * FROM db.rbt WHERE k = 1
----
served locally: true
served via follower read: false

# Reads from us-east-1 now should be remote since we're homed in us-west-1.
refresh-range-descriptor-cache idx=0 table-name=rbt
SELECT * FROM db.rbt WHERE k = 2
----
LAG_BY_CLUSTER_SETTING

trace-sql idx=0
SELECT * FROM db.rbt WHERE k = 1
----
served locally: false

# If we drop us-east-1, replica placement will become deterministic, so we will
# be able to do follower reads from predictable replicas.
# Replica placement becomes deterministic because we have exactly 3 replicas in
# the primary region for the 3 voters and exactly one replica in all non-primary
# regions for our nonvoters. If we had multiple replicas in any non-primary
# region, we wouldn't be able to deterministically say which of those is a
# non-voter.
# This might be overkill to test, but it's worth testing that we retain our
# non-voter placement even on primary region change.
exec-sql idx=0
ALTER DATABASE db SET PRIMARY REGION "us-west-1"
----

exec-sql idx=0
ALTER DATABASE db DROP REGION "us-east-1"
----

exec-sql idx=0
ALTER DATABASE db DROP REGION "us-central-1"
----

wait-for-zone-config-changes db-name=db table-name=rbt num-voters=3 num-non-voters=1 leaseholder=3 voter=4,5 non-voter=7 not-present=0,1,2,6,8
----

# Need to sleep after zone config changes in order for follower reads to work.
# Without sleep, follower reads make calls to the SystemConfig table for
# metadata on past data.
sleep-for-follower-read
----

refresh-range-descriptor-cache idx=6 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

# Cannot still serve from us=central-1
trace-sql idx=6
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: false

# Finally, we'll test out region survivability. We set the database to contain
# all regions except us-west-1 so that we can know deterministically where our
# voters will be.

exec-sql idx=0
ALTER DATABASE db ADD REGION "us-east-1"
----

exec-sql idx=0
ALTER DATABASE db ADD REGION "us-central-1"
----

exec-sql idx=0
ALTER DATABASE db ADD REGION "eu-west-1"
----

exec-sql idx=0
ALTER DATABASE db SET PRIMARY REGION "us-east-1"
----

exec-sql idx=0
ALTER TABLE db.rbt SET LOCALITY REGIONAL BY TABLE IN "us-east-1";
----

exec-sql idx=0
ALTER DATABASE db DROP REGION "us-west-1"
----

wait-for-zone-config-changes db-name=db table-name=rbt num-voters=3 num-non-voters=3 leaseholder=0 voter=1,2 non-voter=6,7,8 not-present=3,4,5
----

# We now modify to region survivability. This should bring us down to 5 replicas
# since we can't have 3 in us-east-1 without losing quorum in the case of
# failure in us-east1.

exec-sql idx=0
ALTER DATABASE db SURVIVE REGION FAILURE
----

# Note that we can't test voter placement in us-east-1 since 2/3 of the nodes
# have replicas, but we don't know which.
wait-for-zone-config-changes db-name=db table-name=rbt num-voters=5 num-non-voters=0 voter=6,7,8
----

# Need to sleep to test follower reads post-zone config change
sleep-for-follower-read
----

# Bulk refresh all caches.

refresh-range-descriptor-cache idx=6 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

refresh-range-descriptor-cache idx=7 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

refresh-range-descriptor-cache idx=8 table-name=rbt
SELECT * FROM db.rbt WHERE k = 1
----
LAG_BY_CLUSTER_SETTING

# Perform follower reads from all regions.

trace-sql idx=6
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: true
served via follower read: true

trace-sql idx=7
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: true
served via follower read: true

trace-sql idx=8
SELECT * FROM db.rbt AS OF SYSTEM TIME follower_read_timestamp() WHERE k = 1
----
served locally: true
served via follower read: true
