# Test that we are able to generate test plans for tests using a
# separate-process deployment mode.

mixed-version-test predecessors=(21.2.11, 22.1.8, 22.2.3, 23.1.4, 23.2.0) num_upgrades=5 deployment_mode=separate-process
----
ok

on-startup name=(create tables)
----
ok

in-mixed-version name=(mixed-version 1)
----
ok

in-mixed-version name=(mixed-version 2)
----
ok

workload name=bank
----
ok


after-upgrade-finalized name=(validate upgrade)
----
ok

plan debug=true
----
Seed:               12345
Upgrades:           v21.2.11 → v22.1.8 → v22.2.3 → v23.1.4 → v23.2.0 → <current>
Deployment mode:    separate-process
Plan:
├── install fixtures for version "v21.2.11" (1) [stage=system:system-setup;tenant:system-setup]
├── start cluster at version "v21.2.11" (2) [stage=system:system-setup;tenant:system-setup]
├── wait for all nodes (:1-4) to acknowledge cluster version '21.2' on system tenant (3) [stage=system:system-setup;tenant:system-setup]
├── upgrade cluster from "v21.2.11" to "v22.1.8"
│   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (4) [stage=system:init;tenant:upgrading-system]
│   ├── upgrade nodes :1-4 from "v21.2.11" to "v22.1.8"
│   │   ├── restart system server on node 4 with binary version v22.1.8 (5) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── restart system server on node 3 with binary version v22.1.8 (6) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── restart system server on node 1 with binary version v22.1.8 (7) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   └── restart system server on node 2 with binary version v22.1.8 (8) [stage=system:last-upgrade;tenant:upgrading-system]
│   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (9) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   └── wait for all nodes (:1-4) to acknowledge cluster version '22.1' on system tenant (10) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
├── upgrade cluster from "v22.1.8" to "v22.2.3"
│   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (11) [stage=system:init;tenant:upgrading-system]
│   ├── upgrade nodes :1-4 from "v22.1.8" to "v22.2.3"
│   │   ├── restart system server on node 4 with binary version v22.2.3 (12) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── restart system server on node 1 with binary version v22.2.3 (13) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── restart system server on node 2 with binary version v22.2.3 (14) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── restart system server on node 3 with binary version v22.2.3 (15) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   └── wait for 1m0s (16) [stage=system:last-upgrade;tenant:upgrading-system]
│   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (17) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   └── wait for all nodes (:1-4) to acknowledge cluster version '22.2' on system tenant (18) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
├── start separate process virtual cluster mixed-version-tenant-cyvju with binary version v22.2.3 (19) [stage=system:tenant-setup;tenant:tenant-setup]
├── wait for all nodes (:1-4) to acknowledge cluster version '22.2' on mixed-version-tenant-cyvju tenant (20) [stage=system:tenant-setup;tenant:tenant-setup]
├── set cluster setting "spanconfig.tenant_limit" to '50000' on mixed-version-tenant-cyvju tenant (21) [stage=system:tenant-setup;tenant:tenant-setup]
├── disable KV and tenant(SQL) rate limiter on mixed-version-tenant-cyvju tenant (22) [stage=system:tenant-setup;tenant:tenant-setup]
├── set cluster setting "server.secondary_tenants.authorization.mode" to 'allow-all' on system tenant (23) [stage=system:tenant-setup;tenant:tenant-setup]
├── delete all-tenants override for the `version` key (24) [stage=system:tenant-setup;tenant:tenant-setup]
├── run startup hooks concurrently
│   ├── run "create tables", after 3m0s delay (25) [stage=system:on-startup;tenant:on-startup]
│   └── run "initialize bank workload", after 3m0s delay (26) [stage=system:on-startup;tenant:on-startup]
├── run "bank workload" (27) [stage=system:background;tenant:background]
├── upgrade cluster from "v22.2.3" to "v23.1.4"
│   ├── upgrade storage cluster
│   │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (28) [stage=system:init;tenant:upgrading-system]
│   │   ├── upgrade nodes :1-4 from "v22.2.3" to "v23.1.4"
│   │   │   ├── restart system server on node 4 with binary version v23.1.4 (29) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── run "mixed-version 2" (30) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 1 with binary version v23.1.4 (31) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── run "mixed-version 1" (32) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 2 with binary version v23.1.4 (33) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   └── restart system server on node 3 with binary version v23.1.4 (34) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (35) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   ├── run "mixed-version 2" (36) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   ├── wait for all nodes (:1-4) to acknowledge cluster version '23.1' on system tenant (37) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   └── run "validate upgrade" (38) [stage=system:after-upgrade-finished;tenant:upgrading-system]
│   └── upgrade tenant
│      ├── upgrade nodes :1-4 from "v22.2.3" to "v23.1.4"
│      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version v23.1.4 (39) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version v23.1.4 (40) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version v23.1.4 (41) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      │   ├── run "mixed-version 2" (42) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version v23.1.4 (43) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      │   └── run "mixed-version 1" (44) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
│      ├── downgrade nodes :1-4 from "v23.1.4" to "v22.2.3"
│      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version v22.2.3 (45) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version v22.2.3 (46) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      │   ├── run "mixed-version 1" (47) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version v22.2.3 (48) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      │   ├── run "mixed-version 2" (49) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      │   └── restart mixed-version-tenant-cyvju server on node 4 with binary version v22.2.3 (50) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
│      ├── upgrade nodes :1-4 from "v22.2.3" to "v23.1.4"
│      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version v23.1.4 (51) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version v23.1.4 (52) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── run "mixed-version 1" (53) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version v23.1.4 (54) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version v23.1.4 (55) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   └── run "mixed-version 2" (56) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      ├── run following steps concurrently
│      │   ├── set `version` to '23.1' on mixed-version-tenant-cyvju tenant, after 0s delay (57) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      │   └── run "mixed-version 2", after 5s delay (58) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      ├── wait for all nodes (:1-4) to acknowledge cluster version '23.1' on mixed-version-tenant-cyvju tenant (59) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      └── run "validate upgrade" (60) [stage=system:upgrading-tenant;tenant:after-upgrade-finished]
├── upgrade cluster from "v23.1.4" to "v23.2.0"
│   ├── upgrade storage cluster
│   │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (61) [stage=system:init;tenant:upgrading-system]
│   │   ├── upgrade nodes :1-4 from "v23.1.4" to "v23.2.0"
│   │   │   ├── restart system server on node 1 with binary version v23.2.0 (62) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 4 with binary version v23.2.0 (63) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 2 with binary version v23.2.0 (64) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 3 with binary version v23.2.0 (65) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   │   └── run mixed-version hooks concurrently
│   │   │      ├── run "mixed-version 1", after 5s delay (66) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   │      └── run "mixed-version 2", after 5s delay (67) [stage=system:temporary-upgrade;tenant:upgrading-system]
│   │   ├── downgrade nodes :1-4 from "v23.2.0" to "v23.1.4"
│   │   │   ├── restart system server on node 3 with binary version v23.1.4 (68) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 2 with binary version v23.1.4 (69) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   │   ├── run mixed-version hooks concurrently
│   │   │   │   ├── run "mixed-version 1", after 0s delay (70) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   │   │   └── run "mixed-version 2", after 3m0s delay (71) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 1 with binary version v23.1.4 (72) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   │   └── restart system server on node 4 with binary version v23.1.4 (73) [stage=system:rollback-upgrade;tenant:upgrading-system]
│   │   ├── upgrade nodes :1-4 from "v23.1.4" to "v23.2.0"
│   │   │   ├── restart system server on node 2 with binary version v23.2.0 (74) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── run "mixed-version 1" (75) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 1 with binary version v23.2.0 (76) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── restart system server on node 3 with binary version v23.2.0 (77) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   ├── run "mixed-version 2" (78) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   │   └── restart system server on node 4 with binary version v23.2.0 (79) [stage=system:last-upgrade;tenant:upgrading-system]
│   │   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (80) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   ├── run mixed-version hooks concurrently
│   │   │   ├── run "mixed-version 1", after 500ms delay (81) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   │   └── run "mixed-version 2", after 500ms delay (82) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   ├── wait for all nodes (:1-4) to acknowledge cluster version '23.2' on system tenant (83) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
│   │   └── run "validate upgrade" (84) [stage=system:after-upgrade-finished;tenant:upgrading-system]
│   └── upgrade tenant
│      ├── upgrade nodes :1-4 from "v23.1.4" to "v23.2.0"
│      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version v23.2.0 (85) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── run "mixed-version 1" (86) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version v23.2.0 (87) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version v23.2.0 (88) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   ├── run "mixed-version 2" (89) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      │   └── restart mixed-version-tenant-cyvju server on node 1 with binary version v23.2.0 (90) [stage=system:upgrading-tenant;tenant:last-upgrade]
│      ├── run following steps concurrently
│      │   ├── set `version` to '23.2' on mixed-version-tenant-cyvju tenant, after 3m0s delay (91) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      │   └── run "mixed-version 1", after 5s delay (92) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      ├── wait for all nodes (:1-4) to acknowledge cluster version '23.2' on mixed-version-tenant-cyvju tenant (93) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
│      └── run "validate upgrade" (94) [stage=system:upgrading-tenant;tenant:after-upgrade-finished]
└── upgrade cluster from "v23.2.0" to "<current>"
   ├── upgrade storage cluster
   │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (95) [stage=system:init;tenant:upgrading-system]
   │   ├── upgrade nodes :1-4 from "v23.2.0" to "<current>"
   │   │   ├── restart system server on node 4 with binary version <current> (96) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   │   ├── run "mixed-version 1" (97) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 2 with binary version <current> (98) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   │   ├── run "mixed-version 2" (99) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 1 with binary version <current> (100) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   │   └── restart system server on node 3 with binary version <current> (101) [stage=system:temporary-upgrade;tenant:upgrading-system]
   │   ├── downgrade nodes :1-4 from "<current>" to "v23.2.0"
   │   │   ├── restart system server on node 1 with binary version v23.2.0 (102) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 4 with binary version v23.2.0 (103) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 3 with binary version v23.2.0 (104) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   │   ├── run mixed-version hooks concurrently
   │   │   │   ├── run "mixed-version 1", after 0s delay (105) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   │   │   └── run "mixed-version 2", after 500ms delay (106) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   │   └── restart system server on node 2 with binary version v23.2.0 (107) [stage=system:rollback-upgrade;tenant:upgrading-system]
   │   ├── upgrade nodes :1-4 from "v23.2.0" to "<current>"
   │   │   ├── restart system server on node 3 with binary version <current> (108) [stage=system:last-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 4 with binary version <current> (109) [stage=system:last-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 2 with binary version <current> (110) [stage=system:last-upgrade;tenant:upgrading-system]
   │   │   ├── run "mixed-version 1" (111) [stage=system:last-upgrade;tenant:upgrading-system]
   │   │   ├── restart system server on node 1 with binary version <current> (112) [stage=system:last-upgrade;tenant:upgrading-system]
   │   │   └── run "mixed-version 2" (113) [stage=system:last-upgrade;tenant:upgrading-system]
   │   ├── allow upgrade to happen on system tenant by resetting `preserve_downgrade_option` (114) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
   │   ├── wait for all nodes (:1-4) to acknowledge cluster version <current> on system tenant (115) [stage=system:running-upgrade-migrations;tenant:upgrading-system,finalizing]
   │   └── run "validate upgrade" (116) [stage=system:after-upgrade-finished;tenant:upgrading-system]
   └── upgrade tenant
      ├── upgrade nodes :1-4 from "v23.2.0" to "<current>"
      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version <current> (117) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version <current> (118) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version <current> (119) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version <current> (120) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      │   └── run mixed-version hooks concurrently
      │      ├── run "mixed-version 1", after 30s delay (121) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      │      └── run "mixed-version 2", after 100ms delay (122) [stage=system:upgrading-tenant;tenant:temporary-upgrade]
      ├── downgrade nodes :1-4 from "<current>" to "v23.2.0"
      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version v23.2.0 (123) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      │   ├── run "mixed-version 1" (124) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version v23.2.0 (125) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version v23.2.0 (126) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      │   ├── run "mixed-version 2" (127) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      │   └── restart mixed-version-tenant-cyvju server on node 3 with binary version v23.2.0 (128) [stage=system:upgrading-tenant;tenant:rollback-upgrade]
      ├── upgrade nodes :1-4 from "v23.2.0" to "<current>"
      │   ├── restart mixed-version-tenant-cyvju server on node 3 with binary version <current> (129) [stage=system:upgrading-tenant;tenant:last-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 2 with binary version <current> (130) [stage=system:upgrading-tenant;tenant:last-upgrade]
      │   ├── run "mixed-version 1" (131) [stage=system:upgrading-tenant;tenant:last-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 1 with binary version <current> (132) [stage=system:upgrading-tenant;tenant:last-upgrade]
      │   ├── restart mixed-version-tenant-cyvju server on node 4 with binary version <current> (133) [stage=system:upgrading-tenant;tenant:last-upgrade]
      │   └── run "mixed-version 2" (134) [stage=system:upgrading-tenant;tenant:last-upgrade]
      ├── run following steps concurrently
      │   ├── set `version` to <current> on mixed-version-tenant-cyvju tenant, after 0s delay (135) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
      │   ├── run "mixed-version 1", after 0s delay (136) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
      │   └── run "mixed-version 2", after 3m0s delay (137) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
      ├── wait for all nodes (:1-4) to acknowledge cluster version <current> on mixed-version-tenant-cyvju tenant (138) [stage=system:upgrading-tenant;tenant:running-upgrade-migrations,finalizing]
      └── run "validate upgrade" (139) [stage=system:upgrading-tenant;tenant:after-upgrade-finished]
