// Copyright (c) 2024 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3_test

import (
	"context"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/format"
	apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/projectcalico/calico/libcalico-go/lib/apiconfig"
	"github.com/projectcalico/calico/libcalico-go/lib/backend"
	"github.com/projectcalico/calico/libcalico-go/lib/clientv3"
	"github.com/projectcalico/calico/libcalico-go/lib/names"
	"github.com/projectcalico/calico/libcalico-go/lib/options"
	"github.com/projectcalico/calico/libcalico-go/lib/testutils"
	"github.com/projectcalico/calico/libcalico-go/lib/watch"
)

func init() {
	// Stop Gomega from chopping off diffs in logs.
	format.MaxLength = 0
}

var _ = testutils.E2eDatastoreDescribe("Tier tests", testutils.DatastoreAll, func(config apiconfig.CalicoAPIConfig) {
	ctx := context.Background()
	defaultOrder := apiv3.DefaultTierOrder
	order1 := 99.999
	order2 := 22.222
	actionDeny := apiv3.Deny
	actionPass := apiv3.Pass
	name1 := "t-1"
	name2 := "t-12"
	defaultName := "default"
	namespace1 := "namespace-1"
	spec1 := apiv3.TierSpec{
		Order:         &order1,
		DefaultAction: &actionDeny,
	}
	spec2 := apiv3.TierSpec{}
	spec2UpdatedOrder := apiv3.TierSpec{
		Order:         &defaultOrder,
		DefaultAction: &actionDeny,
	}
	defaultSpec := apiv3.TierSpec{
		Order:         &defaultOrder,
		DefaultAction: &actionDeny,
	}
	anpOrder := apiv3.AdminNetworkPolicyTierOrder
	anpSpec := apiv3.TierSpec{
		Order:         &anpOrder,
		DefaultAction: &actionPass,
	}

	banpOrder := apiv3.BaselineAdminNetworkPolicyTierOrder
	banpSpec := apiv3.TierSpec{
		Order:         &banpOrder,
		DefaultAction: &actionPass,
	}

	npName1 := name1 + ".networkp-1"
	npSpec1 := apiv3.NetworkPolicySpec{
		Tier:     name1,
		Order:    &order1,
		Ingress:  []apiv3.Rule{testutils.InRule1, testutils.InRule2},
		Egress:   []apiv3.Rule{testutils.EgressRule1, testutils.EgressRule2},
		Selector: "thing == 'value'",
	}

	gnpName1 := name1 + ".globalnetworkp-1"
	gnpSpec1 := apiv3.GlobalNetworkPolicySpec{
		Tier:           name1,
		Order:          &order2,
		Ingress:        []apiv3.Rule{testutils.InRule2, testutils.InRule1},
		Egress:         []apiv3.Rule{testutils.EgressRule2, testutils.EgressRule1},
		Selector:       "thing2 == 'value2'",
		DoNotTrack:     true,
		ApplyOnForward: true,
	}

	gnpName2 := name2 + ".globalnetworkp-1"
	gnpSpec2 := apiv3.GlobalNetworkPolicySpec{
		Tier:           name2,
		Order:          &order2,
		Ingress:        []apiv3.Rule{testutils.InRule2, testutils.InRule1},
		Egress:         []apiv3.Rule{testutils.EgressRule2, testutils.EgressRule1},
		Selector:       "thing2 == 'value2'",
		DoNotTrack:     true,
		ApplyOnForward: true,
	}

	DescribeTable("Tier e2e CRUD tests",
		func(
			name1, name2, npName1, gnpName1, gnpName2, namespace1 string,
			spec1, spec2 apiv3.TierSpec,
			npSpec1 apiv3.NetworkPolicySpec,
			gnpSpec1, gnpSpec2 apiv3.GlobalNetworkPolicySpec,
		) {
			c, err := clientv3.New(config)
			Expect(err).NotTo(HaveOccurred())

			be, err := backend.NewClient(config)
			Expect(err).NotTo(HaveOccurred())
			Expect(be.Clean()).NotTo(HaveOccurred())

			err = c.EnsureInitialized(ctx, "", "")
			Expect(err).NotTo(HaveOccurred())

			By("Creating a tier with nil order should result in a tier with the default order")
			res, outError := c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: "app-tier"},
				Spec:       apiv3.TierSpec{DefaultAction: &actionPass},
			}, options.SetOptions{})
			defaultOrder := apiv3.DefaultTierOrder
			Expect(outError).NotTo(HaveOccurred())
			Expect(res.Name).To(Equal("app-tier"))
			Expect(res.Spec.DefaultAction).To(Equal(&actionPass))
			Expect(res.Spec.Order).To(Equal(&defaultOrder))
			_, outError = c.Tiers().Delete(ctx, "app-tier", options.DeleteOptions{})
			Expect(outError).NotTo(HaveOccurred())

			By("Updating a tier order to nil should result in the default order")
			order := float64(10.0)
			res, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: "app-tier"},
				Spec: apiv3.TierSpec{
					Order:         &order,
					DefaultAction: &actionDeny,
				},
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res.Name).To(Equal("app-tier"))
			Expect(res.Spec.DefaultAction).To(Equal(&actionDeny))
			Expect(res.Spec.Order).To(Equal(&order))
			res, outError = c.Tiers().Update(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{
					Name:              "app-tier",
					ResourceVersion:   res.ResourceVersion,
					CreationTimestamp: res.CreationTimestamp,
					UID:               res.UID,
				},
				Spec: apiv3.TierSpec{DefaultAction: &actionPass},
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res.Name).To(Equal("app-tier"))
			Expect(res.Spec.DefaultAction).To(Equal(&actionPass))
			Expect(res.Spec.Order).To(Equal(&defaultOrder))
			_, outError = c.Tiers().Delete(ctx, "app-tier", options.DeleteOptions{})
			Expect(outError).NotTo(HaveOccurred())

			By("Creating the default tier with an invalid order")
			res, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: defaultName},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(res).To(BeNil())
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).Should(ContainSubstring("default tier order must be 1e+06"))

			By("Cannot delete the default Tier")
			_, outError = c.Tiers().Delete(ctx, defaultName, options.DeleteOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("operation Delete is not supported on default: Cannot delete default tier"))

			By("Getting default Tier")
			defRes, outError := c.Tiers().Get(ctx, defaultName, options.GetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(defRes).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, defaultName, defaultSpec))

			By("Cannot update the default Tier")
			defRes.Spec = spec1
			_, outError = c.Tiers().Update(ctx, defRes, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).Should(ContainSubstring("default tier order must be 1e+06"))

			By("Creating the adminnetworkpolicy tier with an invalid order")
			res, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: names.AdminNetworkPolicyTierName},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(res).To(BeNil())
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).Should(ContainSubstring("adminnetworkpolicy tier order must be 1000"))

			By("Cannot delete the adminnetworkpolicy Tier")
			_, outError = c.Tiers().Delete(ctx, names.AdminNetworkPolicyTierName, options.DeleteOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("operation Delete is not supported on adminnetworkpolicy: Cannot delete adminnetworkpolicy tier"))

			By("Getting adminnetworkpolicy Tier")
			defRes, outError = c.Tiers().Get(ctx, names.AdminNetworkPolicyTierName, options.GetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(defRes).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, names.AdminNetworkPolicyTierName, anpSpec))

			By("Cannot update the adminnetworkpolicy Tier")
			defRes.Spec = spec1
			_, outError = c.Tiers().Update(ctx, defRes, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).Should(ContainSubstring("adminnetworkpolicy tier order must be 1000"))

			By("Updating the Tier before it is created")
			res, outError = c.Tiers().Update(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: name1, ResourceVersion: "1234", CreationTimestamp: metav1.Now(), UID: uid},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(res).To(BeNil())
			Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name1 + ") with error:"))

			By("Attempting to create a new Tier with too long of a name")
			longName := "thisisareallylongnamethatislongerthansixtythreecharactersthatwillnotfitinalabel"
			res, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: longName},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(res).To(BeNil())
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("error with field Metadata.Name = 'thisisareallylongnamethatislongerthansixtythreecharactersthatwillnotfitinalabel' (name is too long by 16 bytes)"))

			By("Attempting to create a new Tier with name1/spec1 and a non-empty ResourceVersion")
			res, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: name1, ResourceVersion: "12345"},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(res).To(BeNil())
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("error with field Metadata.ResourceVersion = '12345' (field must not be set for a Create request)"))

			By("Creating a new Tier with name1/spec1")
			res1, outError := c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: name1},
				Spec:       spec1,
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res1).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))

			// Track the version of the original data for name1.
			rv1_1 := res1.ResourceVersion

			By("Attempting to create the same Tier with name1 but with spec2")
			_, outError = c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: name1},
				Spec:       spec2,
			}, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("resource already exists: Tier(" + name1 + ")"))

			By("Getting Tier (name1) and comparing the output against spec1")
			res, outError = c.Tiers().Get(ctx, name1, options.GetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))
			Expect(res.ResourceVersion).To(Equal(res1.ResourceVersion))

			By("Getting Tier (name2) before it is created")
			_, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name2 + ") with error:"))

			By("Listing all the Tiers, expecting a single result with name1/spec1 plus our default tiers")
			checkAndFilterDefaultTiers := func(outList *apiv3.TierList) []apiv3.Tier {
				// Tiers are returned in name order, and the default ones happen
				// to sort first in these tests.
				Expect(&outList.Items[0]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, names.AdminNetworkPolicyTierName, anpSpec))
				Expect(&outList.Items[1]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, names.BaselineAdminNetworkPolicyTierName, banpSpec))
				Expect(&outList.Items[2]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, defaultName, defaultSpec))
				return outList.Items[3:]
			}
			outList, outError := c.Tiers().List(ctx, options.ListOptions{})
			Expect(outError).NotTo(HaveOccurred())
			nonDefaultTiers := checkAndFilterDefaultTiers(outList)
			Expect(nonDefaultTiers).To(HaveLen(1))
			Expect(&nonDefaultTiers[0]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))

			By("Creating a new Tier with name2/spec2")
			res2, outError := c.Tiers().Create(ctx, &apiv3.Tier{
				ObjectMeta: metav1.ObjectMeta{Name: name2},
				Spec:       spec2,
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res2).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name2, spec2UpdatedOrder))

			By("Getting Tier (name2) and comparing the output against spec2")
			res, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res2).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name2, spec2UpdatedOrder))
			Expect(res.ResourceVersion).To(Equal(res2.ResourceVersion))

			By("Listing all the Tiers, expecting a two results with name1/spec1 and name2/spec2")
			outList, outError = c.Tiers().List(ctx, options.ListOptions{})
			Expect(outError).NotTo(HaveOccurred())
			nonDefaultTiers = checkAndFilterDefaultTiers(outList)
			Expect(nonDefaultTiers).To(HaveLen(2))
			Expect(nonDefaultTiers[0]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))
			Expect(nonDefaultTiers[1]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name2, spec2UpdatedOrder))

			By("Updating Tier name1 with spec2")
			res1.Spec = spec2
			res1, outError = c.Tiers().Update(ctx, res1, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res1).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec2UpdatedOrder))

			// Track the version of the updated name1 data.
			rv1_2 := res1.ResourceVersion

			By("Updating Tier name1 without specifying a resource version")
			res1.Spec = spec1
			res1.ObjectMeta.ResourceVersion = ""
			_, outError = c.Tiers().Update(ctx, res1, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("error with field Metadata.ResourceVersion = '' (field must be set for an Update request)"))

			By("Updating Tier name1 using the previous resource version")
			res1.Spec = spec1
			res1.ResourceVersion = rv1_1
			_, outError = c.Tiers().Update(ctx, res1, options.SetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(Equal("update conflict: Tier(" + name1 + ")"))

			if config.Spec.DatastoreType != apiconfig.Kubernetes {
				By("Getting Tier (name1) with the original resource version and comparing the output against spec1")
				res, outError = c.Tiers().Get(ctx, name1, options.GetOptions{ResourceVersion: rv1_1})
				Expect(outError).NotTo(HaveOccurred())
				Expect(res).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))
				Expect(res.ResourceVersion).To(Equal(rv1_1))
			}

			By("Getting Tier (name1) with the updated resource version and comparing the output against spec2")
			res, outError = c.Tiers().Get(ctx, name1, options.GetOptions{ResourceVersion: rv1_2})
			Expect(outError).NotTo(HaveOccurred())
			Expect(res).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec2UpdatedOrder))
			Expect(res.ResourceVersion).To(Equal(rv1_2))

			if config.Spec.DatastoreType != apiconfig.Kubernetes {
				By("Listing Tiers with the original resource version and checking for a single result with name1/spec1")
				outList, outError = c.Tiers().List(ctx, options.ListOptions{ResourceVersion: rv1_1})
				nonDefaultTiers = checkAndFilterDefaultTiers(outList)
				Expect(outError).NotTo(HaveOccurred())
				Expect(nonDefaultTiers).To(HaveLen(1))
				Expect(&nonDefaultTiers[0]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec1))
			}

			By("Listing Tiers with the latest resource version and checking for two results with name1/spec2 and name2/spec2")
			outList, outError = c.Tiers().List(ctx, options.ListOptions{})
			nonDefaultTiers = checkAndFilterDefaultTiers(outList)
			Expect(outError).NotTo(HaveOccurred())
			Expect(nonDefaultTiers).To(HaveLen(2))
			Expect(nonDefaultTiers[0]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec2UpdatedOrder))
			Expect(nonDefaultTiers[1]).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name2, spec2UpdatedOrder))

			if config.Spec.DatastoreType != apiconfig.Kubernetes {
				By("Deleting Tier (name1) with the old resource version")
				_, outError = c.Tiers().Delete(ctx, name1, options.DeleteOptions{ResourceVersion: rv1_1})
				Expect(outError).To(HaveOccurred())
				Expect(outError.Error()).To(Equal("update conflict: Tier(" + name1 + ")"))
			}

			By("Attempting to delete Tier (name1) when there is a NetworkPolicy in it")
			np1, outError := c.NetworkPolicies().Create(ctx, &apiv3.NetworkPolicy{
				ObjectMeta: metav1.ObjectMeta{Name: npName1, Namespace: namespace1},
				Spec:       npSpec1,
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			npSpec1.Types = []apiv3.PolicyType{apiv3.PolicyTypeIngress, apiv3.PolicyTypeEgress}
			Expect(np1).To(MatchResource(apiv3.KindNetworkPolicy, namespace1, npName1, npSpec1))
			rv_np := np1.ResourceVersion

			dres, outError := c.Tiers().Delete(ctx, name1, options.DeleteOptions{ResourceVersion: rv1_2})
			Expect(outError).To(HaveOccurred())

			dnp, outError := c.NetworkPolicies().Delete(ctx, namespace1, npName1, options.DeleteOptions{ResourceVersion: rv_np})
			Expect(outError).NotTo(HaveOccurred())
			Expect(dnp).To(MatchResource(apiv3.KindNetworkPolicy, namespace1, npName1, npSpec1))

			By("Attempting to delete Tier (name1) when there is a GlobalNetworkPolicy in it")
			gnp1, outError := c.GlobalNetworkPolicies().Create(ctx, &apiv3.GlobalNetworkPolicy{
				ObjectMeta: metav1.ObjectMeta{Name: gnpName1},
				Spec:       gnpSpec1,
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			gnpSpec1.Types = []apiv3.PolicyType{apiv3.PolicyTypeIngress, apiv3.PolicyTypeEgress}
			Expect(gnp1).To(MatchResource(apiv3.KindGlobalNetworkPolicy, testutils.ExpectNoNamespace, gnpName1, gnpSpec1))
			rv_gnp := gnp1.ResourceVersion

			dres, outError = c.Tiers().Delete(ctx, name1, options.DeleteOptions{ResourceVersion: rv1_2})
			Expect(outError).To(HaveOccurred())

			dgnp, outError := c.GlobalNetworkPolicies().Delete(ctx, gnpName1, options.DeleteOptions{ResourceVersion: rv_gnp})
			Expect(outError).NotTo(HaveOccurred())
			Expect(dgnp).To(MatchResource(apiv3.KindGlobalNetworkPolicy, testutils.ExpectNoNamespace, gnpName1, gnpSpec1))
			time.Sleep(1 * time.Second)

			By("Deleting Tier (name1) with the new resource version")
			// Make sure that policies on other tiers are not associated with other tiers
			gnp2, outError := c.GlobalNetworkPolicies().Create(ctx, &apiv3.GlobalNetworkPolicy{
				ObjectMeta: metav1.ObjectMeta{Name: gnpName2},
				Spec:       gnpSpec2,
			}, options.SetOptions{})
			Expect(outError).NotTo(HaveOccurred())
			gnpSpec2.Types = []apiv3.PolicyType{apiv3.PolicyTypeIngress, apiv3.PolicyTypeEgress}
			Expect(gnp2).To(MatchResource(apiv3.KindGlobalNetworkPolicy, testutils.ExpectNoNamespace, gnpName2, gnpSpec2))
			rv_gnp2 := gnp2.ResourceVersion

			dres, outError = c.Tiers().Delete(ctx, name1, options.DeleteOptions{ResourceVersion: rv1_2})
			Expect(outError).NotTo(HaveOccurred())
			Expect(dres).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name1, spec2UpdatedOrder))

			// Delete policy on the other tier
			dgnp, outError = c.GlobalNetworkPolicies().Delete(ctx, gnpName2, options.DeleteOptions{ResourceVersion: rv_gnp2})
			Expect(outError).NotTo(HaveOccurred())
			Expect(dgnp).To(MatchResource(apiv3.KindGlobalNetworkPolicy, testutils.ExpectNoNamespace, gnpName2, gnpSpec2))
			time.Sleep(1 * time.Second)

			if config.Spec.DatastoreType != apiconfig.Kubernetes {
				By("Updating Tier name2 with a 2s TTL and waiting for the entry to be deleted")
				_, outError = c.Tiers().Update(ctx, res2, options.SetOptions{TTL: 2 * time.Second})
				Expect(outError).NotTo(HaveOccurred())
				time.Sleep(1 * time.Second)
				_, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
				Expect(outError).NotTo(HaveOccurred())
				time.Sleep(2 * time.Second)
				_, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
				Expect(outError).To(HaveOccurred())
				Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name2 + ") with error:"))

				By("Creating Tier name2 with a 2s TTL and waiting for the entry to be deleted")
				_, outError = c.Tiers().Create(ctx, &apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: name2},
					Spec:       spec2,
				}, options.SetOptions{TTL: 2 * time.Second})
				Expect(outError).NotTo(HaveOccurred())
				time.Sleep(1 * time.Second)
				_, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
				Expect(outError).NotTo(HaveOccurred())
				time.Sleep(2 * time.Second)
				_, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
				Expect(outError).To(HaveOccurred())
				Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name2 + ") with error:"))
			}

			if config.Spec.DatastoreType == apiconfig.Kubernetes {
				By("Deleting Tier (name2)")
				dres, outError = c.Tiers().Delete(ctx, name2, options.DeleteOptions{})
				Expect(outError).NotTo(HaveOccurred())
				Expect(dres).To(MatchResource(apiv3.KindTier, testutils.ExpectNoNamespace, name2, spec2UpdatedOrder))
			}

			By("Attempting to deleting Tier (name2) again")
			_, outError = c.Tiers().Delete(ctx, name2, options.DeleteOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name2 + ") with error:"))

			By("Listing all Tiers and expecting only the default and adminnetworkpolicy tiers")
			outList, outError = c.Tiers().List(ctx, options.ListOptions{})
			Expect(outError).NotTo(HaveOccurred())
			nonDefaultTiers = checkAndFilterDefaultTiers(outList)
			Expect(nonDefaultTiers).To(HaveLen(0))

			By("Getting Tier (name2) and expecting an error")
			res, outError = c.Tiers().Get(ctx, name2, options.GetOptions{})
			Expect(outError).To(HaveOccurred())
			Expect(outError.Error()).To(ContainSubstring("resource does not exist: Tier(" + name2 + ") with error:"))
		},

		// Test 1: Pass two fully populated TierSpecs and expect the series of operations to succeed.
		Entry("Two fully populated TierSpecs", name1, name2, npName1, gnpName1, gnpName2, namespace1, spec1, spec2, npSpec1, gnpSpec1, gnpSpec2),
	)

	Describe("Tier watch functionality", func() {
		It("should handle watch events for different resource versions and event types", func() {
			c, err := clientv3.New(config)
			Expect(err).NotTo(HaveOccurred())

			be, err := backend.NewClient(config)
			Expect(err).NotTo(HaveOccurred())
			Expect(be.Clean()).NotTo(HaveOccurred())

			By("Listing Tiers with the latest resource version and checking for two results with name1/spec2 and name2/spec2")
			outList, outError := c.Tiers().List(ctx, options.ListOptions{})
			Expect(outError).NotTo(HaveOccurred())
			Expect(outList.Items).To(HaveLen(0))
			rev0 := outList.ResourceVersion

			By("Configuring a Tier name1/spec1 and storing the response")
			outRes1, err := c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: name1},
					Spec:       spec1,
				},
				options.SetOptions{},
			)
			rev1 := outRes1.ResourceVersion

			By("Configuring a Tier name2/spec2 and storing the response")
			outRes2, err := c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: name2},
					Spec:       spec2,
				},
				options.SetOptions{},
			)

			By("Starting a watcher from revision rev1 - this should skip the first creation")
			w, err := c.Tiers().Watch(ctx, options.ListOptions{ResourceVersion: rev1})
			Expect(err).NotTo(HaveOccurred())
			testWatcher1 := testutils.NewTestResourceWatch(config.Spec.DatastoreType, w)
			defer testWatcher1.Stop()

			By("Deleting res1")
			_, err = c.Tiers().Delete(ctx, name1, options.DeleteOptions{})
			Expect(err).NotTo(HaveOccurred())
			time.Sleep(1 * time.Second)

			By("Checking for two events, create res2 and delete re1")
			testWatcher1.ExpectEvents(apiv3.KindTier, []watch.Event{
				{
					Type:   watch.Added,
					Object: outRes2,
				},
				{
					Type:     watch.Deleted,
					Previous: outRes1,
				},
			})
			testWatcher1.Stop()

			By("Starting a watcher from rev0 - this should get all events")
			w, err = c.Tiers().Watch(ctx, options.ListOptions{ResourceVersion: rev0})
			Expect(err).NotTo(HaveOccurred())
			testWatcher2 := testutils.NewTestResourceWatch(config.Spec.DatastoreType, w)
			defer testWatcher2.Stop()

			By("Modifying res2")
			outRes3, err := c.Tiers().Update(
				ctx,
				&apiv3.Tier{
					ObjectMeta: outRes2.ObjectMeta,
					Spec:       spec1,
				},
				options.SetOptions{},
			)
			Expect(err).NotTo(HaveOccurred())
			testWatcher2.ExpectEvents(apiv3.KindTier, []watch.Event{
				{
					Type:   watch.Added,
					Object: outRes1,
				},
				{
					Type:   watch.Added,
					Object: outRes2,
				},
				{
					Type:     watch.Deleted,
					Previous: outRes1,
				},
				{
					Type:     watch.Modified,
					Previous: outRes2,
					Object:   outRes3,
				},
			})
			testWatcher2.Stop()

			// Only etcdv3 supports watching a specific instance of a resource.
			if config.Spec.DatastoreType == apiconfig.EtcdV3 {
				By("Starting a watcher from rev0 watching name1 - this should get all events for name1")
				w, err = c.Tiers().Watch(ctx, options.ListOptions{Name: name1, ResourceVersion: rev0})
				Expect(err).NotTo(HaveOccurred())
				testWatcher2_1 := testutils.NewTestResourceWatch(config.Spec.DatastoreType, w)
				defer testWatcher2_1.Stop()
				testWatcher2_1.ExpectEvents(apiv3.KindTier, []watch.Event{
					{
						Type:   watch.Added,
						Object: outRes1,
					},
					{
						Type:     watch.Deleted,
						Previous: outRes1,
					},
				})
				testWatcher2_1.Stop()
			}

			By("Starting a watcher not specifying a rev - expect the current snapshot")
			w, err = c.Tiers().Watch(ctx, options.ListOptions{})
			Expect(err).NotTo(HaveOccurred())
			testWatcher3 := testutils.NewTestResourceWatch(config.Spec.DatastoreType, w)
			defer testWatcher3.Stop()
			testWatcher3.ExpectEvents(apiv3.KindTier, []watch.Event{
				{
					Type:   watch.Added,
					Object: outRes3,
				},
			})
			testWatcher3.Stop()

			By("Configuring Tier name1/spec1 again and storing the response")
			outRes1, err = c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: name1},
					Spec:       spec1,
				},
				options.SetOptions{},
			)

			By("Starting a watcher not specifying a rev - expect the current snapshot")
			w, err = c.Tiers().Watch(ctx, options.ListOptions{})
			Expect(err).NotTo(HaveOccurred())
			testWatcher4 := testutils.NewTestResourceWatch(config.Spec.DatastoreType, w)
			defer testWatcher4.Stop()
			testWatcher4.ExpectEventsAnyOrder(apiv3.KindTier, []watch.Event{
				{
					Type:   watch.Added,
					Object: outRes1,
				},
				{
					Type:   watch.Added,
					Object: outRes3,
				},
			})

			By("Cleaning the datastore and expecting deletion events for each configured resource (tests prefix deletes results in individual events for each key)")
			Expect(be.Clean()).NotTo(HaveOccurred())
			testWatcher4.ExpectEvents(apiv3.KindTier, []watch.Event{
				{
					Type:     watch.Deleted,
					Previous: outRes1,
				},
				{
					Type:     watch.Deleted,
					Previous: outRes3,
				},
			})
			testWatcher4.Stop()
		})
	})

	Describe("Tier delete functionality", func() {
		var c clientv3.Interface
		var err error

		BeforeEach(func() {
			c, err = clientv3.New(config)
			Expect(err).NotTo(HaveOccurred())

			be, err := backend.NewClient(config)
			Expect(err).NotTo(HaveOccurred())
			Expect(be.Clean()).NotTo(HaveOccurred())

			// These tests use the default tier, so make sure it's created.
			err = c.EnsureInitialized(ctx, "", "")
			Expect(err).NotTo(HaveOccurred())
		})

		It("cannot delete tier `tier-1` if there is a GlobalNetworkPolicy configured in it", func() {
			By("Configuring a Tier called tier-1")
			tier, err := c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: "tier-1"},
					Spec:       spec1,
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Configuring a GlobalNetworkPolicy in tier-1")
			gnp, err := c.GlobalNetworkPolicies().Create(
				ctx,
				&apiv3.GlobalNetworkPolicy{
					ObjectMeta: metav1.ObjectMeta{Name: "tier-1.gnp1"},
					Spec: apiv3.GlobalNetworkPolicySpec{
						Tier:     "tier-1",
						Order:    &order2,
						Ingress:  []apiv3.Rule{},
						Egress:   []apiv3.Rule{},
						Selector: "thing2 == 'value2'",
					},
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting failure)")
			_, err = c.Tiers().Delete(ctx, "tier-1", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).To(HaveOccurred())

			By("Deleting the GNP")
			_, err = c.GlobalNetworkPolicies().Delete(
				ctx, "tier-1.gnp1", options.DeleteOptions{ResourceVersion: gnp.ResourceVersion},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting success)")
			_, err = c.Tiers().Delete(ctx, "tier-1", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).NotTo(HaveOccurred())
		})

		It("cannot delete tier `tier-1` if there is a NetworkPolicy configured in it", func() {
			By("Configuring a Tier called tier-1")
			tier, err := c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: "tier-1"},
					Spec:       spec1,
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Configuring a NetworkPolicy in tier-1")
			np, err := c.NetworkPolicies().Create(
				ctx,
				&apiv3.NetworkPolicy{
					ObjectMeta: metav1.ObjectMeta{Name: "tier-1.np1", Namespace: "namespace-1"},
					Spec: apiv3.NetworkPolicySpec{
						Tier:     "tier-1",
						Order:    &order2,
						Ingress:  []apiv3.Rule{},
						Egress:   []apiv3.Rule{},
						Selector: "thing2 == 'value2'",
					},
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting failure)")
			_, err = c.Tiers().Delete(ctx, "tier-1", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).To(HaveOccurred())

			By("Deleting the NP")
			_, err = c.NetworkPolicies().Delete(
				ctx, "namespace-1", "tier-1.np1", options.DeleteOptions{ResourceVersion: np.ResourceVersion},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting success)")
			_, err = c.Tiers().Delete(ctx, "tier-1", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).NotTo(HaveOccurred())
		})

		if config.Spec.DatastoreType != apiconfig.Kubernetes {
			// The issue with knp tier not being deletable was due to the prefix based enumeration of policies in etcdv3.
			// For kubernetes, we use label matching instead - and so it isn't an issue. We skip this test for kubernetes
			// because it isn'tt possible to create pretend "k8s" policies through the Calico API.
			It("can delete tier `knp` if there is a k8s-backed NetworkPolicy configured", func() {
				By("Configuring a Tier called knp")
				tier, err := c.Tiers().Create(
					ctx,
					&apiv3.Tier{
						ObjectMeta: metav1.ObjectMeta{Name: "knp"},
						Spec:       spec1,
					},
					options.SetOptions{},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Configuring a k8s-backed NetworkPolicy called knp.default.test")
				np, err := c.NetworkPolicies().Create(
					ctx,
					&apiv3.NetworkPolicy{
						ObjectMeta: metav1.ObjectMeta{Name: "knp.default.np", Namespace: "namespace-1"},
						Spec: apiv3.NetworkPolicySpec{
							Tier:     "default",
							Order:    &order2,
							Ingress:  []apiv3.Rule{},
							Egress:   []apiv3.Rule{},
							Selector: "thing2 == 'value2'",
						},
					},
					options.SetOptions{},
				)
				Expect(err).ToNot(HaveOccurred())

				By("Attempting to delete tier-1 (expecting success)")
				_, err = c.Tiers().Delete(ctx, "knp", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
				Expect(err).ToNot(HaveOccurred())

				By("Deleting the NP")
				_, err = c.NetworkPolicies().Delete(
					ctx, "namespace-1", "knp.default.np", options.DeleteOptions{ResourceVersion: np.ResourceVersion},
				)
				Expect(err).ToNot(HaveOccurred())
			})
		}

		It("cannot delete tier `knp` if there is a NetworkPolicy configured in it", func() {
			c, err := clientv3.New(config)
			Expect(err).NotTo(HaveOccurred())

			be, err := backend.NewClient(config)
			Expect(err).NotTo(HaveOccurred())
			Expect(be.Clean()).NotTo(HaveOccurred())

			By("Configuring a Tier called knp")
			tier, err := c.Tiers().Create(
				ctx,
				&apiv3.Tier{
					ObjectMeta: metav1.ObjectMeta{Name: "knp"},
					Spec:       spec1,
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Configuring a NetworkPolicy called knp.default.test")
			np, err := c.NetworkPolicies().Create(
				ctx,
				&apiv3.NetworkPolicy{
					ObjectMeta: metav1.ObjectMeta{Name: "knp.np1", Namespace: "namespace-1"},
					Spec: apiv3.NetworkPolicySpec{
						Tier:     "knp",
						Order:    &order2,
						Ingress:  []apiv3.Rule{},
						Egress:   []apiv3.Rule{},
						Selector: "thing2 == 'value2'",
					},
				},
				options.SetOptions{},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting failure)")
			_, err = c.Tiers().Delete(ctx, "knp", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).To(HaveOccurred())

			By("Deleting the NP")
			_, err = c.NetworkPolicies().Delete(
				ctx, "namespace-1", "knp.np1", options.DeleteOptions{ResourceVersion: np.ResourceVersion},
			)
			Expect(err).ToNot(HaveOccurred())

			By("Attempting to delete tier-1 (expecting success)")
			_, err = c.Tiers().Delete(ctx, "knp", options.DeleteOptions{ResourceVersion: tier.ResourceVersion})
			Expect(err).ToNot(HaveOccurred())
		})
	})
})
