Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/opentracing/opentracing-go v1.2.0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/client_model v0.6.1
Expand Down
4 changes: 2 additions & 2 deletions client/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250 h1:6yUryXKVbKpCNdZWL58/OcZj8NPLUA/xsJYXSbsD59w=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821 h1:ZV/pD/TmTagPWnkmbm/br60HPKsh4Jc+hV3DZog6Pf0=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ require (
github.com/pingcap/errcode v0.3.0
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/pingcap/metering_sdk v0.0.0-20260814062708-9e3b68cd9adf
github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -490,8 +490,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250 h1:6yUryXKVbKpCNdZWL58/OcZj8NPLUA/xsJYXSbsD59w=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821 h1:ZV/pD/TmTagPWnkmbm/br60HPKsh4Jc+hV3DZog6Pf0=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down
17 changes: 17 additions & 0 deletions pkg/core/store_option.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,20 @@ func SetStoreDeployPath(deployPath string) StoreCreateOption {
}
}

// SetStoreTxnProtocolVersionRange sets the txn protocol version range for the store.
// The range reported by the last successful PutStore replaces the previous one entirely:
// a nil range clears the stored value, while a non-nil one (including an empty
// [0, 0] message) is copied to keep its presence.
func SetStoreTxnProtocolVersionRange(versionRange *metapb.TxnProtocolVersionRange) StoreCreateOption {
return func(store *StoreInfo) {
meta := typeutil.DeepClone(store.meta, StoreFactory)
meta.TxnProtocolVersionRange = typeutil.DeepClone(versionRange, func() *metapb.TxnProtocolVersionRange {
return &metapb.TxnProtocolVersionRange{}
})
store.meta = meta
}
}

// SetStoreState sets the state for the store.
func SetStoreState(state metapb.StoreState, physicallyDestroyed ...bool) StoreCreateOption {
return func(store *StoreInfo) {
Expand Down Expand Up @@ -333,6 +347,9 @@ func SetStoreMeta(newMeta *metapb.Store) StoreCreateOption {
meta.Labels = newMeta.GetLabels()
meta.NodeState = newMeta.GetNodeState()
meta.PhysicallyDestroyed = newMeta.GetPhysicallyDestroyed()
meta.TxnProtocolVersionRange = typeutil.DeepClone(newMeta.GetTxnProtocolVersionRange(), func() *metapb.TxnProtocolVersionRange {
return &metapb.TxnProtocolVersionRange{}
})
store.meta = meta
}
}
108 changes: 108 additions & 0 deletions pkg/core/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,114 @@ func TestCloneMetaStore(t *testing.T) {
re.NotEqual(store2.Labels, store.Labels)
}

// requireTxnProtocolVersionRange asserts the presence and the value of the txn
// protocol version range carried by the given store.
func requireTxnProtocolVersionRange(re *require.Assertions, store *StoreInfo, expected *metapb.TxnProtocolVersionRange) {
actual := store.GetMeta().GetTxnProtocolVersionRange()
if expected == nil {
re.Nil(actual)
return
}
re.NotNil(actual)
re.Equal(expected.GetMin(), actual.GetMin())
re.Equal(expected.GetMax(), actual.GetMax())
}

func TestStoreTxnProtocolVersionRange(t *testing.T) {
re := require.New(t)
meta := &metapb.Store{Id: 1, Address: "mock://tikv-1:1", Version: "6.5.0"}
store := NewStoreInfo(meta)
re.Nil(store.GetMeta().GetTxnProtocolVersionRange())

// Set, replace and lower the range by the dedicated option.
store = store.Clone(SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{Min: 0, Max: 2}))
requireTxnProtocolVersionRange(re, store, &metapb.TxnProtocolVersionRange{Min: 0, Max: 2})
store = store.Clone(SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{Min: 0, Max: 1}))
requireTxnProtocolVersionRange(re, store, &metapb.TxnProtocolVersionRange{Min: 0, Max: 1})

// An explicit empty range keeps its presence.
store = store.Clone(SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{}))
re.NotNil(store.GetMeta().GetTxnProtocolVersionRange())
requireTxnProtocolVersionRange(re, store, &metapb.TxnProtocolVersionRange{})

// A missing range clears the stored one.
store = store.Clone(SetStoreTxnProtocolVersionRange(nil))
re.Nil(store.GetMeta().GetTxnProtocolVersionRange())
}

func TestStoreTxnProtocolVersionRangePresenceAndSnapshotIsolation(t *testing.T) {
re := require.New(t)
store := NewStoreInfo(&metapb.Store{Id: 1, Address: "mock://tikv-1:1"})
re.Nil(store.GetMeta().GetTxnProtocolVersionRange())

// An explicit empty range survives the round trip of a full store clone.
emptyRange := store.Clone(SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{}))
re.NotNil(emptyRange.GetMeta().GetTxnProtocolVersionRange())
re.Zero(emptyRange.GetMeta().GetTxnProtocolVersionRange().GetMin())
re.Zero(emptyRange.GetMeta().GetTxnProtocolVersionRange().GetMax())
re.NotNil(typeutil.DeepClone(emptyRange.GetMeta(), StoreFactory).GetTxnProtocolVersionRange())

// Mutating the input range does not affect the generated snapshot.
input := &metapb.TxnProtocolVersionRange{Min: 0, Max: 1}
withRange := store.Clone(SetStoreTxnProtocolVersionRange(input))
requireTxnProtocolVersionRange(re, withRange, &metapb.TxnProtocolVersionRange{Min: 0, Max: 1})
input.Min, input.Max = 3, 4
requireTxnProtocolVersionRange(re, withRange, &metapb.TxnProtocolVersionRange{Min: 0, Max: 1})

// Applying the option does not modify the previous snapshot.
requireTxnProtocolVersionRange(re, store, nil)
cleared := withRange.Clone(SetStoreTxnProtocolVersionRange(nil))
re.Nil(cleared.GetMeta().GetTxnProtocolVersionRange())
requireTxnProtocolVersionRange(re, withRange, &metapb.TxnProtocolVersionRange{Min: 0, Max: 1})

// ShallowClone keeps the meta as is until the option is applied.
shallow := withRange.ShallowClone(SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{Min: 1, Max: 2}))
requireTxnProtocolVersionRange(re, shallow, &metapb.TxnProtocolVersionRange{Min: 1, Max: 2})
requireTxnProtocolVersionRange(re, withRange, &metapb.TxnProtocolVersionRange{Min: 0, Max: 1})

// The range keeps the same presence and value when it is written back to the cache.
storesInfo := NewStoresInfo()
storesInfo.PutStore(withRange)
opts := []StoreCreateOption{SetStoreTxnProtocolVersionRange(&metapb.TxnProtocolVersionRange{Min: 3, Max: 4})}
storesInfo.PutStore(withRange.Clone(opts...), opts...)
requireTxnProtocolVersionRange(re, storesInfo.GetStore(1), &metapb.TxnProtocolVersionRange{Min: 3, Max: 4})
}

func TestSetStoreMetaTxnProtocolVersionRange(t *testing.T) {
re := require.New(t)
newMeta := func(versionRange *metapb.TxnProtocolVersionRange) *metapb.Store {
return &metapb.Store{
Id: 1,
Address: "mock://tikv-1:2",
Version: "6.6.0",
DeployPath: "test/store1-new",
TxnProtocolVersionRange: versionRange,
}
}

store := NewStoreInfo(&metapb.Store{Id: 1, Address: "mock://tikv-1:1", Version: "6.5.0", NodeState: metapb.NodeState_Serving})
heartbeat := time.Now().Add(-time.Minute)
store = store.Clone(SetLastHeartbeatTS(heartbeat))

// SetStoreMeta copies the range reported by the new meta.
updated := store.Clone(SetStoreMeta(newMeta(&metapb.TxnProtocolVersionRange{Min: 0, Max: 2})))
requireTxnProtocolVersionRange(re, updated, &metapb.TxnProtocolVersionRange{Min: 0, Max: 2})
re.Equal("mock://tikv-1:2", updated.GetMeta().GetAddress())
re.Equal("test/store1-new", updated.GetMeta().GetDeployPath())

// An explicit empty range keeps its presence.
emptyRange := store.Clone(SetStoreMeta(newMeta(&metapb.TxnProtocolVersionRange{})))
re.NotNil(emptyRange.GetMeta().GetTxnProtocolVersionRange())
requireTxnProtocolVersionRange(re, emptyRange, &metapb.TxnProtocolVersionRange{})

// A missing range clears the stored one without touching LastHeartbeat nor
// the other fields which are not maintained by SetStoreMeta.
cleared := updated.Clone(SetStoreMeta(newMeta(nil)))
re.Nil(cleared.GetMeta().GetTxnProtocolVersionRange())
re.Equal(heartbeat.UnixNano(), cleared.GetMeta().GetLastHeartbeat())
re.Equal(updated.GetMeta().GetNodeState(), cleared.GetMeta().GetNodeState())
}

func BenchmarkStoreClone(b *testing.B) {
meta := &metapb.Store{Id: 1,
Address: "mock://tikv-1:1",
Expand Down
105 changes: 105 additions & 0 deletions pkg/mcs/router/server/meta/watcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,14 @@ import (
"go.uber.org/goleak"

"github.com/pingcap/failpoint"
"github.com/pingcap/kvproto/pkg/metapb"
"github.com/pingcap/kvproto/pkg/pdpb"

"github.com/tikv/pd/pkg/core"
"github.com/tikv/pd/pkg/storage"
"github.com/tikv/pd/pkg/utils/etcdutil"
"github.com/tikv/pd/pkg/utils/grpcutil"
"github.com/tikv/pd/pkg/utils/keypath"
"github.com/tikv/pd/pkg/utils/testutil"
)

Expand All @@ -47,3 +52,103 @@ func TestNewWatcherWaitLoadFailed(t *testing.T) {
re.Error(err)
re.Nil(watcher)
}

// getStoreFromCache queries the store through the helpers used by the router
// service, so that both the shared SetStoreMeta update path and the query path
// are covered.
func getStoreFromCache(re *require.Assertions, rc *core.BasicCluster, storeID uint64) *metapb.Store {
resp, err := grpcutil.GetStore(rc, &pdpb.GetStoreRequest{StoreId: storeID})
re.NoError(err)
re.NotNil(resp)
re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType())
re.Equal(storeID, resp.GetStore().GetId())
return resp.GetStore()
}

// getAllStoresFromCache queries all the stores through the helpers used by the
// router service.
func getAllStoresFromCache(re *require.Assertions, rc *core.BasicCluster) []*metapb.Store {
resp, err := grpcutil.GetAllStores(rc, &pdpb.GetAllStoresRequest{ExcludeTombstoneStores: true})
re.NoError(err)
re.NotNil(resp)
re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType())
return resp.GetStores()
}

func checkStoreTxnProtocolVersionRange(re *require.Assertions, rc *core.BasicCluster, expected *metapb.TxnProtocolVersionRange) {
check := func(store *metapb.Store) {
actual := store.GetTxnProtocolVersionRange()
if expected == nil {
re.Nil(actual)
return
}
re.NotNil(actual)
re.Equal(expected.GetMin(), actual.GetMin())
re.Equal(expected.GetMax(), actual.GetMax())
}
check(getStoreFromCache(re, rc, 1))
stores := getAllStoresFromCache(re, rc)
re.Len(stores, 1)
check(stores[0])
}

// TestStoreTxnProtocolVersionRange verifies that the router store watcher keeps
// the txn protocol version range of an already loaded store up to date.
func TestStoreTxnProtocolVersionRange(t *testing.T) {
re := require.New(t)
oldClusterID := keypath.ClusterID()
keypath.SetClusterID(1)
t.Cleanup(func() { keypath.SetClusterID(oldClusterID) })

_, client, clean := etcdutil.NewTestEtcdCluster(t, 1, nil)
defer clean()

storage := storage.NewStorageWithEtcdBackend(client)
newStoreMeta := func(versionRange *metapb.TxnProtocolVersionRange) *metapb.Store {
return &metapb.Store{
Id: 1,
Address: "mock://tikv-1:1",
Version: "6.5.0",
State: metapb.StoreState_Up,
NodeState: metapb.NodeState_Serving,
TxnProtocolVersionRange: versionRange,
}
}

// Seed the store before the watcher starts so that the initial load and the
// subsequent updates use different code paths.
re.NoError(storage.SaveStoreMeta(newStoreMeta(nil)))

rc := core.NewBasicCluster()
watcher, err := NewWatcher(context.Background(), client, rc)
re.NoError(err)
defer watcher.Close()
// NewWatcher waits for the initial load, so the seeded store must be visible
// as soon as it returns.
checkStoreTxnProtocolVersionRange(re, rc, nil)

// Update the same store while the watcher keeps running. The binary version
// and the other fields stay unchanged, so the range update can only come
// from the new option.
for _, expected := range []*metapb.TxnProtocolVersionRange{
{Min: 0, Max: 1},
{Min: 0, Max: 2},
{Min: 0, Max: 1},
{},
nil,
} {
re.NoError(storage.SaveStoreMeta(newStoreMeta(expected)))
testutil.Eventually(re, func() bool {
store := rc.GetStore(1)
if store == nil {
return false
}
actual := store.GetMeta().GetTxnProtocolVersionRange()
if expected == nil {
return actual == nil
}
return actual != nil && actual.GetMin() == expected.GetMin() && actual.GetMax() == expected.GetMax()
})
checkStoreTxnProtocolVersionRange(re, rc, expected)
}
}
3 changes: 2 additions & 1 deletion server/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -1573,7 +1573,8 @@ func (c *RaftCluster) putStoreImpl(store *metapb.Store, force bool) error {
core.SetStoreVersion(store.GitHash, store.Version),
core.SetStoreLabels(labels),
core.SetStoreStartTime(store.StartTimestamp),
core.SetStoreDeployPath(store.DeployPath))
core.SetStoreDeployPath(store.DeployPath),
core.SetStoreTxnProtocolVersionRange(store.TxnProtocolVersionRange))
// Update an existed store.
s = s.Clone(opts...)
}
Expand Down
13 changes: 13 additions & 0 deletions server/gc_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -806,6 +806,19 @@ func (s *GrpcServer) GetAllKeyspacesGCStates(ctx context.Context, request *pdpb.
}, nil
}

// WatchGCStates streams the GC states to the caller.
//
// TODO: This is a temporary placeholder, kept only to keep the build green while
// the kvproto dependency is replaced with the fork which carries the txn
// protocol version declaration. That fork is based on a commit which is newer
// than the current checkout and already contains the WatchGCStates API, while
// the real PD implementation (tikv/pd#10498) has not landed yet. Delete this
// placeholder and drop it from the reviewer's checklist once tikv/pd#10498 is
// merged and the kvproto replace directive is removed.
func (*GrpcServer) WatchGCStates(_ *pdpb.WatchGCStatesRequest, _ pdpb.PD_WatchGCStatesServer) error {
return status.Errorf(codes.Unimplemented, "WatchGCStates is not implemented yet")
}

// SetGlobalGCBarrier sets a global GC barrier.
func (s *GrpcServer) SetGlobalGCBarrier(ctx context.Context, request *pdpb.SetGlobalGCBarrierRequest) (*pdpb.SetGlobalGCBarrierResponse, error) {
done, err := s.rateLimitCheck()
Expand Down
2 changes: 1 addition & 1 deletion tests/integrations/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/golang/protobuf v1.5.4
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
github.com/prometheus/client_golang v1.20.5
github.com/prometheus/client_model v0.6.1
Expand Down
4 changes: 2 additions & 2 deletions tests/integrations/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -483,8 +483,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE=
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4=
github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250 h1:6yUryXKVbKpCNdZWL58/OcZj8NPLUA/xsJYXSbsD59w=
github.com/pingcap/kvproto v0.0.0-20260903054228-107095f1d250/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821 h1:ZV/pD/TmTagPWnkmbm/br60HPKsh4Jc+hV3DZog6Pf0=
github.com/pingcap/kvproto v0.0.0-20260918030129-d80b6c5e7821/go.mod h1:z6+aAHB7dBkA+LyinEX+48/ImRJ3jag0Hg0c7wkhEvE=
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
Expand Down
Loading
Loading