Skip to content

Commit 9b329d7

Browse files
nrevannanrevanna
authored
PWX-28882: Fix Nomad support issues for vSphere provider (#137)
* PWX-28882: Fix Nomad support issues for vSphere provider Configmap apis were used in vSphere provider which stopped px from coming up on Nomad. The following changes are made - Store interface moved to cloudops - DriveSetStore interface created with Store as composite. Contains interfaces with DriveSet datastructure. - GetLockTryDuration introduced - Renamed GetLockTimeout/SetLockTimeout to GetLockHoldTimeout/SetLockHoldTimeout. - Implemented IsKeyLocked() and LockWithKey() interfaces in kv_store - Removed noOpStore implementation. It was a stop gap solution for lack of implementation in kv_store Signed-off-by: Naveen Revanna <nrevanna@purestorage.com>
1 parent b4dd76b commit 9b329d7

27 files changed

Lines changed: 2938 additions & 8 deletions

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ require (
1111
github.com/IBM-Cloud/bluemix-go v0.0.0-20220329045155-d2a8118ac5c7
1212
github.com/aws/aws-sdk-go v1.40.39
1313
github.com/codeskyblue/go-sh v0.0.0-20170112005953-b097669b1569
14+
github.com/coreos/etcd v3.3.27+incompatible
1415
github.com/golang/mock v1.4.4
1516
github.com/google/uuid v1.1.2
1617
github.com/hashicorp/go-version v1.2.1
1718
github.com/libopenstorage/openstorage v8.0.1-0.20210603043922-faf638fed3e5+incompatible
1819
github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a
1920
github.com/oracle/oci-go-sdk/v65 v65.13.1
2021
github.com/pborman/uuid v1.2.0
22+
github.com/portworx/kvdb v0.0.0-20230326003017-21a38cf82d4b
2123
github.com/portworx/sched-ops v1.20.4-rc1.0.20211217234328-ead591c0f22d
2224
github.com/prometheus/client_golang v1.9.0
2325
github.com/sirupsen/logrus v1.8.1
@@ -74,6 +76,7 @@ require (
7476
github.com/prometheus/procfs v0.2.0 // indirect
7577
github.com/sony/gobreaker v0.5.0 // indirect
7678
github.com/spf13/pflag v1.0.5 // indirect
79+
github.com/stretchr/objx v0.2.0 // indirect
7780
go.opencensus.io v0.22.4 // indirect
7881
golang.org/x/crypto v0.6.0 // indirect
7982
golang.org/x/net v0.6.0 // indirect

‎go.sum‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,8 @@ github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
317317
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
318318
github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
319319
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
320+
github.com/coreos/etcd v3.3.27+incompatible h1:QIudLb9KeBsE5zyYxd1mjzRSkzLg9Wf9QlRwFgd6oTA=
321+
github.com/coreos/etcd v3.3.27+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
320322
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
321323
github.com/coreos/go-oidc v2.0.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
322324
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@@ -1223,7 +1225,12 @@ github.com/portworx/kvdb v0.0.0-20190105022415-cccaa09abfc9/go.mod h1:Q8YyrNDvPp
12231225
github.com/portworx/kvdb v0.0.0-20191223203141-f42097b1fcd8/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
12241226
github.com/portworx/kvdb v0.0.0-20200311180812-b2c72382d652/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
12251227
github.com/portworx/kvdb v0.0.0-20200723230726-2734b7f40194/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
1228+
github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467 h1:jkqzdbOnejgSN5HG/FLt4enNrozWT/K+nlmaRm3P1II=
12261229
github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
1230+
github.com/portworx/kvdb v0.0.0-20221027170939-f1680e5db6db h1:ycu2xaymWoDkRgIUadkX1XjlKs0RWHtkk5HI/JjF59g=
1231+
github.com/portworx/kvdb v0.0.0-20221027170939-f1680e5db6db/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
1232+
github.com/portworx/kvdb v0.0.0-20230326003017-21a38cf82d4b h1:nP+m9tYQv2cWbN9wAwjlhJU9FZRs7GaYszhLJBLAg/I=
1233+
github.com/portworx/kvdb v0.0.0-20230326003017-21a38cf82d4b/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M=
12271234
github.com/portworx/px-backup-api v1.0.1-0.20200915150042-274508e876ef/go.mod h1:puy7YVXeb6glot1vVCIePIiRLSwB//+rFtN2ZjvXeEw=
12281235
github.com/portworx/pxc v0.33.0/go.mod h1:Tl7hf4K2CDr0XtxzM08sr9H/KsMhscjf9ydb+MnT0U4=
12291236
github.com/portworx/sched-ops v0.0.0-20200123020607-b0799c4686f5/go.mod h1:yb1ypNIiZQAmM7xAWGzO6dydwl/+vNC0WjUm5IvHUEY=

‎store/store.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package store
2+
3+
import (
4+
"fmt"
5+
"github.com/portworx/kvdb"
6+
"time"
7+
)
8+
9+
// PX specific scheduler constants
10+
const (
11+
// Kubernetes identifies kubernetes as the scheduler
12+
Kubernetes = "kubernetes"
13+
)
14+
15+
// Params is the parameters to use for the Store object
16+
type Params struct {
17+
// Kv is the bootstrap kvdb instance
18+
Kv kvdb.Kvdb
19+
// InternalKvdb indicates if PX is using internal kvdb or not
20+
InternalKvdb bool
21+
// SchedulerType indicates the platform pods are running on. e.g Kubernetes
22+
SchedulerType string
23+
}
24+
25+
// Lock identifies a lock taken over CloudDrive store
26+
type Lock struct {
27+
// Key is the name on which the lock is acquired.
28+
// This is used by the callers for logging purpose. Hence public
29+
Key string
30+
// Name of the owner who acquired the lock
31+
owner string
32+
// true if this lock was acquired using LockWithKey() interface
33+
lockedWithKey bool
34+
// lock structure as returned from the KVDB interface
35+
internalLock interface{}
36+
}
37+
38+
// KeyDoesNotExist is error type when the key does not exist
39+
type KeyDoesNotExist struct {
40+
Key string
41+
}
42+
43+
func (e *KeyDoesNotExist) Error() string {
44+
return fmt.Sprintf("key %s does not exist", e.Key)
45+
}
46+
47+
// KeyExists is error type when the key already exist in store
48+
type KeyExists struct {
49+
// Key that exists
50+
Key string
51+
// Message is an optional message to the user
52+
Message string
53+
}
54+
55+
func (e *KeyExists) Error() string {
56+
errMsg := fmt.Sprintf("key %s already exists in store", e.Key)
57+
if len(e.Message) > 0 {
58+
errMsg += " " + e.Message
59+
}
60+
return errMsg
61+
}
62+
63+
// Store provides a set of APIs to CloudDrive to store its metadata
64+
// in a persistent store
65+
type Store interface {
66+
// Lock locks the cloud drive store for a node to perform operations
67+
Lock(owner string) (*Lock, error)
68+
// Unlock unlocks the cloud drive store
69+
Unlock(storeLock *Lock) error
70+
// LockWithKey locks the cloud drive store with an arbitrary key
71+
LockWithKey(owner, key string) (*Lock, error)
72+
// IsKeyLocked checks if the specified key is currently locked
73+
IsKeyLocked(key string) (bool, string, error)
74+
// CreateKey creates the given key with the value
75+
CreateKey(key string, value []byte) error
76+
// PutKey updates the given key with the value
77+
PutKey(key string, value []byte) error
78+
// GetKey returns the value for the given key
79+
GetKey(key string) ([]byte, error)
80+
// DeleteKey deletes the given key
81+
DeleteKey(key string) error
82+
// EnumerateWithKeyPrefix enumerates all keys in the store that begin with the given key
83+
EnumerateWithKeyPrefix(key string) ([]string, error)
84+
}
85+
86+
// GetStoreWithParams returns instance for Store
87+
// kv: bootstrap kvdb
88+
// schedulerType: node scheduler type e.g Kubernetes
89+
// internalKvdb: If the cluster is configured to have internal kvdb
90+
// name: Name for the store
91+
// lockTryDuration: Total time to try acquiring the lock for
92+
// lockHoldTimeout: Once a lock is acquired, if it's held beyond this time, there will be panic
93+
func GetStoreWithParams(
94+
kv kvdb.Kvdb,
95+
schedulerType string,
96+
internalKvdb bool,
97+
name string,
98+
lockTryDuration time.Duration,
99+
lockHoldTimeout time.Duration,
100+
) (Store, error) {
101+
if len(name) == 0 {
102+
return nil, fmt.Errorf("name required to create Store")
103+
}
104+
var (
105+
s Store
106+
err error
107+
)
108+
109+
if internalKvdb && schedulerType == Kubernetes {
110+
s, _, err = newK8sStoreWithParams(name, lockTryDuration, lockHoldTimeout)
111+
} else if internalKvdb && kv == nil {
112+
return nil, fmt.Errorf("bootstrap kvdb cannot be empty")
113+
} else {
114+
// Two cases:
115+
// internal kvdb && kv is not nil
116+
// external kvdb
117+
if !internalKvdb {
118+
if kvdb.Instance() == nil {
119+
return nil, fmt.Errorf("kvdb is not initialized")
120+
}
121+
kv = kvdb.Instance()
122+
}
123+
s, err = newKVStoreWithParams(kv, name, lockTryDuration, lockHoldTimeout)
124+
}
125+
return s, err
126+
}

0 commit comments

Comments
 (0)