Skip to content

Commit 7c9c928

Browse files
kompotkotkompotkot
committed
Added write to file, export prefix and tests
1 parent 0f6e1c6 commit 7c9c928

7 files changed

Lines changed: 465 additions & 142 deletions

File tree

‎scripts/sources/parameters/cmd/aws_ssm.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go
3+
*/
4+
package parameters
5+
6+
import (
7+
"context"
8+
"log"
9+
10+
"github.com/aws/aws-sdk-go-v2/config"
11+
"github.com/aws/aws-sdk-go-v2/service/ssm"
12+
)
13+
14+
// SSMGetParametersAPI and SSMDescribeParametersAPI defines the interface
15+
// for the GetParameters and DescribeParameters function.
16+
// We use this interface to test the function using a mocked service
17+
type SSMGetParametersAPI interface {
18+
GetParameters(
19+
ctx context.Context,
20+
params *ssm.GetParametersInput,
21+
optFns ...func(*ssm.Options),
22+
) (*ssm.GetParametersOutput, error)
23+
}
24+
25+
type SSMDescribeParametersAPI interface {
26+
DescribeParameters(
27+
ctx context.Context,
28+
params *ssm.DescribeParametersInput,
29+
optFns ...func(*ssm.Options),
30+
) (*ssm.DescribeParametersOutput, error)
31+
}
32+
33+
// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter
34+
// Inputs:
35+
// c: is the context of the method call, which includes the AWS Region
36+
// api: is the interface that defines the method call
37+
// input: defines the input arguments to the service call
38+
// Output:
39+
// If success, a GetParametersOutput object containing the result of the service call and nil
40+
// Otherwise, nil and an error from the call to GetParameters
41+
func ExecGetParameters(c context.Context, api SSMGetParametersAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) {
42+
return api.GetParameters(c, input)
43+
}
44+
45+
func ExecDescribeParameters(c context.Context, api SSMDescribeParametersAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) {
46+
return api.DescribeParameters(c, input)
47+
}
48+
49+
// Load the Shared AWS Configuration (~/.aws/config)
50+
func InitAWSClient(ctx context.Context) *ssm.Client {
51+
cfg, err := config.LoadDefaultConfig(ctx)
52+
if err != nil {
53+
log.Fatalln("Failed loading AWS Configuration", err)
54+
}
55+
client := ssm.NewFromConfig(cfg)
56+
57+
return client
58+
}

‎scripts/sources/parameters/cmd/aws_ssm_test.go‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
package parameters
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"io/ioutil"
8+
"testing"
9+
10+
"github.com/aws/aws-sdk-go-v2/aws"
11+
"github.com/aws/aws-sdk-go-v2/service/ssm"
12+
"github.com/aws/aws-sdk-go-v2/service/ssm/types"
13+
)
14+
15+
// Fill fake output data
16+
type SSMGetParametersImpl struct{}
17+
type SSMDescribeParametersImpl struct{}
18+
19+
func (dt SSMGetParametersImpl) GetParameters(
20+
ctx context.Context,
21+
params *ssm.GetParametersInput,
22+
optFns ...func(*ssm.Options),
23+
) (*ssm.GetParametersOutput, error) {
24+
parameters := []types.Parameter{}
25+
for _, d := range globalData {
26+
parameters = append(parameters, types.Parameter{
27+
Name: aws.String(d.Name),
28+
Value: aws.String(d.Value),
29+
})
30+
}
31+
output := &ssm.GetParametersOutput{
32+
Parameters: parameters,
33+
}
34+
35+
return output, nil
36+
}
37+
func (dt SSMDescribeParametersImpl) DescribeParameters(
38+
ctx context.Context,
39+
params *ssm.DescribeParametersInput,
40+
optFns ...func(*ssm.Options),
41+
) (*ssm.DescribeParametersOutput, error) {
42+
43+
// TODO(kompotkot): How to test filters?
44+
parameters := []types.ParameterMetadata{
45+
{Name: aws.String("/test/dev/t1")},
46+
{Name: aws.String("/test/dev/t2")},
47+
}
48+
output := &ssm.DescribeParametersOutput{
49+
Parameters: parameters,
50+
}
51+
52+
return output, nil
53+
}
54+
55+
type DataTags struct {
56+
Product string `json:"Product"`
57+
}
58+
59+
type Data struct {
60+
Name string `json:"Name"`
61+
Value string `json:"Value"`
62+
Tags []DataTags `json:"Tags"`
63+
}
64+
65+
var globalData []Data
66+
67+
var globalParameterKeys []string
68+
69+
func populateData(t *testing.T) error {
70+
content, err := ioutil.ReadFile("../data.json")
71+
if err != nil {
72+
return err
73+
}
74+
75+
contentStr := string(content)
76+
err = json.Unmarshal([]byte(contentStr), &globalData)
77+
if err != nil {
78+
return nil
79+
}
80+
81+
return nil
82+
}
83+
84+
func TestDescribeParameters(t *testing.T) {
85+
err := populateData(t)
86+
if err != nil {
87+
t.Fatal("Failed to populate data")
88+
}
89+
90+
api := &SSMDescribeParametersImpl{}
91+
92+
flags := Flags{ProductTag: "test"}
93+
94+
// Test DescribeParameters
95+
parameterKeys := FetchKeysOfParameters(
96+
context.Background(),
97+
*api,
98+
flags,
99+
)
100+
if len(parameterKeys) != 2 {
101+
// TODO(kompotkot): Extract length of parameters from data.json
102+
t.Logf("Length of parameter keys should be 2, but got %d", len(parameterKeys))
103+
t.Fail()
104+
}
105+
106+
for _, p := range parameterKeys {
107+
globalParameterKeys = append(globalParameterKeys, p)
108+
}
109+
}
110+
111+
func TestGetParameters(t *testing.T) {
112+
parameterKeyChunks := GenerateChunks(globalParameterKeys, 10)
113+
114+
api := &SSMGetParametersImpl{}
115+
116+
flags := Flags{Export: false}
117+
118+
parameters := FetchParameters(
119+
context.Background(),
120+
*api,
121+
parameterKeyChunks,
122+
flags,
123+
)
124+
if len(parameters) != 2 {
125+
// TODO(kompotkot): Extract length of parameters from data.json
126+
t.Logf("Length of parameters should be 2, but got %d", len(parameters))
127+
t.Fail()
128+
}
129+
fmt.Println(parameters)
130+
}

‎scripts/sources/parameters/cmd/data.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package parameters
2+
3+
// Parameter structure for storing final result from AWS SSM
4+
type Parameter struct {
5+
Name string
6+
Value string
7+
Export string
8+
}
9+
10+
// Contains command-line flags defined by user
11+
type Flags struct {
12+
Export bool
13+
MaxResults int
14+
Outfile string
15+
ProductTag string
16+
Update bool
17+
}

‎scripts/sources/parameters/cmd/parameters.go‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package parameters
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"log"
8+
"os"
9+
"os/signal"
10+
11+
"github.com/aws/aws-sdk-go-v2/service/ssm"
12+
"github.com/aws/aws-sdk-go-v2/service/ssm/types"
13+
)
14+
15+
// Fetch values for parameters
16+
// Inputs:
17+
// chunks: list of lists with parameter key values
18+
func FetchParameters(ctx context.Context, api SSMGetParametersAPI, chunks [][]string, flags Flags) []Parameter {
19+
var parameters []Parameter
20+
21+
for _, chunk := range chunks {
22+
getInput := &ssm.GetParametersInput{
23+
Names: chunk,
24+
}
25+
results, err := ExecGetParameters(ctx, api, getInput)
26+
if err != nil {
27+
log.Fatal(err)
28+
}
29+
30+
for _, p := range results.Parameters {
31+
parameter := Parameter{
32+
Name: *p.Name, Value: *p.Value,
33+
}
34+
if flags.Export {
35+
parameter.Export = "export "
36+
}
37+
parameters = append(parameters, parameter)
38+
}
39+
}
40+
log.Println("Retrieved values for parameters")
41+
42+
return parameters
43+
}
44+
45+
// Fetch list of parameter keys from AWS with defined filters
46+
func FetchKeysOfParameters(
47+
ctx context.Context,
48+
api SSMDescribeParametersAPI,
49+
flags Flags,
50+
) []string {
51+
var parameters []string
52+
53+
// Set parameter filters
54+
filterKey := "tag:Product"
55+
parameterFilters := []types.ParameterStringFilter{
56+
{
57+
Key: &filterKey,
58+
Values: []string{flags.ProductTag},
59+
},
60+
}
61+
describeInput := &ssm.DescribeParametersInput{
62+
MaxResults: int32(flags.MaxResults),
63+
ParameterFilters: parameterFilters,
64+
}
65+
n := 0
66+
for {
67+
// Fetch list of parameter keys
68+
results, err := ExecDescribeParameters(ctx, api, describeInput)
69+
if err != nil {
70+
log.Fatal(err)
71+
}
72+
for _, p := range results.Parameters {
73+
parameters = append(parameters, *p.Name)
74+
}
75+
76+
// If there are no more parameters break
77+
if results.NextToken == nil {
78+
break
79+
}
80+
describeInput.NextToken = *&results.NextToken
81+
82+
n++
83+
if n >= 50 {
84+
log.Fatal("To many iterations over DescribeParameters loop")
85+
}
86+
}
87+
log.Printf("Retrieved %d parameters", len(parameters))
88+
89+
return parameters
90+
}
91+
92+
// Split list of reports on nested lists
93+
func GenerateChunks(flatSlice []string, chunkSize int) [][]string {
94+
if len(flatSlice) == 0 {
95+
return [][]string{}
96+
}
97+
98+
chunks := make([][]string, 0, len(flatSlice)/chunkSize+1)
99+
100+
for i, v := range flatSlice {
101+
if i%chunkSize == 0 {
102+
chunks = append(chunks, make([]string, 0, chunkSize))
103+
}
104+
chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v)
105+
}
106+
107+
return chunks
108+
}
109+
110+
// WriteToFile generate or update existing file and
111+
// flash to it environment variables
112+
func WriteToFile(parameters []Parameter, outfile string, update bool, export bool) {
113+
flag := os.O_TRUNC | os.O_CREATE | os.O_WRONLY
114+
if update {
115+
flag = os.O_APPEND | os.O_CREATE | os.O_WRONLY
116+
}
117+
118+
f, err := os.OpenFile(
119+
outfile,
120+
flag,
121+
0644,
122+
)
123+
if err != nil {
124+
log.Fatalf("Unable to open file %s, error: %s", outfile, err)
125+
}
126+
defer f.Close()
127+
128+
parametersStr := ""
129+
for _, p := range parameters {
130+
parametersStr += fmt.Sprintf("%s%s=%s\n", p.Export, p.Name, p.Value)
131+
}
132+
if _, err := f.WriteString(parametersStr); err != nil {
133+
log.Fatalf("Unable to write to file %s, error: %s", outfile, err)
134+
}
135+
}
136+
137+
// HandleSignals process Ctrl+C and all script interruptions
138+
func HandleSignals(cancel context.CancelFunc) {
139+
sigCh := make(chan os.Signal)
140+
signal.Notify(sigCh, os.Interrupt)
141+
for {
142+
sig := <-sigCh
143+
switch sig {
144+
case os.Interrupt:
145+
cancel()
146+
return
147+
}
148+
}
149+
}
150+
151+
func Extract() {
152+
var flags Flags
153+
flag.BoolVar(&flags.Export, "export", false, "Add prefix 'export' to each parameter")
154+
flag.IntVar(&flags.MaxResults, "max", 3, "The maximum number of items to return for call to AWS")
155+
flag.StringVar(&flags.Outfile, "outfile", "", "Output file where parameters will be saved")
156+
flag.StringVar(&flags.ProductTag, "product", "", "Product tag")
157+
flag.BoolVar(&flags.Update, "update", false, "Update existing file if exists (by default the file will be overwritten)")
158+
flag.Parse()
159+
160+
if flags.ProductTag == "" {
161+
log.Fatalln("Please specify the tag of product")
162+
}
163+
164+
ctx, cancel := context.WithCancel(context.Background())
165+
go HandleSignals(cancel)
166+
167+
client := InitAWSClient(ctx)
168+
169+
parameterKeys := FetchKeysOfParameters(ctx, client, flags)
170+
171+
// Split slice of parameter keys to chunks by 10 (max len allowed by AWS)
172+
// and fetch values for required parameters
173+
parameterKeyChunks := GenerateChunks(parameterKeys, 10)
174+
if len(parameterKeyChunks) == 0 {
175+
log.Fatalln("Nothing to generate, empty slice provided")
176+
}
177+
parameters := FetchParameters(ctx, client, parameterKeyChunks, flags)
178+
179+
if flags.Outfile != "" {
180+
WriteToFile(parameters, flags.Outfile, flags.Update, flags.Export)
181+
} else {
182+
for _, p := range parameters {
183+
fmt.Printf("%s%s=%s\n", p.Export, p.Name, p.Value)
184+
}
185+
}
186+
}

0 commit comments

Comments
 (0)