Skip to content

ECS Scaling at 20% CPU usage. #16693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
connoromalleyatwork wants to merge 1 commit into
base: main
Choose a base branch
from
Open

ECS Scaling at 20% CPU usage. #16693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion dotcom-rendering/cdk/bin/cdk.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { App } from 'aws-cdk-lib';
import { App, Duration } from 'aws-cdk-lib';
import { InstanceClass, InstanceSize, InstanceType } from 'aws-cdk-lib/aws-ec2';
import type { RenderingCDKStackProps } from '../lib/renderingStack';
import { RenderingCDKStack } from '../lib/renderingStack';
Expand Down Expand Up @@ -115,6 +115,15 @@ export const TagPageRenderingPropsCODE: RenderingCDKStackProps = {
imageIdentifier: getImageIdentifier(),
taskCpu: 1024,
taskMemoryLimitMiB: 2048,
scaling: {
minimumTasks: 1,
maximumTasks: 9,
cpuScaling: {
targetValue: 20,
scaleInCooldown: Duration.seconds(60),
scaleOutCooldown: Duration.seconds(60),
},
},
},
};

Expand Down Expand Up @@ -158,6 +167,17 @@ export const TagPageRenderingPropsPROD: RenderingCDKStackProps = {
imageIdentifier: getImageIdentifier(),
taskCpu: 2048,
taskMemoryLimitMiB: 4096,
scaling: {
minimumTasks: 9,
maximumTasks: 90,
cpuScaling: {
targetValue: 20,
// TODO: Tune cooldown values.
// https://docs.aws.amazon.com/autoscaling/application/userguide/target-tracking-scaling-policy-overview.html#target-tracking-cooldown
scaleInCooldown: Duration.seconds(60),
scaleOutCooldown: Duration.seconds(60),
Comment on lines +177 to +178

@connoromalleyatwork connoromalleyatwork Sep 15, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The default is Duration.seconds(300), which I actually think I like more as it gives more time to prevent fluctuations in capacity.

We had the 60 seconds before this change when doing our load testing, however I think that may have been to speed up scaling events to reduce wait time to watch them happen. Not for any meaningful reason.

},
},
},
};

Expand Down
42 changes: 39 additions & 3 deletions dotcom-rendering/cdk/lib/__snapshots__/renderingStack.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2109,7 +2109,7 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering CODE
"EcsTaskDefinitionTaskRoleB7B6D8DD",
],
"Properties": {
"MaxCapacity": 2,
"MaxCapacity": 9,
"MinCapacity": 1,
"ResourceId": {
"Fn::Join": [
Expand Down Expand Up @@ -3667,6 +3667,24 @@ systemctl start tag-page-rendering",
},
"Type": "AWS::IAM::InstanceProfile",
},
"tagpagerenderingCpuScaling1742C101": {
"Properties": {
"PolicyName": "TagPageRenderingCODEtagpagerenderingCpuScalingAFEBC723",
"PolicyType": "TargetTrackingScaling",
"ScalingTargetId": {
"Ref": "EcsServiceTaskCountTarget02FCCE22",
},
"TargetTrackingScalingPolicyConfiguration": {
"PredefinedMetricSpecification": {
"PredefinedMetricType": "ECSServiceAverageCPUUtilizationHighResolution",
},
"ScaleInCooldown": 60,
"ScaleOutCooldown": 60,
"TargetValue": 20,
},
},
"Type": "AWS::ApplicationAutoScaling::ScalingPolicy",
},
"tagpagerenderingEcsClusterE7696595": {
"Properties": {
"ClusterSettings": [
Expand Down Expand Up @@ -4260,8 +4278,8 @@ exports[`The RenderingCDKStack matches the snapshot for Tag Page Rendering PROD
"EcsTaskDefinitionTaskRoleB7B6D8DD",
],
"Properties": {
"MaxCapacity": 2,
"MinCapacity": 1,
"MaxCapacity": 90,
"MinCapacity": 9,
"ResourceId": {
"Fn::Join": [
"",
Expand Down Expand Up @@ -6261,6 +6279,24 @@ systemctl start tag-page-rendering",
},
"Type": "AWS::IAM::InstanceProfile",
},
"tagpagerenderingCpuScaling1742C101": {
"Properties": {
"PolicyName": "TagPageRenderingPRODtagpagerenderingCpuScaling4582D272",
"PolicyType": "TargetTrackingScaling",
"ScalingTargetId": {
"Ref": "EcsServiceTaskCountTarget02FCCE22",
},
"TargetTrackingScalingPolicyConfiguration": {
"PredefinedMetricSpecification": {
"PredefinedMetricType": "ECSServiceAverageCPUUtilizationHighResolution",
},
"ScaleInCooldown": 60,
"ScaleOutCooldown": 60,
"TargetValue": 20,
},
},
"Type": "AWS::ApplicationAutoScaling::ScalingPolicy",
},
"tagpagerenderingEcsClusterE7696595": {
"Properties": {
"ClusterSettings": [
Expand Down
11 changes: 6 additions & 5 deletions dotcom-rendering/cdk/lib/renderingStack.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
} from '@guardian/cdk/lib/constructs/core';
import { GuCname } from '@guardian/cdk/lib/constructs/dns/dns-records';
import { GuAllowPolicy } from '@guardian/cdk/lib/constructs/iam';
import type { GuLoadBalancedAppExperimentalProps } from '@guardian/cdk/lib/experimental/patterns/gu-load-balanced-app';
import { GuLoadBalancedAppExperimental } from '@guardian/cdk/lib/experimental/patterns/gu-load-balanced-app';
import type { GuAsgCapacity } from '@guardian/cdk/lib/types';
import { aws_cloudwatch, type App as CDKApp, Duration } from 'aws-cdk-lib';
Expand Down Expand Up @@ -67,6 +68,10 @@ export interface RenderingCDKStackProps extends Omit<GuStackProps, 'stack'> {
* Memory in MB for ECS task
*/
taskMemoryLimitMiB: number;

scaling: NonNullable<
GuLoadBalancedAppExperimentalProps['ecsProps']
>['scaling'];
};
}

Expand Down Expand Up @@ -296,10 +301,7 @@ export class RenderingCDKStack extends CDKStack {

memoryLimitMiB: ecsProps.taskMemoryLimitMiB,
cpu: ecsProps.taskCpu,
scaling: {
minimumTasks: 1,
maximumTasks: 2,
},
scaling: ecsProps.scaling,
},

// Route all traffic to EC2
Expand All @@ -312,7 +314,6 @@ export class RenderingCDKStack extends CDKStack {

if (app.ecsService) {
const { taskDefinition } = app.ecsService;

const ecsEnvVars: Record<string, string> = {
// Custom environment variables needed by the application
NODE_ENV: 'production',
Expand Down
Loading
Loading