Skip to content
This repository was archived by the owner on May 6, 2025. It is now read-only.

Prevent cpu reservation starvation when scaling in #7

Open
cbankston wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Prevent cpu reservation starvation when scaling in #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Once the draining process is complete, the instance will be terminated.
#### Changable Parameters:
* SCALE_IN_CPU_TH = 30 `# Below this EC2 average metric scaling would take action`
* SCALE_IN_MEM_TH = 60 `# Below this cluster average metric scaling would take action`
* FUTURE_CPU_TH = 40 `# Below this future metric scaling would take action`
Copy link
Copy Markdown
Contributor Author

@cbankston cbankston Oct 23, 2018

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me know if you want to default this to 30 so that it doesn't break any existing functionality.

Copy link
Copy Markdown
Owner

@omerxx omerxx Jan 26, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cbankston WDYT?
Plus, can you handle the conflicts?

* FUTURE_MEM_TH = 70 `# Below this future metric scaling would take action`
* ECS_AVOID_STR = 'awseb' `# Use this to avoid clusters containing a specific string (i.e ElasticBeanstalk clusters)`

Expand All @@ -31,7 +32,7 @@ Once the draining process is complete, the instance will be terminated.

#### Flow logic
* Iterate over existing ECS clusters
* Check a cluster's ability to scale-in based on predicted future memory reservation capacity
* Check a cluster's ability to scale-in based on predicted future cpu and memory reservation capacity
* Look for empty hosts the can be scaled
* Look for least utilized host
* Choose a candidate and put in draining state
Expand Down
38 changes: 22 additions & 16 deletions ecscale.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

SCALE_IN_CPU_TH = 30
SCALE_IN_MEM_TH = 60
FUTURE_CPU_TH = 40
FUTURE_MEM_TH = 70
ECS_AVOID_STR = 'awseb'
logline = {}
Expand All @@ -19,12 +20,12 @@ def clusters(ecsClient):
return [cluster for cluster in response['clusterArns'] if ECS_AVOID_STR not in cluster]


def cluster_memory_reservation(cwClient, clusterName):
# Return cluster mem reservation average per minute cloudwatch metric
def cluster_metric(cwClient, clusterName, metricName):
# Return cluster average per minute cloudwatch metric
try:
response = cwClient.get_metric_statistics(
Namespace='AWS/ECS',
MetricName='MemoryReservation',
MetricName=metricName,
Dimensions=[
{
'Name': 'ClusterName',
Expand All @@ -39,7 +40,7 @@ def cluster_memory_reservation(cwClient, clusterName):
return response['Datapoints'][0]['Average']

except Exception:
logger({'ClusterMemoryError': 'Could not retrieve mem reservation for {}'.format(clusterName)})
logger({'ClusterMetricError': 'Could not retrieve {} for {}'.format(metricName, clusterName)})


def find_asg(clusterName, asgData):
Expand Down Expand Up @@ -167,18 +168,18 @@ def drain_instance(containerInstanceId, ecsClient, clusterArn):
logger({'DrainingError': e})


def future_reservation(activeContainerDescribed, clusterMemReservation):
# If the cluster were to scale in an instance, calculate the effect on mem reservation
# return cluster_mem_reserve*num_of_ec2 / num_of_ec2-1
def future_metric(activeContainerDescribed, metricValue):
# If the cluster were to scale in an instance, calculate the effect on the given metric value
# return metric_value*num_of_ec2 / num_of_ec2-1
numOfEc2 = len(activeContainerDescribed['containerInstances'])
if numOfEc2 > 1:
futureMem = (clusterMemReservation*numOfEc2) / (numOfEc2-1)
futureValue = (metricValue*numOfEc2) / (numOfEc2-1)
else:
return 100

print '*** Current: {} | Future : {}'.format(clusterMemReservation, futureMem)
print '*** Current: {} | Future : {}'.format(metricValue, futureValue)

return futureMem
return futureValue


def asg_scaleable(asgData, clusterName):
Expand All @@ -195,7 +196,8 @@ def retrieve_cluster_data(ecsClient, cwClient, asgClient, cluster):
clusterName = cluster.split('/')[1]
print '*** {} ***'.format(clusterName)
activeContainerInstances = ecsClient.list_container_instances(cluster=cluster, status='ACTIVE')
clusterMemReservation = cluster_memory_reservation(cwClient, clusterName)
clusterCpuReservation = cluster_metric(cwClient, clusterName, 'CPUReservation')
clusterMemReservation = cluster_metric(cwClient, clusterName, 'MemoryReservation')

if activeContainerInstances['containerInstanceArns']:
activeContainerDescribed = ecsClient.describe_container_instances(cluster=cluster, containerInstances=activeContainerInstances['containerInstanceArns'])
Expand All @@ -213,6 +215,7 @@ def retrieve_cluster_data(ecsClient, cwClient, asgClient, cluster):

dataObj = {
'clusterName': clusterName,
'clusterCpuReservation': clusterCpuReservation,
'clusterMemReservation': clusterMemReservation,
'activeContainerDescribed': activeContainerDescribed,
'drainingInstances': drainingInstances,
Expand Down Expand Up @@ -246,6 +249,7 @@ def main(run='normal'):
continue
else:
clusterName = clusterData['clusterName']
clusterCpuReservation = clusterData['clusterCpuReservation']
clusterMemReservation = clusterData['clusterMemReservation']
activeContainerDescribed = clusterData['activeContainerDescribed']
drainingInstances = clusterData['drainingInstances']
Expand All @@ -256,9 +260,11 @@ def main(run='normal'):
print '{}: in Minimum state, skipping'.format(clusterName)
continue

if (clusterMemReservation < FUTURE_MEM_TH and
future_reservation(activeContainerDescribed, clusterMemReservation) < FUTURE_MEM_TH):
# Future memory levels allow scale
if (clusterCpuReservation < FUTURE_CPU_TH and
clusterMemReservation < FUTURE_MEM_TH and
future_metric(activeContainerDescribed, clusterCpuReservation) < FUTURE_CPU_TH and
future_metric(activeContainerDescribed, clusterMemReservation) < FUTURE_MEM_TH):
# Future reservation levels allow scale
if emptyInstances.keys():
# There are empty instances
for instanceId, containerInstId in emptyInstances.iteritems():
Expand All @@ -268,8 +274,8 @@ def main(run='normal'):
print 'Draining empty instance {}'.format(instanceId)
drain_instance(containerInstId, ecsClient, cluster)

if (clusterMemReservation < SCALE_IN_MEM_TH):
# Cluster mem reservation level requires scale
if (clusterCpuReservation < SCALE_IN_CPU_TH and clusterMemReservation < SCALE_IN_MEM_TH):
# Cluster reservation level requires scale
if (ec2_avg_cpu_utilization(clusterName, asgData, cwClient) < SCALE_IN_CPU_TH):
instanceToScale = scale_in_instance(cluster, activeContainerDescribed)['containerInstanceArn']
if run == 'dry':
Expand Down