shariq.site

Run an SSM Command against a set of EC2 Instances

A quick script today to run an SSM Command against a set of EC2 Instances. You can selectively target instance based on a TAG.

import boto3
import os
from time import sleep

lambda_func_name = os.getenv("AWS_LAMBDA_FUNCTION_NAME", "")

if lambda_func_name == "":  # We are not running in AWS
    boto3.setup_default_session(profile_name='<profile_name>')

ec2_client = boto3.client('ec2', region_name='ap-southeast-2')
ssm_client = boto3.client('ssm', region_name='ap-southeast-2')  # use region code in which you are working

sleep_duration = 2

def get_instances(os):

    instances = []

    # Check running or stopped instances
    response = ec2_client.describe_instances(
        Filters=[
            {
                'Name': 'instance-state-name',
                'Values': ['running']
            }
        ])

    # Iterate over instance(s)
    for r in response['Reservations']:
        for inst in r['Instances']:
            inst_id = inst['InstanceId']
            tags = inst['Tags']

            ins_tag = ""

            for tag in tags:
                if "OSFamily" in tag['Key']:
                    ins_tag = (tag['Value'])
                    break
                else:
                    ins_tag = "NA"

            if ins_tag == os:
                instances.append(inst_id)

    return instances

def ssm_run_command(instance_id, cmd, os_family):

    document = "AWS-RunPowerShellScript"

    if (os_family == "Linux"):
        document = "AWS-RunShellScript"

    response = ssm_client.send_command(
        InstanceIds=[
            instance_id  # use instance id on which you want to execute, even multiple is allowed
        ],
        DocumentName=document,
        Parameters={
            'commands': [
                cmd
            ]
        },
    )

    #print(response)

    sleep(sleep_duration)  # Seconds to wait for command to execute

    command_id = response['Command']['CommandId']
    output = ssm_client.get_command_invocation(CommandId = command_id, InstanceId = instance_id)

    return output['StandardOutputContent']

def check_time_zone_windows():

    os_family = "Windows"

    instances = get_instances(os_family)
    command = "[System.TimeZone]::CurrentTimeZone.StandardName"

    instances_with_wrong_tz = []

    for instance in instances:
        print("Checking instance: " + instance)
        time_zone = ssm_run_command(instance, command, os_family)
        #print(time_zone)

        if ( time_zone[0:3] != "AUS"):
            instances_with_wrong_tz.append(instance)

    result_str = ""

    if (len(instances_with_wrong_tz) > 0):

        result_str = "Following " + os_family + " instances have wrong Timezone:\n\n"

        for instance in instances_with_wrong_tz:
            result_str = result_str + instance + "\n\n"

    return result_str

def check_time_zone_linux():
    os_family = "Linux"

    instances = get_instances(os_family)
    command = 'date +"%Z"'

    instances_with_wrong_tz = []

    for instance in instances:
        print("Checking instance: " + instance)
        time_zone = ssm_run_command(instance, command, os_family)
        #print(time_zone)

        if (time_zone[0:2] != "AE"):
            instances_with_wrong_tz.append(instance)

    result_str = ""

    if (len(instances_with_wrong_tz) > 0):

        result_str = "Following " + os_family + " instances have wrong Timezone:\n\n"

        for instance in instances_with_wrong_tz:
            result_str = result_str + instance + "\n\n"

    return result_str

def lambda_handler(event, context):

    result_lin = check_time_zone_linux()
    result_win = check_time_zone_windows()

    result = result_lin + "\n\n" + result_win

    print(result)

if __name__ == "__main__":
    lambda_handler(0, 0)

Find missing Tags on EC2 Instances

Find missing Tags on EC2 Instances

# This script look for missing tags on EC2 instances
# Initialize 5 Environment variables tag1...tag5
# Usual tags to check can be as follows
#  - cpm backup
#  - monitor_site24x7
#  - Project
#  - Environment
#  - Owner

import boto3
import logging
import os

lambda_func_name = os.getenv("AWS_LAMBDA_FUNCTION_NAME", "")

if lambda_func_name == "":  # We are not running in AWS
    boto3.setup_default_session(profile_name='iconwater')

# setup simple logging for INFO
logger = logging.getLogger()
logger.setLevel(logging.INFO)

# define the connection
ec2 = boto3.resource('ec2')

def send_alert(alert_data):
    topic_arn = os.getenv("TopicARN", "")

    if topic_arn == "":
        print("send_alert: Missing topic ARN. Returning without sending alert.")
        return

    subject = os.getenv('CustomerID', '') + " - Missing EC2 Instances Tag Check"
    message = "Missing EC2 Instances Tag Check Results: \n\n" + alert_data

    print("send_alert: *** Sending alert ***")
    print("send_alert: Message: {0}".format(message))

    client = boto3.client('sns')
    response = client.publish(TargetArn=topic_arn,
                              Message=message,
                              Subject=subject)

def find_instances_with_missing_tags(tag_to_check):
    result_str = ""

    client = boto3.client('ec2')

    client = boto3.client('ec2', region_name='ap-southeast-2')
    # Check running or stopped instances
    response = client.describe_instances(
        Filters=[
            {
                'Name': 'instance-state-name',
                'Values': ['running', 'stopped']
            }
        ])
    # Iterate over instance(s)
    for r in response['Reservations']:
        for inst in r['Instances']:
            inst_id = inst['InstanceId']
            tags = inst['Tags']
            # Check the Name tag
            for tag in tags:
                if 'Name' in tag['Key']:
                    ins_name = (tag['Value'])
                    break
                else:
                    ins_name = "{No-Name}"

            for tag in tags:
                if tag_to_check in tag['Key']:
                    ins_tag = (tag['Value'])
                    break
                else:
                    ins_tag = "NA"

            if ins_tag == "NA":
                s = "Tag '{}' missing for instance {} ({})\n\n".format(tag_to_check, ins_name, inst['InstanceId'])
                # print (s)
                result_str = result_str + s
                # else:
                #    print("Tag '{}' present for instance {} ({})".format(tag_to_check, ins_name, inst['InstanceId']))

    return result_str

def lambda_handler(event, context):
    tag1 = os.getenv("tag1", "")
    tag2 = os.getenv("tag2", "")
    tag3 = os.getenv("tag3", "")
    tag4 = os.getenv("tag4", "")
    tag5 = os.getenv("tag5", "")

    s = ""

    if tag1 != "": s = s + find_instances_with_missing_tags(tag1)
    if tag2 != "": s = s + find_instances_with_missing_tags(tag2)
    if tag3 != "": s = s + find_instances_with_missing_tags(tag3)
    if tag4 != "": s = s + find_instances_with_missing_tags(tag4)
    if tag5 != "": s = s + find_instances_with_missing_tags(tag5)

    if s != "":
        print(s)
        send_alert(s)

    return 0

if __name__ == "__main__":
    lambda_handler(0, 0)

Find EC2 instances not reporting to SSM (Python)

Find EC2 instances not reporting to SSM

# This script look for Instances not reporting/online in SSM

import boto3
import os

lambda_func_name = os.getenv("AWS_LAMBDA_FUNCTION_NAME", "")

if lambda_func_name == "":  # We are not running in AWS
    boto3.setup_default_session(profile_name='iconwater')

ec2 = boto3.resource('ec2')

instances_missing_in_ssm = []

def send_alert():
    alert_data = ""
    topic_arn = os.getenv("TopicARN", "")

    if topic_arn == "":
        print("send_alert: Missing topic ARN. Returning without sending alert.")
        return

    for m in instances_missing_in_ssm:
        alert_data  = alert_data + m +"\n\n"

    subject = os.getenv('CustomerID', '') + " - Missing Managed (SSM) Instances"
    message = "The following instances are offline or not reporting to SSM: \n\n" + alert_data

    print("send_alert: *** Sending alert ***")
    print("send_alert: Message: {0}".format(message))

    client = boto3.client('sns')
    response = client.publish(TargetArn=topic_arn,
                              Message=message,
                              Subject=subject)

def check_instance_ssm_status(instance_id):

    #print ("Checking {}".format(instance_id))

    client_ssm = boto3.client('ssm', region_name='ap-southeast-2')
    # Check running or stopped instances
    response = client_ssm.describe_instance_information(
        InstanceInformationFilterList=[
            {
                'key': 'InstanceIds',
                'valueSet': [
                    instance_id,
                ]
            },
        ]
    )

    #print (response)

    for r in response['InstanceInformationList']:
        #print(r)
        return r['PingStatus']

def find_instances_not_with_ssm():
    result_str = ""

    client_ec2 = boto3.client('ec2', region_name='ap-southeast-2')
    # Check running or stopped instances
    response = client_ec2.describe_instances(
        Filters=[
            {
                'Name': 'instance-state-name',
                'Values': ['running', 'stopped']
            }
        ])
    # Iterate over instance(s)
    for r in response['Reservations']:
        for inst in r['Instances']:
            inst_id = inst['InstanceId']
            tags = inst['Tags']
            # Check the Name tag
            for tag in tags:
                if 'Name' in tag['Key']:
                    ins_name = (tag['Value'])
                    break
                else:
                    ins_name = "{No-Name}"

            inst_id = inst['InstanceId']
            ssm_status = check_instance_ssm_status(inst_id)

            if ssm_status != 'Online':
                instances_missing_in_ssm.append(inst_id)

            #s = "{} ({})".format(ins_name, inst['InstanceId'])

    return len(instances_missing_in_ssm)

def lambda_handler(event, context):

    ret = find_instances_not_with_ssm()

    if (ret > 0):
        send_alert()

    return 0

if __name__ == "__main__":
    lambda_handler(0, 0)

Enable Cost Allocation Tags to differentiate project based billing

When running in an AWS public cloud environment, many times there is a need to dissect the billing across different projects for accounting and accrual purposes. AWS provides a mechanism to aggregate related platform costs using a feature known as Cost Allocation Tags. With this feature you can designate Tags on your AWS resources to track costs on a detailed level.

From the AWS Documentation:

Activating tags for cost allocation tells AWS that the associated cost data for these tags should be made available throughout the billing pipeline. Once activated, cost allocation tags can be used as a dimension of grouping and filtering in Cost Explorer, as well as for refining AWS budget criteria.

For example, to view cost allocation based on various project resources in your AWS account, you can tag these resources (EC2 instances, S3 buckets, etc) with a tag named “Project”. Next the Project tag can be activated as a Cost Allocation Tag. From then on AWS will include this tag in associated cost data to allow for filtering based in the tag in Cost Explorer reports.

Let’s walk through the steps of setting this up:

1. Log in to your AWS Management Console
2. Tag all the resources with a Tag Key as Project and Value as per your various projects. Understand that this may not be possible for every resource type.
3. Navigate to My Billing Dashboard > Cost Allocation Tags
4. Under User-Defined Cost Allocation Tags section, select the tag “Project” and click the “Activate” button.

Once a tag is activated it will take around 24 hours for billing data to appear under this tag.

Next, to view the costs under a project, do the following:

1. Log in to your AWS Management Console
2. Navigate to My Billing Dashboard > Cost Explorer
3. Click “Launch Cost Explorer”
4. On the right side of the page under Filters section, click the Tag filter and select the Project tag, then the Tag Value to filter cost by the Project

As you can see from the screenshot below, now we can see exactly how much each project is costing per day (or month, if selected)

Some important points to consider:

• Cost allocation tagging is “managed” via the master billing account at the root of the AWS organization. If your account is part of an organization you will have to contact this account administrator to enable the cost allocation tags.
• The error message in the previous screenshot will always appear in tenancies not allocated the management permission.
• Some resources notably bandwidth charges cannot be tagged and thus cannot be accounted under cost allocation tagging. A common pattern in such cases is to calculate percentage cost on each project and cost the unaccounted charges based on this percentage.

Lunch with Kloud colleagues

One line Chrome installer in Windows

It is often said that Internet Explorer is the best browser to Download Chrome :) Well you cna use Powershell as well:

$LocalTempDir = $env:TEMP; $ChromeInstaller = "ChromeInstaller.exe"; (new-object    System.Net.WebClient).DownloadFile('http://dl.google.com/chrome/install/375.126/chrome_installer.exe', "$LocalTempDir\$ChromeInstaller"); & "$LocalTempDir\$ChromeInstaller" /silent /install; $Process2Monitor =  "ChromeInstaller"; Do { $ProcessesFound = Get-Process | ?{$Process2Monitor -contains $_.Name} | Select-Object -ExpandProperty Name; If ($ProcessesFound) { "Still running: $($ProcessesFound -join ', ')" | Write-Host; Start-Sleep -Seconds 2 } else { rm "$LocalTempDir\$ChromeInstaller" -ErrorAction SilentlyContinue -Verbose } } Until (!$ProcessesFound)

Find Critical updates required for the System (RHEL)

sudo yum updateinfo list --security --sec-severity=Critical

Hanging out with Kloud team