AWS Cloudformation template for Lambda access to Elasticache redis (private subnet) and DynamoDB (public subnet)

The lambda code zip file

  • Create a dynamo table, make sure to replace “MyDynamoTable” below with its name
  • Create an index.js file, stick this inside it:
// private access
var redis = require('redis');
var rClient = redis.createClient(process.env.RedisPort, process.env.RedisEndpoint);

// public access
var AWS = require('aws-sdk');
var dynamodb = new AWS.DynamoDB();

exports.handler = function (event, context, callback) {
    context.callbackWaitsForEmptyEventLoop = false;
    dynamodb.scan({TableName: 'MyDynamoTable'}, function(err, data) {
        if (err){
            return callback(err);
        }else{
            return callback(null, data);
        }
    });
};

It will be the code run by the lambda. We’re establishing a redis connection and a dynamo connection from the same lambda.

  • Next to your index.js file, create a node_modules and install redis inside inside it (or just “npm install redis” from that folder, and it will do the work for you.
  • zip the whole thing thing and put it on s3. I used this command to do it: “zip -r file.zip index.js node_modules”. It should look something like this:

Screen Shot 2016-12-30 at 1.31.29 PM.png

  • Upload to S3, and replace the bucket name in the template below “YOUR-BUCKET-NAME-GOES-HERE”. in the lambda section.

The Cloudformation template

  • Beware! You’re paying for the resources this template creates. Specifically, the redis instance (t2.micro) and the NAT gateways are not cheap.
  • This is the Cloudformation that will create all the necessary resources for you:
{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Resources": {
    "VPC": {
      "Type": "AWS::EC2::VPC",
      "Properties": { "CidrBlock": "10.0.0.0/16" }
    },
    "PublicSubnet": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "CidrBlock": "10.0.0.0/24"
      }
    },
    "PrivateSubnet": {
      "Type": "AWS::EC2::Subnet",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "CidrBlock": "10.0.1.0/24"
      }
    },
    "InternetGateway": {
      "Type": "AWS::EC2::InternetGateway"
    },
    "AttachGateway": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "VpcId": { "Ref": "VPC" },
        "InternetGatewayId": { "Ref": "InternetGateway" }
      }
    },
    "PublicRouteTable": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "VPC" }
      }
    },
    "PrivateRouteTable": {
      "Type": "AWS::EC2::RouteTable",
      "Properties": {
        "VpcId": { "Ref": "VPC" }
      }
    },
    "PublicRoute": {
      "Type": "AWS::EC2::Route",
      "DependsOn": "AttachGateway",
      "Properties": {
        "RouteTableId": {
          "Ref": "PublicRouteTable"
        },
        "DestinationCidrBlock": "0.0.0.0/0",
        "GatewayId": { "Ref": "InternetGateway" }
      }
    },
    "PrivateRoute": {
      "Type": "AWS::EC2::Route",
      "DependsOn": "AttachGateway",
      "Properties": {
        "RouteTableId": { "Ref": "PrivateRouteTable" },
        "DestinationCidrBlock": "0.0.0.0/0",
        "NatGatewayId": { "Ref": "NatGateway" }
      }
    },
    "NatGateway": {
      "Type": "AWS::EC2::NatGateway",
      "Properties": {
        "AllocationId": {
          "Fn::GetAtt": [ "ElasticIp", "AllocationId"]
        },
        "SubnetId": { "Ref": "PublicSubnet" }
      }
    },
    "GatewayAttachment": {
      "Type": "AWS::EC2::VPCGatewayAttachment",
      "Properties": {
        "VpcId": { "Ref": "VPC"},
        "InternetGatewayId": { "Ref": "InternetGateway" }
      }
    },
    "ElasticIp": {
      "Type": "AWS::EC2::EIP",
      "DependsOn": "GatewayAttachment",
      "Properties": { "Domain": "vpc" }
    },
    "PublicSubnetRouteTableAssociation": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PublicSubnet" },
        "RouteTableId": { "Ref": "PublicRouteTable"}
      }
    },
    "PrivateSubnetRouteTableAssociation": {
      "Type": "AWS::EC2::SubnetRouteTableAssociation",
      "Properties": {
        "SubnetId": { "Ref": "PrivateSubnet" },
        "RouteTableId": { "Ref": "PrivateRouteTable" }
      }
    },
    "RedisInstance": {
      "Type": "AWS::ElastiCache::CacheCluster",
      "Properties": {
        "CacheNodeType": "cache.t2.micro",
        "CacheSubnetGroupName": { "Ref": "CacheSubnetGroup" },
        "VpcSecurityGroupIds": [ {"Ref": "RedisSecurityGroup"}
        ],
        "Engine": "redis",
        "NumCacheNodes": "1",
        "PreferredMaintenanceWindow": "Tue:18:00-Tue:19:00"
      }
    },
    "RedisSecurityGroup": {
      "Type": "AWS::EC2::SecurityGroup",
      "DependsOn": "CacheSubnetGroup",
      "Properties": {
        "GroupDescription": "Security Group for Redis",
        "VpcId": { "Ref": "VPC" },
        "SecurityGroupIngress": [
          {
            "IpProtocol": "tcp",
            "FromPort": "6379",
            "ToPort": "6379",
            "CidrIp": "10.0.1.0/24"
          }
        ]
      }
    },
    "CacheSubnetGroup": {
      "Type": "AWS::ElastiCache::SubnetGroup",
      "DependsOn": [ "PrivateSubnet"],
      "Properties": {
        "Description": "Subnet Group",
        "SubnetIds": [ {"Ref": "PrivateSubnet"}]
      }
    },
    "ReallyGreatLamba": {
      "Type": "AWS::Lambda::Function",
      "DependsOn": "RedisInstance",
      "Properties": {
        "Environment": {
          "Variables": {
            "RedisEndpoint": { "Fn::GetAtt": [ "RedisInstance", "RedisEndpoint.Address"]},
            "RedisPort": { "Fn::GetAtt": ["RedisInstance","RedisEndpoint.Port"]
            }
          }
        },
        "Code": {
          "S3Bucket": "YOUR-BUCKET-NAME-GOES-HERE",
          "S3Key": "file.zip"
        },
        "Role": {
          "Fn::GetAtt": ["MyLambdaRole","Arn"]
        },
        "VpcConfig": {
          "SecurityGroupIds": [{"Ref": "RedisSecurityGroup"}],
          "SubnetIds": [{"Ref": "PrivateSubnet"}]
        },
        "Description": "A Lambda that can talk to redis and the public internet at once",
        "Handler": "index.handler",
        "Runtime": "nodejs4.3",
        "Timeout": 10,
        "MemorySize": 128
      }
    },
    "MyLambdaRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [{
              "Effect": "Allow",
              "Principal": {"Service": "lambda.amazonaws.com"},
              "Action": ["sts:AssumeRole"]
            }
          ]
        },
        "Policies": [{
            "PolicyName": "adminPolicy",
            "PolicyDocument": {
              "Statement": [{
                  "Effect": "Allow",
                  "Action": "*",
                  "Resource": "*"
                }
              ]
            }
          }
        ]
      }
    }
  }
}

The easiest way to execute this template is probably through the template designer. Once you’ve finished the previous steps, paste the template in the “template” section, hit the “refresh” button at the top right if you want to visualize the stack, and then “create the stack” at the top left.

Screen Shot 2016-12-30 at 3.22.31 PM.png

Once you get through the Create Stack steps (the only required field is the stack name), it might take 5 or 10 minutes to create all the resources for you. You can track progress in the Cloudformation section of the AWS UI.

Screen Shot 2016-12-30 at 3.28.31 PM.png

What this creates

  • A VPC with 2 subnets, 1 private and 1 public
  • A managed NAT instance to access the internet from your lambda (called an “internet gateway” in AWS terms.
  • All the wiring for your VPC to work:
    • 2 routes
    • 2 route gateways
    • 2 route table associations
    • a VPC gateway attachment
    • a cache subnet group
  • A t2.micro redis instance
  • A security group to access redis
  • A lambda
  • An IAM role for your lambda

For the visually-inclined:

new-designer (1).png

As mentioned before, don’t forget that these things cost money! Don’t forget to shut them down when you’re done using them.

Finally, I should also point out that this setup is NOT production-ready. You’ll want to make sure you are more highly-available (multi-az), the IAM role create has way too liberal permissions and you may need larger subnets if you plan to have a lot of lambdas running (256 IPs right now).

Fun gotchas and random observations

  • If you are using the new node4.3 runtime (as we are here), the signature for your lambda handler has changed to “(event, context, callback)”. By default, calling “callback” while you have persistent connections (like redis here) will not return as expected, unless you explicitly clean it up. It will instead time out and refuse to return as expected. This is why we need to add:
context.callbackWaitsForEmptyEventLoop = false;

inside the lambda handler. You can read more about it here, but the important part is:

The default value is true. This property is useful only to modify the default behavior of the callback. By default, the callback will wait until the Node.js runtime event loop is empty before freezing the process and returning the results to the caller. You can set this property to false to request AWS Lambda to freeze the process soon after the callback is called, even if there are events in the event loop. AWS Lambda will freeze the process, any state data and the events in the Node.js event loop (any remaining events in the event loop processed when the Lambda function is called next and if AWS Lambda chooses to use the frozen process).

  • Lambda recently announced support for environment variables, and it’s clear for a use case like this, how powerful they can be. We’re creating a redis cluster with a single machine, and referencing its endpoint and port in the environment variable section of the lambda resource definition:
"Environment": {
  "Variables": {
    "RedisEndpoint": { "Fn::GetAtt": [ "RedisInstance", "RedisEndpoint.Address"]},
    "RedisPort": { "Fn::GetAtt": ["RedisInstance","RedisEndpoint.Port"]
    }
  }
}

We can then access the endpoint and port inside our lambda with

console.log(process.env.RedisPort, process.env.RedisEndpoint)