go-s3-lambda/template.yaml at master · marekq/go-s3-lambda · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
AWSTemplateFormatVersion: '2010-09-09'
Transform: 'AWS::Serverless-2016-10-31'
Description: An AWS Serverless Specification template describing your function.

Parameters:
  # specify an existing S3 bucket with some objects stored
  SourceBucket:
    Type: String
    Description: Name of the existing source S3 bucket with files.

Resources:
  # option 1 - resources for Go Lambda using the S3 path as input
  # for this scenario, the Lambda function needs to have IAM access to the bucket

  # queue - create sqs queue
  S3PathSQS:
    Type: AWS::SQS::Queue

  # table - create a dynamodb table to store results
  S3PathTable:
    Type: 'AWS::DynamoDB::Table'
    Properties:
      AttributeDefinitions:
      - AttributeName: Fileurl
        AttributeType: S
      - AttributeName: Md5
        AttributeType: S
      KeySchema:
      - AttributeName: Fileurl
        KeyType: HASH
      - AttributeName: Md5
        KeyType: RANGE
      BillingMode: PAY_PER_REQUEST

  # receiver - create s3 path lambda that receives sqs messages
  S3PathLambda:
    Type: 'AWS::Serverless::Function'
    Properties:
      CodeUri: s3lambda/
      Handler: s3lambda
      Tracing: Active
      Runtime: go1.x
      Events:
        S3PathEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt S3PathSQS.Arn
            BatchSize: 10
      Description: 'Lambda function which processes S3 paths submitted over SQS'
      MemorySize: 256
      Timeout: 5
      Policies:
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - 's3:Get*'
              Resource: !Sub "arn:aws:s3:::${SourceBucket}/*"
        - DynamoDBCrudPolicy:
            TableName: !Ref S3PathTable
      Environment:
        Variables:
          s3bucket: !Ref SourceBucket
          ddbtable: !Ref S3PathTable
          lambdamode: "s3path"

  # sender - create s3 path load generator lambda
  S3PathLoadgenLambda:
    Type: 'AWS::Serverless::Function'
    Properties:
      CodeUri: s3loadgen/
      Handler: s3loadgen
      Tracing: Active
      Runtime: go1.x
      Description: 'Lambda function which generates S3 paths submitted to SQS'
      MemorySize: 512
      Timeout: 180
      Policies:
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - 's3:List*'
              Resource: !Sub "arn:aws:s3:::${SourceBucket}"
        - SQSSendMessagePolicy:
            QueueName: !GetAtt S3PathSQS.QueueName
      Environment:
        Variables:
          sqsqueue: !Ref S3PathSQS
          s3bucket: !Ref SourceBucket
          lambdamode: "s3path"

  # option 2 - resources for Go Lambda using a signed S3 URL
  # for this scenario, the Lambda function needs to IAM access to the bucket, so it may be easier to use cross account

  # queue - create sqs queue
  SignedUrlSQS:
    Type: AWS::SQS::Queue

  # table - create a dynamodb table to store results
  SignedURLTable:
    Type: 'AWS::DynamoDB::Table'
    Properties:
      AttributeDefinitions:
      - AttributeName: Fileurl
        AttributeType: S
      - AttributeName: Md5
        AttributeType: S
      KeySchema:
      - AttributeName: Fileurl
        KeyType: HASH
      - AttributeName: Md5
        KeyType: RANGE
      BillingMode: PAY_PER_REQUEST

  # receiver - create s3 signed url lambda that receives sqs messages
  SignedUrlLambda:
    Type: 'AWS::Serverless::Function'
    Properties:
      CodeUri: s3lambda/
      Handler: s3lambda
      Tracing: Active
      Runtime: go1.x
      Events:
        SignedUrlEvent:
          Type: SQS
          Properties:
            Queue: !GetAtt SignedUrlSQS.Arn
            BatchSize: 10
      Description: 'Lambda function which retrieves S3 signed URLs submitted over SQS using a HTTP GET request'
      MemorySize: 256
      Timeout: 5
      Policies:
        - DynamoDBCrudPolicy:
            TableName: !Ref SignedURLTable
      Environment:
        Variables:
          ddbtable: !Ref SignedURLTable
          s3bucket: !Ref SourceBucket
          lambdamode: "s3signed"

  # sender - create a s3 signed url lambda load generator lambda
  SignedUrlLoadgenLambda:
    Type: 'AWS::Serverless::Function'
    Properties:
      CodeUri: s3loadgen/
      Handler: s3loadgen
      Tracing: Active
      Runtime: go1.x
      Description: 'Lambda function which generates signed S3 URLs and submits them to SQS'
      MemorySize: 512
      Timeout: 180
      Policies:
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - 's3:List*'
                - 's3:Get*'
              Resource: !Sub "arn:aws:s3:::${SourceBucket}"
        - Version: '2012-10-17'
          Statement:
            - Effect: Allow
              Action:
                - 's3:List*'
                - 's3:Get*'
              Resource: !Sub "arn:aws:s3:::${SourceBucket}/*"
        - SQSSendMessagePolicy:
            QueueName: !GetAtt SignedUrlSQS.QueueName
      Environment:
        Variables:
          sqsqueue: !Ref SignedUrlSQS
          s3bucket: !Ref SourceBucket
          lambdamode: "s3signed"