Skip to content

Commit 5c78147

Browse files
gfairesgfaires
authored
OpenSearch Serverless migration (#689)
* Move from Managed OpenSearch to OpenSearch Serverless * Add id field to similar items response * Implemented propagation wait time for Data Access policy * Ensured search service can still be run locally
1 parent 86ef0c2 commit 5c78147

File tree

22 files changed

+493
-204
lines changed

22 files changed

+493
-204
lines changed

‎aws/cloudformation-templates/base/_template.yaml

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ Resources:
162162
PinpointAppId: !Ref PinpointAppId
163163
GitHubUser: !Ref GitHubUser
164164
GitHubBranch: !Ref GitHubBranch
165+
ProductsCollectionArn: !GetAtt OpenSearch.Outputs.CollectionArn
165166

166167
# Tables
167168
Tables:
@@ -203,8 +204,8 @@ Resources:
203204
Condition: CreateOpenSearchServiceLinkedRole
204205
Type: AWS::IAM::ServiceLinkedRole
205206
Properties:
206-
AWSServiceName: opensearchservice.amazonaws.com
207-
Description: "Role for OpenSearch to access resources in VPC"
207+
AWSServiceName: observability.aoss.amazonaws.com
208+
Description: "Role for OpenSearch to put metrics in CloudWatch"
208209

209210
OpenSearchRoleWaitHandle:
210211
Condition: CreateOpenSearchServiceLinkedRole
@@ -232,7 +233,7 @@ Resources:
232233
TemplateURL: !Sub https://s3.amazonaws.com/${ResourceBucket}/${ResourceBucketRelativePath}cloudformation-templates/base/opensearch.yaml
233234
Parameters:
234235
VpcId: !GetAtt VPC.Outputs.VpcId
235-
Subnet1: !GetAtt VPC.Outputs.Subnet1
236+
PrivateSubnets: !GetAtt VPC.Outputs.Subnets
236237

237238
# SSM Parameters
238239
SSMParameters:
@@ -357,13 +358,13 @@ Outputs:
357358
Description: Service Discovery Namespace.
358359
Value: !GetAtt ServiceDiscovery.Outputs.ServiceDiscoveryNamespace
359360

360-
OpenSearchDomainEndpoint:
361-
Description: OpenSearch Domain
362-
Value: !GetAtt OpenSearch.Outputs.DomainEndpoint
361+
OpenSearchCollectionEndpoint:
362+
Description: OpenSearch Collection Endpoint
363+
Value: !GetAtt OpenSearch.Outputs.CollectionEndpoint
363364

364-
OpenSearchDomainArn:
365-
Description: OpenSearch Domain
366-
Value: !GetAtt OpenSearch.Outputs.DomainArn
365+
OpenSearchCollectionArn:
366+
Description: OpenSearch Collection Arn
367+
Value: !GetAtt OpenSearch.Outputs.CollectionArn
367368

368369
OpenSearchSecurityGroupId:
369370
Description: OpenSearch Security Group Id

‎aws/cloudformation-templates/base/notebook.yaml

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ Parameters:
4444

4545
PinpointAppId:
4646
Type: String
47+
48+
ProductsCollectionArn:
49+
Type: String
4750

4851
Conditions:
4952
UseDefaultGitHubUser: !Equals
@@ -146,10 +149,14 @@ Resources:
146149
Effect: "Allow"
147150
Action:
148151
- servicediscovery:DiscoverInstances
149-
- es:ListDomainNames
150-
- es:DescribeDomain
151-
- es:ListTags
152+
- aoss:ListCollections
152153
Resource: "*"
154+
-
155+
Effect: "Allow"
156+
Action:
157+
- aoss:APIAccessAll
158+
- aoss:BatchGetCollection
159+
Resource: !Ref ProductsCollectionArn
153160
-
154161
PolicyName: "1-Personalize"
155162
PolicyDocument:
@@ -370,6 +377,32 @@ Resources:
370377
- logs:CreateLogGroup
371378
Resource: '*'
372379

380+
DataAccessPolicy:
381+
Type: AWS::OpenSearchServerless::AccessPolicy
382+
Properties:
383+
Name: retaildemostore-notebook
384+
Type: data
385+
Description: Access policy to allow SageMaker Notebook access to the retail demo store collections
386+
Policy:
387+
!Sub |
388+
[{
389+
"Description": "Access for SageMaker notebook",
390+
"Rules": [
391+
{
392+
"ResourceType": "index",
393+
"Resource": [
394+
"index/retaildemostore-products/*"
395+
],
396+
"Permission": [
397+
"aoss:*"
398+
]
399+
}
400+
],
401+
"Principal": [
402+
"${ExecutionRole.Arn}"
403+
]
404+
}]
405+
373406
Outputs:
374407
NotebookInstanceId:
375408
Value: !Ref NotebookInstance

‎aws/cloudformation-templates/base/opensearch.yaml

Lines changed: 70 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -6,54 +6,14 @@ Description: >
66
77
Parameters:
88

9-
Subnet1:
9+
PrivateSubnets:
1010
Type: String
1111

1212
VpcId:
1313
Type: String
1414

1515
Resources:
1616

17-
18-
OpenSearchDomain:
19-
Type: AWS::OpenSearchService::Domain
20-
Properties:
21-
EngineVersion: OpenSearch_2.3
22-
ClusterConfig:
23-
InstanceCount: 1
24-
InstanceType: t3.small.search
25-
ZoneAwarenessEnabled: false
26-
DomainEndpointOptions:
27-
EnforceHTTPS: true
28-
NodeToNodeEncryptionOptions:
29-
Enabled: true
30-
EBSOptions:
31-
EBSEnabled: true
32-
Iops: 0
33-
VolumeSize: 10
34-
VolumeType: gp2
35-
AccessPolicies:
36-
Version: 2012-10-17
37-
Statement:
38-
- Effect: Allow
39-
Principal:
40-
AWS: '*'
41-
Action: 'es:*'
42-
Resource: '*'
43-
AdvancedOptions:
44-
rest.action.multi.allow_explicit_index: 'true'
45-
EncryptionAtRestOptions:
46-
Enabled: true
47-
VPCOptions:
48-
SubnetIds:
49-
- !Ref Subnet1
50-
SecurityGroupIds:
51-
- !GetAtt SecurityGroup.GroupId
52-
Tags:
53-
-
54-
Key: "Name"
55-
Value: "retaildemostore"
56-
5717
SecurityGroup:
5818
Type: AWS::EC2::SecurityGroup
5919
Properties:
@@ -64,13 +24,77 @@ Resources:
6424
IpProtocol: tcp
6525
ToPort: 443
6626
CidrIp: 0.0.0.0/0
27+
28+
AOSSVpcEndpoint:
29+
Type: AWS::OpenSearchServerless::VpcEndpoint
30+
Properties:
31+
Name: aoss-retaildemostore-vpcendpoint
32+
VpcId: !Ref VpcId
33+
SubnetIds: !Split [",", !Ref PrivateSubnets ]
34+
SecurityGroupIds:
35+
- !Ref SecurityGroup
36+
37+
EncryptionPolicy:
38+
Type: AWS::OpenSearchServerless::SecurityPolicy
39+
Properties:
40+
Name: retaildemostore-security-policy
41+
Type: encryption
42+
Description: Encryption policy for the retail demo store collections
43+
Policy:
44+
!Sub |
45+
{
46+
"Rules": [
47+
{
48+
"ResourceType": "collection",
49+
"Resource": [
50+
"collection/retaildemostore*"
51+
]
52+
}
53+
],
54+
"AWSOwnedKey": true
55+
}
56+
57+
NetworkPolicy:
58+
Type: AWS::OpenSearchServerless::SecurityPolicy
59+
Properties:
60+
Name: retaildemostore-network-policy
61+
Type: network
62+
Description: Network policy for the retail demo store collections
63+
Policy:
64+
!Sub |
65+
[{
66+
"Rules": [
67+
{
68+
"ResourceType": "collection",
69+
"Resource": [
70+
"collection/retaildemostore*"
71+
]
72+
}
73+
],
74+
"AllowFromPublic": false,
75+
"SourceVPCEs": [
76+
"${AOSSVpcEndpoint}"
77+
]
78+
}]
79+
80+
Collection:
81+
Type: AWS::OpenSearchServerless::Collection
82+
DependsOn: EncryptionPolicy
83+
Properties:
84+
Name: retaildemostore-products
85+
Type: SEARCH
86+
StandbyReplicas: DISABLED
87+
Description: Collection to hold product data
88+
Tags:
89+
- Key: "Name"
90+
Value: "retaildemostore"
6791

6892
Outputs:
69-
DomainArn:
70-
Value: !GetAtt OpenSearchDomain.DomainArn
93+
CollectionArn:
94+
Value: !GetAtt Collection.Arn
7195

72-
DomainEndpoint:
73-
Value: !GetAtt OpenSearchDomain.DomainEndpoint
96+
CollectionEndpoint:
97+
Value: !GetAtt Collection.CollectionEndpoint
7498

7599
SecurityGroupId:
76-
Value: !Ref SecurityGroup
100+
Value: !Ref SecurityGroup

‎aws/cloudformation-templates/deployment-support.yaml

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,7 @@ Parameters:
5050
- 'No'
5151
Default: 'No'
5252

53-
Subnet1:
54-
Type: String
55-
56-
Subnet2:
53+
Subnets:
5754
Type: String
5855

5956
LambdaVpcSecurityGroup:
@@ -62,10 +59,10 @@ Parameters:
6259
OpenSearchSecurityGroupId:
6360
Type: String
6461

65-
OpenSearchDomainArn:
62+
OpenSearchCollectionArn:
6663
Type: String
6764

68-
OpenSearchDomainEndpoint:
65+
OpenSearchCollectionEndpoint:
6966
Type: String
7067

7168
ParameterIVSVideoChannelMap:
@@ -172,9 +169,7 @@ Resources:
172169
VpcConfig:
173170
SecurityGroupIds:
174171
- !Ref LambdaVpcSecurityGroup
175-
SubnetIds:
176-
- !Ref Subnet1
177-
- !Ref Subnet2
172+
SubnetIds: !Split [",", !Ref Subnets ]
178173
Environment:
179174
Variables:
180175
ProductsServiceUrl: !Ref ProductsServiceExternalUrl
@@ -240,9 +235,7 @@ Resources:
240235
VpcConfig:
241236
SecurityGroupIds:
242237
- !Ref LambdaVpcSecurityGroup
243-
SubnetIds:
244-
- !Ref Subnet1
245-
- !Ref Subnet2
238+
SubnetIds: !Split [",", !Ref Subnets ]
246239
Environment:
247240
Variables:
248241
bucket: !Ref StackBucketName
@@ -489,63 +482,70 @@ Resources:
489482
VpcConfig:
490483
SecurityGroupIds:
491484
- !Ref OpenSearchSecurityGroupId
492-
SubnetIds:
493-
- !Ref Subnet1
494-
- !Ref Subnet2
485+
SubnetIds: !Split [",", !Ref Subnets ]
495486

496487
OpenSearchPreIndexLambdaExecutionRole:
497488
Condition: DeployPreIndexOpenSearch
498-
Type: 'AWS::IAM::Role'
489+
Type: AWS::IAM::Role
499490
Properties:
500491
AssumeRolePolicyDocument:
501492
Version: 2012-10-17
502493
Statement:
503494
- Effect: Allow
504495
Principal:
505-
Service:
506-
- lambda.amazonaws.com
507-
Action:
508-
- 'sts:AssumeRole'
509-
Path: /service-role/
510-
ManagedPolicyArns:
511-
- arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess
496+
Service: lambda.amazonaws.com
497+
Action: sts:AssumeRole
498+
Path: /
512499
Policies:
513-
- PolicyName: root
500+
- PolicyName: OpenSearchServerless
514501
PolicyDocument:
515502
Version: 2012-10-17
516503
Statement:
517504
- Effect: Allow
518-
Action:
519-
- logs:CreateLogGroup
520-
- ec2:CreateNetworkInterface
521-
- ec2:DeleteNetworkInterface
522-
- ec2:DescribeNetworkInterfaces
523-
Resource: '*'
524-
- Effect: Allow
525-
Action:
526-
- logs:CreateLogStream
527-
- logs:PutLogEvents
528-
Resource:
529-
- !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*OpenSearch*:log-stream:*'
530-
- !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*OpenSearch*'
531-
- Effect: Allow
532-
Action:
533-
- es:ESHttpDelete
534-
- es:ESHttpGet
535-
- es:ESHttpPost
536-
- es:ESHttpPut
537-
Resource: !Ref OpenSearchDomainArn
505+
Action: aoss:APIAccessAll
506+
Resource: !Ref OpenSearchCollectionArn
507+
ManagedPolicyArns:
508+
- arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess
509+
- arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
538510

539511
# Custom resource to launch opensearch preindex function
540512
CustomLaunchOpenSearchPreIndexLambdaFunction:
541513
Condition: DeployPreIndexOpenSearch
542514
Type: Custom::CustomLambdaOpenSearch
543515
Properties:
544516
ServiceToken: !GetAtt OpenSearchPreIndexLambdaFunction.Arn
545-
OpenSearchDomainEndpoint: !Ref OpenSearchDomainEndpoint
517+
OpenSearchCollectionEndpoint: !Ref OpenSearchCollectionEndpoint
546518
Bucket: !Ref ResourceBucket
547519
File: !Sub '${ResourceBucketRelativePath}data/products.yaml'
548520

521+
DataAccessPolicy:
522+
Type: AWS::OpenSearchServerless::AccessPolicy
523+
Condition: DeployPreIndexOpenSearch
524+
DependsOn: OpenSearchPreIndexLambdaExecutionRole
525+
Properties:
526+
Name: retaildemostore-index-lambda
527+
Type: data
528+
Description: !Sub Access policy to allow pre-index lambda access to retail demo store collections
529+
Policy:
530+
!Sub |
531+
[{
532+
"Description": "Access for Pre-index lambda",
533+
"Rules": [
534+
{
535+
"ResourceType": "index",
536+
"Resource": [
537+
"index/retaildemostore-products/*"
538+
],
539+
"Permission": [
540+
"aoss:*"
541+
]
542+
}
543+
],
544+
"Principal": [
545+
"${OpenSearchPreIndexLambdaExecutionRole.Arn}"
546+
]
547+
}]
548+
549549
####################### Pre-Create Pinpoint Workshop #######################
550550

551551
PinpointPreCreateLambdaFunction:

0 commit comments

Comments
 (0)