Create role and policy used to deploy CDP environments for CDE
The Cloudera Data Engineering (CDE) control plane requires a role and policies to create CDP environments. In this step, you create a common policy for creating environments, as well as a policy that is specific to CDE environments.
The following two policies are created in this step:
- Compute infrastructure restricted IAM policy - A common policy for all data services deployed on CDP.
- CDE restricted IAM policy - A policy with additional permissions for CDE.
There are two options for the timing of attaching the role: during environment creation, or prior to enabling the CML data service.
Option #1: During environment creation
The Cloudbreak environment creation UI should be set up as shown here:
Option #2: Prior to enabling CDE data service
If the Cloudbreak environment has already been created, you can create and attach the Compute infrastructure Restricted IAM policy and CDE restricted IAM policy to the existing cross-account role associated with the environment.
To view the existing cross-account role, in the Environments section of the CDP management console, on the Summary tab, see Credentials.
Compute (Liftie) Restricted IAM policy
Replace the following placeholders in the JSON file:
- [YOUR-ACCOUNT-ID] with your account ID in use.
- [YOUR-IAM-ROLE-NAME] with the IAM restricted role associated with this policy.
- [YOUR-SUBNET-ARN-*] supplied during the Cloudbreak Environment(s) creation. Note: Please provide all the subnets present in all the Cloudbreak Environment(s) that you intend to use it for the experience. If at any point a new Cloudbreak Environment is created or an existing one is updated for subnets, the same should be updated here.
- [YOUR-IDBROKER-ROLE-NAME] with the ID Broker Role name in use.
- [YOUR-LOG-ROLE-NAME] with the Log Role name in use.
- [YOUR-KMS-CUSTOMER-MANAGED-KEY-ARN] with KMS key ARN.
- [YOUR-ACCOUNT-REGION] with the AWS region.
{
"Version":"2012-10-17",
"Id":"ComputePolicy_v9",
"Statement":[
{
"Sid":"SimulatePrincipalPolicy",
"Effect":"Allow",
"Action":[
"iam:SimulatePrincipalPolicy"
],
"Resource":[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/[YOUR-IAM-ROLE-NAME]"
]
},
{
"Sid":"RestrictedPermissionsViaClouderaRequestTag",
"Effect":"Allow",
"Action":[
"cloudformation:CreateStack",
"cloudformation:CreateChangeSet",
"ec2:createTags",
"eks:TagResource"
],
"Resource":"*",
"Condition":{
"StringLike":{
"aws:RequestTag/Cloudera-Resource-Name":[
"crn:cdp:*"
]
}
}
},
{
"Sid":"RestrictedPermissionsViaClouderaResourceTag",
"Effect":"Allow",
"Action":[
"autoscaling:DetachInstances",
"autoscaling:ResumeProcesses",
"autoscaling:SetDesiredCapacity",
"autoscaling:SuspendProcesses",
"autoscaling:UpdateAutoScalingGroup",
"autoscaling:DeleteTags",
"autoscaling:TerminateInstanceInAutoScalingGroup",
"cloudformation:DeleteStack",
"cloudformation:DescribeStacks"
],
"Resource":"*",
"Condition":{
"StringLike":{
"aws:ResourceTag/Cloudera-Resource-Name":[
"crn:cdp:*"
]
}
}
},
{
"Sid":"RestrictedPermissionsViaCloudFormation",
"Effect":"Allow",
"Action":[
"ec2:CreateSecurityGroup",
"ec2:DeleteSecurityGroup",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:RevokeSecurityGroupIngress",
"ec2:AuthorizeSecurityGroupEgress",
"ec2:RevokeSecurityGroupEgress",
"ec2:CreateLaunchTemplate",
"ec2:DeleteLaunchTemplate",
"autoscaling:CreateAutoScalingGroup",
"autoscaling:DeleteAutoScalingGroup",
"autoscaling:CreateOrUpdateTags",
"autoscaling:CreateLaunchConfiguration",
"eks:CreateCluster",
"eks:DeleteCluster"
],
"Resource":"*",
"Condition":{
"ForAnyValue:StringEquals":{
"aws:CalledVia":[
"cloudformation.amazonaws.com"
]
}
}
},
{
"Sid":"RestrictedEC2PermissionsViaClouderaResourceTag",
"Effect":"Allow",
"Action":[
"ec2:RebootInstances",
"ec2:StartInstances",
"ec2:StopInstances",
"ec2:TerminateInstances"
],
"Resource":[
"*"
],
"Condition":{
"ForAnyValue:StringLike":{
"ec2:ResourceTag/Cloudera-Resource-Name":[
"crn:cdp:*"
]
}
}
},
{
"Sid":"RestrictedIamPermissionsToClouderaResources",
"Effect":"Allow",
"Action":[
"iam:PassRole"
],
"Resource":[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/[YOUR-IDBROKER-ROLE-NAME]",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/[YOUR-LOG-ROLE-NAME]",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/liftie-*-eks-service-role",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/liftie-*-eks-worker-nodes",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/cdp-eks-master-role",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/cdp-liftie-instance-profile"
]
},
{
"Sid":"RestrictedKMSPermissionsUsingCustomerProvidedKey",
"Effect":"Allow",
"Action":[
"kms:CreateGrant",
"kms:DescribeKey",
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*"
],
"Resource":[
"[YOUR-KMS-CUSTOMER-MANAGED-KEY-ARN]"
]
},
{
"Sid": "AllowCreateDeleteTagsForSubnets",
"Effect": "Allow",
"Action": [
"ec2:CreateTags",
"ec2:DeleteTags"
],
"Resource": [
"[YOUR-SUBNET-ARN-1]",
"[YOUR-SUBNET-ARN-2]"
....
]
},
{
"Sid": "OtherPermissionsViaCloudFormation",
"Effect": "Allow",
"Action": [
"autoscaling:DescribeScheduledActions",
"autoscaling:DescribeTags",
"autoscaling:DescribeAutoScalingInstances",
"autoscaling:DescribeLaunchConfigurations",
"autoscaling:DeleteLaunchConfiguration",
"autoscaling:DescribeScalingActivities",
"dynamodb:DescribeTable",
"ec2:DeletePlacementGroup",
"ec2:DescribeAccountAttributes",
"ec2:DescribeImages",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeLaunchTemplates",
"ec2:DescribePlacementGroups",
"ec2:DescribeRegions",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeVolumes"
],
"Resource": [
"*"
],
"Condition":{
"ForAnyValue:StringEquals":{
"aws:CalledVia":[
"cloudformation.amazonaws.com"
]
}
}
},
{
"Sid": "OtherPermissionsViaClouderaResourceTag",
"Effect": "Allow",
"Action": [
"cloudformation:DescribeChangeSet",
"cloudformation:DeleteChangeSet",
"cloudformation:ExecuteChangeSet",
"cloudformation:CancelUpdateStack",
"cloudformation:ContinueUpdateRollback",
"cloudformation:ListStacks",
"cloudformation:DescribeStackEvents",
"cloudformation:DescribeStackResource",
"cloudformation:DescribeStackResources",
"cloudwatch:deleteAlarms",
"cloudwatch:putMetricAlarm",
"logs:DescribeLogStreams",
"logs:FilterLogEvents",
"ec2:AttachVolume",
"ec2:CreateNetworkInterface",
"ec2:CreateVolume",
"ec2:DeleteVolume",
"ec2:ModifyInstanceAttribute",
"ec2:RunInstances",
"eks:ListUpdates",
"eks:UpdateClusterConfig",
"eks:UpdateClusterVersion",
"eks:DescribeUpdate",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRoleTags",
"iam:RemoveRoleFromInstanceProfile",
"iam:TagRole",
"iam:UntagRole"
],
"Resource": [
"*"
],
"Condition":{
"StringLike":{
"aws:ResourceTag/Cloudera-Resource-Name":[
"crn:cdp:*"
]
}
}
},
{
"Sid":"OtherPermissions",
"Effect":"Allow",
"Action":[
"autoscaling:DescribeAutoScalingGroups",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CreateLaunchTemplateVersion",
"ec2:CreatePlacementGroup",
"ec2:DeleteKeyPair",
"ec2:DeleteNetworkInterface",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeInstanceTypes",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcs",
"ec2:ImportKeyPair",
"ec2:UpdateSecurityGroupRuleDescriptionsIngress",
"ec2:GetInstanceTypesFromInstanceRequirements",
"eks:DescribeCluster",
"elasticloadbalancing:DescribeLoadBalancers",
"iam:GetRole",
"iam:ListRoles",
"iam:GetInstanceProfile"
],
"Resource":[
"*"
]
},
{
"Sid":"SSMPermissions",
"Effect":"Allow",
"Action":[
"ssm:DescribeParameters",
"ssm:GetParameter",
"ssm:GetParameters",
"ssm:GetParameterHistory",
"ssm:GetParametersByPath"
],
"Resource":[
"arn:aws:ssm:*:*:parameter/aws/service/eks/optimized-ami/*"
]
},
{
"Sid":"CfDeny",
"Effect":"Deny",
"Action":[
"cloudformation:*"
],
"Resource":[
"*"
],
"Condition":{
"ForAnyValue:StringLike":{
"cloudformation:ImportResourceTypes":[
"*"
]
}
}
},
{
"Sid":"ForAutoscalingLinkedRole",
"Effect":"Allow",
"Action":[
"iam:CreateServiceLinkedRole"
],
"Resource":[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/aws-service-role/autoscaling-plans.amazonaws.com/AWSServiceRoleForAutoScalingPlans_EC2AutoScaling"
],
"Condition":{
"StringLike":{
"iam:AWSServiceName":"autoscaling-plans.amazonaws.com"
}
}
},
{
"Sid":"ForEksLinkedRole",
"Effect":"Allow",
"Action":[
"iam:CreateServiceLinkedRole"
],
"Resource":[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/aws-service-role/eks.amazonaws.com/AWSServiceRoleForEKS"
],
"Condition":{
"StringLike":{
"iam:AWSServiceName":"eks.amazonaws.com"
}
}
}
]
}
Supporting Customer Managed CMKs
Along with providing the KMS Customer Managed Customer Master Key (CMK) for volume encryption
in the policy section with Sid:
RestrictedKMSPermissionsUsingCustomerProvidedKey, you need to verify that the policy
for the Customer Managed Customer Master Key (CMK) at KMS (this is not an IAM policy) has the
following two permission blocks defined for AWSServiceRoleForAutoScaling:
{
"Sid": "AllowAutoscalingAndCDPCrossAccountRoleUseOfTheCMK",
"Effect": "Allow",
"Principal":
{
"AWS":
[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/CDP-CROSSACCOUNT-ROLE",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"
]
},
"Action":
[
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*"
},
{
"Sid": "AllowAutoscalingAndCDPCrossAccountRoleToCreateGrantsOfTheCMK",
"Effect": "Allow",
"Principal":
{
"AWS":
[
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/CDP-CROSSACCOUNT-ROLE",
"arn:aws:iam::[YOUR-ACCOUNT-ID]:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling"
]
},
"Action":
[
"kms:CreateGrant"
],
"Resource": "*",
"Condition":
{
"Bool":
{
"kms:GrantIsForAWSResource": "true"
}
}
},
{
"Sid": "AllowCreateGrantToLiftieCluster",
"Effect": "Allow",
"Principal":
{
"AWS": "arn:aws:iam::[YOUR-ACCOUNT-ID]:role/CDP-CROSSACCOUNT-ROLE"
},
"Action": "kms:CreateGrant",
"Resource": "*",
"Condition":
{
"StringEquals":
{
"aws:CalledViaFirst": "cloudformation.amazonaws.com"
},
"ForAllValues:StringEquals":
{
"kms:GrantOperations":
[
"Encrypt",
"Decrypt"
]
}
}
},
{
"Sid": "AllowEKSAccessToEBS",
"Effect": "Allow",
"Principal":
{
"AWS": "*"
},
"Action":
[
"kms:CreateGrant",
"kms:Encrypt",
"kms:Decrypt",
"kms:ReEncrypt*",
"kms:GenerateDataKey*",
"kms:DescribeKey"
],
"Resource": "*",
"Condition":
{
"StringEquals":
{
"kms:CallerAccount": "[YOUR-ACCOUNT-ID]",
"kms:viaService": "ec2.[YOUR-ACCOUNT-REGION].amazonaws.com"
}
}
}After the policy is attached, the KMS service page will show the CMS as having the policy attached, similar to this screen shot:
CDE restricted IAM policy
Replace the following placeholders in the JSON file:
- [YOUR-ACCOUNT-ID] with your account ID in use.
- [YOUR-IAM-ROLE-NAME] with the IAM restricted role with which this policy would be associated with.
{
"Id": "CDEPolicy_v2",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "ElasticFileSystem",
"Action": [
"elasticfilesystem:CreateMountTarget",
"elasticfilesystem:DescribeFileSystems",
"elasticfilesystem:TagResource",
"elasticfilesystem:ClientMount"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Sid": "ElasticFileSystemRequest",
"Action": [
"elasticfilesystem:CreateFileSystem",
"elasticfilesystem:CreateTags"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Condition": {
"StringLike": {
"aws:RequestTag/Cloudera-Resource-Name": [
"crn:cdp:de:*"
]
}
}
},
{
"Sid": "ElasticFileSystemResource",
"Action": [
"elasticfilesystem:DescribeFileSystemPolicy",
"elasticfilesystem:DeleteFileSystem",
"elasticfilesystem:DescribeMountTargets",
"elasticfilesystem:DeleteMountTarget",
"elasticfilesystem:PutFileSystemPolicy"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/Cloudera-Resource-Name": [
"crn:cdp:de:*"
]
}
}
},
{
"Sid": "CloudWatch",
"Action": [
"cloudwatch:GetMetricData"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Sid": "ElasticLoadBalancing",
"Action": [
"elasticloadbalancing:DescribeTags"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
{
"Sid": "RelationalDatabaseServiceRequest",
"Action": [
"rds:CreateDBCluster",
"rds:CreateDBSubnetGroup",
"rds:AddTagsToResource",
"rds:CreateDBInstance"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Condition": {
"StringLike": {
"aws:RequestTag/Cloudera-Resource-Name": [
"crn:cdp:de:*"
]
}
}
},
{
"Sid": "RelationalDatabaseServiceResource",
"Action": [
"rds:DeleteDBSubnetGroup",
"rds:DescribeDBInstances",
"rds:ModifyDBInstance",
"rds:DeleteDBInstance"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Condition": {
"StringLike": {
"aws:ResourceTag/Cloudera-Resource-Name": [
"crn:cdp:de:*"
]
}
}
},
{
"Sid": "RelationalDatabaseService",
"Action": [
"rds:DescribeDBEngineVersions"
],
"Effect": "Allow",
"Resource": [
"*"
]
}
]
}
