Cluster template structure
The template below is an example of a 7.2.2 Data Engineering HA cluster template in JSON view.
Many of the default templates contain properties for services that aren't displayed when you select a default template in the user interface, for example Hive on Tez or Sqoop in the default Data Engineering HA template below. If you are creating a custom template and you need to know more about the properties and services in a template, refer to the Cloudera Manager configuration properties for each Cloudera Runtime service. These properties can be helpful if you are trying to tune Yarn or Hive, for example. You can also copy a service like Sqoop from the JSON of one template to another.
{
"cdhVersion": "7.2.2",
"displayName": "dataengineering ha",
"services": [
{
"refName": "zookeeper",
"serviceType": "ZOOKEEPER",
"roleConfigGroups": [
{
"refName": "zookeeper-SERVER-BASE",
"roleType": "SERVER",
"base": true
}
]
},
{
"refName": "hdfs",
"serviceType": "HDFS",
"serviceConfigs": [
{
"name": "zookeeper_service",
"ref": "zookeeper"
},
{
"name": "hdfs_verify_ec_with_topology_enabled",
"value": false
}
],
"roleConfigGroups": [
{
"refName": "hdfs-NAMENODE-BASE",
"roleType": "NAMENODE",
"base": true,
"configs": [
{
"name": "fs_trash_interval",
"value": "0"
},
{
"name": "fs_trash_checkpoint_interval",
"value": "0"
},
{
"name": "erasure_coding_default_policy",
"value": " "
}
]
},
{
"refName": "hdfs-FAILOVERCONTROLLER-BASE",
"roleType": "FAILOVERCONTROLLER",
"base": true
},
{
"refName": "hdfs-JOURNALNODE-BASE",
"roleType": "JOURNALNODE",
"base": true
},
{
"refName": "hdfs-DATANODE-BASE",
"roleType": "DATANODE",
"base": true
},
{
"refName": "hdfs-SECONDARYNAMENODE-BASE",
"roleType": "SECONDARYNAMENODE",
"configs": [
{
"name": "fs_checkpoint_dir_list",
"value": "/should_not_be_required_in_HA_setup"
}
],
"base": true
},
{
"refName": "hdfs-BALANCER-BASE",
"roleType": "BALANCER",
"base": true
},
{
"refName": "hdfs-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true,
"configs": [
{
"name": "dfs_client_use_trash",
"value": false
},
{
"name": "hdfs_client_env_safety_valve",
"value": "HADOOP_OPTS=\"-Dorg.wildfly.openssl.path=/usr/lib64 ${HADOOP_OPTS}\""
}
]
}
]
},
{
"refName": "hms",
"serviceType": "HIVE",
"displayName": "Hive Metastore",
"roleConfigGroups": [
{
"refName": "hms-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true
},
{
"refName": "hms-HIVEMETASTORE-BASE",
"roleType": "HIVEMETASTORE",
"base": true
}
]
},
{
"refName": "hive",
"serviceType": "HIVE_ON_TEZ",
"displayName": "Hive",
"serviceConfigs": [
{
"name": "tez_container_size",
"value": "4096"
},
{
"name": "tez_auto_reducer_parallelism",
"value": "false"
},
{
"name": "hive_service_config_safety_valve",
"value": "<property><name>fs.s3a.ssl.channel.mode</name><value>openssl</value></property><property><name>hive.txn.acid.dir.cache.duration</name><value>0</value></property>"
}
],
"roleConfigGroups": [
{
"refName": "hive-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true
},
{
"refName": "hive-HIVESERVER2-BASE",
"roleType": "HIVESERVER2",
"base": true,
"configs": [
{
"name": "hive_server2_transport_mode",
"value": "http"
},
{
"name": "hiveserver2_mv_files_thread",
"value": 20
},
{
"name": "hiveserver2_load_dynamic_partitions_thread_count",
"value": 20
}
]
}
]
},
{
"refName": "hue",
"serviceType": "HUE",
"serviceConfigs": [
{
"name": "hue_service_safety_valve",
"value": "[desktop]\napp_blacklist=spark,zookeeper,hbase,impala,search,sqoop,security,pig"
}
],
"roleConfigGroups": [
{
"refName": "hue-HUE_SERVER-BASE",
"roleType": "HUE_SERVER",
"base": true
},
{
"refName": "hue-HUE_LOAD_BALANCER-BASE",
"roleType": "HUE_LOAD_BALANCER",
"base": true
}
]
},
{
"refName": "livy",
"serviceType": "LIVY",
"roleConfigGroups": [
{
"refName": "livy-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true
},
{
"refName": "livy-LIVY_SERVER-BASE",
"roleType": "LIVY_SERVER",
"base": true
}
]
},
{
"refName": "oozie",
"serviceType": "OOZIE",
"roleConfigGroups": [
{
"refName": "oozie-OOZIE_SERVER-BASE",
"roleType": "OOZIE_SERVER",
"configs": [
{
"name": "oozie_config_safety_valve",
"value": "<property><name>oozie.service.HadoopAccessorService.nameNode.whitelist</name><value></value></property>"
}
],
"base": true
}
]
},
{
"refName": "sqoop",
"serviceType": "SQOOP_CLIENT",
"roleConfigGroups": [
{
"refName": "sqoop-SQOOP_CLIENT-GATEWAY-BASE",
"roleType": "GATEWAY",
"configs": [],
"base": true
}
]
},
{
"refName": "yarn",
"serviceType": "YARN",
"serviceConfigs": [
{
"name": "yarn_admin_acl",
"value": "yarn,hive,hdfs,mapred"
}
],
"roleConfigGroups": [
{
"refName": "yarn-RESOURCEMANAGER-BASE",
"roleType": "RESOURCEMANAGER",
"base": true,
"configs": [
{
"name": "resourcemanager_config_safety_valve",
"value": "<property><name>yarn.scheduler.configuration.store.class</name><value>zk</value></property>"
},
{
"name": "yarn_resourcemanager_scheduler_class",
"value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"
},
{
"name": "yarn_scheduler_capacity_resource_calculator",
"value": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator"
},
{
"name": "resourcemanager_capacity_scheduler_configuration",
"value": "<configuration><property><name>yarn.scheduler.capacity.root.queues</name><value>default</value></property><property><name>yarn.scheduler.capacity.root.capacity</name><value>100</value></property><property><name>yarn.scheduler.capacity.root.default.capacity</name><value>100</value></property><property><name>yarn.scheduler.capacity.root.acl_submit_applications</name><value> </value></property><property><name>yarn.scheduler.capacity.root.acl_administer_queue</name><value> </value></property><property><name>yarn.scheduler.capacity.root.default.acl_submit_applications</name><value>*</value></property><property><name>yarn.scheduler.capacity.root.default.minimum-user-limit-percent</name><value>100</value></property><property><name>yarn.scheduler.capacity.maximum-am-resource-percent</name><value>0.33</value></property><property><name>yarn.scheduler.capacity.node-locality-delay</name><value>0</value></property><property><name>yarn.scheduler.capacity.schedule-asynchronously.maximum-threads</name><value>1</value></property><property><name>yarn.scheduler.capacity.schedule-asynchronously.scheduling-interval-ms</name><value>10</value></property></configuration>"
}
]
},
{
"refName": "yarn-NODEMANAGER-WORKER",
"roleType": "NODEMANAGER",
"base": false
},
{
"refName": "yarn-NODEMANAGER-COMPUTE",
"roleType": "NODEMANAGER",
"base": false
},
{
"refName": "yarn-JOBHISTORY-BASE",
"roleType": "JOBHISTORY",
"base": true
},
{
"refName": "yarn-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true,
"configs": [
{
"name": "mapreduce_map_memory_mb",
"value": 4096
},
{
"name": "mapreduce_reduce_memory_mb",
"value": 4096
},
{
"name": "mapreduce_client_env_safety_valve",
"value": "HADOOP_OPTS=\"-Dorg.wildfly.openssl.path=/usr/lib64 ${HADOOP_OPTS}\""
}
]
}
]
},
{
"refName": "spark_on_yarn",
"serviceType": "SPARK_ON_YARN",
"roleConfigGroups": [
{
"refName": "spark_on_yarn-SPARK_YARN_HISTORY_SERVER-BASE",
"roleType": "SPARK_YARN_HISTORY_SERVER",
"base": true
},
{
"refName": "spark_on_yarn-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true,
"configs": [
{
"name": "spark-conf/spark-defaults.conf_client_config_safety_valve",
"value": "spark.hadoop.fs.s3a.ssl.channel.mode=openssl"
}
]
}
]
},
{
"refName": "tez",
"serviceType": "TEZ",
"serviceConfigs": [
{
"name": "yarn_service",
"ref": "yarn"
},
{
"name": "tez.am.container.reuse.non-local-fallback.enabled",
"value": "true"
},
{
"name": "tez.am.container.reuse.locality.delay-allocation-millis",
"value": "0"
},
{
"name": "tez.am.launch.cmd-opts",
"value": "-XX:+PrintGCDetails -verbose:gc -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
},
{
"name": "tez.task.launch.cmd-opts",
"value": "-XX:+PrintGCDetails -verbose:gc -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
},
{
"name": "tez.grouping.split-waves",
"value": 1.4
},
{
"name": "tez.grouping.min-size",
"value": 268435456
},
{
"name": "tez.grouping.max-size",
"value": 268435456
}
],
"roleConfigGroups": [
{
"refName": "tez-GATEWAY-BASE",
"roleType": "GATEWAY",
"base": true
}
]
},
{
"refName": "das",
"serviceType": "DAS",
"roleConfigGroups": [
{
"refName": "das-DAS_WEBAPP",
"roleType": "DAS_WEBAPP",
"base": true,
"configs": [
{
"name": "data_analytics_studio_admin_users",
"value": "*"
}
]
},
{
"refName": "das-DAS_EVENT_PROCESSOR",
"roleType": "DAS_EVENT_PROCESSOR",
"base": true
}
]
},
{
"refName": "knox",
"roleConfigGroups": [
{
"base": true,
"refName": "knox-KNOX_GATEWAY-BASE",
"roleType": "KNOX_GATEWAY"
}
],
"serviceType": "KNOX"
},
{
"refName": "zeppelin",
"serviceType": "ZEPPELIN",
"serviceConfigs": [
{
"name": "yarn_service",
"ref": "yarn"
},
{
"name": "hdfs_service",
"ref": "hdfs"
},
{
"name": "spark_on_yarn_service",
"ref": "spark_on_yarn"
}
],
"roleConfigGroups": [
{
"refName": "zeppelin-ZEPPELIN_SERVER-BASE",
"roleType": "ZEPPELIN_SERVER",
"base": true
}
]
},
{
"refName": "queuemanager",
"serviceType": "QUEUEMANAGER",
"serviceConfigs": [
{
"name": "kerberos.auth.enabled",
"value": "true"
}
],
"roleConfigGroups": [
{
"refName": "yarn-QUEUEMANAGER_WEBAPP-BASE",
"roleType": "QUEUEMANAGER_WEBAPP",
"base": true
},
{
"refName": "yarn-QUEUEMANAGER_STORE-BASE",
"roleType": "QUEUEMANAGER_STORE",
"base": true
}
]
}
],
"hostTemplates": [
{
"refName": "gateway",
"cardinality": 0,
"roleConfigGroupsRefNames": [
"hdfs-GATEWAY-BASE",
"hive-GATEWAY-BASE",
"hms-GATEWAY-BASE",
"livy-GATEWAY-BASE",
"spark_on_yarn-GATEWAY-BASE",
"tez-GATEWAY-BASE",
"yarn-GATEWAY-BASE"
]
},
{
"refName": "master",
"cardinality": 2,
"roleConfigGroupsRefNames": [
"hdfs-FAILOVERCONTROLLER-BASE",
"hdfs-NAMENODE-BASE",
"hdfs-GATEWAY-BASE",
"hive-GATEWAY-BASE",
"hive-HIVESERVER2-BASE",
"hms-GATEWAY-BASE",
"hms-HIVEMETASTORE-BASE",
"hue-HUE_LOAD_BALANCER-BASE",
"hue-HUE_SERVER-BASE",
"knox-KNOX_GATEWAY-BASE",
"livy-GATEWAY-BASE",
"spark_on_yarn-GATEWAY-BASE",
"tez-GATEWAY-BASE",
"yarn-RESOURCEMANAGER-BASE",
"hdfs-JOURNALNODE-BASE",
"zookeeper-SERVER-BASE",
"yarn-GATEWAY-BASE"
]
},
{
"refName": "manager",
"cardinality": 1,
"roleConfigGroupsRefNames": [
"hdfs-BALANCER-BASE",
"hdfs-GATEWAY-BASE",
"hive-GATEWAY-BASE",
"hms-GATEWAY-BASE",
"livy-GATEWAY-BASE",
"livy-LIVY_SERVER-BASE",
"spark_on_yarn-GATEWAY-BASE",
"spark_on_yarn-SPARK_YARN_HISTORY_SERVER-BASE",
"tez-GATEWAY-BASE",
"yarn-JOBHISTORY-BASE",
"oozie-OOZIE_SERVER-BASE",
"sqoop-SQOOP_CLIENT-GATEWAY-BASE",
"zeppelin-ZEPPELIN_SERVER-BASE",
"das-DAS_EVENT_PROCESSOR",
"das-DAS_WEBAPP",
"knox-KNOX_GATEWAY-BASE",
"hdfs-JOURNALNODE-BASE",
"zookeeper-SERVER-BASE",
"yarn-QUEUEMANAGER_WEBAPP-BASE",
"yarn-QUEUEMANAGER_STORE-BASE",
"yarn-GATEWAY-BASE"
]
},
{
"refName": "worker",
"cardinality": 3,
"roleConfigGroupsRefNames": [
"hdfs-DATANODE-BASE",
"hdfs-GATEWAY-BASE",
"hive-GATEWAY-BASE",
"hms-GATEWAY-BASE",
"livy-GATEWAY-BASE",
"spark_on_yarn-GATEWAY-BASE",
"tez-GATEWAY-BASE",
"yarn-NODEMANAGER-WORKER",
"yarn-GATEWAY-BASE"
]
},
{
"refName": "compute",
"cardinality": 0,
"roleConfigGroupsRefNames": [
"hdfs-GATEWAY-BASE",
"hive-GATEWAY-BASE",
"hms-GATEWAY-BASE",
"livy-GATEWAY-BASE",
"spark_on_yarn-GATEWAY-BASE",
"tez-GATEWAY-BASE",
"yarn-NODEMANAGER-COMPUTE",
"yarn-GATEWAY-BASE"
]
}
]
}
