Cluster template structure

The template below is an example of a 7.2.2 Data Engineering HA cluster template in JSON view.

Many of the default templates contain properties for services that aren't displayed when you select a default template in the user interface, for example Hive on Tez or Sqoop in the default Data Engineering HA template below. If you are creating a custom template and you need to know more about the properties and services in a template, refer to the Cloudera Manager configuration properties for each Cloudera Runtime service. These properties can be helpful if you are trying to tune Yarn or Hive, for example. You can also copy a service like Sqoop from the JSON of one template to another.

{
  "cdhVersion": "7.2.2",
  "displayName": "dataengineering ha",
  "services": [
    {
      "refName": "zookeeper",
      "serviceType": "ZOOKEEPER",
      "roleConfigGroups": [
        {
          "refName": "zookeeper-SERVER-BASE",
          "roleType": "SERVER",
          "base": true
        }
      ]
    },
    {
      "refName": "hdfs",
      "serviceType": "HDFS",
      "serviceConfigs": [
        {
          "name": "zookeeper_service",
          "ref": "zookeeper"
        },
        {
          "name": "hdfs_verify_ec_with_topology_enabled",
          "value": false
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "hdfs-NAMENODE-BASE",
          "roleType": "NAMENODE",
          "base": true,
          "configs": [
            {
              "name": "fs_trash_interval",
              "value": "0"
            },
            {
              "name": "fs_trash_checkpoint_interval",
              "value": "0"
            },
            {
              "name": "erasure_coding_default_policy",
              "value": " "
            }
          ]
        },
        {
          "refName": "hdfs-FAILOVERCONTROLLER-BASE",
          "roleType": "FAILOVERCONTROLLER",
          "base": true
        },
        {
          "refName": "hdfs-JOURNALNODE-BASE",
          "roleType": "JOURNALNODE",
          "base": true
        },
        {
          "refName": "hdfs-DATANODE-BASE",
          "roleType": "DATANODE",
          "base": true
        },
        {
          "refName": "hdfs-SECONDARYNAMENODE-BASE",
          "roleType": "SECONDARYNAMENODE",
          "configs": [
            {
              "name": "fs_checkpoint_dir_list",
              "value": "/should_not_be_required_in_HA_setup"
            }
          ],
          "base": true
        },
        {
          "refName": "hdfs-BALANCER-BASE",
          "roleType": "BALANCER",
          "base": true
        },
        {
          "refName": "hdfs-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true,
          "configs": [
            {
              "name": "dfs_client_use_trash",
              "value": false
            },
            {
              "name": "hdfs_client_env_safety_valve",
              "value": "HADOOP_OPTS=\"-Dorg.wildfly.openssl.path=/usr/lib64 ${HADOOP_OPTS}\""
            }
          ]
        }
      ]
    },
    {
      "refName": "hms",
      "serviceType": "HIVE",
      "displayName": "Hive Metastore",
      "roleConfigGroups": [
        {
          "refName": "hms-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true
        },
        {
          "refName": "hms-HIVEMETASTORE-BASE",
          "roleType": "HIVEMETASTORE",
          "base": true
        }
      ]
    },
    {
      "refName": "hive",
      "serviceType": "HIVE_ON_TEZ",
      "displayName": "Hive",
      "serviceConfigs": [
        {
          "name": "tez_container_size",
          "value": "4096"
        },
        {
          "name": "tez_auto_reducer_parallelism",
          "value": "false"
        },
        {
          "name": "hive_service_config_safety_valve",
          "value": "<property><name>fs.s3a.ssl.channel.mode</name><value>openssl</value></property><property><name>hive.txn.acid.dir.cache.duration</name><value>0</value></property>"
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "hive-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true
        },
        {
          "refName": "hive-HIVESERVER2-BASE",
          "roleType": "HIVESERVER2",
          "base": true,
          "configs": [
            {
              "name": "hive_server2_transport_mode",
              "value": "http"
            },
            {
              "name": "hiveserver2_mv_files_thread",
              "value": 20
            },
            {
              "name": "hiveserver2_load_dynamic_partitions_thread_count",
              "value": 20
            }
          ]
        }
      ]
    },
    {
      "refName": "hue",
      "serviceType": "HUE",
      "serviceConfigs": [
        {
          "name": "hue_service_safety_valve",
          "value": "[desktop]\napp_blacklist=spark,zookeeper,hbase,impala,search,sqoop,security,pig"
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "hue-HUE_SERVER-BASE",
          "roleType": "HUE_SERVER",
          "base": true
        },
        {
          "refName": "hue-HUE_LOAD_BALANCER-BASE",
          "roleType": "HUE_LOAD_BALANCER",
          "base": true
        }
      ]
    },
    {
      "refName": "livy",
      "serviceType": "LIVY",
      "roleConfigGroups": [
        {
          "refName": "livy-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true
        },
        {
          "refName": "livy-LIVY_SERVER-BASE",
          "roleType": "LIVY_SERVER",
          "base": true
        }
      ]
    },
    {
      "refName": "oozie",
      "serviceType": "OOZIE",
      "roleConfigGroups": [
        {
          "refName": "oozie-OOZIE_SERVER-BASE",
          "roleType": "OOZIE_SERVER",
          "configs": [
            {
              "name": "oozie_config_safety_valve",
              "value": "<property><name>oozie.service.HadoopAccessorService.nameNode.whitelist</name><value></value></property>"
            }
          ],
          "base": true
        }
      ]
    },
    {
      "refName": "sqoop",
      "serviceType": "SQOOP_CLIENT",
      "roleConfigGroups": [
        {
          "refName": "sqoop-SQOOP_CLIENT-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "configs": [],
          "base": true
        }
      ]
    },
    {
      "refName": "yarn",
      "serviceType": "YARN",
      "serviceConfigs": [
        {
          "name": "yarn_admin_acl",
          "value": "yarn,hive,hdfs,mapred"
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "yarn-RESOURCEMANAGER-BASE",
          "roleType": "RESOURCEMANAGER",
          "base": true,
          "configs": [
            {
              "name": "resourcemanager_config_safety_valve",
              "value": "<property><name>yarn.scheduler.configuration.store.class</name><value>zk</value></property>"
            },
            {
              "name": "yarn_resourcemanager_scheduler_class",
              "value": "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"
            },
            {
              "name": "yarn_scheduler_capacity_resource_calculator",
              "value": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator"
            },
            {
              "name": "resourcemanager_capacity_scheduler_configuration",
              "value": "<configuration><property><name>yarn.scheduler.capacity.root.queues</name><value>default</value></property><property><name>yarn.scheduler.capacity.root.capacity</name><value>100</value></property><property><name>yarn.scheduler.capacity.root.default.capacity</name><value>100</value></property><property><name>yarn.scheduler.capacity.root.acl_submit_applications</name><value> </value></property><property><name>yarn.scheduler.capacity.root.acl_administer_queue</name><value> </value></property><property><name>yarn.scheduler.capacity.root.default.acl_submit_applications</name><value>*</value></property><property><name>yarn.scheduler.capacity.root.default.minimum-user-limit-percent</name><value>100</value></property><property><name>yarn.scheduler.capacity.maximum-am-resource-percent</name><value>0.33</value></property><property><name>yarn.scheduler.capacity.node-locality-delay</name><value>0</value></property><property><name>yarn.scheduler.capacity.schedule-asynchronously.maximum-threads</name><value>1</value></property><property><name>yarn.scheduler.capacity.schedule-asynchronously.scheduling-interval-ms</name><value>10</value></property></configuration>"
            }
          ]
        },
        {
          "refName": "yarn-NODEMANAGER-WORKER",
          "roleType": "NODEMANAGER",
          "base": false
        },
        {
          "refName": "yarn-NODEMANAGER-COMPUTE",
          "roleType": "NODEMANAGER",
          "base": false
        },
        {
          "refName": "yarn-JOBHISTORY-BASE",
          "roleType": "JOBHISTORY",
          "base": true
        },
        {
          "refName": "yarn-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true,
          "configs": [
            {
              "name": "mapreduce_map_memory_mb",
              "value": 4096
            },
            {
              "name": "mapreduce_reduce_memory_mb",
              "value": 4096
            },
            {
              "name": "mapreduce_client_env_safety_valve",
              "value": "HADOOP_OPTS=\"-Dorg.wildfly.openssl.path=/usr/lib64 ${HADOOP_OPTS}\""
            }
          ]
        }
      ]
    },
    {
      "refName": "spark_on_yarn",
      "serviceType": "SPARK_ON_YARN",
      "roleConfigGroups": [
        {
          "refName": "spark_on_yarn-SPARK_YARN_HISTORY_SERVER-BASE",
          "roleType": "SPARK_YARN_HISTORY_SERVER",
          "base": true
        },
        {
          "refName": "spark_on_yarn-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true,
          "configs": [
            {
              "name": "spark-conf/spark-defaults.conf_client_config_safety_valve",
              "value": "spark.hadoop.fs.s3a.ssl.channel.mode=openssl"
            }
          ]
        }
      ]
    },
    {
      "refName": "tez",
      "serviceType": "TEZ",
      "serviceConfigs": [
        {
          "name": "yarn_service",
          "ref": "yarn"
        },
        {
          "name": "tez.am.container.reuse.non-local-fallback.enabled",
          "value": "true"
        },
        {
          "name": "tez.am.container.reuse.locality.delay-allocation-millis",
          "value": "0"
        },
        {
          "name": "tez.am.launch.cmd-opts",
          "value": "-XX:+PrintGCDetails -verbose:gc -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
        },
        {
          "name": "tez.task.launch.cmd-opts",
          "value": "-XX:+PrintGCDetails -verbose:gc -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
        },
        {
          "name": "tez.grouping.split-waves",
          "value": 1.4
        },
        {
          "name": "tez.grouping.min-size",
          "value": 268435456
        },
        {
          "name": "tez.grouping.max-size",
          "value": 268435456
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "tez-GATEWAY-BASE",
          "roleType": "GATEWAY",
          "base": true
        }
      ]
    },
    {
      "refName": "das",
      "serviceType": "DAS",
      "roleConfigGroups": [
        {
          "refName": "das-DAS_WEBAPP",
          "roleType": "DAS_WEBAPP",
          "base": true,
          "configs": [
            {
              "name": "data_analytics_studio_admin_users",
              "value": "*"
            }
          ]
        },
        {
          "refName": "das-DAS_EVENT_PROCESSOR",
          "roleType": "DAS_EVENT_PROCESSOR",
          "base": true
        }
      ]
    },
    {
      "refName": "knox",
      "roleConfigGroups": [
        {
          "base": true,
          "refName": "knox-KNOX_GATEWAY-BASE",
          "roleType": "KNOX_GATEWAY"
        }
      ],
      "serviceType": "KNOX"
    },
    {
      "refName": "zeppelin",
      "serviceType": "ZEPPELIN",
      "serviceConfigs": [
        {
          "name": "yarn_service",
          "ref": "yarn"
        },
        {
          "name": "hdfs_service",
          "ref": "hdfs"
        },
        {
          "name": "spark_on_yarn_service",
          "ref": "spark_on_yarn"
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "zeppelin-ZEPPELIN_SERVER-BASE",
          "roleType": "ZEPPELIN_SERVER",
          "base": true
        }
      ]
    },
    {
      "refName": "queuemanager",
      "serviceType": "QUEUEMANAGER",
      "serviceConfigs": [
        {
          "name": "kerberos.auth.enabled",
          "value": "true"
        }
      ],
      "roleConfigGroups": [
        {
          "refName": "yarn-QUEUEMANAGER_WEBAPP-BASE",
          "roleType": "QUEUEMANAGER_WEBAPP",
          "base": true
        },
        {
          "refName": "yarn-QUEUEMANAGER_STORE-BASE",
          "roleType": "QUEUEMANAGER_STORE",
          "base": true
        }
      ]
    }
  ],
  "hostTemplates": [
    {
      "refName": "gateway",
      "cardinality": 0,
      "roleConfigGroupsRefNames": [
        "hdfs-GATEWAY-BASE",
        "hive-GATEWAY-BASE",
        "hms-GATEWAY-BASE",
        "livy-GATEWAY-BASE",
        "spark_on_yarn-GATEWAY-BASE",
        "tez-GATEWAY-BASE",
        "yarn-GATEWAY-BASE"
      ]
    },
    {
      "refName": "master",
      "cardinality": 2,
      "roleConfigGroupsRefNames": [
        "hdfs-FAILOVERCONTROLLER-BASE",
        "hdfs-NAMENODE-BASE",
        "hdfs-GATEWAY-BASE",
        "hive-GATEWAY-BASE",
        "hive-HIVESERVER2-BASE",
        "hms-GATEWAY-BASE",
        "hms-HIVEMETASTORE-BASE",
        "hue-HUE_LOAD_BALANCER-BASE",
        "hue-HUE_SERVER-BASE",
        "knox-KNOX_GATEWAY-BASE",
        "livy-GATEWAY-BASE",
        "spark_on_yarn-GATEWAY-BASE",
        "tez-GATEWAY-BASE",
        "yarn-RESOURCEMANAGER-BASE",
        "hdfs-JOURNALNODE-BASE",
        "zookeeper-SERVER-BASE",
        "yarn-GATEWAY-BASE"
      ]
    },
    {
      "refName": "manager",
      "cardinality": 1,
      "roleConfigGroupsRefNames": [
        "hdfs-BALANCER-BASE",
        "hdfs-GATEWAY-BASE",
        "hive-GATEWAY-BASE",
        "hms-GATEWAY-BASE",
        "livy-GATEWAY-BASE",
        "livy-LIVY_SERVER-BASE",
        "spark_on_yarn-GATEWAY-BASE",
        "spark_on_yarn-SPARK_YARN_HISTORY_SERVER-BASE",
        "tez-GATEWAY-BASE",
        "yarn-JOBHISTORY-BASE",
        "oozie-OOZIE_SERVER-BASE",
        "sqoop-SQOOP_CLIENT-GATEWAY-BASE",
        "zeppelin-ZEPPELIN_SERVER-BASE",
        "das-DAS_EVENT_PROCESSOR",
        "das-DAS_WEBAPP",
        "knox-KNOX_GATEWAY-BASE",
        "hdfs-JOURNALNODE-BASE",
        "zookeeper-SERVER-BASE",
        "yarn-QUEUEMANAGER_WEBAPP-BASE",
        "yarn-QUEUEMANAGER_STORE-BASE",
        "yarn-GATEWAY-BASE"
      ]
    },
    {
      "refName": "worker",
      "cardinality": 3,
      "roleConfigGroupsRefNames": [
        "hdfs-DATANODE-BASE",
        "hdfs-GATEWAY-BASE",
        "hive-GATEWAY-BASE",
        "hms-GATEWAY-BASE",
        "livy-GATEWAY-BASE",
        "spark_on_yarn-GATEWAY-BASE",
        "tez-GATEWAY-BASE",
        "yarn-NODEMANAGER-WORKER",
        "yarn-GATEWAY-BASE"
      ]
    },
    {
      "refName": "compute",
      "cardinality": 0,
      "roleConfigGroupsRefNames": [
        "hdfs-GATEWAY-BASE",
        "hive-GATEWAY-BASE",
        "hms-GATEWAY-BASE",
        "livy-GATEWAY-BASE",
        "spark_on_yarn-GATEWAY-BASE",
        "tez-GATEWAY-BASE",
        "yarn-NODEMANAGER-COMPUTE",
        "yarn-GATEWAY-BASE"
      ]
    }
  ]
}