Metadata Policy Expressions
A metadata policy expression allows you to specify certain metadata extraction policy properties using Java expressions instead of string literals. The supported properties are: entity name and description, key-value pairs, and JMS notification message.
You must declare classes accessed in the expression in the policy's Import Statements field. A metadata policy expression must evaluate to a string.
Metadata policy expressions are not enabled by default. To enable metadata policy expressions, follow the procedure in Enabling and Disabling Metadata Policy Expression Input.
Including Entity Properties in Policy Expressions
To include entity properties in property expressions, use the entity.get method, which takes a property and a return type:entity.get(XXProperties.Property, return_type)XXProperties.Property is the Java enumerated value representing an entity property, where
- XX is FSEntity, HiveColumn, HiveDatabase, HivePartition, HiveQueryExecution, HiveQueryPart, HiveQuery, HiveTable, HiveView, JobExecution, Job, WorkflowInstance, Workflow, PigField, PigOperationExecution, PigOperation, PigRelation, SqoopExportSubOperation, SqoopImportSubOperation, SqoopOperationExecution, SqoopQueryOperation, SqoopTableExportOperation, or SqoopTableImportOperation.
- Property is one of the properties listed in Entity Property Enum Reference.
Metadata Policy Expression Examples
- Set a filesystem entity name to the original name concatenated with the entity type:
entity.get(FSEntityProperties.ORIGINAL_NAME, Object.class) + " " + entity.get(FSEntityProperties.TYPE, Object.class)
Import Statements:import com.cloudera.nav.hdfs.model.FSEntityProperties;
- Add the entity's creation date to the entity name:
entity.get(FSEntityProperties.ORIGINAL_NAME, Object.class) + " - " + new SimpleDateFormat("yyyy-MM-dd").format(entity.get(FSEntityProperties.CREATED, Instant.class).toDate())
Import Statements:import com.cloudera.nav.hdfs.model.FSEntityProperties; import java.text.SimpleDateFormat; import org.joda.time.Instant;
- Set the key-value pair: retain_util-seven years from today's local time:
new DateTime().plusYears(7).toLocalDateTime().toString("MMM dd yyyy", Locale.US)
Import statements:import org.joda.time.DateTime; import java.util.Locale;
Entity Property Enum Reference
The following reference lists the Java enumerated values for retrieving properties of each entity type.com.cloudera.nav.hdfs.model.FSEntityProperties public enum FSEntityProperties implements PropertyEnum { PERMISSIONS, // Return type: java.lang.String TYPE, // Return type: java.lang.String SIZE, // Return type: java.lang.Long OWNER, // Return type: java.lang.String LAST_MODIFIED, // Return type: org.joda.time.Instant SOURCE_TYPE, // Return type: java.lang.String DELETED, // Return type: java.lang.Boolean FILE_SYSTEM_PATH, // Return type: java.lang.String CREATED, // Return type: org.joda.time.Instant LAST_ACCESSED, // Return type: org.joda.time.Instant GROUP, // Return type: java.lang.String MIME_TYPE, // Return type: java.lang.String DELETE_TIME, // Return type: java.lang.Long NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveColumnProperties public enum HiveColumnProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String DELETED, // Return type: java.lang.Boolean DATA_TYPE, // Return type: java.lang.String ORIGINAL_DESCRIPTION, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveDatabaseProperties public enum HiveDatabaseProperties implements PropertyEnum { TYPE, // Return type: java.lang.String ORIGINAL_DESCRIPTION, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String DELETED, // Return type: java.lang.Boolean FILE_SYSTEM_PATH, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HivePartitionProperties public enum HivePartitionProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String DELETED, // Return type: java.lang.Boolean FILE_SYSTEM_PATH, // Return type: java.lang.String CREATED, // Return type: org.joda.time.Instant LAST_ACCESSED, // Return type: org.joda.time.Instant COL_VALUES, // Return type: java.util.List NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveQueryExecutionProperties public enum HiveQueryExecutionProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String ENDED, // Return type: org.joda.time.Instant INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection STARTED, // Return type: org.joda.time.Instant PRINCIPAL, // Return type: java.lang.String WF_INST_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveQueryPartProperties public enum HiveQueryPartProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveQueryProperties public enum HiveQueryProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection QUERY_TEXT, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveTableProperties public enum HiveTableProperties implements PropertyEnum { OWNER, // Return type: java.lang.String INPUT_FORMAT, // Return type: java.lang.String OUTPUT_FORMAT, // Return type: java.lang.String DELETED, // Return type: java.lang.Boolean FILE_SYSTEM_PATH, // Return type: java.lang.String COMPRESSED, // Return type: java.lang.Boolean PARTITION_COL_NAMES, // Return type: java.util.List CLUSTERED_BY_COL_NAMES, // Return type: java.util.List SORT_BY_COL_NAMES, // Return type: java.util.List SER_DE_NAME, // Return type: java.lang.String SER_DE_LIB_NAME, // Return type: java.lang.String TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String CREATED, // Return type: org.joda.time.Instant LAST_ACCESSED, // Return type: org.joda.time.Instant NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.hive.model.HiveViewProperties public enum HiveViewProperties implements PropertyEnum { DELETED, // Return type: java.lang.Boolean QUERY_TEXT, // Return type: java.lang.String TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String CREATED, // Return type: org.joda.time.Instant LAST_ACCESSED, // Return type: org.joda.time.Instant NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.mapreduce.model.JobExecutionProperties public enum JobExecutionProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String JOB_ID, // Return type: java.lang.String ENDED, // Return type: org.joda.time.Instant INPUT_RECURSIVE, // Return type: boolean TYPE, // Return type: java.lang.String INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection STARTED, // Return type: org.joda.time.Instant PRINCIPAL, // Return type: java.lang.String WF_INST_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.mapreduce.model.JobProperties public enum JobProperties implements PropertyEnum { ORIGINAL_NAME, // Return type: java.lang.String INPUT_FORMAT, // Return type: java.lang.String OUTPUT_FORMAT, // Return type: java.lang.String OUTPUT_KEY, // Return type: java.lang.String OUTPUT_VALUE, // Return type: java.lang.String MAPPER, // Return type: java.lang.String REDUCER, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.oozie.model.WorkflowInstanceProperties public enum WorkflowInstanceProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String CREATED, // Return type: org.joda.time.Instant JOB_ID, // Return type: java.lang.String STATUS, // Return type: java.lang.String ENDED, // Return type: org.joda.time.Instant INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection STARTED, // Return type: org.joda.time.Instant PRINCIPAL, // Return type: java.lang.String WF_INST_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.oozie.model.WorkflowProperties public enum WorkflowProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.pig.model.PigFieldProperties public enum PigFieldProperties implements PropertyEnum { TYPE, // Return type: java.lang.String INDEX, // Return type: int SOURCE_TYPE, // Return type: java.lang.String DATA_TYPE, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.pig.model.PigOperationExecutionProperties public enum PigOperationExecutionProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String ENDED, // Return type: org.joda.time.Instant INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection STARTED, // Return type: org.joda.time.Instant PRINCIPAL, // Return type: java.lang.String WF_INST_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.pig.model.PigOperationProperties public enum PigOperationProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String OPERATION_TYPE, // Return type: java.lang.String SCRIPT_ID, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.pig.model.PigRelationProperties public enum PigRelationProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String FILE_SYSTEM_PATH, // Return type: java.lang.String SCRIPT_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopExportSubOperationProperties public enum SqoopExportSubOperationProperties implements PropertyEnum { TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String INPUTS, // Return type: java.util.Collection FIELD_INDEX, // Return type: int NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopImportSubOperationProperties public enum SqoopImportSubOperationProperties implements PropertyEnum { DB_COLUMN_EXPRESSION, // Return type: java.lang.String TYPE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String INPUTS, // Return type: java.util.Collection FIELD_INDEX, // Return type: int NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopOperationExecutionProperties public enum SqoopOperationExecutionProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String ENDED, // Return type: org.joda.time.Instant INPUTS, // Return type: java.util.Collection OUTPUTS, // Return type: java.util.Collection STARTED, // Return type: org.joda.time.Instant PRINCIPAL, // Return type: java.lang.String WF_INST_ID, // Return type: java.lang.String NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopQueryOperationProperties public enum SqoopQueryOperationProperties implements PropertyEnum { SOURCE_TYPE, // Return type: java.lang.String INPUTS, // Return type: java.util.Collection QUERY_TEXT, // Return type: java.lang.String DB_USER, // Return type: java.lang.String DB_URL, // Return type: java.lang.String OPERATION_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopTableExportOperationProperties public enum SqoopTableExportOperationProperties implements PropertyEnum { DB_TABLE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String DB_USER, // Return type: java.lang.String DB_URL, // Return type: java.lang.String OPERATION_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }
com.cloudera.nav.sqoop.model.SqoopTableImportOperationProperties public enum SqoopTableImportOperationProperties implements PropertyEnum { DB_TABLE, // Return type: java.lang.String DB_WHERE, // Return type: java.lang.String SOURCE_TYPE, // Return type: java.lang.String DB_USER, // Return type: java.lang.String DB_URL, // Return type: java.lang.String OPERATION_TYPE, // Return type: java.lang.String TYPE, // Return type: java.lang.String WF_IDS, // Return type: java.util.Collection NAME, // Return type: java.lang.String ORIGINAL_NAME, // Return type: java.lang.String USER_ENTITY, // Return type: boolean SOURCE_ID, // Return type: java.lang.String EXTRACTOR_RUN_ID, // Return type: java.lang.String PARENT_PATH; // Return type: java.lang.String }