001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.conf;
020
021 import java.io.BufferedInputStream;
022 import java.io.DataInput;
023 import java.io.DataOutput;
024 import java.io.File;
025 import java.io.FileInputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.InputStreamReader;
029 import java.io.OutputStream;
030 import java.io.OutputStreamWriter;
031 import java.io.Reader;
032 import java.io.Writer;
033 import java.lang.ref.WeakReference;
034 import java.net.InetSocketAddress;
035 import java.net.URL;
036 import java.util.ArrayList;
037 import java.util.Arrays;
038 import java.util.Collection;
039 import java.util.Collections;
040 import java.util.Enumeration;
041 import java.util.HashMap;
042 import java.util.HashSet;
043 import java.util.Iterator;
044 import java.util.LinkedList;
045 import java.util.List;
046 import java.util.ListIterator;
047 import java.util.Map;
048 import java.util.Map.Entry;
049 import java.util.Properties;
050 import java.util.Set;
051 import java.util.StringTokenizer;
052 import java.util.WeakHashMap;
053 import java.util.concurrent.CopyOnWriteArrayList;
054 import java.util.regex.Matcher;
055 import java.util.regex.Pattern;
056 import java.util.regex.PatternSyntaxException;
057
058 import javax.xml.parsers.DocumentBuilder;
059 import javax.xml.parsers.DocumentBuilderFactory;
060 import javax.xml.parsers.ParserConfigurationException;
061 import javax.xml.transform.Transformer;
062 import javax.xml.transform.TransformerException;
063 import javax.xml.transform.TransformerFactory;
064 import javax.xml.transform.dom.DOMSource;
065 import javax.xml.transform.stream.StreamResult;
066
067 import org.apache.commons.logging.Log;
068 import org.apache.commons.logging.LogFactory;
069 import org.apache.hadoop.classification.InterfaceAudience;
070 import org.apache.hadoop.classification.InterfaceStability;
071 import org.apache.hadoop.fs.FileSystem;
072 import org.apache.hadoop.fs.Path;
073 import org.apache.hadoop.fs.CommonConfigurationKeys;
074 import org.apache.hadoop.io.Writable;
075 import org.apache.hadoop.io.WritableUtils;
076 import org.apache.hadoop.net.NetUtils;
077 import org.apache.hadoop.util.ReflectionUtils;
078 import org.apache.hadoop.util.StringInterner;
079 import org.apache.hadoop.util.StringUtils;
080 import org.codehaus.jackson.JsonFactory;
081 import org.codehaus.jackson.JsonGenerator;
082 import org.w3c.dom.DOMException;
083 import org.w3c.dom.Document;
084 import org.w3c.dom.Element;
085 import org.w3c.dom.Node;
086 import org.w3c.dom.NodeList;
087 import org.w3c.dom.Text;
088 import org.xml.sax.SAXException;
089 import com.google.common.base.Preconditions;
090
091 /**
092 * Provides access to configuration parameters.
093 *
094 * <h4 id="Resources">Resources</h4>
095 *
096 * <p>Configurations are specified by resources. A resource contains a set of
097 * name/value pairs as XML data. Each resource is named by either a
098 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
099 * then the classpath is examined for a file with that name. If named by a
100 * <code>Path</code>, then the local filesystem is examined directly, without
101 * referring to the classpath.
102 *
103 * <p>Unless explicitly turned off, Hadoop by default specifies two
104 * resources, loaded in-order from the classpath: <ol>
105 * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
106 * </tt>: Read-only defaults for hadoop.</li>
107 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
108 * installation.</li>
109 * </ol>
110 * Applications may add additional resources, which are loaded
111 * subsequent to these resources in the order they are added.
112 *
113 * <h4 id="FinalParams">Final Parameters</h4>
114 *
115 * <p>Configuration parameters may be declared <i>final</i>.
116 * Once a resource declares a value final, no subsequently-loaded
117 * resource can alter that value.
118 * For example, one might define a final parameter with:
119 * <tt><pre>
120 * <property>
121 * <name>dfs.hosts.include</name>
122 * <value>/etc/hadoop/conf/hosts.include</value>
123 * <b><final>true</final></b>
124 * </property></pre></tt>
125 *
126 * Administrators typically define parameters as final in
127 * <tt>core-site.xml</tt> for values that user applications may not alter.
128 *
129 * <h4 id="VariableExpansion">Variable Expansion</h4>
130 *
131 * <p>Value strings are first processed for <i>variable expansion</i>. The
132 * available properties are:<ol>
133 * <li>Other properties defined in this Configuration; and, if a name is
134 * undefined here,</li>
135 * <li>Properties in {@link System#getProperties()}.</li>
136 * </ol>
137 *
138 * <p>For example, if a configuration resource contains the following property
139 * definitions:
140 * <tt><pre>
141 * <property>
142 * <name>basedir</name>
143 * <value>/user/${<i>user.name</i>}</value>
144 * </property>
145 *
146 * <property>
147 * <name>tempdir</name>
148 * <value>${<i>basedir</i>}/tmp</value>
149 * </property></pre></tt>
150 *
151 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
152 * will be resolved to another property in this Configuration, while
153 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
154 * of the System property with that name.
155 */
156 @InterfaceAudience.Public
157 @InterfaceStability.Stable
158 public class Configuration implements Iterable<Map.Entry<String,String>>,
159 Writable {
160 private static final Log LOG =
161 LogFactory.getLog(Configuration.class);
162
163 private boolean quietmode = true;
164
165 private static class Resource {
166 private final Object resource;
167 private final String name;
168
169 public Resource(Object resource) {
170 this(resource, resource.toString());
171 }
172
173 public Resource(Object resource, String name) {
174 this.resource = resource;
175 this.name = name;
176 }
177
178 public String getName(){
179 return name;
180 }
181
182 public Object getResource() {
183 return resource;
184 }
185
186 @Override
187 public String toString() {
188 return name;
189 }
190 }
191
192 /**
193 * List of configuration resources.
194 */
195 private ArrayList<Resource> resources = new ArrayList<Resource>();
196
197 /**
198 * The value reported as the setting resource when a key is set
199 * by code rather than a file resource by dumpConfiguration.
200 */
201 static final String UNKNOWN_RESOURCE = "Unknown";
202
203
204 /**
205 * List of configuration parameters marked <b>final</b>.
206 */
207 private Set<String> finalParameters = new HashSet<String>();
208
209 private boolean loadDefaults = true;
210
211 /**
212 * Configuration objects
213 */
214 private static final WeakHashMap<Configuration,Object> REGISTRY =
215 new WeakHashMap<Configuration,Object>();
216
217 /**
218 * List of default Resources. Resources are loaded in the order of the list
219 * entries
220 */
221 private static final CopyOnWriteArrayList<String> defaultResources =
222 new CopyOnWriteArrayList<String>();
223
224 private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
225 CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
226
227 /**
228 * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
229 */
230 private static final Class<?> NEGATIVE_CACHE_SENTINEL =
231 NegativeCacheSentinel.class;
232
233 /**
234 * Stores the mapping of key to the resource which modifies or loads
235 * the key most recently
236 */
237 private HashMap<String, String[]> updatingResource;
238
239 /**
240 * Class to keep the information about the keys which replace the deprecated
241 * ones.
242 *
243 * This class stores the new keys which replace the deprecated keys and also
244 * gives a provision to have a custom message for each of the deprecated key
245 * that is being replaced. It also provides method to get the appropriate
246 * warning message which can be logged whenever the deprecated key is used.
247 */
248 private static class DeprecatedKeyInfo {
249 private String[] newKeys;
250 private String customMessage;
251 private boolean accessed;
252 DeprecatedKeyInfo(String[] newKeys, String customMessage) {
253 this.newKeys = newKeys;
254 this.customMessage = customMessage;
255 accessed = false;
256 }
257
258 /**
259 * Method to provide the warning message. It gives the custom message if
260 * non-null, and default message otherwise.
261 * @param key the associated deprecated key.
262 * @return message that is to be logged when a deprecated key is used.
263 */
264 private final String getWarningMessage(String key) {
265 String warningMessage;
266 if(customMessage == null) {
267 StringBuilder message = new StringBuilder(key);
268 String deprecatedKeySuffix = " is deprecated. Instead, use ";
269 message.append(deprecatedKeySuffix);
270 for (int i = 0; i < newKeys.length; i++) {
271 message.append(newKeys[i]);
272 if(i != newKeys.length-1) {
273 message.append(", ");
274 }
275 }
276 warningMessage = message.toString();
277 }
278 else {
279 warningMessage = customMessage;
280 }
281 accessed = true;
282 return warningMessage;
283 }
284 }
285
286 /**
287 * Stores the deprecated keys, the new keys which replace the deprecated keys
288 * and custom message(if any provided).
289 */
290 private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap =
291 new HashMap<String, DeprecatedKeyInfo>();
292
293 /**
294 * Stores a mapping from superseding keys to the keys which they deprecate.
295 */
296 private static Map<String, String> reverseDeprecatedKeyMap =
297 new HashMap<String, String>();
298
299 /**
300 * Adds the deprecated key to the deprecation map.
301 * It does not override any existing entries in the deprecation map.
302 * This is to be used only by the developers in order to add deprecation of
303 * keys, and attempts to call this method after loading resources once,
304 * would lead to <tt>UnsupportedOperationException</tt>
305 *
306 * If a key is deprecated in favor of multiple keys, they are all treated as
307 * aliases of each other, and setting any one of them resets all the others
308 * to the new value.
309 *
310 * @param key
311 * @param newKeys
312 * @param customMessage
313 * @deprecated use {@link #addDeprecation(String key, String newKey,
314 String customMessage)} instead
315 */
316 @Deprecated
317 public synchronized static void addDeprecation(String key, String[] newKeys,
318 String customMessage) {
319 if (key == null || key.length() == 0 ||
320 newKeys == null || newKeys.length == 0) {
321 throw new IllegalArgumentException();
322 }
323 if (!isDeprecated(key)) {
324 DeprecatedKeyInfo newKeyInfo;
325 newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
326 deprecatedKeyMap.put(key, newKeyInfo);
327 for (String newKey : newKeys) {
328 reverseDeprecatedKeyMap.put(newKey, key);
329 }
330 }
331 }
332
333 /**
334 * Adds the deprecated key to the deprecation map.
335 * It does not override any existing entries in the deprecation map.
336 * This is to be used only by the developers in order to add deprecation of
337 * keys, and attempts to call this method after loading resources once,
338 * would lead to <tt>UnsupportedOperationException</tt>
339 *
340 * @param key
341 * @param newKey
342 * @param customMessage
343 */
344 public synchronized static void addDeprecation(String key, String newKey,
345 String customMessage) {
346 addDeprecation(key, new String[] {newKey}, customMessage);
347 }
348
349 /**
350 * Adds the deprecated key to the deprecation map when no custom message
351 * is provided.
352 * It does not override any existing entries in the deprecation map.
353 * This is to be used only by the developers in order to add deprecation of
354 * keys, and attempts to call this method after loading resources once,
355 * would lead to <tt>UnsupportedOperationException</tt>
356 *
357 * If a key is deprecated in favor of multiple keys, they are all treated as
358 * aliases of each other, and setting any one of them resets all the others
359 * to the new value.
360 *
361 * @param key Key that is to be deprecated
362 * @param newKeys list of keys that take up the values of deprecated key
363 * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
364 */
365 @Deprecated
366 public synchronized static void addDeprecation(String key, String[] newKeys) {
367 addDeprecation(key, newKeys, null);
368 }
369
370 /**
371 * Adds the deprecated key to the deprecation map when no custom message
372 * is provided.
373 * It does not override any existing entries in the deprecation map.
374 * This is to be used only by the developers in order to add deprecation of
375 * keys, and attempts to call this method after loading resources once,
376 * would lead to <tt>UnsupportedOperationException</tt>
377 *
378 * @param key Key that is to be deprecated
379 * @param newKey key that takes up the value of deprecated key
380 */
381 public synchronized static void addDeprecation(String key, String newKey) {
382 addDeprecation(key, new String[] {newKey}, null);
383 }
384
385 /**
386 * checks whether the given <code>key</code> is deprecated.
387 *
388 * @param key the parameter which is to be checked for deprecation
389 * @return <code>true</code> if the key is deprecated and
390 * <code>false</code> otherwise.
391 */
392 public static boolean isDeprecated(String key) {
393 return deprecatedKeyMap.containsKey(key);
394 }
395
396 /**
397 * Returns the alternate name for a key if the property name is deprecated
398 * or if deprecates a property name.
399 *
400 * @param name property name.
401 * @return alternate name.
402 */
403 private String[] getAlternateNames(String name) {
404 String altNames[] = null;
405 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
406 if (keyInfo == null) {
407 altNames = (reverseDeprecatedKeyMap.get(name) != null ) ?
408 new String [] {reverseDeprecatedKeyMap.get(name)} : null;
409 if(altNames != null && altNames.length > 0) {
410 //To help look for other new configs for this deprecated config
411 keyInfo = deprecatedKeyMap.get(altNames[0]);
412 }
413 }
414 if(keyInfo != null && keyInfo.newKeys.length > 0) {
415 List<String> list = new ArrayList<String>();
416 if(altNames != null) {
417 list.addAll(Arrays.asList(altNames));
418 }
419 list.addAll(Arrays.asList(keyInfo.newKeys));
420 altNames = list.toArray(new String[list.size()]);
421 }
422 return altNames;
423 }
424
425 /**
426 * Checks for the presence of the property <code>name</code> in the
427 * deprecation map. Returns the first of the list of new keys if present
428 * in the deprecation map or the <code>name</code> itself. If the property
429 * is not presently set but the property map contains an entry for the
430 * deprecated key, the value of the deprecated key is set as the value for
431 * the provided property name.
432 *
433 * @param name the property name
434 * @return the first property in the list of properties mapping
435 * the <code>name</code> or the <code>name</code> itself.
436 */
437 private String[] handleDeprecation(String name) {
438 ArrayList<String > names = new ArrayList<String>();
439 if (isDeprecated(name)) {
440 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
441 warnOnceIfDeprecated(name);
442 for (String newKey : keyInfo.newKeys) {
443 if(newKey != null) {
444 names.add(newKey);
445 }
446 }
447 }
448 if(names.size() == 0) {
449 names.add(name);
450 }
451 for(String n : names) {
452 String deprecatedKey = reverseDeprecatedKeyMap.get(n);
453 if (deprecatedKey != null && !getOverlay().containsKey(n) &&
454 getOverlay().containsKey(deprecatedKey)) {
455 getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
456 getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
457 }
458 }
459 return names.toArray(new String[names.size()]);
460 }
461
462 private void handleDeprecation() {
463 LOG.debug("Handling deprecation for all properties in config...");
464 Set<Object> keys = new HashSet<Object>();
465 keys.addAll(getProps().keySet());
466 for (Object item: keys) {
467 LOG.debug("Handling deprecation for " + (String)item);
468 handleDeprecation((String)item);
469 }
470 }
471
472 static{
473 //print deprecation warning if hadoop-site.xml is found in classpath
474 ClassLoader cL = Thread.currentThread().getContextClassLoader();
475 if (cL == null) {
476 cL = Configuration.class.getClassLoader();
477 }
478 if(cL.getResource("hadoop-site.xml")!=null) {
479 LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
480 "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
481 + "mapred-site.xml and hdfs-site.xml to override properties of " +
482 "core-default.xml, mapred-default.xml and hdfs-default.xml " +
483 "respectively");
484 }
485 addDefaultResource("core-default.xml");
486 addDefaultResource("core-site.xml");
487 //Add code for managing deprecated key mapping
488 //for example
489 //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
490 //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
491 //so get or set of oldKey1 will correctly populate/access values of
492 //newkey1 and newkey2
493 addDeprecatedKeys();
494 }
495
496 private Properties properties;
497 private Properties overlay;
498 private ClassLoader classLoader;
499 {
500 classLoader = Thread.currentThread().getContextClassLoader();
501 if (classLoader == null) {
502 classLoader = Configuration.class.getClassLoader();
503 }
504 }
505
506 /** A new configuration. */
507 public Configuration() {
508 this(true);
509 }
510
511 /** A new configuration where the behavior of reading from the default
512 * resources can be turned off.
513 *
514 * If the parameter {@code loadDefaults} is false, the new instance
515 * will not load resources from the default files.
516 * @param loadDefaults specifies whether to load from the default files
517 */
518 public Configuration(boolean loadDefaults) {
519 this.loadDefaults = loadDefaults;
520 updatingResource = new HashMap<String, String[]>();
521 synchronized(Configuration.class) {
522 REGISTRY.put(this, null);
523 }
524 }
525
526 /**
527 * A new configuration with the same settings cloned from another.
528 *
529 * @param other the configuration from which to clone settings.
530 */
531 @SuppressWarnings("unchecked")
532 public Configuration(Configuration other) {
533 this.resources = (ArrayList<Resource>) other.resources.clone();
534 synchronized(other) {
535 if (other.properties != null) {
536 this.properties = (Properties)other.properties.clone();
537 }
538
539 if (other.overlay!=null) {
540 this.overlay = (Properties)other.overlay.clone();
541 }
542
543 this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
544 }
545
546 this.finalParameters = new HashSet<String>(other.finalParameters);
547 synchronized(Configuration.class) {
548 REGISTRY.put(this, null);
549 }
550 this.classLoader = other.classLoader;
551 this.loadDefaults = other.loadDefaults;
552 setQuietMode(other.getQuietMode());
553 }
554
555 /**
556 * Add a default resource. Resources are loaded in the order of the resources
557 * added.
558 * @param name file name. File should be present in the classpath.
559 */
560 public static synchronized void addDefaultResource(String name) {
561 if(!defaultResources.contains(name)) {
562 defaultResources.add(name);
563 for(Configuration conf : REGISTRY.keySet()) {
564 if(conf.loadDefaults) {
565 conf.reloadConfiguration();
566 }
567 }
568 }
569 }
570
571 /**
572 * Add a configuration resource.
573 *
574 * The properties of this resource will override properties of previously
575 * added resources, unless they were marked <a href="#Final">final</a>.
576 *
577 * @param name resource to be added, the classpath is examined for a file
578 * with that name.
579 */
580 public void addResource(String name) {
581 addResourceObject(new Resource(name));
582 }
583
584 /**
585 * Add a configuration resource.
586 *
587 * The properties of this resource will override properties of previously
588 * added resources, unless they were marked <a href="#Final">final</a>.
589 *
590 * @param url url of the resource to be added, the local filesystem is
591 * examined directly to find the resource, without referring to
592 * the classpath.
593 */
594 public void addResource(URL url) {
595 addResourceObject(new Resource(url));
596 }
597
598 /**
599 * Add a configuration resource.
600 *
601 * The properties of this resource will override properties of previously
602 * added resources, unless they were marked <a href="#Final">final</a>.
603 *
604 * @param file file-path of resource to be added, the local filesystem is
605 * examined directly to find the resource, without referring to
606 * the classpath.
607 */
608 public void addResource(Path file) {
609 addResourceObject(new Resource(file));
610 }
611
612 /**
613 * Add a configuration resource.
614 *
615 * The properties of this resource will override properties of previously
616 * added resources, unless they were marked <a href="#Final">final</a>.
617 *
618 * WARNING: The contents of the InputStream will be cached, by this method.
619 * So use this sparingly because it does increase the memory consumption.
620 *
621 * @param in InputStream to deserialize the object from. In will be read from
622 * when a get or set is called next. After it is read the stream will be
623 * closed.
624 */
625 public void addResource(InputStream in) {
626 addResourceObject(new Resource(in));
627 }
628
629 /**
630 * Add a configuration resource.
631 *
632 * The properties of this resource will override properties of previously
633 * added resources, unless they were marked <a href="#Final">final</a>.
634 *
635 * @param in InputStream to deserialize the object from.
636 * @param name the name of the resource because InputStream.toString is not
637 * very descriptive some times.
638 */
639 public void addResource(InputStream in, String name) {
640 addResourceObject(new Resource(in, name));
641 }
642
643
644 /**
645 * Reload configuration from previously added resources.
646 *
647 * This method will clear all the configuration read from the added
648 * resources, and final parameters. This will make the resources to
649 * be read again before accessing the values. Values that are added
650 * via set methods will overlay values read from the resources.
651 */
652 public synchronized void reloadConfiguration() {
653 properties = null; // trigger reload
654 finalParameters.clear(); // clear site-limits
655 }
656
657 private synchronized void addResourceObject(Resource resource) {
658 resources.add(resource); // add to resources
659 reloadConfiguration();
660 }
661
662 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
663 private static int MAX_SUBST = 20;
664
665 private String substituteVars(String expr) {
666 if (expr == null) {
667 return null;
668 }
669 Matcher match = varPat.matcher("");
670 String eval = expr;
671 for(int s=0; s<MAX_SUBST; s++) {
672 match.reset(eval);
673 if (!match.find()) {
674 return eval;
675 }
676 String var = match.group();
677 var = var.substring(2, var.length()-1); // remove ${ .. }
678 String val = null;
679 try {
680 val = System.getProperty(var);
681 } catch(SecurityException se) {
682 LOG.warn("Unexpected SecurityException in Configuration", se);
683 }
684 if (val == null) {
685 val = getRaw(var);
686 }
687 if (val == null) {
688 return eval; // return literal ${var}: var is unbound
689 }
690 // substitute
691 eval = eval.substring(0, match.start())+val+eval.substring(match.end());
692 }
693 throw new IllegalStateException("Variable substitution depth too large: "
694 + MAX_SUBST + " " + expr);
695 }
696
697 /**
698 * Get the value of the <code>name</code> property, <code>null</code> if
699 * no such property exists. If the key is deprecated, it returns the value of
700 * the first key which replaces the deprecated key and is not null
701 *
702 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
703 * before being returned.
704 *
705 * @param name the property name.
706 * @return the value of the <code>name</code> or its replacing property,
707 * or null if no such property exists.
708 */
709 public String get(String name) {
710 String[] names = handleDeprecation(name);
711 String result = null;
712 for(String n : names) {
713 result = substituteVars(getProps().getProperty(n));
714 }
715 return result;
716 }
717
718 /**
719 * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
720 * <code>null</code> if no such property exists.
721 * If the key is deprecated, it returns the value of
722 * the first key which replaces the deprecated key and is not null
723 *
724 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
725 * before being returned.
726 *
727 * @param name the property name.
728 * @return the value of the <code>name</code> or its replacing property,
729 * or null if no such property exists.
730 */
731 public String getTrimmed(String name) {
732 String value = get(name);
733
734 if (null == value) {
735 return null;
736 } else {
737 return value.trim();
738 }
739 }
740
741 /**
742 * Get the value of the <code>name</code> property, without doing
743 * <a href="#VariableExpansion">variable expansion</a>.If the key is
744 * deprecated, it returns the value of the first key which replaces
745 * the deprecated key and is not null.
746 *
747 * @param name the property name.
748 * @return the value of the <code>name</code> property or
749 * its replacing property and null if no such property exists.
750 */
751 public String getRaw(String name) {
752 String[] names = handleDeprecation(name);
753 String result = null;
754 for(String n : names) {
755 result = getProps().getProperty(n);
756 }
757 return result;
758 }
759
760 /**
761 * Set the <code>value</code> of the <code>name</code> property. If
762 * <code>name</code> is deprecated or there is a deprecated name associated to it,
763 * it sets the value to both names.
764 *
765 * @param name property name.
766 * @param value property value.
767 */
768 public void set(String name, String value) {
769 set(name, value, null);
770 }
771
772 /**
773 * Set the <code>value</code> of the <code>name</code> property. If
774 * <code>name</code> is deprecated or there is a deprecated name associated to it,
775 * it sets the value to both names.
776 *
777 * @param name property name.
778 * @param value property value.
779 * @param source the place that this configuration value came from
780 * (For debugging).
781 * @throws IllegalArgumentException when the value or name is null.
782 */
783 public void set(String name, String value, String source) {
784 Preconditions.checkArgument(
785 name != null,
786 "Property name must not be null");
787 Preconditions.checkArgument(
788 value != null,
789 "Property value must not be null");
790 if (deprecatedKeyMap.isEmpty()) {
791 getProps();
792 }
793 getOverlay().setProperty(name, value);
794 getProps().setProperty(name, value);
795 if(source == null) {
796 updatingResource.put(name, new String[] {"programatically"});
797 } else {
798 updatingResource.put(name, new String[] {source});
799 }
800 String[] altNames = getAlternateNames(name);
801 if (altNames != null && altNames.length > 0) {
802 String altSource = "because " + name + " is deprecated";
803 for(String altName : altNames) {
804 if(!altName.equals(name)) {
805 getOverlay().setProperty(altName, value);
806 getProps().setProperty(altName, value);
807 updatingResource.put(altName, new String[] {altSource});
808 }
809 }
810 }
811 warnOnceIfDeprecated(name);
812 }
813
814 private void warnOnceIfDeprecated(String name) {
815 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
816 if (keyInfo != null && !keyInfo.accessed) {
817 LOG.warn(keyInfo.getWarningMessage(name));
818 }
819 }
820
821 /**
822 * Unset a previously set property.
823 */
824 public synchronized void unset(String name) {
825 String[] altNames = getAlternateNames(name);
826 getOverlay().remove(name);
827 getProps().remove(name);
828 if (altNames !=null && altNames.length > 0) {
829 for(String altName : altNames) {
830 getOverlay().remove(altName);
831 getProps().remove(altName);
832 }
833 }
834 }
835
836 /**
837 * Sets a property if it is currently unset.
838 * @param name the property name
839 * @param value the new value
840 */
841 public synchronized void setIfUnset(String name, String value) {
842 if (get(name) == null) {
843 set(name, value);
844 }
845 }
846
847 private synchronized Properties getOverlay() {
848 if (overlay==null){
849 overlay=new Properties();
850 }
851 return overlay;
852 }
853
854 /**
855 * Get the value of the <code>name</code>. If the key is deprecated,
856 * it returns the value of the first key which replaces the deprecated key
857 * and is not null.
858 * If no such property exists,
859 * then <code>defaultValue</code> is returned.
860 *
861 * @param name property name.
862 * @param defaultValue default value.
863 * @return property value, or <code>defaultValue</code> if the property
864 * doesn't exist.
865 */
866 public String get(String name, String defaultValue) {
867 String[] names = handleDeprecation(name);
868 String result = null;
869 for(String n : names) {
870 result = substituteVars(getProps().getProperty(n, defaultValue));
871 }
872 return result;
873 }
874
875 /**
876 * Get the value of the <code>name</code> property as an <code>int</code>.
877 *
878 * If no such property exists, the provided default value is returned,
879 * or if the specified value is not a valid <code>int</code>,
880 * then an error is thrown.
881 *
882 * @param name property name.
883 * @param defaultValue default value.
884 * @throws NumberFormatException when the value is invalid
885 * @return property value as an <code>int</code>,
886 * or <code>defaultValue</code>.
887 */
888 public int getInt(String name, int defaultValue) {
889 String valueString = getTrimmed(name);
890 if (valueString == null)
891 return defaultValue;
892 String hexString = getHexDigits(valueString);
893 if (hexString != null) {
894 return Integer.parseInt(hexString, 16);
895 }
896 return Integer.parseInt(valueString);
897 }
898
899 /**
900 * Get the value of the <code>name</code> property as a set of comma-delimited
901 * <code>int</code> values.
902 *
903 * If no such property exists, an empty array is returned.
904 *
905 * @param name property name
906 * @return property value interpreted as an array of comma-delimited
907 * <code>int</code> values
908 */
909 public int[] getInts(String name) {
910 String[] strings = getTrimmedStrings(name);
911 int[] ints = new int[strings.length];
912 for (int i = 0; i < strings.length; i++) {
913 ints[i] = Integer.parseInt(strings[i]);
914 }
915 return ints;
916 }
917
918 /**
919 * Set the value of the <code>name</code> property to an <code>int</code>.
920 *
921 * @param name property name.
922 * @param value <code>int</code> value of the property.
923 */
924 public void setInt(String name, int value) {
925 set(name, Integer.toString(value));
926 }
927
928
929 /**
930 * Get the value of the <code>name</code> property as a <code>long</code>.
931 * If no such property exists, the provided default value is returned,
932 * or if the specified value is not a valid <code>long</code>,
933 * then an error is thrown.
934 *
935 * @param name property name.
936 * @param defaultValue default value.
937 * @throws NumberFormatException when the value is invalid
938 * @return property value as a <code>long</code>,
939 * or <code>defaultValue</code>.
940 */
941 public long getLong(String name, long defaultValue) {
942 String valueString = getTrimmed(name);
943 if (valueString == null)
944 return defaultValue;
945 String hexString = getHexDigits(valueString);
946 if (hexString != null) {
947 return Long.parseLong(hexString, 16);
948 }
949 return Long.parseLong(valueString);
950 }
951
952 /**
953 * Get the value of the <code>name</code> property as a <code>long</code> or
954 * human readable format. If no such property exists, the provided default
955 * value is returned, or if the specified value is not a valid
956 * <code>long</code> or human readable format, then an error is thrown. You
957 * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
958 * t(tera), p(peta), e(exa)
959 *
960 * @param name property name.
961 * @param defaultValue default value.
962 * @throws NumberFormatException when the value is invalid
963 * @return property value as a <code>long</code>,
964 * or <code>defaultValue</code>.
965 */
966 public long getLongBytes(String name, long defaultValue) {
967 String valueString = getTrimmed(name);
968 if (valueString == null)
969 return defaultValue;
970 return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
971 }
972
973 private String getHexDigits(String value) {
974 boolean negative = false;
975 String str = value;
976 String hexString = null;
977 if (value.startsWith("-")) {
978 negative = true;
979 str = value.substring(1);
980 }
981 if (str.startsWith("0x") || str.startsWith("0X")) {
982 hexString = str.substring(2);
983 if (negative) {
984 hexString = "-" + hexString;
985 }
986 return hexString;
987 }
988 return null;
989 }
990
991 /**
992 * Set the value of the <code>name</code> property to a <code>long</code>.
993 *
994 * @param name property name.
995 * @param value <code>long</code> value of the property.
996 */
997 public void setLong(String name, long value) {
998 set(name, Long.toString(value));
999 }
1000
1001 /**
1002 * Get the value of the <code>name</code> property as a <code>float</code>.
1003 * If no such property exists, the provided default value is returned,
1004 * or if the specified value is not a valid <code>float</code>,
1005 * then an error is thrown.
1006 *
1007 * @param name property name.
1008 * @param defaultValue default value.
1009 * @throws NumberFormatException when the value is invalid
1010 * @return property value as a <code>float</code>,
1011 * or <code>defaultValue</code>.
1012 */
1013 public float getFloat(String name, float defaultValue) {
1014 String valueString = getTrimmed(name);
1015 if (valueString == null)
1016 return defaultValue;
1017 return Float.parseFloat(valueString);
1018 }
1019 /**
1020 * Set the value of the <code>name</code> property to a <code>float</code>.
1021 *
1022 * @param name property name.
1023 * @param value property value.
1024 */
1025 public void setFloat(String name, float value) {
1026 set(name,Float.toString(value));
1027 }
1028
1029 /**
1030 * Get the value of the <code>name</code> property as a <code>boolean</code>.
1031 * If no such property is specified, or if the specified value is not a valid
1032 * <code>boolean</code>, then <code>defaultValue</code> is returned.
1033 *
1034 * @param name property name.
1035 * @param defaultValue default value.
1036 * @return property value as a <code>boolean</code>,
1037 * or <code>defaultValue</code>.
1038 */
1039 public boolean getBoolean(String name, boolean defaultValue) {
1040 String valueString = getTrimmed(name);
1041 if (null == valueString || "".equals(valueString)) {
1042 return defaultValue;
1043 }
1044
1045 valueString = valueString.toLowerCase();
1046
1047 if ("true".equals(valueString))
1048 return true;
1049 else if ("false".equals(valueString))
1050 return false;
1051 else return defaultValue;
1052 }
1053
1054 /**
1055 * Set the value of the <code>name</code> property to a <code>boolean</code>.
1056 *
1057 * @param name property name.
1058 * @param value <code>boolean</code> value of the property.
1059 */
1060 public void setBoolean(String name, boolean value) {
1061 set(name, Boolean.toString(value));
1062 }
1063
1064 /**
1065 * Set the given property, if it is currently unset.
1066 * @param name property name
1067 * @param value new value
1068 */
1069 public void setBooleanIfUnset(String name, boolean value) {
1070 setIfUnset(name, Boolean.toString(value));
1071 }
1072
1073 /**
1074 * Set the value of the <code>name</code> property to the given type. This
1075 * is equivalent to <code>set(<name>, value.toString())</code>.
1076 * @param name property name
1077 * @param value new value
1078 */
1079 public <T extends Enum<T>> void setEnum(String name, T value) {
1080 set(name, value.toString());
1081 }
1082
1083 /**
1084 * Return value matching this enumerated type.
1085 * @param name Property name
1086 * @param defaultValue Value returned if no mapping exists
1087 * @throws IllegalArgumentException If mapping is illegal for the type
1088 * provided
1089 */
1090 public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1091 final String val = get(name);
1092 return null == val
1093 ? defaultValue
1094 : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1095 }
1096
1097 /**
1098 * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1099 * If no such property is specified, or if the specified value is not a valid
1100 * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1101 *
1102 * @param name property name
1103 * @param defaultValue default value
1104 * @return property value as a compiled Pattern, or defaultValue
1105 */
1106 public Pattern getPattern(String name, Pattern defaultValue) {
1107 String valString = get(name);
1108 if (null == valString || "".equals(valString)) {
1109 return defaultValue;
1110 }
1111 try {
1112 return Pattern.compile(valString);
1113 } catch (PatternSyntaxException pse) {
1114 LOG.warn("Regular expression '" + valString + "' for property '" +
1115 name + "' not valid. Using default", pse);
1116 return defaultValue;
1117 }
1118 }
1119
1120 /**
1121 * Set the given property to <code>Pattern</code>.
1122 * If the pattern is passed as null, sets the empty pattern which results in
1123 * further calls to getPattern(...) returning the default value.
1124 *
1125 * @param name property name
1126 * @param pattern new value
1127 */
1128 public void setPattern(String name, Pattern pattern) {
1129 if (null == pattern) {
1130 set(name, null);
1131 } else {
1132 set(name, pattern.pattern());
1133 }
1134 }
1135
1136 /**
1137 * Gets information about why a property was set. Typically this is the
1138 * path to the resource objects (file, URL, etc.) the property came from, but
1139 * it can also indicate that it was set programatically, or because of the
1140 * command line.
1141 *
1142 * @param name - The property name to get the source of.
1143 * @return null - If the property or its source wasn't found. Otherwise,
1144 * returns a list of the sources of the resource. The older sources are
1145 * the first ones in the list. So for example if a configuration is set from
1146 * the command line, and then written out to a file that is read back in the
1147 * first entry would indicate that it was set from the command line, while
1148 * the second one would indicate the file that the new configuration was read
1149 * in from.
1150 */
1151 @InterfaceStability.Unstable
1152 public synchronized String[] getPropertySources(String name) {
1153 if (properties == null) {
1154 // If properties is null, it means a resource was newly added
1155 // but the props were cleared so as to load it upon future
1156 // requests. So lets force a load by asking a properties list.
1157 getProps();
1158 }
1159 // Return a null right away if our properties still
1160 // haven't loaded or the resource mapping isn't defined
1161 if (properties == null || updatingResource == null) {
1162 return null;
1163 } else {
1164 String[] source = updatingResource.get(name);
1165 if(source == null) {
1166 return null;
1167 } else {
1168 return Arrays.copyOf(source, source.length);
1169 }
1170 }
1171 }
1172
1173 /**
1174 * A class that represents a set of positive integer ranges. It parses
1175 * strings of the form: "2-3,5,7-" where ranges are separated by comma and
1176 * the lower/upper bounds are separated by dash. Either the lower or upper
1177 * bound may be omitted meaning all values up to or over. So the string
1178 * above means 2, 3, 5, and 7, 8, 9, ...
1179 */
1180 public static class IntegerRanges implements Iterable<Integer>{
1181 private static class Range {
1182 int start;
1183 int end;
1184 }
1185
1186 private static class RangeNumberIterator implements Iterator<Integer> {
1187 Iterator<Range> internal;
1188 int at;
1189 int end;
1190
1191 public RangeNumberIterator(List<Range> ranges) {
1192 if (ranges != null) {
1193 internal = ranges.iterator();
1194 }
1195 at = -1;
1196 end = -2;
1197 }
1198
1199 @Override
1200 public boolean hasNext() {
1201 if (at <= end) {
1202 return true;
1203 } else if (internal != null){
1204 return internal.hasNext();
1205 }
1206 return false;
1207 }
1208
1209 @Override
1210 public Integer next() {
1211 if (at <= end) {
1212 at++;
1213 return at - 1;
1214 } else if (internal != null){
1215 Range found = internal.next();
1216 if (found != null) {
1217 at = found.start;
1218 end = found.end;
1219 at++;
1220 return at - 1;
1221 }
1222 }
1223 return null;
1224 }
1225
1226 @Override
1227 public void remove() {
1228 throw new UnsupportedOperationException();
1229 }
1230 };
1231
1232 List<Range> ranges = new ArrayList<Range>();
1233
1234 public IntegerRanges() {
1235 }
1236
1237 public IntegerRanges(String newValue) {
1238 StringTokenizer itr = new StringTokenizer(newValue, ",");
1239 while (itr.hasMoreTokens()) {
1240 String rng = itr.nextToken().trim();
1241 String[] parts = rng.split("-", 3);
1242 if (parts.length < 1 || parts.length > 2) {
1243 throw new IllegalArgumentException("integer range badly formed: " +
1244 rng);
1245 }
1246 Range r = new Range();
1247 r.start = convertToInt(parts[0], 0);
1248 if (parts.length == 2) {
1249 r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1250 } else {
1251 r.end = r.start;
1252 }
1253 if (r.start > r.end) {
1254 throw new IllegalArgumentException("IntegerRange from " + r.start +
1255 " to " + r.end + " is invalid");
1256 }
1257 ranges.add(r);
1258 }
1259 }
1260
1261 /**
1262 * Convert a string to an int treating empty strings as the default value.
1263 * @param value the string value
1264 * @param defaultValue the value for if the string is empty
1265 * @return the desired integer
1266 */
1267 private static int convertToInt(String value, int defaultValue) {
1268 String trim = value.trim();
1269 if (trim.length() == 0) {
1270 return defaultValue;
1271 }
1272 return Integer.parseInt(trim);
1273 }
1274
1275 /**
1276 * Is the given value in the set of ranges
1277 * @param value the value to check
1278 * @return is the value in the ranges?
1279 */
1280 public boolean isIncluded(int value) {
1281 for(Range r: ranges) {
1282 if (r.start <= value && value <= r.end) {
1283 return true;
1284 }
1285 }
1286 return false;
1287 }
1288
1289 /**
1290 * @return true if there are no values in this range, else false.
1291 */
1292 public boolean isEmpty() {
1293 return ranges == null || ranges.isEmpty();
1294 }
1295
1296 @Override
1297 public String toString() {
1298 StringBuilder result = new StringBuilder();
1299 boolean first = true;
1300 for(Range r: ranges) {
1301 if (first) {
1302 first = false;
1303 } else {
1304 result.append(',');
1305 }
1306 result.append(r.start);
1307 result.append('-');
1308 result.append(r.end);
1309 }
1310 return result.toString();
1311 }
1312
1313 @Override
1314 public Iterator<Integer> iterator() {
1315 return new RangeNumberIterator(ranges);
1316 }
1317
1318 }
1319
1320 /**
1321 * Parse the given attribute as a set of integer ranges
1322 * @param name the attribute name
1323 * @param defaultValue the default value if it is not set
1324 * @return a new set of ranges from the configured value
1325 */
1326 public IntegerRanges getRange(String name, String defaultValue) {
1327 return new IntegerRanges(get(name, defaultValue));
1328 }
1329
1330 /**
1331 * Get the comma delimited values of the <code>name</code> property as
1332 * a collection of <code>String</code>s.
1333 * If no such property is specified then empty collection is returned.
1334 * <p>
1335 * This is an optimized version of {@link #getStrings(String)}
1336 *
1337 * @param name property name.
1338 * @return property value as a collection of <code>String</code>s.
1339 */
1340 public Collection<String> getStringCollection(String name) {
1341 String valueString = get(name);
1342 return StringUtils.getStringCollection(valueString);
1343 }
1344
1345 /**
1346 * Get the comma delimited values of the <code>name</code> property as
1347 * an array of <code>String</code>s.
1348 * If no such property is specified then <code>null</code> is returned.
1349 *
1350 * @param name property name.
1351 * @return property value as an array of <code>String</code>s,
1352 * or <code>null</code>.
1353 */
1354 public String[] getStrings(String name) {
1355 String valueString = get(name);
1356 return StringUtils.getStrings(valueString);
1357 }
1358
1359 /**
1360 * Get the comma delimited values of the <code>name</code> property as
1361 * an array of <code>String</code>s.
1362 * If no such property is specified then default value is returned.
1363 *
1364 * @param name property name.
1365 * @param defaultValue The default value
1366 * @return property value as an array of <code>String</code>s,
1367 * or default value.
1368 */
1369 public String[] getStrings(String name, String... defaultValue) {
1370 String valueString = get(name);
1371 if (valueString == null) {
1372 return defaultValue;
1373 } else {
1374 return StringUtils.getStrings(valueString);
1375 }
1376 }
1377
1378 /**
1379 * Get the comma delimited values of the <code>name</code> property as
1380 * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
1381 * If no such property is specified then empty <code>Collection</code> is returned.
1382 *
1383 * @param name property name.
1384 * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>
1385 */
1386 public Collection<String> getTrimmedStringCollection(String name) {
1387 String valueString = get(name);
1388 if (null == valueString) {
1389 Collection<String> empty = new ArrayList<String>();
1390 return empty;
1391 }
1392 return StringUtils.getTrimmedStringCollection(valueString);
1393 }
1394
1395 /**
1396 * Get the comma delimited values of the <code>name</code> property as
1397 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1398 * If no such property is specified then an empty array is returned.
1399 *
1400 * @param name property name.
1401 * @return property value as an array of trimmed <code>String</code>s,
1402 * or empty array.
1403 */
1404 public String[] getTrimmedStrings(String name) {
1405 String valueString = get(name);
1406 return StringUtils.getTrimmedStrings(valueString);
1407 }
1408
1409 /**
1410 * Get the comma delimited values of the <code>name</code> property as
1411 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1412 * If no such property is specified then default value is returned.
1413 *
1414 * @param name property name.
1415 * @param defaultValue The default value
1416 * @return property value as an array of trimmed <code>String</code>s,
1417 * or default value.
1418 */
1419 public String[] getTrimmedStrings(String name, String... defaultValue) {
1420 String valueString = get(name);
1421 if (null == valueString) {
1422 return defaultValue;
1423 } else {
1424 return StringUtils.getTrimmedStrings(valueString);
1425 }
1426 }
1427
1428 /**
1429 * Set the array of string values for the <code>name</code> property as
1430 * as comma delimited values.
1431 *
1432 * @param name property name.
1433 * @param values The values
1434 */
1435 public void setStrings(String name, String... values) {
1436 set(name, StringUtils.arrayToString(values));
1437 }
1438
1439 /**
1440 * Get the socket address for <code>name</code> property as a
1441 * <code>InetSocketAddress</code>.
1442 * @param name property name.
1443 * @param defaultAddress the default value
1444 * @param defaultPort the default port
1445 * @return InetSocketAddress
1446 */
1447 public InetSocketAddress getSocketAddr(
1448 String name, String defaultAddress, int defaultPort) {
1449 final String address = get(name, defaultAddress);
1450 return NetUtils.createSocketAddr(address, defaultPort, name);
1451 }
1452
1453 /**
1454 * Set the socket address for the <code>name</code> property as
1455 * a <code>host:port</code>.
1456 */
1457 public void setSocketAddr(String name, InetSocketAddress addr) {
1458 set(name, NetUtils.getHostPortString(addr));
1459 }
1460
1461 /**
1462 * Set the socket address a client can use to connect for the
1463 * <code>name</code> property as a <code>host:port</code>. The wildcard
1464 * address is replaced with the local host's address.
1465 * @param name property name.
1466 * @param addr InetSocketAddress of a listener to store in the given property
1467 * @return InetSocketAddress for clients to connect
1468 */
1469 public InetSocketAddress updateConnectAddr(String name,
1470 InetSocketAddress addr) {
1471 final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1472 setSocketAddr(name, connectAddr);
1473 return connectAddr;
1474 }
1475
1476 /**
1477 * Load a class by name.
1478 *
1479 * @param name the class name.
1480 * @return the class object.
1481 * @throws ClassNotFoundException if the class is not found.
1482 */
1483 public Class<?> getClassByName(String name) throws ClassNotFoundException {
1484 Class<?> ret = getClassByNameOrNull(name);
1485 if (ret == null) {
1486 throw new ClassNotFoundException("Class " + name + " not found");
1487 }
1488 return ret;
1489 }
1490
1491 /**
1492 * Load a class by name, returning null rather than throwing an exception
1493 * if it couldn't be loaded. This is to avoid the overhead of creating
1494 * an exception.
1495 *
1496 * @param name the class name
1497 * @return the class object, or null if it could not be found.
1498 */
1499 public Class<?> getClassByNameOrNull(String name) {
1500 Map<String, WeakReference<Class<?>>> map;
1501
1502 synchronized (CACHE_CLASSES) {
1503 map = CACHE_CLASSES.get(classLoader);
1504 if (map == null) {
1505 map = Collections.synchronizedMap(
1506 new WeakHashMap<String, WeakReference<Class<?>>>());
1507 CACHE_CLASSES.put(classLoader, map);
1508 }
1509 }
1510
1511 Class<?> clazz = null;
1512 WeakReference<Class<?>> ref = map.get(name);
1513 if (ref != null) {
1514 clazz = ref.get();
1515 }
1516
1517 if (clazz == null) {
1518 try {
1519 clazz = Class.forName(name, true, classLoader);
1520 } catch (ClassNotFoundException e) {
1521 // Leave a marker that the class isn't found
1522 map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
1523 return null;
1524 }
1525 // two putters can race here, but they'll put the same class
1526 map.put(name, new WeakReference<Class<?>>(clazz));
1527 return clazz;
1528 } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
1529 return null; // not found
1530 } else {
1531 // cache hit
1532 return clazz;
1533 }
1534 }
1535
1536 /**
1537 * Get the value of the <code>name</code> property
1538 * as an array of <code>Class</code>.
1539 * The value of the property specifies a list of comma separated class names.
1540 * If no such property is specified, then <code>defaultValue</code> is
1541 * returned.
1542 *
1543 * @param name the property name.
1544 * @param defaultValue default value.
1545 * @return property value as a <code>Class[]</code>,
1546 * or <code>defaultValue</code>.
1547 */
1548 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1549 String[] classnames = getTrimmedStrings(name);
1550 if (classnames == null)
1551 return defaultValue;
1552 try {
1553 Class<?>[] classes = new Class<?>[classnames.length];
1554 for(int i = 0; i < classnames.length; i++) {
1555 classes[i] = getClassByName(classnames[i]);
1556 }
1557 return classes;
1558 } catch (ClassNotFoundException e) {
1559 throw new RuntimeException(e);
1560 }
1561 }
1562
1563 /**
1564 * Get the value of the <code>name</code> property as a <code>Class</code>.
1565 * If no such property is specified, then <code>defaultValue</code> is
1566 * returned.
1567 *
1568 * @param name the class name.
1569 * @param defaultValue default value.
1570 * @return property value as a <code>Class</code>,
1571 * or <code>defaultValue</code>.
1572 */
1573 public Class<?> getClass(String name, Class<?> defaultValue) {
1574 String valueString = getTrimmed(name);
1575 if (valueString == null)
1576 return defaultValue;
1577 try {
1578 return getClassByName(valueString);
1579 } catch (ClassNotFoundException e) {
1580 throw new RuntimeException(e);
1581 }
1582 }
1583
1584 /**
1585 * Get the value of the <code>name</code> property as a <code>Class</code>
1586 * implementing the interface specified by <code>xface</code>.
1587 *
1588 * If no such property is specified, then <code>defaultValue</code> is
1589 * returned.
1590 *
1591 * An exception is thrown if the returned class does not implement the named
1592 * interface.
1593 *
1594 * @param name the class name.
1595 * @param defaultValue default value.
1596 * @param xface the interface implemented by the named class.
1597 * @return property value as a <code>Class</code>,
1598 * or <code>defaultValue</code>.
1599 */
1600 public <U> Class<? extends U> getClass(String name,
1601 Class<? extends U> defaultValue,
1602 Class<U> xface) {
1603 try {
1604 Class<?> theClass = getClass(name, defaultValue);
1605 if (theClass != null && !xface.isAssignableFrom(theClass))
1606 throw new RuntimeException(theClass+" not "+xface.getName());
1607 else if (theClass != null)
1608 return theClass.asSubclass(xface);
1609 else
1610 return null;
1611 } catch (Exception e) {
1612 throw new RuntimeException(e);
1613 }
1614 }
1615
1616 /**
1617 * Get the value of the <code>name</code> property as a <code>List</code>
1618 * of objects implementing the interface specified by <code>xface</code>.
1619 *
1620 * An exception is thrown if any of the classes does not exist, or if it does
1621 * not implement the named interface.
1622 *
1623 * @param name the property name.
1624 * @param xface the interface implemented by the classes named by
1625 * <code>name</code>.
1626 * @return a <code>List</code> of objects implementing <code>xface</code>.
1627 */
1628 @SuppressWarnings("unchecked")
1629 public <U> List<U> getInstances(String name, Class<U> xface) {
1630 List<U> ret = new ArrayList<U>();
1631 Class<?>[] classes = getClasses(name);
1632 for (Class<?> cl: classes) {
1633 if (!xface.isAssignableFrom(cl)) {
1634 throw new RuntimeException(cl + " does not implement " + xface);
1635 }
1636 ret.add((U)ReflectionUtils.newInstance(cl, this));
1637 }
1638 return ret;
1639 }
1640
1641 /**
1642 * Set the value of the <code>name</code> property to the name of a
1643 * <code>theClass</code> implementing the given interface <code>xface</code>.
1644 *
1645 * An exception is thrown if <code>theClass</code> does not implement the
1646 * interface <code>xface</code>.
1647 *
1648 * @param name property name.
1649 * @param theClass property value.
1650 * @param xface the interface implemented by the named class.
1651 */
1652 public void setClass(String name, Class<?> theClass, Class<?> xface) {
1653 if (!xface.isAssignableFrom(theClass))
1654 throw new RuntimeException(theClass+" not "+xface.getName());
1655 set(name, theClass.getName());
1656 }
1657
1658 /**
1659 * Get a local file under a directory named by <i>dirsProp</i> with
1660 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1661 * then one is chosen based on <i>path</i>'s hash code. If the selected
1662 * directory does not exist, an attempt is made to create it.
1663 *
1664 * @param dirsProp directory in which to locate the file.
1665 * @param path file-path.
1666 * @return local file under the directory with the given path.
1667 */
1668 public Path getLocalPath(String dirsProp, String path)
1669 throws IOException {
1670 String[] dirs = getTrimmedStrings(dirsProp);
1671 int hashCode = path.hashCode();
1672 FileSystem fs = FileSystem.getLocal(this);
1673 for (int i = 0; i < dirs.length; i++) { // try each local dir
1674 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1675 Path file = new Path(dirs[index], path);
1676 Path dir = file.getParent();
1677 if (fs.mkdirs(dir) || fs.exists(dir)) {
1678 return file;
1679 }
1680 }
1681 LOG.warn("Could not make " + path +
1682 " in local directories from " + dirsProp);
1683 for(int i=0; i < dirs.length; i++) {
1684 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1685 LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1686 }
1687 throw new IOException("No valid local directories in property: "+dirsProp);
1688 }
1689
1690 /**
1691 * Get a local file name under a directory named in <i>dirsProp</i> with
1692 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1693 * then one is chosen based on <i>path</i>'s hash code. If the selected
1694 * directory does not exist, an attempt is made to create it.
1695 *
1696 * @param dirsProp directory in which to locate the file.
1697 * @param path file-path.
1698 * @return local file under the directory with the given path.
1699 */
1700 public File getFile(String dirsProp, String path)
1701 throws IOException {
1702 String[] dirs = getTrimmedStrings(dirsProp);
1703 int hashCode = path.hashCode();
1704 for (int i = 0; i < dirs.length; i++) { // try each local dir
1705 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1706 File file = new File(dirs[index], path);
1707 File dir = file.getParentFile();
1708 if (dir.exists() || dir.mkdirs()) {
1709 return file;
1710 }
1711 }
1712 throw new IOException("No valid local directories in property: "+dirsProp);
1713 }
1714
1715 /**
1716 * Get the {@link URL} for the named resource.
1717 *
1718 * @param name resource name.
1719 * @return the url for the named resource.
1720 */
1721 public URL getResource(String name) {
1722 return classLoader.getResource(name);
1723 }
1724
1725 /**
1726 * Get an input stream attached to the configuration resource with the
1727 * given <code>name</code>.
1728 *
1729 * @param name configuration resource name.
1730 * @return an input stream attached to the resource.
1731 */
1732 public InputStream getConfResourceAsInputStream(String name) {
1733 try {
1734 URL url= getResource(name);
1735
1736 if (url == null) {
1737 LOG.info(name + " not found");
1738 return null;
1739 } else {
1740 LOG.info("found resource " + name + " at " + url);
1741 }
1742
1743 return url.openStream();
1744 } catch (Exception e) {
1745 return null;
1746 }
1747 }
1748
1749 /**
1750 * Get a {@link Reader} attached to the configuration resource with the
1751 * given <code>name</code>.
1752 *
1753 * @param name configuration resource name.
1754 * @return a reader attached to the resource.
1755 */
1756 public Reader getConfResourceAsReader(String name) {
1757 try {
1758 URL url= getResource(name);
1759
1760 if (url == null) {
1761 LOG.info(name + " not found");
1762 return null;
1763 } else {
1764 LOG.info("found resource " + name + " at " + url);
1765 }
1766
1767 return new InputStreamReader(url.openStream());
1768 } catch (Exception e) {
1769 return null;
1770 }
1771 }
1772
1773 protected synchronized Properties getProps() {
1774 if (properties == null) {
1775 properties = new Properties();
1776 HashMap<String, String[]> backup =
1777 new HashMap<String, String[]>(updatingResource);
1778 loadResources(properties, resources, quietmode);
1779 if (overlay!= null) {
1780 properties.putAll(overlay);
1781 for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1782 String key = (String)item.getKey();
1783 updatingResource.put(key, backup.get(key));
1784 }
1785 }
1786 }
1787 return properties;
1788 }
1789
1790 /**
1791 * Return the number of keys in the configuration.
1792 *
1793 * @return number of keys in the configuration.
1794 */
1795 public int size() {
1796 return getProps().size();
1797 }
1798
1799 /**
1800 * Clears all keys from the configuration.
1801 */
1802 public void clear() {
1803 getProps().clear();
1804 getOverlay().clear();
1805 }
1806
1807 /**
1808 * Get an {@link Iterator} to go through the list of <code>String</code>
1809 * key-value pairs in the configuration.
1810 *
1811 * @return an iterator over the entries.
1812 */
1813 @Override
1814 public Iterator<Map.Entry<String, String>> iterator() {
1815 // Get a copy of just the string to string pairs. After the old object
1816 // methods that allow non-strings to be put into configurations are removed,
1817 // we could replace properties with a Map<String,String> and get rid of this
1818 // code.
1819 Map<String,String> result = new HashMap<String,String>();
1820 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1821 if (item.getKey() instanceof String &&
1822 item.getValue() instanceof String) {
1823 result.put((String) item.getKey(), (String) item.getValue());
1824 }
1825 }
1826 return result.entrySet().iterator();
1827 }
1828
1829 private Document parse(DocumentBuilder builder, URL url)
1830 throws IOException, SAXException {
1831 if (!quietmode) {
1832 LOG.info("parsing URL " + url);
1833 }
1834 if (url == null) {
1835 return null;
1836 }
1837 return parse(builder, url.openStream(), url.toString());
1838 }
1839
1840 private Document parse(DocumentBuilder builder, InputStream is,
1841 String systemId) throws IOException, SAXException {
1842 if (!quietmode) {
1843 LOG.info("parsing input stream " + is);
1844 }
1845 if (is == null) {
1846 return null;
1847 }
1848 try {
1849 return (systemId == null) ? builder.parse(is) : builder.parse(is,
1850 systemId);
1851 } finally {
1852 is.close();
1853 }
1854 }
1855
1856 private void loadResources(Properties properties,
1857 ArrayList<Resource> resources,
1858 boolean quiet) {
1859 if(loadDefaults) {
1860 for (String resource : defaultResources) {
1861 loadResource(properties, new Resource(resource), quiet);
1862 }
1863
1864 //support the hadoop-site.xml as a deprecated case
1865 if(getResource("hadoop-site.xml")!=null) {
1866 loadResource(properties, new Resource("hadoop-site.xml"), quiet);
1867 }
1868 }
1869
1870 for (int i = 0; i < resources.size(); i++) {
1871 Resource ret = loadResource(properties, resources.get(i), quiet);
1872 if (ret != null) {
1873 resources.set(i, ret);
1874 }
1875 }
1876 }
1877
1878 private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
1879 String name = UNKNOWN_RESOURCE;
1880 try {
1881 Object resource = wrapper.getResource();
1882 name = wrapper.getName();
1883
1884 DocumentBuilderFactory docBuilderFactory
1885 = DocumentBuilderFactory.newInstance();
1886 //ignore all comments inside the xml file
1887 docBuilderFactory.setIgnoringComments(true);
1888
1889 //allow includes in the xml file
1890 docBuilderFactory.setNamespaceAware(true);
1891 try {
1892 docBuilderFactory.setXIncludeAware(true);
1893 } catch (UnsupportedOperationException e) {
1894 LOG.error("Failed to set setXIncludeAware(true) for parser "
1895 + docBuilderFactory
1896 + ":" + e,
1897 e);
1898 }
1899 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
1900 Document doc = null;
1901 Element root = null;
1902 boolean returnCachedProperties = false;
1903
1904 if (resource instanceof URL) { // an URL resource
1905 doc = parse(builder, (URL)resource);
1906 } else if (resource instanceof String) { // a CLASSPATH resource
1907 URL url = getResource((String)resource);
1908 doc = parse(builder, url);
1909 } else if (resource instanceof Path) { // a file resource
1910 // Can't use FileSystem API or we get an infinite loop
1911 // since FileSystem uses Configuration API. Use java.io.File instead.
1912 File file = new File(((Path)resource).toUri().getPath())
1913 .getAbsoluteFile();
1914 if (file.exists()) {
1915 if (!quiet) {
1916 LOG.info("parsing File " + file);
1917 }
1918 doc = parse(builder, new BufferedInputStream(
1919 new FileInputStream(file)), ((Path)resource).toString());
1920 }
1921 } else if (resource instanceof InputStream) {
1922 doc = parse(builder, (InputStream) resource, null);
1923 returnCachedProperties = true;
1924 } else if (resource instanceof Properties) {
1925 overlay(properties, (Properties)resource);
1926 } else if (resource instanceof Element) {
1927 root = (Element)resource;
1928 }
1929
1930 if (doc == null && root == null) {
1931 if (quiet)
1932 return null;
1933 throw new RuntimeException(resource + " not found");
1934 }
1935
1936 if (root == null) {
1937 root = doc.getDocumentElement();
1938 }
1939 Properties toAddTo = properties;
1940 if(returnCachedProperties) {
1941 toAddTo = new Properties();
1942 }
1943 if (!"configuration".equals(root.getTagName()))
1944 LOG.fatal("bad conf file: top-level element not <configuration>");
1945 NodeList props = root.getChildNodes();
1946 for (int i = 0; i < props.getLength(); i++) {
1947 Node propNode = props.item(i);
1948 if (!(propNode instanceof Element))
1949 continue;
1950 Element prop = (Element)propNode;
1951 if ("configuration".equals(prop.getTagName())) {
1952 loadResource(toAddTo, new Resource(prop, name), quiet);
1953 continue;
1954 }
1955 if (!"property".equals(prop.getTagName()))
1956 LOG.warn("bad conf file: element not <property>");
1957 NodeList fields = prop.getChildNodes();
1958 String attr = null;
1959 String value = null;
1960 boolean finalParameter = false;
1961 LinkedList<String> source = new LinkedList<String>();
1962 for (int j = 0; j < fields.getLength(); j++) {
1963 Node fieldNode = fields.item(j);
1964 if (!(fieldNode instanceof Element))
1965 continue;
1966 Element field = (Element)fieldNode;
1967 if ("name".equals(field.getTagName()) && field.hasChildNodes())
1968 attr = StringInterner.weakIntern(
1969 ((Text)field.getFirstChild()).getData().trim());
1970 if ("value".equals(field.getTagName()) && field.hasChildNodes())
1971 value = StringInterner.weakIntern(
1972 ((Text)field.getFirstChild()).getData());
1973 if ("final".equals(field.getTagName()) && field.hasChildNodes())
1974 finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
1975 if ("source".equals(field.getTagName()) && field.hasChildNodes())
1976 source.add(StringInterner.weakIntern(
1977 ((Text)field.getFirstChild()).getData()));
1978 }
1979 source.add(name);
1980
1981 // Ignore this parameter if it has already been marked as 'final'
1982 if (attr != null) {
1983 if (deprecatedKeyMap.containsKey(attr)) {
1984 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
1985 keyInfo.accessed = false;
1986 for (String key:keyInfo.newKeys) {
1987 // update new keys with deprecated key's value
1988 loadProperty(toAddTo, name, key, value, finalParameter,
1989 source.toArray(new String[source.size()]));
1990 }
1991 }
1992 else {
1993 loadProperty(toAddTo, name, attr, value, finalParameter,
1994 source.toArray(new String[source.size()]));
1995 }
1996 }
1997 }
1998
1999 if (returnCachedProperties) {
2000 overlay(properties, toAddTo);
2001 return new Resource(toAddTo, name);
2002 }
2003 return null;
2004 } catch (IOException e) {
2005 LOG.fatal("error parsing conf " + name, e);
2006 throw new RuntimeException(e);
2007 } catch (DOMException e) {
2008 LOG.fatal("error parsing conf " + name, e);
2009 throw new RuntimeException(e);
2010 } catch (SAXException e) {
2011 LOG.fatal("error parsing conf " + name, e);
2012 throw new RuntimeException(e);
2013 } catch (ParserConfigurationException e) {
2014 LOG.fatal("error parsing conf " + name , e);
2015 throw new RuntimeException(e);
2016 }
2017 }
2018
2019 private void overlay(Properties to, Properties from) {
2020 for (Entry<Object, Object> entry: from.entrySet()) {
2021 to.put(entry.getKey(), entry.getValue());
2022 }
2023 }
2024
2025 private void loadProperty(Properties properties, String name, String attr,
2026 String value, boolean finalParameter, String[] source) {
2027 if (value != null) {
2028 if (!finalParameters.contains(attr)) {
2029 properties.setProperty(attr, value);
2030 updatingResource.put(attr, source);
2031 } else {
2032 LOG.warn(name+":an attempt to override final parameter: "+attr
2033 +"; Ignoring.");
2034 }
2035 }
2036 if (finalParameter) {
2037 finalParameters.add(attr);
2038 }
2039 }
2040
2041 /**
2042 * Write out the non-default properties in this configuration to the given
2043 * {@link OutputStream}.
2044 *
2045 * @param out the output stream to write to.
2046 */
2047 public void writeXml(OutputStream out) throws IOException {
2048 writeXml(new OutputStreamWriter(out));
2049 }
2050
2051 /**
2052 * Write out the non-default properties in this configuration to the given
2053 * {@link Writer}.
2054 *
2055 * @param out the writer to write to.
2056 */
2057 public void writeXml(Writer out) throws IOException {
2058 Document doc = asXmlDocument();
2059
2060 try {
2061 DOMSource source = new DOMSource(doc);
2062 StreamResult result = new StreamResult(out);
2063 TransformerFactory transFactory = TransformerFactory.newInstance();
2064 Transformer transformer = transFactory.newTransformer();
2065
2066 // Important to not hold Configuration log while writing result, since
2067 // 'out' may be an HDFS stream which needs to lock this configuration
2068 // from another thread.
2069 transformer.transform(source, result);
2070 } catch (TransformerException te) {
2071 throw new IOException(te);
2072 }
2073 }
2074
2075 /**
2076 * Return the XML DOM corresponding to this Configuration.
2077 */
2078 private synchronized Document asXmlDocument() throws IOException {
2079 Document doc;
2080 try {
2081 doc =
2082 DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2083 } catch (ParserConfigurationException pe) {
2084 throw new IOException(pe);
2085 }
2086 Element conf = doc.createElement("configuration");
2087 doc.appendChild(conf);
2088 conf.appendChild(doc.createTextNode("\n"));
2089 handleDeprecation(); //ensure properties is set and deprecation is handled
2090 for (Enumeration e = properties.keys(); e.hasMoreElements();) {
2091 String name = (String)e.nextElement();
2092 Object object = properties.get(name);
2093 String value = null;
2094 if (object instanceof String) {
2095 value = (String) object;
2096 }else {
2097 continue;
2098 }
2099 Element propNode = doc.createElement("property");
2100 conf.appendChild(propNode);
2101
2102 Element nameNode = doc.createElement("name");
2103 nameNode.appendChild(doc.createTextNode(name));
2104 propNode.appendChild(nameNode);
2105
2106 Element valueNode = doc.createElement("value");
2107 valueNode.appendChild(doc.createTextNode(value));
2108 propNode.appendChild(valueNode);
2109
2110 if (updatingResource != null) {
2111 String[] sources = updatingResource.get(name);
2112 if(sources != null) {
2113 for(String s : sources) {
2114 Element sourceNode = doc.createElement("source");
2115 sourceNode.appendChild(doc.createTextNode(s));
2116 propNode.appendChild(sourceNode);
2117 }
2118 }
2119 }
2120
2121 conf.appendChild(doc.createTextNode("\n"));
2122 }
2123 return doc;
2124 }
2125
2126 /**
2127 * Writes out all the parameters and their properties (final and resource) to
2128 * the given {@link Writer}
2129 * The format of the output would be
2130 * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2131 * key2.isFinal,key2.resource}... ] }
2132 * It does not output the parameters of the configuration object which is
2133 * loaded from an input stream.
2134 * @param out the Writer to write to
2135 * @throws IOException
2136 */
2137 public static void dumpConfiguration(Configuration config,
2138 Writer out) throws IOException {
2139 JsonFactory dumpFactory = new JsonFactory();
2140 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2141 dumpGenerator.writeStartObject();
2142 dumpGenerator.writeFieldName("properties");
2143 dumpGenerator.writeStartArray();
2144 dumpGenerator.flush();
2145 synchronized (config) {
2146 for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2147 dumpGenerator.writeStartObject();
2148 dumpGenerator.writeStringField("key", (String) item.getKey());
2149 dumpGenerator.writeStringField("value",
2150 config.get((String) item.getKey()));
2151 dumpGenerator.writeBooleanField("isFinal",
2152 config.finalParameters.contains(item.getKey()));
2153 String[] resources = config.updatingResource.get(item.getKey());
2154 String resource = UNKNOWN_RESOURCE;
2155 if(resources != null && resources.length > 0) {
2156 resource = resources[0];
2157 }
2158 dumpGenerator.writeStringField("resource", resource);
2159 dumpGenerator.writeEndObject();
2160 }
2161 }
2162 dumpGenerator.writeEndArray();
2163 dumpGenerator.writeEndObject();
2164 dumpGenerator.flush();
2165 }
2166
2167 /**
2168 * Get the {@link ClassLoader} for this job.
2169 *
2170 * @return the correct class loader.
2171 */
2172 public ClassLoader getClassLoader() {
2173 return classLoader;
2174 }
2175
2176 /**
2177 * Set the class loader that will be used to load the various objects.
2178 *
2179 * @param classLoader the new class loader.
2180 */
2181 public void setClassLoader(ClassLoader classLoader) {
2182 this.classLoader = classLoader;
2183 }
2184
2185 @Override
2186 public String toString() {
2187 StringBuilder sb = new StringBuilder();
2188 sb.append("Configuration: ");
2189 if(loadDefaults) {
2190 toString(defaultResources, sb);
2191 if(resources.size()>0) {
2192 sb.append(", ");
2193 }
2194 }
2195 toString(resources, sb);
2196 return sb.toString();
2197 }
2198
2199 private <T> void toString(List<T> resources, StringBuilder sb) {
2200 ListIterator<T> i = resources.listIterator();
2201 while (i.hasNext()) {
2202 if (i.nextIndex() != 0) {
2203 sb.append(", ");
2204 }
2205 sb.append(i.next());
2206 }
2207 }
2208
2209 /**
2210 * Set the quietness-mode.
2211 *
2212 * In the quiet-mode, error and informational messages might not be logged.
2213 *
2214 * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2215 * to turn it off.
2216 */
2217 public synchronized void setQuietMode(boolean quietmode) {
2218 this.quietmode = quietmode;
2219 }
2220
2221 synchronized boolean getQuietMode() {
2222 return this.quietmode;
2223 }
2224
2225 /** For debugging. List non-default properties to the terminal and exit. */
2226 public static void main(String[] args) throws Exception {
2227 new Configuration().writeXml(System.out);
2228 }
2229
2230 @Override
2231 public void readFields(DataInput in) throws IOException {
2232 clear();
2233 int size = WritableUtils.readVInt(in);
2234 for(int i=0; i < size; ++i) {
2235 String key = org.apache.hadoop.io.Text.readString(in);
2236 String value = org.apache.hadoop.io.Text.readString(in);
2237 set(key, value);
2238 String sources[] = WritableUtils.readCompressedStringArray(in);
2239 updatingResource.put(key, sources);
2240 }
2241 }
2242
2243 //@Override
2244 @Override
2245 public void write(DataOutput out) throws IOException {
2246 Properties props = getProps();
2247 WritableUtils.writeVInt(out, props.size());
2248 for(Map.Entry<Object, Object> item: props.entrySet()) {
2249 org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2250 org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2251 WritableUtils.writeCompressedStringArray(out,
2252 updatingResource.get(item.getKey()));
2253 }
2254 }
2255
2256 /**
2257 * get keys matching the the regex
2258 * @param regex
2259 * @return Map<String,String> with matching keys
2260 */
2261 public Map<String,String> getValByRegex(String regex) {
2262 Pattern p = Pattern.compile(regex);
2263
2264 Map<String,String> result = new HashMap<String,String>();
2265 Matcher m;
2266
2267 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2268 if (item.getKey() instanceof String &&
2269 item.getValue() instanceof String) {
2270 m = p.matcher((String)item.getKey());
2271 if(m.find()) { // match
2272 result.put((String) item.getKey(), (String) item.getValue());
2273 }
2274 }
2275 }
2276 return result;
2277 }
2278
2279 //Load deprecated keys in common
2280 private static void addDeprecatedKeys() {
2281 Configuration.addDeprecation("topology.script.file.name",
2282 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
2283 Configuration.addDeprecation("topology.script.number.args",
2284 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
2285 Configuration.addDeprecation("hadoop.configured.node.mapping",
2286 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
2287 Configuration.addDeprecation("topology.node.switch.mapping.impl",
2288 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
2289 Configuration.addDeprecation("dfs.df.interval",
2290 new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
2291 Configuration.addDeprecation("hadoop.native.lib",
2292 new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
2293 Configuration.addDeprecation("fs.default.name",
2294 new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
2295 Configuration.addDeprecation("dfs.umaskmode",
2296 new String[]{CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY});
2297 }
2298
2299 /**
2300 * A unique class which is used as a sentinel value in the caching
2301 * for getClassByName. {@see Configuration#getClassByNameOrNull(String)}
2302 */
2303 private static abstract class NegativeCacheSentinel {}
2304
2305 public static void dumpDeprecatedKeys() {
2306 for (Map.Entry<String, DeprecatedKeyInfo> entry : deprecatedKeyMap.entrySet()) {
2307 String newKeys = "";
2308 for (String newKey : entry.getValue().newKeys) {
2309 newKeys += newKey + "\t";
2310 }
2311 System.out.println(entry.getKey() + "\t" + newKeys);
2312 }
2313 }
2314 }