001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.mapred; 020 021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString; 022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString; 023 024 import java.io.DataInput; 025 import java.io.DataOutput; 026 import java.io.IOException; 027 import java.text.ParseException; 028 import java.util.Collection; 029 import java.util.Iterator; 030 031 import org.apache.commons.collections.IteratorUtils; 032 import org.apache.commons.logging.Log; 033 import org.apache.hadoop.classification.InterfaceAudience; 034 import org.apache.hadoop.classification.InterfaceStability; 035 import org.apache.hadoop.mapreduce.FileSystemCounter; 036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; 037 import org.apache.hadoop.mapreduce.counters.AbstractCounters; 038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase; 039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; 040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; 041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; 042 import org.apache.hadoop.mapreduce.counters.GenericCounter; 043 import org.apache.hadoop.mapreduce.counters.Limits; 044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; 045 import org.apache.hadoop.mapreduce.util.CountersStrings; 046 047 import com.google.common.collect.Iterators; 048 049 /** 050 * A set of named counters. 051 * 052 * <p><code>Counters</code> represent global counters, defined either by the 053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of 054 * any {@link Enum} type.</p> 055 * 056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of 057 * counters from a particular <code>Enum</code> class. 058 */ 059 @InterfaceAudience.Public 060 @InterfaceStability.Stable 061 public class Counters 062 extends AbstractCounters<Counters.Counter, Counters.Group> { 063 064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX; 065 066 public Counters() { 067 super(groupFactory); 068 } 069 070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { 071 super(newCounters, groupFactory); 072 } 073 074 /** 075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters 076 * @param newCounters new Counters 077 * @return old Counters instance corresponding to newCounters 078 */ 079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { 080 return new Counters(newCounters); 081 } 082 083 public synchronized Group getGroup(String groupName) { 084 return super.getGroup(groupName); 085 } 086 087 @SuppressWarnings("unchecked") 088 public synchronized Collection<String> getGroupNames() { 089 return IteratorUtils.toList(super.getGroupNames().iterator()); 090 } 091 092 public synchronized String makeCompactString() { 093 StringBuilder builder = new StringBuilder(); 094 boolean first = true; 095 for(Group group: this){ 096 for(Counter counter: group) { 097 if (first) { 098 first = false; 099 } else { 100 builder.append(','); 101 } 102 builder.append(group.getDisplayName()); 103 builder.append('.'); 104 builder.append(counter.getDisplayName()); 105 builder.append(':'); 106 builder.append(counter.getCounter()); 107 } 108 } 109 return builder.toString(); 110 } 111 112 /** 113 * A counter record, comprising its name and value. 114 */ 115 @InterfaceAudience.Public 116 @InterfaceStability.Stable 117 public static class Counter implements org.apache.hadoop.mapreduce.Counter { 118 org.apache.hadoop.mapreduce.Counter realCounter; 119 120 Counter(org.apache.hadoop.mapreduce.Counter counter) { 121 this.realCounter = counter; 122 } 123 124 public Counter() { 125 this(new GenericCounter()); 126 } 127 128 @SuppressWarnings("deprecation") 129 @Override 130 public void setDisplayName(String displayName) { 131 realCounter.setDisplayName(displayName); 132 } 133 134 @Override 135 public String getName() { 136 return realCounter.getName(); 137 } 138 139 @Override 140 public String getDisplayName() { 141 return realCounter.getDisplayName(); 142 } 143 144 @Override 145 public long getValue() { 146 return realCounter.getValue(); 147 } 148 149 @Override 150 public void setValue(long value) { 151 realCounter.setValue(value); 152 } 153 154 @Override 155 public void increment(long incr) { 156 realCounter.increment(incr); 157 } 158 159 @Override 160 public void write(DataOutput out) throws IOException { 161 realCounter.write(out); 162 } 163 164 @Override 165 public void readFields(DataInput in) throws IOException { 166 realCounter.readFields(in); 167 } 168 169 /** 170 * Returns the compact stringified version of the counter in the format 171 * [(actual-name)(display-name)(value)] 172 * @return the stringified result 173 */ 174 public String makeEscapedCompactString() { 175 return toEscapedCompactString(realCounter); 176 } 177 178 /** 179 * Checks for (content) equality of two (basic) counters 180 * @param counter to compare 181 * @return true if content equals 182 * @deprecated 183 */ 184 @Deprecated 185 public boolean contentEquals(Counter counter) { 186 return realCounter.equals(counter.getUnderlyingCounter()); 187 } 188 189 /** 190 * @return the value of the counter 191 */ 192 public long getCounter() { 193 return realCounter.getValue(); 194 } 195 196 @Override 197 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() { 198 return realCounter; 199 } 200 201 @Override 202 public synchronized boolean equals(Object genericRight) { 203 if (genericRight instanceof Counter) { 204 synchronized (genericRight) { 205 Counter right = (Counter) genericRight; 206 return getName().equals(right.getName()) && 207 getDisplayName().equals(right.getDisplayName()) && 208 getValue() == right.getValue(); 209 } 210 } 211 return false; 212 } 213 214 @Override 215 public int hashCode() { 216 return realCounter.hashCode(); 217 } 218 } 219 220 221 /** 222 * <code>Group</code> of counters, comprising of counters from a particular 223 * counter {@link Enum} class. 224 * 225 * <p><code>Group</code>handles localization of the class name and the 226 * counter names.</p> 227 */ 228 @InterfaceAudience.Public 229 @InterfaceStability.Stable 230 public static class Group implements CounterGroupBase<Counter> { 231 private CounterGroupBase<Counter> realGroup; 232 233 protected Group() { 234 realGroup = null; 235 } 236 237 Group(GenericGroup group) { 238 this.realGroup = group; 239 } 240 Group(FSGroupImpl group) { 241 this.realGroup = group; 242 } 243 244 @SuppressWarnings({ "unchecked", "rawtypes" }) 245 Group(FrameworkGroupImpl group) { 246 this.realGroup = group; 247 } 248 249 /** 250 * @param counterName the name of the counter 251 * @return the value of the specified counter, or 0 if the counter does 252 * not exist. 253 */ 254 public long getCounter(String counterName) { 255 return getCounterValue(realGroup, counterName); 256 } 257 258 /** 259 * @return the compact stringified version of the group in the format 260 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings 261 * for the counters within. 262 */ 263 public String makeEscapedCompactString() { 264 return toEscapedCompactString(realGroup); 265 } 266 267 /** 268 * Get the counter for the given id and create it if it doesn't exist. 269 * @param id the numeric id of the counter within the group 270 * @param name the internal counter name 271 * @return the counter 272 * @deprecated use {@link #findCounter(String)} instead 273 */ 274 @Deprecated 275 public Counter getCounter(int id, String name) { 276 return findCounter(name); 277 } 278 279 /** 280 * Get the counter for the given name and create it if it doesn't exist. 281 * @param name the internal counter name 282 * @return the counter 283 */ 284 public Counter getCounterForName(String name) { 285 return findCounter(name); 286 } 287 288 @Override 289 public void write(DataOutput out) throws IOException { 290 realGroup.write(out); 291 } 292 293 @Override 294 public void readFields(DataInput in) throws IOException { 295 realGroup.readFields(in); 296 } 297 298 @Override 299 public Iterator<Counter> iterator() { 300 return realGroup.iterator(); 301 } 302 303 @Override 304 public String getName() { 305 return realGroup.getName(); 306 } 307 308 @Override 309 public String getDisplayName() { 310 return realGroup.getDisplayName(); 311 } 312 313 @Override 314 public void setDisplayName(String displayName) { 315 realGroup.setDisplayName(displayName); 316 } 317 318 @Override 319 public void addCounter(Counter counter) { 320 realGroup.addCounter(counter); 321 } 322 323 @Override 324 public Counter addCounter(String name, String displayName, long value) { 325 return realGroup.addCounter(name, displayName, value); 326 } 327 328 @Override 329 public Counter findCounter(String counterName, String displayName) { 330 return realGroup.findCounter(counterName, displayName); 331 } 332 333 @Override 334 public Counter findCounter(String counterName, boolean create) { 335 return realGroup.findCounter(counterName, create); 336 } 337 338 @Override 339 public Counter findCounter(String counterName) { 340 return realGroup.findCounter(counterName); 341 } 342 343 @Override 344 public int size() { 345 return realGroup.size(); 346 } 347 348 @Override 349 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) { 350 realGroup.incrAllCounters(rightGroup); 351 } 352 353 @Override 354 public CounterGroupBase<Counter> getUnderlyingGroup() { 355 return realGroup; 356 } 357 358 @Override 359 public synchronized boolean equals(Object genericRight) { 360 if (genericRight instanceof CounterGroupBase<?>) { 361 @SuppressWarnings("unchecked") 362 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 363 genericRight).getUnderlyingGroup(); 364 return Iterators.elementsEqual(iterator(), right.iterator()); 365 } 366 return false; 367 } 368 369 @Override 370 public int hashCode() { 371 return realGroup.hashCode(); 372 } 373 } 374 375 // All the group impls need this for legacy group interface 376 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) { 377 Counter counter = group.findCounter(counterName, false); 378 if (counter != null) return counter.getValue(); 379 return 0L; 380 } 381 382 // Mix the generic group implementation into the Group interface 383 private static class GenericGroup extends AbstractCounterGroup<Counter> { 384 385 GenericGroup(String name, String displayName, Limits limits) { 386 super(name, displayName, limits); 387 } 388 389 @Override 390 protected Counter newCounter(String counterName, String displayName, 391 long value) { 392 return new Counter(new GenericCounter(counterName, displayName, value)); 393 } 394 395 @Override 396 protected Counter newCounter() { 397 return new Counter(); 398 } 399 400 @Override 401 public CounterGroupBase<Counter> getUnderlyingGroup() { 402 return this; 403 } 404 } 405 406 // Mix the framework group implementation into the Group interface 407 private static class FrameworkGroupImpl<T extends Enum<T>> 408 extends FrameworkCounterGroup<T, Counter> { 409 410 FrameworkGroupImpl(Class<T> cls) { 411 super(cls); 412 } 413 414 @Override 415 protected Counter newCounter(T key) { 416 return new Counter(new FrameworkCounter<T>(key, getName())); 417 } 418 419 @Override 420 public CounterGroupBase<Counter> getUnderlyingGroup() { 421 return this; 422 } 423 } 424 425 // Mix the file system counter group implementation into the Group interface 426 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> { 427 428 @Override 429 protected Counter newCounter(String scheme, FileSystemCounter key) { 430 return new Counter(new FSCounter(scheme, key)); 431 } 432 433 @Override 434 public CounterGroupBase<Counter> getUnderlyingGroup() { 435 return this; 436 } 437 } 438 439 public synchronized Counter findCounter(String group, String name) { 440 if (name.equals("MAP_INPUT_BYTES")) { 441 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + 442 "Use FileInputFormatCounters as group name and " + 443 " BYTES_READ as counter name instead"); 444 return findCounter(FileInputFormatCounter.BYTES_READ); 445 } 446 return getGroup(group).getCounterForName(name); 447 } 448 449 /** 450 * Provide factory methods for counter group factory implementation. 451 * See also the GroupFactory in 452 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} 453 */ 454 static class GroupFactory extends CounterGroupFactory<Counter, Group> { 455 456 @Override 457 protected <T extends Enum<T>> 458 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) { 459 return new FrameworkGroupFactory<Group>() { 460 @Override public Group newGroup(String name) { 461 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package 462 } 463 }; 464 } 465 466 @Override 467 protected Group newGenericGroup(String name, String displayName, 468 Limits limits) { 469 return new Group(new GenericGroup(name, displayName, limits)); 470 } 471 472 @Override 473 protected Group newFileSystemGroup() { 474 return new Group(new FSGroupImpl()); 475 } 476 } 477 478 private static final GroupFactory groupFactory = new GroupFactory(); 479 480 /** 481 * Find a counter by using strings 482 * @param group the name of the group 483 * @param id the id of the counter within the group (0 to N-1) 484 * @param name the internal name of the counter 485 * @return the counter for that name 486 * @deprecated use {@link #findCounter(String, String)} instead 487 */ 488 @Deprecated 489 public Counter findCounter(String group, int id, String name) { 490 return findCounter(group, name); 491 } 492 493 /** 494 * Increments the specified counter by the specified amount, creating it if 495 * it didn't already exist. 496 * @param key identifies a counter 497 * @param amount amount by which counter is to be incremented 498 */ 499 public void incrCounter(Enum<?> key, long amount) { 500 findCounter(key).increment(amount); 501 } 502 503 /** 504 * Increments the specified counter by the specified amount, creating it if 505 * it didn't already exist. 506 * @param group the name of the group 507 * @param counter the internal name of the counter 508 * @param amount amount by which counter is to be incremented 509 */ 510 public void incrCounter(String group, String counter, long amount) { 511 findCounter(group, counter).increment(amount); 512 } 513 514 /** 515 * Returns current value of the specified counter, or 0 if the counter 516 * does not exist. 517 * @param key the counter enum to lookup 518 * @return the counter value or 0 if counter not found 519 */ 520 public synchronized long getCounter(Enum<?> key) { 521 return findCounter(key).getValue(); 522 } 523 524 /** 525 * Increments multiple counters by their amounts in another Counters 526 * instance. 527 * @param other the other Counters instance 528 */ 529 public synchronized void incrAllCounters(Counters other) { 530 for (Group otherGroup: other) { 531 Group group = getGroup(otherGroup.getName()); 532 group.setDisplayName(otherGroup.getDisplayName()); 533 for (Counter otherCounter : otherGroup) { 534 Counter counter = group.getCounterForName(otherCounter.getName()); 535 counter.setDisplayName(otherCounter.getDisplayName()); 536 counter.increment(otherCounter.getValue()); 537 } 538 } 539 } 540 541 /** 542 * @return the total number of counters 543 * @deprecated use {@link #countCounters()} instead 544 */ 545 public int size() { 546 return countCounters(); 547 } 548 549 /** 550 * Convenience method for computing the sum of two sets of counters. 551 * @param a the first counters 552 * @param b the second counters 553 * @return a new summed counters object 554 */ 555 public static Counters sum(Counters a, Counters b) { 556 Counters counters = new Counters(); 557 counters.incrAllCounters(a); 558 counters.incrAllCounters(b); 559 return counters; 560 } 561 562 /** 563 * Logs the current counter values. 564 * @param log The log to use. 565 */ 566 public void log(Log log) { 567 log.info("Counters: " + size()); 568 for(Group group: this) { 569 log.info(" " + group.getDisplayName()); 570 for (Counter counter: group) { 571 log.info(" " + counter.getDisplayName() + "=" + 572 counter.getCounter()); 573 } 574 } 575 } 576 577 /** 578 * Represent the counter in a textual format that can be converted back to 579 * its object form 580 * @return the string in the following format 581 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* 582 */ 583 public String makeEscapedCompactString() { 584 return toEscapedCompactString(this); 585 } 586 587 /** 588 * Convert a stringified (by {@link #makeEscapedCompactString()} counter 589 * representation into a counter object. 590 * @param compactString to parse 591 * @return a new counters object 592 * @throws ParseException 593 */ 594 public static Counters fromEscapedCompactString(String compactString) 595 throws ParseException { 596 return parseEscapedCompactString(compactString, new Counters()); 597 } 598 }