001/* 002 * Copyright 2008-2014 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2014 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.LinkedHashMap; 030import java.util.LinkedHashSet; 031import java.util.List; 032import java.util.concurrent.CyclicBarrier; 033import java.util.concurrent.Semaphore; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicLong; 036 037import com.unboundid.ldap.sdk.LDAPConnection; 038import com.unboundid.ldap.sdk.LDAPConnectionOptions; 039import com.unboundid.ldap.sdk.LDAPException; 040import com.unboundid.ldap.sdk.ResultCode; 041import com.unboundid.ldap.sdk.SearchScope; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.util.ColumnFormatter; 044import com.unboundid.util.FixedRateBarrier; 045import com.unboundid.util.FormattableColumn; 046import com.unboundid.util.HorizontalAlignment; 047import com.unboundid.util.LDAPCommandLineTool; 048import com.unboundid.util.ObjectPair; 049import com.unboundid.util.OutputFormat; 050import com.unboundid.util.RateAdjustor; 051import com.unboundid.util.ResultCodeCounter; 052import com.unboundid.util.ThreadSafety; 053import com.unboundid.util.ThreadSafetyLevel; 054import com.unboundid.util.WakeableSleeper; 055import com.unboundid.util.ValuePattern; 056import com.unboundid.util.args.ArgumentException; 057import com.unboundid.util.args.ArgumentParser; 058import com.unboundid.util.args.BooleanArgument; 059import com.unboundid.util.args.FileArgument; 060import com.unboundid.util.args.IntegerArgument; 061import com.unboundid.util.args.ScopeArgument; 062import com.unboundid.util.args.StringArgument; 063 064import static com.unboundid.util.Debug.*; 065import static com.unboundid.util.StaticUtils.*; 066 067 068 069/** 070 * This class provides a tool that can be used to search an LDAP directory 071 * server repeatedly using multiple threads. It can help provide an estimate of 072 * the search performance that a directory server is able to achieve. Either or 073 * both of the base DN and the search filter may be a value pattern as 074 * described in the {@link ValuePattern} class. This makes it possible to 075 * search over a range of entries rather than repeatedly performing searches 076 * with the same base DN and filter. 077 * <BR><BR> 078 * Some of the APIs demonstrated by this example include: 079 * <UL> 080 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 081 * package)</LI> 082 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 083 * package)</LI> 084 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 085 * package)</LI> 086 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 087 * </UL> 088 * <BR><BR> 089 * All of the necessary information is provided using command line arguments. 090 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 091 * class, as well as the following additional arguments: 092 * <UL> 093 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 094 * for the searches. This must be provided. It may be a simple DN, or it 095 * may be a value pattern to express a range of base DNs.</LI> 096 * <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the 097 * search. The scope value should be one of "base", "one", "sub", or 098 * "subord". If this isn't specified, then a scope of "sub" will be 099 * used.</LI> 100 * <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for 101 * the searches. This must be provided. It may be a simple filter, or it 102 * may be a value pattern to express a range of filters.</LI> 103 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an 104 * attribute that should be included in entries returned from the server. 105 * If this is not provided, then all user attributes will be requested. 106 * This may include special tokens that the server may interpret, like 107 * "1.1" to indicate that no attributes should be returned, "*", for all 108 * user attributes, or "+" for all operational attributes. Multiple 109 * attributes may be requested with multiple instances of this 110 * argument.</LI> 111 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 112 * concurrent threads to use when performing the searches. If this is not 113 * provided, then a default of one thread will be used.</LI> 114 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 115 * time in seconds between lines out output. If this is not provided, 116 * then a default interval duration of five seconds will be used.</LI> 117 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 118 * intervals for which to run. If this is not provided, then it will 119 * run forever.</LI> 120 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search 121 * iterations that should be performed on a connection before that 122 * connection is closed and replaced with a newly-established (and 123 * authenticated, if appropriate) connection.</LI> 124 * <LI>"-r {searches-per-second}" or "--ratePerSecond {searches-per-second}" 125 * -- specifies the target number of searches to perform per second. It 126 * is still necessary to specify a sufficient number of threads for 127 * achieving this rate. If this option is not provided, then the tool 128 * will run at the maximum rate for the specified number of threads.</LI> 129 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 130 * information needed to allow the tool to vary the target rate over time. 131 * If this option is not provided, then the tool will either use a fixed 132 * target rate as specified by the "--ratePerSecond" argument, or it will 133 * run at the maximum rate.</LI> 134 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 135 * which sample data will be written illustrating and describing the 136 * format of the file expected to be used in conjunction with the 137 * "--variableRateData" argument.</LI> 138 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 139 * complete before beginning overall statistics collection.</LI> 140 * <LI>"--timestampFormat {format}" -- specifies the format to use for 141 * timestamps included before each output line. The format may be one of 142 * "none" (for no timestamps), "with-date" (to include both the date and 143 * the time), or "without-date" (to include only time time).</LI> 144 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 145 * authorization v2 control to request that the operation be processed 146 * using an alternate authorization identity. In this case, the bind DN 147 * should be that of a user that has permission to use this control. The 148 * authorization identity may be a value pattern.</LI> 149 * <LI>"-a" or "--asynchronous" -- Indicates that searches should be performed 150 * in asynchronous mode, in which the client will not wait for a response 151 * to a previous request before sending the next request. Either the 152 * "--ratePerSecond" or "--maxOutstandingRequests" arguments must be 153 * provided to limit the number of outstanding requests.</LI> 154 * <LI>"-O {num}" or "--maxOutstandingRequests {num}" -- Specifies the maximum 155 * number of outstanding requests that will be allowed in asynchronous 156 * mode.</LI> 157 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 158 * result codes for failed operations should not be displayed.</LI> 159 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 160 * display-friendly format.</LI> 161 * </UL> 162 */ 163@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 164public final class SearchRate 165 extends LDAPCommandLineTool 166 implements Serializable 167{ 168 /** 169 * The serial version UID for this serializable class. 170 */ 171 private static final long serialVersionUID = 3345838530404592182L; 172 173 174 175 // Indicates whether a request has been made to stop running. 176 private final AtomicBoolean stopRequested; 177 178 // The argument used to indicate whether to operate in asynchronous mode. 179 private BooleanArgument asynchronousMode; 180 181 // The argument used to indicate whether to generate output in CSV format. 182 private BooleanArgument csvFormat; 183 184 // The argument used to indicate whether to suppress information about error 185 // result codes. 186 private BooleanArgument suppressErrors; 187 188 // The argument used to specify the collection interval. 189 private IntegerArgument collectionInterval; 190 191 // The argument used to specify the number of search iterations on a 192 // connection before it is closed and re-established. 193 private IntegerArgument iterationsBeforeReconnect; 194 195 // The argument used to specify the maximum number of outstanding asynchronous 196 // requests. 197 private IntegerArgument maxOutstandingRequests; 198 199 // The argument used to specify the number of intervals. 200 private IntegerArgument numIntervals; 201 202 // The argument used to specify the number of threads. 203 private IntegerArgument numThreads; 204 205 // The argument used to specify the seed to use for the random number 206 // generator. 207 private IntegerArgument randomSeed; 208 209 // The target rate of searches per second. 210 private IntegerArgument ratePerSecond; 211 212 // The argument used to specify a variable rate file. 213 private FileArgument sampleRateFile; 214 215 // The argument used to specify a variable rate file. 216 private FileArgument variableRateData; 217 218 // The number of warm-up intervals to perform. 219 private IntegerArgument warmUpIntervals; 220 221 // The argument used to specify the scope for the searches. 222 private ScopeArgument scopeArg; 223 224 // The argument used to specify the attributes to return. 225 private StringArgument attributes; 226 227 // The argument used to specify the base DNs for the searches. 228 private StringArgument baseDN; 229 230 // The argument used to specify the filters for the searches. 231 private StringArgument filter; 232 233 // The argument used to specify the proxied authorization identity. 234 private StringArgument proxyAs; 235 236 // The argument used to specify the timestamp format. 237 private StringArgument timestampFormat; 238 239 // The thread currently being used to run the searchrate tool. 240 private volatile Thread runningThread; 241 242 // A wakeable sleeper that will be used to sleep between reporting intervals. 243 private final WakeableSleeper sleeper; 244 245 246 247 /** 248 * Parse the provided command line arguments and make the appropriate set of 249 * changes. 250 * 251 * @param args The command line arguments provided to this program. 252 */ 253 public static void main(final String[] args) 254 { 255 final ResultCode resultCode = main(args, System.out, System.err); 256 if (resultCode != ResultCode.SUCCESS) 257 { 258 System.exit(resultCode.intValue()); 259 } 260 } 261 262 263 264 /** 265 * Parse the provided command line arguments and make the appropriate set of 266 * changes. 267 * 268 * @param args The command line arguments provided to this program. 269 * @param outStream The output stream to which standard out should be 270 * written. It may be {@code null} if output should be 271 * suppressed. 272 * @param errStream The output stream to which standard error should be 273 * written. It may be {@code null} if error messages 274 * should be suppressed. 275 * 276 * @return A result code indicating whether the processing was successful. 277 */ 278 public static ResultCode main(final String[] args, 279 final OutputStream outStream, 280 final OutputStream errStream) 281 { 282 final SearchRate searchRate = new SearchRate(outStream, errStream); 283 return searchRate.runTool(args); 284 } 285 286 287 288 /** 289 * Creates a new instance of this tool. 290 * 291 * @param outStream The output stream to which standard out should be 292 * written. It may be {@code null} if output should be 293 * suppressed. 294 * @param errStream The output stream to which standard error should be 295 * written. It may be {@code null} if error messages 296 * should be suppressed. 297 */ 298 public SearchRate(final OutputStream outStream, final OutputStream errStream) 299 { 300 super(outStream, errStream); 301 302 stopRequested = new AtomicBoolean(false); 303 sleeper = new WakeableSleeper(); 304 } 305 306 307 308 /** 309 * Retrieves the name for this tool. 310 * 311 * @return The name for this tool. 312 */ 313 @Override() 314 public String getToolName() 315 { 316 return "searchrate"; 317 } 318 319 320 321 /** 322 * Retrieves the description for this tool. 323 * 324 * @return The description for this tool. 325 */ 326 @Override() 327 public String getToolDescription() 328 { 329 return "Perform repeated searches against an " + 330 "LDAP directory server."; 331 } 332 333 334 335 /** 336 * Retrieves the version string for this tool. 337 * 338 * @return The version string for this tool. 339 */ 340 @Override() 341 public String getToolVersion() 342 { 343 return Version.NUMERIC_VERSION_STRING; 344 } 345 346 347 348 /** 349 * Adds the arguments used by this program that aren't already provided by the 350 * generic {@code LDAPCommandLineTool} framework. 351 * 352 * @param parser The argument parser to which the arguments should be added. 353 * 354 * @throws ArgumentException If a problem occurs while adding the arguments. 355 */ 356 @Override() 357 public void addNonLDAPArguments(final ArgumentParser parser) 358 throws ArgumentException 359 { 360 String description = "The base DN to use for the searches. It may be a " + 361 "simple DN or a value pattern to specify a range of DNs (e.g., " + 362 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). This must be " + 363 "provided."; 364 baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description); 365 parser.addArgument(baseDN); 366 367 368 description = "The scope to use for the searches. It should be 'base', " + 369 "'one', 'sub', or 'subord'. If this is not provided, then " + 370 "a default scope of 'sub' will be used."; 371 scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description, 372 SearchScope.SUB); 373 parser.addArgument(scopeArg); 374 375 376 description = "The filter to use for the searches. It may be a simple " + 377 "filter or a value pattern to specify a range of filters " + 378 "(e.g., \"(uid=user.[1-1000])\"). This must be provided."; 379 filter = new StringArgument('f', "filter", true, 1, "{filter}", 380 description); 381 parser.addArgument(filter); 382 383 384 description = "The name of an attribute to include in entries returned " + 385 "from the searches. Multiple attributes may be requested " + 386 "by providing this argument multiple times. If no request " + 387 "attributes are provided, then the entries returned will " + 388 "include all user attributes."; 389 attributes = new StringArgument('A', "attribute", false, 0, "{name}", 390 description); 391 parser.addArgument(attributes); 392 393 394 description = "The number of threads to use to perform the searches. If " + 395 "this is not provided, then a default of one thread will " + 396 "be used."; 397 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 398 description, 1, Integer.MAX_VALUE, 1); 399 parser.addArgument(numThreads); 400 401 402 description = "The length of time in seconds between output lines. If " + 403 "this is not provided, then a default interval of five " + 404 "seconds will be used."; 405 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 406 "{num}", description, 1, 407 Integer.MAX_VALUE, 5); 408 parser.addArgument(collectionInterval); 409 410 411 description = "The maximum number of intervals for which to run. If " + 412 "this is not provided, then the tool will run until it is " + 413 "interrupted."; 414 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 415 description, 1, Integer.MAX_VALUE, 416 Integer.MAX_VALUE); 417 parser.addArgument(numIntervals); 418 419 description = "The number of search iterations that should be processed " + 420 "on a connection before that connection is closed and " + 421 "replaced with a newly-established (and authenticated, if " + 422 "appropriate) connection. If this is not provided, then " + 423 "connections will not be periodically closed and " + 424 "re-established."; 425 iterationsBeforeReconnect = new IntegerArgument(null, 426 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 427 parser.addArgument(iterationsBeforeReconnect); 428 429 description = "The target number of searches to perform per second. It " + 430 "is still necessary to specify a sufficient number of " + 431 "threads for achieving this rate. If neither this option " + 432 "nor --variableRateData is provided, then the tool will " + 433 "run at the maximum rate for the specified number of " + 434 "threads."; 435 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 436 "{searches-per-second}", description, 437 1, Integer.MAX_VALUE); 438 parser.addArgument(ratePerSecond); 439 440 final String variableRateDataArgName = "variableRateData"; 441 final String generateSampleRateFileArgName = "generateSampleRateFile"; 442 description = RateAdjustor.getVariableRateDataArgumentDescription( 443 generateSampleRateFileArgName); 444 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 445 "{path}", description, true, true, true, 446 false); 447 parser.addArgument(variableRateData); 448 449 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 450 variableRateDataArgName); 451 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 452 false, 1, "{path}", description, false, 453 true, true, false); 454 sampleRateFile.setUsageArgument(true); 455 parser.addArgument(sampleRateFile); 456 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 457 458 description = "The number of intervals to complete before beginning " + 459 "overall statistics collection. Specifying a nonzero " + 460 "number of warm-up intervals gives the client and server " + 461 "a chance to warm up without skewing performance results."; 462 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 463 "{num}", description, 0, Integer.MAX_VALUE, 0); 464 parser.addArgument(warmUpIntervals); 465 466 description = "Indicates the format to use for timestamps included in " + 467 "the output. A value of 'none' indicates that no " + 468 "timestamps should be included. A value of 'with-date' " + 469 "indicates that both the date and the time should be " + 470 "included. A value of 'without-date' indicates that only " + 471 "the time should be included."; 472 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3); 473 allowedFormats.add("none"); 474 allowedFormats.add("with-date"); 475 allowedFormats.add("without-date"); 476 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 477 "{format}", description, allowedFormats, "none"); 478 parser.addArgument(timestampFormat); 479 480 description = "Indicates that the proxied authorization control (as " + 481 "defined in RFC 4370) should be used to request that " + 482 "operations be processed using an alternate authorization " + 483 "identity."; 484 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 485 description); 486 parser.addArgument(proxyAs); 487 488 description = "Indicates that the client should operate in asynchronous " + 489 "mode, in which it will not be necessary to wait for a " + 490 "response to a previous request before sending the next " + 491 "request. Either the '--ratePerSecond' or the " + 492 "'--maxOutstandingRequests' argument must be provided to " + 493 "limit the number of outstanding requests."; 494 asynchronousMode = new BooleanArgument('a', "asynchronous", description); 495 parser.addArgument(asynchronousMode); 496 497 description = "Specifies the maximum number of outstanding requests " + 498 "that should be allowed when operating in asynchronous mode."; 499 maxOutstandingRequests = new IntegerArgument('O', "maxOutstandingRequests", 500 false, 1, "{num}", description, 1, Integer.MAX_VALUE, (Integer) null); 501 parser.addArgument(maxOutstandingRequests); 502 503 description = "Indicates that information about the result codes for " + 504 "failed operations should not be displayed."; 505 suppressErrors = new BooleanArgument(null, 506 "suppressErrorResultCodes", 1, description); 507 parser.addArgument(suppressErrors); 508 509 description = "Generate output in CSV format rather than a " + 510 "display-friendly format"; 511 csvFormat = new BooleanArgument('c', "csv", 1, description); 512 parser.addArgument(csvFormat); 513 514 description = "Specifies the seed to use for the random number generator."; 515 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 516 description); 517 parser.addArgument(randomSeed); 518 519 520 parser.addDependentArgumentSet(asynchronousMode, ratePerSecond, 521 maxOutstandingRequests); 522 parser.addDependentArgumentSet(maxOutstandingRequests, asynchronousMode); 523 } 524 525 526 527 /** 528 * Indicates whether this tool supports creating connections to multiple 529 * servers. If it is to support multiple servers, then the "--hostname" and 530 * "--port" arguments will be allowed to be provided multiple times, and 531 * will be required to be provided the same number of times. The same type of 532 * communication security and bind credentials will be used for all servers. 533 * 534 * @return {@code true} if this tool supports creating connections to 535 * multiple servers, or {@code false} if not. 536 */ 537 @Override() 538 protected boolean supportsMultipleServers() 539 { 540 return true; 541 } 542 543 544 545 /** 546 * Retrieves the connection options that should be used for connections 547 * created for use with this tool. 548 * 549 * @return The connection options that should be used for connections created 550 * for use with this tool. 551 */ 552 @Override() 553 public LDAPConnectionOptions getConnectionOptions() 554 { 555 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 556 options.setAutoReconnect(true); 557 options.setUseSynchronousMode(! asynchronousMode.isPresent()); 558 return options; 559 } 560 561 562 563 /** 564 * Performs the actual processing for this tool. In this case, it gets a 565 * connection to the directory server and uses it to perform the requested 566 * searches. 567 * 568 * @return The result code for the processing that was performed. 569 */ 570 @Override() 571 public ResultCode doToolProcessing() 572 { 573 runningThread = Thread.currentThread(); 574 575 try 576 { 577 return doToolProcessingInternal(); 578 } 579 finally 580 { 581 runningThread = null; 582 } 583 } 584 585 586 587 /** 588 * Performs the actual processing for this tool. In this case, it gets a 589 * connection to the directory server and uses it to perform the requested 590 * searches. 591 * 592 * @return The result code for the processing that was performed. 593 */ 594 private ResultCode doToolProcessingInternal() 595 { 596 // If the sample rate file argument was specified, then generate the sample 597 // variable rate data file and return. 598 if (sampleRateFile.isPresent()) 599 { 600 try 601 { 602 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 603 return ResultCode.SUCCESS; 604 } 605 catch (final Exception e) 606 { 607 debugException(e); 608 err("An error occurred while trying to write sample variable data " + 609 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 610 "': ", getExceptionMessage(e)); 611 return ResultCode.LOCAL_ERROR; 612 } 613 } 614 615 616 // Determine the random seed to use. 617 final Long seed; 618 if (randomSeed.isPresent()) 619 { 620 seed = Long.valueOf(randomSeed.getValue()); 621 } 622 else 623 { 624 seed = null; 625 } 626 627 // Create value patterns for the base DN, filter, and proxied authorization 628 // DN. 629 final ValuePattern dnPattern; 630 try 631 { 632 dnPattern = new ValuePattern(baseDN.getValue(), seed); 633 } 634 catch (final ParseException pe) 635 { 636 debugException(pe); 637 err("Unable to parse the base DN value pattern: ", pe.getMessage()); 638 return ResultCode.PARAM_ERROR; 639 } 640 641 final ValuePattern filterPattern; 642 try 643 { 644 filterPattern = new ValuePattern(filter.getValue(), seed); 645 } 646 catch (final ParseException pe) 647 { 648 debugException(pe); 649 err("Unable to parse the filter pattern: ", pe.getMessage()); 650 return ResultCode.PARAM_ERROR; 651 } 652 653 final ValuePattern authzIDPattern; 654 if (proxyAs.isPresent()) 655 { 656 try 657 { 658 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 659 } 660 catch (final ParseException pe) 661 { 662 debugException(pe); 663 err("Unable to parse the proxied authorization pattern: ", 664 pe.getMessage()); 665 return ResultCode.PARAM_ERROR; 666 } 667 } 668 else 669 { 670 authzIDPattern = null; 671 } 672 673 674 // Get the attributes to return. 675 final String[] attrs; 676 if (attributes.isPresent()) 677 { 678 final List<String> attrList = attributes.getValues(); 679 attrs = new String[attrList.size()]; 680 attrList.toArray(attrs); 681 } 682 else 683 { 684 attrs = NO_STRINGS; 685 } 686 687 688 // If the --ratePerSecond option was specified, then limit the rate 689 // accordingly. 690 FixedRateBarrier fixedRateBarrier = null; 691 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 692 { 693 // We might not have a rate per second if --variableRateData is specified. 694 // The rate typically doesn't matter except when we have warm-up 695 // intervals. In this case, we'll run at the max rate. 696 final int intervalSeconds = collectionInterval.getValue(); 697 final int ratePerInterval = 698 (ratePerSecond.getValue() == null) 699 ? Integer.MAX_VALUE 700 : ratePerSecond.getValue() * intervalSeconds; 701 fixedRateBarrier = 702 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 703 } 704 705 706 // If --variableRateData was specified, then initialize a RateAdjustor. 707 RateAdjustor rateAdjustor = null; 708 if (variableRateData.isPresent()) 709 { 710 try 711 { 712 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 713 ratePerSecond.getValue(), variableRateData.getValue()); 714 } 715 catch (final IOException e) 716 { 717 debugException(e); 718 err("Initializing the variable rates failed: " + e.getMessage()); 719 return ResultCode.PARAM_ERROR; 720 } 721 catch (final IllegalArgumentException e) 722 { 723 debugException(e); 724 err("Initializing the variable rates failed: " + e.getMessage()); 725 return ResultCode.PARAM_ERROR; 726 } 727 } 728 729 730 // If the --maxOutstandingRequests option was specified, then create the 731 // semaphore used to enforce that limit. 732 final Semaphore asyncSemaphore; 733 if (maxOutstandingRequests.isPresent()) 734 { 735 asyncSemaphore = new Semaphore(maxOutstandingRequests.getValue()); 736 } 737 else 738 { 739 asyncSemaphore = null; 740 } 741 742 743 // Determine whether to include timestamps in the output and if so what 744 // format should be used for them. 745 final boolean includeTimestamp; 746 final String timeFormat; 747 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 748 { 749 includeTimestamp = true; 750 timeFormat = "dd/MM/yyyy HH:mm:ss"; 751 } 752 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 753 { 754 includeTimestamp = true; 755 timeFormat = "HH:mm:ss"; 756 } 757 else 758 { 759 includeTimestamp = false; 760 timeFormat = null; 761 } 762 763 764 // Determine whether any warm-up intervals should be run. 765 final long totalIntervals; 766 final boolean warmUp; 767 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 768 if (remainingWarmUpIntervals > 0) 769 { 770 warmUp = true; 771 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 772 } 773 else 774 { 775 warmUp = true; 776 totalIntervals = 0L + numIntervals.getValue(); 777 } 778 779 780 // Create the table that will be used to format the output. 781 final OutputFormat outputFormat; 782 if (csvFormat.isPresent()) 783 { 784 outputFormat = OutputFormat.CSV; 785 } 786 else 787 { 788 outputFormat = OutputFormat.COLUMNS; 789 } 790 791 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 792 timeFormat, outputFormat, " ", 793 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 794 "Searches/Sec"), 795 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 796 "Avg Dur ms"), 797 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 798 "Entries/Srch"), 799 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 800 "Errors/Sec"), 801 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 802 "Searches/Sec"), 803 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 804 "Avg Dur ms")); 805 806 807 // Create values to use for statistics collection. 808 final AtomicLong searchCounter = new AtomicLong(0L); 809 final AtomicLong entryCounter = new AtomicLong(0L); 810 final AtomicLong errorCounter = new AtomicLong(0L); 811 final AtomicLong searchDurations = new AtomicLong(0L); 812 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 813 814 815 // Determine the length of each interval in milliseconds. 816 final long intervalMillis = 1000L * collectionInterval.getValue(); 817 818 819 // Create the threads to use for the searches. 820 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 821 final SearchRateThread[] threads = 822 new SearchRateThread[numThreads.getValue()]; 823 for (int i=0; i < threads.length; i++) 824 { 825 final LDAPConnection connection; 826 try 827 { 828 connection = getConnection(); 829 } 830 catch (final LDAPException le) 831 { 832 debugException(le); 833 err("Unable to connect to the directory server: ", 834 getExceptionMessage(le)); 835 return le.getResultCode(); 836 } 837 838 threads[i] = new SearchRateThread(this, i, connection, 839 asynchronousMode.isPresent(), dnPattern, scopeArg.getValue(), 840 filterPattern, attrs, authzIDPattern, 841 iterationsBeforeReconnect.getValue(), barrier, searchCounter, 842 entryCounter, searchDurations, errorCounter, rcCounter, 843 fixedRateBarrier, asyncSemaphore); 844 threads[i].start(); 845 } 846 847 848 // Display the table header. 849 for (final String headerLine : formatter.getHeaderLines(true)) 850 { 851 out(headerLine); 852 } 853 854 855 // Start the RateAdjustor before the threads so that the initial value is 856 // in place before any load is generated unless we're doing a warm-up in 857 // which case, we'll start it after the warm-up is complete. 858 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 859 { 860 rateAdjustor.start(); 861 } 862 863 864 // Indicate that the threads can start running. 865 try 866 { 867 barrier.await(); 868 } 869 catch (final Exception e) 870 { 871 debugException(e); 872 } 873 874 long overallStartTime = System.nanoTime(); 875 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 876 877 878 boolean setOverallStartTime = false; 879 long lastDuration = 0L; 880 long lastNumEntries = 0L; 881 long lastNumErrors = 0L; 882 long lastNumSearches = 0L; 883 long lastEndTime = System.nanoTime(); 884 for (long i=0; i < totalIntervals; i++) 885 { 886 if (rateAdjustor != null) 887 { 888 if (! rateAdjustor.isAlive()) 889 { 890 out("All of the rates in " + variableRateData.getValue().getName() + 891 " have been completed."); 892 break; 893 } 894 } 895 896 final long startTimeMillis = System.currentTimeMillis(); 897 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 898 nextIntervalStartTime += intervalMillis; 899 if (sleepTimeMillis > 0) 900 { 901 sleeper.sleep(sleepTimeMillis); 902 } 903 904 if (stopRequested.get()) 905 { 906 break; 907 } 908 909 final long endTime = System.nanoTime(); 910 final long intervalDuration = endTime - lastEndTime; 911 912 final long numSearches; 913 final long numEntries; 914 final long numErrors; 915 final long totalDuration; 916 if (warmUp && (remainingWarmUpIntervals > 0)) 917 { 918 numSearches = searchCounter.getAndSet(0L); 919 numEntries = entryCounter.getAndSet(0L); 920 numErrors = errorCounter.getAndSet(0L); 921 totalDuration = searchDurations.getAndSet(0L); 922 } 923 else 924 { 925 numSearches = searchCounter.get(); 926 numEntries = entryCounter.get(); 927 numErrors = errorCounter.get(); 928 totalDuration = searchDurations.get(); 929 } 930 931 final long recentNumSearches = numSearches - lastNumSearches; 932 final long recentNumEntries = numEntries - lastNumEntries; 933 final long recentNumErrors = numErrors - lastNumErrors; 934 final long recentDuration = totalDuration - lastDuration; 935 936 final double numSeconds = intervalDuration / 1000000000.0d; 937 final double recentSearchRate = recentNumSearches / numSeconds; 938 final double recentErrorRate = recentNumErrors / numSeconds; 939 940 final double recentAvgDuration; 941 final double recentEntriesPerSearch; 942 if (recentNumSearches > 0L) 943 { 944 recentEntriesPerSearch = 1.0d * recentNumEntries / recentNumSearches; 945 recentAvgDuration = 1.0d * recentDuration / recentNumSearches / 1000000; 946 } 947 else 948 { 949 recentEntriesPerSearch = 0.0d; 950 recentAvgDuration = 0.0d; 951 } 952 953 954 if (warmUp && (remainingWarmUpIntervals > 0)) 955 { 956 out(formatter.formatRow(recentSearchRate, recentAvgDuration, 957 recentEntriesPerSearch, recentErrorRate, "warming up", 958 "warming up")); 959 960 remainingWarmUpIntervals--; 961 if (remainingWarmUpIntervals == 0) 962 { 963 out("Warm-up completed. Beginning overall statistics collection."); 964 setOverallStartTime = true; 965 if (rateAdjustor != null) 966 { 967 rateAdjustor.start(); 968 } 969 } 970 } 971 else 972 { 973 if (setOverallStartTime) 974 { 975 overallStartTime = lastEndTime; 976 setOverallStartTime = false; 977 } 978 979 final double numOverallSeconds = 980 (endTime - overallStartTime) / 1000000000.0d; 981 final double overallSearchRate = numSearches / numOverallSeconds; 982 983 final double overallAvgDuration; 984 if (numSearches > 0L) 985 { 986 overallAvgDuration = 1.0d * totalDuration / numSearches / 1000000; 987 } 988 else 989 { 990 overallAvgDuration = 0.0d; 991 } 992 993 out(formatter.formatRow(recentSearchRate, recentAvgDuration, 994 recentEntriesPerSearch, recentErrorRate, overallSearchRate, 995 overallAvgDuration)); 996 997 lastNumSearches = numSearches; 998 lastNumEntries = numEntries; 999 lastNumErrors = numErrors; 1000 lastDuration = totalDuration; 1001 } 1002 1003 final List<ObjectPair<ResultCode,Long>> rcCounts = 1004 rcCounter.getCounts(true); 1005 if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty())) 1006 { 1007 err("\tError Results:"); 1008 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1009 { 1010 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1011 } 1012 } 1013 1014 lastEndTime = endTime; 1015 } 1016 1017 1018 // Shut down the RateAdjustor if we have one. 1019 if (rateAdjustor != null) 1020 { 1021 rateAdjustor.shutDown(); 1022 } 1023 1024 1025 // Stop all of the threads. 1026 ResultCode resultCode = ResultCode.SUCCESS; 1027 for (final SearchRateThread t : threads) 1028 { 1029 t.signalShutdown(); 1030 } 1031 for (final SearchRateThread t : threads) 1032 { 1033 final ResultCode r = t.waitForShutdown(); 1034 if (resultCode == ResultCode.SUCCESS) 1035 { 1036 resultCode = r; 1037 } 1038 } 1039 1040 return resultCode; 1041 } 1042 1043 1044 1045 /** 1046 * Requests that this tool stop running. This method will attempt to wait 1047 * for all threads to complete before returning control to the caller. 1048 */ 1049 public void stopRunning() 1050 { 1051 stopRequested.set(true); 1052 sleeper.wakeup(); 1053 1054 final Thread t = runningThread; 1055 if (t != null) 1056 { 1057 try 1058 { 1059 t.join(); 1060 } 1061 catch (final Exception e) 1062 { 1063 debugException(e); 1064 } 1065 } 1066 } 1067 1068 1069 1070 /** 1071 * Retrieves the maximum number of outstanding requests that may be in 1072 * progress at any time, if appropriate. 1073 * 1074 * @return The maximum number of outstanding requests that may be in progress 1075 * at any time, or -1 if the tool was not configured to perform 1076 * asynchronous searches with a maximum number of outstanding 1077 * requests. 1078 */ 1079 int getMaxOutstandingRequests() 1080 { 1081 if (maxOutstandingRequests.isPresent()) 1082 { 1083 return maxOutstandingRequests.getValue(); 1084 } 1085 else 1086 { 1087 return -1; 1088 } 1089 } 1090 1091 1092 1093 /** 1094 * {@inheritDoc} 1095 */ 1096 @Override() 1097 public LinkedHashMap<String[],String> getExampleUsages() 1098 { 1099 final LinkedHashMap<String[],String> examples = 1100 new LinkedHashMap<String[],String>(2); 1101 1102 String[] args = 1103 { 1104 "--hostname", "server.example.com", 1105 "--port", "389", 1106 "--bindDN", "uid=admin,dc=example,dc=com", 1107 "--bindPassword", "password", 1108 "--baseDN", "dc=example,dc=com", 1109 "--scope", "sub", 1110 "--filter", "(uid=user.[1-1000000])", 1111 "--attribute", "givenName", 1112 "--attribute", "sn", 1113 "--attribute", "mail", 1114 "--numThreads", "10" 1115 }; 1116 String description = 1117 "Test search performance by searching randomly across a set " + 1118 "of one million users located below 'dc=example,dc=com' with ten " + 1119 "concurrent threads. The entries returned to the client will " + 1120 "include the givenName, sn, and mail attributes."; 1121 examples.put(args, description); 1122 1123 args = new String[] 1124 { 1125 "--generateSampleRateFile", "variable-rate-data.txt" 1126 }; 1127 description = 1128 "Generate a sample variable rate definition file that may be used " + 1129 "in conjunction with the --variableRateData argument. The sample " + 1130 "file will include comments that describe the format for data to be " + 1131 "included in this file."; 1132 examples.put(args, description); 1133 1134 return examples; 1135 } 1136}