001/* 002 * Copyright 2008-2018 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2018 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.ArrayList; 030import java.util.LinkedHashMap; 031import java.util.LinkedHashSet; 032import java.util.List; 033import java.util.concurrent.CyclicBarrier; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicLong; 036 037import com.unboundid.ldap.sdk.Control; 038import com.unboundid.ldap.sdk.LDAPConnection; 039import com.unboundid.ldap.sdk.LDAPConnectionOptions; 040import com.unboundid.ldap.sdk.LDAPException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.ldap.sdk.controls.AssertionRequestControl; 044import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl; 045import com.unboundid.ldap.sdk.controls.PostReadRequestControl; 046import com.unboundid.ldap.sdk.controls.PreReadRequestControl; 047import com.unboundid.util.ColumnFormatter; 048import com.unboundid.util.Debug; 049import com.unboundid.util.FixedRateBarrier; 050import com.unboundid.util.FormattableColumn; 051import com.unboundid.util.HorizontalAlignment; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.ObjectPair; 054import com.unboundid.util.OutputFormat; 055import com.unboundid.util.RateAdjustor; 056import com.unboundid.util.ResultCodeCounter; 057import com.unboundid.util.StaticUtils; 058import com.unboundid.util.ThreadSafety; 059import com.unboundid.util.ThreadSafetyLevel; 060import com.unboundid.util.ValuePattern; 061import com.unboundid.util.WakeableSleeper; 062import com.unboundid.util.args.ArgumentException; 063import com.unboundid.util.args.ArgumentParser; 064import com.unboundid.util.args.BooleanArgument; 065import com.unboundid.util.args.ControlArgument; 066import com.unboundid.util.args.FileArgument; 067import com.unboundid.util.args.FilterArgument; 068import com.unboundid.util.args.IntegerArgument; 069import com.unboundid.util.args.StringArgument; 070 071 072 073/** 074 * This class provides a tool that can be used to perform repeated modifications 075 * in an LDAP directory server using multiple threads. It can help provide an 076 * estimate of the modify performance that a directory server is able to 077 * achieve. The target entry DN may be a value pattern as described in the 078 * {@link ValuePattern} class. This makes it possible to modify a range of 079 * entries rather than repeatedly updating the same entry. 080 * <BR><BR> 081 * Some of the APIs demonstrated by this example include: 082 * <UL> 083 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 084 * package)</LI> 085 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 086 * package)</LI> 087 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 088 * package)</LI> 089 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 090 * </UL> 091 * <BR><BR> 092 * All of the necessary information is provided using command line arguments. 093 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 094 * class, as well as the following additional arguments: 095 * <UL> 096 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the 097 * entry to be modified. This must be provided. It may be a simple DN, 098 * or it may be a value pattern to express a range of entry DNs.</LI> 099 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the 100 * attribute to modify. Multiple attributes may be modified by providing 101 * multiple instances of this argument. At least one attribute must be 102 * provided.</LI> 103 * <LI>"--valuePattern {pattern}" -- specifies the pattern to use to generate 104 * the value to use for each modification. If this argument is provided, 105 * then neither the "--valueLength" nor "--characterSet" arguments may be 106 * given.</LI> 107 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 108 * use for the values of the target attributes. If this is not provided, 109 * then a default length of 10 bytes will be used.</LI> 110 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 111 * characters that will be used to generate the values to use for the 112 * target attributes. It should only include ASCII characters. Values 113 * will be generated from randomly-selected characters from this set. If 114 * this is not provided, then a default set of lowercase alphabetic 115 * characters will be used.</LI> 116 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 117 * concurrent threads to use when performing the modifications. If this 118 * is not provided, then a default of one thread will be used.</LI> 119 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 120 * time in seconds between lines out output. If this is not provided, 121 * then a default interval duration of five seconds will be used.</LI> 122 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 123 * intervals for which to run. If this is not provided, then it will 124 * run forever.</LI> 125 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify 126 * iterations that should be performed on a connection before that 127 * connection is closed and replaced with a newly-established (and 128 * authenticated, if appropriate) connection.</LI> 129 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}" 130 * -- specifies the target number of modifies to perform per second. It 131 * is still necessary to specify a sufficient number of threads for 132 * achieving this rate. If this option is not provided, then the tool 133 * will run at the maximum rate for the specified number of threads.</LI> 134 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 135 * information needed to allow the tool to vary the target rate over time. 136 * If this option is not provided, then the tool will either use a fixed 137 * target rate as specified by the "--ratePerSecond" argument, or it will 138 * run at the maximum rate.</LI> 139 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 140 * which sample data will be written illustrating and describing the 141 * format of the file expected to be used in conjunction with the 142 * "--variableRateData" argument.</LI> 143 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 144 * complete before beginning overall statistics collection.</LI> 145 * <LI>"--timestampFormat {format}" -- specifies the format to use for 146 * timestamps included before each output line. The format may be one of 147 * "none" (for no timestamps), "with-date" (to include both the date and 148 * the time), or "without-date" (to include only time time).</LI> 149 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 150 * authorization v2 control to request that the operation be processed 151 * using an alternate authorization identity. In this case, the bind DN 152 * should be that of a user that has permission to use this control. The 153 * authorization identity may be a value pattern.</LI> 154 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 155 * result codes for failed operations should not be displayed.</LI> 156 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 157 * display-friendly format.</LI> 158 * </UL> 159 */ 160@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 161public final class ModRate 162 extends LDAPCommandLineTool 163 implements Serializable 164{ 165 /** 166 * The serial version UID for this serializable class. 167 */ 168 private static final long serialVersionUID = 2709717414202815822L; 169 170 171 172 // Indicates whether a request has been made to stop running. 173 private final AtomicBoolean stopRequested; 174 175 // The argument used to indicate whether to generate output in CSV format. 176 private BooleanArgument csvFormat; 177 178 // Indicates that the tool should use the increment modification type instead 179 // of replace. 180 private BooleanArgument increment; 181 182 // Indicates that modify requests should include the permissive modify request 183 // control. 184 private BooleanArgument permissiveModify; 185 186 // The argument used to indicate whether to suppress information about error 187 // result codes. 188 private BooleanArgument suppressErrorsArgument; 189 190 // The argument used to indicate that a generic control should be included in 191 // the request. 192 private ControlArgument control; 193 194 // The argument used to specify a variable rate file. 195 private FileArgument sampleRateFile; 196 197 // The argument used to specify a variable rate file. 198 private FileArgument variableRateData; 199 200 // Indicates that modify requests should include the assertion request control 201 // with the specified filter. 202 private FilterArgument assertionFilter; 203 204 // The argument used to specify the collection interval. 205 private IntegerArgument collectionInterval; 206 207 // The increment amount to use when performing an increment instead of a 208 // replace. 209 private IntegerArgument incrementAmount; 210 211 // The argument used to specify the number of modify iterations on a 212 // connection before it is closed and re-established. 213 private IntegerArgument iterationsBeforeReconnect; 214 215 // The argument used to specify the number of intervals. 216 private IntegerArgument numIntervals; 217 218 // The argument used to specify the number of threads. 219 private IntegerArgument numThreads; 220 221 // The argument used to specify the seed to use for the random number 222 // generator. 223 private IntegerArgument randomSeed; 224 225 // The target rate of modifies per second. 226 private IntegerArgument ratePerSecond; 227 228 // The number of values to include in the replace modification. 229 private IntegerArgument valueCount; 230 231 // The argument used to specify the length of the values to generate. 232 private IntegerArgument valueLength; 233 234 // The number of warm-up intervals to perform. 235 private IntegerArgument warmUpIntervals; 236 237 // The argument used to specify the name of the attribute to modify. 238 private StringArgument attribute; 239 240 // The argument used to specify the set of characters to use when generating 241 // values. 242 private StringArgument characterSet; 243 244 // The argument used to specify the DNs of the entries to modify. 245 private StringArgument entryDN; 246 247 // Indicates that modify requests should include the post-read request control 248 // to request the specified attribute. 249 private StringArgument postReadAttribute; 250 251 // Indicates that modify requests should include the pre-read request control 252 // to request the specified attribute. 253 private StringArgument preReadAttribute; 254 255 // The argument used to specify the proxied authorization identity. 256 private StringArgument proxyAs; 257 258 // The argument used to specify the timestamp format. 259 private StringArgument timestampFormat; 260 261 // The argument used to specify the pattern to use to generate values. 262 private StringArgument valuePattern; 263 264 // The thread currently being used to run the searchrate tool. 265 private volatile Thread runningThread; 266 267 // A wakeable sleeper that will be used to sleep between reporting intervals. 268 private final WakeableSleeper sleeper; 269 270 271 272 /** 273 * Parse the provided command line arguments and make the appropriate set of 274 * changes. 275 * 276 * @param args The command line arguments provided to this program. 277 */ 278 public static void main(final String[] args) 279 { 280 final ResultCode resultCode = main(args, System.out, System.err); 281 if (resultCode != ResultCode.SUCCESS) 282 { 283 System.exit(resultCode.intValue()); 284 } 285 } 286 287 288 289 /** 290 * Parse the provided command line arguments and make the appropriate set of 291 * changes. 292 * 293 * @param args The command line arguments provided to this program. 294 * @param outStream The output stream to which standard out should be 295 * written. It may be {@code null} if output should be 296 * suppressed. 297 * @param errStream The output stream to which standard error should be 298 * written. It may be {@code null} if error messages 299 * should be suppressed. 300 * 301 * @return A result code indicating whether the processing was successful. 302 */ 303 public static ResultCode main(final String[] args, 304 final OutputStream outStream, 305 final OutputStream errStream) 306 { 307 final ModRate modRate = new ModRate(outStream, errStream); 308 return modRate.runTool(args); 309 } 310 311 312 313 /** 314 * Creates a new instance of this tool. 315 * 316 * @param outStream The output stream to which standard out should be 317 * written. It may be {@code null} if output should be 318 * suppressed. 319 * @param errStream The output stream to which standard error should be 320 * written. It may be {@code null} if error messages 321 * should be suppressed. 322 */ 323 public ModRate(final OutputStream outStream, final OutputStream errStream) 324 { 325 super(outStream, errStream); 326 327 stopRequested = new AtomicBoolean(false); 328 sleeper = new WakeableSleeper(); 329 } 330 331 332 333 /** 334 * Retrieves the name for this tool. 335 * 336 * @return The name for this tool. 337 */ 338 @Override() 339 public String getToolName() 340 { 341 return "modrate"; 342 } 343 344 345 346 /** 347 * Retrieves the description for this tool. 348 * 349 * @return The description for this tool. 350 */ 351 @Override() 352 public String getToolDescription() 353 { 354 return "Perform repeated modifications against " + 355 "an LDAP directory server."; 356 } 357 358 359 360 /** 361 * Retrieves the version string for this tool. 362 * 363 * @return The version string for this tool. 364 */ 365 @Override() 366 public String getToolVersion() 367 { 368 return Version.NUMERIC_VERSION_STRING; 369 } 370 371 372 373 /** 374 * Indicates whether this tool should provide support for an interactive mode, 375 * in which the tool offers a mode in which the arguments can be provided in 376 * a text-driven menu rather than requiring them to be given on the command 377 * line. If interactive mode is supported, it may be invoked using the 378 * "--interactive" argument. Alternately, if interactive mode is supported 379 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 380 * interactive mode may be invoked by simply launching the tool without any 381 * arguments. 382 * 383 * @return {@code true} if this tool supports interactive mode, or 384 * {@code false} if not. 385 */ 386 @Override() 387 public boolean supportsInteractiveMode() 388 { 389 return true; 390 } 391 392 393 394 /** 395 * Indicates whether this tool defaults to launching in interactive mode if 396 * the tool is invoked without any command-line arguments. This will only be 397 * used if {@link #supportsInteractiveMode()} returns {@code true}. 398 * 399 * @return {@code true} if this tool defaults to using interactive mode if 400 * launched without any command-line arguments, or {@code false} if 401 * not. 402 */ 403 @Override() 404 public boolean defaultsToInteractiveMode() 405 { 406 return true; 407 } 408 409 410 411 /** 412 * Indicates whether this tool should provide arguments for redirecting output 413 * to a file. If this method returns {@code true}, then the tool will offer 414 * an "--outputFile" argument that will specify the path to a file to which 415 * all standard output and standard error content will be written, and it will 416 * also offer a "--teeToStandardOut" argument that can only be used if the 417 * "--outputFile" argument is present and will cause all output to be written 418 * to both the specified output file and to standard output. 419 * 420 * @return {@code true} if this tool should provide arguments for redirecting 421 * output to a file, or {@code false} if not. 422 */ 423 @Override() 424 protected boolean supportsOutputFile() 425 { 426 return true; 427 } 428 429 430 431 /** 432 * Indicates whether this tool should default to interactively prompting for 433 * the bind password if a password is required but no argument was provided 434 * to indicate how to get the password. 435 * 436 * @return {@code true} if this tool should default to interactively 437 * prompting for the bind password, or {@code false} if not. 438 */ 439 @Override() 440 protected boolean defaultToPromptForBindPassword() 441 { 442 return true; 443 } 444 445 446 447 /** 448 * Indicates whether this tool supports the use of a properties file for 449 * specifying default values for arguments that aren't specified on the 450 * command line. 451 * 452 * @return {@code true} if this tool supports the use of a properties file 453 * for specifying default values for arguments that aren't specified 454 * on the command line, or {@code false} if not. 455 */ 456 @Override() 457 public boolean supportsPropertiesFile() 458 { 459 return true; 460 } 461 462 463 464 /** 465 * Indicates whether the LDAP-specific arguments should include alternate 466 * versions of all long identifiers that consist of multiple words so that 467 * they are available in both camelCase and dash-separated versions. 468 * 469 * @return {@code true} if this tool should provide multiple versions of 470 * long identifiers for LDAP-specific arguments, or {@code false} if 471 * not. 472 */ 473 @Override() 474 protected boolean includeAlternateLongIdentifiers() 475 { 476 return true; 477 } 478 479 480 481 /** 482 * {@inheritDoc} 483 */ 484 @Override() 485 protected boolean logToolInvocationByDefault() 486 { 487 return true; 488 } 489 490 491 492 /** 493 * Adds the arguments used by this program that aren't already provided by the 494 * generic {@code LDAPCommandLineTool} framework. 495 * 496 * @param parser The argument parser to which the arguments should be added. 497 * 498 * @throws ArgumentException If a problem occurs while adding the arguments. 499 */ 500 @Override() 501 public void addNonLDAPArguments(final ArgumentParser parser) 502 throws ArgumentException 503 { 504 String description = "The DN of the entry to modify. It may be a simple " + 505 "DN or a value pattern to specify a range of DN (e.g., " + 506 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 507 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 508 "value pattern syntax. This must be provided."; 509 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description); 510 entryDN.setArgumentGroupName("Modification Arguments"); 511 entryDN.addLongIdentifier("entry-dn", true); 512 parser.addArgument(entryDN); 513 514 515 description = "The name of the attribute to modify. Multiple attributes " + 516 "may be specified by providing this argument multiple " + 517 "times. At least one attribute must be specified."; 518 attribute = new StringArgument('A', "attribute", true, 0, "{name}", 519 description); 520 attribute.setArgumentGroupName("Modification Arguments"); 521 parser.addArgument(attribute); 522 523 524 description = "The pattern to use to generate values for the replace " + 525 "modifications. If this is provided, then neither the " + 526 "--valueLength argument nor the --characterSet arguments " + 527 "may be provided."; 528 valuePattern = new StringArgument(null, "valuePattern", false, 1, 529 "{pattern}", description); 530 valuePattern.setArgumentGroupName("Modification Arguments"); 531 valuePattern.addLongIdentifier("value-pattern", true); 532 parser.addArgument(valuePattern); 533 534 535 description = "The length in bytes to use when generating values for the " + 536 "replace modifications. If this is not provided, then a " + 537 "default length of ten bytes will be used."; 538 valueLength = new IntegerArgument('l', "valueLength", false, 1, "{num}", 539 description, 1, Integer.MAX_VALUE); 540 valueLength.setArgumentGroupName("Modification Arguments"); 541 valueLength.addLongIdentifier("value-length", true); 542 parser.addArgument(valueLength); 543 544 545 description = "The number of values to include in replace " + 546 "modifications. If this is not provided, then a default " + 547 "of one value will be used."; 548 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}", 549 description, 0, Integer.MAX_VALUE, 1); 550 valueCount.setArgumentGroupName("Modification Arguments"); 551 valueCount.addLongIdentifier("value-count", true); 552 parser.addArgument(valueCount); 553 554 555 description = "Indicates that the tool should use the increment " + 556 "modification type rather than the replace modification " + 557 "type."; 558 increment = new BooleanArgument(null, "increment", 1, description); 559 increment.setArgumentGroupName("Modification Arguments"); 560 parser.addArgument(increment); 561 562 563 description = "The amount by which to increment values when using the " + 564 "increment modification type. The amount may be negative " + 565 "if values should be decremented rather than incremented. " + 566 "If this is not provided, then a default increment amount " + 567 "of one will be used."; 568 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1, 569 null, description, Integer.MIN_VALUE, 570 Integer.MAX_VALUE, 1); 571 incrementAmount.setArgumentGroupName("Modification Arguments"); 572 incrementAmount.addLongIdentifier("increment-amount", true); 573 parser.addArgument(incrementAmount); 574 575 576 description = "The set of characters to use to generate the values for " + 577 "the modifications. It should only include ASCII " + 578 "characters. If this is not provided, then a default set " + 579 "of lowercase alphabetic characters will be used."; 580 characterSet = new StringArgument('C', "characterSet", false, 1, "{chars}", 581 description); 582 characterSet.setArgumentGroupName("Modification Arguments"); 583 characterSet.addLongIdentifier("character-set", true); 584 parser.addArgument(characterSet); 585 586 587 description = "Indicates that modify requests should include the " + 588 "assertion request control with the specified filter."; 589 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1, 590 "{filter}", description); 591 assertionFilter.setArgumentGroupName("Request Control Arguments"); 592 assertionFilter.addLongIdentifier("assertion-filter", true); 593 parser.addArgument(assertionFilter); 594 595 596 description = "Indicates that modify requests should include the " + 597 "permissive modify request control."; 598 permissiveModify = new BooleanArgument(null, "permissiveModify", 1, 599 description); 600 permissiveModify.setArgumentGroupName("Request Control Arguments"); 601 permissiveModify.addLongIdentifier("permissive-modify", true); 602 parser.addArgument(permissiveModify); 603 604 605 description = "Indicates that modify requests should include the " + 606 "pre-read request control with the specified requested " + 607 "attribute. This argument may be provided multiple times " + 608 "to indicate that multiple requested attributes should be " + 609 "included in the pre-read request control."; 610 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0, 611 "{attribute}", description); 612 preReadAttribute.setArgumentGroupName("Request Control Arguments"); 613 preReadAttribute.addLongIdentifier("pre-read-attribute", true); 614 parser.addArgument(preReadAttribute); 615 616 617 description = "Indicates that modify requests should include the " + 618 "post-read request control with the specified requested " + 619 "attribute. This argument may be provided multiple times " + 620 "to indicate that multiple requested attributes should be " + 621 "included in the post-read request control."; 622 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0, 623 "{attribute}", description); 624 postReadAttribute.setArgumentGroupName("Request Control Arguments"); 625 postReadAttribute.addLongIdentifier("post-read-attribute", true); 626 parser.addArgument(postReadAttribute); 627 628 629 description = "Indicates that the proxied authorization control (as " + 630 "defined in RFC 4370) should be used to request that " + 631 "operations be processed using an alternate authorization " + 632 "identity. This may be a simple authorization ID or it " + 633 "may be a value pattern to specify a range of " + 634 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 635 " for complete details about the value pattern syntax."; 636 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 637 description); 638 proxyAs.setArgumentGroupName("Request Control Arguments"); 639 proxyAs.addLongIdentifier("proxy-as", true); 640 parser.addArgument(proxyAs); 641 642 643 description = "Indicates that modify requests should include the " + 644 "specified request control. This may be provided multiple " + 645 "times to include multiple request controls."; 646 control = new ControlArgument('J', "control", false, 0, null, description); 647 control.setArgumentGroupName("Request Control Arguments"); 648 parser.addArgument(control); 649 650 651 description = "The number of threads to use to perform the " + 652 "modifications. If this is not provided, a single thread " + 653 "will be used."; 654 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 655 description, 1, Integer.MAX_VALUE, 1); 656 numThreads.setArgumentGroupName("Rate Management Arguments"); 657 numThreads.addLongIdentifier("num-threads", true); 658 parser.addArgument(numThreads); 659 660 661 description = "The length of time in seconds between output lines. If " + 662 "this is not provided, then a default interval of five " + 663 "seconds will be used."; 664 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 665 "{num}", description, 1, 666 Integer.MAX_VALUE, 5); 667 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 668 collectionInterval.addLongIdentifier("interval-duration", true); 669 parser.addArgument(collectionInterval); 670 671 672 description = "The maximum number of intervals for which to run. If " + 673 "this is not provided, then the tool will run until it is " + 674 "interrupted."; 675 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 676 description, 1, Integer.MAX_VALUE, 677 Integer.MAX_VALUE); 678 numIntervals.setArgumentGroupName("Rate Management Arguments"); 679 numIntervals.addLongIdentifier("num-intervals", true); 680 parser.addArgument(numIntervals); 681 682 description = "The number of modify iterations that should be processed " + 683 "on a connection before that connection is closed and " + 684 "replaced with a newly-established (and authenticated, if " + 685 "appropriate) connection. If this is not provided, then " + 686 "connections will not be periodically closed and " + 687 "re-established."; 688 iterationsBeforeReconnect = new IntegerArgument(null, 689 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 690 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 691 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect", 692 true); 693 parser.addArgument(iterationsBeforeReconnect); 694 695 description = "The target number of modifies to perform per second. It " + 696 "is still necessary to specify a sufficient number of " + 697 "threads for achieving this rate. If neither this option " + 698 "nor --variableRateData is provided, then the tool will " + 699 "run at the maximum rate for the specified number of " + 700 "threads."; 701 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 702 "{modifies-per-second}", description, 703 1, Integer.MAX_VALUE); 704 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 705 ratePerSecond.addLongIdentifier("rate-per-second", true); 706 parser.addArgument(ratePerSecond); 707 708 final String variableRateDataArgName = "variableRateData"; 709 final String generateSampleRateFileArgName = "generateSampleRateFile"; 710 description = RateAdjustor.getVariableRateDataArgumentDescription( 711 generateSampleRateFileArgName); 712 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 713 "{path}", description, true, true, true, 714 false); 715 variableRateData.setArgumentGroupName("Rate Management Arguments"); 716 variableRateData.addLongIdentifier("variable-rate-data", true); 717 parser.addArgument(variableRateData); 718 719 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 720 variableRateDataArgName); 721 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 722 false, 1, "{path}", description, false, 723 true, true, false); 724 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 725 sampleRateFile.addLongIdentifier("generate-sample-rate-file", true); 726 sampleRateFile.setUsageArgument(true); 727 parser.addArgument(sampleRateFile); 728 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 729 730 description = "The number of intervals to complete before beginning " + 731 "overall statistics collection. Specifying a nonzero " + 732 "number of warm-up intervals gives the client and server " + 733 "a chance to warm up without skewing performance results."; 734 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 735 "{num}", description, 0, Integer.MAX_VALUE, 0); 736 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 737 warmUpIntervals.addLongIdentifier("warm-up-intervals", true); 738 parser.addArgument(warmUpIntervals); 739 740 description = "Indicates the format to use for timestamps included in " + 741 "the output. A value of 'none' indicates that no " + 742 "timestamps should be included. A value of 'with-date' " + 743 "indicates that both the date and the time should be " + 744 "included. A value of 'without-date' indicates that only " + 745 "the time should be included."; 746 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<>(3); 747 allowedFormats.add("none"); 748 allowedFormats.add("with-date"); 749 allowedFormats.add("without-date"); 750 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 751 "{format}", description, allowedFormats, "none"); 752 timestampFormat.addLongIdentifier("timestamp-format", true); 753 parser.addArgument(timestampFormat); 754 755 description = "Indicates that information about the result codes for " + 756 "failed operations should not be displayed."; 757 suppressErrorsArgument = new BooleanArgument(null, 758 "suppressErrorResultCodes", 1, description); 759 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes", 760 true); 761 parser.addArgument(suppressErrorsArgument); 762 763 description = "Generate output in CSV format rather than a " + 764 "display-friendly format"; 765 csvFormat = new BooleanArgument('c', "csv", 1, description); 766 parser.addArgument(csvFormat); 767 768 description = "Specifies the seed to use for the random number generator."; 769 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 770 description); 771 randomSeed.addLongIdentifier("random-seed", true); 772 parser.addArgument(randomSeed); 773 774 775 // The incrementAmount argument can only be used if the increment argument 776 // is provided. 777 parser.addDependentArgumentSet(incrementAmount, increment); 778 779 780 // None of the valueLength, valueCount, characterSet, or valuePattern 781 // arguments can be used if the increment argument is provided. 782 parser.addExclusiveArgumentSet(increment, valueLength); 783 parser.addExclusiveArgumentSet(increment, valueCount); 784 parser.addExclusiveArgumentSet(increment, characterSet); 785 parser.addExclusiveArgumentSet(increment, valuePattern); 786 787 788 // The valuePattern argument cannot be used with either the valueLength or 789 // characterSet arguments. 790 parser.addExclusiveArgumentSet(valuePattern, valueLength); 791 parser.addExclusiveArgumentSet(valuePattern, characterSet); 792 } 793 794 795 796 /** 797 * Indicates whether this tool supports creating connections to multiple 798 * servers. If it is to support multiple servers, then the "--hostname" and 799 * "--port" arguments will be allowed to be provided multiple times, and 800 * will be required to be provided the same number of times. The same type of 801 * communication security and bind credentials will be used for all servers. 802 * 803 * @return {@code true} if this tool supports creating connections to 804 * multiple servers, or {@code false} if not. 805 */ 806 @Override() 807 protected boolean supportsMultipleServers() 808 { 809 return true; 810 } 811 812 813 814 /** 815 * Retrieves the connection options that should be used for connections 816 * created for use with this tool. 817 * 818 * @return The connection options that should be used for connections created 819 * for use with this tool. 820 */ 821 @Override() 822 public LDAPConnectionOptions getConnectionOptions() 823 { 824 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 825 options.setUseSynchronousMode(true); 826 return options; 827 } 828 829 830 831 /** 832 * Performs the actual processing for this tool. In this case, it gets a 833 * connection to the directory server and uses it to perform the requested 834 * modifications. 835 * 836 * @return The result code for the processing that was performed. 837 */ 838 @Override() 839 public ResultCode doToolProcessing() 840 { 841 runningThread = Thread.currentThread(); 842 843 try 844 { 845 return doToolProcessingInternal(); 846 } 847 finally 848 { 849 runningThread = null; 850 } 851 852 } 853 854 855 /** 856 * Performs the actual processing for this tool. In this case, it gets a 857 * connection to the directory server and uses it to perform the requested 858 * modifications. 859 * 860 * @return The result code for the processing that was performed. 861 */ 862 private ResultCode doToolProcessingInternal() 863 { 864 // If the sample rate file argument was specified, then generate the sample 865 // variable rate data file and return. 866 if (sampleRateFile.isPresent()) 867 { 868 try 869 { 870 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 871 return ResultCode.SUCCESS; 872 } 873 catch (final Exception e) 874 { 875 Debug.debugException(e); 876 err("An error occurred while trying to write sample variable data " + 877 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 878 "': ", StaticUtils.getExceptionMessage(e)); 879 return ResultCode.LOCAL_ERROR; 880 } 881 } 882 883 884 // Determine the random seed to use. 885 final Long seed; 886 if (randomSeed.isPresent()) 887 { 888 seed = Long.valueOf(randomSeed.getValue()); 889 } 890 else 891 { 892 seed = null; 893 } 894 895 // Create the value patterns for the target entry DN and proxied 896 // authorization identities. 897 final ValuePattern dnPattern; 898 try 899 { 900 dnPattern = new ValuePattern(entryDN.getValue(), seed); 901 } 902 catch (final ParseException pe) 903 { 904 Debug.debugException(pe); 905 err("Unable to parse the entry DN value pattern: ", pe.getMessage()); 906 return ResultCode.PARAM_ERROR; 907 } 908 909 final ValuePattern authzIDPattern; 910 if (proxyAs.isPresent()) 911 { 912 try 913 { 914 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 915 } 916 catch (final ParseException pe) 917 { 918 Debug.debugException(pe); 919 err("Unable to parse the proxied authorization pattern: ", 920 pe.getMessage()); 921 return ResultCode.PARAM_ERROR; 922 } 923 } 924 else 925 { 926 authzIDPattern = null; 927 } 928 929 930 // Get the set of controls to include in modify requests. 931 final ArrayList<Control> controlList = new ArrayList<>(5); 932 if (assertionFilter.isPresent()) 933 { 934 controlList.add(new AssertionRequestControl(assertionFilter.getValue())); 935 } 936 937 if (permissiveModify.isPresent()) 938 { 939 controlList.add(new PermissiveModifyRequestControl()); 940 } 941 942 if (preReadAttribute.isPresent()) 943 { 944 final List<String> attrList = preReadAttribute.getValues(); 945 final String[] attrArray = new String[attrList.size()]; 946 attrList.toArray(attrArray); 947 controlList.add(new PreReadRequestControl(attrArray)); 948 } 949 950 if (postReadAttribute.isPresent()) 951 { 952 final List<String> attrList = postReadAttribute.getValues(); 953 final String[] attrArray = new String[attrList.size()]; 954 attrList.toArray(attrArray); 955 controlList.add(new PostReadRequestControl(attrArray)); 956 } 957 958 if (control.isPresent()) 959 { 960 controlList.addAll(control.getValues()); 961 } 962 963 final Control[] controlArray = new Control[controlList.size()]; 964 controlList.toArray(controlArray); 965 966 967 // Get the names of the attributes to modify. 968 final String[] attrs = new String[attribute.getValues().size()]; 969 attribute.getValues().toArray(attrs); 970 971 972 // If the --ratePerSecond option was specified, then limit the rate 973 // accordingly. 974 FixedRateBarrier fixedRateBarrier = null; 975 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 976 { 977 // We might not have a rate per second if --variableRateData is specified. 978 // The rate typically doesn't matter except when we have warm-up 979 // intervals. In this case, we'll run at the max rate. 980 final int intervalSeconds = collectionInterval.getValue(); 981 final int ratePerInterval = 982 (ratePerSecond.getValue() == null) 983 ? Integer.MAX_VALUE 984 : ratePerSecond.getValue() * intervalSeconds; 985 fixedRateBarrier = 986 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 987 } 988 989 990 // If --variableRateData was specified, then initialize a RateAdjustor. 991 RateAdjustor rateAdjustor = null; 992 if (variableRateData.isPresent()) 993 { 994 try 995 { 996 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 997 ratePerSecond.getValue(), variableRateData.getValue()); 998 } 999 catch (final IOException | IllegalArgumentException e) 1000 { 1001 Debug.debugException(e); 1002 err("Initializing the variable rates failed: " + e.getMessage()); 1003 return ResultCode.PARAM_ERROR; 1004 } 1005 } 1006 1007 1008 // Determine whether to include timestamps in the output and if so what 1009 // format should be used for them. 1010 final boolean includeTimestamp; 1011 final String timeFormat; 1012 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 1013 { 1014 includeTimestamp = true; 1015 timeFormat = "dd/MM/yyyy HH:mm:ss"; 1016 } 1017 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 1018 { 1019 includeTimestamp = true; 1020 timeFormat = "HH:mm:ss"; 1021 } 1022 else 1023 { 1024 includeTimestamp = false; 1025 timeFormat = null; 1026 } 1027 1028 1029 // Determine whether any warm-up intervals should be run. 1030 final long totalIntervals; 1031 final boolean warmUp; 1032 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 1033 if (remainingWarmUpIntervals > 0) 1034 { 1035 warmUp = true; 1036 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 1037 } 1038 else 1039 { 1040 warmUp = true; 1041 totalIntervals = 0L + numIntervals.getValue(); 1042 } 1043 1044 1045 // Create the table that will be used to format the output. 1046 final OutputFormat outputFormat; 1047 if (csvFormat.isPresent()) 1048 { 1049 outputFormat = OutputFormat.CSV; 1050 } 1051 else 1052 { 1053 outputFormat = OutputFormat.COLUMNS; 1054 } 1055 1056 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 1057 timeFormat, outputFormat, " ", 1058 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1059 "Mods/Sec"), 1060 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1061 "Avg Dur ms"), 1062 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1063 "Errors/Sec"), 1064 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1065 "Mods/Sec"), 1066 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1067 "Avg Dur ms")); 1068 1069 1070 // Create values to use for statistics collection. 1071 final AtomicLong modCounter = new AtomicLong(0L); 1072 final AtomicLong errorCounter = new AtomicLong(0L); 1073 final AtomicLong modDurations = new AtomicLong(0L); 1074 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 1075 1076 1077 // Determine the length of each interval in milliseconds. 1078 final long intervalMillis = 1000L * collectionInterval.getValue(); 1079 1080 1081 // Create the threads to use for the modifications. 1082 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 1083 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()]; 1084 for (int i=0; i < threads.length; i++) 1085 { 1086 final LDAPConnection connection; 1087 try 1088 { 1089 connection = getConnection(); 1090 } 1091 catch (final LDAPException le) 1092 { 1093 Debug.debugException(le); 1094 err("Unable to connect to the directory server: ", 1095 StaticUtils.getExceptionMessage(le)); 1096 return le.getResultCode(); 1097 } 1098 1099 final String valuePatternString; 1100 if (valuePattern.isPresent()) 1101 { 1102 valuePatternString = valuePattern.getValue(); 1103 } 1104 else 1105 { 1106 final int length; 1107 if (valueLength.isPresent()) 1108 { 1109 length = valueLength.getValue(); 1110 } 1111 else 1112 { 1113 length = 10; 1114 } 1115 1116 final String charSet; 1117 if (characterSet.isPresent()) 1118 { 1119 charSet = 1120 characterSet.getValue().replace("]", "]]").replace("[", "[["); 1121 } 1122 else 1123 { 1124 charSet = "abcdefghijklmnopqrstuvwxyz"; 1125 } 1126 1127 valuePatternString = "random:" + length + ':' + charSet; 1128 } 1129 1130 final ValuePattern parsedValuePattern; 1131 try 1132 { 1133 parsedValuePattern = new ValuePattern(valuePatternString); 1134 } 1135 catch (final ParseException e) 1136 { 1137 Debug.debugException(e); 1138 err(e.getMessage()); 1139 return ResultCode.PARAM_ERROR; 1140 } 1141 1142 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs, 1143 parsedValuePattern, valueCount.getValue(), increment.isPresent(), 1144 incrementAmount.getValue(), controlArray, authzIDPattern, 1145 iterationsBeforeReconnect.getValue(), barrier, 1146 modCounter, modDurations, errorCounter, rcCounter, fixedRateBarrier); 1147 threads[i].start(); 1148 } 1149 1150 1151 // Display the table header. 1152 for (final String headerLine : formatter.getHeaderLines(true)) 1153 { 1154 out(headerLine); 1155 } 1156 1157 1158 // Start the RateAdjustor before the threads so that the initial value is 1159 // in place before any load is generated unless we're doing a warm-up in 1160 // which case, we'll start it after the warm-up is complete. 1161 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1162 { 1163 rateAdjustor.start(); 1164 } 1165 1166 1167 // Indicate that the threads can start running. 1168 try 1169 { 1170 barrier.await(); 1171 } 1172 catch (final Exception e) 1173 { 1174 Debug.debugException(e); 1175 } 1176 1177 long overallStartTime = System.nanoTime(); 1178 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1179 1180 1181 boolean setOverallStartTime = false; 1182 long lastDuration = 0L; 1183 long lastNumErrors = 0L; 1184 long lastNumMods = 0L; 1185 long lastEndTime = System.nanoTime(); 1186 for (long i=0; i < totalIntervals; i++) 1187 { 1188 if (rateAdjustor != null) 1189 { 1190 if (! rateAdjustor.isAlive()) 1191 { 1192 out("All of the rates in " + variableRateData.getValue().getName() + 1193 " have been completed."); 1194 break; 1195 } 1196 } 1197 1198 final long startTimeMillis = System.currentTimeMillis(); 1199 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1200 nextIntervalStartTime += intervalMillis; 1201 if (sleepTimeMillis > 0) 1202 { 1203 sleeper.sleep(sleepTimeMillis); 1204 } 1205 1206 if (stopRequested.get()) 1207 { 1208 break; 1209 } 1210 1211 final long endTime = System.nanoTime(); 1212 final long intervalDuration = endTime - lastEndTime; 1213 1214 final long numMods; 1215 final long numErrors; 1216 final long totalDuration; 1217 if (warmUp && (remainingWarmUpIntervals > 0)) 1218 { 1219 numMods = modCounter.getAndSet(0L); 1220 numErrors = errorCounter.getAndSet(0L); 1221 totalDuration = modDurations.getAndSet(0L); 1222 } 1223 else 1224 { 1225 numMods = modCounter.get(); 1226 numErrors = errorCounter.get(); 1227 totalDuration = modDurations.get(); 1228 } 1229 1230 final long recentNumMods = numMods - lastNumMods; 1231 final long recentNumErrors = numErrors - lastNumErrors; 1232 final long recentDuration = totalDuration - lastDuration; 1233 1234 final double numSeconds = intervalDuration / 1_000_000_000.0d; 1235 final double recentModRate = recentNumMods / numSeconds; 1236 final double recentErrorRate = recentNumErrors / numSeconds; 1237 1238 final double recentAvgDuration; 1239 if (recentNumMods > 0L) 1240 { 1241 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1_000_000; 1242 } 1243 else 1244 { 1245 recentAvgDuration = 0.0d; 1246 } 1247 1248 if (warmUp && (remainingWarmUpIntervals > 0)) 1249 { 1250 out(formatter.formatRow(recentModRate, recentAvgDuration, 1251 recentErrorRate, "warming up", "warming up")); 1252 1253 remainingWarmUpIntervals--; 1254 if (remainingWarmUpIntervals == 0) 1255 { 1256 out("Warm-up completed. Beginning overall statistics collection."); 1257 setOverallStartTime = true; 1258 if (rateAdjustor != null) 1259 { 1260 rateAdjustor.start(); 1261 } 1262 } 1263 } 1264 else 1265 { 1266 if (setOverallStartTime) 1267 { 1268 overallStartTime = lastEndTime; 1269 setOverallStartTime = false; 1270 } 1271 1272 final double numOverallSeconds = 1273 (endTime - overallStartTime) / 1_000_000_000.0d; 1274 final double overallAuthRate = numMods / numOverallSeconds; 1275 1276 final double overallAvgDuration; 1277 if (numMods > 0L) 1278 { 1279 overallAvgDuration = 1.0d * totalDuration / numMods / 1_000_000; 1280 } 1281 else 1282 { 1283 overallAvgDuration = 0.0d; 1284 } 1285 1286 out(formatter.formatRow(recentModRate, recentAvgDuration, 1287 recentErrorRate, overallAuthRate, overallAvgDuration)); 1288 1289 lastNumMods = numMods; 1290 lastNumErrors = numErrors; 1291 lastDuration = totalDuration; 1292 } 1293 1294 final List<ObjectPair<ResultCode,Long>> rcCounts = 1295 rcCounter.getCounts(true); 1296 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty())) 1297 { 1298 err("\tError Results:"); 1299 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1300 { 1301 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1302 } 1303 } 1304 1305 lastEndTime = endTime; 1306 } 1307 1308 // Shut down the RateAdjustor if we have one. 1309 if (rateAdjustor != null) 1310 { 1311 rateAdjustor.shutDown(); 1312 } 1313 1314 // Stop all of the threads. 1315 ResultCode resultCode = ResultCode.SUCCESS; 1316 for (final ModRateThread t : threads) 1317 { 1318 final ResultCode r = t.stopRunning(); 1319 if (resultCode == ResultCode.SUCCESS) 1320 { 1321 resultCode = r; 1322 } 1323 } 1324 1325 return resultCode; 1326 } 1327 1328 1329 1330 /** 1331 * Requests that this tool stop running. This method will attempt to wait 1332 * for all threads to complete before returning control to the caller. 1333 */ 1334 public void stopRunning() 1335 { 1336 stopRequested.set(true); 1337 sleeper.wakeup(); 1338 1339 final Thread t = runningThread; 1340 if (t != null) 1341 { 1342 try 1343 { 1344 t.join(); 1345 } 1346 catch (final Exception e) 1347 { 1348 Debug.debugException(e); 1349 1350 if (e instanceof InterruptedException) 1351 { 1352 Thread.currentThread().interrupt(); 1353 } 1354 } 1355 } 1356 } 1357 1358 1359 1360 /** 1361 * {@inheritDoc} 1362 */ 1363 @Override() 1364 public LinkedHashMap<String[],String> getExampleUsages() 1365 { 1366 final LinkedHashMap<String[],String> examples = new LinkedHashMap<>(2); 1367 1368 String[] args = 1369 { 1370 "--hostname", "server.example.com", 1371 "--port", "389", 1372 "--bindDN", "uid=admin,dc=example,dc=com", 1373 "--bindPassword", "password", 1374 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com", 1375 "--attribute", "description", 1376 "--valueLength", "12", 1377 "--numThreads", "10" 1378 }; 1379 String description = 1380 "Test modify performance by randomly selecting entries across a set " + 1381 "of one million users located below 'ou=People,dc=example,dc=com' " + 1382 "with ten concurrent threads and replacing the values for the " + 1383 "description attribute with a string of 12 randomly-selected " + 1384 "lowercase alphabetic characters."; 1385 examples.put(args, description); 1386 1387 args = new String[] 1388 { 1389 "--generateSampleRateFile", "variable-rate-data.txt" 1390 }; 1391 description = 1392 "Generate a sample variable rate definition file that may be used " + 1393 "in conjunction with the --variableRateData argument. The sample " + 1394 "file will include comments that describe the format for data to be " + 1395 "included in this file."; 1396 examples.put(args, description); 1397 1398 return examples; 1399 } 1400}