001/* 002 * Copyright 2008-2017 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2017 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.IOException; 026import java.io.OutputStream; 027import java.io.Serializable; 028import java.text.ParseException; 029import java.util.ArrayList; 030import java.util.LinkedHashMap; 031import java.util.LinkedHashSet; 032import java.util.List; 033import java.util.Random; 034import java.util.concurrent.CyclicBarrier; 035import java.util.concurrent.atomic.AtomicBoolean; 036import java.util.concurrent.atomic.AtomicLong; 037 038import com.unboundid.ldap.sdk.Control; 039import com.unboundid.ldap.sdk.LDAPConnection; 040import com.unboundid.ldap.sdk.LDAPConnectionOptions; 041import com.unboundid.ldap.sdk.LDAPException; 042import com.unboundid.ldap.sdk.ResultCode; 043import com.unboundid.ldap.sdk.Version; 044import com.unboundid.ldap.sdk.controls.AssertionRequestControl; 045import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl; 046import com.unboundid.ldap.sdk.controls.PostReadRequestControl; 047import com.unboundid.ldap.sdk.controls.PreReadRequestControl; 048import com.unboundid.util.ColumnFormatter; 049import com.unboundid.util.FixedRateBarrier; 050import com.unboundid.util.FormattableColumn; 051import com.unboundid.util.HorizontalAlignment; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.ObjectPair; 054import com.unboundid.util.OutputFormat; 055import com.unboundid.util.RateAdjustor; 056import com.unboundid.util.ResultCodeCounter; 057import com.unboundid.util.ThreadSafety; 058import com.unboundid.util.ThreadSafetyLevel; 059import com.unboundid.util.ValuePattern; 060import com.unboundid.util.WakeableSleeper; 061import com.unboundid.util.args.ArgumentException; 062import com.unboundid.util.args.ArgumentParser; 063import com.unboundid.util.args.BooleanArgument; 064import com.unboundid.util.args.ControlArgument; 065import com.unboundid.util.args.FileArgument; 066import com.unboundid.util.args.FilterArgument; 067import com.unboundid.util.args.IntegerArgument; 068import com.unboundid.util.args.StringArgument; 069 070import static com.unboundid.util.Debug.*; 071import static com.unboundid.util.StaticUtils.*; 072 073 074 075/** 076 * This class provides a tool that can be used to perform repeated modifications 077 * in an LDAP directory server using multiple threads. It can help provide an 078 * estimate of the modify performance that a directory server is able to 079 * achieve. The target entry DN may be a value pattern as described in the 080 * {@link ValuePattern} class. This makes it possible to modify a range of 081 * entries rather than repeatedly updating the same entry. 082 * <BR><BR> 083 * Some of the APIs demonstrated by this example include: 084 * <UL> 085 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 086 * package)</LI> 087 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 088 * package)</LI> 089 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk} 090 * package)</LI> 091 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI> 092 * </UL> 093 * <BR><BR> 094 * All of the necessary information is provided using command line arguments. 095 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 096 * class, as well as the following additional arguments: 097 * <UL> 098 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the 099 * entry to be modified. This must be provided. It may be a simple DN, 100 * or it may be a value pattern to express a range of entry DNs.</LI> 101 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the 102 * attribute to modify. Multiple attributes may be modified by providing 103 * multiple instances of this argument. At least one attribute must be 104 * provided.</LI> 105 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to 106 * use for the values of the target attributes. If this is not provided, 107 * then a default length of 10 bytes will be used.</LI> 108 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of 109 * characters that will be used to generate the values to use for the 110 * target attributes. It should only include ASCII characters. Values 111 * will be generated from randomly-selected characters from this set. If 112 * this is not provided, then a default set of lowercase alphabetic 113 * characters will be used.</LI> 114 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 115 * concurrent threads to use when performing the modifications. If this 116 * is not provided, then a default of one thread will be used.</LI> 117 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of 118 * time in seconds between lines out output. If this is not provided, 119 * then a default interval duration of five seconds will be used.</LI> 120 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of 121 * intervals for which to run. If this is not provided, then it will 122 * run forever.</LI> 123 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify 124 * iterations that should be performed on a connection before that 125 * connection is closed and replaced with a newly-established (and 126 * authenticated, if appropriate) connection.</LI> 127 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}" 128 * -- specifies the target number of modifies to perform per second. It 129 * is still necessary to specify a sufficient number of threads for 130 * achieving this rate. If this option is not provided, then the tool 131 * will run at the maximum rate for the specified number of threads.</LI> 132 * <LI>"--variableRateData {path}" -- specifies the path to a file containing 133 * information needed to allow the tool to vary the target rate over time. 134 * If this option is not provided, then the tool will either use a fixed 135 * target rate as specified by the "--ratePerSecond" argument, or it will 136 * run at the maximum rate.</LI> 137 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to 138 * which sample data will be written illustrating and describing the 139 * format of the file expected to be used in conjunction with the 140 * "--variableRateData" argument.</LI> 141 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to 142 * complete before beginning overall statistics collection.</LI> 143 * <LI>"--timestampFormat {format}" -- specifies the format to use for 144 * timestamps included before each output line. The format may be one of 145 * "none" (for no timestamps), "with-date" (to include both the date and 146 * the time), or "without-date" (to include only time time).</LI> 147 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied 148 * authorization v2 control to request that the operation be processed 149 * using an alternate authorization identity. In this case, the bind DN 150 * should be that of a user that has permission to use this control. The 151 * authorization identity may be a value pattern.</LI> 152 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the 153 * result codes for failed operations should not be displayed.</LI> 154 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a 155 * display-friendly format.</LI> 156 * </UL> 157 */ 158@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 159public final class ModRate 160 extends LDAPCommandLineTool 161 implements Serializable 162{ 163 /** 164 * The serial version UID for this serializable class. 165 */ 166 private static final long serialVersionUID = 2709717414202815822L; 167 168 169 170 // Indicates whether a request has been made to stop running. 171 private final AtomicBoolean stopRequested; 172 173 // The argument used to indicate whether to generate output in CSV format. 174 private BooleanArgument csvFormat; 175 176 // Indicates that the tool should use the increment modification type instead 177 // of replace. 178 private BooleanArgument increment; 179 180 // Indicates that modify requests should include the permissive modify request 181 // control. 182 private BooleanArgument permissiveModify; 183 184 // The argument used to indicate whether to suppress information about error 185 // result codes. 186 private BooleanArgument suppressErrorsArgument; 187 188 // The argument used to indicate that a generic control should be included in 189 // the request. 190 private ControlArgument control; 191 192 // The argument used to specify a variable rate file. 193 private FileArgument sampleRateFile; 194 195 // The argument used to specify a variable rate file. 196 private FileArgument variableRateData; 197 198 // Indicates that modify requests should include the assertion request control 199 // with the specified filter. 200 private FilterArgument assertionFilter; 201 202 // The argument used to specify the collection interval. 203 private IntegerArgument collectionInterval; 204 205 // The increment amount to use when performing an increment instead of a 206 // replace. 207 private IntegerArgument incrementAmount; 208 209 // The argument used to specify the number of modify iterations on a 210 // connection before it is closed and re-established. 211 private IntegerArgument iterationsBeforeReconnect; 212 213 // The argument used to specify the number of intervals. 214 private IntegerArgument numIntervals; 215 216 // The argument used to specify the number of threads. 217 private IntegerArgument numThreads; 218 219 // The argument used to specify the seed to use for the random number 220 // generator. 221 private IntegerArgument randomSeed; 222 223 // The target rate of modifies per second. 224 private IntegerArgument ratePerSecond; 225 226 // The number of values to include in the replace modification. 227 private IntegerArgument valueCount; 228 229 // The argument used to specify the length of the values to generate. 230 private IntegerArgument valueLength; 231 232 // The number of warm-up intervals to perform. 233 private IntegerArgument warmUpIntervals; 234 235 // The argument used to specify the name of the attribute to modify. 236 private StringArgument attribute; 237 238 // The argument used to specify the set of characters to use when generating 239 // values. 240 private StringArgument characterSet; 241 242 // The argument used to specify the DNs of the entries to modify. 243 private StringArgument entryDN; 244 245 // Indicates that modify requests should include the post-read request control 246 // to request the specified attribute. 247 private StringArgument postReadAttribute; 248 249 // Indicates that modify requests should include the pre-read request control 250 // to request the specified attribute. 251 private StringArgument preReadAttribute; 252 253 // The argument used to specify the proxied authorization identity. 254 private StringArgument proxyAs; 255 256 // The argument used to specify the timestamp format. 257 private StringArgument timestampFormat; 258 259 // The thread currently being used to run the searchrate tool. 260 private volatile Thread runningThread; 261 262 // A wakeable sleeper that will be used to sleep between reporting intervals. 263 private final WakeableSleeper sleeper; 264 265 266 267 /** 268 * Parse the provided command line arguments and make the appropriate set of 269 * changes. 270 * 271 * @param args The command line arguments provided to this program. 272 */ 273 public static void main(final String[] args) 274 { 275 final ResultCode resultCode = main(args, System.out, System.err); 276 if (resultCode != ResultCode.SUCCESS) 277 { 278 System.exit(resultCode.intValue()); 279 } 280 } 281 282 283 284 /** 285 * Parse the provided command line arguments and make the appropriate set of 286 * changes. 287 * 288 * @param args The command line arguments provided to this program. 289 * @param outStream The output stream to which standard out should be 290 * written. It may be {@code null} if output should be 291 * suppressed. 292 * @param errStream The output stream to which standard error should be 293 * written. It may be {@code null} if error messages 294 * should be suppressed. 295 * 296 * @return A result code indicating whether the processing was successful. 297 */ 298 public static ResultCode main(final String[] args, 299 final OutputStream outStream, 300 final OutputStream errStream) 301 { 302 final ModRate modRate = new ModRate(outStream, errStream); 303 return modRate.runTool(args); 304 } 305 306 307 308 /** 309 * Creates a new instance of this tool. 310 * 311 * @param outStream The output stream to which standard out should be 312 * written. It may be {@code null} if output should be 313 * suppressed. 314 * @param errStream The output stream to which standard error should be 315 * written. It may be {@code null} if error messages 316 * should be suppressed. 317 */ 318 public ModRate(final OutputStream outStream, final OutputStream errStream) 319 { 320 super(outStream, errStream); 321 322 stopRequested = new AtomicBoolean(false); 323 sleeper = new WakeableSleeper(); 324 } 325 326 327 328 /** 329 * Retrieves the name for this tool. 330 * 331 * @return The name for this tool. 332 */ 333 @Override() 334 public String getToolName() 335 { 336 return "modrate"; 337 } 338 339 340 341 /** 342 * Retrieves the description for this tool. 343 * 344 * @return The description for this tool. 345 */ 346 @Override() 347 public String getToolDescription() 348 { 349 return "Perform repeated modifications against " + 350 "an LDAP directory server."; 351 } 352 353 354 355 /** 356 * Retrieves the version string for this tool. 357 * 358 * @return The version string for this tool. 359 */ 360 @Override() 361 public String getToolVersion() 362 { 363 return Version.NUMERIC_VERSION_STRING; 364 } 365 366 367 368 /** 369 * Indicates whether this tool should provide support for an interactive mode, 370 * in which the tool offers a mode in which the arguments can be provided in 371 * a text-driven menu rather than requiring them to be given on the command 372 * line. If interactive mode is supported, it may be invoked using the 373 * "--interactive" argument. Alternately, if interactive mode is supported 374 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 375 * interactive mode may be invoked by simply launching the tool without any 376 * arguments. 377 * 378 * @return {@code true} if this tool supports interactive mode, or 379 * {@code false} if not. 380 */ 381 @Override() 382 public boolean supportsInteractiveMode() 383 { 384 return true; 385 } 386 387 388 389 /** 390 * Indicates whether this tool defaults to launching in interactive mode if 391 * the tool is invoked without any command-line arguments. This will only be 392 * used if {@link #supportsInteractiveMode()} returns {@code true}. 393 * 394 * @return {@code true} if this tool defaults to using interactive mode if 395 * launched without any command-line arguments, or {@code false} if 396 * not. 397 */ 398 @Override() 399 public boolean defaultsToInteractiveMode() 400 { 401 return true; 402 } 403 404 405 406 /** 407 * Indicates whether this tool should provide arguments for redirecting output 408 * to a file. If this method returns {@code true}, then the tool will offer 409 * an "--outputFile" argument that will specify the path to a file to which 410 * all standard output and standard error content will be written, and it will 411 * also offer a "--teeToStandardOut" argument that can only be used if the 412 * "--outputFile" argument is present and will cause all output to be written 413 * to both the specified output file and to standard output. 414 * 415 * @return {@code true} if this tool should provide arguments for redirecting 416 * output to a file, or {@code false} if not. 417 */ 418 @Override() 419 protected boolean supportsOutputFile() 420 { 421 return true; 422 } 423 424 425 426 /** 427 * Indicates whether this tool should default to interactively prompting for 428 * the bind password if a password is required but no argument was provided 429 * to indicate how to get the password. 430 * 431 * @return {@code true} if this tool should default to interactively 432 * prompting for the bind password, or {@code false} if not. 433 */ 434 @Override() 435 protected boolean defaultToPromptForBindPassword() 436 { 437 return true; 438 } 439 440 441 442 /** 443 * Indicates whether this tool supports the use of a properties file for 444 * specifying default values for arguments that aren't specified on the 445 * command line. 446 * 447 * @return {@code true} if this tool supports the use of a properties file 448 * for specifying default values for arguments that aren't specified 449 * on the command line, or {@code false} if not. 450 */ 451 @Override() 452 public boolean supportsPropertiesFile() 453 { 454 return true; 455 } 456 457 458 459 /** 460 * Indicates whether the LDAP-specific arguments should include alternate 461 * versions of all long identifiers that consist of multiple words so that 462 * they are available in both camelCase and dash-separated versions. 463 * 464 * @return {@code true} if this tool should provide multiple versions of 465 * long identifiers for LDAP-specific arguments, or {@code false} if 466 * not. 467 */ 468 @Override() 469 protected boolean includeAlternateLongIdentifiers() 470 { 471 return true; 472 } 473 474 475 476 /** 477 * Adds the arguments used by this program that aren't already provided by the 478 * generic {@code LDAPCommandLineTool} framework. 479 * 480 * @param parser The argument parser to which the arguments should be added. 481 * 482 * @throws ArgumentException If a problem occurs while adding the arguments. 483 */ 484 @Override() 485 public void addNonLDAPArguments(final ArgumentParser parser) 486 throws ArgumentException 487 { 488 String description = "The DN of the entry to modify. It may be a simple " + 489 "DN or a value pattern to specify a range of DN (e.g., " + 490 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " + 491 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " + 492 "value pattern syntax. This must be provided."; 493 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description); 494 entryDN.setArgumentGroupName("Modification Arguments"); 495 entryDN.addLongIdentifier("entry-dn"); 496 parser.addArgument(entryDN); 497 498 499 description = "The name of the attribute to modify. Multiple attributes " + 500 "may be specified by providing this argument multiple " + 501 "times. At least one attribute must be specified."; 502 attribute = new StringArgument('A', "attribute", true, 0, "{name}", 503 description); 504 attribute.setArgumentGroupName("Modification Arguments"); 505 parser.addArgument(attribute); 506 507 508 description = "The length in bytes to use when generating values for the " + 509 "replace modifications. If this is not provided, then a " + 510 "default length of ten bytes will be used."; 511 valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}", 512 description, 1, Integer.MAX_VALUE, 10); 513 valueLength.setArgumentGroupName("Modification Arguments"); 514 valueLength.addLongIdentifier("value-length"); 515 parser.addArgument(valueLength); 516 517 518 description = "The number of values to include in replace " + 519 "modifications. If this is not provided, then a default " + 520 "of one value will be used."; 521 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}", 522 description, 0, Integer.MAX_VALUE, 1); 523 valueCount.setArgumentGroupName("Modification Arguments"); 524 valueCount.addLongIdentifier("value-count"); 525 parser.addArgument(valueCount); 526 527 528 description = "Indicates that the tool should use the increment " + 529 "modification type rather than the replace modification " + 530 "type."; 531 increment = new BooleanArgument(null, "increment", 1, description); 532 increment.setArgumentGroupName("Modification Arguments"); 533 parser.addArgument(increment); 534 535 536 description = "The amount by which to increment values when using the " + 537 "increment modification type. The amount may be negative " + 538 "if values should be decremented rather than incremented. " + 539 "If this is not provided, then a default increment amount " + 540 "of one will be used."; 541 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1, 542 null, description, Integer.MIN_VALUE, 543 Integer.MAX_VALUE, 1); 544 incrementAmount.setArgumentGroupName("Modification Arguments"); 545 incrementAmount.addLongIdentifier("increment-amount"); 546 parser.addArgument(incrementAmount); 547 548 549 description = "The set of characters to use to generate the values for " + 550 "the modifications. It should only include ASCII " + 551 "characters. If this is not provided, then a default set " + 552 "of lowercase alphabetic characters will be used."; 553 characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}", 554 description, 555 "abcdefghijklmnopqrstuvwxyz"); 556 characterSet.setArgumentGroupName("Modification Arguments"); 557 characterSet.addLongIdentifier("character-set"); 558 parser.addArgument(characterSet); 559 560 561 description = "Indicates that modify requests should include the " + 562 "assertion request control with the specified filter."; 563 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1, 564 "{filter}", description); 565 assertionFilter.setArgumentGroupName("Request Control Arguments"); 566 assertionFilter.addLongIdentifier("assertion-filter"); 567 parser.addArgument(assertionFilter); 568 569 570 description = "Indicates that modify requests should include the " + 571 "permissive modify request control."; 572 permissiveModify = new BooleanArgument(null, "permissiveModify", 1, 573 description); 574 permissiveModify.setArgumentGroupName("Request Control Arguments"); 575 permissiveModify.addLongIdentifier("permissive-modify"); 576 parser.addArgument(permissiveModify); 577 578 579 description = "Indicates that modify requests should include the " + 580 "pre-read request control with the specified requested " + 581 "attribute. This argument may be provided multiple times " + 582 "to indicate that multiple requested attributes should be " + 583 "included in the pre-read request control."; 584 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0, 585 "{attribute}", description); 586 preReadAttribute.setArgumentGroupName("Request Control Arguments"); 587 preReadAttribute.addLongIdentifier("pre-read-attribute"); 588 parser.addArgument(preReadAttribute); 589 590 591 description = "Indicates that modify requests should include the " + 592 "post-read request control with the specified requested " + 593 "attribute. This argument may be provided multiple times " + 594 "to indicate that multiple requested attributes should be " + 595 "included in the post-read request control."; 596 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0, 597 "{attribute}", description); 598 postReadAttribute.setArgumentGroupName("Request Control Arguments"); 599 postReadAttribute.addLongIdentifier("post-read-attribute"); 600 parser.addArgument(postReadAttribute); 601 602 603 description = "Indicates that the proxied authorization control (as " + 604 "defined in RFC 4370) should be used to request that " + 605 "operations be processed using an alternate authorization " + 606 "identity. This may be a simple authorization ID or it " + 607 "may be a value pattern to specify a range of " + 608 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL + 609 " for complete details about the value pattern syntax."; 610 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}", 611 description); 612 proxyAs.setArgumentGroupName("Request Control Arguments"); 613 proxyAs.addLongIdentifier("proxy-as"); 614 parser.addArgument(proxyAs); 615 616 617 description = "Indicates that modify requests should include the " + 618 "specified request control. This may be provided multiple " + 619 "times to include multiple request controls."; 620 control = new ControlArgument('J', "control", false, 0, null, description); 621 control.setArgumentGroupName("Request Control Arguments"); 622 parser.addArgument(control); 623 624 625 description = "The number of threads to use to perform the " + 626 "modifications. If this is not provided, a single thread " + 627 "will be used."; 628 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 629 description, 1, Integer.MAX_VALUE, 1); 630 numThreads.setArgumentGroupName("Rate Management Arguments"); 631 numThreads.addLongIdentifier("num-threads"); 632 parser.addArgument(numThreads); 633 634 635 description = "The length of time in seconds between output lines. If " + 636 "this is not provided, then a default interval of five " + 637 "seconds will be used."; 638 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1, 639 "{num}", description, 1, 640 Integer.MAX_VALUE, 5); 641 collectionInterval.setArgumentGroupName("Rate Management Arguments"); 642 collectionInterval.addLongIdentifier("interval-duration"); 643 parser.addArgument(collectionInterval); 644 645 646 description = "The maximum number of intervals for which to run. If " + 647 "this is not provided, then the tool will run until it is " + 648 "interrupted."; 649 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}", 650 description, 1, Integer.MAX_VALUE, 651 Integer.MAX_VALUE); 652 numIntervals.setArgumentGroupName("Rate Management Arguments"); 653 numIntervals.addLongIdentifier("num-intervals"); 654 parser.addArgument(numIntervals); 655 656 description = "The number of modify iterations that should be processed " + 657 "on a connection before that connection is closed and " + 658 "replaced with a newly-established (and authenticated, if " + 659 "appropriate) connection. If this is not provided, then " + 660 "connections will not be periodically closed and " + 661 "re-established."; 662 iterationsBeforeReconnect = new IntegerArgument(null, 663 "iterationsBeforeReconnect", false, 1, "{num}", description, 0); 664 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments"); 665 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect"); 666 parser.addArgument(iterationsBeforeReconnect); 667 668 description = "The target number of modifies to perform per second. It " + 669 "is still necessary to specify a sufficient number of " + 670 "threads for achieving this rate. If neither this option " + 671 "nor --variableRateData is provided, then the tool will " + 672 "run at the maximum rate for the specified number of " + 673 "threads."; 674 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1, 675 "{modifies-per-second}", description, 676 1, Integer.MAX_VALUE); 677 ratePerSecond.setArgumentGroupName("Rate Management Arguments"); 678 ratePerSecond.addLongIdentifier("rate-per-second"); 679 parser.addArgument(ratePerSecond); 680 681 final String variableRateDataArgName = "variableRateData"; 682 final String generateSampleRateFileArgName = "generateSampleRateFile"; 683 description = RateAdjustor.getVariableRateDataArgumentDescription( 684 generateSampleRateFileArgName); 685 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1, 686 "{path}", description, true, true, true, 687 false); 688 variableRateData.setArgumentGroupName("Rate Management Arguments"); 689 variableRateData.addLongIdentifier("variable-rate-data"); 690 parser.addArgument(variableRateData); 691 692 description = RateAdjustor.getGenerateSampleVariableRateFileDescription( 693 variableRateDataArgName); 694 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName, 695 false, 1, "{path}", description, false, 696 true, true, false); 697 sampleRateFile.setArgumentGroupName("Rate Management Arguments"); 698 sampleRateFile.addLongIdentifier("generate-sample-rate-file"); 699 sampleRateFile.setUsageArgument(true); 700 parser.addArgument(sampleRateFile); 701 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile); 702 703 description = "The number of intervals to complete before beginning " + 704 "overall statistics collection. Specifying a nonzero " + 705 "number of warm-up intervals gives the client and server " + 706 "a chance to warm up without skewing performance results."; 707 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1, 708 "{num}", description, 0, Integer.MAX_VALUE, 0); 709 warmUpIntervals.setArgumentGroupName("Rate Management Arguments"); 710 warmUpIntervals.addLongIdentifier("warm-up-intervals"); 711 parser.addArgument(warmUpIntervals); 712 713 description = "Indicates the format to use for timestamps included in " + 714 "the output. A value of 'none' indicates that no " + 715 "timestamps should be included. A value of 'with-date' " + 716 "indicates that both the date and the time should be " + 717 "included. A value of 'without-date' indicates that only " + 718 "the time should be included."; 719 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3); 720 allowedFormats.add("none"); 721 allowedFormats.add("with-date"); 722 allowedFormats.add("without-date"); 723 timestampFormat = new StringArgument(null, "timestampFormat", true, 1, 724 "{format}", description, allowedFormats, "none"); 725 timestampFormat.addLongIdentifier("timestamp-format"); 726 parser.addArgument(timestampFormat); 727 728 description = "Indicates that information about the result codes for " + 729 "failed operations should not be displayed."; 730 suppressErrorsArgument = new BooleanArgument(null, 731 "suppressErrorResultCodes", 1, description); 732 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes"); 733 parser.addArgument(suppressErrorsArgument); 734 735 description = "Generate output in CSV format rather than a " + 736 "display-friendly format"; 737 csvFormat = new BooleanArgument('c', "csv", 1, description); 738 parser.addArgument(csvFormat); 739 740 description = "Specifies the seed to use for the random number generator."; 741 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}", 742 description); 743 randomSeed.addLongIdentifier("random-seed"); 744 parser.addArgument(randomSeed); 745 746 747 // The incrementAmount argument can only be used if the increment argument 748 // is provided. 749 parser.addDependentArgumentSet(incrementAmount, increment); 750 751 752 // Neither the valueLength nor valueCount arguments can be used if the 753 // increment argument is provided. 754 parser.addExclusiveArgumentSet(increment, valueLength); 755 parser.addExclusiveArgumentSet(increment, valueCount); 756 } 757 758 759 760 /** 761 * Indicates whether this tool supports creating connections to multiple 762 * servers. If it is to support multiple servers, then the "--hostname" and 763 * "--port" arguments will be allowed to be provided multiple times, and 764 * will be required to be provided the same number of times. The same type of 765 * communication security and bind credentials will be used for all servers. 766 * 767 * @return {@code true} if this tool supports creating connections to 768 * multiple servers, or {@code false} if not. 769 */ 770 @Override() 771 protected boolean supportsMultipleServers() 772 { 773 return true; 774 } 775 776 777 778 /** 779 * Retrieves the connection options that should be used for connections 780 * created for use with this tool. 781 * 782 * @return The connection options that should be used for connections created 783 * for use with this tool. 784 */ 785 @Override() 786 public LDAPConnectionOptions getConnectionOptions() 787 { 788 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 789 options.setUseSynchronousMode(true); 790 return options; 791 } 792 793 794 795 /** 796 * Performs the actual processing for this tool. In this case, it gets a 797 * connection to the directory server and uses it to perform the requested 798 * modifications. 799 * 800 * @return The result code for the processing that was performed. 801 */ 802 @Override() 803 public ResultCode doToolProcessing() 804 { 805 runningThread = Thread.currentThread(); 806 807 try 808 { 809 return doToolProcessingInternal(); 810 } 811 finally 812 { 813 runningThread = null; 814 } 815 816 } 817 818 819 /** 820 * Performs the actual processing for this tool. In this case, it gets a 821 * connection to the directory server and uses it to perform the requested 822 * modifications. 823 * 824 * @return The result code for the processing that was performed. 825 */ 826 private ResultCode doToolProcessingInternal() 827 { 828 // If the sample rate file argument was specified, then generate the sample 829 // variable rate data file and return. 830 if (sampleRateFile.isPresent()) 831 { 832 try 833 { 834 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue()); 835 return ResultCode.SUCCESS; 836 } 837 catch (final Exception e) 838 { 839 debugException(e); 840 err("An error occurred while trying to write sample variable data " + 841 "rate file '", sampleRateFile.getValue().getAbsolutePath(), 842 "': ", getExceptionMessage(e)); 843 return ResultCode.LOCAL_ERROR; 844 } 845 } 846 847 848 // Determine the random seed to use. 849 final Long seed; 850 if (randomSeed.isPresent()) 851 { 852 seed = Long.valueOf(randomSeed.getValue()); 853 } 854 else 855 { 856 seed = null; 857 } 858 859 // Create the value patterns for the target entry DN and proxied 860 // authorization identities. 861 final ValuePattern dnPattern; 862 try 863 { 864 dnPattern = new ValuePattern(entryDN.getValue(), seed); 865 } 866 catch (final ParseException pe) 867 { 868 debugException(pe); 869 err("Unable to parse the entry DN value pattern: ", pe.getMessage()); 870 return ResultCode.PARAM_ERROR; 871 } 872 873 final ValuePattern authzIDPattern; 874 if (proxyAs.isPresent()) 875 { 876 try 877 { 878 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed); 879 } 880 catch (final ParseException pe) 881 { 882 debugException(pe); 883 err("Unable to parse the proxied authorization pattern: ", 884 pe.getMessage()); 885 return ResultCode.PARAM_ERROR; 886 } 887 } 888 else 889 { 890 authzIDPattern = null; 891 } 892 893 894 // Get the set of controls to include in modify requests. 895 final ArrayList<Control> controlList = new ArrayList<Control>(5); 896 if (assertionFilter.isPresent()) 897 { 898 controlList.add(new AssertionRequestControl(assertionFilter.getValue())); 899 } 900 901 if (permissiveModify.isPresent()) 902 { 903 controlList.add(new PermissiveModifyRequestControl()); 904 } 905 906 if (preReadAttribute.isPresent()) 907 { 908 final List<String> attrList = preReadAttribute.getValues(); 909 final String[] attrArray = new String[attrList.size()]; 910 attrList.toArray(attrArray); 911 controlList.add(new PreReadRequestControl(attrArray)); 912 } 913 914 if (postReadAttribute.isPresent()) 915 { 916 final List<String> attrList = postReadAttribute.getValues(); 917 final String[] attrArray = new String[attrList.size()]; 918 attrList.toArray(attrArray); 919 controlList.add(new PostReadRequestControl(attrArray)); 920 } 921 922 if (control.isPresent()) 923 { 924 controlList.addAll(control.getValues()); 925 } 926 927 final Control[] controlArray = new Control[controlList.size()]; 928 controlList.toArray(controlArray); 929 930 931 // Get the names of the attributes to modify. 932 final String[] attrs = new String[attribute.getValues().size()]; 933 attribute.getValues().toArray(attrs); 934 935 936 // Get the character set as a byte array. 937 final byte[] charSet = getBytes(characterSet.getValue()); 938 939 940 // If the --ratePerSecond option was specified, then limit the rate 941 // accordingly. 942 FixedRateBarrier fixedRateBarrier = null; 943 if (ratePerSecond.isPresent() || variableRateData.isPresent()) 944 { 945 // We might not have a rate per second if --variableRateData is specified. 946 // The rate typically doesn't matter except when we have warm-up 947 // intervals. In this case, we'll run at the max rate. 948 final int intervalSeconds = collectionInterval.getValue(); 949 final int ratePerInterval = 950 (ratePerSecond.getValue() == null) 951 ? Integer.MAX_VALUE 952 : ratePerSecond.getValue() * intervalSeconds; 953 fixedRateBarrier = 954 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval); 955 } 956 957 958 // If --variableRateData was specified, then initialize a RateAdjustor. 959 RateAdjustor rateAdjustor = null; 960 if (variableRateData.isPresent()) 961 { 962 try 963 { 964 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier, 965 ratePerSecond.getValue(), variableRateData.getValue()); 966 } 967 catch (final IOException e) 968 { 969 debugException(e); 970 err("Initializing the variable rates failed: " + e.getMessage()); 971 return ResultCode.PARAM_ERROR; 972 } 973 catch (final IllegalArgumentException e) 974 { 975 debugException(e); 976 err("Initializing the variable rates failed: " + e.getMessage()); 977 return ResultCode.PARAM_ERROR; 978 } 979 } 980 981 982 // Determine whether to include timestamps in the output and if so what 983 // format should be used for them. 984 final boolean includeTimestamp; 985 final String timeFormat; 986 if (timestampFormat.getValue().equalsIgnoreCase("with-date")) 987 { 988 includeTimestamp = true; 989 timeFormat = "dd/MM/yyyy HH:mm:ss"; 990 } 991 else if (timestampFormat.getValue().equalsIgnoreCase("without-date")) 992 { 993 includeTimestamp = true; 994 timeFormat = "HH:mm:ss"; 995 } 996 else 997 { 998 includeTimestamp = false; 999 timeFormat = null; 1000 } 1001 1002 1003 // Determine whether any warm-up intervals should be run. 1004 final long totalIntervals; 1005 final boolean warmUp; 1006 int remainingWarmUpIntervals = warmUpIntervals.getValue(); 1007 if (remainingWarmUpIntervals > 0) 1008 { 1009 warmUp = true; 1010 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals; 1011 } 1012 else 1013 { 1014 warmUp = true; 1015 totalIntervals = 0L + numIntervals.getValue(); 1016 } 1017 1018 1019 // Create the table that will be used to format the output. 1020 final OutputFormat outputFormat; 1021 if (csvFormat.isPresent()) 1022 { 1023 outputFormat = OutputFormat.CSV; 1024 } 1025 else 1026 { 1027 outputFormat = OutputFormat.COLUMNS; 1028 } 1029 1030 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp, 1031 timeFormat, outputFormat, " ", 1032 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1033 "Mods/Sec"), 1034 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1035 "Avg Dur ms"), 1036 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent", 1037 "Errors/Sec"), 1038 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1039 "Mods/Sec"), 1040 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall", 1041 "Avg Dur ms")); 1042 1043 1044 // Create values to use for statistics collection. 1045 final AtomicLong modCounter = new AtomicLong(0L); 1046 final AtomicLong errorCounter = new AtomicLong(0L); 1047 final AtomicLong modDurations = new AtomicLong(0L); 1048 final ResultCodeCounter rcCounter = new ResultCodeCounter(); 1049 1050 1051 // Determine the length of each interval in milliseconds. 1052 final long intervalMillis = 1000L * collectionInterval.getValue(); 1053 1054 1055 // Create a random number generator to use for seeding the per-thread 1056 // generators. 1057 final Random random = new Random(); 1058 1059 1060 // Create the threads to use for the modifications. 1061 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1); 1062 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()]; 1063 for (int i=0; i < threads.length; i++) 1064 { 1065 final LDAPConnection connection; 1066 try 1067 { 1068 connection = getConnection(); 1069 } 1070 catch (final LDAPException le) 1071 { 1072 debugException(le); 1073 err("Unable to connect to the directory server: ", 1074 getExceptionMessage(le)); 1075 return le.getResultCode(); 1076 } 1077 1078 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs, 1079 charSet, valueLength.getValue(), valueCount.getValue(), 1080 increment.isPresent(), incrementAmount.getValue(), controlArray, 1081 authzIDPattern, random.nextLong(), 1082 iterationsBeforeReconnect.getValue(), barrier, modCounter, 1083 modDurations, errorCounter, rcCounter, fixedRateBarrier); 1084 threads[i].start(); 1085 } 1086 1087 1088 // Display the table header. 1089 for (final String headerLine : formatter.getHeaderLines(true)) 1090 { 1091 out(headerLine); 1092 } 1093 1094 1095 // Start the RateAdjustor before the threads so that the initial value is 1096 // in place before any load is generated unless we're doing a warm-up in 1097 // which case, we'll start it after the warm-up is complete. 1098 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0)) 1099 { 1100 rateAdjustor.start(); 1101 } 1102 1103 1104 // Indicate that the threads can start running. 1105 try 1106 { 1107 barrier.await(); 1108 } 1109 catch (final Exception e) 1110 { 1111 debugException(e); 1112 } 1113 1114 long overallStartTime = System.nanoTime(); 1115 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis; 1116 1117 1118 boolean setOverallStartTime = false; 1119 long lastDuration = 0L; 1120 long lastNumErrors = 0L; 1121 long lastNumMods = 0L; 1122 long lastEndTime = System.nanoTime(); 1123 for (long i=0; i < totalIntervals; i++) 1124 { 1125 if (rateAdjustor != null) 1126 { 1127 if (! rateAdjustor.isAlive()) 1128 { 1129 out("All of the rates in " + variableRateData.getValue().getName() + 1130 " have been completed."); 1131 break; 1132 } 1133 } 1134 1135 final long startTimeMillis = System.currentTimeMillis(); 1136 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis; 1137 nextIntervalStartTime += intervalMillis; 1138 if (sleepTimeMillis > 0) 1139 { 1140 sleeper.sleep(sleepTimeMillis); 1141 } 1142 1143 if (stopRequested.get()) 1144 { 1145 break; 1146 } 1147 1148 final long endTime = System.nanoTime(); 1149 final long intervalDuration = endTime - lastEndTime; 1150 1151 final long numMods; 1152 final long numErrors; 1153 final long totalDuration; 1154 if (warmUp && (remainingWarmUpIntervals > 0)) 1155 { 1156 numMods = modCounter.getAndSet(0L); 1157 numErrors = errorCounter.getAndSet(0L); 1158 totalDuration = modDurations.getAndSet(0L); 1159 } 1160 else 1161 { 1162 numMods = modCounter.get(); 1163 numErrors = errorCounter.get(); 1164 totalDuration = modDurations.get(); 1165 } 1166 1167 final long recentNumMods = numMods - lastNumMods; 1168 final long recentNumErrors = numErrors - lastNumErrors; 1169 final long recentDuration = totalDuration - lastDuration; 1170 1171 final double numSeconds = intervalDuration / 1000000000.0d; 1172 final double recentModRate = recentNumMods / numSeconds; 1173 final double recentErrorRate = recentNumErrors / numSeconds; 1174 1175 final double recentAvgDuration; 1176 if (recentNumMods > 0L) 1177 { 1178 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1000000; 1179 } 1180 else 1181 { 1182 recentAvgDuration = 0.0d; 1183 } 1184 1185 if (warmUp && (remainingWarmUpIntervals > 0)) 1186 { 1187 out(formatter.formatRow(recentModRate, recentAvgDuration, 1188 recentErrorRate, "warming up", "warming up")); 1189 1190 remainingWarmUpIntervals--; 1191 if (remainingWarmUpIntervals == 0) 1192 { 1193 out("Warm-up completed. Beginning overall statistics collection."); 1194 setOverallStartTime = true; 1195 if (rateAdjustor != null) 1196 { 1197 rateAdjustor.start(); 1198 } 1199 } 1200 } 1201 else 1202 { 1203 if (setOverallStartTime) 1204 { 1205 overallStartTime = lastEndTime; 1206 setOverallStartTime = false; 1207 } 1208 1209 final double numOverallSeconds = 1210 (endTime - overallStartTime) / 1000000000.0d; 1211 final double overallAuthRate = numMods / numOverallSeconds; 1212 1213 final double overallAvgDuration; 1214 if (numMods > 0L) 1215 { 1216 overallAvgDuration = 1.0d * totalDuration / numMods / 1000000; 1217 } 1218 else 1219 { 1220 overallAvgDuration = 0.0d; 1221 } 1222 1223 out(formatter.formatRow(recentModRate, recentAvgDuration, 1224 recentErrorRate, overallAuthRate, overallAvgDuration)); 1225 1226 lastNumMods = numMods; 1227 lastNumErrors = numErrors; 1228 lastDuration = totalDuration; 1229 } 1230 1231 final List<ObjectPair<ResultCode,Long>> rcCounts = 1232 rcCounter.getCounts(true); 1233 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty())) 1234 { 1235 err("\tError Results:"); 1236 for (final ObjectPair<ResultCode,Long> p : rcCounts) 1237 { 1238 err("\t", p.getFirst().getName(), ": ", p.getSecond()); 1239 } 1240 } 1241 1242 lastEndTime = endTime; 1243 } 1244 1245 // Shut down the RateAdjustor if we have one. 1246 if (rateAdjustor != null) 1247 { 1248 rateAdjustor.shutDown(); 1249 } 1250 1251 // Stop all of the threads. 1252 ResultCode resultCode = ResultCode.SUCCESS; 1253 for (final ModRateThread t : threads) 1254 { 1255 final ResultCode r = t.stopRunning(); 1256 if (resultCode == ResultCode.SUCCESS) 1257 { 1258 resultCode = r; 1259 } 1260 } 1261 1262 return resultCode; 1263 } 1264 1265 1266 1267 /** 1268 * Requests that this tool stop running. This method will attempt to wait 1269 * for all threads to complete before returning control to the caller. 1270 */ 1271 public void stopRunning() 1272 { 1273 stopRequested.set(true); 1274 sleeper.wakeup(); 1275 1276 final Thread t = runningThread; 1277 if (t != null) 1278 { 1279 try 1280 { 1281 t.join(); 1282 } 1283 catch (final Exception e) 1284 { 1285 debugException(e); 1286 1287 if (e instanceof InterruptedException) 1288 { 1289 Thread.currentThread().interrupt(); 1290 } 1291 } 1292 } 1293 } 1294 1295 1296 1297 /** 1298 * {@inheritDoc} 1299 */ 1300 @Override() 1301 public LinkedHashMap<String[],String> getExampleUsages() 1302 { 1303 final LinkedHashMap<String[],String> examples = 1304 new LinkedHashMap<String[],String>(2); 1305 1306 String[] args = 1307 { 1308 "--hostname", "server.example.com", 1309 "--port", "389", 1310 "--bindDN", "uid=admin,dc=example,dc=com", 1311 "--bindPassword", "password", 1312 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com", 1313 "--attribute", "description", 1314 "--valueLength", "12", 1315 "--numThreads", "10" 1316 }; 1317 String description = 1318 "Test modify performance by randomly selecting entries across a set " + 1319 "of one million users located below 'ou=People,dc=example,dc=com' " + 1320 "with ten concurrent threads and replacing the values for the " + 1321 "description attribute with a string of 12 randomly-selected " + 1322 "lowercase alphabetic characters."; 1323 examples.put(args, description); 1324 1325 args = new String[] 1326 { 1327 "--generateSampleRateFile", "variable-rate-data.txt" 1328 }; 1329 description = 1330 "Generate a sample variable rate definition file that may be used " + 1331 "in conjunction with the --variableRateData argument. The sample " + 1332 "file will include comments that describe the format for data to be " + 1333 "included in this file."; 1334 examples.put(args, description); 1335 1336 return examples; 1337 } 1338}