001/* 002 * Copyright 2013-2017 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2013-2017 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.OutputStream; 026import java.util.Collections; 027import java.util.LinkedHashMap; 028import java.util.List; 029import java.util.Map; 030import java.util.TreeMap; 031import java.util.concurrent.atomic.AtomicLong; 032 033import com.unboundid.asn1.ASN1OctetString; 034import com.unboundid.ldap.sdk.Attribute; 035import com.unboundid.ldap.sdk.DN; 036import com.unboundid.ldap.sdk.Filter; 037import com.unboundid.ldap.sdk.LDAPConnectionOptions; 038import com.unboundid.ldap.sdk.LDAPConnectionPool; 039import com.unboundid.ldap.sdk.LDAPException; 040import com.unboundid.ldap.sdk.LDAPSearchException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.SearchRequest; 043import com.unboundid.ldap.sdk.SearchResult; 044import com.unboundid.ldap.sdk.SearchResultEntry; 045import com.unboundid.ldap.sdk.SearchResultReference; 046import com.unboundid.ldap.sdk.SearchResultListener; 047import com.unboundid.ldap.sdk.SearchScope; 048import com.unboundid.ldap.sdk.Version; 049import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl; 050import com.unboundid.util.Debug; 051import com.unboundid.util.LDAPCommandLineTool; 052import com.unboundid.util.StaticUtils; 053import com.unboundid.util.ThreadSafety; 054import com.unboundid.util.ThreadSafetyLevel; 055import com.unboundid.util.args.ArgumentException; 056import com.unboundid.util.args.ArgumentParser; 057import com.unboundid.util.args.DNArgument; 058import com.unboundid.util.args.IntegerArgument; 059import com.unboundid.util.args.StringArgument; 060 061 062 063/** 064 * This class provides a tool that may be used to identify references to entries 065 * that do not exist. This tool can be useful for verifying existing data in 066 * directory servers that provide support for referential integrity. 067 * <BR><BR> 068 * All of the necessary information is provided using command line arguments. 069 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 070 * class, as well as the following additional arguments: 071 * <UL> 072 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 073 * for the searches. At least one base DN must be provided.</LI> 074 * <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute 075 * that is expected to contain references to other entries. This 076 * attribute should be indexed for equality searches, and its values 077 * should be DNs. At least one attribute must be provided.</LI> 078 * <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search 079 * to find entries with references to other entries should use the simple 080 * paged results control to iterate across entries in fixed-size pages 081 * rather than trying to use a single search to identify all entries that 082 * reference other entries.</LI> 083 * </UL> 084 */ 085@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 086public final class IdentifyReferencesToMissingEntries 087 extends LDAPCommandLineTool 088 implements SearchResultListener 089{ 090 /** 091 * The serial version UID for this serializable class. 092 */ 093 private static final long serialVersionUID = 1981894839719501258L; 094 095 096 097 // The number of entries examined so far. 098 private final AtomicLong entriesExamined; 099 100 // The argument used to specify the base DNs to use for searches. 101 private DNArgument baseDNArgument; 102 103 // The argument used to specify the search page size. 104 private IntegerArgument pageSizeArgument; 105 106 // The connection to use for retrieving referenced entries. 107 private LDAPConnectionPool getReferencedEntriesPool; 108 109 // A map with counts of missing references by attribute type. 110 private final Map<String,AtomicLong> missingReferenceCounts; 111 112 // The names of the attributes for which to find missing references. 113 private String[] attributes; 114 115 // The argument used to specify the attributes for which to find missing 116 // references. 117 private StringArgument attributeArgument; 118 119 120 121 /** 122 * Parse the provided command line arguments and perform the appropriate 123 * processing. 124 * 125 * @param args The command line arguments provided to this program. 126 */ 127 public static void main(final String... args) 128 { 129 final ResultCode resultCode = main(args, System.out, System.err); 130 if (resultCode != ResultCode.SUCCESS) 131 { 132 System.exit(resultCode.intValue()); 133 } 134 } 135 136 137 138 /** 139 * Parse the provided command line arguments and perform the appropriate 140 * processing. 141 * 142 * @param args The command line arguments provided to this program. 143 * @param outStream The output stream to which standard out should be 144 * written. It may be {@code null} if output should be 145 * suppressed. 146 * @param errStream The output stream to which standard error should be 147 * written. It may be {@code null} if error messages 148 * should be suppressed. 149 * 150 * @return A result code indicating whether the processing was successful. 151 */ 152 public static ResultCode main(final String[] args, 153 final OutputStream outStream, 154 final OutputStream errStream) 155 { 156 final IdentifyReferencesToMissingEntries tool = 157 new IdentifyReferencesToMissingEntries(outStream, errStream); 158 return tool.runTool(args); 159 } 160 161 162 163 /** 164 * Creates a new instance of this tool. 165 * 166 * @param outStream The output stream to which standard out should be 167 * written. It may be {@code null} if output should be 168 * suppressed. 169 * @param errStream The output stream to which standard error should be 170 * written. It may be {@code null} if error messages 171 * should be suppressed. 172 */ 173 public IdentifyReferencesToMissingEntries(final OutputStream outStream, 174 final OutputStream errStream) 175 { 176 super(outStream, errStream); 177 178 baseDNArgument = null; 179 pageSizeArgument = null; 180 attributeArgument = null; 181 getReferencedEntriesPool = null; 182 183 entriesExamined = new AtomicLong(0L); 184 missingReferenceCounts = new TreeMap<String, AtomicLong>(); 185 } 186 187 188 189 /** 190 * Retrieves the name of this tool. It should be the name of the command used 191 * to invoke this tool. 192 * 193 * @return The name for this tool. 194 */ 195 @Override() 196 public String getToolName() 197 { 198 return "identify-references-to-missing-entries"; 199 } 200 201 202 203 /** 204 * Retrieves a human-readable description for this tool. 205 * 206 * @return A human-readable description for this tool. 207 */ 208 @Override() 209 public String getToolDescription() 210 { 211 return "This tool may be used to identify entries containing one or more " + 212 "attributes which reference entries that do not exist. This may " + 213 "require the ability to perform unindexed searches and/or the " + 214 "ability to use the simple paged results control."; 215 } 216 217 218 219 /** 220 * Retrieves a version string for this tool, if available. 221 * 222 * @return A version string for this tool, or {@code null} if none is 223 * available. 224 */ 225 @Override() 226 public String getToolVersion() 227 { 228 return Version.NUMERIC_VERSION_STRING; 229 } 230 231 232 233 /** 234 * Indicates whether this tool should provide support for an interactive mode, 235 * in which the tool offers a mode in which the arguments can be provided in 236 * a text-driven menu rather than requiring them to be given on the command 237 * line. If interactive mode is supported, it may be invoked using the 238 * "--interactive" argument. Alternately, if interactive mode is supported 239 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 240 * interactive mode may be invoked by simply launching the tool without any 241 * arguments. 242 * 243 * @return {@code true} if this tool supports interactive mode, or 244 * {@code false} if not. 245 */ 246 @Override() 247 public boolean supportsInteractiveMode() 248 { 249 return true; 250 } 251 252 253 254 /** 255 * Indicates whether this tool defaults to launching in interactive mode if 256 * the tool is invoked without any command-line arguments. This will only be 257 * used if {@link #supportsInteractiveMode()} returns {@code true}. 258 * 259 * @return {@code true} if this tool defaults to using interactive mode if 260 * launched without any command-line arguments, or {@code false} if 261 * not. 262 */ 263 @Override() 264 public boolean defaultsToInteractiveMode() 265 { 266 return true; 267 } 268 269 270 271 /** 272 * Indicates whether this tool should provide arguments for redirecting output 273 * to a file. If this method returns {@code true}, then the tool will offer 274 * an "--outputFile" argument that will specify the path to a file to which 275 * all standard output and standard error content will be written, and it will 276 * also offer a "--teeToStandardOut" argument that can only be used if the 277 * "--outputFile" argument is present and will cause all output to be written 278 * to both the specified output file and to standard output. 279 * 280 * @return {@code true} if this tool should provide arguments for redirecting 281 * output to a file, or {@code false} if not. 282 */ 283 @Override() 284 protected boolean supportsOutputFile() 285 { 286 return true; 287 } 288 289 290 291 /** 292 * Indicates whether this tool should default to interactively prompting for 293 * the bind password if a password is required but no argument was provided 294 * to indicate how to get the password. 295 * 296 * @return {@code true} if this tool should default to interactively 297 * prompting for the bind password, or {@code false} if not. 298 */ 299 @Override() 300 protected boolean defaultToPromptForBindPassword() 301 { 302 return true; 303 } 304 305 306 307 /** 308 * Indicates whether this tool supports the use of a properties file for 309 * specifying default values for arguments that aren't specified on the 310 * command line. 311 * 312 * @return {@code true} if this tool supports the use of a properties file 313 * for specifying default values for arguments that aren't specified 314 * on the command line, or {@code false} if not. 315 */ 316 @Override() 317 public boolean supportsPropertiesFile() 318 { 319 return true; 320 } 321 322 323 324 /** 325 * Indicates whether the LDAP-specific arguments should include alternate 326 * versions of all long identifiers that consist of multiple words so that 327 * they are available in both camelCase and dash-separated versions. 328 * 329 * @return {@code true} if this tool should provide multiple versions of 330 * long identifiers for LDAP-specific arguments, or {@code false} if 331 * not. 332 */ 333 @Override() 334 protected boolean includeAlternateLongIdentifiers() 335 { 336 return true; 337 } 338 339 340 341 /** 342 * Adds the arguments needed by this command-line tool to the provided 343 * argument parser which are not related to connecting or authenticating to 344 * the directory server. 345 * 346 * @param parser The argument parser to which the arguments should be added. 347 * 348 * @throws ArgumentException If a problem occurs while adding the arguments. 349 */ 350 @Override() 351 public void addNonLDAPArguments(final ArgumentParser parser) 352 throws ArgumentException 353 { 354 String description = "The search base DN(s) to use to find entries with " + 355 "references to other entries. At least one base DN must be " + 356 "specified."; 357 baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}", 358 description); 359 baseDNArgument.addLongIdentifier("base-dn"); 360 parser.addArgument(baseDNArgument); 361 362 description = "The attribute(s) for which to find missing references. " + 363 "At least one attribute must be specified, and each attribute " + 364 "must be indexed for equality searches and have values which are DNs."; 365 attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}", 366 description); 367 parser.addArgument(attributeArgument); 368 369 description = "The maximum number of entries to retrieve at a time when " + 370 "attempting to find entries with references to other entries. This " + 371 "requires that the authenticated user have permission to use the " + 372 "simple paged results control, but it can avoid problems with the " + 373 "server sending entries too quickly for the client to handle. By " + 374 "default, the simple paged results control will not be used."; 375 pageSizeArgument = 376 new IntegerArgument('z', "simplePageSize", false, 1, "{num}", 377 description, 1, Integer.MAX_VALUE); 378 pageSizeArgument.addLongIdentifier("simple-page-size"); 379 parser.addArgument(pageSizeArgument); 380 } 381 382 383 384 /** 385 * Retrieves the connection options that should be used for connections that 386 * are created with this command line tool. Subclasses may override this 387 * method to use a custom set of connection options. 388 * 389 * @return The connection options that should be used for connections that 390 * are created with this command line tool. 391 */ 392 @Override() 393 public LDAPConnectionOptions getConnectionOptions() 394 { 395 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 396 397 options.setUseSynchronousMode(true); 398 options.setResponseTimeoutMillis(0L); 399 400 return options; 401 } 402 403 404 405 /** 406 * Performs the core set of processing for this tool. 407 * 408 * @return A result code that indicates whether the processing completed 409 * successfully. 410 */ 411 @Override() 412 public ResultCode doToolProcessing() 413 { 414 // Establish a connection to the target directory server to use for 415 // finding references to entries. 416 final LDAPConnectionPool findReferencesPool; 417 try 418 { 419 findReferencesPool = getConnectionPool(1, 1); 420 findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true); 421 } 422 catch (final LDAPException le) 423 { 424 Debug.debugException(le); 425 err("Unable to establish a connection to the directory server: ", 426 StaticUtils.getExceptionMessage(le)); 427 return le.getResultCode(); 428 } 429 430 try 431 { 432 // Establish a second connection to use for retrieving referenced entries. 433 try 434 { 435 getReferencedEntriesPool = getConnectionPool(1,1); 436 getReferencedEntriesPool. 437 setRetryFailedOperationsDueToInvalidConnections(true); 438 } 439 catch (final LDAPException le) 440 { 441 Debug.debugException(le); 442 err("Unable to establish a connection to the directory server: ", 443 StaticUtils.getExceptionMessage(le)); 444 return le.getResultCode(); 445 } 446 447 448 // Get the set of attributes for which to find missing references. 449 final List<String> attrList = attributeArgument.getValues(); 450 attributes = new String[attrList.size()]; 451 attrList.toArray(attributes); 452 453 454 // Construct a search filter that will be used to find all entries with 455 // references to other entries. 456 final Filter filter; 457 if (attributes.length == 1) 458 { 459 filter = Filter.createPresenceFilter(attributes[0]); 460 missingReferenceCounts.put(attributes[0], new AtomicLong(0L)); 461 } 462 else 463 { 464 final Filter[] orComps = new Filter[attributes.length]; 465 for (int i=0; i < attributes.length; i++) 466 { 467 orComps[i] = Filter.createPresenceFilter(attributes[i]); 468 missingReferenceCounts.put(attributes[i], new AtomicLong(0L)); 469 } 470 filter = Filter.createORFilter(orComps); 471 } 472 473 474 // Iterate across all of the search base DNs and perform searches to find 475 // missing references. 476 for (final DN baseDN : baseDNArgument.getValues()) 477 { 478 ASN1OctetString cookie = null; 479 do 480 { 481 final SearchRequest searchRequest = new SearchRequest(this, 482 baseDN.toString(), SearchScope.SUB, filter, attributes); 483 if (pageSizeArgument.isPresent()) 484 { 485 searchRequest.addControl(new SimplePagedResultsControl( 486 pageSizeArgument.getValue(), cookie, false)); 487 } 488 489 SearchResult searchResult; 490 try 491 { 492 searchResult = findReferencesPool.search(searchRequest); 493 } 494 catch (final LDAPSearchException lse) 495 { 496 Debug.debugException(lse); 497 try 498 { 499 searchResult = findReferencesPool.search(searchRequest); 500 } 501 catch (final LDAPSearchException lse2) 502 { 503 Debug.debugException(lse2); 504 searchResult = lse2.getSearchResult(); 505 } 506 } 507 508 if (searchResult.getResultCode() != ResultCode.SUCCESS) 509 { 510 err("An error occurred while attempting to search for missing " + 511 "references to entries below " + baseDN + ": " + 512 searchResult.getDiagnosticMessage()); 513 return searchResult.getResultCode(); 514 } 515 516 final SimplePagedResultsControl pagedResultsResponse; 517 try 518 { 519 pagedResultsResponse = SimplePagedResultsControl.get(searchResult); 520 } 521 catch (final LDAPException le) 522 { 523 Debug.debugException(le); 524 err("An error occurred while attempting to decode a simple " + 525 "paged results response control in the response to a " + 526 "search for entries below " + baseDN + ": " + 527 StaticUtils.getExceptionMessage(le)); 528 return le.getResultCode(); 529 } 530 531 if (pagedResultsResponse != null) 532 { 533 if (pagedResultsResponse.moreResultsToReturn()) 534 { 535 cookie = pagedResultsResponse.getCookie(); 536 } 537 else 538 { 539 cookie = null; 540 } 541 } 542 } 543 while (cookie != null); 544 } 545 546 547 // See if there were any missing references found. 548 boolean missingReferenceFound = false; 549 for (final Map.Entry<String,AtomicLong> e : 550 missingReferenceCounts.entrySet()) 551 { 552 final long numMissing = e.getValue().get(); 553 if (numMissing > 0L) 554 { 555 if (! missingReferenceFound) 556 { 557 err(); 558 missingReferenceFound = true; 559 } 560 561 err("Found " + numMissing + ' ' + e.getKey() + 562 " references to entries that do not exist."); 563 } 564 } 565 566 if (missingReferenceFound) 567 { 568 return ResultCode.CONSTRAINT_VIOLATION; 569 } 570 else 571 { 572 out("No references were found to entries that do not exist."); 573 return ResultCode.SUCCESS; 574 } 575 } 576 finally 577 { 578 findReferencesPool.close(); 579 580 if (getReferencedEntriesPool != null) 581 { 582 getReferencedEntriesPool.close(); 583 } 584 } 585 } 586 587 588 589 /** 590 * Retrieves a map that correlates the number of missing references found by 591 * attribute type. 592 * 593 * @return A map that correlates the number of missing references found by 594 * attribute type. 595 */ 596 public Map<String,AtomicLong> getMissingReferenceCounts() 597 { 598 return Collections.unmodifiableMap(missingReferenceCounts); 599 } 600 601 602 603 /** 604 * Retrieves a set of information that may be used to generate example usage 605 * information. Each element in the returned map should consist of a map 606 * between an example set of arguments and a string that describes the 607 * behavior of the tool when invoked with that set of arguments. 608 * 609 * @return A set of information that may be used to generate example usage 610 * information. It may be {@code null} or empty if no example usage 611 * information is available. 612 */ 613 @Override() 614 public LinkedHashMap<String[],String> getExampleUsages() 615 { 616 final LinkedHashMap<String[],String> exampleMap = 617 new LinkedHashMap<String[],String>(1); 618 619 final String[] args = 620 { 621 "--hostname", "server.example.com", 622 "--port", "389", 623 "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com", 624 "--bindPassword", "password", 625 "--baseDN", "dc=example,dc=com", 626 "--attribute", "member", 627 "--attribute", "uniqueMember", 628 "--simplePageSize", "100" 629 }; 630 exampleMap.put(args, 631 "Identify all entries below dc=example,dc=com in which either the " + 632 "member or uniqueMember attribute references an entry that " + 633 "does not exist."); 634 635 return exampleMap; 636 } 637 638 639 640 /** 641 * Indicates that the provided search result entry has been returned by the 642 * server and may be processed by this search result listener. 643 * 644 * @param searchEntry The search result entry that has been returned by the 645 * server. 646 */ 647 public void searchEntryReturned(final SearchResultEntry searchEntry) 648 { 649 try 650 { 651 // Find attributes which references to entries that do not exist. 652 for (final String attr : attributes) 653 { 654 final List<Attribute> attrList = 655 searchEntry.getAttributesWithOptions(attr, null); 656 for (final Attribute a : attrList) 657 { 658 for (final String value : a.getValues()) 659 { 660 try 661 { 662 final SearchResultEntry e = 663 getReferencedEntriesPool.getEntry(value, "1.1"); 664 if (e == null) 665 { 666 err("Entry '", searchEntry.getDN(), "' includes attribute ", 667 a.getName(), " that references entry '", value, 668 "' which does not exist."); 669 missingReferenceCounts.get(attr).incrementAndGet(); 670 } 671 } 672 catch (final LDAPException le) 673 { 674 Debug.debugException(le); 675 err("An error occurred while attempting to determine whether " + 676 "entry '" + value + "' referenced in attribute " + 677 a.getName() + " of entry '" + searchEntry.getDN() + 678 "' exists: " + StaticUtils.getExceptionMessage(le)); 679 missingReferenceCounts.get(attr).incrementAndGet(); 680 } 681 } 682 } 683 } 684 } 685 finally 686 { 687 final long count = entriesExamined.incrementAndGet(); 688 if ((count % 1000L) == 0L) 689 { 690 out(count, " entries examined"); 691 } 692 } 693 } 694 695 696 697 /** 698 * Indicates that the provided search result reference has been returned by 699 * the server and may be processed by this search result listener. 700 * 701 * @param searchReference The search result reference that has been returned 702 * by the server. 703 */ 704 public void searchReferenceReturned( 705 final SearchResultReference searchReference) 706 { 707 // No implementation is required. This tool will not follow referrals. 708 } 709}