001/*
002 * Copyright 2013-2017 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2013-2017 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.OutputStream;
026import java.util.Collections;
027import java.util.LinkedHashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.atomic.AtomicLong;
032
033import com.unboundid.asn1.ASN1OctetString;
034import com.unboundid.ldap.sdk.Attribute;
035import com.unboundid.ldap.sdk.DN;
036import com.unboundid.ldap.sdk.Filter;
037import com.unboundid.ldap.sdk.LDAPConnectionOptions;
038import com.unboundid.ldap.sdk.LDAPConnectionPool;
039import com.unboundid.ldap.sdk.LDAPException;
040import com.unboundid.ldap.sdk.LDAPSearchException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.SearchRequest;
043import com.unboundid.ldap.sdk.SearchResult;
044import com.unboundid.ldap.sdk.SearchResultEntry;
045import com.unboundid.ldap.sdk.SearchResultReference;
046import com.unboundid.ldap.sdk.SearchResultListener;
047import com.unboundid.ldap.sdk.SearchScope;
048import com.unboundid.ldap.sdk.Version;
049import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
050import com.unboundid.util.Debug;
051import com.unboundid.util.LDAPCommandLineTool;
052import com.unboundid.util.StaticUtils;
053import com.unboundid.util.ThreadSafety;
054import com.unboundid.util.ThreadSafetyLevel;
055import com.unboundid.util.args.ArgumentException;
056import com.unboundid.util.args.ArgumentParser;
057import com.unboundid.util.args.DNArgument;
058import com.unboundid.util.args.IntegerArgument;
059import com.unboundid.util.args.StringArgument;
060
061
062
063/**
064 * This class provides a tool that may be used to identify references to entries
065 * that do not exist.  This tool can be useful for verifying existing data in
066 * directory servers that provide support for referential integrity.
067 * <BR><BR>
068 * All of the necessary information is provided using command line arguments.
069 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
070 * class, as well as the following additional arguments:
071 * <UL>
072 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
073 *       for the searches.  At least one base DN must be provided.</LI>
074 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
075 *       that is expected to contain references to other entries.  This
076 *       attribute should be indexed for equality searches, and its values
077 *       should be DNs.  At least one attribute must be provided.</LI>
078 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
079 *       to find entries with references to other entries should use the simple
080 *       paged results control to iterate across entries in fixed-size pages
081 *       rather than trying to use a single search to identify all entries that
082 *       reference other entries.</LI>
083 * </UL>
084 */
085@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
086public final class IdentifyReferencesToMissingEntries
087       extends LDAPCommandLineTool
088       implements SearchResultListener
089{
090  /**
091   * The serial version UID for this serializable class.
092   */
093  private static final long serialVersionUID = 1981894839719501258L;
094
095
096
097  // The number of entries examined so far.
098  private final AtomicLong entriesExamined;
099
100  // The argument used to specify the base DNs to use for searches.
101  private DNArgument baseDNArgument;
102
103  // The argument used to specify the search page size.
104  private IntegerArgument pageSizeArgument;
105
106  // The connection to use for retrieving referenced entries.
107  private LDAPConnectionPool getReferencedEntriesPool;
108
109  // A map with counts of missing references by attribute type.
110  private final Map<String,AtomicLong> missingReferenceCounts;
111
112  // The names of the attributes for which to find missing references.
113  private String[] attributes;
114
115  // The argument used to specify the attributes for which to find missing
116  // references.
117  private StringArgument attributeArgument;
118
119
120
121  /**
122   * Parse the provided command line arguments and perform the appropriate
123   * processing.
124   *
125   * @param  args  The command line arguments provided to this program.
126   */
127  public static void main(final String... args)
128  {
129    final ResultCode resultCode = main(args, System.out, System.err);
130    if (resultCode != ResultCode.SUCCESS)
131    {
132      System.exit(resultCode.intValue());
133    }
134  }
135
136
137
138  /**
139   * Parse the provided command line arguments and perform the appropriate
140   * processing.
141   *
142   * @param  args       The command line arguments provided to this program.
143   * @param  outStream  The output stream to which standard out should be
144   *                    written.  It may be {@code null} if output should be
145   *                    suppressed.
146   * @param  errStream  The output stream to which standard error should be
147   *                    written.  It may be {@code null} if error messages
148   *                    should be suppressed.
149   *
150   * @return A result code indicating whether the processing was successful.
151   */
152  public static ResultCode main(final String[] args,
153                                final OutputStream outStream,
154                                final OutputStream errStream)
155  {
156    final IdentifyReferencesToMissingEntries tool =
157         new IdentifyReferencesToMissingEntries(outStream, errStream);
158    return tool.runTool(args);
159  }
160
161
162
163  /**
164   * Creates a new instance of this tool.
165   *
166   * @param  outStream  The output stream to which standard out should be
167   *                    written.  It may be {@code null} if output should be
168   *                    suppressed.
169   * @param  errStream  The output stream to which standard error should be
170   *                    written.  It may be {@code null} if error messages
171   *                    should be suppressed.
172   */
173  public IdentifyReferencesToMissingEntries(final OutputStream outStream,
174                                            final OutputStream errStream)
175  {
176    super(outStream, errStream);
177
178    baseDNArgument = null;
179    pageSizeArgument = null;
180    attributeArgument = null;
181    getReferencedEntriesPool = null;
182
183    entriesExamined = new AtomicLong(0L);
184    missingReferenceCounts = new TreeMap<String, AtomicLong>();
185  }
186
187
188
189  /**
190   * Retrieves the name of this tool.  It should be the name of the command used
191   * to invoke this tool.
192   *
193   * @return  The name for this tool.
194   */
195  @Override()
196  public String getToolName()
197  {
198    return "identify-references-to-missing-entries";
199  }
200
201
202
203  /**
204   * Retrieves a human-readable description for this tool.
205   *
206   * @return  A human-readable description for this tool.
207   */
208  @Override()
209  public String getToolDescription()
210  {
211    return "This tool may be used to identify entries containing one or more " +
212         "attributes which reference entries that do not exist.  This may " +
213         "require the ability to perform unindexed searches and/or the " +
214         "ability to use the simple paged results control.";
215  }
216
217
218
219  /**
220   * Retrieves a version string for this tool, if available.
221   *
222   * @return  A version string for this tool, or {@code null} if none is
223   *          available.
224   */
225  @Override()
226  public String getToolVersion()
227  {
228    return Version.NUMERIC_VERSION_STRING;
229  }
230
231
232
233  /**
234   * Indicates whether this tool should provide support for an interactive mode,
235   * in which the tool offers a mode in which the arguments can be provided in
236   * a text-driven menu rather than requiring them to be given on the command
237   * line.  If interactive mode is supported, it may be invoked using the
238   * "--interactive" argument.  Alternately, if interactive mode is supported
239   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
240   * interactive mode may be invoked by simply launching the tool without any
241   * arguments.
242   *
243   * @return  {@code true} if this tool supports interactive mode, or
244   *          {@code false} if not.
245   */
246  @Override()
247  public boolean supportsInteractiveMode()
248  {
249    return true;
250  }
251
252
253
254  /**
255   * Indicates whether this tool defaults to launching in interactive mode if
256   * the tool is invoked without any command-line arguments.  This will only be
257   * used if {@link #supportsInteractiveMode()} returns {@code true}.
258   *
259   * @return  {@code true} if this tool defaults to using interactive mode if
260   *          launched without any command-line arguments, or {@code false} if
261   *          not.
262   */
263  @Override()
264  public boolean defaultsToInteractiveMode()
265  {
266    return true;
267  }
268
269
270
271  /**
272   * Indicates whether this tool should provide arguments for redirecting output
273   * to a file.  If this method returns {@code true}, then the tool will offer
274   * an "--outputFile" argument that will specify the path to a file to which
275   * all standard output and standard error content will be written, and it will
276   * also offer a "--teeToStandardOut" argument that can only be used if the
277   * "--outputFile" argument is present and will cause all output to be written
278   * to both the specified output file and to standard output.
279   *
280   * @return  {@code true} if this tool should provide arguments for redirecting
281   *          output to a file, or {@code false} if not.
282   */
283  @Override()
284  protected boolean supportsOutputFile()
285  {
286    return true;
287  }
288
289
290
291  /**
292   * Indicates whether this tool should default to interactively prompting for
293   * the bind password if a password is required but no argument was provided
294   * to indicate how to get the password.
295   *
296   * @return  {@code true} if this tool should default to interactively
297   *          prompting for the bind password, or {@code false} if not.
298   */
299  @Override()
300  protected boolean defaultToPromptForBindPassword()
301  {
302    return true;
303  }
304
305
306
307  /**
308   * Indicates whether this tool supports the use of a properties file for
309   * specifying default values for arguments that aren't specified on the
310   * command line.
311   *
312   * @return  {@code true} if this tool supports the use of a properties file
313   *          for specifying default values for arguments that aren't specified
314   *          on the command line, or {@code false} if not.
315   */
316  @Override()
317  public boolean supportsPropertiesFile()
318  {
319    return true;
320  }
321
322
323
324  /**
325   * Indicates whether the LDAP-specific arguments should include alternate
326   * versions of all long identifiers that consist of multiple words so that
327   * they are available in both camelCase and dash-separated versions.
328   *
329   * @return  {@code true} if this tool should provide multiple versions of
330   *          long identifiers for LDAP-specific arguments, or {@code false} if
331   *          not.
332   */
333  @Override()
334  protected boolean includeAlternateLongIdentifiers()
335  {
336    return true;
337  }
338
339
340
341  /**
342   * Adds the arguments needed by this command-line tool to the provided
343   * argument parser which are not related to connecting or authenticating to
344   * the directory server.
345   *
346   * @param  parser  The argument parser to which the arguments should be added.
347   *
348   * @throws  ArgumentException  If a problem occurs while adding the arguments.
349   */
350  @Override()
351  public void addNonLDAPArguments(final ArgumentParser parser)
352         throws ArgumentException
353  {
354    String description = "The search base DN(s) to use to find entries with " +
355         "references to other entries.  At least one base DN must be " +
356         "specified.";
357    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
358         description);
359    baseDNArgument.addLongIdentifier("base-dn");
360    parser.addArgument(baseDNArgument);
361
362    description = "The attribute(s) for which to find missing references.  " +
363         "At least one attribute must be specified, and each attribute " +
364         "must be indexed for equality searches and have values which are DNs.";
365    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
366         description);
367    parser.addArgument(attributeArgument);
368
369    description = "The maximum number of entries to retrieve at a time when " +
370         "attempting to find entries with references to other entries.  This " +
371         "requires that the authenticated user have permission to use the " +
372         "simple paged results control, but it can avoid problems with the " +
373         "server sending entries too quickly for the client to handle.  By " +
374         "default, the simple paged results control will not be used.";
375    pageSizeArgument =
376         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
377              description, 1, Integer.MAX_VALUE);
378    pageSizeArgument.addLongIdentifier("simple-page-size");
379    parser.addArgument(pageSizeArgument);
380  }
381
382
383
384  /**
385   * Retrieves the connection options that should be used for connections that
386   * are created with this command line tool.  Subclasses may override this
387   * method to use a custom set of connection options.
388   *
389   * @return  The connection options that should be used for connections that
390   *          are created with this command line tool.
391   */
392  @Override()
393  public LDAPConnectionOptions getConnectionOptions()
394  {
395    final LDAPConnectionOptions options = new LDAPConnectionOptions();
396
397    options.setUseSynchronousMode(true);
398    options.setResponseTimeoutMillis(0L);
399
400    return options;
401  }
402
403
404
405  /**
406   * Performs the core set of processing for this tool.
407   *
408   * @return  A result code that indicates whether the processing completed
409   *          successfully.
410   */
411  @Override()
412  public ResultCode doToolProcessing()
413  {
414    // Establish a connection to the target directory server to use for
415    // finding references to entries.
416    final LDAPConnectionPool findReferencesPool;
417    try
418    {
419      findReferencesPool = getConnectionPool(1, 1);
420      findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
421    }
422    catch (final LDAPException le)
423    {
424      Debug.debugException(le);
425      err("Unable to establish a connection to the directory server:  ",
426           StaticUtils.getExceptionMessage(le));
427      return le.getResultCode();
428    }
429
430    try
431    {
432      // Establish a second connection to use for retrieving referenced entries.
433      try
434      {
435        getReferencedEntriesPool = getConnectionPool(1,1);
436        getReferencedEntriesPool.
437             setRetryFailedOperationsDueToInvalidConnections(true);
438      }
439      catch (final LDAPException le)
440      {
441        Debug.debugException(le);
442        err("Unable to establish a connection to the directory server:  ",
443             StaticUtils.getExceptionMessage(le));
444        return le.getResultCode();
445      }
446
447
448      // Get the set of attributes for which to find missing references.
449      final List<String> attrList = attributeArgument.getValues();
450      attributes = new String[attrList.size()];
451      attrList.toArray(attributes);
452
453
454      // Construct a search filter that will be used to find all entries with
455      // references to other entries.
456      final Filter filter;
457      if (attributes.length == 1)
458      {
459        filter = Filter.createPresenceFilter(attributes[0]);
460        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
461      }
462      else
463      {
464        final Filter[] orComps = new Filter[attributes.length];
465        for (int i=0; i < attributes.length; i++)
466        {
467          orComps[i] = Filter.createPresenceFilter(attributes[i]);
468          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
469        }
470        filter = Filter.createORFilter(orComps);
471      }
472
473
474      // Iterate across all of the search base DNs and perform searches to find
475      // missing references.
476      for (final DN baseDN : baseDNArgument.getValues())
477      {
478        ASN1OctetString cookie = null;
479        do
480        {
481          final SearchRequest searchRequest = new SearchRequest(this,
482               baseDN.toString(), SearchScope.SUB, filter, attributes);
483          if (pageSizeArgument.isPresent())
484          {
485            searchRequest.addControl(new SimplePagedResultsControl(
486                 pageSizeArgument.getValue(), cookie, false));
487          }
488
489          SearchResult searchResult;
490          try
491          {
492            searchResult = findReferencesPool.search(searchRequest);
493          }
494          catch (final LDAPSearchException lse)
495          {
496            Debug.debugException(lse);
497            try
498            {
499              searchResult = findReferencesPool.search(searchRequest);
500            }
501            catch (final LDAPSearchException lse2)
502            {
503              Debug.debugException(lse2);
504              searchResult = lse2.getSearchResult();
505            }
506          }
507
508          if (searchResult.getResultCode() != ResultCode.SUCCESS)
509          {
510            err("An error occurred while attempting to search for missing " +
511                 "references to entries below " + baseDN + ":  " +
512                 searchResult.getDiagnosticMessage());
513            return searchResult.getResultCode();
514          }
515
516          final SimplePagedResultsControl pagedResultsResponse;
517          try
518          {
519            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
520          }
521          catch (final LDAPException le)
522          {
523            Debug.debugException(le);
524            err("An error occurred while attempting to decode a simple " +
525                 "paged results response control in the response to a " +
526                 "search for entries below " + baseDN + ":  " +
527                 StaticUtils.getExceptionMessage(le));
528            return le.getResultCode();
529          }
530
531          if (pagedResultsResponse != null)
532          {
533            if (pagedResultsResponse.moreResultsToReturn())
534            {
535              cookie = pagedResultsResponse.getCookie();
536            }
537            else
538            {
539              cookie = null;
540            }
541          }
542        }
543        while (cookie != null);
544      }
545
546
547      // See if there were any missing references found.
548      boolean missingReferenceFound = false;
549      for (final Map.Entry<String,AtomicLong> e :
550           missingReferenceCounts.entrySet())
551      {
552        final long numMissing = e.getValue().get();
553        if (numMissing > 0L)
554        {
555          if (! missingReferenceFound)
556          {
557            err();
558            missingReferenceFound = true;
559          }
560
561          err("Found " + numMissing + ' ' + e.getKey() +
562               " references to entries that do not exist.");
563        }
564      }
565
566      if (missingReferenceFound)
567      {
568        return ResultCode.CONSTRAINT_VIOLATION;
569      }
570      else
571      {
572        out("No references were found to entries that do not exist.");
573        return ResultCode.SUCCESS;
574      }
575    }
576    finally
577    {
578      findReferencesPool.close();
579
580      if (getReferencedEntriesPool != null)
581      {
582        getReferencedEntriesPool.close();
583      }
584    }
585  }
586
587
588
589  /**
590   * Retrieves a map that correlates the number of missing references found by
591   * attribute type.
592   *
593   * @return  A map that correlates the number of missing references found by
594   *          attribute type.
595   */
596  public Map<String,AtomicLong> getMissingReferenceCounts()
597  {
598    return Collections.unmodifiableMap(missingReferenceCounts);
599  }
600
601
602
603  /**
604   * Retrieves a set of information that may be used to generate example usage
605   * information.  Each element in the returned map should consist of a map
606   * between an example set of arguments and a string that describes the
607   * behavior of the tool when invoked with that set of arguments.
608   *
609   * @return  A set of information that may be used to generate example usage
610   *          information.  It may be {@code null} or empty if no example usage
611   *          information is available.
612   */
613  @Override()
614  public LinkedHashMap<String[],String> getExampleUsages()
615  {
616    final LinkedHashMap<String[],String> exampleMap =
617         new LinkedHashMap<String[],String>(1);
618
619    final String[] args =
620    {
621      "--hostname", "server.example.com",
622      "--port", "389",
623      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
624      "--bindPassword", "password",
625      "--baseDN", "dc=example,dc=com",
626      "--attribute", "member",
627      "--attribute", "uniqueMember",
628      "--simplePageSize", "100"
629    };
630    exampleMap.put(args,
631         "Identify all entries below dc=example,dc=com in which either the " +
632              "member or uniqueMember attribute references an entry that " +
633              "does not exist.");
634
635    return exampleMap;
636  }
637
638
639
640  /**
641   * Indicates that the provided search result entry has been returned by the
642   * server and may be processed by this search result listener.
643   *
644   * @param  searchEntry  The search result entry that has been returned by the
645   *                      server.
646   */
647  public void searchEntryReturned(final SearchResultEntry searchEntry)
648  {
649    try
650    {
651      // Find attributes which references to entries that do not exist.
652      for (final String attr : attributes)
653      {
654        final List<Attribute> attrList =
655             searchEntry.getAttributesWithOptions(attr, null);
656        for (final Attribute a : attrList)
657        {
658          for (final String value : a.getValues())
659          {
660            try
661            {
662              final SearchResultEntry e =
663                   getReferencedEntriesPool.getEntry(value, "1.1");
664              if (e == null)
665              {
666                err("Entry '", searchEntry.getDN(), "' includes attribute ",
667                     a.getName(), " that references entry '", value,
668                     "' which does not exist.");
669                missingReferenceCounts.get(attr).incrementAndGet();
670              }
671            }
672            catch (final LDAPException le)
673            {
674              Debug.debugException(le);
675              err("An error occurred while attempting to determine whether " +
676                   "entry '" + value + "' referenced in attribute " +
677                   a.getName() + " of entry '" + searchEntry.getDN() +
678                   "' exists:  " + StaticUtils.getExceptionMessage(le));
679              missingReferenceCounts.get(attr).incrementAndGet();
680            }
681          }
682        }
683      }
684    }
685    finally
686    {
687      final long count = entriesExamined.incrementAndGet();
688      if ((count % 1000L) == 0L)
689      {
690        out(count, " entries examined");
691      }
692    }
693  }
694
695
696
697  /**
698   * Indicates that the provided search result reference has been returned by
699   * the server and may be processed by this search result listener.
700   *
701   * @param  searchReference  The search result reference that has been returned
702   *                          by the server.
703   */
704  public void searchReferenceReturned(
705                   final SearchResultReference searchReference)
706  {
707    // No implementation is required.  This tool will not follow referrals.
708  }
709}