001/*
002 * Copyright 2008-2017 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2017 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.InputStream;
028import java.io.IOException;
029import java.io.OutputStream;
030import java.util.ArrayList;
031import java.util.Iterator;
032import java.util.TreeMap;
033import java.util.LinkedHashMap;
034import java.util.List;
035import java.util.concurrent.atomic.AtomicLong;
036import java.util.zip.GZIPInputStream;
037
038import com.unboundid.ldap.sdk.Entry;
039import com.unboundid.ldap.sdk.LDAPConnection;
040import com.unboundid.ldap.sdk.LDAPException;
041import com.unboundid.ldap.sdk.ResultCode;
042import com.unboundid.ldap.sdk.Version;
043import com.unboundid.ldap.sdk.schema.Schema;
044import com.unboundid.ldap.sdk.schema.EntryValidator;
045import com.unboundid.ldif.DuplicateValueBehavior;
046import com.unboundid.ldif.LDIFException;
047import com.unboundid.ldif.LDIFReader;
048import com.unboundid.ldif.LDIFReaderEntryTranslator;
049import com.unboundid.ldif.LDIFWriter;
050import com.unboundid.util.LDAPCommandLineTool;
051import com.unboundid.util.ThreadSafety;
052import com.unboundid.util.ThreadSafetyLevel;
053import com.unboundid.util.args.ArgumentException;
054import com.unboundid.util.args.ArgumentParser;
055import com.unboundid.util.args.BooleanArgument;
056import com.unboundid.util.args.FileArgument;
057import com.unboundid.util.args.IntegerArgument;
058import com.unboundid.util.args.StringArgument;
059
060import static com.unboundid.util.StaticUtils.*;
061
062
063
064/**
065 * This class provides a simple tool that can be used to validate that the
066 * contents of an LDIF file are valid.  This includes ensuring that the contents
067 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
068 * conforms to the server schema.  It will obtain the schema by connecting to
069 * the server and retrieving the default schema (i.e., the schema which governs
070 * the root DSE).  By default, a thorough set of validation will be performed,
071 * but it is possible to disable certain types of validation.
072 * <BR><BR>
073 * Some of the APIs demonstrated by this example include:
074 * <UL>
075 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
076 *       package)</LI>
077 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
078 *       package)</LI>
079 *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
080 *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
081 *       package)</LI>
082 * </UL>
083 * <BR><BR>
084 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
085 * class (to obtain the information to use to connect to the server to read the
086 * schema), as well as the following additional arguments:
087 * <UL>
088 *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
089 *       containing files with schema definitions.  If this argument is
090 *       provided, then no attempt will be made to communicate with a directory
091 *       server.</LI>
092 *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
093 *       file to be validated.</LI>
094 *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
095 *       compressed.</LI>
096 *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
097 *       to be written with information about all entries that failed
098 *       validation.</LI>
099 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
100 *       concurrent threads to use when processing the LDIF.  If this is not
101 *       provided, then a default of one thread will be used.</LI>
102 *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
103 *       process should ignore validation failures due to entries that contain
104 *       object classes not defined in the server schema.</LI>
105 *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
106 *       should ignore validation failures due to entries that contain
107 *       attributes not defined in the server schema.</LI>
108 *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
109 *       ignore validation failures due to entries with malformed DNs.</LI>
110 *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
111 *       should ignore validation failures due to entries that contain an RDN
112 *       attribute value that is not present in the set of entry
113 *       attributes.</LI>
114 *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
115 *       process should ignore validation failures due to entries that either do
116 *       not have a structural object class or that have multiple structural
117 *       object classes.</LI>
118 *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
119 *       process should ignore validation failures due to entries containing
120 *       auxiliary classes that are not allowed by a DIT content rule, or
121 *       abstract classes that are not subclassed by an auxiliary or structural
122 *       class contained in the entry.</LI>
123 *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
124 *       should ignore validation failures due to entries including attributes
125 *       that are not allowed or are explicitly prohibited by a DIT content
126 *       rule.</LI>
127 *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
128 *       should ignore validation failures due to entries missing required
129 *       attributes.</LI>
130 *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
131 *       process should ignore validation failures due to single-valued
132 *       attributes containing multiple values.</LI>
133 *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
134 *       should ignore validation failures due to attribute values which violate
135 *       the associated attribute syntax.</LI>
136 *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
137 *       process should ignore validation failures due to attribute values which
138 *       violate the associated attribute syntax, but only for the specified
139 *       attribute types.</LI>
140 *   <LI>"--ignoreNameForms" -- indicates that the validation process should
141 *       ignore validation failures due to name form violations (in which the
142 *       entry's RDN does not comply with the associated name form).</LI>
143 * </UL>
144 */
145@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
146public final class ValidateLDIF
147       extends LDAPCommandLineTool
148       implements LDIFReaderEntryTranslator
149{
150  /**
151   * The end-of-line character for this platform.
152   */
153  private static final String EOL = System.getProperty("line.separator", "\n");
154
155
156
157  // The arguments used by this program.
158  private BooleanArgument ignoreDuplicateValues;
159  private BooleanArgument ignoreUndefinedObjectClasses;
160  private BooleanArgument ignoreUndefinedAttributes;
161  private BooleanArgument ignoreMalformedDNs;
162  private BooleanArgument ignoreMissingRDNValues;
163  private BooleanArgument ignoreMissingSuperiorObjectClasses;
164  private BooleanArgument ignoreStructuralObjectClasses;
165  private BooleanArgument ignoreProhibitedObjectClasses;
166  private BooleanArgument ignoreProhibitedAttributes;
167  private BooleanArgument ignoreMissingAttributes;
168  private BooleanArgument ignoreSingleValuedAttributes;
169  private BooleanArgument ignoreAttributeSyntax;
170  private BooleanArgument ignoreNameForms;
171  private BooleanArgument isCompressed;
172  private FileArgument    schemaDirectory;
173  private FileArgument    ldifFile;
174  private FileArgument    rejectFile;
175  private IntegerArgument numThreads;
176  private StringArgument  ignoreSyntaxViolationsForAttribute;
177
178  // The counter used to keep track of the number of entries processed.
179  private final AtomicLong entriesProcessed = new AtomicLong(0L);
180
181  // The counter used to keep track of the number of entries that could not be
182  // parsed as valid entries.
183  private final AtomicLong malformedEntries = new AtomicLong(0L);
184
185  // The entry validator that will be used to validate the entries.
186  private EntryValidator entryValidator;
187
188  // The LDIF writer that will be used to write rejected entries.
189  private LDIFWriter rejectWriter;
190
191
192
193  /**
194   * Parse the provided command line arguments and make the appropriate set of
195   * changes.
196   *
197   * @param  args  The command line arguments provided to this program.
198   */
199  public static void main(final String[] args)
200  {
201    final ResultCode resultCode = main(args, System.out, System.err);
202    if (resultCode != ResultCode.SUCCESS)
203    {
204      System.exit(resultCode.intValue());
205    }
206  }
207
208
209
210  /**
211   * Parse the provided command line arguments and make the appropriate set of
212   * changes.
213   *
214   * @param  args       The command line arguments provided to this program.
215   * @param  outStream  The output stream to which standard out should be
216   *                    written.  It may be {@code null} if output should be
217   *                    suppressed.
218   * @param  errStream  The output stream to which standard error should be
219   *                    written.  It may be {@code null} if error messages
220   *                    should be suppressed.
221   *
222   * @return  A result code indicating whether the processing was successful.
223   */
224  public static ResultCode main(final String[] args,
225                                final OutputStream outStream,
226                                final OutputStream errStream)
227  {
228    final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
229    return validateLDIF.runTool(args);
230  }
231
232
233
234  /**
235   * Creates a new instance of this tool.
236   *
237   * @param  outStream  The output stream to which standard out should be
238   *                    written.  It may be {@code null} if output should be
239   *                    suppressed.
240   * @param  errStream  The output stream to which standard error should be
241   *                    written.  It may be {@code null} if error messages
242   *                    should be suppressed.
243   */
244  public ValidateLDIF(final OutputStream outStream,
245                      final OutputStream errStream)
246  {
247    super(outStream, errStream);
248  }
249
250
251
252  /**
253   * Retrieves the name for this tool.
254   *
255   * @return  The name for this tool.
256   */
257  @Override()
258  public String getToolName()
259  {
260    return "validate-ldif";
261  }
262
263
264
265  /**
266   * Retrieves the description for this tool.
267   *
268   * @return  The description for this tool.
269   */
270  @Override()
271  public String getToolDescription()
272  {
273    return "Validate the contents of an LDIF file " +
274           "against the server schema.";
275  }
276
277
278
279  /**
280   * Retrieves the version string for this tool.
281   *
282   * @return  The version string for this tool.
283   */
284  @Override()
285  public String getToolVersion()
286  {
287    return Version.NUMERIC_VERSION_STRING;
288  }
289
290
291
292  /**
293   * Indicates whether this tool should provide support for an interactive mode,
294   * in which the tool offers a mode in which the arguments can be provided in
295   * a text-driven menu rather than requiring them to be given on the command
296   * line.  If interactive mode is supported, it may be invoked using the
297   * "--interactive" argument.  Alternately, if interactive mode is supported
298   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
299   * interactive mode may be invoked by simply launching the tool without any
300   * arguments.
301   *
302   * @return  {@code true} if this tool supports interactive mode, or
303   *          {@code false} if not.
304   */
305  @Override()
306  public boolean supportsInteractiveMode()
307  {
308    return true;
309  }
310
311
312
313  /**
314   * Indicates whether this tool defaults to launching in interactive mode if
315   * the tool is invoked without any command-line arguments.  This will only be
316   * used if {@link #supportsInteractiveMode()} returns {@code true}.
317   *
318   * @return  {@code true} if this tool defaults to using interactive mode if
319   *          launched without any command-line arguments, or {@code false} if
320   *          not.
321   */
322  @Override()
323  public boolean defaultsToInteractiveMode()
324  {
325    return true;
326  }
327
328
329
330  /**
331   * Indicates whether this tool should provide arguments for redirecting output
332   * to a file.  If this method returns {@code true}, then the tool will offer
333   * an "--outputFile" argument that will specify the path to a file to which
334   * all standard output and standard error content will be written, and it will
335   * also offer a "--teeToStandardOut" argument that can only be used if the
336   * "--outputFile" argument is present and will cause all output to be written
337   * to both the specified output file and to standard output.
338   *
339   * @return  {@code true} if this tool should provide arguments for redirecting
340   *          output to a file, or {@code false} if not.
341   */
342  @Override()
343  protected boolean supportsOutputFile()
344  {
345    return true;
346  }
347
348
349
350  /**
351   * Indicates whether this tool should default to interactively prompting for
352   * the bind password if a password is required but no argument was provided
353   * to indicate how to get the password.
354   *
355   * @return  {@code true} if this tool should default to interactively
356   *          prompting for the bind password, or {@code false} if not.
357   */
358  @Override()
359  protected boolean defaultToPromptForBindPassword()
360  {
361    return true;
362  }
363
364
365
366  /**
367   * Indicates whether this tool supports the use of a properties file for
368   * specifying default values for arguments that aren't specified on the
369   * command line.
370   *
371   * @return  {@code true} if this tool supports the use of a properties file
372   *          for specifying default values for arguments that aren't specified
373   *          on the command line, or {@code false} if not.
374   */
375  @Override()
376  public boolean supportsPropertiesFile()
377  {
378    return true;
379  }
380
381
382
383  /**
384   * Indicates whether the LDAP-specific arguments should include alternate
385   * versions of all long identifiers that consist of multiple words so that
386   * they are available in both camelCase and dash-separated versions.
387   *
388   * @return  {@code true} if this tool should provide multiple versions of
389   *          long identifiers for LDAP-specific arguments, or {@code false} if
390   *          not.
391   */
392  @Override()
393  protected boolean includeAlternateLongIdentifiers()
394  {
395    return true;
396  }
397
398
399
400  /**
401   * Adds the arguments used by this program that aren't already provided by the
402   * generic {@code LDAPCommandLineTool} framework.
403   *
404   * @param  parser  The argument parser to which the arguments should be added.
405   *
406   * @throws  ArgumentException  If a problem occurs while adding the arguments.
407   */
408  @Override()
409  public void addNonLDAPArguments(final ArgumentParser parser)
410         throws ArgumentException
411  {
412    String description = "The path to the LDIF file to process.";
413    ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
414                                true, true, true, false);
415    ldifFile.addLongIdentifier("ldif-file");
416    parser.addArgument(ldifFile);
417
418    description = "Indicates that the specified LDIF file is compressed " +
419                  "using gzip compression.";
420    isCompressed = new BooleanArgument('c', "isCompressed", description);
421    isCompressed.addLongIdentifier("is-compressed");
422    parser.addArgument(isCompressed);
423
424    description = "The path to the file to which rejected entries should be " +
425                  "written.";
426    rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
427                                  description, false, true, true, false);
428    rejectFile.addLongIdentifier("reject-file");
429    parser.addArgument(rejectFile);
430
431    description = "The path to a directory containing one or more LDIF files " +
432                  "with the schema information to use.  If this is provided, " +
433                  "then no LDAP communication will be performed.";
434    schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
435         "{path}", description, true, true, false, true);
436    schemaDirectory.addLongIdentifier("schema-directory");
437    parser.addArgument(schemaDirectory);
438
439    description = "The number of threads to use when processing the LDIF file.";
440    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
441         description, 1, Integer.MAX_VALUE, 1);
442    numThreads.addLongIdentifier("num-threads");
443    parser.addArgument(numThreads);
444
445    description = "Ignore validation failures due to entries containing " +
446                  "duplicate values for the same attribute.";
447    ignoreDuplicateValues =
448         new BooleanArgument(null, "ignoreDuplicateValues", description);
449    ignoreDuplicateValues.setArgumentGroupName(
450         "Validation Strictness Arguments");
451    ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values");
452    parser.addArgument(ignoreDuplicateValues);
453
454    description = "Ignore validation failures due to object classes not " +
455                  "defined in the schema.";
456    ignoreUndefinedObjectClasses =
457         new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
458    ignoreUndefinedObjectClasses.setArgumentGroupName(
459         "Validation Strictness Arguments");
460    ignoreUndefinedObjectClasses.addLongIdentifier(
461         "ignore-undefined-object-classes");
462    parser.addArgument(ignoreUndefinedObjectClasses);
463
464    description = "Ignore validation failures due to attributes not defined " +
465                  "in the schema.";
466    ignoreUndefinedAttributes =
467         new BooleanArgument(null, "ignoreUndefinedAttributes", description);
468    ignoreUndefinedAttributes.setArgumentGroupName(
469         "Validation Strictness Arguments");
470    ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes");
471    parser.addArgument(ignoreUndefinedAttributes);
472
473    description = "Ignore validation failures due to entries with malformed " +
474                  "DNs.";
475    ignoreMalformedDNs =
476         new BooleanArgument(null, "ignoreMalformedDNs", description);
477    ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
478    ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns");
479    parser.addArgument(ignoreMalformedDNs);
480
481    description = "Ignore validation failures due to entries with RDN " +
482                  "attribute values that are missing from the set of entry " +
483                  "attributes.";
484    ignoreMissingRDNValues =
485         new BooleanArgument(null, "ignoreMissingRDNValues", description);
486    ignoreMissingRDNValues.setArgumentGroupName(
487         "Validation Strictness Arguments");
488    ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values");
489    parser.addArgument(ignoreMissingRDNValues);
490
491    description = "Ignore validation failures due to entries without exactly " +
492                  "structural object class.";
493    ignoreStructuralObjectClasses =
494         new BooleanArgument(null, "ignoreStructuralObjectClasses",
495                             description);
496    ignoreStructuralObjectClasses.setArgumentGroupName(
497         "Validation Strictness Arguments");
498    ignoreStructuralObjectClasses.addLongIdentifier(
499         "ignore-structural-object-classes");
500    parser.addArgument(ignoreStructuralObjectClasses);
501
502    description = "Ignore validation failures due to entries with object " +
503                  "classes that are not allowed.";
504    ignoreProhibitedObjectClasses =
505         new BooleanArgument(null, "ignoreProhibitedObjectClasses",
506                             description);
507    ignoreProhibitedObjectClasses.setArgumentGroupName(
508         "Validation Strictness Arguments");
509    ignoreProhibitedObjectClasses.addLongIdentifier(
510         "ignore-prohibited-object-classes");
511    parser.addArgument(ignoreProhibitedObjectClasses);
512
513    description = "Ignore validation failures due to entries that are " +
514                  "one or more superior object classes.";
515    ignoreMissingSuperiorObjectClasses =
516         new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
517              description);
518    ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
519         "Validation Strictness Arguments");
520    ignoreMissingSuperiorObjectClasses.addLongIdentifier(
521         "ignore-missing-superior-object-classes");
522    parser.addArgument(ignoreMissingSuperiorObjectClasses);
523
524    description = "Ignore validation failures due to entries with attributes " +
525                  "that are not allowed.";
526    ignoreProhibitedAttributes =
527         new BooleanArgument(null, "ignoreProhibitedAttributes", description);
528    ignoreProhibitedAttributes.setArgumentGroupName(
529         "Validation Strictness Arguments");
530    ignoreProhibitedAttributes.addLongIdentifier(
531         "ignore-prohibited-attributes");
532    parser.addArgument(ignoreProhibitedAttributes);
533
534    description = "Ignore validation failures due to entries missing " +
535                  "required attributes.";
536    ignoreMissingAttributes =
537         new BooleanArgument(null, "ignoreMissingAttributes", description);
538    ignoreMissingAttributes.setArgumentGroupName(
539         "Validation Strictness Arguments");
540    ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes");
541    parser.addArgument(ignoreMissingAttributes);
542
543    description = "Ignore validation failures due to entries with multiple " +
544                  "values for single-valued attributes.";
545    ignoreSingleValuedAttributes =
546         new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
547    ignoreSingleValuedAttributes.setArgumentGroupName(
548         "Validation Strictness Arguments");
549    ignoreSingleValuedAttributes.addLongIdentifier(
550         "ignore-single-valued-attributes");
551    parser.addArgument(ignoreSingleValuedAttributes);
552
553    description = "Ignore validation failures due to entries with attribute " +
554                  "values that violate their associated syntax.  If this is " +
555                  "provided, then no attribute syntax violations will be " +
556                  "flagged.  If this is not provided, then all attribute " +
557                  "syntax violations will be flagged except for violations " +
558                  "in those attributes excluded by the " +
559                  "--ignoreSyntaxViolationsForAttribute argument.";
560    ignoreAttributeSyntax =
561         new BooleanArgument(null, "ignoreAttributeSyntax", description);
562    ignoreAttributeSyntax.setArgumentGroupName(
563         "Validation Strictness Arguments");
564    ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax");
565    parser.addArgument(ignoreAttributeSyntax);
566
567    description = "The name or OID of an attribute for which to ignore " +
568                  "validation failures due to violations of the associated " +
569                  "attribute syntax.  This argument can only be used if the " +
570                  "--ignoreAttributeSyntax argument is not provided.";
571    ignoreSyntaxViolationsForAttribute = new StringArgument(null,
572         "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
573    ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
574         "Validation Strictness Arguments");
575    ignoreSyntaxViolationsForAttribute.addLongIdentifier(
576         "ignore-syntax-violations-for-attribute");
577    parser.addArgument(ignoreSyntaxViolationsForAttribute);
578
579    description = "Ignore validation failures due to entries with RDNs " +
580                  "that violate the associated name form definition.";
581    ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
582    ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
583    ignoreNameForms.addLongIdentifier("ignore-name-forms");
584    parser.addArgument(ignoreNameForms);
585
586
587    // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
588    // cannot be used together.
589    parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
590         ignoreSyntaxViolationsForAttribute);
591  }
592
593
594
595  /**
596   * Performs the actual processing for this tool.  In this case, it gets a
597   * connection to the directory server and uses it to retrieve the server
598   * schema.  It then reads the LDIF file and validates each entry accordingly.
599   *
600   * @return  The result code for the processing that was performed.
601   */
602  @Override()
603  public ResultCode doToolProcessing()
604  {
605    // Get the connection to the directory server and use it to read the schema.
606    final Schema schema;
607    if (schemaDirectory.isPresent())
608    {
609      final File schemaDir = schemaDirectory.getValue();
610
611      try
612      {
613        final TreeMap<String,File> fileMap = new TreeMap<String,File>();
614        for (final File f : schemaDir.listFiles())
615        {
616          final String name = f.getName();
617          if (f.isFile() && name.endsWith(".ldif"))
618          {
619            fileMap.put(name, f);
620          }
621        }
622
623        if (fileMap.isEmpty())
624        {
625          err("No LDIF files found in directory " +
626              schemaDir.getAbsolutePath());
627          return ResultCode.PARAM_ERROR;
628        }
629
630        final ArrayList<File> fileList = new ArrayList<File>(fileMap.values());
631        schema = Schema.getSchema(fileList);
632      }
633      catch (Exception e)
634      {
635        err("Unable to read schema from files in directory " +
636            schemaDir.getAbsolutePath() + ":  " + getExceptionMessage(e));
637        return ResultCode.LOCAL_ERROR;
638      }
639    }
640    else
641    {
642      try
643      {
644        final LDAPConnection connection = getConnection();
645        schema = connection.getSchema();
646        connection.close();
647      }
648      catch (LDAPException le)
649      {
650        err("Unable to connect to the directory server and read the schema:  ",
651            le.getMessage());
652        return le.getResultCode();
653      }
654    }
655
656
657    // Create the entry validator and initialize its configuration.
658    entryValidator = new EntryValidator(schema);
659    entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
660    entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
661    entryValidator.setCheckEntryMissingRDNValues(
662         !ignoreMissingRDNValues.isPresent());
663    entryValidator.setCheckMissingAttributes(
664         !ignoreMissingAttributes.isPresent());
665    entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
666    entryValidator.setCheckProhibitedAttributes(
667         !ignoreProhibitedAttributes.isPresent());
668    entryValidator.setCheckProhibitedObjectClasses(
669         !ignoreProhibitedObjectClasses.isPresent());
670    entryValidator.setCheckMissingSuperiorObjectClasses(
671         !ignoreMissingSuperiorObjectClasses.isPresent());
672    entryValidator.setCheckSingleValuedAttributes(
673         !ignoreSingleValuedAttributes.isPresent());
674    entryValidator.setCheckStructuralObjectClasses(
675         !ignoreStructuralObjectClasses.isPresent());
676    entryValidator.setCheckUndefinedAttributes(
677         !ignoreUndefinedAttributes.isPresent());
678    entryValidator.setCheckUndefinedObjectClasses(
679         !ignoreUndefinedObjectClasses.isPresent());
680
681    if (ignoreSyntaxViolationsForAttribute.isPresent())
682    {
683      entryValidator.setIgnoreSyntaxViolationAttributeTypes(
684           ignoreSyntaxViolationsForAttribute.getValues());
685    }
686
687
688    // Create an LDIF reader that can be used to read through the LDIF file.
689    final LDIFReader ldifReader;
690    rejectWriter = null;
691    try
692    {
693      InputStream inputStream = new FileInputStream(ldifFile.getValue());
694      if (isCompressed.isPresent())
695      {
696        inputStream = new GZIPInputStream(inputStream);
697      }
698      ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
699    }
700    catch (Exception e)
701    {
702      err("Unable to open the LDIF reader:  ", getExceptionMessage(e));
703      return ResultCode.LOCAL_ERROR;
704    }
705
706    ldifReader.setSchema(schema);
707    if (ignoreDuplicateValues.isPresent())
708    {
709      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
710    }
711    else
712    {
713      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
714    }
715
716    try
717    {
718      // Create an LDIF writer that can be used to write information about
719      // rejected entries.
720      try
721      {
722        if (rejectFile.isPresent())
723        {
724          rejectWriter = new LDIFWriter(rejectFile.getValue());
725        }
726      }
727      catch (Exception e)
728      {
729        err("Unable to create the reject writer:  ", getExceptionMessage(e));
730        return ResultCode.LOCAL_ERROR;
731      }
732
733      ResultCode resultCode = ResultCode.SUCCESS;
734      while (true)
735      {
736        try
737        {
738          final Entry e = ldifReader.readEntry();
739          if (e == null)
740          {
741            // Because we're performing parallel processing and returning null
742            // from the translate method, LDIFReader.readEntry() should never
743            // return a non-null value.  However, it can throw an LDIFException
744            // if it encounters an invalid entry, or an IOException if there's
745            // a problem reading from the file, so we should still iterate
746            // through all of the entries to catch and report on those problems.
747            break;
748          }
749        }
750        catch (LDIFException le)
751        {
752          malformedEntries.incrementAndGet();
753
754          if (resultCode == ResultCode.SUCCESS)
755          {
756            resultCode = ResultCode.DECODING_ERROR;
757          }
758
759          if (rejectWriter != null)
760          {
761            try
762            {
763              rejectWriter.writeComment(
764                   "Unable to parse an entry read from LDIF:", false, false);
765              if (le.mayContinueReading())
766              {
767                rejectWriter.writeComment(getExceptionMessage(le), false, true);
768              }
769              else
770              {
771                rejectWriter.writeComment(getExceptionMessage(le), false,
772                                          false);
773                rejectWriter.writeComment("Unable to continue LDIF processing.",
774                                          false, true);
775                err("Aborting LDIF processing:  ", getExceptionMessage(le));
776                return ResultCode.LOCAL_ERROR;
777              }
778            }
779            catch (IOException ioe)
780            {
781              err("Unable to write to the reject file:",
782                  getExceptionMessage(ioe));
783              err("LDIF parse failure that triggered the rejection:  ",
784                  getExceptionMessage(le));
785              return ResultCode.LOCAL_ERROR;
786            }
787          }
788        }
789        catch (IOException ioe)
790        {
791
792          if (rejectWriter != null)
793          {
794            try
795            {
796              rejectWriter.writeComment("I/O error reading from LDIF:", false,
797                                        false);
798              rejectWriter.writeComment(getExceptionMessage(ioe), false,
799                                        true);
800              return ResultCode.LOCAL_ERROR;
801            }
802            catch (Exception ex)
803            {
804              err("I/O error reading from LDIF:", getExceptionMessage(ioe));
805              return ResultCode.LOCAL_ERROR;
806            }
807          }
808        }
809      }
810
811      if (malformedEntries.get() > 0)
812      {
813        out(malformedEntries.get() + " entries were malformed and could not " +
814            "be read from the LDIF file.");
815      }
816
817      if (entryValidator.getInvalidEntries() > 0)
818      {
819        if (resultCode == ResultCode.SUCCESS)
820        {
821          resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
822        }
823
824        for (final String s : entryValidator.getInvalidEntrySummary(true))
825        {
826          out(s);
827        }
828      }
829      else
830      {
831        if (malformedEntries.get() == 0)
832        {
833          out("No errors were encountered.");
834        }
835      }
836
837      return resultCode;
838    }
839    finally
840    {
841      try
842      {
843        ldifReader.close();
844      }
845      catch (Exception e) {}
846
847      try
848      {
849        if (rejectWriter != null)
850        {
851          rejectWriter.close();
852        }
853      }
854      catch (Exception e) {}
855    }
856  }
857
858
859
860  /**
861   * Examines the provided entry to determine whether it conforms to the
862   * server schema.
863   *
864   * @param  entry           The entry to be examined.
865   * @param  firstLineNumber The line number of the LDIF source on which the
866   *                         provided entry begins.
867   *
868   * @return  The updated entry.  This method will always return {@code null}
869   *          because all of the real processing needed for the entry is
870   *          performed in this method and the entry isn't needed any more
871   *          after this method is done.
872   */
873  public Entry translate(final Entry entry, final long firstLineNumber)
874  {
875    final ArrayList<String> invalidReasons = new ArrayList<String>(5);
876    if (! entryValidator.entryIsValid(entry, invalidReasons))
877    {
878      if (rejectWriter != null)
879      {
880        synchronized (this)
881        {
882          try
883          {
884            rejectWriter.writeEntry(entry, listToString(invalidReasons));
885          }
886          catch (IOException ioe) {}
887        }
888      }
889    }
890
891    final long numEntries = entriesProcessed.incrementAndGet();
892    if ((numEntries % 1000L) == 0L)
893    {
894      out("Processed ", numEntries, " entries.");
895    }
896
897    return null;
898  }
899
900
901
902  /**
903   * Converts the provided list of strings into a single string.  It will
904   * contain line breaks after all but the last element.
905   *
906   * @param  l  The list of strings to convert to a single string.
907   *
908   * @return  The string from the provided list, or {@code null} if the provided
909   *          list is empty or {@code null}.
910   */
911  private static String listToString(final List<String> l)
912  {
913    if ((l == null) || (l.isEmpty()))
914    {
915      return null;
916    }
917
918    final StringBuilder buffer = new StringBuilder();
919    final Iterator<String> iterator = l.iterator();
920    while (iterator.hasNext())
921    {
922      buffer.append(iterator.next());
923      if (iterator.hasNext())
924      {
925        buffer.append(EOL);
926      }
927    }
928
929    return buffer.toString();
930  }
931
932
933
934  /**
935   * {@inheritDoc}
936   */
937  @Override()
938  public LinkedHashMap<String[],String> getExampleUsages()
939  {
940    final LinkedHashMap<String[],String> examples =
941         new LinkedHashMap<String[],String>(2);
942
943    String[] args =
944    {
945      "--hostname", "server.example.com",
946      "--port", "389",
947      "--ldifFile", "data.ldif",
948      "--rejectFile", "rejects.ldif",
949      "--numThreads", "4"
950    };
951    String description =
952         "Validate the contents of the 'data.ldif' file using the schema " +
953         "defined in the specified directory server using four concurrent " +
954         "threads.  All types of validation will be performed, and " +
955         "information about any errors will be written to the 'rejects.ldif' " +
956         "file.";
957    examples.put(args, description);
958
959
960    args = new String[]
961    {
962      "--schemaDirectory", "/ds/config/schema",
963      "--ldifFile", "data.ldif",
964      "--rejectFile", "rejects.ldif",
965      "--ignoreStructuralObjectClasses",
966      "--ignoreAttributeSyntax"
967    };
968    description =
969         "Validate the contents of the 'data.ldif' file using the schema " +
970         "defined in LDIF files contained in the /ds/config/schema directory " +
971         "using a single thread.  Any errors resulting from entries that do " +
972         "not have exactly one structural object class or from values which " +
973         "violate the syntax for their associated attribute types will be " +
974         "ignored.  Information about any other failures will be written to " +
975         "the 'rejects.ldif' file.";
976    examples.put(args, description);
977
978    return examples;
979  }
980
981
982
983  /**
984   * @return EntryValidator
985   *
986   * Returns the EntryValidator
987   */
988  public EntryValidator getEntryValidator()
989  {
990    return entryValidator;
991  }
992}