001/*
002 * Copyright 2016-2017 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2016-2017 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.transformations;
022
023
024
025import java.io.ByteArrayInputStream;
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.FileOutputStream;
029import java.io.InputStream;
030import java.io.OutputStream;
031import java.util.ArrayList;
032import java.util.Iterator;
033import java.util.LinkedHashMap;
034import java.util.List;
035import java.util.TreeMap;
036import java.util.concurrent.atomic.AtomicLong;
037import java.util.zip.GZIPInputStream;
038import java.util.zip.GZIPOutputStream;
039
040import com.unboundid.ldap.sdk.Attribute;
041import com.unboundid.ldap.sdk.DN;
042import com.unboundid.ldap.sdk.Entry;
043import com.unboundid.ldap.sdk.LDAPException;
044import com.unboundid.ldap.sdk.ResultCode;
045import com.unboundid.ldap.sdk.Version;
046import com.unboundid.ldap.sdk.schema.Schema;
047import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
048import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
049import com.unboundid.ldif.LDIFException;
050import com.unboundid.ldif.LDIFReader;
051import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
052import com.unboundid.ldif.LDIFReaderEntryTranslator;
053import com.unboundid.ldif.LDIFRecord;
054import com.unboundid.util.AggregateInputStream;
055import com.unboundid.util.ByteStringBuffer;
056import com.unboundid.util.CommandLineTool;
057import com.unboundid.util.Debug;
058import com.unboundid.util.StaticUtils;
059import com.unboundid.util.ThreadSafety;
060import com.unboundid.util.ThreadSafetyLevel;
061import com.unboundid.util.args.ArgumentException;
062import com.unboundid.util.args.ArgumentParser;
063import com.unboundid.util.args.BooleanArgument;
064import com.unboundid.util.args.DNArgument;
065import com.unboundid.util.args.FileArgument;
066import com.unboundid.util.args.FilterArgument;
067import com.unboundid.util.args.IntegerArgument;
068import com.unboundid.util.args.ScopeArgument;
069import com.unboundid.util.args.StringArgument;
070
071import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
072
073
074
075/**
076 * This class provides a command-line tool that can be used to apply a number of
077 * transformations to an LDIF file.  The transformations that can be applied
078 * include:
079 * <UL>
080 *   <LI>
081 *     It can scramble the values of a specified set of attributes in a manner
082 *     that attempts to preserve the syntax and consistently scrambles the same
083 *     value to the same representation.
084 *   </LI>
085 *   <LI>
086 *     It can strip a specified set of attributes out of entries.
087 *   </LI>
088 *   <LI>
089 *     It can redact the values of a specified set of attributes, to indicate
090 *     that the values are there but providing no information about what their
091 *     values are.
092 *   </LI>
093 *   <LI>
094 *     It can replace the values of a specified attribute with a given set of
095 *     values.
096 *   </LI>
097 *   <LI>
098 *     It can add an attribute with a given set of values to any entry that does
099 *     not contain that attribute.
100 *   </LI>
101 *   <LI>
102 *     It can replace the values of a specified attribute with a value that
103 *     contains a sequentially-incrementing counter.
104 *   </LI>
105 *   <LI>
106 *     It can strip entries matching a given base DN, scope, and filter out of
107 *     the LDIF file.
108 *   </LI>
109 *   <LI>
110 *     It can perform DN mapping, so that entries that exist below one base DN
111 *     are moved below a different base DN.
112 *   </LI>
113 *   <LI>
114 *     It can perform attribute mapping, to replace uses of one attribute name
115 *     with another.
116 *   </LI>
117 * </UL>
118 */
119@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
120public final class TransformLDIF
121       extends CommandLineTool
122       implements LDIFReaderEntryTranslator
123{
124  /**
125   * The maximum length of any message to write to standard output or standard
126   * error.
127   */
128  private static final int MAX_OUTPUT_LINE_LENGTH =
129       StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
130
131
132
133  // The arguments for use by this program.
134  private BooleanArgument addToExistingValues = null;
135  private BooleanArgument appendToTargetLDIF = null;
136  private BooleanArgument compressTarget = null;
137  private BooleanArgument excludeNonMatchingEntries = null;
138  private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
139  private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
140  private BooleanArgument hideRedactedValueCount = null;
141  private BooleanArgument processDNs = null;
142  private BooleanArgument sourceCompressed = null;
143  private BooleanArgument sourceContainsChangeRecords = null;
144  private BooleanArgument sourceFromStandardInput = null;
145  private BooleanArgument targetToStandardOutput = null;
146  private DNArgument addAttributeBaseDN = null;
147  private DNArgument excludeEntryBaseDN = null;
148  private DNArgument flattenBaseDN = null;
149  private DNArgument moveSubtreeFrom = null;
150  private DNArgument moveSubtreeTo = null;
151  private FileArgument schemaPath = null;
152  private FileArgument sourceLDIF = null;
153  private FileArgument targetLDIF = null;
154  private FilterArgument addAttributeFilter = null;
155  private FilterArgument excludeEntryFilter = null;
156  private FilterArgument flattenExcludeFilter = null;
157  private IntegerArgument initialSequentialValue = null;
158  private IntegerArgument numThreads = null;
159  private IntegerArgument randomSeed = null;
160  private IntegerArgument sequentialValueIncrement = null;
161  private IntegerArgument wrapColumn = null;
162  private ScopeArgument addAttributeScope = null;
163  private ScopeArgument excludeEntryScope = null;
164  private StringArgument addAttributeName = null;
165  private StringArgument addAttributeValue = null;
166  private StringArgument excludeAttribute = null;
167  private StringArgument redactAttribute = null;
168  private StringArgument renameAttributeFrom = null;
169  private StringArgument renameAttributeTo = null;
170  private StringArgument replaceValuesAttribute = null;
171  private StringArgument replacementValue = null;
172  private StringArgument scrambleAttribute = null;
173  private StringArgument scrambleJSONField = null;
174  private StringArgument sequentialAttribute = null;
175  private StringArgument textAfterSequentialValue = null;
176  private StringArgument textBeforeSequentialValue = null;
177
178  // A set of thread-local byte stream buffers that will be used to construct
179  // the LDIF representations of records.
180  private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
181       new ThreadLocal<ByteStringBuffer>();
182
183
184
185  /**
186   * Invokes this tool with the provided set of arguments.
187   *
188   * @param  args  The command-line arguments provided to this program.
189   */
190  public static void main(final String... args)
191  {
192    final ResultCode resultCode = main(System.out, System.err, args);
193    if (resultCode != ResultCode.SUCCESS)
194    {
195      System.exit(resultCode.intValue());
196    }
197  }
198
199
200
201  /**
202   * Invokes this tool with the provided set of arguments.
203   *
204   * @param  out   The output stream to use for standard output.  It may be
205   *               {@code null} if standard output should be suppressed.
206   * @param  err   The output stream to use for standard error.  It may be
207   *               {@code null} if standard error should be suppressed.
208   * @param  args  The command-line arguments provided to this program.
209   *
210   * @return  A result code indicating whether processing completed
211   *          successfully.
212   */
213  public static ResultCode main(final OutputStream out, final OutputStream err,
214                                final String... args)
215  {
216    final TransformLDIF tool = new TransformLDIF(out, err);
217    return tool.runTool(args);
218  }
219
220
221
222  /**
223   * Creates a new instance of this tool with the provided information.
224   *
225   * @param  out  The output stream to use for standard output.  It may be
226   *              {@code null} if standard output should be suppressed.
227   * @param  err  The output stream to use for standard error.  It may be
228   *              {@code null} if standard error should be suppressed.
229   */
230  public TransformLDIF(final OutputStream out, final OutputStream err)
231  {
232    super(out, err);
233  }
234
235
236
237  /**
238   * {@inheritDoc}
239   */
240  @Override()
241  public String getToolName()
242  {
243    return "transform-ldif";
244  }
245
246
247
248  /**
249   * {@inheritDoc}
250   */
251  @Override()
252  public String getToolDescription()
253  {
254    return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
255  }
256
257
258
259  /**
260   * {@inheritDoc}
261   */
262  @Override()
263  public String getToolVersion()
264  {
265    return Version.NUMERIC_VERSION_STRING;
266  }
267
268
269
270  /**
271   * {@inheritDoc}
272   */
273  @Override()
274  public boolean supportsInteractiveMode()
275  {
276    return true;
277  }
278
279
280
281  /**
282   * {@inheritDoc}
283   */
284  @Override()
285  public boolean defaultsToInteractiveMode()
286  {
287    return true;
288  }
289
290
291
292  /**
293   * {@inheritDoc}
294   */
295  @Override()
296  public boolean supportsPropertiesFile()
297  {
298    return true;
299  }
300
301
302
303  /**
304   * {@inheritDoc}
305   */
306  @Override()
307  public void addToolArguments(final ArgumentParser parser)
308         throws ArgumentException
309  {
310    // Add arguments pertaining to the source and target LDIF files.
311    sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
312         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
313         false);
314    sourceLDIF.addLongIdentifier("inputLDIF");
315    sourceLDIF.addLongIdentifier("source-ldif");
316    sourceLDIF.addLongIdentifier("input-ldif");
317    sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
318    parser.addArgument(sourceLDIF);
319
320    sourceFromStandardInput = new BooleanArgument(null,
321         "sourceFromStandardInput", 1,
322         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
323    sourceFromStandardInput.addLongIdentifier("source-from-standard-input");
324    sourceFromStandardInput.setArgumentGroupName(
325         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
326    parser.addArgument(sourceFromStandardInput);
327    parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
328    parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
329
330    targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
331         INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
332         false);
333    targetLDIF.addLongIdentifier("outputLDIF");
334    targetLDIF.addLongIdentifier("target-ldif");
335    targetLDIF.addLongIdentifier("output-ldif");
336    targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
337    parser.addArgument(targetLDIF);
338
339    targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
340         1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
341    targetToStandardOutput.addLongIdentifier("target-to-standard-output");
342    targetToStandardOutput.setArgumentGroupName(
343         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
344    parser.addArgument(targetToStandardOutput);
345    parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
346
347    sourceContainsChangeRecords = new BooleanArgument(null,
348         "sourceContainsChangeRecords",
349         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
350    sourceContainsChangeRecords.addLongIdentifier(
351         "source-contains-change-records");
352    sourceContainsChangeRecords.setArgumentGroupName(
353         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
354    parser.addArgument(sourceContainsChangeRecords);
355
356    appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
357         INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
358    appendToTargetLDIF.addLongIdentifier("append-to-target-ldif");
359    appendToTargetLDIF.setArgumentGroupName(
360         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
361    parser.addArgument(appendToTargetLDIF);
362    parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
363
364    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
365         INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
366    wrapColumn.addLongIdentifier("wrap-column");
367    wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
368    parser.addArgument(wrapColumn);
369
370    sourceCompressed = new BooleanArgument('C', "sourceCompressed",
371         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
372    sourceCompressed.addLongIdentifier("inputCompressed");
373    sourceCompressed.addLongIdentifier("source-compressed");
374    sourceCompressed.addLongIdentifier("input-compressed");
375    sourceCompressed.setArgumentGroupName(
376         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
377    parser.addArgument(sourceCompressed);
378
379    compressTarget = new BooleanArgument('c', "compressTarget",
380         INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
381    compressTarget.addLongIdentifier("compressOutput");
382    compressTarget.addLongIdentifier("compress");
383    compressTarget.addLongIdentifier("compress-target");
384    compressTarget.addLongIdentifier("compress-output");
385    compressTarget.setArgumentGroupName(
386         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
387    parser.addArgument(compressTarget);
388
389
390    // Add arguments pertaining to attribute scrambling.
391    scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
392         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
393         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
394    scrambleAttribute.addLongIdentifier("attributeName");
395    scrambleAttribute.addLongIdentifier("scramble-attribute");
396    scrambleAttribute.addLongIdentifier("attribute-name");
397    scrambleAttribute.setArgumentGroupName(
398         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
399    parser.addArgument(scrambleAttribute);
400
401    scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
402         INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
403         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
404              scrambleAttribute.getIdentifierString()));
405    scrambleJSONField.addLongIdentifier("scramble-json-field");
406    scrambleJSONField.setArgumentGroupName(
407         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
408    parser.addArgument(scrambleJSONField);
409    parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
410
411    randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
412         INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
413    randomSeed.addLongIdentifier("random-seed");
414    randomSeed.setArgumentGroupName(
415         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
416    parser.addArgument(randomSeed);
417
418
419    // Add arguments pertaining to replacing attribute values with a generated
420    // value using a sequential counter.
421    sequentialAttribute = new StringArgument('S', "sequentialAttribute",
422         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
423         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
424              sourceContainsChangeRecords.getIdentifierString()));
425    sequentialAttribute.addLongIdentifier("sequentialAttributeName");
426    sequentialAttribute.addLongIdentifier("sequential-attribute");
427    sequentialAttribute.addLongIdentifier("sequential-attribute-name");
428    sequentialAttribute.setArgumentGroupName(
429         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
430    parser.addArgument(sequentialAttribute);
431    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
432         sequentialAttribute);
433
434    initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
435         false, 1, null,
436         INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
437              sequentialAttribute.getIdentifierString()));
438    initialSequentialValue.addLongIdentifier("initial-sequential-value");
439    initialSequentialValue.setArgumentGroupName(
440         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
441    parser.addArgument(initialSequentialValue);
442    parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
443
444    sequentialValueIncrement = new IntegerArgument(null,
445         "sequentialValueIncrement", false, 1, null,
446         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
447              sequentialAttribute.getIdentifierString()));
448    sequentialValueIncrement.addLongIdentifier("sequential-value-increment");
449    sequentialValueIncrement.setArgumentGroupName(
450         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
451    parser.addArgument(sequentialValueIncrement);
452    parser.addDependentArgumentSet(sequentialValueIncrement,
453         sequentialAttribute);
454
455    textBeforeSequentialValue = new StringArgument(null,
456         "textBeforeSequentialValue", false, 1, null,
457         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
458              sequentialAttribute.getIdentifierString()));
459    textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value");
460    textBeforeSequentialValue.setArgumentGroupName(
461         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
462    parser.addArgument(textBeforeSequentialValue);
463    parser.addDependentArgumentSet(textBeforeSequentialValue,
464         sequentialAttribute);
465
466    textAfterSequentialValue = new StringArgument(null,
467         "textAfterSequentialValue", false, 1, null,
468         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
469              sequentialAttribute.getIdentifierString()));
470    textAfterSequentialValue.addLongIdentifier("text-after-sequential-value");
471    textAfterSequentialValue.setArgumentGroupName(
472         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
473    parser.addArgument(textAfterSequentialValue);
474    parser.addDependentArgumentSet(textAfterSequentialValue,
475         sequentialAttribute);
476
477
478    // Add arguments pertaining to attribute value replacement.
479    replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
480         false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
481         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
482              sourceContainsChangeRecords.getIdentifierString()));
483    replaceValuesAttribute.addLongIdentifier("replace-values-attribute");
484    replaceValuesAttribute.setArgumentGroupName(
485         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
486    parser.addArgument(replaceValuesAttribute);
487    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
488         replaceValuesAttribute);
489
490    replacementValue = new StringArgument(null, "replacementValue", false, 0,
491         null,
492         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
493              replaceValuesAttribute.getIdentifierString()));
494    replacementValue.addLongIdentifier("replacement-value");
495    replacementValue.setArgumentGroupName(
496         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
497    parser.addArgument(replacementValue);
498    parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
499    parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
500
501
502    // Add arguments pertaining to adding missing attributes.
503    addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
504         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
505         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
506              "--addAttributeValue",
507              sourceContainsChangeRecords.getIdentifierString()));
508    addAttributeName.addLongIdentifier("add-attribute-name");
509    addAttributeName.setArgumentGroupName(
510         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
511    parser.addArgument(addAttributeName);
512    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
513         addAttributeName);
514
515    addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
516         null,
517         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
518              addAttributeName.getIdentifierString()));
519    addAttributeValue.addLongIdentifier("add-attribute-value");
520    addAttributeValue.setArgumentGroupName(
521         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
522    parser.addArgument(addAttributeValue);
523    parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
524    parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
525
526    addToExistingValues = new BooleanArgument(null, "addToExistingValues",
527         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
528              addAttributeName.getIdentifierString(),
529              addAttributeValue.getIdentifierString()));
530    addToExistingValues.addLongIdentifier("add-to-existing-values");
531    addToExistingValues.setArgumentGroupName(
532         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
533    parser.addArgument(addToExistingValues);
534    parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
535
536    addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
537         null,
538         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
539              addAttributeName.getIdentifierString()));
540    addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn");
541    addAttributeBaseDN.setArgumentGroupName(
542         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
543    parser.addArgument(addAttributeBaseDN);
544    parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
545
546    addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
547         null,
548         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
549              addAttributeBaseDN.getIdentifierString(),
550              addAttributeName.getIdentifierString()));
551    addAttributeScope.addLongIdentifier("add-attribute-scope");
552    addAttributeScope.setArgumentGroupName(
553         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
554    parser.addArgument(addAttributeScope);
555    parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
556
557    addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
558         1, null,
559         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
560              addAttributeName.getIdentifierString()));
561    addAttributeFilter.addLongIdentifier("add-attribute-filter");
562    addAttributeFilter.setArgumentGroupName(
563         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
564    parser.addArgument(addAttributeFilter);
565    parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
566
567
568    // Add arguments pertaining to renaming attributes.
569    renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
570         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
571         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
572    renameAttributeFrom.addLongIdentifier("rename-attribute-from");
573    renameAttributeFrom.setArgumentGroupName(
574         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
575    parser.addArgument(renameAttributeFrom);
576
577    renameAttributeTo = new StringArgument(null, "renameAttributeTo",
578         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
579         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
580              renameAttributeFrom.getIdentifierString()));
581    renameAttributeTo.addLongIdentifier("rename-attribute-to");
582    renameAttributeTo.setArgumentGroupName(
583         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
584    parser.addArgument(renameAttributeTo);
585    parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
586    parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
587
588
589    // Add arguments pertaining to flattening subtrees.
590    flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
591         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
592    flattenBaseDN.addLongIdentifier("flatten-base-dn");
593    flattenBaseDN.setArgumentGroupName(
594         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
595    parser.addArgument(flattenBaseDN);
596    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
597         flattenBaseDN);
598
599    flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
600         "flattenAddOmittedRDNAttributesToEntry", 1,
601         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
602    flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
603         "flatten-add-omitted-rdn-attributes-to-entry");
604    flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
605         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
606    parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
607    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
608         flattenBaseDN);
609
610    flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
611         "flattenAddOmittedRDNAttributesToRDN", 1,
612         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
613    flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
614         "flatten-add-omitted-rdn-attributes-to-rdn");
615    flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
616         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
617    parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
618    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
619         flattenBaseDN);
620
621    flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
622         false, 1, null,
623         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
624    flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter");
625    flattenExcludeFilter.setArgumentGroupName(
626         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
627    parser.addArgument(flattenExcludeFilter);
628    parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
629
630
631    // Add arguments pertaining to moving subtrees.
632    moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
633         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
634    moveSubtreeFrom.addLongIdentifier("move-subtree-from");
635    moveSubtreeFrom.setArgumentGroupName(
636         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
637    parser.addArgument(moveSubtreeFrom);
638
639    moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
640         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
641              moveSubtreeFrom.getIdentifierString()));
642    moveSubtreeTo.addLongIdentifier("move-subtree-to");
643    moveSubtreeTo.setArgumentGroupName(
644         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
645    parser.addArgument(moveSubtreeTo);
646    parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
647    parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
648
649
650    // Add arguments pertaining to redacting attribute values.
651    redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
652         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
653         INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
654    redactAttribute.addLongIdentifier("redact-attribute");
655    redactAttribute.setArgumentGroupName(
656         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
657    parser.addArgument(redactAttribute);
658
659    hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
660         INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
661    hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count");
662    hideRedactedValueCount.setArgumentGroupName(
663         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
664    parser.addArgument(hideRedactedValueCount);
665    parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
666
667
668    // Add arguments pertaining to excluding attributes and entries.
669    excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
670         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
671         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
672    excludeAttribute.addLongIdentifier("suppressAttribute");
673    excludeAttribute.addLongIdentifier("exclude-attribute");
674    excludeAttribute.addLongIdentifier("suppress-attribute");
675    excludeAttribute.setArgumentGroupName(
676         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
677    parser.addArgument(excludeAttribute);
678
679    excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
680         null,
681         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
682              sourceContainsChangeRecords.getIdentifierString()));
683    excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN");
684    excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn");
685    excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn");
686    excludeEntryBaseDN.setArgumentGroupName(
687         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
688    parser.addArgument(excludeEntryBaseDN);
689    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
690         excludeEntryBaseDN);
691
692    excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
693         null,
694         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
695              sourceContainsChangeRecords.getIdentifierString()));
696    excludeEntryScope.addLongIdentifier("suppressEntryScope");
697    excludeEntryScope.addLongIdentifier("exclude-entry-scope");
698    excludeEntryScope.addLongIdentifier("suppress-entry-scope");
699    excludeEntryScope.setArgumentGroupName(
700         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
701    parser.addArgument(excludeEntryScope);
702    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
703         excludeEntryScope);
704
705    excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
706         1, null,
707         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
708              sourceContainsChangeRecords.getIdentifierString()));
709    excludeEntryFilter.addLongIdentifier("suppressEntryFilter");
710    excludeEntryFilter.addLongIdentifier("exclude-entry-filter");
711    excludeEntryFilter.addLongIdentifier("suppress-entry-filter");
712    excludeEntryFilter.setArgumentGroupName(
713         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
714    parser.addArgument(excludeEntryFilter);
715    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
716         excludeEntryFilter);
717
718    excludeNonMatchingEntries = new BooleanArgument(null,
719         "excludeNonMatchingEntries",
720         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
721    excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries");
722    excludeNonMatchingEntries.setArgumentGroupName(
723         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
724    parser.addArgument(excludeNonMatchingEntries);
725    parser.addDependentArgumentSet(excludeNonMatchingEntries,
726         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
727
728
729    // Add the remaining arguments.
730    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
731         INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
732         true, true, false, false);
733    schemaPath.addLongIdentifier("schemaFile");
734    schemaPath.addLongIdentifier("schemaDirectory");
735    schemaPath.addLongIdentifier("schema-path");
736    schemaPath.addLongIdentifier("schema-file");
737    schemaPath.addLongIdentifier("schema-directory");
738    parser.addArgument(schemaPath);
739
740    numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
741         INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
742         1);
743    numThreads.addLongIdentifier("num-threads");
744    parser.addArgument(numThreads);
745
746    processDNs = new BooleanArgument('d', "processDNs",
747         INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
748    processDNs.addLongIdentifier("process-dns");
749    parser.addArgument(processDNs);
750
751
752    // Ensure that at least one kind of transformation was requested.
753    parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
754         replaceValuesAttribute, addAttributeName, renameAttributeFrom,
755         flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
756         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
757  }
758
759
760
761  /**
762   * {@inheritDoc}
763   */
764  @Override()
765  public void doExtendedArgumentValidation()
766         throws ArgumentException
767  {
768    // Ideally, exactly one of the targetLDIF and targetToStandardOutput
769    // arguments should always be provided.  But in order to preserve backward
770    // compatibility with a legacy scramble-ldif tool, we will allow both to be
771    // omitted if either --scrambleAttribute or --sequentialArgument is
772    // provided.  In that case, the path of the output file will be the path of
773    // the first input file with ".scrambled" appended to it.
774    if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
775    {
776      if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
777      {
778        throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
779             targetLDIF.getIdentifierString(),
780             targetToStandardOutput.getIdentifierString()));
781      }
782    }
783
784
785    // Make sure that the --renameAttributeFrom and --renameAttributeTo
786    // arguments were provided an equal number of times.
787    final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
788    final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
789    if (renameFromOccurrences != renameToOccurrences)
790    {
791      throw new ArgumentException(
792           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
793                renameAttributeFrom.getIdentifierString(),
794                renameAttributeTo.getIdentifierString()));
795    }
796
797
798    // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
799    // provided an equal number of times.
800    final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
801    final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
802    if (moveFromOccurrences != moveToOccurrences)
803    {
804      throw new ArgumentException(
805           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
806                moveSubtreeFrom.getIdentifierString(),
807                moveSubtreeTo.getIdentifierString()));
808    }
809  }
810
811
812
813  /**
814   * {@inheritDoc}
815   */
816  @Override()
817  public ResultCode doToolProcessing()
818  {
819    final Schema schema;
820    try
821    {
822      schema = getSchema();
823    }
824    catch (final LDAPException le)
825    {
826      wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
827      return le.getResultCode();
828    }
829
830
831    // Create the translators to use to apply the transformations.
832    final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
833         new ArrayList<LDIFReaderEntryTranslator>(10);
834    final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
835         new ArrayList<LDIFReaderChangeRecordTranslator>(10);
836
837    final AtomicLong excludedEntryCount = new AtomicLong(0L);
838    createTranslators(entryTranslators, changeRecordTranslators,
839         schema, excludedEntryCount);
840
841    final AggregateLDIFReaderEntryTranslator entryTranslator =
842         new AggregateLDIFReaderEntryTranslator(entryTranslators);
843    final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
844         new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
845
846
847    // Determine the path to the target file to be written.
848    final File targetFile;
849    if (targetLDIF.isPresent())
850    {
851      targetFile = targetLDIF.getValue();
852    }
853    else if (targetToStandardOutput.isPresent())
854    {
855      targetFile = null;
856    }
857    else
858    {
859      targetFile =
860           new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
861    }
862
863
864    // Create the LDIF reader.
865    final LDIFReader ldifReader;
866    try
867    {
868      InputStream inputStream;
869      if (sourceLDIF.isPresent())
870      {
871        final List<File> sourceFiles = sourceLDIF.getValues();
872        final ArrayList<InputStream> fileInputStreams =
873             new ArrayList<InputStream>(2*sourceFiles.size());
874        for (final File f : sourceFiles)
875        {
876          if (! fileInputStreams.isEmpty())
877          {
878            // Go ahead and ensure that there are at least new end-of-line
879            // markers between each file.  Otherwise, it's possible for entries
880            // to run together.
881            final byte[] doubleEOL = new byte[StaticUtils.EOL_BYTES.length * 2];
882            System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL, 0,
883                 StaticUtils.EOL_BYTES.length);
884            System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL,
885                 StaticUtils.EOL_BYTES.length, StaticUtils.EOL_BYTES.length);
886            fileInputStreams.add(new ByteArrayInputStream(doubleEOL));
887          }
888          fileInputStreams.add(new FileInputStream(f));
889        }
890
891        if (fileInputStreams.size() == 1)
892        {
893          inputStream = fileInputStreams.get(0);
894        }
895        else
896        {
897          inputStream = new AggregateInputStream(fileInputStreams);
898        }
899      }
900      else
901      {
902        inputStream = System.in;
903      }
904
905      if (sourceCompressed.isPresent())
906      {
907        inputStream = new GZIPInputStream(inputStream);
908      }
909
910      ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
911           entryTranslator, changeRecordTranslator);
912      if (schema != null)
913      {
914        ldifReader.setSchema(schema);
915      }
916    }
917    catch (final Exception e)
918    {
919      Debug.debugException(e);
920      wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
921           ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
922                StaticUtils.getExceptionMessage(e)));
923      return ResultCode.LOCAL_ERROR;
924    }
925
926
927    ResultCode resultCode = ResultCode.SUCCESS;
928    OutputStream outputStream = null;
929processingBlock:
930    try
931    {
932      // Create the output stream to use to write the transformed data.
933      try
934      {
935        if (targetFile == null)
936        {
937          outputStream = getOut();
938        }
939        else
940        {
941          outputStream =
942               new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
943        }
944
945        if (compressTarget.isPresent())
946        {
947          outputStream = new GZIPOutputStream(outputStream);
948        }
949      }
950      catch (final Exception e)
951      {
952        Debug.debugException(e);
953        wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
954             ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
955                  targetFile.getAbsolutePath(),
956                  StaticUtils.getExceptionMessage(e)));
957        resultCode = ResultCode.LOCAL_ERROR;
958        break processingBlock;
959      }
960
961
962      // Read the source data one record at a time.  The transformations will
963      // automatically be applied by the LDIF reader's translators, and even if
964      // there are multiple reader threads, we're guaranteed to get the results
965      // in the right order.
966      long entriesWritten = 0L;
967      while (true)
968      {
969        final LDIFRecord ldifRecord;
970        try
971        {
972          ldifRecord = ldifReader.readLDIFRecord();
973        }
974        catch (final LDIFException le)
975        {
976          Debug.debugException(le);
977          if (le.mayContinueReading())
978          {
979            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
980                 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
981                      StaticUtils.getExceptionMessage(le)));
982            if (resultCode == ResultCode.SUCCESS)
983            {
984              resultCode = ResultCode.PARAM_ERROR;
985            }
986            continue;
987          }
988          else
989          {
990            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
991                 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
992                      StaticUtils.getExceptionMessage(le)));
993            if (resultCode == ResultCode.SUCCESS)
994            {
995              resultCode = ResultCode.PARAM_ERROR;
996            }
997            break processingBlock;
998          }
999        }
1000        catch (final Exception e)
1001        {
1002          Debug.debugException(e);
1003          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1004               ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1005                    StaticUtils.getExceptionMessage(e)));
1006          resultCode = ResultCode.LOCAL_ERROR;
1007          break processingBlock;
1008        }
1009
1010
1011        // If the LDIF record is null, then we've run out of records so we're
1012        // done.
1013        if (ldifRecord == null)
1014        {
1015          break;
1016        }
1017
1018
1019        // Write the record to the output stream.
1020        try
1021        {
1022          if (ldifRecord instanceof PreEncodedLDIFEntry)
1023          {
1024            outputStream.write(
1025                 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1026          }
1027          else
1028          {
1029            final ByteStringBuffer buffer = getBuffer();
1030            if (wrapColumn.isPresent())
1031            {
1032              ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1033            }
1034            else
1035            {
1036              ldifRecord.toLDIF(buffer, 0);
1037            }
1038            buffer.append(StaticUtils.EOL_BYTES);
1039            buffer.write(outputStream);
1040          }
1041        }
1042        catch (final Exception e)
1043        {
1044          Debug.debugException(e);
1045          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1046               ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1047                    StaticUtils.getExceptionMessage(e)));
1048          resultCode = ResultCode.LOCAL_ERROR;
1049          break processingBlock;
1050        }
1051
1052
1053        // If we've written a multiple of 1000 entries, print a progress
1054        // message.
1055        entriesWritten++;
1056        if ((! targetToStandardOutput.isPresent()) &&
1057            ((entriesWritten % 1000L) == 0))
1058        {
1059          final long numExcluded = excludedEntryCount.get();
1060          if (numExcluded > 0L)
1061          {
1062            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1063                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1064                      entriesWritten, numExcluded));
1065          }
1066          else
1067          {
1068            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1069                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1070                      entriesWritten));
1071          }
1072        }
1073      }
1074
1075
1076      if (! targetToStandardOutput.isPresent())
1077      {
1078        final long numExcluded = excludedEntryCount.get();
1079        if (numExcluded > 0L)
1080        {
1081          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1082               INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1083                    numExcluded));
1084        }
1085        else
1086        {
1087          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1088               INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1089        }
1090      }
1091    }
1092    finally
1093    {
1094      if (outputStream != null)
1095      {
1096        try
1097        {
1098          outputStream.close();
1099        }
1100        catch (final Exception e)
1101        {
1102          Debug.debugException(e);
1103          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1104               ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1105                    targetFile.getAbsolutePath(),
1106                    StaticUtils.getExceptionMessage(e)));
1107          if (resultCode == ResultCode.SUCCESS)
1108          {
1109            resultCode = ResultCode.LOCAL_ERROR;
1110          }
1111        }
1112      }
1113
1114      try
1115      {
1116        ldifReader.close();
1117      }
1118      catch (final Exception e)
1119      {
1120        Debug.debugException(e);
1121        // We can ignore this.
1122      }
1123    }
1124
1125
1126    return resultCode;
1127  }
1128
1129
1130
1131  /**
1132   * Retrieves the schema that should be used for processing.
1133   *
1134   * @return  The schema that was created.
1135   *
1136   * @throws  LDAPException  If a problem is encountered while retrieving the
1137   *                         schema.
1138   */
1139  private Schema getSchema()
1140          throws LDAPException
1141  {
1142    // If any schema paths were specified, then load the schema only from those
1143    // paths.
1144    if (schemaPath.isPresent())
1145    {
1146      final ArrayList<File> schemaFiles = new ArrayList<File>(10);
1147      for (final File path : schemaPath.getValues())
1148      {
1149        if (path.isFile())
1150        {
1151          schemaFiles.add(path);
1152        }
1153        else
1154        {
1155          final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1156          for (final File schemaDirFile : path.listFiles())
1157          {
1158            final String name = schemaDirFile.getName();
1159            if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1160            {
1161              fileMap.put(name, schemaDirFile);
1162            }
1163          }
1164          schemaFiles.addAll(fileMap.values());
1165        }
1166      }
1167
1168      if (schemaFiles.isEmpty())
1169      {
1170        throw new LDAPException(ResultCode.PARAM_ERROR,
1171             ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1172                  schemaPath.getIdentifierString()));
1173      }
1174      else
1175      {
1176        try
1177        {
1178          return Schema.getSchema(schemaFiles);
1179        }
1180        catch (final Exception e)
1181        {
1182          Debug.debugException(e);
1183          throw new LDAPException(ResultCode.LOCAL_ERROR,
1184               ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1185                    StaticUtils.getExceptionMessage(e)));
1186        }
1187      }
1188    }
1189    else
1190    {
1191      // If the INSTANCE_ROOT environment variable is set and it refers to a
1192      // directory that has a config/schema subdirectory that has one or more
1193      // schema files in it, then read the schema from that directory.
1194      try
1195      {
1196        final String instanceRootStr = System.getenv("INSTANCE_ROOT");
1197        if (instanceRootStr != null)
1198        {
1199          final File instanceRoot = new File(instanceRootStr);
1200          final File configDir = new File(instanceRoot, "config");
1201          final File schemaDir = new File(configDir, "schema");
1202          if (schemaDir.exists())
1203          {
1204            final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1205            for (final File schemaDirFile : schemaDir.listFiles())
1206            {
1207              final String name = schemaDirFile.getName();
1208              if (schemaDirFile.isFile() &&
1209                  name.toLowerCase().endsWith(".ldif"))
1210              {
1211                fileMap.put(name, schemaDirFile);
1212              }
1213            }
1214
1215            if (! fileMap.isEmpty())
1216            {
1217              return Schema.getSchema(new ArrayList<File>(fileMap.values()));
1218            }
1219          }
1220        }
1221      }
1222      catch (final Exception e)
1223      {
1224        Debug.debugException(e);
1225      }
1226    }
1227
1228
1229    // If we've gotten here, then just return null and the tool will try to use
1230    // the default standard schema.
1231    return null;
1232  }
1233
1234
1235
1236  /**
1237   * Creates the entry and change record translators that will be used to
1238   * perform the transformations.
1239   *
1240   * @param  entryTranslators         A list to which all created entry
1241   *                                  translators should be written.
1242   * @param  changeRecordTranslators  A list to which all created change record
1243   *                                  translators should be written.
1244   * @param  schema                   The schema to use when processing.
1245   * @param  excludedEntryCount       A counter used to keep track of the number
1246   *                                  of entries that have been excluded from
1247   *                                  the result set.
1248   */
1249  private void createTranslators(
1250       final List<LDIFReaderEntryTranslator> entryTranslators,
1251       final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1252       final Schema schema, final AtomicLong excludedEntryCount)
1253  {
1254    if (scrambleAttribute.isPresent())
1255    {
1256      final Long seed;
1257      if (randomSeed.isPresent())
1258      {
1259        seed = randomSeed.getValue().longValue();
1260      }
1261      else
1262      {
1263        seed = null;
1264      }
1265
1266      final ScrambleAttributeTransformation t =
1267           new ScrambleAttributeTransformation(schema, seed,
1268                processDNs.isPresent(), scrambleAttribute.getValues(),
1269                scrambleJSONField.getValues());
1270      entryTranslators.add(t);
1271      changeRecordTranslators.add(t);
1272    }
1273
1274    if (sequentialAttribute.isPresent())
1275    {
1276      final long initialValue;
1277      if (initialSequentialValue.isPresent())
1278      {
1279        initialValue = initialSequentialValue.getValue().longValue();
1280      }
1281      else
1282      {
1283        initialValue = 0L;
1284      }
1285
1286      final long incrementAmount;
1287      if (sequentialValueIncrement.isPresent())
1288      {
1289        incrementAmount = sequentialValueIncrement.getValue().longValue();
1290      }
1291      else
1292      {
1293        incrementAmount = 1L;
1294      }
1295
1296      for (final String attrName : sequentialAttribute.getValues())
1297      {
1298
1299
1300        final ReplaceWithCounterTransformation t =
1301             new ReplaceWithCounterTransformation(schema, attrName,
1302                  initialValue, incrementAmount,
1303                  textBeforeSequentialValue.getValue(),
1304                  textAfterSequentialValue.getValue(), processDNs.isPresent());
1305        entryTranslators.add(t);
1306      }
1307    }
1308
1309    if (replaceValuesAttribute.isPresent())
1310    {
1311      final ReplaceAttributeTransformation t =
1312           new ReplaceAttributeTransformation(schema,
1313                replaceValuesAttribute.getValue(),
1314                replacementValue.getValues());
1315      entryTranslators.add(t);
1316    }
1317
1318    if (addAttributeName.isPresent())
1319    {
1320      final AddAttributeTransformation t = new AddAttributeTransformation(
1321           schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1322           addAttributeFilter.getValue(),
1323           new Attribute(addAttributeName.getValue(), schema,
1324                addAttributeValue.getValues()),
1325           (! addToExistingValues.isPresent()));
1326      entryTranslators.add(t);
1327    }
1328
1329    if (renameAttributeFrom.isPresent())
1330    {
1331      final Iterator<String> renameFromIterator =
1332           renameAttributeFrom.getValues().iterator();
1333      final Iterator<String> renameToIterator =
1334           renameAttributeTo.getValues().iterator();
1335      while (renameFromIterator.hasNext())
1336      {
1337        final RenameAttributeTransformation t =
1338             new RenameAttributeTransformation(schema,
1339                  renameFromIterator.next(), renameToIterator.next(),
1340                  processDNs.isPresent());
1341        entryTranslators.add(t);
1342        changeRecordTranslators.add(t);
1343      }
1344    }
1345
1346    if (flattenBaseDN.isPresent())
1347    {
1348      final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1349           schema, flattenBaseDN.getValue(),
1350           flattenAddOmittedRDNAttributesToEntry.isPresent(),
1351           flattenAddOmittedRDNAttributesToRDN.isPresent(),
1352           flattenExcludeFilter.getValue());
1353      entryTranslators.add(t);
1354    }
1355
1356    if (moveSubtreeFrom.isPresent())
1357    {
1358      final Iterator<DN> moveFromIterator =
1359           moveSubtreeFrom.getValues().iterator();
1360      final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1361      while (moveFromIterator.hasNext())
1362      {
1363        final MoveSubtreeTransformation t =
1364             new MoveSubtreeTransformation(moveFromIterator.next(),
1365                  moveToIterator.next());
1366        entryTranslators.add(t);
1367        changeRecordTranslators.add(t);
1368      }
1369    }
1370
1371    if (redactAttribute.isPresent())
1372    {
1373      final RedactAttributeTransformation t = new RedactAttributeTransformation(
1374           schema, processDNs.isPresent(),
1375           (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1376      entryTranslators.add(t);
1377      changeRecordTranslators.add(t);
1378    }
1379
1380    if (excludeAttribute.isPresent())
1381    {
1382      final ExcludeAttributeTransformation t =
1383           new ExcludeAttributeTransformation(schema,
1384                excludeAttribute.getValues());
1385      entryTranslators.add(t);
1386      changeRecordTranslators.add(t);
1387    }
1388
1389    if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1390        excludeEntryFilter.isPresent())
1391    {
1392      final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1393           schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1394           excludeEntryFilter.getValue(),
1395           (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1396      entryTranslators.add(t);
1397    }
1398
1399    entryTranslators.add(this);
1400  }
1401
1402
1403
1404  /**
1405   * {@inheritDoc}
1406   */
1407  @Override()
1408  public LinkedHashMap<String[],String> getExampleUsages()
1409  {
1410    final LinkedHashMap<String[],String> examples =
1411         new LinkedHashMap<String[],String>(4);
1412
1413    examples.put(
1414         new String[]
1415         {
1416           "--sourceLDIF", "input.ldif",
1417           "--targetLDIF", "scrambled.ldif",
1418           "--scrambleAttribute", "givenName",
1419           "--scrambleAttribute", "sn",
1420           "--scrambleAttribute", "cn",
1421           "--numThreads", "10",
1422           "--schemaPath", "/ds/config/schema",
1423           "--processDNs"
1424         },
1425         INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1426
1427    examples.put(
1428         new String[]
1429         {
1430           "--sourceLDIF", "input.ldif",
1431           "--targetLDIF", "sequential.ldif",
1432           "--sequentialAttribute", "uid",
1433           "--initialSequentialValue", "1",
1434           "--sequentialValueIncrement", "1",
1435           "--textBeforeSequentialValue", "user.",
1436           "--numThreads", "10",
1437           "--schemaPath", "/ds/config/schema",
1438           "--processDNs"
1439         },
1440         INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1441
1442    examples.put(
1443         new String[]
1444         {
1445           "--sourceLDIF", "input.ldif",
1446           "--targetLDIF", "added-organization.ldif",
1447           "--addAttributeName", "o",
1448           "--addAttributeValue", "Example Corp.",
1449           "--addAttributeFilter", "(objectClass=person)",
1450           "--numThreads", "10",
1451           "--schemaPath", "/ds/config/schema"
1452         },
1453         INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1454
1455    examples.put(
1456         new String[]
1457         {
1458           "--sourceLDIF", "input.ldif",
1459           "--targetLDIF", "rebased.ldif",
1460           "--moveSubtreeFrom", "o=example.com",
1461           "--moveSubtreeTo", "dc=example,dc=com",
1462           "--numThreads", "10",
1463           "--schemaPath", "/ds/config/schema"
1464         },
1465         INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1466
1467    return examples;
1468  }
1469
1470
1471
1472  /**
1473   * {@inheritDoc}
1474   */
1475  public Entry translate(final Entry original, final long firstLineNumber)
1476         throws LDIFException
1477  {
1478    final ByteStringBuffer buffer = getBuffer();
1479    if (wrapColumn.isPresent())
1480    {
1481      original.toLDIF(buffer, wrapColumn.getValue());
1482    }
1483    else
1484    {
1485      original.toLDIF(buffer, 0);
1486    }
1487    buffer.append(StaticUtils.EOL_BYTES);
1488
1489    return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1490  }
1491
1492
1493
1494  /**
1495   * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1496   *
1497   * @return  A byte string buffer that can be used to perform LDIF encoding.
1498   */
1499  private ByteStringBuffer getBuffer()
1500  {
1501    ByteStringBuffer buffer = byteStringBuffers.get();
1502    if (buffer == null)
1503    {
1504      buffer = new ByteStringBuffer();
1505      byteStringBuffers.set(buffer);
1506    }
1507    else
1508    {
1509      buffer.clear();
1510    }
1511
1512    return buffer;
1513  }
1514}