001/* 002 * Copyright 2016-2017 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2016-2017 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.transformations; 022 023 024 025import java.io.ByteArrayInputStream; 026import java.io.File; 027import java.io.FileInputStream; 028import java.io.FileOutputStream; 029import java.io.InputStream; 030import java.io.OutputStream; 031import java.util.ArrayList; 032import java.util.Iterator; 033import java.util.LinkedHashMap; 034import java.util.List; 035import java.util.TreeMap; 036import java.util.concurrent.atomic.AtomicLong; 037import java.util.zip.GZIPInputStream; 038import java.util.zip.GZIPOutputStream; 039 040import com.unboundid.ldap.sdk.Attribute; 041import com.unboundid.ldap.sdk.DN; 042import com.unboundid.ldap.sdk.Entry; 043import com.unboundid.ldap.sdk.LDAPException; 044import com.unboundid.ldap.sdk.ResultCode; 045import com.unboundid.ldap.sdk.Version; 046import com.unboundid.ldap.sdk.schema.Schema; 047import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 048import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 049import com.unboundid.ldif.LDIFException; 050import com.unboundid.ldif.LDIFReader; 051import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 052import com.unboundid.ldif.LDIFReaderEntryTranslator; 053import com.unboundid.ldif.LDIFRecord; 054import com.unboundid.util.AggregateInputStream; 055import com.unboundid.util.ByteStringBuffer; 056import com.unboundid.util.CommandLineTool; 057import com.unboundid.util.Debug; 058import com.unboundid.util.StaticUtils; 059import com.unboundid.util.ThreadSafety; 060import com.unboundid.util.ThreadSafetyLevel; 061import com.unboundid.util.args.ArgumentException; 062import com.unboundid.util.args.ArgumentParser; 063import com.unboundid.util.args.BooleanArgument; 064import com.unboundid.util.args.DNArgument; 065import com.unboundid.util.args.FileArgument; 066import com.unboundid.util.args.FilterArgument; 067import com.unboundid.util.args.IntegerArgument; 068import com.unboundid.util.args.ScopeArgument; 069import com.unboundid.util.args.StringArgument; 070 071import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 072 073 074 075/** 076 * This class provides a command-line tool that can be used to apply a number of 077 * transformations to an LDIF file. The transformations that can be applied 078 * include: 079 * <UL> 080 * <LI> 081 * It can scramble the values of a specified set of attributes in a manner 082 * that attempts to preserve the syntax and consistently scrambles the same 083 * value to the same representation. 084 * </LI> 085 * <LI> 086 * It can strip a specified set of attributes out of entries. 087 * </LI> 088 * <LI> 089 * It can redact the values of a specified set of attributes, to indicate 090 * that the values are there but providing no information about what their 091 * values are. 092 * </LI> 093 * <LI> 094 * It can replace the values of a specified attribute with a given set of 095 * values. 096 * </LI> 097 * <LI> 098 * It can add an attribute with a given set of values to any entry that does 099 * not contain that attribute. 100 * </LI> 101 * <LI> 102 * It can replace the values of a specified attribute with a value that 103 * contains a sequentially-incrementing counter. 104 * </LI> 105 * <LI> 106 * It can strip entries matching a given base DN, scope, and filter out of 107 * the LDIF file. 108 * </LI> 109 * <LI> 110 * It can perform DN mapping, so that entries that exist below one base DN 111 * are moved below a different base DN. 112 * </LI> 113 * <LI> 114 * It can perform attribute mapping, to replace uses of one attribute name 115 * with another. 116 * </LI> 117 * </UL> 118 */ 119@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 120public final class TransformLDIF 121 extends CommandLineTool 122 implements LDIFReaderEntryTranslator 123{ 124 /** 125 * The maximum length of any message to write to standard output or standard 126 * error. 127 */ 128 private static final int MAX_OUTPUT_LINE_LENGTH = 129 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 130 131 132 133 // The arguments for use by this program. 134 private BooleanArgument addToExistingValues = null; 135 private BooleanArgument appendToTargetLDIF = null; 136 private BooleanArgument compressTarget = null; 137 private BooleanArgument excludeNonMatchingEntries = null; 138 private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null; 139 private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 140 private BooleanArgument hideRedactedValueCount = null; 141 private BooleanArgument processDNs = null; 142 private BooleanArgument sourceCompressed = null; 143 private BooleanArgument sourceContainsChangeRecords = null; 144 private BooleanArgument sourceFromStandardInput = null; 145 private BooleanArgument targetToStandardOutput = null; 146 private DNArgument addAttributeBaseDN = null; 147 private DNArgument excludeEntryBaseDN = null; 148 private DNArgument flattenBaseDN = null; 149 private DNArgument moveSubtreeFrom = null; 150 private DNArgument moveSubtreeTo = null; 151 private FileArgument schemaPath = null; 152 private FileArgument sourceLDIF = null; 153 private FileArgument targetLDIF = null; 154 private FilterArgument addAttributeFilter = null; 155 private FilterArgument excludeEntryFilter = null; 156 private FilterArgument flattenExcludeFilter = null; 157 private IntegerArgument initialSequentialValue = null; 158 private IntegerArgument numThreads = null; 159 private IntegerArgument randomSeed = null; 160 private IntegerArgument sequentialValueIncrement = null; 161 private IntegerArgument wrapColumn = null; 162 private ScopeArgument addAttributeScope = null; 163 private ScopeArgument excludeEntryScope = null; 164 private StringArgument addAttributeName = null; 165 private StringArgument addAttributeValue = null; 166 private StringArgument excludeAttribute = null; 167 private StringArgument redactAttribute = null; 168 private StringArgument renameAttributeFrom = null; 169 private StringArgument renameAttributeTo = null; 170 private StringArgument replaceValuesAttribute = null; 171 private StringArgument replacementValue = null; 172 private StringArgument scrambleAttribute = null; 173 private StringArgument scrambleJSONField = null; 174 private StringArgument sequentialAttribute = null; 175 private StringArgument textAfterSequentialValue = null; 176 private StringArgument textBeforeSequentialValue = null; 177 178 // A set of thread-local byte stream buffers that will be used to construct 179 // the LDIF representations of records. 180 private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 181 new ThreadLocal<ByteStringBuffer>(); 182 183 184 185 /** 186 * Invokes this tool with the provided set of arguments. 187 * 188 * @param args The command-line arguments provided to this program. 189 */ 190 public static void main(final String... args) 191 { 192 final ResultCode resultCode = main(System.out, System.err, args); 193 if (resultCode != ResultCode.SUCCESS) 194 { 195 System.exit(resultCode.intValue()); 196 } 197 } 198 199 200 201 /** 202 * Invokes this tool with the provided set of arguments. 203 * 204 * @param out The output stream to use for standard output. It may be 205 * {@code null} if standard output should be suppressed. 206 * @param err The output stream to use for standard error. It may be 207 * {@code null} if standard error should be suppressed. 208 * @param args The command-line arguments provided to this program. 209 * 210 * @return A result code indicating whether processing completed 211 * successfully. 212 */ 213 public static ResultCode main(final OutputStream out, final OutputStream err, 214 final String... args) 215 { 216 final TransformLDIF tool = new TransformLDIF(out, err); 217 return tool.runTool(args); 218 } 219 220 221 222 /** 223 * Creates a new instance of this tool with the provided information. 224 * 225 * @param out The output stream to use for standard output. It may be 226 * {@code null} if standard output should be suppressed. 227 * @param err The output stream to use for standard error. It may be 228 * {@code null} if standard error should be suppressed. 229 */ 230 public TransformLDIF(final OutputStream out, final OutputStream err) 231 { 232 super(out, err); 233 } 234 235 236 237 /** 238 * {@inheritDoc} 239 */ 240 @Override() 241 public String getToolName() 242 { 243 return "transform-ldif"; 244 } 245 246 247 248 /** 249 * {@inheritDoc} 250 */ 251 @Override() 252 public String getToolDescription() 253 { 254 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 255 } 256 257 258 259 /** 260 * {@inheritDoc} 261 */ 262 @Override() 263 public String getToolVersion() 264 { 265 return Version.NUMERIC_VERSION_STRING; 266 } 267 268 269 270 /** 271 * {@inheritDoc} 272 */ 273 @Override() 274 public boolean supportsInteractiveMode() 275 { 276 return true; 277 } 278 279 280 281 /** 282 * {@inheritDoc} 283 */ 284 @Override() 285 public boolean defaultsToInteractiveMode() 286 { 287 return true; 288 } 289 290 291 292 /** 293 * {@inheritDoc} 294 */ 295 @Override() 296 public boolean supportsPropertiesFile() 297 { 298 return true; 299 } 300 301 302 303 /** 304 * {@inheritDoc} 305 */ 306 @Override() 307 public void addToolArguments(final ArgumentParser parser) 308 throws ArgumentException 309 { 310 // Add arguments pertaining to the source and target LDIF files. 311 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 312 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 313 false); 314 sourceLDIF.addLongIdentifier("inputLDIF"); 315 sourceLDIF.addLongIdentifier("source-ldif"); 316 sourceLDIF.addLongIdentifier("input-ldif"); 317 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 318 parser.addArgument(sourceLDIF); 319 320 sourceFromStandardInput = new BooleanArgument(null, 321 "sourceFromStandardInput", 1, 322 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 323 sourceFromStandardInput.addLongIdentifier("source-from-standard-input"); 324 sourceFromStandardInput.setArgumentGroupName( 325 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 326 parser.addArgument(sourceFromStandardInput); 327 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 328 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 329 330 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 331 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 332 false); 333 targetLDIF.addLongIdentifier("outputLDIF"); 334 targetLDIF.addLongIdentifier("target-ldif"); 335 targetLDIF.addLongIdentifier("output-ldif"); 336 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 337 parser.addArgument(targetLDIF); 338 339 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 340 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 341 targetToStandardOutput.addLongIdentifier("target-to-standard-output"); 342 targetToStandardOutput.setArgumentGroupName( 343 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 344 parser.addArgument(targetToStandardOutput); 345 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 346 347 sourceContainsChangeRecords = new BooleanArgument(null, 348 "sourceContainsChangeRecords", 349 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 350 sourceContainsChangeRecords.addLongIdentifier( 351 "source-contains-change-records"); 352 sourceContainsChangeRecords.setArgumentGroupName( 353 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 354 parser.addArgument(sourceContainsChangeRecords); 355 356 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 357 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 358 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif"); 359 appendToTargetLDIF.setArgumentGroupName( 360 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 361 parser.addArgument(appendToTargetLDIF); 362 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 363 364 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 365 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 366 wrapColumn.addLongIdentifier("wrap-column"); 367 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 368 parser.addArgument(wrapColumn); 369 370 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 371 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 372 sourceCompressed.addLongIdentifier("inputCompressed"); 373 sourceCompressed.addLongIdentifier("source-compressed"); 374 sourceCompressed.addLongIdentifier("input-compressed"); 375 sourceCompressed.setArgumentGroupName( 376 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 377 parser.addArgument(sourceCompressed); 378 379 compressTarget = new BooleanArgument('c', "compressTarget", 380 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 381 compressTarget.addLongIdentifier("compressOutput"); 382 compressTarget.addLongIdentifier("compress"); 383 compressTarget.addLongIdentifier("compress-target"); 384 compressTarget.addLongIdentifier("compress-output"); 385 compressTarget.setArgumentGroupName( 386 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 387 parser.addArgument(compressTarget); 388 389 390 // Add arguments pertaining to attribute scrambling. 391 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 392 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 393 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 394 scrambleAttribute.addLongIdentifier("attributeName"); 395 scrambleAttribute.addLongIdentifier("scramble-attribute"); 396 scrambleAttribute.addLongIdentifier("attribute-name"); 397 scrambleAttribute.setArgumentGroupName( 398 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 399 parser.addArgument(scrambleAttribute); 400 401 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 402 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 403 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 404 scrambleAttribute.getIdentifierString())); 405 scrambleJSONField.addLongIdentifier("scramble-json-field"); 406 scrambleJSONField.setArgumentGroupName( 407 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 408 parser.addArgument(scrambleJSONField); 409 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 410 411 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 412 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 413 randomSeed.addLongIdentifier("random-seed"); 414 randomSeed.setArgumentGroupName( 415 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 416 parser.addArgument(randomSeed); 417 418 419 // Add arguments pertaining to replacing attribute values with a generated 420 // value using a sequential counter. 421 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 422 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 423 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 424 sourceContainsChangeRecords.getIdentifierString())); 425 sequentialAttribute.addLongIdentifier("sequentialAttributeName"); 426 sequentialAttribute.addLongIdentifier("sequential-attribute"); 427 sequentialAttribute.addLongIdentifier("sequential-attribute-name"); 428 sequentialAttribute.setArgumentGroupName( 429 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 430 parser.addArgument(sequentialAttribute); 431 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 432 sequentialAttribute); 433 434 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 435 false, 1, null, 436 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 437 sequentialAttribute.getIdentifierString())); 438 initialSequentialValue.addLongIdentifier("initial-sequential-value"); 439 initialSequentialValue.setArgumentGroupName( 440 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 441 parser.addArgument(initialSequentialValue); 442 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 443 444 sequentialValueIncrement = new IntegerArgument(null, 445 "sequentialValueIncrement", false, 1, null, 446 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 447 sequentialAttribute.getIdentifierString())); 448 sequentialValueIncrement.addLongIdentifier("sequential-value-increment"); 449 sequentialValueIncrement.setArgumentGroupName( 450 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 451 parser.addArgument(sequentialValueIncrement); 452 parser.addDependentArgumentSet(sequentialValueIncrement, 453 sequentialAttribute); 454 455 textBeforeSequentialValue = new StringArgument(null, 456 "textBeforeSequentialValue", false, 1, null, 457 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 458 sequentialAttribute.getIdentifierString())); 459 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value"); 460 textBeforeSequentialValue.setArgumentGroupName( 461 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 462 parser.addArgument(textBeforeSequentialValue); 463 parser.addDependentArgumentSet(textBeforeSequentialValue, 464 sequentialAttribute); 465 466 textAfterSequentialValue = new StringArgument(null, 467 "textAfterSequentialValue", false, 1, null, 468 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 469 sequentialAttribute.getIdentifierString())); 470 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value"); 471 textAfterSequentialValue.setArgumentGroupName( 472 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 473 parser.addArgument(textAfterSequentialValue); 474 parser.addDependentArgumentSet(textAfterSequentialValue, 475 sequentialAttribute); 476 477 478 // Add arguments pertaining to attribute value replacement. 479 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 480 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 481 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 482 sourceContainsChangeRecords.getIdentifierString())); 483 replaceValuesAttribute.addLongIdentifier("replace-values-attribute"); 484 replaceValuesAttribute.setArgumentGroupName( 485 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 486 parser.addArgument(replaceValuesAttribute); 487 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 488 replaceValuesAttribute); 489 490 replacementValue = new StringArgument(null, "replacementValue", false, 0, 491 null, 492 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 493 replaceValuesAttribute.getIdentifierString())); 494 replacementValue.addLongIdentifier("replacement-value"); 495 replacementValue.setArgumentGroupName( 496 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 497 parser.addArgument(replacementValue); 498 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 499 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 500 501 502 // Add arguments pertaining to adding missing attributes. 503 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 504 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 505 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 506 "--addAttributeValue", 507 sourceContainsChangeRecords.getIdentifierString())); 508 addAttributeName.addLongIdentifier("add-attribute-name"); 509 addAttributeName.setArgumentGroupName( 510 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 511 parser.addArgument(addAttributeName); 512 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 513 addAttributeName); 514 515 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 516 null, 517 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 518 addAttributeName.getIdentifierString())); 519 addAttributeValue.addLongIdentifier("add-attribute-value"); 520 addAttributeValue.setArgumentGroupName( 521 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 522 parser.addArgument(addAttributeValue); 523 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 524 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 525 526 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 527 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 528 addAttributeName.getIdentifierString(), 529 addAttributeValue.getIdentifierString())); 530 addToExistingValues.addLongIdentifier("add-to-existing-values"); 531 addToExistingValues.setArgumentGroupName( 532 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 533 parser.addArgument(addToExistingValues); 534 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 535 536 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 537 null, 538 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 539 addAttributeName.getIdentifierString())); 540 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn"); 541 addAttributeBaseDN.setArgumentGroupName( 542 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 543 parser.addArgument(addAttributeBaseDN); 544 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 545 546 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 547 null, 548 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 549 addAttributeBaseDN.getIdentifierString(), 550 addAttributeName.getIdentifierString())); 551 addAttributeScope.addLongIdentifier("add-attribute-scope"); 552 addAttributeScope.setArgumentGroupName( 553 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 554 parser.addArgument(addAttributeScope); 555 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 556 557 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 558 1, null, 559 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 560 addAttributeName.getIdentifierString())); 561 addAttributeFilter.addLongIdentifier("add-attribute-filter"); 562 addAttributeFilter.setArgumentGroupName( 563 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 564 parser.addArgument(addAttributeFilter); 565 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 566 567 568 // Add arguments pertaining to renaming attributes. 569 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 570 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 571 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 572 renameAttributeFrom.addLongIdentifier("rename-attribute-from"); 573 renameAttributeFrom.setArgumentGroupName( 574 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 575 parser.addArgument(renameAttributeFrom); 576 577 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 578 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 579 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 580 renameAttributeFrom.getIdentifierString())); 581 renameAttributeTo.addLongIdentifier("rename-attribute-to"); 582 renameAttributeTo.setArgumentGroupName( 583 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 584 parser.addArgument(renameAttributeTo); 585 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 586 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 587 588 589 // Add arguments pertaining to flattening subtrees. 590 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 591 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 592 flattenBaseDN.addLongIdentifier("flatten-base-dn"); 593 flattenBaseDN.setArgumentGroupName( 594 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 595 parser.addArgument(flattenBaseDN); 596 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 597 flattenBaseDN); 598 599 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 600 "flattenAddOmittedRDNAttributesToEntry", 1, 601 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 602 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 603 "flatten-add-omitted-rdn-attributes-to-entry"); 604 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 605 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 606 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 607 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 608 flattenBaseDN); 609 610 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 611 "flattenAddOmittedRDNAttributesToRDN", 1, 612 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 613 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 614 "flatten-add-omitted-rdn-attributes-to-rdn"); 615 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 616 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 617 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 618 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 619 flattenBaseDN); 620 621 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 622 false, 1, null, 623 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 624 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter"); 625 flattenExcludeFilter.setArgumentGroupName( 626 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 627 parser.addArgument(flattenExcludeFilter); 628 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 629 630 631 // Add arguments pertaining to moving subtrees. 632 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 633 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 634 moveSubtreeFrom.addLongIdentifier("move-subtree-from"); 635 moveSubtreeFrom.setArgumentGroupName( 636 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 637 parser.addArgument(moveSubtreeFrom); 638 639 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 640 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 641 moveSubtreeFrom.getIdentifierString())); 642 moveSubtreeTo.addLongIdentifier("move-subtree-to"); 643 moveSubtreeTo.setArgumentGroupName( 644 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 645 parser.addArgument(moveSubtreeTo); 646 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 647 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 648 649 650 // Add arguments pertaining to redacting attribute values. 651 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 652 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 653 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 654 redactAttribute.addLongIdentifier("redact-attribute"); 655 redactAttribute.setArgumentGroupName( 656 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 657 parser.addArgument(redactAttribute); 658 659 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 660 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 661 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count"); 662 hideRedactedValueCount.setArgumentGroupName( 663 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 664 parser.addArgument(hideRedactedValueCount); 665 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 666 667 668 // Add arguments pertaining to excluding attributes and entries. 669 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 670 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 671 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 672 excludeAttribute.addLongIdentifier("suppressAttribute"); 673 excludeAttribute.addLongIdentifier("exclude-attribute"); 674 excludeAttribute.addLongIdentifier("suppress-attribute"); 675 excludeAttribute.setArgumentGroupName( 676 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 677 parser.addArgument(excludeAttribute); 678 679 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 680 null, 681 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 682 sourceContainsChangeRecords.getIdentifierString())); 683 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN"); 684 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn"); 685 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn"); 686 excludeEntryBaseDN.setArgumentGroupName( 687 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 688 parser.addArgument(excludeEntryBaseDN); 689 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 690 excludeEntryBaseDN); 691 692 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 693 null, 694 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 695 sourceContainsChangeRecords.getIdentifierString())); 696 excludeEntryScope.addLongIdentifier("suppressEntryScope"); 697 excludeEntryScope.addLongIdentifier("exclude-entry-scope"); 698 excludeEntryScope.addLongIdentifier("suppress-entry-scope"); 699 excludeEntryScope.setArgumentGroupName( 700 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 701 parser.addArgument(excludeEntryScope); 702 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 703 excludeEntryScope); 704 705 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 706 1, null, 707 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 708 sourceContainsChangeRecords.getIdentifierString())); 709 excludeEntryFilter.addLongIdentifier("suppressEntryFilter"); 710 excludeEntryFilter.addLongIdentifier("exclude-entry-filter"); 711 excludeEntryFilter.addLongIdentifier("suppress-entry-filter"); 712 excludeEntryFilter.setArgumentGroupName( 713 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 714 parser.addArgument(excludeEntryFilter); 715 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 716 excludeEntryFilter); 717 718 excludeNonMatchingEntries = new BooleanArgument(null, 719 "excludeNonMatchingEntries", 720 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 721 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries"); 722 excludeNonMatchingEntries.setArgumentGroupName( 723 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 724 parser.addArgument(excludeNonMatchingEntries); 725 parser.addDependentArgumentSet(excludeNonMatchingEntries, 726 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 727 728 729 // Add the remaining arguments. 730 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 731 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 732 true, true, false, false); 733 schemaPath.addLongIdentifier("schemaFile"); 734 schemaPath.addLongIdentifier("schemaDirectory"); 735 schemaPath.addLongIdentifier("schema-path"); 736 schemaPath.addLongIdentifier("schema-file"); 737 schemaPath.addLongIdentifier("schema-directory"); 738 parser.addArgument(schemaPath); 739 740 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 741 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 742 1); 743 numThreads.addLongIdentifier("num-threads"); 744 parser.addArgument(numThreads); 745 746 processDNs = new BooleanArgument('d', "processDNs", 747 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 748 processDNs.addLongIdentifier("process-dns"); 749 parser.addArgument(processDNs); 750 751 752 // Ensure that at least one kind of transformation was requested. 753 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 754 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 755 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 756 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 757 } 758 759 760 761 /** 762 * {@inheritDoc} 763 */ 764 @Override() 765 public void doExtendedArgumentValidation() 766 throws ArgumentException 767 { 768 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 769 // arguments should always be provided. But in order to preserve backward 770 // compatibility with a legacy scramble-ldif tool, we will allow both to be 771 // omitted if either --scrambleAttribute or --sequentialArgument is 772 // provided. In that case, the path of the output file will be the path of 773 // the first input file with ".scrambled" appended to it. 774 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 775 { 776 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 777 { 778 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 779 targetLDIF.getIdentifierString(), 780 targetToStandardOutput.getIdentifierString())); 781 } 782 } 783 784 785 // Make sure that the --renameAttributeFrom and --renameAttributeTo 786 // arguments were provided an equal number of times. 787 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 788 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 789 if (renameFromOccurrences != renameToOccurrences) 790 { 791 throw new ArgumentException( 792 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 793 renameAttributeFrom.getIdentifierString(), 794 renameAttributeTo.getIdentifierString())); 795 } 796 797 798 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 799 // provided an equal number of times. 800 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 801 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 802 if (moveFromOccurrences != moveToOccurrences) 803 { 804 throw new ArgumentException( 805 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 806 moveSubtreeFrom.getIdentifierString(), 807 moveSubtreeTo.getIdentifierString())); 808 } 809 } 810 811 812 813 /** 814 * {@inheritDoc} 815 */ 816 @Override() 817 public ResultCode doToolProcessing() 818 { 819 final Schema schema; 820 try 821 { 822 schema = getSchema(); 823 } 824 catch (final LDAPException le) 825 { 826 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 827 return le.getResultCode(); 828 } 829 830 831 // Create the translators to use to apply the transformations. 832 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 833 new ArrayList<LDIFReaderEntryTranslator>(10); 834 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 835 new ArrayList<LDIFReaderChangeRecordTranslator>(10); 836 837 final AtomicLong excludedEntryCount = new AtomicLong(0L); 838 createTranslators(entryTranslators, changeRecordTranslators, 839 schema, excludedEntryCount); 840 841 final AggregateLDIFReaderEntryTranslator entryTranslator = 842 new AggregateLDIFReaderEntryTranslator(entryTranslators); 843 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 844 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 845 846 847 // Determine the path to the target file to be written. 848 final File targetFile; 849 if (targetLDIF.isPresent()) 850 { 851 targetFile = targetLDIF.getValue(); 852 } 853 else if (targetToStandardOutput.isPresent()) 854 { 855 targetFile = null; 856 } 857 else 858 { 859 targetFile = 860 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 861 } 862 863 864 // Create the LDIF reader. 865 final LDIFReader ldifReader; 866 try 867 { 868 InputStream inputStream; 869 if (sourceLDIF.isPresent()) 870 { 871 final List<File> sourceFiles = sourceLDIF.getValues(); 872 final ArrayList<InputStream> fileInputStreams = 873 new ArrayList<InputStream>(2*sourceFiles.size()); 874 for (final File f : sourceFiles) 875 { 876 if (! fileInputStreams.isEmpty()) 877 { 878 // Go ahead and ensure that there are at least new end-of-line 879 // markers between each file. Otherwise, it's possible for entries 880 // to run together. 881 final byte[] doubleEOL = new byte[StaticUtils.EOL_BYTES.length * 2]; 882 System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL, 0, 883 StaticUtils.EOL_BYTES.length); 884 System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL, 885 StaticUtils.EOL_BYTES.length, StaticUtils.EOL_BYTES.length); 886 fileInputStreams.add(new ByteArrayInputStream(doubleEOL)); 887 } 888 fileInputStreams.add(new FileInputStream(f)); 889 } 890 891 if (fileInputStreams.size() == 1) 892 { 893 inputStream = fileInputStreams.get(0); 894 } 895 else 896 { 897 inputStream = new AggregateInputStream(fileInputStreams); 898 } 899 } 900 else 901 { 902 inputStream = System.in; 903 } 904 905 if (sourceCompressed.isPresent()) 906 { 907 inputStream = new GZIPInputStream(inputStream); 908 } 909 910 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 911 entryTranslator, changeRecordTranslator); 912 if (schema != null) 913 { 914 ldifReader.setSchema(schema); 915 } 916 } 917 catch (final Exception e) 918 { 919 Debug.debugException(e); 920 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 921 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 922 StaticUtils.getExceptionMessage(e))); 923 return ResultCode.LOCAL_ERROR; 924 } 925 926 927 ResultCode resultCode = ResultCode.SUCCESS; 928 OutputStream outputStream = null; 929processingBlock: 930 try 931 { 932 // Create the output stream to use to write the transformed data. 933 try 934 { 935 if (targetFile == null) 936 { 937 outputStream = getOut(); 938 } 939 else 940 { 941 outputStream = 942 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 943 } 944 945 if (compressTarget.isPresent()) 946 { 947 outputStream = new GZIPOutputStream(outputStream); 948 } 949 } 950 catch (final Exception e) 951 { 952 Debug.debugException(e); 953 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 954 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 955 targetFile.getAbsolutePath(), 956 StaticUtils.getExceptionMessage(e))); 957 resultCode = ResultCode.LOCAL_ERROR; 958 break processingBlock; 959 } 960 961 962 // Read the source data one record at a time. The transformations will 963 // automatically be applied by the LDIF reader's translators, and even if 964 // there are multiple reader threads, we're guaranteed to get the results 965 // in the right order. 966 long entriesWritten = 0L; 967 while (true) 968 { 969 final LDIFRecord ldifRecord; 970 try 971 { 972 ldifRecord = ldifReader.readLDIFRecord(); 973 } 974 catch (final LDIFException le) 975 { 976 Debug.debugException(le); 977 if (le.mayContinueReading()) 978 { 979 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 980 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 981 StaticUtils.getExceptionMessage(le))); 982 if (resultCode == ResultCode.SUCCESS) 983 { 984 resultCode = ResultCode.PARAM_ERROR; 985 } 986 continue; 987 } 988 else 989 { 990 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 991 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 992 StaticUtils.getExceptionMessage(le))); 993 if (resultCode == ResultCode.SUCCESS) 994 { 995 resultCode = ResultCode.PARAM_ERROR; 996 } 997 break processingBlock; 998 } 999 } 1000 catch (final Exception e) 1001 { 1002 Debug.debugException(e); 1003 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1004 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1005 StaticUtils.getExceptionMessage(e))); 1006 resultCode = ResultCode.LOCAL_ERROR; 1007 break processingBlock; 1008 } 1009 1010 1011 // If the LDIF record is null, then we've run out of records so we're 1012 // done. 1013 if (ldifRecord == null) 1014 { 1015 break; 1016 } 1017 1018 1019 // Write the record to the output stream. 1020 try 1021 { 1022 if (ldifRecord instanceof PreEncodedLDIFEntry) 1023 { 1024 outputStream.write( 1025 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1026 } 1027 else 1028 { 1029 final ByteStringBuffer buffer = getBuffer(); 1030 if (wrapColumn.isPresent()) 1031 { 1032 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1033 } 1034 else 1035 { 1036 ldifRecord.toLDIF(buffer, 0); 1037 } 1038 buffer.append(StaticUtils.EOL_BYTES); 1039 buffer.write(outputStream); 1040 } 1041 } 1042 catch (final Exception e) 1043 { 1044 Debug.debugException(e); 1045 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1046 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1047 StaticUtils.getExceptionMessage(e))); 1048 resultCode = ResultCode.LOCAL_ERROR; 1049 break processingBlock; 1050 } 1051 1052 1053 // If we've written a multiple of 1000 entries, print a progress 1054 // message. 1055 entriesWritten++; 1056 if ((! targetToStandardOutput.isPresent()) && 1057 ((entriesWritten % 1000L) == 0)) 1058 { 1059 final long numExcluded = excludedEntryCount.get(); 1060 if (numExcluded > 0L) 1061 { 1062 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1063 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1064 entriesWritten, numExcluded)); 1065 } 1066 else 1067 { 1068 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1069 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1070 entriesWritten)); 1071 } 1072 } 1073 } 1074 1075 1076 if (! targetToStandardOutput.isPresent()) 1077 { 1078 final long numExcluded = excludedEntryCount.get(); 1079 if (numExcluded > 0L) 1080 { 1081 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1082 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1083 numExcluded)); 1084 } 1085 else 1086 { 1087 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1088 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1089 } 1090 } 1091 } 1092 finally 1093 { 1094 if (outputStream != null) 1095 { 1096 try 1097 { 1098 outputStream.close(); 1099 } 1100 catch (final Exception e) 1101 { 1102 Debug.debugException(e); 1103 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1104 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1105 targetFile.getAbsolutePath(), 1106 StaticUtils.getExceptionMessage(e))); 1107 if (resultCode == ResultCode.SUCCESS) 1108 { 1109 resultCode = ResultCode.LOCAL_ERROR; 1110 } 1111 } 1112 } 1113 1114 try 1115 { 1116 ldifReader.close(); 1117 } 1118 catch (final Exception e) 1119 { 1120 Debug.debugException(e); 1121 // We can ignore this. 1122 } 1123 } 1124 1125 1126 return resultCode; 1127 } 1128 1129 1130 1131 /** 1132 * Retrieves the schema that should be used for processing. 1133 * 1134 * @return The schema that was created. 1135 * 1136 * @throws LDAPException If a problem is encountered while retrieving the 1137 * schema. 1138 */ 1139 private Schema getSchema() 1140 throws LDAPException 1141 { 1142 // If any schema paths were specified, then load the schema only from those 1143 // paths. 1144 if (schemaPath.isPresent()) 1145 { 1146 final ArrayList<File> schemaFiles = new ArrayList<File>(10); 1147 for (final File path : schemaPath.getValues()) 1148 { 1149 if (path.isFile()) 1150 { 1151 schemaFiles.add(path); 1152 } 1153 else 1154 { 1155 final TreeMap<String,File> fileMap = new TreeMap<String,File>(); 1156 for (final File schemaDirFile : path.listFiles()) 1157 { 1158 final String name = schemaDirFile.getName(); 1159 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1160 { 1161 fileMap.put(name, schemaDirFile); 1162 } 1163 } 1164 schemaFiles.addAll(fileMap.values()); 1165 } 1166 } 1167 1168 if (schemaFiles.isEmpty()) 1169 { 1170 throw new LDAPException(ResultCode.PARAM_ERROR, 1171 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1172 schemaPath.getIdentifierString())); 1173 } 1174 else 1175 { 1176 try 1177 { 1178 return Schema.getSchema(schemaFiles); 1179 } 1180 catch (final Exception e) 1181 { 1182 Debug.debugException(e); 1183 throw new LDAPException(ResultCode.LOCAL_ERROR, 1184 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1185 StaticUtils.getExceptionMessage(e))); 1186 } 1187 } 1188 } 1189 else 1190 { 1191 // If the INSTANCE_ROOT environment variable is set and it refers to a 1192 // directory that has a config/schema subdirectory that has one or more 1193 // schema files in it, then read the schema from that directory. 1194 try 1195 { 1196 final String instanceRootStr = System.getenv("INSTANCE_ROOT"); 1197 if (instanceRootStr != null) 1198 { 1199 final File instanceRoot = new File(instanceRootStr); 1200 final File configDir = new File(instanceRoot, "config"); 1201 final File schemaDir = new File(configDir, "schema"); 1202 if (schemaDir.exists()) 1203 { 1204 final TreeMap<String,File> fileMap = new TreeMap<String,File>(); 1205 for (final File schemaDirFile : schemaDir.listFiles()) 1206 { 1207 final String name = schemaDirFile.getName(); 1208 if (schemaDirFile.isFile() && 1209 name.toLowerCase().endsWith(".ldif")) 1210 { 1211 fileMap.put(name, schemaDirFile); 1212 } 1213 } 1214 1215 if (! fileMap.isEmpty()) 1216 { 1217 return Schema.getSchema(new ArrayList<File>(fileMap.values())); 1218 } 1219 } 1220 } 1221 } 1222 catch (final Exception e) 1223 { 1224 Debug.debugException(e); 1225 } 1226 } 1227 1228 1229 // If we've gotten here, then just return null and the tool will try to use 1230 // the default standard schema. 1231 return null; 1232 } 1233 1234 1235 1236 /** 1237 * Creates the entry and change record translators that will be used to 1238 * perform the transformations. 1239 * 1240 * @param entryTranslators A list to which all created entry 1241 * translators should be written. 1242 * @param changeRecordTranslators A list to which all created change record 1243 * translators should be written. 1244 * @param schema The schema to use when processing. 1245 * @param excludedEntryCount A counter used to keep track of the number 1246 * of entries that have been excluded from 1247 * the result set. 1248 */ 1249 private void createTranslators( 1250 final List<LDIFReaderEntryTranslator> entryTranslators, 1251 final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators, 1252 final Schema schema, final AtomicLong excludedEntryCount) 1253 { 1254 if (scrambleAttribute.isPresent()) 1255 { 1256 final Long seed; 1257 if (randomSeed.isPresent()) 1258 { 1259 seed = randomSeed.getValue().longValue(); 1260 } 1261 else 1262 { 1263 seed = null; 1264 } 1265 1266 final ScrambleAttributeTransformation t = 1267 new ScrambleAttributeTransformation(schema, seed, 1268 processDNs.isPresent(), scrambleAttribute.getValues(), 1269 scrambleJSONField.getValues()); 1270 entryTranslators.add(t); 1271 changeRecordTranslators.add(t); 1272 } 1273 1274 if (sequentialAttribute.isPresent()) 1275 { 1276 final long initialValue; 1277 if (initialSequentialValue.isPresent()) 1278 { 1279 initialValue = initialSequentialValue.getValue().longValue(); 1280 } 1281 else 1282 { 1283 initialValue = 0L; 1284 } 1285 1286 final long incrementAmount; 1287 if (sequentialValueIncrement.isPresent()) 1288 { 1289 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1290 } 1291 else 1292 { 1293 incrementAmount = 1L; 1294 } 1295 1296 for (final String attrName : sequentialAttribute.getValues()) 1297 { 1298 1299 1300 final ReplaceWithCounterTransformation t = 1301 new ReplaceWithCounterTransformation(schema, attrName, 1302 initialValue, incrementAmount, 1303 textBeforeSequentialValue.getValue(), 1304 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1305 entryTranslators.add(t); 1306 } 1307 } 1308 1309 if (replaceValuesAttribute.isPresent()) 1310 { 1311 final ReplaceAttributeTransformation t = 1312 new ReplaceAttributeTransformation(schema, 1313 replaceValuesAttribute.getValue(), 1314 replacementValue.getValues()); 1315 entryTranslators.add(t); 1316 } 1317 1318 if (addAttributeName.isPresent()) 1319 { 1320 final AddAttributeTransformation t = new AddAttributeTransformation( 1321 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1322 addAttributeFilter.getValue(), 1323 new Attribute(addAttributeName.getValue(), schema, 1324 addAttributeValue.getValues()), 1325 (! addToExistingValues.isPresent())); 1326 entryTranslators.add(t); 1327 } 1328 1329 if (renameAttributeFrom.isPresent()) 1330 { 1331 final Iterator<String> renameFromIterator = 1332 renameAttributeFrom.getValues().iterator(); 1333 final Iterator<String> renameToIterator = 1334 renameAttributeTo.getValues().iterator(); 1335 while (renameFromIterator.hasNext()) 1336 { 1337 final RenameAttributeTransformation t = 1338 new RenameAttributeTransformation(schema, 1339 renameFromIterator.next(), renameToIterator.next(), 1340 processDNs.isPresent()); 1341 entryTranslators.add(t); 1342 changeRecordTranslators.add(t); 1343 } 1344 } 1345 1346 if (flattenBaseDN.isPresent()) 1347 { 1348 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1349 schema, flattenBaseDN.getValue(), 1350 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1351 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1352 flattenExcludeFilter.getValue()); 1353 entryTranslators.add(t); 1354 } 1355 1356 if (moveSubtreeFrom.isPresent()) 1357 { 1358 final Iterator<DN> moveFromIterator = 1359 moveSubtreeFrom.getValues().iterator(); 1360 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1361 while (moveFromIterator.hasNext()) 1362 { 1363 final MoveSubtreeTransformation t = 1364 new MoveSubtreeTransformation(moveFromIterator.next(), 1365 moveToIterator.next()); 1366 entryTranslators.add(t); 1367 changeRecordTranslators.add(t); 1368 } 1369 } 1370 1371 if (redactAttribute.isPresent()) 1372 { 1373 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1374 schema, processDNs.isPresent(), 1375 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1376 entryTranslators.add(t); 1377 changeRecordTranslators.add(t); 1378 } 1379 1380 if (excludeAttribute.isPresent()) 1381 { 1382 final ExcludeAttributeTransformation t = 1383 new ExcludeAttributeTransformation(schema, 1384 excludeAttribute.getValues()); 1385 entryTranslators.add(t); 1386 changeRecordTranslators.add(t); 1387 } 1388 1389 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1390 excludeEntryFilter.isPresent()) 1391 { 1392 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1393 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1394 excludeEntryFilter.getValue(), 1395 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1396 entryTranslators.add(t); 1397 } 1398 1399 entryTranslators.add(this); 1400 } 1401 1402 1403 1404 /** 1405 * {@inheritDoc} 1406 */ 1407 @Override() 1408 public LinkedHashMap<String[],String> getExampleUsages() 1409 { 1410 final LinkedHashMap<String[],String> examples = 1411 new LinkedHashMap<String[],String>(4); 1412 1413 examples.put( 1414 new String[] 1415 { 1416 "--sourceLDIF", "input.ldif", 1417 "--targetLDIF", "scrambled.ldif", 1418 "--scrambleAttribute", "givenName", 1419 "--scrambleAttribute", "sn", 1420 "--scrambleAttribute", "cn", 1421 "--numThreads", "10", 1422 "--schemaPath", "/ds/config/schema", 1423 "--processDNs" 1424 }, 1425 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1426 1427 examples.put( 1428 new String[] 1429 { 1430 "--sourceLDIF", "input.ldif", 1431 "--targetLDIF", "sequential.ldif", 1432 "--sequentialAttribute", "uid", 1433 "--initialSequentialValue", "1", 1434 "--sequentialValueIncrement", "1", 1435 "--textBeforeSequentialValue", "user.", 1436 "--numThreads", "10", 1437 "--schemaPath", "/ds/config/schema", 1438 "--processDNs" 1439 }, 1440 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1441 1442 examples.put( 1443 new String[] 1444 { 1445 "--sourceLDIF", "input.ldif", 1446 "--targetLDIF", "added-organization.ldif", 1447 "--addAttributeName", "o", 1448 "--addAttributeValue", "Example Corp.", 1449 "--addAttributeFilter", "(objectClass=person)", 1450 "--numThreads", "10", 1451 "--schemaPath", "/ds/config/schema" 1452 }, 1453 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1454 1455 examples.put( 1456 new String[] 1457 { 1458 "--sourceLDIF", "input.ldif", 1459 "--targetLDIF", "rebased.ldif", 1460 "--moveSubtreeFrom", "o=example.com", 1461 "--moveSubtreeTo", "dc=example,dc=com", 1462 "--numThreads", "10", 1463 "--schemaPath", "/ds/config/schema" 1464 }, 1465 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1466 1467 return examples; 1468 } 1469 1470 1471 1472 /** 1473 * {@inheritDoc} 1474 */ 1475 public Entry translate(final Entry original, final long firstLineNumber) 1476 throws LDIFException 1477 { 1478 final ByteStringBuffer buffer = getBuffer(); 1479 if (wrapColumn.isPresent()) 1480 { 1481 original.toLDIF(buffer, wrapColumn.getValue()); 1482 } 1483 else 1484 { 1485 original.toLDIF(buffer, 0); 1486 } 1487 buffer.append(StaticUtils.EOL_BYTES); 1488 1489 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1490 } 1491 1492 1493 1494 /** 1495 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1496 * 1497 * @return A byte string buffer that can be used to perform LDIF encoding. 1498 */ 1499 private ByteStringBuffer getBuffer() 1500 { 1501 ByteStringBuffer buffer = byteStringBuffers.get(); 1502 if (buffer == null) 1503 { 1504 buffer = new ByteStringBuffer(); 1505 byteStringBuffers.set(buffer); 1506 } 1507 else 1508 { 1509 buffer.clear(); 1510 } 1511 1512 return buffer; 1513 } 1514}