001/*
002 * Copyright 2007-2014 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2014 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldif;
022
023
024
025import java.io.BufferedReader;
026import java.io.BufferedWriter;
027import java.io.File;
028import java.io.FileInputStream;
029import java.io.FileWriter;
030import java.io.InputStream;
031import java.io.InputStreamReader;
032import java.io.IOException;
033import java.text.ParseException;
034import java.util.ArrayList;
035import java.util.Collection;
036import java.util.Iterator;
037import java.util.LinkedHashMap;
038import java.util.List;
039import java.util.concurrent.BlockingQueue;
040import java.util.concurrent.ArrayBlockingQueue;
041import java.util.concurrent.TimeUnit;
042import java.util.concurrent.atomic.AtomicBoolean;
043import java.nio.charset.Charset;
044
045import com.unboundid.asn1.ASN1OctetString;
046import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
047import com.unboundid.ldap.matchingrules.MatchingRule;
048import com.unboundid.ldap.sdk.Attribute;
049import com.unboundid.ldap.sdk.Control;
050import com.unboundid.ldap.sdk.Entry;
051import com.unboundid.ldap.sdk.Modification;
052import com.unboundid.ldap.sdk.ModificationType;
053import com.unboundid.ldap.sdk.LDAPException;
054import com.unboundid.ldap.sdk.schema.Schema;
055import com.unboundid.util.AggregateInputStream;
056import com.unboundid.util.Base64;
057import com.unboundid.util.LDAPSDKThreadFactory;
058import com.unboundid.util.ThreadSafety;
059import com.unboundid.util.ThreadSafetyLevel;
060import com.unboundid.util.parallel.AsynchronousParallelProcessor;
061import com.unboundid.util.parallel.Result;
062import com.unboundid.util.parallel.ParallelProcessor;
063import com.unboundid.util.parallel.Processor;
064
065import static com.unboundid.ldif.LDIFMessages.*;
066import static com.unboundid.util.Debug.*;
067import static com.unboundid.util.StaticUtils.*;
068import static com.unboundid.util.Validator.*;
069
070/**
071 * This class provides an LDIF reader, which can be used to read and decode
072 * entries and change records from a data source using the LDAP Data Interchange
073 * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
074 * <BR>
075 * This class is not synchronized.  If multiple threads read from the
076 * LDIFReader, they must be synchronized externally.
077 * <BR><BR>
078 * <H2>Example</H2>
079 * The following example iterates through all entries contained in an LDIF file
080 * and attempts to add them to a directory server:
081 * <PRE>
082 * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
083 *
084 * int entriesRead = 0;
085 * int entriesAdded = 0;
086 * int errorsEncountered = 0;
087 * while (true)
088 * {
089 *   Entry entry;
090 *   try
091 *   {
092 *     entry = ldifReader.readEntry();
093 *     if (entry == null)
094 *     {
095 *       // All entries have been read.
096 *       break;
097 *     }
098 *
099 *     entriesRead++;
100 *   }
101 *   catch (LDIFException le)
102 *   {
103 *     errorsEncountered++;
104 *     if (le.mayContinueReading())
105 *     {
106 *       // A recoverable error occurred while attempting to read a change
107 *       // record, at or near line number le.getLineNumber()
108 *       // The entry will be skipped, but we'll try to keep reading from the
109 *       // LDIF file.
110 *       continue;
111 *     }
112 *     else
113 *     {
114 *       // An unrecoverable error occurred while attempting to read an entry
115 *       // at or near line number le.getLineNumber()
116 *       // No further LDIF processing will be performed.
117 *       break;
118 *     }
119 *   }
120 *   catch (IOException ioe)
121 *   {
122 *     // An I/O error occurred while attempting to read from the LDIF file.
123 *     // No further LDIF processing will be performed.
124 *     errorsEncountered++;
125 *     break;
126 *   }
127 *
128 *   LDAPResult addResult;
129 *   try
130 *   {
131 *     addResult = connection.add(entry);
132 *     // If we got here, then the change should have been processed
133 *     // successfully.
134 *     entriesAdded++;
135 *   }
136 *   catch (LDAPException le)
137 *   {
138 *     // If we got here, then the change attempt failed.
139 *     addResult = le.toLDAPResult();
140 *     errorsEncountered++;
141 *   }
142 * }
143 *
144 * ldifReader.close();
145 * </PRE>
146 */
147@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
148public final class LDIFReader
149{
150  /**
151   * The default buffer size (128KB) that will be used when reading from the
152   * data source.
153   */
154  public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
155
156
157
158  /*
159   * When processing asynchronously, this determines how many of the allocated
160   * worker threads are used to parse each batch of read entries.
161   */
162  private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
163
164
165
166  /**
167   * When processing asynchronously, this specifies the size of the pending and
168   * completed queues.
169   */
170  private static final int ASYNC_QUEUE_SIZE = 500;
171
172
173
174  /**
175   * Special entry used internally to signal that the LDIFReaderEntryTranslator
176   * has signalled that a read Entry should be skipped by returning null,
177   * which normally implies EOF.
178   */
179  private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
180
181
182
183  /**
184   * The default base path that will be prepended to relative paths.  It will
185   * end with a trailing slash.
186   */
187  private static final String DEFAULT_RELATIVE_BASE_PATH;
188  static
189  {
190    final File currentDir;
191    String currentDirString = System.getProperty("user.dir");
192    if (currentDirString == null)
193    {
194      currentDir = new File(".");
195    }
196    else
197    {
198      currentDir = new File(currentDirString);
199    }
200
201    final String currentDirAbsolutePath = currentDir.getAbsolutePath();
202    if (currentDirAbsolutePath.endsWith(File.separator))
203    {
204      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
205    }
206    else
207    {
208      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
209    }
210  }
211
212
213
214  // The buffered reader that will be used to read LDIF data.
215  private final BufferedReader reader;
216
217  // The behavior that should be exhibited when encountering duplicate attribute
218  // values.
219  private volatile DuplicateValueBehavior duplicateValueBehavior;
220
221  // A line number counter.
222  private long lineNumberCounter = 0;
223
224  private final LDIFReaderEntryTranslator entryTranslator;
225
226  // The schema that will be used when processing, if applicable.
227  private Schema schema;
228
229  // Specifies the base path that will be prepended to relative paths for file
230  // URLs.
231  private volatile String relativeBasePath;
232
233  // The behavior that should be exhibited with regard to illegal trailing
234  // spaces in attribute values.
235  private volatile TrailingSpaceBehavior trailingSpaceBehavior;
236
237  // True iff we are processing asynchronously.
238  private final boolean isAsync;
239
240  //
241  // The following only apply to asynchronous processing.
242  //
243
244  // Parses entries asynchronously.
245  private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord>
246       asyncParser;
247
248  // Set to true when the end of the input is reached.
249  private final AtomicBoolean asyncParsingComplete;
250
251  // The records that have been read and parsed.
252  private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>>
253       asyncParsedRecords;
254
255
256
257  /**
258   * Creates a new LDIF reader that will read data from the specified file.
259   *
260   * @param  path  The path to the file from which the data is to be read.  It
261   *               must not be {@code null}.
262   *
263   * @throws  IOException  If a problem occurs while opening the file for
264   *                       reading.
265   */
266  public LDIFReader(final String path)
267         throws IOException
268  {
269    this(new FileInputStream(path));
270  }
271
272
273
274  /**
275   * Creates a new LDIF reader that will read data from the specified file
276   * and parses the LDIF records asynchronously using the specified number of
277   * threads.
278   *
279   * @param  path  The path to the file from which the data is to be read.  It
280   *               must not be {@code null}.
281   * @param  numParseThreads  If this value is greater than zero, then the
282   *                          specified number of threads will be used to
283   *                          asynchronously read and parse the LDIF file.
284   *
285   * @throws  IOException  If a problem occurs while opening the file for
286   *                       reading.
287   *
288   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
289   *      constructor for more details about asynchronous processing.
290   */
291  public LDIFReader(final String path, final int numParseThreads)
292         throws IOException
293  {
294    this(new FileInputStream(path), numParseThreads);
295  }
296
297
298
299  /**
300   * Creates a new LDIF reader that will read data from the specified file.
301   *
302   * @param  file  The file from which the data is to be read.  It must not be
303   *               {@code null}.
304   *
305   * @throws  IOException  If a problem occurs while opening the file for
306   *                       reading.
307   */
308  public LDIFReader(final File file)
309         throws IOException
310  {
311    this(new FileInputStream(file));
312  }
313
314
315
316  /**
317   * Creates a new LDIF reader that will read data from the specified file
318   * and optionally parses the LDIF records asynchronously using the specified
319   * number of threads.
320   *
321   * @param  file             The file from which the data is to be read.  It
322   *                          must not be {@code null}.
323   * @param  numParseThreads  If this value is greater than zero, then the
324   *                          specified number of threads will be used to
325   *                          asynchronously read and parse the LDIF file.
326   *
327   * @throws  IOException  If a problem occurs while opening the file for
328   *                       reading.
329   */
330  public LDIFReader(final File file, final int numParseThreads)
331         throws IOException
332  {
333    this(new FileInputStream(file), numParseThreads);
334  }
335
336
337
338  /**
339   * Creates a new LDIF reader that will read data from the specified files in
340   * the order in which they are provided and optionally parses the LDIF records
341   * asynchronously using the specified number of threads.
342   *
343   * @param  files            The files from which the data is to be read.  It
344   *                          must not be {@code null} or empty.
345   * @param  numParseThreads  If this value is greater than zero, then the
346   *                          specified number of threads will be used to
347   *                          asynchronously read and parse the LDIF file.
348   * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
349   *                          before they are returned.  This is normally
350   *                          {@code null}, which causes entries to be returned
351   *                          unaltered. This is particularly useful when
352   *                          parsing the input file in parallel because the
353   *                          entry translation is also done in parallel.
354   *
355   * @throws  IOException  If a problem occurs while opening the file for
356   *                       reading.
357   */
358  public LDIFReader(final File[] files, final int numParseThreads,
359                    final LDIFReaderEntryTranslator entryTranslator)
360         throws IOException
361  {
362    this(createAggregateInputStream(files), numParseThreads, entryTranslator);
363  }
364
365
366
367  /**
368   * Creates a new aggregate input stream that will read data from the specified
369   * files.  If there are multiple files, then a "padding" file will be inserted
370   * between them to ensure that there is at least one blank line between the
371   * end of one file and the beginning of another.
372   *
373   * @param  files  The files from which the data is to be read.  It must not be
374   *                {@code null} or empty.
375   *
376   * @return  The input stream to use to read data from the provided files.
377   *
378   * @throws  IOException  If a problem is encountered while attempting to
379   *                       create the input stream.
380   */
381  private static InputStream createAggregateInputStream(final File... files)
382          throws IOException
383  {
384    if (files.length == 0)
385    {
386      throw new IOException(ERR_READ_NO_LDIF_FILES.get());
387    }
388    else if (files.length == 1)
389    {
390      return new FileInputStream(files[0]);
391    }
392    else
393    {
394      final File spacerFile =
395           File.createTempFile("ldif-reader-spacer", ".ldif");
396      spacerFile.deleteOnExit();
397
398      final BufferedWriter spacerWriter =
399           new BufferedWriter(new FileWriter(spacerFile));
400      try
401      {
402        spacerWriter.newLine();
403        spacerWriter.newLine();
404      }
405      finally
406      {
407        spacerWriter.close();
408      }
409
410      final File[] returnArray = new File[(files.length * 2) - 1];
411      returnArray[0] = files[0];
412
413      int pos = 1;
414      for (int i=1; i < files.length; i++)
415      {
416        returnArray[pos++] = spacerFile;
417        returnArray[pos++] = files[i];
418      }
419
420      return new AggregateInputStream(returnArray);
421    }
422  }
423
424
425
426  /**
427   * Creates a new LDIF reader that will read data from the provided input
428   * stream.
429   *
430   * @param  inputStream  The input stream from which the data is to be read.
431   *                      It must not be {@code null}.
432   */
433  public LDIFReader(final InputStream inputStream)
434  {
435    this(inputStream, 0);
436  }
437
438
439
440  /**
441   * Creates a new LDIF reader that will read data from the specified stream
442   * and parses the LDIF records asynchronously using the specified number of
443   * threads.
444   *
445   * @param  inputStream  The input stream from which the data is to be read.
446   *                      It must not be {@code null}.
447   * @param  numParseThreads  If this value is greater than zero, then the
448   *                          specified number of threads will be used to
449   *                          asynchronously read and parse the LDIF file.
450   *
451   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
452   *      constructor for more details about asynchronous processing.
453   */
454  public LDIFReader(final InputStream inputStream, final int numParseThreads)
455  {
456    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
457    this(new BufferedReader(new InputStreamReader(inputStream,
458                                                  Charset.forName("UTF-8")),
459                            DEFAULT_BUFFER_SIZE),
460         numParseThreads);
461  }
462
463
464
465  /**
466   * Creates a new LDIF reader that will read data from the specified stream
467   * and parses the LDIF records asynchronously using the specified number of
468   * threads.
469   *
470   * @param  inputStream  The input stream from which the data is to be read.
471   *                      It must not be {@code null}.
472   * @param  numParseThreads  If this value is greater than zero, then the
473   *                          specified number of threads will be used to
474   *                          asynchronously read and parse the LDIF file.
475   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
476   *                         entries before they are returned.  This is normally
477   *                         {@code null}, which causes entries to be returned
478   *                         unaltered. This is particularly useful when parsing
479   *                         the input file in parallel because the entry
480   *                         translation is also done in parallel.
481   *
482   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
483   *      constructor for more details about asynchronous processing.
484   */
485  public LDIFReader(final InputStream inputStream, final int numParseThreads,
486                    final LDIFReaderEntryTranslator entryTranslator)
487  {
488    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
489    this(new BufferedReader(new InputStreamReader(inputStream,
490                                                  Charset.forName("UTF-8")),
491                            DEFAULT_BUFFER_SIZE),
492         numParseThreads, entryTranslator);
493  }
494
495
496
497  /**
498   * Creates a new LDIF reader that will use the provided buffered reader to
499   * read the LDIF data.  The encoding of the underlying Reader must be set to
500   * "UTF-8" as required by RFC 2849.
501   *
502   * @param  reader  The buffered reader that will be used to read the LDIF
503   *                 data.  It must not be {@code null}.
504   */
505  public LDIFReader(final BufferedReader reader)
506  {
507    this(reader, 0);
508  }
509
510
511
512  /**
513   * Creates a new LDIF reader that will read data from the specified buffered
514   * reader and parses the LDIF records asynchronously using the specified
515   * number of threads.  The encoding of the underlying Reader must be set to
516   * "UTF-8" as required by RFC 2849.
517   *
518   * @param reader The buffered reader that will be used to read the LDIF data.
519   *               It must not be {@code null}.
520   * @param  numParseThreads  If this value is greater than zero, then the
521   *                          specified number of threads will be used to
522   *                          asynchronously read and parse the LDIF file.
523   *
524   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
525   *      constructor for more details about asynchronous processing.
526   */
527  public LDIFReader(final BufferedReader reader, final int numParseThreads)
528  {
529    this(reader, numParseThreads, null);
530  }
531
532
533
534  /**
535   * Creates a new LDIF reader that will read data from the specified buffered
536   * reader and parses the LDIF records asynchronously using the specified
537   * number of threads.  The encoding of the underlying Reader must be set to
538   * "UTF-8" as required by RFC 2849.
539   *
540   * @param reader The buffered reader that will be used to read the LDIF data.
541   *               It must not be {@code null}.
542   * @param  numParseThreads  If this value is greater than zero, then the
543   *                          specified number of threads will be used to
544   *                          asynchronously read and parse the LDIF file.
545   *                          This should only be set to greater than zero when
546   *                          performance analysis has demonstrated that reading
547   *                          and parsing the LDIF is a bottleneck.  The default
548   *                          synchronous processing is normally fast enough.
549   *                          There is little benefit in passing in a value
550   *                          greater than four (unless there is an
551   *                          LDIFReaderEntryTranslator that does time-consuming
552   *                          processing).  A value of zero implies the
553   *                          default behavior of reading and parsing LDIF
554   *                          records synchronously when one of the read
555   *                          methods is called.
556   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
557   *                         entries before they are returned.  This is normally
558   *                         {@code null}, which causes entries to be returned
559   *                         unaltered. This is particularly useful when parsing
560   *                         the input file in parallel because the entry
561   *                         translation is also done in parallel.
562   */
563  public LDIFReader(final BufferedReader reader,
564                    final int numParseThreads,
565                    final LDIFReaderEntryTranslator entryTranslator)
566  {
567    ensureNotNull(reader);
568    ensureTrue(numParseThreads >= 0,
569               "LDIFReader.numParseThreads must not be negative.");
570
571    this.reader = reader;
572    this.entryTranslator = entryTranslator;
573
574    duplicateValueBehavior = DuplicateValueBehavior.STRIP;
575    trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
576
577    relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
578
579    if (numParseThreads == 0)
580    {
581      isAsync = false;
582      asyncParser = null;
583      asyncParsingComplete = null;
584      asyncParsedRecords = null;
585    }
586    else
587    {
588      isAsync = true;
589      asyncParsingComplete = new AtomicBoolean(false);
590
591      // Decodes entries in parallel.
592      final LDAPSDKThreadFactory threadFactory =
593           new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
594      final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
595           new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>(
596                new RecordParser(), threadFactory, numParseThreads,
597                ASYNC_MIN_PER_PARSING_THREAD);
598
599      final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
600           ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE);
601
602      // The output queue must be a little more than twice as big as the input
603      // queue to more easily handle being shutdown in the middle of processing
604      // when the queues are full and threads are blocked.
605      asyncParsedRecords = new ArrayBlockingQueue
606           <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100);
607
608      asyncParser = new AsynchronousParallelProcessor
609           <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser,
610                                            asyncParsedRecords);
611
612      final LineReaderThread lineReaderThread = new LineReaderThread();
613      lineReaderThread.start();
614    }
615  }
616
617
618
619  /**
620   * Reads entries from the LDIF file with the specified path and returns them
621   * as a {@code List}.  This is a convenience method that should only be used
622   * for data sets that are small enough so that running out of memory isn't a
623   * concern.
624   *
625   * @param  path  The path to the LDIF file containing the entries to be read.
626   *
627   * @return  A list of the entries read from the given LDIF file.
628   *
629   * @throws  IOException  If a problem occurs while attempting to read data
630   *                       from the specified file.
631   *
632   * @throws  LDIFException  If a problem is encountered while attempting to
633   *                         decode data read as LDIF.
634   */
635  public static List<Entry> readEntries(final String path)
636         throws IOException, LDIFException
637  {
638    return readEntries(new LDIFReader(path));
639  }
640
641
642
643  /**
644   * Reads entries from the specified LDIF file and returns them as a
645   * {@code List}.  This is a convenience method that should only be used for
646   * data sets that are small enough so that running out of memory isn't a
647   * concern.
648   *
649   * @param  file  A reference to the LDIF file containing the entries to be
650   *               read.
651   *
652   * @return  A list of the entries read from the given LDIF file.
653   *
654   * @throws  IOException  If a problem occurs while attempting to read data
655   *                       from the specified file.
656   *
657   * @throws  LDIFException  If a problem is encountered while attempting to
658   *                         decode data read as LDIF.
659   */
660  public static List<Entry> readEntries(final File file)
661         throws IOException, LDIFException
662  {
663    return readEntries(new LDIFReader(file));
664  }
665
666
667
668  /**
669   * Reads and decodes LDIF entries from the provided input stream and
670   * returns them as a {@code List}.  This is a convenience method that should
671   * only be used for data sets that are small enough so that running out of
672   * memory isn't a concern.
673   *
674   * @param  inputStream  The input stream from which the entries should be
675   *                      read.  The input stream will be closed before
676   *                      returning.
677   *
678   * @return  A list of the entries read from the given input stream.
679   *
680   * @throws  IOException  If a problem occurs while attempting to read data
681   *                       from the input stream.
682   *
683   * @throws  LDIFException  If a problem is encountered while attempting to
684   *                         decode data read as LDIF.
685   */
686  public static List<Entry> readEntries(final InputStream inputStream)
687         throws IOException, LDIFException
688  {
689    return readEntries(new LDIFReader(inputStream));
690  }
691
692
693
694  /**
695   * Reads entries from the provided LDIF reader and returns them as a list.
696   *
697   * @param  reader  The reader from which the entries should be read.  It will
698   *                 be closed before returning.
699   *
700   * @return  A list of the entries read from the provided reader.
701   *
702   * @throws  IOException  If a problem was encountered while attempting to read
703   *                       data from the LDIF data source.
704   *
705   * @throws  LDIFException  If a problem is encountered while attempting to
706   *                         decode data read as LDIF.
707   */
708  private static List<Entry> readEntries(final LDIFReader reader)
709          throws IOException, LDIFException
710  {
711    try
712    {
713      final ArrayList<Entry> entries = new ArrayList<Entry>(10);
714      while (true)
715      {
716        final Entry e = reader.readEntry();
717        if (e == null)
718        {
719          break;
720        }
721
722        entries.add(e);
723      }
724
725      return entries;
726    }
727    finally
728    {
729      reader.close();
730    }
731  }
732
733
734
735  /**
736   * Closes this LDIF reader and the underlying LDIF source.
737   *
738   * @throws  IOException  If a problem occurs while closing the underlying LDIF
739   *                       source.
740   */
741  public void close()
742         throws IOException
743  {
744    reader.close();
745
746    if (isAsync())
747    {
748      // Closing the reader will trigger the LineReaderThread to complete, but
749      // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
750      // this, we clear out the completed output queue, which is larger than
751      // the input queue, so the LineReaderThread will stop reading and
752      // shutdown the asyncParser.
753      asyncParsedRecords.clear();
754    }
755  }
756
757
758
759  /**
760   * Indicates whether to ignore any duplicate values encountered while reading
761   * LDIF records.
762   *
763   * @return  {@code true} if duplicate values should be ignored, or
764   *          {@code false} if any LDIF records containing duplicate values
765   *          should be rejected.
766   *
767   * @deprecated  Use the {@link #getDuplicateValueBehavior} method instead.
768   */
769  @Deprecated()
770  public boolean ignoreDuplicateValues()
771  {
772    return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
773  }
774
775
776
777  /**
778   * Specifies whether to ignore any duplicate values encountered while reading
779   * LDIF records.
780   *
781   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
782   *                                attribute values encountered while reading
783   *                                LDIF records.
784   *
785   * @deprecated  Use the {@link #setDuplicateValueBehavior} method instead.
786   */
787  @Deprecated()
788  public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
789  {
790    if (ignoreDuplicateValues)
791    {
792      duplicateValueBehavior = DuplicateValueBehavior.STRIP;
793    }
794    else
795    {
796      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
797    }
798  }
799
800
801
802  /**
803   * Retrieves the behavior that should be exhibited if the LDIF reader
804   * encounters an entry with duplicate values.
805   *
806   * @return  The behavior that should be exhibited if the LDIF reader
807   *          encounters an entry with duplicate values.
808   */
809  public DuplicateValueBehavior getDuplicateValueBehavior()
810  {
811    return duplicateValueBehavior;
812  }
813
814
815
816  /**
817   * Specifies the behavior that should be exhibited if the LDIF reader
818   * encounters an entry with duplicate values.
819   *
820   * @param  duplicateValueBehavior  The behavior that should be exhibited if
821   *                                 the LDIF reader encounters an entry with
822   *                                 duplicate values.
823   */
824  public void setDuplicateValueBehavior(
825                   final DuplicateValueBehavior duplicateValueBehavior)
826  {
827    this.duplicateValueBehavior = duplicateValueBehavior;
828  }
829
830
831
832  /**
833   * Indicates whether to strip off any illegal trailing spaces that may appear
834   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
835   * specification strongly recommends that any value which legitimately
836   * contains trailing spaces be base64-encoded, and any spaces which appear
837   * after the end of non-base64-encoded values may therefore be considered
838   * invalid.  If any such trailing spaces are encountered in an LDIF record and
839   * they are not to be stripped, then an {@link LDIFException} will be thrown
840   * for that record.
841   * <BR><BR>
842   * Note that this applies only to spaces after the end of a value, and not to
843   * spaces which may appear at the end of a line for a value that is wrapped
844   * and continued on the next line.
845   *
846   * @return  {@code true} if illegal trailing spaces should be stripped off, or
847   *          {@code false} if LDIF records containing illegal trailing spaces
848   *          should be rejected.
849   *
850   * @deprecated  Use the {@link #getTrailingSpaceBehavior} method instead.
851   */
852  @Deprecated()
853  public boolean stripTrailingSpaces()
854  {
855    return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
856  }
857
858
859
860  /**
861   * Specifies whether to strip off any illegal trailing spaces that may appear
862   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
863   * specification strongly recommends that any value which legitimately
864   * contains trailing spaces be base64-encoded, and any spaces which appear
865   * after the end of non-base64-encoded values may therefore be considered
866   * invalid.  If any such trailing spaces are encountered in an LDIF record and
867   * they are not to be stripped, then an {@link LDIFException} will be thrown
868   * for that record.
869   * <BR><BR>
870   * Note that this applies only to spaces after the end of a value, and not to
871   * spaces which may appear at the end of a line for a value that is wrapped
872   * and continued on the next line.
873   *
874   * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
875   *                              trailing spaces, or {@code false} if LDIF
876   *                              records containing them should be rejected.
877   *
878   * @deprecated  Use the {@link #setTrailingSpaceBehavior} method instead.
879   */
880  @Deprecated()
881  public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
882  {
883    trailingSpaceBehavior = stripTrailingSpaces
884         ? TrailingSpaceBehavior.STRIP
885         : TrailingSpaceBehavior.REJECT;
886  }
887
888
889
890  /**
891   * Retrieves the behavior that should be exhibited when encountering attribute
892   * values which are not base64-encoded but contain trailing spaces.  The LDIF
893   * specification strongly recommends that any value which legitimately
894   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
895   * may be configured to automatically strip these spaces, to preserve them, or
896   * to reject any entry or change record containing them.
897   *
898   * @return  The behavior that should be exhibited when encountering attribute
899   *          values which are not base64-encoded but contain trailing spaces.
900   */
901  public TrailingSpaceBehavior getTrailingSpaceBehavior()
902  {
903    return trailingSpaceBehavior;
904  }
905
906
907
908  /**
909   * Specifies the behavior that should be exhibited when encountering attribute
910   * values which are not base64-encoded but contain trailing spaces.  The LDIF
911   * specification strongly recommends that any value which legitimately
912   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
913   * may be configured to automatically strip these spaces, to preserve them, or
914   * to reject any entry or change record containing them.
915   *
916   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
917   *                                encountering attribute values which are not
918   *                                base64-encoded but contain trailing spaces.
919   */
920  public void setTrailingSpaceBehavior(
921                   final TrailingSpaceBehavior trailingSpaceBehavior)
922  {
923    this.trailingSpaceBehavior = trailingSpaceBehavior;
924  }
925
926
927
928  /**
929   * Retrieves the base path that will be prepended to relative paths in order
930   * to obtain an absolute path.  This will only be used for "file:" URLs that
931   * have paths which do not begin with a slash.
932   *
933   * @return  The base path that will be prepended to relative paths in order to
934   *          obtain an absolute path.
935   */
936  public String getRelativeBasePath()
937  {
938    return relativeBasePath;
939  }
940
941
942
943  /**
944   * Specifies the base path that will be prepended to relative paths in order
945   * to obtain an absolute path.  This will only be used for "file:" URLs that
946   * have paths which do not begin with a space.
947   *
948   * @param  relativeBasePath  The base path that will be prepended to relative
949   *                           paths in order to obtain an absolute path.
950   */
951  public void setRelativeBasePath(final String relativeBasePath)
952  {
953    setRelativeBasePath(new File(relativeBasePath));
954  }
955
956
957
958  /**
959   * Specifies the base path that will be prepended to relative paths in order
960   * to obtain an absolute path.  This will only be used for "file:" URLs that
961   * have paths which do not begin with a space.
962   *
963   * @param  relativeBasePath  The base path that will be prepended to relative
964   *                           paths in order to obtain an absolute path.
965   */
966  public void setRelativeBasePath(final File relativeBasePath)
967  {
968    final String path = relativeBasePath.getAbsolutePath();
969    if (path.endsWith(File.separator))
970    {
971      this.relativeBasePath = path;
972    }
973    else
974    {
975      this.relativeBasePath = path + File.separator;
976    }
977  }
978
979
980
981  /**
982   * Retrieves the schema that will be used when reading LDIF records, if
983   * defined.
984   *
985   * @return  The schema that will be used when reading LDIF records, or
986   *          {@code null} if no schema should be used and all attributes should
987   *          be treated as case-insensitive strings.
988   */
989  public Schema getSchema()
990  {
991    return schema;
992  }
993
994
995
996  /**
997   * Specifies the schema that should be used when reading LDIF records.
998   *
999   * @param  schema  The schema that should be used when reading LDIF records,
1000   *                 or {@code null} if no schema should be used and all
1001   *                 attributes should be treated as case-insensitive strings.
1002   */
1003  public void setSchema(final Schema schema)
1004  {
1005    this.schema = schema;
1006  }
1007
1008
1009
1010  /**
1011   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1012   * change record.
1013   *
1014   * @return  The record read from the LDIF source, or {@code null} if there are
1015   *          no more entries to be read.
1016   *
1017   * @throws  IOException  If a problem occurs while trying to read from the
1018   *                       LDIF source.
1019   *
1020   * @throws  LDIFException  If the data read could not be parsed as an entry or
1021   *                         an LDIF change record.
1022   */
1023  public LDIFRecord readLDIFRecord()
1024         throws IOException, LDIFException
1025  {
1026    if (isAsync())
1027    {
1028      return readLDIFRecordAsync();
1029    }
1030    else
1031    {
1032      return readLDIFRecordInternal();
1033    }
1034  }
1035
1036
1037
1038  /**
1039   * Reads an entry from the LDIF source.
1040   *
1041   * @return  The entry read from the LDIF source, or {@code null} if there are
1042   *          no more entries to be read.
1043   *
1044   * @throws  IOException  If a problem occurs while attempting to read from the
1045   *                       LDIF source.
1046   *
1047   * @throws  LDIFException  If the data read could not be parsed as an entry.
1048   */
1049  public Entry readEntry()
1050         throws IOException, LDIFException
1051  {
1052    if (isAsync())
1053    {
1054      return readEntryAsync();
1055    }
1056    else
1057    {
1058      return readEntryInternal();
1059    }
1060  }
1061
1062
1063
1064  /**
1065   * Reads an LDIF change record from the LDIF source.  The LDIF record must
1066   * have a changetype.
1067   *
1068   * @return  The change record read from the LDIF source, or {@code null} if
1069   *          there are no more records to be read.
1070   *
1071   * @throws  IOException  If a problem occurs while attempting to read from the
1072   *                       LDIF source.
1073   *
1074   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1075   *                         change record.
1076   */
1077  public LDIFChangeRecord readChangeRecord()
1078         throws IOException, LDIFException
1079  {
1080    return readChangeRecord(false);
1081  }
1082
1083
1084
1085  /**
1086   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1087   * record does not have a changetype, then it may be assumed to be an add
1088   * change record.
1089   *
1090   * @param  defaultAdd  Indicates whether an LDIF record not containing a
1091   *                     changetype should be retrieved as an add change record.
1092   *                     If this is {@code false} and the record read does not
1093   *                     include a changetype, then an {@link LDIFException}
1094   *                     will be thrown.
1095   *
1096   * @return  The change record read from the LDIF source, or {@code null} if
1097   *          there are no more records to be read.
1098   *
1099   * @throws  IOException  If a problem occurs while attempting to read from the
1100   *                       LDIF source.
1101   *
1102   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1103   *                         change record.
1104   */
1105  public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1106         throws IOException, LDIFException
1107  {
1108    if (isAsync())
1109    {
1110      return readChangeRecordAsync(defaultAdd);
1111    }
1112    else
1113    {
1114      return readChangeRecordInternal(defaultAdd);
1115    }
1116  }
1117
1118
1119
1120  /**
1121   * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1122   * thread.
1123   *
1124   * @return  The next parsed record or {@code null} if there are no more
1125   *          records to read.
1126   *
1127   * @throws IOException  If IOException was thrown when reading or parsing
1128   *                      the record.
1129   *
1130   * @throws LDIFException If LDIFException was thrown parsing the record.
1131   */
1132  private LDIFRecord readLDIFRecordAsync()
1133          throws IOException, LDIFException
1134  {
1135    final Result<UnparsedLDIFRecord, LDIFRecord> result =
1136         readLDIFRecordResultAsync();
1137    if (result == null)
1138    {
1139      return null;
1140    }
1141    else
1142    {
1143      return result.getOutput();
1144    }
1145  }
1146
1147
1148
1149  /**
1150   * Reads an entry asynchronously from the LDIF source.
1151   *
1152   * @return The entry read from the LDIF source, or {@code null} if there are
1153   *         no more entries to be read.
1154   *
1155   * @throws IOException   If a problem occurs while attempting to read from the
1156   *                       LDIF source.
1157   * @throws LDIFException If the data read could not be parsed as an entry.
1158   */
1159  private Entry readEntryAsync()
1160          throws IOException, LDIFException
1161  {
1162    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1163    LDIFRecord record = null;
1164    while (record == null)
1165    {
1166      result = readLDIFRecordResultAsync();
1167      if (result == null)
1168      {
1169        return null;
1170      }
1171
1172      record = result.getOutput();
1173
1174      // This is a special value that means we should skip this Entry.  We have
1175      // to use something different than null because null means EOF.
1176      if (record == SKIP_ENTRY)
1177      {
1178        record = null;
1179      }
1180    }
1181
1182    if (!(record instanceof Entry))
1183    {
1184      try
1185      {
1186        // Some LDIFChangeRecord can be converted to an Entry.  This is really
1187        // an edge case though.
1188        return ((LDIFChangeRecord)record).toEntry();
1189      }
1190      catch (LDIFException e)
1191      {
1192        debugException(e);
1193        final long firstLineNumber = result.getInput().getFirstLineNumber();
1194        throw new LDIFException(e.getExceptionMessage(),
1195                                firstLineNumber, true, e);
1196      }
1197    }
1198
1199    return (Entry) record;
1200  }
1201
1202
1203
1204  /**
1205   * Reads an LDIF change record from the LDIF source asynchronously.
1206   * Optionally, if the LDIF record does not have a changetype, then it may be
1207   * assumed to be an add change record.
1208   *
1209   * @param defaultAdd Indicates whether an LDIF record not containing a
1210   *                   changetype should be retrieved as an add change record.
1211   *                   If this is {@code false} and the record read does not
1212   *                   include a changetype, then an {@link LDIFException} will
1213   *                   be thrown.
1214   *
1215   * @return The change record read from the LDIF source, or {@code null} if
1216   *         there are no more records to be read.
1217   *
1218   * @throws IOException   If a problem occurs while attempting to read from the
1219   *                       LDIF source.
1220   * @throws LDIFException If the data read could not be parsed as an LDIF
1221   *                       change record.
1222   */
1223  private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1224          throws IOException, LDIFException
1225  {
1226    final Result<UnparsedLDIFRecord, LDIFRecord> result =
1227         readLDIFRecordResultAsync();
1228    if (result == null)
1229    {
1230      return null;
1231    }
1232
1233    final LDIFRecord record = result.getOutput();
1234    if (record instanceof LDIFChangeRecord)
1235    {
1236      return (LDIFChangeRecord) record;
1237    }
1238    else if (record instanceof Entry)
1239    {
1240      if (defaultAdd)
1241      {
1242        return new LDIFAddChangeRecord((Entry) record);
1243      }
1244      else
1245      {
1246        final long firstLineNumber = result.getInput().getFirstLineNumber();
1247        throw new LDIFException(
1248             ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1249             true);
1250      }
1251    }
1252
1253    throw new AssertionError("LDIFRecords must either be an Entry or an " +
1254                             "LDIFChangeRecord");
1255  }
1256
1257
1258
1259  /**
1260   * Reads the next LDIF record, which was read and parsed asynchronously by
1261   * separate threads.
1262   *
1263   * @return  The next LDIF record or {@code null} if there are no more records.
1264   *
1265   * @throws  IOException  If a problem occurs while attempting to read from the
1266   *                       LDIF source.
1267   *
1268   * @throws  LDIFException  If the data read could not be parsed as an entry.
1269   */
1270  private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1271          throws IOException, LDIFException
1272  {
1273    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1274
1275    // If the asynchronous reading and parsing is complete, then we don't have
1276    // to block waiting for the next record to show up on the queue.  If there
1277    // isn't a record there, then return null (EOF) right away.
1278    if (asyncParsingComplete.get())
1279    {
1280      result = asyncParsedRecords.poll();
1281    }
1282    else
1283    {
1284      try
1285      {
1286        // We probably could just do a asyncParsedRecords.take() here, but
1287        // there are some edge case error scenarios where
1288        // asyncParsingComplete might be set without a special EOF sentinel
1289        // Result enqueued.  So to guard against this, we have a very cautious
1290        // polling interval of 1 second.  During normal processing, we never
1291        // have to wait for this to expire, when there is something to do
1292        // (like shutdown).
1293        while ((result == null) && (!asyncParsingComplete.get()))
1294        {
1295          result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1296        }
1297
1298        // There's a very small chance that we missed the value, so double-check
1299        if (result == null)
1300        {
1301          result = asyncParsedRecords.poll();
1302        }
1303      }
1304      catch (InterruptedException e)
1305      {
1306        debugException(e);
1307        throw new IOException(getExceptionMessage(e));
1308      }
1309    }
1310    if (result == null)
1311    {
1312      return null;
1313    }
1314
1315    rethrow(result.getFailureCause());
1316
1317    // Check if we reached the end of the input
1318    final UnparsedLDIFRecord unparsedRecord = result.getInput();
1319    if (unparsedRecord.isEOF())
1320    {
1321      // This might have been set already by the LineReaderThread, but
1322      // just in case it hasn't gotten to it yet, do so here.
1323      asyncParsingComplete.set(true);
1324
1325      // Enqueue this EOF result again for any other thread that might be
1326      // blocked in asyncParsedRecords.take() even though having multiple
1327      // threads call this method concurrently breaks the contract of this
1328      // class.
1329      try
1330      {
1331        asyncParsedRecords.put(result);
1332      }
1333      catch (InterruptedException e)
1334      {
1335        // We shouldn't ever get interrupted because the put won't ever block.
1336        // Once we are done reading, this is the only item left in the queue,
1337        // so we should always be able to re-enqueue it.
1338        debugException(e);
1339      }
1340      return null;
1341    }
1342
1343    return result;
1344  }
1345
1346
1347
1348  /**
1349   * Indicates whether this LDIF reader was constructed to perform asynchronous
1350   * processing.
1351   *
1352   * @return  {@code true} if this LDIFReader was constructed to perform
1353   *          asynchronous processing, or {@code false} if not.
1354   */
1355  private boolean isAsync()
1356  {
1357    return isAsync;
1358  }
1359
1360
1361
1362  /**
1363   * If not {@code null}, rethrows the specified Throwable as either an
1364   * IOException or LDIFException.
1365   *
1366   * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1367   *           is thrown.
1368   *
1369   * @throws IOException   If t is an IOException or a checked Exception that
1370   *                       is not an LDIFException.
1371   * @throws LDIFException  If t is an LDIFException.
1372   */
1373  static void rethrow(final Throwable t)
1374         throws IOException, LDIFException
1375  {
1376    if (t == null)
1377    {
1378      return;
1379    }
1380
1381    if (t instanceof IOException)
1382    {
1383      throw (IOException) t;
1384    }
1385    else if (t instanceof LDIFException)
1386    {
1387      throw (LDIFException) t;
1388    }
1389    else if (t instanceof RuntimeException)
1390    {
1391      throw (RuntimeException) t;
1392    }
1393    else if (t instanceof Error)
1394    {
1395      throw (Error) t;
1396    }
1397    else
1398    {
1399      throw new IOException(getExceptionMessage(t));
1400    }
1401  }
1402
1403
1404
1405  /**
1406   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1407   * change record.
1408   *
1409   * @return The record read from the LDIF source, or {@code null} if there are
1410   *         no more entries to be read.
1411   *
1412   * @throws IOException   If a problem occurs while trying to read from the
1413   *                       LDIF source.
1414   * @throws LDIFException If the data read could not be parsed as an entry or
1415   *                       an LDIF change record.
1416   */
1417  private LDIFRecord readLDIFRecordInternal()
1418       throws IOException, LDIFException
1419  {
1420    final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1421    return decodeRecord(unparsedRecord, relativeBasePath);
1422  }
1423
1424
1425
1426  /**
1427   * Reads an entry from the LDIF source.
1428   *
1429   * @return The entry read from the LDIF source, or {@code null} if there are
1430   *         no more entries to be read.
1431   *
1432   * @throws IOException   If a problem occurs while attempting to read from the
1433   *                       LDIF source.
1434   * @throws LDIFException If the data read could not be parsed as an entry.
1435   */
1436  private Entry readEntryInternal()
1437       throws IOException, LDIFException
1438  {
1439    Entry e = null;
1440    while (e == null)
1441    {
1442      final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1443      if (unparsedRecord.isEOF())
1444      {
1445        return null;
1446      }
1447
1448      e = decodeEntry(unparsedRecord, relativeBasePath);
1449      debugLDIFRead(e);
1450
1451      if (entryTranslator != null)
1452      {
1453        e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1454      }
1455    }
1456    return e;
1457  }
1458
1459
1460
1461  /**
1462   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1463   * record does not have a changetype, then it may be assumed to be an add
1464   * change record.
1465   *
1466   * @param defaultAdd Indicates whether an LDIF record not containing a
1467   *                   changetype should be retrieved as an add change record.
1468   *                   If this is {@code false} and the record read does not
1469   *                   include a changetype, then an {@link LDIFException} will
1470   *                   be thrown.
1471   *
1472   * @return The change record read from the LDIF source, or {@code null} if
1473   *         there are no more records to be read.
1474   *
1475   * @throws IOException   If a problem occurs while attempting to read from the
1476   *                       LDIF source.
1477   * @throws LDIFException If the data read could not be parsed as an LDIF
1478   *                       change record.
1479   */
1480  private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1481       throws IOException, LDIFException
1482  {
1483    final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1484    if (unparsedRecord.isEOF())
1485    {
1486      return null;
1487    }
1488
1489    final LDIFChangeRecord r =
1490         decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd);
1491    debugLDIFRead(r);
1492    return r;
1493  }
1494
1495
1496
1497  /**
1498   * Reads a record (either an entry or a change record) from the LDIF source
1499   * and places it in the line list.
1500   *
1501   * @return  The line number for the first line of the entry that was read.
1502   *
1503   * @throws  IOException  If a problem occurs while attempting to read from the
1504   *                       LDIF source.
1505   *
1506   * @throws  LDIFException  If the data read could not be parsed as a valid
1507   *                         LDIF record.
1508   */
1509  private UnparsedLDIFRecord readUnparsedRecord()
1510         throws IOException, LDIFException
1511  {
1512    final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20);
1513    boolean lastWasComment = false;
1514    long firstLineNumber = lineNumberCounter + 1;
1515    while (true)
1516    {
1517      final String line = reader.readLine();
1518      lineNumberCounter++;
1519
1520      if (line == null)
1521      {
1522        // We've hit the end of the LDIF source.  If we haven't read any entry
1523        // data, then return null.  Otherwise, the last entry wasn't followed by
1524        // a blank line, which is OK, and we should decode that entry.
1525        if (lineList.isEmpty())
1526        {
1527          return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1528               duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1529        }
1530        else
1531        {
1532          break;
1533        }
1534      }
1535
1536      if (line.length() == 0)
1537      {
1538        // It's a blank line.  If we have read entry data, then this signals the
1539        // end of the entry.  Otherwise, it's an extra space between entries,
1540        // which is OK.
1541        lastWasComment = false;
1542        if (lineList.isEmpty())
1543        {
1544          firstLineNumber++;
1545          continue;
1546        }
1547        else
1548        {
1549          break;
1550        }
1551      }
1552
1553      if (line.charAt(0) == ' ')
1554      {
1555        // The line starts with a space, which means that it must be a
1556        // continuation of the previous line.  This is true even if the last
1557        // line was a comment.
1558        if (lastWasComment)
1559        {
1560          // What we've read is part of a comment, so we don't care about its
1561          // content.
1562        }
1563        else if (lineList.isEmpty())
1564        {
1565          throw new LDIFException(
1566                         ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1567                         lineNumberCounter, false);
1568        }
1569        else
1570        {
1571          lineList.get(lineList.size() - 1).append(line.substring(1));
1572          lastWasComment = false;
1573        }
1574      }
1575      else if (line.charAt(0) == '#')
1576      {
1577        lastWasComment = true;
1578      }
1579      else
1580      {
1581        // We want to make sure that we skip over the "version:" line if it
1582        // exists, but that should only occur at the beginning of an entry where
1583        // it can't be confused with a possible "version" attribute.
1584        if (lineList.isEmpty() && line.startsWith("version:"))
1585        {
1586          lastWasComment = true;
1587        }
1588        else
1589        {
1590          lineList.add(new StringBuilder(line));
1591          lastWasComment = false;
1592        }
1593      }
1594    }
1595
1596    return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1597         trailingSpaceBehavior, schema, firstLineNumber);
1598  }
1599
1600
1601
1602  /**
1603   * Decodes the provided set of LDIF lines as an entry.  The provided set of
1604   * lines must contain exactly one entry.  Long lines may be wrapped as per the
1605   * LDIF specification, and it is acceptable to have one or more blank lines
1606   * following the entry.
1607   *
1608   * @param  ldifLines  The set of lines that comprise the LDIF representation
1609   *                    of the entry.  It must not be {@code null} or empty.
1610   *
1611   * @return  The entry read from LDIF.
1612   *
1613   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1614   *                         entry.
1615   */
1616  public static Entry decodeEntry(final String... ldifLines)
1617         throws LDIFException
1618  {
1619    final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1620         TrailingSpaceBehavior.REJECT, null, ldifLines),
1621         DEFAULT_RELATIVE_BASE_PATH);
1622    debugLDIFRead(e);
1623    return e;
1624  }
1625
1626
1627
1628  /**
1629   * Decodes the provided set of LDIF lines as an entry.  The provided set of
1630   * lines must contain exactly one entry.  Long lines may be wrapped as per the
1631   * LDIF specification, and it is acceptable to have one or more blank lines
1632   * following the entry.
1633   *
1634   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1635   *                                attribute values encountered while parsing.
1636   * @param  schema                 The schema to use when parsing the record,
1637   *                                if applicable.
1638   * @param  ldifLines              The set of lines that comprise the LDIF
1639   *                                representation of the entry.  It must not be
1640   *                                {@code null} or empty.
1641   *
1642   * @return  The entry read from LDIF.
1643   *
1644   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1645   *                         entry.
1646   */
1647  public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1648                                  final Schema schema,
1649                                  final String... ldifLines)
1650         throws LDIFException
1651  {
1652    final Entry e = decodeEntry(prepareRecord(
1653              (ignoreDuplicateValues
1654                   ? DuplicateValueBehavior.STRIP
1655                   : DuplicateValueBehavior.REJECT),
1656              TrailingSpaceBehavior.REJECT, schema, ldifLines),
1657         DEFAULT_RELATIVE_BASE_PATH);
1658    debugLDIFRead(e);
1659    return e;
1660  }
1661
1662
1663
1664  /**
1665   * Decodes the provided set of LDIF lines as an LDIF change record.  The
1666   * provided set of lines must contain exactly one change record and it must
1667   * include a changetype.  Long lines may be wrapped as per the LDIF
1668   * specification, and it is acceptable to have one or more blank lines
1669   * following the entry.
1670   *
1671   * @param  ldifLines  The set of lines that comprise the LDIF representation
1672   *                    of the change record.  It must not be {@code null} or
1673   *                    empty.
1674   *
1675   * @return  The change record read from LDIF.
1676   *
1677   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1678   *                         change record.
1679   */
1680  public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1681         throws LDIFException
1682  {
1683    return decodeChangeRecord(false, ldifLines);
1684  }
1685
1686
1687
1688  /**
1689   * Decodes the provided set of LDIF lines as an LDIF change record.  The
1690   * provided set of lines must contain exactly one change record.  Long lines
1691   * may be wrapped as per the LDIF specification, and it is acceptable to have
1692   * one or more blank lines following the entry.
1693   *
1694   * @param  defaultAdd  Indicates whether an LDIF record not containing a
1695   *                     changetype should be retrieved as an add change record.
1696   *                     If this is {@code false} and the record read does not
1697   *                     include a changetype, then an {@link LDIFException}
1698   *                     will be thrown.
1699   * @param  ldifLines  The set of lines that comprise the LDIF representation
1700   *                    of the change record.  It must not be {@code null} or
1701   *                    empty.
1702   *
1703   * @return  The change record read from LDIF.
1704   *
1705   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1706   *                         change record.
1707   */
1708  public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1709                                                    final String... ldifLines)
1710         throws LDIFException
1711  {
1712    final LDIFChangeRecord r =
1713         decodeChangeRecord(
1714              prepareRecord(DuplicateValueBehavior.STRIP,
1715                   TrailingSpaceBehavior.REJECT, null, ldifLines),
1716              DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1717    debugLDIFRead(r);
1718    return r;
1719  }
1720
1721
1722
1723  /**
1724   * Decodes the provided set of LDIF lines as an LDIF change record.  The
1725   * provided set of lines must contain exactly one change record.  Long lines
1726   * may be wrapped as per the LDIF specification, and it is acceptable to have
1727   * one or more blank lines following the entry.
1728   *
1729   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1730   *                                attribute values encountered while parsing.
1731   * @param  schema                 The schema to use when processing the change
1732   *                                record, or {@code null} if no schema should
1733   *                                be used and all values should be treated as
1734   *                                case-insensitive strings.
1735   * @param  defaultAdd             Indicates whether an LDIF record not
1736   *                                containing a changetype should be retrieved
1737   *                                as an add change record.  If this is
1738   *                                {@code false} and the record read does not
1739   *                                include a changetype, then an
1740   *                                {@link LDIFException} will be thrown.
1741   * @param  ldifLines              The set of lines that comprise the LDIF
1742   *                                representation of the change record.  It
1743   *                                must not be {@code null} or empty.
1744   *
1745   * @return  The change record read from LDIF.
1746   *
1747   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1748   *                         change record.
1749   */
1750  public static LDIFChangeRecord decodeChangeRecord(
1751                                      final boolean ignoreDuplicateValues,
1752                                      final Schema schema,
1753                                      final boolean defaultAdd,
1754                                      final String... ldifLines)
1755         throws LDIFException
1756  {
1757    final LDIFChangeRecord r = decodeChangeRecord(
1758         prepareRecord(
1759              (ignoreDuplicateValues
1760                   ? DuplicateValueBehavior.STRIP
1761                   : DuplicateValueBehavior.REJECT),
1762              TrailingSpaceBehavior.REJECT, schema, ldifLines),
1763         DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1764    debugLDIFRead(r);
1765    return r;
1766  }
1767
1768
1769
1770  /**
1771   * Parses the provided set of lines into a list of {@code StringBuilder}
1772   * objects suitable for decoding into an entry or LDIF change record.
1773   * Comments will be ignored and wrapped lines will be unwrapped.
1774   *
1775   * @param  duplicateValueBehavior  The behavior that should be exhibited if
1776   *                                 the LDIF reader encounters an entry with
1777   *                                 duplicate values.
1778   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
1779   *                                 encountering attribute values which are not
1780   *                                 base64-encoded but contain trailing spaces.
1781   * @param  schema                  The schema to use when parsing the record,
1782   *                                 if applicable.
1783   * @param  ldifLines               The set of lines that comprise the record
1784   *                                 to decode.  It must not be {@code null} or
1785   *                                 empty.
1786   *
1787   * @return  The prepared list of {@code StringBuilder} objects ready to be
1788   *          decoded.
1789   *
1790   * @throws  LDIFException  If the provided lines do not contain valid LDIF
1791   *                         content.
1792   */
1793  private static UnparsedLDIFRecord prepareRecord(
1794                      final DuplicateValueBehavior duplicateValueBehavior,
1795                      final TrailingSpaceBehavior trailingSpaceBehavior,
1796                      final Schema schema, final String... ldifLines)
1797          throws LDIFException
1798  {
1799    ensureNotNull(ldifLines);
1800    ensureFalse(ldifLines.length == 0,
1801                "LDIFReader.prepareRecord.ldifLines must not be empty.");
1802
1803    boolean lastWasComment = false;
1804    final ArrayList<StringBuilder> lineList =
1805         new ArrayList<StringBuilder>(ldifLines.length);
1806    for (int i=0; i < ldifLines.length; i++)
1807    {
1808      final String line = ldifLines[i];
1809      if (line.length() == 0)
1810      {
1811        // This is only acceptable if there are no more non-empty lines in the
1812        // array.
1813        for (int j=i+1; j < ldifLines.length; j++)
1814        {
1815          if (ldifLines[j].length() > 0)
1816          {
1817            throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
1818                                    ldifLines, null);
1819          }
1820
1821          // If we've gotten here, then we know that we're at the end of the
1822          // entry.  If we have read data, then we can decode it as an entry.
1823          // Otherwise, there was no real data in the provided LDIF lines.
1824          if (lineList.isEmpty())
1825          {
1826            throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
1827                                    ldifLines, null);
1828          }
1829          else
1830          {
1831            return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1832                 trailingSpaceBehavior, schema, 0);
1833          }
1834        }
1835      }
1836
1837      if (line.charAt(0) == ' ')
1838      {
1839        if (i > 0)
1840        {
1841          if (! lastWasComment)
1842          {
1843            lineList.get(lineList.size() - 1).append(line.substring(1));
1844          }
1845        }
1846        else
1847        {
1848          throw new LDIFException(
1849                         ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
1850                         true, ldifLines, null);
1851        }
1852      }
1853      else if (line.charAt(0) == '#')
1854      {
1855        lastWasComment = true;
1856      }
1857      else
1858      {
1859        lineList.add(new StringBuilder(line));
1860        lastWasComment = false;
1861      }
1862    }
1863
1864    if (lineList.isEmpty())
1865    {
1866      throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
1867    }
1868    else
1869    {
1870      return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1871           trailingSpaceBehavior, schema, 0);
1872    }
1873  }
1874
1875
1876
1877  /**
1878   * Decodes the unparsed record that was read from the LDIF source.  It may be
1879   * either an entry or an LDIF change record.
1880   *
1881   * @param  unparsedRecord    The unparsed LDIF record that was read from the
1882   *                           input.  It must not be {@code null} or empty.
1883   * @param  relativeBasePath  The base path that will be prepended to relative
1884   *                           paths in order to obtain an absolute path.
1885   *
1886   * @return  The parsed record, or {@code null} if there are no more entries to
1887   *          be read.
1888   *
1889   * @throws  LDIFException  If the data read could not be parsed as an entry or
1890   *                         an LDIF change record.
1891   */
1892  private static LDIFRecord decodeRecord(
1893                                 final UnparsedLDIFRecord unparsedRecord,
1894                                 final String relativeBasePath)
1895       throws LDIFException
1896  {
1897    // If there was an error reading from the input, then we rethrow it here.
1898    final Exception readError = unparsedRecord.getFailureCause();
1899    if (readError != null)
1900    {
1901      if (readError instanceof LDIFException)
1902      {
1903        // If the error was an LDIFException, which will normally be the case,
1904        // then rethrow it with all of the same state.  We could just
1905        //   throw (LDIFException) readError;
1906        // but that's considered bad form.
1907        final LDIFException ldifEx = (LDIFException) readError;
1908        throw new LDIFException(ldifEx.getMessage(),
1909                                ldifEx.getLineNumber(),
1910                                ldifEx.mayContinueReading(),
1911                                ldifEx.getDataLines(),
1912                                ldifEx.getCause());
1913      }
1914      else
1915      {
1916        throw new LDIFException(getExceptionMessage(readError),
1917                                -1, true, readError);
1918      }
1919    }
1920
1921    if (unparsedRecord.isEOF())
1922    {
1923      return null;
1924    }
1925
1926    final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
1927    if (unparsedRecord.getLineList() == null)
1928    {
1929      return null;  // We can get here if there was an error reading the lines.
1930    }
1931
1932    final LDIFRecord r;
1933    if (lineList.size() == 1)
1934    {
1935      r = decodeEntry(unparsedRecord, relativeBasePath);
1936    }
1937    else
1938    {
1939      final String lowerSecondLine = toLowerCase(lineList.get(1).toString());
1940      if (lowerSecondLine.startsWith("control:") ||
1941          lowerSecondLine.startsWith("changetype:"))
1942      {
1943        r = decodeChangeRecord(unparsedRecord, relativeBasePath, true);
1944      }
1945      else
1946      {
1947        r = decodeEntry(unparsedRecord, relativeBasePath);
1948      }
1949    }
1950
1951    debugLDIFRead(r);
1952    return r;
1953  }
1954
1955
1956
1957  /**
1958   * Decodes the provided set of LDIF lines as an entry.  The provided list must
1959   * not contain any blank lines or comments, and lines are not allowed to be
1960   * wrapped.
1961   *
1962   * @param  unparsedRecord   The unparsed LDIF record that was read from the
1963   *                          input.  It must not be {@code null} or empty.
1964   * @param  relativeBasePath  The base path that will be prepended to relative
1965   *                           paths in order to obtain an absolute path.
1966   *
1967   * @return  The entry read from LDIF.
1968   *
1969   * @throws  LDIFException  If the provided LDIF data cannot be read as an
1970   *                         entry.
1971   */
1972  private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
1973                                   final String relativeBasePath)
1974          throws LDIFException
1975  {
1976    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
1977    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
1978
1979    final Iterator<StringBuilder> iterator = ldifLines.iterator();
1980
1981    // The first line must start with either "version:" or "dn:".  If the first
1982    // line starts with "version:" then the second must start with "dn:".
1983    StringBuilder line = iterator.next();
1984    handleTrailingSpaces(line, null, firstLineNumber,
1985         unparsedRecord.getTrailingSpaceBehavior());
1986    int colonPos = line.indexOf(":");
1987    if ((colonPos > 0) &&
1988        line.substring(0, colonPos).equalsIgnoreCase("version"))
1989    {
1990      // The first line is "version:".  Under most conditions, this will be
1991      // handled by the LDIF reader, but this can happen if you call
1992      // decodeEntry with a set of data that includes a version.  At any rate,
1993      // read the next line, which must specify the DN.
1994      line = iterator.next();
1995      handleTrailingSpaces(line, null, firstLineNumber,
1996           unparsedRecord.getTrailingSpaceBehavior());
1997    }
1998
1999    colonPos = line.indexOf(":");
2000    if ((colonPos < 0) ||
2001         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2002    {
2003      throw new LDIFException(
2004           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2005           firstLineNumber, true, ldifLines, null);
2006    }
2007
2008    final String dn;
2009    final int length = line.length();
2010    if (length == (colonPos+1))
2011    {
2012      // The colon was the last character on the line.  This is acceptable and
2013      // indicates that the entry has the null DN.
2014      dn = "";
2015    }
2016    else if (line.charAt(colonPos+1) == ':')
2017    {
2018      // Skip over any spaces leading up to the value, and then the rest of the
2019      // string is the base64-encoded DN.
2020      int pos = colonPos+2;
2021      while ((pos < length) && (line.charAt(pos) == ' '))
2022      {
2023        pos++;
2024      }
2025
2026      try
2027      {
2028        final byte[] dnBytes = Base64.decode(line.substring(pos));
2029        dn = new String(dnBytes, "UTF-8");
2030      }
2031      catch (final ParseException pe)
2032      {
2033        debugException(pe);
2034        throw new LDIFException(
2035                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2036                                                            pe.getMessage()),
2037                       firstLineNumber, true, ldifLines, pe);
2038      }
2039      catch (final Exception e)
2040      {
2041        debugException(e);
2042        throw new LDIFException(
2043                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2044                       firstLineNumber, true, ldifLines, e);
2045      }
2046    }
2047    else
2048    {
2049      // Skip over any spaces leading up to the value, and then the rest of the
2050      // string is the DN.
2051      int pos = colonPos+1;
2052      while ((pos < length) && (line.charAt(pos) == ' '))
2053      {
2054        pos++;
2055      }
2056
2057      dn = line.substring(pos);
2058    }
2059
2060
2061    // The remaining lines must be the attributes for the entry.  However, we
2062    // will allow the case in which an entry does not have any attributes, to be
2063    // able to support reading search result entries in which no attributes were
2064    // returned.
2065    if (! iterator.hasNext())
2066    {
2067      return new Entry(dn, unparsedRecord.getSchema());
2068    }
2069
2070    return new Entry(dn, unparsedRecord.getSchema(),
2071         parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2072              unparsedRecord.getTrailingSpaceBehavior(),
2073              unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2074              firstLineNumber));
2075  }
2076
2077
2078
2079  /**
2080   * Decodes the provided set of LDIF lines as a change record.  The provided
2081   * list must not contain any blank lines or comments, and lines are not
2082   * allowed to be wrapped.
2083   *
2084   * @param  unparsedRecord    The unparsed LDIF record that was read from the
2085   *                           input.  It must not be {@code null} or empty.
2086   * @param  relativeBasePath  The base path that will be prepended to relative
2087   *                           paths in order to obtain an absolute path.
2088   * @param  defaultAdd        Indicates whether an LDIF record not containing a
2089   *                           changetype should be retrieved as an add change
2090   *                           record.  If this is {@code false} and the record
2091   *                           read does not include a changetype, then an
2092   *                           {@link LDIFException} will be thrown.
2093   *
2094   * @return  The change record read from LDIF.
2095   *
2096   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2097   *                         change record.
2098   */
2099  private static LDIFChangeRecord decodeChangeRecord(
2100                                       final UnparsedLDIFRecord unparsedRecord,
2101                                       final String relativeBasePath,
2102                                       final boolean defaultAdd)
2103          throws LDIFException
2104  {
2105    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2106    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2107
2108    Iterator<StringBuilder> iterator = ldifLines.iterator();
2109
2110    // The first line must start with either "version:" or "dn:".  If the first
2111    // line starts with "version:" then the second must start with "dn:".
2112    StringBuilder line = iterator.next();
2113    handleTrailingSpaces(line, null, firstLineNumber,
2114         unparsedRecord.getTrailingSpaceBehavior());
2115    int colonPos = line.indexOf(":");
2116    int linesRead = 1;
2117    if ((colonPos > 0) &&
2118        line.substring(0, colonPos).equalsIgnoreCase("version"))
2119    {
2120      // The first line is "version:".  Under most conditions, this will be
2121      // handled by the LDIF reader, but this can happen if you call
2122      // decodeEntry with a set of data that includes a version.  At any rate,
2123      // read the next line, which must specify the DN.
2124      line = iterator.next();
2125      linesRead++;
2126      handleTrailingSpaces(line, null, firstLineNumber,
2127           unparsedRecord.getTrailingSpaceBehavior());
2128    }
2129
2130    colonPos = line.indexOf(":");
2131    if ((colonPos < 0) ||
2132         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2133    {
2134      throw new LDIFException(
2135           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2136           firstLineNumber, true, ldifLines, null);
2137    }
2138
2139    final String dn;
2140    int length = line.length();
2141    if (length == (colonPos+1))
2142    {
2143      // The colon was the last character on the line.  This is acceptable and
2144      // indicates that the entry has the null DN.
2145      dn = "";
2146    }
2147    else if (line.charAt(colonPos+1) == ':')
2148    {
2149      // Skip over any spaces leading up to the value, and then the rest of the
2150      // string is the base64-encoded DN.
2151      int pos = colonPos+2;
2152      while ((pos < length) && (line.charAt(pos) == ' '))
2153      {
2154        pos++;
2155      }
2156
2157      try
2158      {
2159        final byte[] dnBytes = Base64.decode(line.substring(pos));
2160        dn = new String(dnBytes, "UTF-8");
2161      }
2162      catch (final ParseException pe)
2163      {
2164        debugException(pe);
2165        throw new LDIFException(
2166                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2167                                                               pe.getMessage()),
2168                       firstLineNumber, true, ldifLines, pe);
2169      }
2170      catch (final Exception e)
2171      {
2172        debugException(e);
2173        throw new LDIFException(
2174                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2175                                                               e),
2176                       firstLineNumber, true, ldifLines, e);
2177      }
2178    }
2179    else
2180    {
2181      // Skip over any spaces leading up to the value, and then the rest of the
2182      // string is the DN.
2183      int pos = colonPos+1;
2184      while ((pos < length) && (line.charAt(pos) == ' '))
2185      {
2186        pos++;
2187      }
2188
2189      dn = line.substring(pos);
2190    }
2191
2192
2193    // An LDIF change record may contain zero or more controls, with the end of
2194    // the controls signified by the changetype.  The changetype element must be
2195    // present, unless defaultAdd is true in which case the first thing that is
2196    // neither control or changetype will trigger the start of add attribute
2197    // parsing.
2198    if (! iterator.hasNext())
2199    {
2200      throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2201                              firstLineNumber, true, ldifLines, null);
2202    }
2203
2204    String changeType = null;
2205    ArrayList<Control> controls = null;
2206    while (true)
2207    {
2208      line = iterator.next();
2209      handleTrailingSpaces(line, dn, firstLineNumber,
2210           unparsedRecord.getTrailingSpaceBehavior());
2211      colonPos = line.indexOf(":");
2212      if (colonPos < 0)
2213      {
2214        throw new LDIFException(
2215             ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2216             firstLineNumber, true, ldifLines, null);
2217      }
2218
2219      final String token = toLowerCase(line.substring(0, colonPos));
2220      if (token.equals("control"))
2221      {
2222        if (controls == null)
2223        {
2224          controls = new ArrayList<Control>(5);
2225        }
2226
2227        controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2228             relativeBasePath));
2229      }
2230      else if (token.equals("changetype"))
2231      {
2232        changeType =
2233             decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2234        break;
2235      }
2236      else if (defaultAdd)
2237      {
2238        // The line we read wasn't a control or changetype declaration, so we'll
2239        // assume it's an attribute in an add record.  However, we're not ready
2240        // for that yet, and since we can't rewind an iterator we'll create a
2241        // new one that hasn't yet gotten to this line.
2242        changeType = "add";
2243        iterator = ldifLines.iterator();
2244        for (int i=0; i < linesRead; i++)
2245        {
2246          iterator.next();
2247        }
2248        break;
2249      }
2250      else
2251      {
2252        throw new LDIFException(
2253             ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2254                  firstLineNumber),
2255             firstLineNumber, true, ldifLines, null);
2256      }
2257
2258      linesRead++;
2259    }
2260
2261
2262    // Make sure that the change type is acceptable and then decode the rest of
2263    // the change record accordingly.
2264    final String lowerChangeType = toLowerCase(changeType);
2265    if (lowerChangeType.equals("add"))
2266    {
2267      // There must be at least one more line.  If not, then that's an error.
2268      // Otherwise, parse the rest of the data as attribute-value pairs.
2269      if (iterator.hasNext())
2270      {
2271        final Collection<Attribute> attrs =
2272             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2273                  unparsedRecord.getTrailingSpaceBehavior(),
2274                  unparsedRecord.getSchema(), ldifLines, iterator,
2275                  relativeBasePath, firstLineNumber);
2276        final Attribute[] attributes = new Attribute[attrs.size()];
2277        final Iterator<Attribute> attrIterator = attrs.iterator();
2278        for (int i=0; i < attributes.length; i++)
2279        {
2280          attributes[i] = attrIterator.next();
2281        }
2282
2283        return new LDIFAddChangeRecord(dn, attributes, controls);
2284      }
2285      else
2286      {
2287        throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2288                                firstLineNumber, true, ldifLines, null);
2289      }
2290    }
2291    else if (lowerChangeType.equals("delete"))
2292    {
2293      // There shouldn't be any more data.  If there is, then that's an error.
2294      // Otherwise, we can just return the delete change record with what we
2295      // already know.
2296      if (iterator.hasNext())
2297      {
2298        throw new LDIFException(
2299                       ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2300                       firstLineNumber, true, ldifLines, null);
2301      }
2302      else
2303      {
2304        return new LDIFDeleteChangeRecord(dn, controls);
2305      }
2306    }
2307    else if (lowerChangeType.equals("modify"))
2308    {
2309      // There must be at least one more line.  If not, then that's an error.
2310      // Otherwise, parse the rest of the data as a set of modifications.
2311      if (iterator.hasNext())
2312      {
2313        final Modification[] mods = parseModifications(dn,
2314             unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2315             firstLineNumber);
2316        return new LDIFModifyChangeRecord(dn, mods, controls);
2317      }
2318      else
2319      {
2320        throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2321                                firstLineNumber, true, ldifLines, null);
2322      }
2323    }
2324    else if (lowerChangeType.equals("moddn") ||
2325             lowerChangeType.equals("modrdn"))
2326    {
2327      // There must be at least one more line.  If not, then that's an error.
2328      // Otherwise, parse the rest of the data as a set of modifications.
2329      if (iterator.hasNext())
2330      {
2331        return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2332             unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2333      }
2334      else
2335      {
2336        throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2337                                firstLineNumber, true, ldifLines, null);
2338      }
2339    }
2340    else
2341    {
2342      throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2343                                                         firstLineNumber),
2344                              firstLineNumber, true, ldifLines, null);
2345    }
2346  }
2347
2348
2349
2350  /**
2351   * Decodes information about a control from the provided line.
2352   *
2353   * @param  line              The line to process.
2354   * @param  colonPos          The position of the colon that separates the
2355   *                           control token string from tbe encoded control.
2356   * @param  firstLineNumber   The line number for the start of the record.
2357   * @param  ldifLines         The lines that comprise the LDIF representation
2358   *                           of the full record being parsed.
2359   * @param  relativeBasePath  The base path that will be prepended to relative
2360   *                           paths in order to obtain an absolute path.
2361   *
2362   * @return  The decoded control.
2363   *
2364   * @throws  LDIFException  If a problem is encountered while trying to decode
2365   *                         the changetype.
2366   */
2367  private static Control decodeControl(final StringBuilder line,
2368                                       final int colonPos,
2369                                       final long firstLineNumber,
2370                                       final ArrayList<StringBuilder> ldifLines,
2371                                       final String relativeBasePath)
2372          throws LDIFException
2373  {
2374    final String controlString;
2375    int length = line.length();
2376    if (length == (colonPos+1))
2377    {
2378      // The colon was the last character on the line.  This is not
2379      // acceptable.
2380      throw new LDIFException(
2381           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2382           firstLineNumber, true, ldifLines, null);
2383    }
2384    else if (line.charAt(colonPos+1) == ':')
2385    {
2386      // Skip over any spaces leading up to the value, and then the rest of
2387      // the string is the base64-encoded control representation.  This is
2388      // unusual and unnecessary, but is nevertheless acceptable.
2389      int pos = colonPos+2;
2390      while ((pos < length) && (line.charAt(pos) == ' '))
2391      {
2392        pos++;
2393      }
2394
2395      try
2396      {
2397        final byte[] controlBytes = Base64.decode(line.substring(pos));
2398        controlString =  new String(controlBytes, "UTF-8");
2399      }
2400      catch (final ParseException pe)
2401      {
2402        debugException(pe);
2403        throw new LDIFException(
2404                       ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2405                            firstLineNumber, pe.getMessage()),
2406                       firstLineNumber, true, ldifLines, pe);
2407      }
2408      catch (final Exception e)
2409      {
2410        debugException(e);
2411        throw new LDIFException(
2412             ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2413             firstLineNumber, true, ldifLines, e);
2414      }
2415    }
2416    else
2417    {
2418      // Skip over any spaces leading up to the value, and then the rest of
2419      // the string is the encoded control.
2420      int pos = colonPos+1;
2421      while ((pos < length) && (line.charAt(pos) == ' '))
2422      {
2423        pos++;
2424      }
2425
2426      controlString = line.substring(pos);
2427    }
2428
2429    // If the resulting control definition is empty, then that's invalid.
2430    if (controlString.length() == 0)
2431    {
2432      throw new LDIFException(
2433           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2434           firstLineNumber, true, ldifLines, null);
2435    }
2436
2437
2438    // The first element of the control must be the OID, and it must be followed
2439    // by a space (to separate it from the criticality), a colon (to separate it
2440    // from the value and indicate a default criticality of false), or the end
2441    // of the line (to indicate a default criticality of false and no value).
2442    String oid = null;
2443    boolean hasCriticality = false;
2444    boolean hasValue = false;
2445    int pos = 0;
2446    length = controlString.length();
2447    while (pos < length)
2448    {
2449      final char c = controlString.charAt(pos);
2450      if (c == ':')
2451      {
2452        // This indicates that there is no criticality and that the value
2453        // immediately follows the OID.
2454        oid = controlString.substring(0, pos++);
2455        hasValue = true;
2456        break;
2457      }
2458      else if (c == ' ')
2459      {
2460        // This indicates that there is a criticality.  We don't know anything
2461        // about the presence of a value yet.
2462        oid = controlString.substring(0, pos++);
2463        hasCriticality = true;
2464        break;
2465      }
2466      else
2467      {
2468        pos++;
2469      }
2470    }
2471
2472    if (oid == null)
2473    {
2474      // This indicates that the string representation of the control is only
2475      // the OID.
2476      return new Control(controlString, false);
2477    }
2478
2479
2480    // See if we need to read the criticality.  If so, then do so now.
2481    // Otherwise, assume a default criticality of false.
2482    final boolean isCritical;
2483    if (hasCriticality)
2484    {
2485      // Skip over any spaces before the criticality.
2486      while (controlString.charAt(pos) == ' ')
2487      {
2488        pos++;
2489      }
2490
2491      // Read until we find a colon or the end of the string.
2492      final int criticalityStartPos = pos;
2493      while (pos < length)
2494      {
2495        final char c = controlString.charAt(pos);
2496        if (c == ':')
2497        {
2498          hasValue = true;
2499          break;
2500        }
2501        else
2502        {
2503          pos++;
2504        }
2505      }
2506
2507      final String criticalityString =
2508           toLowerCase(controlString.substring(criticalityStartPos, pos));
2509      if (criticalityString.equals("true"))
2510      {
2511        isCritical = true;
2512      }
2513      else if (criticalityString.equals("false"))
2514      {
2515        isCritical = false;
2516      }
2517      else
2518      {
2519        throw new LDIFException(
2520             ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2521                  firstLineNumber),
2522             firstLineNumber, true, ldifLines, null);
2523      }
2524
2525      if (hasValue)
2526      {
2527        pos++;
2528      }
2529    }
2530    else
2531    {
2532      isCritical = false;
2533    }
2534
2535    // See if we need to read the value.  If so, then do so now.  It may be
2536    // a string, or it may be base64-encoded.  It could conceivably even be read
2537    // from a URL.
2538    final ASN1OctetString value;
2539    if (hasValue)
2540    {
2541      // The character immediately after the colon that precedes the value may
2542      // be one of the following:
2543      // - A second colon (optionally followed by a single space) to indicate
2544      //   that the value is base64-encoded.
2545      // - A less-than symbol to indicate that the value should be read from a
2546      //   location specified by a URL.
2547      // - A single space that precedes the non-base64-encoded value.
2548      // - The first character of the non-base64-encoded value.
2549      switch (controlString.charAt(pos))
2550      {
2551        case ':':
2552          try
2553          {
2554            if (controlString.length() == (pos+1))
2555            {
2556              value = new ASN1OctetString();
2557            }
2558            else if (controlString.charAt(pos+1) == ' ')
2559            {
2560              value = new ASN1OctetString(
2561                   Base64.decode(controlString.substring(pos+2)));
2562            }
2563            else
2564            {
2565              value = new ASN1OctetString(
2566                   Base64.decode(controlString.substring(pos+1)));
2567            }
2568          }
2569          catch (final Exception e)
2570          {
2571            debugException(e);
2572            throw new LDIFException(
2573                 ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2574                      firstLineNumber, getExceptionMessage(e)),
2575                 firstLineNumber, true, ldifLines, e);
2576          }
2577          break;
2578        case '<':
2579          try
2580          {
2581            final String urlString;
2582            if (controlString.charAt(pos+1) == ' ')
2583            {
2584              urlString = controlString.substring(pos+2);
2585            }
2586            else
2587            {
2588              urlString = controlString.substring(pos+1);
2589            }
2590            value = new ASN1OctetString(retrieveURLBytes(urlString,
2591                 relativeBasePath, firstLineNumber));
2592          }
2593          catch (final Exception e)
2594          {
2595            debugException(e);
2596            throw new LDIFException(
2597                 ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2598                      firstLineNumber, getExceptionMessage(e)),
2599                 firstLineNumber, true, ldifLines, e);
2600          }
2601          break;
2602        case ' ':
2603          value = new ASN1OctetString(controlString.substring(pos+1));
2604          break;
2605        default:
2606          value = new ASN1OctetString(controlString.substring(pos));
2607          break;
2608      }
2609    }
2610    else
2611    {
2612      value = null;
2613    }
2614
2615    return new Control(oid, isCritical, value);
2616  }
2617
2618
2619
2620  /**
2621   * Decodes the changetype element from the provided line.
2622   *
2623   * @param  line             The line to process.
2624   * @param  colonPos         The position of the colon that separates the
2625   *                          changetype string from its value.
2626   * @param  firstLineNumber  The line number for the start of the record.
2627   * @param  ldifLines        The lines that comprise the LDIF representation of
2628   *                          the full record being parsed.
2629   *
2630   * @return  The decoded changetype string.
2631   *
2632   * @throws  LDIFException  If a problem is encountered while trying to decode
2633   *                         the changetype.
2634   */
2635  private static String decodeChangeType(final StringBuilder line,
2636                             final int colonPos, final long firstLineNumber,
2637                             final ArrayList<StringBuilder> ldifLines)
2638          throws LDIFException
2639  {
2640    final int length = line.length();
2641    if (length == (colonPos+1))
2642    {
2643      // The colon was the last character on the line.  This is not
2644      // acceptable.
2645      throw new LDIFException(
2646           ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2647           true, ldifLines, null);
2648    }
2649    else if (line.charAt(colonPos+1) == ':')
2650    {
2651      // Skip over any spaces leading up to the value, and then the rest of
2652      // the string is the base64-encoded changetype.  This is unusual and
2653      // unnecessary, but is nevertheless acceptable.
2654      int pos = colonPos+2;
2655      while ((pos < length) && (line.charAt(pos) == ' '))
2656      {
2657        pos++;
2658      }
2659
2660      try
2661      {
2662        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
2663        return new String(changeTypeBytes, "UTF-8");
2664      }
2665      catch (final ParseException pe)
2666      {
2667        debugException(pe);
2668        throw new LDIFException(
2669                       ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
2670                                                            pe.getMessage()),
2671                       firstLineNumber, true, ldifLines, pe);
2672      }
2673      catch (final Exception e)
2674      {
2675        debugException(e);
2676        throw new LDIFException(
2677             ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
2678             firstLineNumber, true, ldifLines, e);
2679      }
2680    }
2681    else
2682    {
2683      // Skip over any spaces leading up to the value, and then the rest of
2684      // the string is the changetype.
2685      int pos = colonPos+1;
2686      while ((pos < length) && (line.charAt(pos) == ' '))
2687      {
2688        pos++;
2689      }
2690
2691      return line.substring(pos);
2692    }
2693  }
2694
2695
2696
2697  /**
2698   * Parses the data available through the provided iterator as a collection of
2699   * attributes suitable for use in an entry or an add change record.
2700   *
2701   * @param  dn                      The DN of the record being read.
2702   * @param  duplicateValueBehavior  The behavior that should be exhibited if
2703   *                                 the LDIF reader encounters an entry with
2704   *                                 duplicate values.
2705   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2706   *                                 encountering attribute values which are not
2707   *                                 base64-encoded but contain trailing spaces.
2708   * @param  schema                  The schema to use when parsing the
2709   *                                 attributes, or {@code null} if none is
2710   *                                 needed.
2711   * @param  ldifLines               The lines that comprise the LDIF
2712   *                                 representation of the full record being
2713   *                                 parsed.
2714   * @param  iterator                The iterator to use to access the attribute
2715   *                                 lines.
2716   * @param  relativeBasePath        The base path that will be prepended to
2717   *                                 relative paths in order to obtain an
2718   *                                 absolute path.
2719   * @param  firstLineNumber         The line number for the start of the
2720   *                                 record.
2721   *
2722   * @return  The collection of attributes that were read.
2723   *
2724   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2725   *                         set of attributes.
2726   */
2727  private static ArrayList<Attribute> parseAttributes(final String dn,
2728       final DuplicateValueBehavior duplicateValueBehavior,
2729       final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
2730       final ArrayList<StringBuilder> ldifLines,
2731       final Iterator<StringBuilder> iterator, final String relativeBasePath,
2732       final long firstLineNumber)
2733          throws LDIFException
2734  {
2735    final LinkedHashMap<String,Object> attributes =
2736         new LinkedHashMap<String,Object>(ldifLines.size());
2737    while (iterator.hasNext())
2738    {
2739      final StringBuilder line = iterator.next();
2740      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
2741      final int colonPos = line.indexOf(":");
2742      if (colonPos <= 0)
2743      {
2744        throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
2745                                firstLineNumber, true, ldifLines, null);
2746      }
2747
2748      final String attributeName = line.substring(0, colonPos);
2749      final String lowerName     = toLowerCase(attributeName);
2750
2751      final MatchingRule matchingRule;
2752      if (schema == null)
2753      {
2754        matchingRule = CaseIgnoreStringMatchingRule.getInstance();
2755      }
2756      else
2757      {
2758        matchingRule =
2759             MatchingRule.selectEqualityMatchingRule(attributeName, schema);
2760      }
2761
2762      Attribute attr;
2763      final LDIFAttribute ldifAttr;
2764      final Object attrObject = attributes.get(lowerName);
2765      if (attrObject == null)
2766      {
2767        attr     = null;
2768        ldifAttr = null;
2769      }
2770      else
2771      {
2772        if (attrObject instanceof Attribute)
2773        {
2774          attr     = (Attribute) attrObject;
2775          ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
2776                                       attr.getRawValues()[0]);
2777          attributes.put(lowerName, ldifAttr);
2778        }
2779        else
2780        {
2781          attr     = null;
2782          ldifAttr = (LDIFAttribute) attrObject;
2783        }
2784      }
2785
2786      final int length = line.length();
2787      if (length == (colonPos+1))
2788      {
2789        // This means that the attribute has a zero-length value, which is
2790        // acceptable.
2791        if (attrObject == null)
2792        {
2793          attr = new Attribute(attributeName, "");
2794          attributes.put(lowerName, attr);
2795        }
2796        else
2797        {
2798          try
2799          {
2800            if (! ldifAttr.addValue(new ASN1OctetString(),
2801                       duplicateValueBehavior))
2802            {
2803              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2804              {
2805                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2806                     firstLineNumber, attributeName), firstLineNumber, true,
2807                     ldifLines, null);
2808              }
2809            }
2810          }
2811          catch (LDAPException le)
2812          {
2813            throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2814                 firstLineNumber, attributeName, getExceptionMessage(le)),
2815                 firstLineNumber, true, ldifLines, le);
2816          }
2817        }
2818      }
2819      else if (line.charAt(colonPos+1) == ':')
2820      {
2821        // Skip over any spaces leading up to the value, and then the rest of
2822        // the string is the base64-encoded attribute value.
2823        int pos = colonPos+2;
2824        while ((pos < length) && (line.charAt(pos) == ' '))
2825        {
2826          pos++;
2827        }
2828
2829        try
2830        {
2831          final byte[] valueBytes = Base64.decode(line.substring(pos));
2832          if (attrObject == null)
2833          {
2834            attr = new Attribute(attributeName, valueBytes);
2835            attributes.put(lowerName, attr);
2836          }
2837          else
2838          {
2839            try
2840            {
2841              if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
2842                         duplicateValueBehavior))
2843              {
2844                if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2845                {
2846                  throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2847                       firstLineNumber, attributeName), firstLineNumber, true,
2848                       ldifLines, null);
2849                }
2850              }
2851            }
2852            catch (LDAPException le)
2853            {
2854              throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2855                   firstLineNumber, attributeName, getExceptionMessage(le)),
2856                   firstLineNumber, true, ldifLines, le);
2857            }
2858          }
2859        }
2860        catch (final ParseException pe)
2861        {
2862          debugException(pe);
2863          throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
2864                                       attributeName,  firstLineNumber,
2865                                       pe.getMessage()),
2866                                  firstLineNumber, true, ldifLines, pe);
2867        }
2868      }
2869      else if (line.charAt(colonPos+1) == '<')
2870      {
2871        // Skip over any spaces leading up to the value, and then the rest of
2872        // the string is a URL that indicates where to get the real content.
2873        // At the present time, we'll only support the file URLs.
2874        int pos = colonPos+2;
2875        while ((pos < length) && (line.charAt(pos) == ' '))
2876        {
2877          pos++;
2878        }
2879
2880        final byte[] urlBytes;
2881        final String urlString = line.substring(pos);
2882        try
2883        {
2884          urlBytes =
2885               retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
2886        }
2887        catch (final Exception e)
2888        {
2889          debugException(e);
2890          throw new LDIFException(
2891               ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2892                    firstLineNumber, e),
2893               firstLineNumber, true, ldifLines, e);
2894        }
2895
2896        if (attrObject == null)
2897        {
2898          attr = new Attribute(attributeName, urlBytes);
2899          attributes.put(lowerName, attr);
2900        }
2901        else
2902        {
2903          try
2904          {
2905            if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
2906                 duplicateValueBehavior))
2907            {
2908              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2909              {
2910                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2911                     firstLineNumber, attributeName), firstLineNumber, true,
2912                     ldifLines, null);
2913              }
2914            }
2915          }
2916          catch (final LDIFException le)
2917          {
2918            debugException(le);
2919            throw le;
2920          }
2921          catch (final Exception e)
2922          {
2923            debugException(e);
2924            throw new LDIFException(
2925                 ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2926                      firstLineNumber, e),
2927                 firstLineNumber, true, ldifLines, e);
2928          }
2929        }
2930      }
2931      else
2932      {
2933        // Skip over any spaces leading up to the value, and then the rest of
2934        // the string is the value.
2935        int pos = colonPos+1;
2936        while ((pos < length) && (line.charAt(pos) == ' '))
2937        {
2938          pos++;
2939        }
2940
2941        final String valueString = line.substring(pos);
2942        if (attrObject == null)
2943        {
2944          attr = new Attribute(attributeName, valueString);
2945          attributes.put(lowerName, attr);
2946        }
2947        else
2948        {
2949          try
2950          {
2951            if (! ldifAttr.addValue(new ASN1OctetString(valueString),
2952                       duplicateValueBehavior))
2953            {
2954              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2955              {
2956                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2957                     firstLineNumber, attributeName), firstLineNumber, true,
2958                     ldifLines, null);
2959              }
2960            }
2961          }
2962          catch (LDAPException le)
2963          {
2964            throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2965                 firstLineNumber, attributeName, getExceptionMessage(le)),
2966                 firstLineNumber, true, ldifLines, le);
2967          }
2968        }
2969      }
2970    }
2971
2972    final ArrayList<Attribute> attrList =
2973         new ArrayList<Attribute>(attributes.size());
2974    for (final Object o : attributes.values())
2975    {
2976      if (o instanceof Attribute)
2977      {
2978        attrList.add((Attribute) o);
2979      }
2980      else
2981      {
2982        attrList.add(((LDIFAttribute) o).toAttribute());
2983      }
2984    }
2985
2986    return attrList;
2987  }
2988
2989
2990
2991  /**
2992   * Retrieves the bytes that make up the file referenced by the given URL.
2993   *
2994   * @param  urlString         The string representation of the URL to retrieve.
2995   * @param  relativeBasePath  The base path that will be prepended to relative
2996   *                           paths in order to obtain an absolute path.
2997   * @param  firstLineNumber   The line number for the start of the record.
2998   *
2999   * @return  The bytes contained in the specified file, or an empty array if
3000   *          the specified file is empty.
3001   *
3002   * @throws  LDIFException  If the provided URL is malformed or references a
3003   *                         nonexistent file.
3004   *
3005   * @throws  IOException  If a problem is encountered while attempting to read
3006   *                       from the target file.
3007   */
3008  private static byte[] retrieveURLBytes(final String urlString,
3009                                         final String relativeBasePath,
3010                                         final long firstLineNumber)
3011          throws LDIFException, IOException
3012  {
3013    int pos;
3014    String path;
3015    final String lowerURLString = toLowerCase(urlString);
3016    if (lowerURLString.startsWith("file:/"))
3017    {
3018      pos = 6;
3019      while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3020      {
3021        pos++;
3022      }
3023
3024      path = urlString.substring(pos-1);
3025    }
3026    else if (lowerURLString.startsWith("file:"))
3027    {
3028      // A file: URL that doesn't include a slash will be interpreted as a
3029      // relative path.
3030      path = relativeBasePath + urlString.substring(5);
3031    }
3032    else
3033    {
3034      throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3035           firstLineNumber, true);
3036    }
3037
3038    final File f = new File(path);
3039    if (! f.exists())
3040    {
3041      throw new LDIFException(
3042           ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3043           firstLineNumber, true);
3044    }
3045
3046    // In order to conserve memory, we'll only allow values to be read from
3047    // files no larger than 10 megabytes.
3048    final long fileSize = f.length();
3049    if (fileSize > (10 * 1024 * 1024))
3050    {
3051      throw new LDIFException(
3052           ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3053                (10*1024*1024)),
3054           firstLineNumber, true);
3055    }
3056
3057    int fileBytesRemaining = (int) fileSize;
3058    final byte[] fileData = new byte[(int) fileSize];
3059    final FileInputStream fis = new FileInputStream(f);
3060    try
3061    {
3062      int fileBytesRead = 0;
3063      while (fileBytesRead < fileSize)
3064      {
3065        final int bytesRead =
3066             fis.read(fileData, fileBytesRead, fileBytesRemaining);
3067        if (bytesRead < 0)
3068        {
3069          // We hit the end of the file before we expected to.  This shouldn't
3070          // happen unless the file size changed since we first looked at it,
3071          // which we won't allow.
3072          throw new LDIFException(
3073               ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3074                    f.getAbsolutePath()),
3075               firstLineNumber, true);
3076        }
3077
3078        fileBytesRead      += bytesRead;
3079        fileBytesRemaining -= bytesRead;
3080      }
3081
3082      if (fis.read() != -1)
3083      {
3084        // There is still more data to read.  This shouldn't happen unless the
3085        // file size changed since we first looked at it, which we won't allow.
3086        throw new LDIFException(
3087             ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3088             firstLineNumber, true);
3089      }
3090    }
3091    finally
3092    {
3093      fis.close();
3094    }
3095
3096    return fileData;
3097  }
3098
3099
3100
3101  /**
3102   * Parses the data available through the provided iterator into an array of
3103   * modifications suitable for use in a modify change record.
3104   *
3105   * @param  dn                     The DN of the entry being parsed.
3106   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3107   *                                encountering attribute values which are not
3108   *                                base64-encoded but contain trailing spaces.
3109   * @param  ldifLines              The lines that comprise the LDIF
3110   *                                representation of the full record being
3111   *                                parsed.
3112   * @param  iterator               The iterator to use to access the
3113   *                                modification data.
3114   * @param  firstLineNumber        The line number for the start of the record.
3115   *
3116   * @return  An array containing the modifications that were read.
3117   *
3118   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3119   *                         set of modifications.
3120   */
3121  private static Modification[] parseModifications(final String dn,
3122       final TrailingSpaceBehavior trailingSpaceBehavior,
3123       final ArrayList<StringBuilder> ldifLines,
3124       final Iterator<StringBuilder> iterator, final long firstLineNumber)
3125       throws LDIFException
3126  {
3127    final ArrayList<Modification> modList =
3128         new ArrayList<Modification>(ldifLines.size());
3129
3130    while (iterator.hasNext())
3131    {
3132      // The first line must start with "add:", "delete:", "replace:", or
3133      // "increment:" followed by an attribute name.
3134      StringBuilder line = iterator.next();
3135      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3136      int colonPos = line.indexOf(":");
3137      if (colonPos < 0)
3138      {
3139        throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3140                                firstLineNumber, true, ldifLines, null);
3141      }
3142
3143      final ModificationType modType;
3144      final String modTypeStr = toLowerCase(line.substring(0, colonPos));
3145      if (modTypeStr.equals("add"))
3146      {
3147        modType = ModificationType.ADD;
3148      }
3149      else if (modTypeStr.equals("delete"))
3150      {
3151        modType = ModificationType.DELETE;
3152      }
3153      else if (modTypeStr.equals("replace"))
3154      {
3155        modType = ModificationType.REPLACE;
3156      }
3157      else if (modTypeStr.equals("increment"))
3158      {
3159        modType = ModificationType.INCREMENT;
3160      }
3161      else
3162      {
3163        throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3164                                     firstLineNumber),
3165                                firstLineNumber, true, ldifLines, null);
3166      }
3167
3168      final String attributeName;
3169      int length = line.length();
3170      if (length == (colonPos+1))
3171      {
3172        // The colon was the last character on the line.  This is not
3173        // acceptable.
3174        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3175                                     firstLineNumber),
3176                                firstLineNumber, true, ldifLines, null);
3177      }
3178      else if (line.charAt(colonPos+1) == ':')
3179      {
3180        // Skip over any spaces leading up to the value, and then the rest of
3181        // the string is the base64-encoded attribute name.
3182        int pos = colonPos+2;
3183        while ((pos < length) && (line.charAt(pos) == ' '))
3184        {
3185          pos++;
3186        }
3187
3188        try
3189        {
3190          final byte[] dnBytes = Base64.decode(line.substring(pos));
3191          attributeName = new String(dnBytes, "UTF-8");
3192        }
3193        catch (final ParseException pe)
3194        {
3195          debugException(pe);
3196          throw new LDIFException(
3197               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3198                    firstLineNumber, pe.getMessage()),
3199               firstLineNumber, true, ldifLines, pe);
3200        }
3201        catch (final Exception e)
3202        {
3203          debugException(e);
3204          throw new LDIFException(
3205               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3206                    firstLineNumber, e),
3207               firstLineNumber, true, ldifLines, e);
3208        }
3209      }
3210      else
3211      {
3212        // Skip over any spaces leading up to the value, and then the rest of
3213        // the string is the attribute name.
3214        int pos = colonPos+1;
3215        while ((pos < length) && (line.charAt(pos) == ' '))
3216        {
3217          pos++;
3218        }
3219
3220        attributeName = line.substring(pos);
3221      }
3222
3223      if (attributeName.length() == 0)
3224      {
3225        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3226                                     firstLineNumber),
3227                                firstLineNumber, true, ldifLines, null);
3228      }
3229
3230
3231      // The next zero or more lines may be the set of attribute values.  Keep
3232      // reading until we reach the end of the iterator or until we find a line
3233      // with just a "-".
3234      final ArrayList<ASN1OctetString> valueList =
3235           new ArrayList<ASN1OctetString>(ldifLines.size());
3236      while (iterator.hasNext())
3237      {
3238        line = iterator.next();
3239        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3240        if (line.toString().equals("-"))
3241        {
3242          break;
3243        }
3244
3245        colonPos = line.indexOf(":");
3246        if (colonPos < 0)
3247        {
3248          throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3249                                  firstLineNumber, true, ldifLines, null);
3250        }
3251        else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3252        {
3253          throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3254                                       firstLineNumber,
3255                                       line.substring(0, colonPos),
3256                                       attributeName),
3257                                  firstLineNumber, true, ldifLines, null);
3258        }
3259
3260        final ASN1OctetString value;
3261        length = line.length();
3262        if (length == (colonPos+1))
3263        {
3264          // The colon was the last character on the line.  This is fine.
3265          value = new ASN1OctetString();
3266        }
3267        else if (line.charAt(colonPos+1) == ':')
3268        {
3269          // Skip over any spaces leading up to the value, and then the rest of
3270          // the string is the base64-encoded value.  This is unusual and
3271          // unnecessary, but is nevertheless acceptable.
3272          int pos = colonPos+2;
3273          while ((pos < length) && (line.charAt(pos) == ' '))
3274          {
3275            pos++;
3276          }
3277
3278          try
3279          {
3280            value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3281          }
3282          catch (final ParseException pe)
3283          {
3284            debugException(pe);
3285            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3286                 attributeName, firstLineNumber, pe.getMessage()),
3287                 firstLineNumber, true, ldifLines, pe);
3288          }
3289          catch (final Exception e)
3290          {
3291            debugException(e);
3292            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3293                                         firstLineNumber, e),
3294                                    firstLineNumber, true, ldifLines, e);
3295          }
3296        }
3297        else
3298        {
3299          // Skip over any spaces leading up to the value, and then the rest of
3300          // the string is the value.
3301          int pos = colonPos+1;
3302          while ((pos < length) && (line.charAt(pos) == ' '))
3303          {
3304            pos++;
3305          }
3306
3307          value = new ASN1OctetString(line.substring(pos));
3308        }
3309
3310        valueList.add(value);
3311      }
3312
3313      final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3314      valueList.toArray(values);
3315
3316      // If it's an add modification type, then there must be at least one
3317      // value.
3318      if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3319          (values.length == 0))
3320      {
3321        throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3322                                     firstLineNumber),
3323                                firstLineNumber, true, ldifLines, null);
3324      }
3325
3326      // If it's an increment modification type, then there must be exactly one
3327      // value.
3328      if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3329          (values.length != 1))
3330      {
3331        throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3332                                     firstLineNumber, attributeName),
3333                                firstLineNumber, true, ldifLines, null);
3334      }
3335
3336      modList.add(new Modification(modType, attributeName, values));
3337    }
3338
3339    final Modification[] mods = new Modification[modList.size()];
3340    modList.toArray(mods);
3341    return mods;
3342  }
3343
3344
3345
3346  /**
3347   * Parses the data available through the provided iterator as the body of a
3348   * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3349   * newsuperior lines).
3350   *
3351   * @param  ldifLines              The lines that comprise the LDIF
3352   *                                representation of the full record being
3353   *                                parsed.
3354   * @param  iterator               The iterator to use to access the modify DN
3355   *                                data.
3356   * @param  dn                     The current DN of the entry.
3357   * @param  controls               The set of controls to include in the change
3358   *                                record.
3359   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3360   *                                encountering attribute values which are not
3361   *                                base64-encoded but contain trailing spaces.
3362   * @param  firstLineNumber        The line number for the start of the record.
3363   *
3364   * @return  The decoded modify DN change record.
3365   *
3366   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3367   *                         modify DN change record.
3368   */
3369  private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3370       final ArrayList<StringBuilder> ldifLines,
3371       final Iterator<StringBuilder> iterator, final String dn,
3372       final List<Control> controls,
3373       final TrailingSpaceBehavior trailingSpaceBehavior,
3374       final long firstLineNumber)
3375       throws LDIFException
3376  {
3377    // The next line must be the new RDN, and it must start with "newrdn:".
3378    StringBuilder line = iterator.next();
3379    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3380    int colonPos = line.indexOf(":");
3381    if ((colonPos < 0) ||
3382        (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3383    {
3384      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3385                                   firstLineNumber),
3386                              firstLineNumber, true, ldifLines, null);
3387    }
3388
3389    final String newRDN;
3390    int length = line.length();
3391    if (length == (colonPos+1))
3392    {
3393      // The colon was the last character on the line.  This is not acceptable.
3394      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3395                                   firstLineNumber),
3396                              firstLineNumber, true, ldifLines, null);
3397    }
3398    else if (line.charAt(colonPos+1) == ':')
3399    {
3400      // Skip over any spaces leading up to the value, and then the rest of the
3401      // string is the base64-encoded new RDN.
3402      int pos = colonPos+2;
3403      while ((pos < length) && (line.charAt(pos) == ' '))
3404      {
3405        pos++;
3406      }
3407
3408      try
3409      {
3410        final byte[] dnBytes = Base64.decode(line.substring(pos));
3411        newRDN = new String(dnBytes, "UTF-8");
3412      }
3413      catch (final ParseException pe)
3414      {
3415        debugException(pe);
3416        throw new LDIFException(
3417             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3418                                                               pe.getMessage()),
3419             firstLineNumber, true, ldifLines, pe);
3420      }
3421      catch (final Exception e)
3422      {
3423        debugException(e);
3424        throw new LDIFException(
3425             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3426                                                               e),
3427             firstLineNumber, true, ldifLines, e);
3428      }
3429    }
3430    else
3431    {
3432      // Skip over any spaces leading up to the value, and then the rest of the
3433      // string is the new RDN.
3434      int pos = colonPos+1;
3435      while ((pos < length) && (line.charAt(pos) == ' '))
3436      {
3437        pos++;
3438      }
3439
3440      newRDN = line.substring(pos);
3441    }
3442
3443    if (newRDN.length() == 0)
3444    {
3445      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3446                                   firstLineNumber),
3447                              firstLineNumber, true, ldifLines, null);
3448    }
3449
3450
3451    // The next line must be the deleteOldRDN flag, and it must start with
3452    // 'deleteoldrdn:'.
3453    if (! iterator.hasNext())
3454    {
3455      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3456                                   firstLineNumber),
3457                              firstLineNumber, true, ldifLines, null);
3458    }
3459
3460    line = iterator.next();
3461    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3462    colonPos = line.indexOf(":");
3463    if ((colonPos < 0) ||
3464        (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3465    {
3466      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3467                                   firstLineNumber),
3468                              firstLineNumber, true, ldifLines, null);
3469    }
3470
3471    final String deleteOldRDNStr;
3472    length = line.length();
3473    if (length == (colonPos+1))
3474    {
3475      // The colon was the last character on the line.  This is not acceptable.
3476      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3477                                   firstLineNumber),
3478                              firstLineNumber, true, ldifLines, null);
3479    }
3480    else if (line.charAt(colonPos+1) == ':')
3481    {
3482      // Skip over any spaces leading up to the value, and then the rest of the
3483      // string is the base64-encoded value.  This is unusual and
3484      // unnecessary, but is nevertheless acceptable.
3485      int pos = colonPos+2;
3486      while ((pos < length) && (line.charAt(pos) == ' '))
3487      {
3488        pos++;
3489      }
3490
3491      try
3492      {
3493        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3494        deleteOldRDNStr = new String(changeTypeBytes, "UTF-8");
3495      }
3496      catch (final ParseException pe)
3497      {
3498        debugException(pe);
3499        throw new LDIFException(
3500             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3501                  firstLineNumber, pe.getMessage()),
3502             firstLineNumber, true, ldifLines, pe);
3503      }
3504      catch (final Exception e)
3505      {
3506        debugException(e);
3507        throw new LDIFException(
3508             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3509                  firstLineNumber, e),
3510             firstLineNumber, true, ldifLines, e);
3511      }
3512    }
3513    else
3514    {
3515      // Skip over any spaces leading up to the value, and then the rest of the
3516      // string is the value.
3517      int pos = colonPos+1;
3518      while ((pos < length) && (line.charAt(pos) == ' '))
3519      {
3520        pos++;
3521      }
3522
3523      deleteOldRDNStr = line.substring(pos);
3524    }
3525
3526    final boolean deleteOldRDN;
3527    if (deleteOldRDNStr.equals("0"))
3528    {
3529      deleteOldRDN = false;
3530    }
3531    else if (deleteOldRDNStr.equals("1"))
3532    {
3533      deleteOldRDN = true;
3534    }
3535    else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3536             deleteOldRDNStr.equalsIgnoreCase("no"))
3537    {
3538      // This is technically illegal, but we'll allow it.
3539      deleteOldRDN = false;
3540    }
3541    else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3542             deleteOldRDNStr.equalsIgnoreCase("yes"))
3543    {
3544      // This is also technically illegal, but we'll allow it.
3545      deleteOldRDN = false;
3546    }
3547    else
3548    {
3549      throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3550                                   deleteOldRDNStr, firstLineNumber),
3551                              firstLineNumber, true, ldifLines, null);
3552    }
3553
3554
3555    // If there is another line, then it must be the new superior DN and it must
3556    // start with "newsuperior:".  If this is absent, then it's fine.
3557    final String newSuperiorDN;
3558    if (iterator.hasNext())
3559    {
3560      line = iterator.next();
3561      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3562      colonPos = line.indexOf(":");
3563      if ((colonPos < 0) ||
3564          (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3565      {
3566        throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3567                                     firstLineNumber),
3568                                firstLineNumber, true, ldifLines, null);
3569      }
3570
3571      length = line.length();
3572      if (length == (colonPos+1))
3573      {
3574        // The colon was the last character on the line.  This is fine.
3575        newSuperiorDN = "";
3576      }
3577      else if (line.charAt(colonPos+1) == ':')
3578      {
3579        // Skip over any spaces leading up to the value, and then the rest of
3580        // the string is the base64-encoded new superior DN.
3581        int pos = colonPos+2;
3582        while ((pos < length) && (line.charAt(pos) == ' '))
3583        {
3584          pos++;
3585        }
3586
3587        try
3588        {
3589          final byte[] dnBytes = Base64.decode(line.substring(pos));
3590          newSuperiorDN = new String(dnBytes, "UTF-8");
3591        }
3592        catch (final ParseException pe)
3593        {
3594          debugException(pe);
3595          throw new LDIFException(
3596               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3597                    firstLineNumber, pe.getMessage()),
3598               firstLineNumber, true, ldifLines, pe);
3599        }
3600        catch (final Exception e)
3601        {
3602          debugException(e);
3603          throw new LDIFException(
3604               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3605                    firstLineNumber, e),
3606               firstLineNumber, true, ldifLines, e);
3607        }
3608      }
3609      else
3610      {
3611        // Skip over any spaces leading up to the value, and then the rest of
3612        // the string is the new superior DN.
3613        int pos = colonPos+1;
3614        while ((pos < length) && (line.charAt(pos) == ' '))
3615        {
3616          pos++;
3617        }
3618
3619        newSuperiorDN = line.substring(pos);
3620      }
3621    }
3622    else
3623    {
3624      newSuperiorDN = null;
3625    }
3626
3627
3628    // There must not be any more lines.
3629    if (iterator.hasNext())
3630    {
3631      throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
3632                              firstLineNumber, true, ldifLines, null);
3633    }
3634
3635    return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
3636         newSuperiorDN, controls);
3637  }
3638
3639
3640
3641  /**
3642   * Examines the line contained in the provided buffer to determine whether it
3643   * may contain one or more illegal trailing spaces.  If it does, then those
3644   * spaces will either be stripped out or an exception will be thrown to
3645   * indicate that they are illegal.
3646   *
3647   * @param  buffer                 The buffer to be examined.
3648   * @param  dn                     The DN of the LDIF record being parsed.  It
3649   *                                may be {@code null} if the DN is not yet
3650   *                                known (e.g., because the provided line is
3651   *                                expected to contain that DN).
3652   * @param  firstLineNumber        The approximate line number in the LDIF
3653   *                                source on which the LDIF record begins.
3654   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3655   *                                encountering attribute values which are not
3656   *                                base64-encoded but contain trailing spaces.
3657   *
3658   * @throws  LDIFException  If the line contained in the provided buffer ends
3659   *                         with one or more illegal trailing spaces and
3660   *                         {@code stripTrailingSpaces} was provided with a
3661   *                         value of {@code false}.
3662   */
3663  private static void handleTrailingSpaces(final StringBuilder buffer,
3664                           final String dn, final long firstLineNumber,
3665                           final TrailingSpaceBehavior trailingSpaceBehavior)
3666          throws LDIFException
3667  {
3668    int pos = buffer.length() - 1;
3669    boolean trailingFound = false;
3670    while ((pos >= 0) && (buffer.charAt(pos) == ' '))
3671    {
3672      trailingFound = true;
3673      pos--;
3674    }
3675
3676    if (trailingFound && (buffer.charAt(pos) != ':'))
3677    {
3678      switch (trailingSpaceBehavior)
3679      {
3680        case STRIP:
3681          buffer.setLength(pos+1);
3682          break;
3683
3684        case REJECT:
3685          if (dn == null)
3686          {
3687            throw new LDIFException(
3688                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
3689                      buffer.toString()),
3690                 firstLineNumber, true);
3691          }
3692          else
3693          {
3694            throw new LDIFException(
3695                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
3696                      firstLineNumber, buffer.toString()),
3697                 firstLineNumber, true);
3698          }
3699
3700        case RETAIN:
3701        default:
3702          // No action will be taken.
3703          break;
3704      }
3705    }
3706  }
3707
3708
3709
3710  /**
3711   * This represents an unparsed LDIFRecord.  It stores the line number of the
3712   * first line of the record and each line of the record.
3713   */
3714  private static final class UnparsedLDIFRecord
3715  {
3716    private final ArrayList<StringBuilder> lineList;
3717    private final long firstLineNumber;
3718    private final Exception failureCause;
3719    private final boolean isEOF;
3720    private final DuplicateValueBehavior duplicateValueBehavior;
3721    private final Schema schema;
3722    private final TrailingSpaceBehavior trailingSpaceBehavior;
3723
3724
3725
3726    /**
3727     * Constructor.
3728     *
3729     * @param  lineList                The lines that comprise the LDIF record.
3730     * @param  duplicateValueBehavior  The behavior to exhibit if the entry
3731     *                                 contains duplicate attribute values.
3732     * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
3733     *                                 encountering trailing spaces in
3734     *                                 non-base64-encoded attribute values.
3735     * @param  schema                  The schema to use when parsing, if
3736     *                                 applicable.
3737     * @param  firstLineNumber         The first line number of the LDIF record.
3738     */
3739    private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
3740                 final DuplicateValueBehavior duplicateValueBehavior,
3741                 final TrailingSpaceBehavior trailingSpaceBehavior,
3742                 final Schema schema, final long firstLineNumber)
3743    {
3744      this.lineList               = lineList;
3745      this.firstLineNumber        = firstLineNumber;
3746      this.duplicateValueBehavior = duplicateValueBehavior;
3747      this.trailingSpaceBehavior  = trailingSpaceBehavior;
3748      this.schema                 = schema;
3749
3750      failureCause = null;
3751      isEOF =
3752           (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
3753    }
3754
3755
3756
3757    /**
3758     * Constructor.
3759     *
3760     * @param failureCause  The Exception thrown when reading from the input.
3761     */
3762    private UnparsedLDIFRecord(final Exception failureCause)
3763    {
3764      this.failureCause = failureCause;
3765
3766      lineList               = null;
3767      firstLineNumber        = 0;
3768      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
3769      trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
3770      schema                 = null;
3771      isEOF                  = false;
3772    }
3773
3774
3775
3776    /**
3777     * Return the lines that comprise the LDIF record.
3778     *
3779     * @return  The lines that comprise the LDIF record.
3780     */
3781    private ArrayList<StringBuilder> getLineList()
3782    {
3783      return lineList;
3784    }
3785
3786
3787
3788    /**
3789     * Retrieves the behavior to exhibit when encountering duplicate attribute
3790     * values.
3791     *
3792     * @return  The behavior to exhibit when encountering duplicate attribute
3793     *          values.
3794     */
3795    private DuplicateValueBehavior getDuplicateValueBehavior()
3796    {
3797      return duplicateValueBehavior;
3798    }
3799
3800
3801
3802    /**
3803     * Retrieves the behavior that should be exhibited when encountering
3804     * attribute values which are not base64-encoded but contain trailing
3805     * spaces.  The LDIF specification strongly recommends that any value which
3806     * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
3807     * LDIF parser may be configured to automatically strip these spaces, to
3808     * preserve them, or to reject any entry or change record containing them.
3809     *
3810     * @return  The behavior that should be exhibited when encountering
3811     *          attribute values which are not base64-encoded but contain
3812     *          trailing spaces.
3813     */
3814    private TrailingSpaceBehavior getTrailingSpaceBehavior()
3815    {
3816      return trailingSpaceBehavior;
3817    }
3818
3819
3820
3821    /**
3822     * Retrieves the schema that should be used when parsing the record, if
3823     * applicable.
3824     *
3825     * @return  The schema that should be used when parsing the record, or
3826     *          {@code null} if none should be used.
3827     */
3828    private Schema getSchema()
3829    {
3830      return schema;
3831    }
3832
3833
3834
3835    /**
3836     * Return the first line number of the LDIF record.
3837     *
3838     * @return  The first line number of the LDIF record.
3839     */
3840    private long getFirstLineNumber()
3841    {
3842      return firstLineNumber;
3843    }
3844
3845
3846
3847    /**
3848     * Return {@code true} iff the end of the input was reached.
3849     *
3850     * @return  {@code true} iff the end of the input was reached.
3851     */
3852    private boolean isEOF()
3853    {
3854      return isEOF;
3855    }
3856
3857
3858
3859    /**
3860     * Returns the reason that reading the record lines failed.  This normally
3861     * is only non-null if something bad happened to the input stream (like
3862     * a disk read error).
3863     *
3864     * @return  The reason that reading the record lines failed.
3865     */
3866    private Exception getFailureCause()
3867    {
3868      return failureCause;
3869    }
3870  }
3871
3872
3873  /**
3874   * When processing in asynchronous mode, this thread is responsible for
3875   * reading the raw unparsed records from the input and submitting them for
3876   * processing.
3877   */
3878  private final class LineReaderThread
3879       extends Thread
3880  {
3881    /**
3882     * Constructor.
3883     */
3884    private LineReaderThread()
3885    {
3886      super("Asynchronous LDIF line reader");
3887      setDaemon(true);
3888    }
3889
3890
3891
3892    /**
3893     * Reads raw, unparsed records from the input and submits them for
3894     * processing until the input is finished or closed.
3895     */
3896    @Override()
3897    public void run()
3898    {
3899      try
3900      {
3901        boolean stopProcessing = false;
3902        while (!stopProcessing)
3903        {
3904          UnparsedLDIFRecord unparsedRecord = null;
3905          try
3906          {
3907            unparsedRecord = readUnparsedRecord();
3908          }
3909          catch (IOException e)
3910          {
3911            debugException(e);
3912            unparsedRecord = new UnparsedLDIFRecord(e);
3913            stopProcessing = true;
3914          }
3915          catch (Exception e)
3916          {
3917            debugException(e);
3918            unparsedRecord = new UnparsedLDIFRecord(e);
3919          }
3920
3921          try
3922          {
3923            asyncParser.submit(unparsedRecord);
3924          }
3925          catch (InterruptedException e)
3926          {
3927            debugException(e);
3928            // If this thread is interrupted, then someone wants us to stop
3929            // processing, so that's what we'll do.
3930            stopProcessing = true;
3931          }
3932
3933          if ((unparsedRecord == null) || (unparsedRecord.isEOF()))
3934          {
3935            stopProcessing = true;
3936          }
3937        }
3938      }
3939      finally
3940      {
3941        try
3942        {
3943          asyncParser.shutdown();
3944        }
3945        catch (InterruptedException e)
3946        {
3947          debugException(e);
3948        }
3949        finally
3950        {
3951          asyncParsingComplete.set(true);
3952        }
3953      }
3954    }
3955  }
3956
3957
3958
3959  /**
3960   * Used to parse Records asynchronously.
3961   */
3962  private final class RecordParser implements Processor<UnparsedLDIFRecord,
3963                                                        LDIFRecord>
3964  {
3965    /**
3966     * {@inheritDoc}
3967     */
3968    public LDIFRecord process(final UnparsedLDIFRecord input)
3969           throws LDIFException
3970    {
3971      LDIFRecord record = decodeRecord(input, relativeBasePath);
3972
3973      if ((record instanceof Entry) && (entryTranslator != null))
3974      {
3975        record = entryTranslator.translate((Entry) record,
3976             input.getFirstLineNumber());
3977
3978        if (record == null)
3979        {
3980          record = SKIP_ENTRY;
3981        }
3982      }
3983      return record;
3984    }
3985  }
3986}