001/*
002 * Copyright 2013-2014 UnboundID Corp.
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2013-2014 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.OutputStream;
026import java.util.Collections;
027import java.util.LinkedHashMap;
028import java.util.List;
029import java.util.Map;
030import java.util.TreeMap;
031import java.util.concurrent.atomic.AtomicLong;
032
033import com.unboundid.asn1.ASN1OctetString;
034import com.unboundid.ldap.sdk.Attribute;
035import com.unboundid.ldap.sdk.DN;
036import com.unboundid.ldap.sdk.Filter;
037import com.unboundid.ldap.sdk.LDAPConnection;
038import com.unboundid.ldap.sdk.LDAPException;
039import com.unboundid.ldap.sdk.LDAPSearchException;
040import com.unboundid.ldap.sdk.ResultCode;
041import com.unboundid.ldap.sdk.SearchRequest;
042import com.unboundid.ldap.sdk.SearchResult;
043import com.unboundid.ldap.sdk.SearchResultEntry;
044import com.unboundid.ldap.sdk.SearchResultReference;
045import com.unboundid.ldap.sdk.SearchResultListener;
046import com.unboundid.ldap.sdk.SearchScope;
047import com.unboundid.ldap.sdk.Version;
048import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
049import com.unboundid.util.Debug;
050import com.unboundid.util.LDAPCommandLineTool;
051import com.unboundid.util.StaticUtils;
052import com.unboundid.util.ThreadSafety;
053import com.unboundid.util.ThreadSafetyLevel;
054import com.unboundid.util.args.ArgumentException;
055import com.unboundid.util.args.ArgumentParser;
056import com.unboundid.util.args.DNArgument;
057import com.unboundid.util.args.IntegerArgument;
058import com.unboundid.util.args.StringArgument;
059
060
061
062/**
063 * This class provides a tool that may be used to identify references to entries
064 * that do not exist.  This tool can be useful for verifying existing data in
065 * directory servers that provide support for referential integrity.
066 * <BR><BR>
067 * All of the necessary information is provided using command line arguments.
068 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
069 * class, as well as the following additional arguments:
070 * <UL>
071 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
072 *       for the searches.  At least one base DN must be provided.</LI>
073 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
074 *       that is expected to contain references to other entries.  This
075 *       attribute should be indexed for equality searches, and its values
076 *       should be DNs.  At least one attribute must be provided.</LI>
077 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
078 *       to find entries with references to other entries should use the simple
079 *       paged results control to iterate across entries in fixed-size pages
080 *       rather than trying to use a single search to identify all entries that
081 *       reference other entries.</LI>
082 * </UL>
083 */
084@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
085public final class IdentifyReferencesToMissingEntries
086       extends LDAPCommandLineTool
087       implements SearchResultListener
088{
089  /**
090   * The serial version UID for this serializable class.
091   */
092  private static final long serialVersionUID = 1981894839719501258L;
093
094
095
096  // The number of entries examined so far.
097  private final AtomicLong entriesExamined;
098
099  // The argument used to specify the base DNs to use for searches.
100  private DNArgument baseDNArgument;
101
102  // The argument used to specify the search page size.
103  private IntegerArgument pageSizeArgument;
104
105  // The connection to use for retrieving referenced entries.
106  private LDAPConnection getReferencedEntriesConnection;
107
108  // A map with counts of missing references by attribute type.
109  private final Map<String,AtomicLong> missingReferenceCounts;
110
111  // The names of the attributes for which to find missing references.
112  private String[] attributes;
113
114  // The argument used to specify the attributes for which to find missing
115  // references.
116  private StringArgument attributeArgument;
117
118
119
120  /**
121   * Parse the provided command line arguments and perform the appropriate
122   * processing.
123   *
124   * @param  args  The command line arguments provided to this program.
125   */
126  public static void main(final String... args)
127  {
128    final ResultCode resultCode = main(args, System.out, System.err);
129    if (resultCode != ResultCode.SUCCESS)
130    {
131      System.exit(resultCode.intValue());
132    }
133  }
134
135
136
137  /**
138   * Parse the provided command line arguments and perform the appropriate
139   * processing.
140   *
141   * @param  args       The command line arguments provided to this program.
142   * @param  outStream  The output stream to which standard out should be
143   *                    written.  It may be {@code null} if output should be
144   *                    suppressed.
145   * @param  errStream  The output stream to which standard error should be
146   *                    written.  It may be {@code null} if error messages
147   *                    should be suppressed.
148   *
149   * @return A result code indicating whether the processing was successful.
150   */
151  public static ResultCode main(final String[] args,
152                                final OutputStream outStream,
153                                final OutputStream errStream)
154  {
155    final IdentifyReferencesToMissingEntries tool =
156         new IdentifyReferencesToMissingEntries(outStream, errStream);
157    return tool.runTool(args);
158  }
159
160
161
162  /**
163   * Creates a new instance of this tool.
164   *
165   * @param  outStream  The output stream to which standard out should be
166   *                    written.  It may be {@code null} if output should be
167   *                    suppressed.
168   * @param  errStream  The output stream to which standard error should be
169   *                    written.  It may be {@code null} if error messages
170   *                    should be suppressed.
171   */
172  public IdentifyReferencesToMissingEntries(final OutputStream outStream,
173                                            final OutputStream errStream)
174  {
175    super(outStream, errStream);
176
177    baseDNArgument = null;
178    pageSizeArgument = null;
179    attributeArgument = null;
180    getReferencedEntriesConnection = null;
181
182    entriesExamined = new AtomicLong(0L);
183    missingReferenceCounts = new TreeMap<String, AtomicLong>();
184  }
185
186
187
188  /**
189   * Retrieves the name of this tool.  It should be the name of the command used
190   * to invoke this tool.
191   *
192   * @return  The name for this tool.
193   */
194  @Override()
195  public String getToolName()
196  {
197    return "identify-references-to-missing-entries";
198  }
199
200
201
202  /**
203   * Retrieves a human-readable description for this tool.
204   *
205   * @return  A human-readable description for this tool.
206   */
207  @Override()
208  public String getToolDescription()
209  {
210    return "This tool may be used to identify entries containing one or more " +
211         "attributes which reference entries that do not exist.  This may " +
212         "require the ability to perform unindexed searches and/or the " +
213         "ability to use the simple paged results control.";
214  }
215
216
217
218  /**
219   * Retrieves a version string for this tool, if available.
220   *
221   * @return  A version string for this tool, or {@code null} if none is
222   *          available.
223   */
224  @Override()
225  public String getToolVersion()
226  {
227    return Version.NUMERIC_VERSION_STRING;
228  }
229
230
231
232  /**
233   * Adds the arguments needed by this command-line tool to the provided
234   * argument parser which are not related to connecting or authenticating to
235   * the directory server.
236   *
237   * @param  parser  The argument parser to which the arguments should be added.
238   *
239   * @throws  ArgumentException  If a problem occurs while adding the arguments.
240   */
241  @Override()
242  public void addNonLDAPArguments(final ArgumentParser parser)
243         throws ArgumentException
244  {
245    String description = "The search base DN(s) to use to find entries with " +
246         "references to other entries.  At least one base DN must be " +
247         "specified.";
248    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
249         description);
250    parser.addArgument(baseDNArgument);
251
252    description = "The attribute(s) for which to find missing references.  " +
253         "At least one attribute must be specified, and each attribute " +
254         "must be indexed for equality searches and have values which are DNs.";
255    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
256         description);
257    parser.addArgument(attributeArgument);
258
259    description = "The maximum number of entries to retrieve at a time when " +
260         "attempting to find entries with references to other entries.  This " +
261         "requires that the authenticated user have permission to use the " +
262         "simple paged results control, but it can avoid problems with the " +
263         "server sending entries too quickly for the client to handle.  By " +
264         "default, the simple paged results control will not be used.";
265    pageSizeArgument =
266         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
267              description, 1, Integer.MAX_VALUE);
268    parser.addArgument(pageSizeArgument);
269  }
270
271
272
273  /**
274   * Performs the core set of processing for this tool.
275   *
276   * @return  A result code that indicates whether the processing completed
277   *          successfully.
278   */
279  @Override()
280  public ResultCode doToolProcessing()
281  {
282    // Establish a connection to the target directory server to use for
283    // finding references to entries.
284    final LDAPConnection findReferencesConnection;
285    try
286    {
287      findReferencesConnection = getConnection();
288    }
289    catch (final LDAPException le)
290    {
291      Debug.debugException(le);
292      err("Unable to establish a connection to the directory server:  ",
293           StaticUtils.getExceptionMessage(le));
294      return le.getResultCode();
295    }
296
297    try
298    {
299      // Establish a second connection to use for retrieving referenced entries.
300      try
301      {
302        getReferencedEntriesConnection = getConnection();
303      }
304      catch (final LDAPException le)
305      {
306        Debug.debugException(le);
307        err("Unable to establish a connection to the directory server:  ",
308             StaticUtils.getExceptionMessage(le));
309        return le.getResultCode();
310      }
311
312
313      // Get the set of attributes for which to find missing references.
314      final List<String> attrList = attributeArgument.getValues();
315      attributes = new String[attrList.size()];
316      attrList.toArray(attributes);
317
318
319      // Construct a search filter that will be used to find all entries with
320      // references to other entries.
321      final Filter filter;
322      if (attributes.length == 1)
323      {
324        filter = Filter.createPresenceFilter(attributes[0]);
325        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
326      }
327      else
328      {
329        final Filter[] orComps = new Filter[attributes.length];
330        for (int i=0; i < attributes.length; i++)
331        {
332          orComps[i] = Filter.createPresenceFilter(attributes[i]);
333          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
334        }
335        filter = Filter.createORFilter(orComps);
336      }
337
338
339      // Iterate across all of the search base DNs and perform searches to find
340      // missing references.
341      for (final DN baseDN : baseDNArgument.getValues())
342      {
343        ASN1OctetString cookie = null;
344        do
345        {
346          final SearchRequest searchRequest = new SearchRequest(this,
347               baseDN.toString(), SearchScope.SUB, filter, attributes);
348          if (pageSizeArgument.isPresent())
349          {
350            searchRequest.addControl(new SimplePagedResultsControl(
351                 pageSizeArgument.getValue(), cookie, false));
352          }
353
354          SearchResult searchResult;
355          try
356          {
357            searchResult = findReferencesConnection.search(searchRequest);
358          }
359          catch (final LDAPSearchException lse)
360          {
361            Debug.debugException(lse);
362            searchResult = lse.getSearchResult();
363          }
364
365          if (searchResult.getResultCode() != ResultCode.SUCCESS)
366          {
367            err("An error occurred while attempting to search for missing " +
368                 "references to entries below " + baseDN + ":  " +
369                 searchResult.getDiagnosticMessage());
370            return searchResult.getResultCode();
371          }
372
373          final SimplePagedResultsControl pagedResultsResponse;
374          try
375          {
376            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
377          }
378          catch (final LDAPException le)
379          {
380            Debug.debugException(le);
381            err("An error occurred while attempting to decode a simple " +
382                 "paged results response control in the response to a " +
383                 "search for entries below " + baseDN + ":  " +
384                 StaticUtils.getExceptionMessage(le));
385            return le.getResultCode();
386          }
387
388          if (pagedResultsResponse != null)
389          {
390            if (pagedResultsResponse.moreResultsToReturn())
391            {
392              cookie = pagedResultsResponse.getCookie();
393            }
394            else
395            {
396              cookie = null;
397            }
398          }
399        }
400        while (cookie != null);
401      }
402
403
404      // See if there were any missing references found.
405      boolean missingReferenceFound = false;
406      for (final Map.Entry<String,AtomicLong> e :
407           missingReferenceCounts.entrySet())
408      {
409        final long numMissing = e.getValue().get();
410        if (numMissing > 0L)
411        {
412          if (! missingReferenceFound)
413          {
414            err();
415            missingReferenceFound = true;
416          }
417
418          err("Found " + numMissing + ' ' + e.getKey() +
419               " references to entries that do not exist.");
420        }
421      }
422
423      if (missingReferenceFound)
424      {
425        return ResultCode.CONSTRAINT_VIOLATION;
426      }
427      else
428      {
429        out("No references were found to entries that do not exist.");
430        return ResultCode.SUCCESS;
431      }
432    }
433    finally
434    {
435      findReferencesConnection.close();
436
437      if (getReferencedEntriesConnection != null)
438      {
439        getReferencedEntriesConnection.close();
440      }
441    }
442  }
443
444
445
446  /**
447   * Retrieves a map that correlates the number of missing references found by
448   * attribute type.
449   *
450   * @return  A map that correlates the number of missing references found by
451   *          attribute type.
452   */
453  public Map<String,AtomicLong> getMissingReferenceCounts()
454  {
455    return Collections.unmodifiableMap(missingReferenceCounts);
456  }
457
458
459
460  /**
461   * Retrieves a set of information that may be used to generate example usage
462   * information.  Each element in the returned map should consist of a map
463   * between an example set of arguments and a string that describes the
464   * behavior of the tool when invoked with that set of arguments.
465   *
466   * @return  A set of information that may be used to generate example usage
467   *          information.  It may be {@code null} or empty if no example usage
468   *          information is available.
469   */
470  @Override()
471  public LinkedHashMap<String[],String> getExampleUsages()
472  {
473    final LinkedHashMap<String[],String> exampleMap =
474         new LinkedHashMap<String[],String>(1);
475
476    final String[] args =
477    {
478      "--hostname", "server.example.com",
479      "--port", "389",
480      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
481      "--bindPassword", "password",
482      "--baseDN", "dc=example,dc=com",
483      "--attribute", "member",
484      "--attribute", "uniqueMember",
485      "--simplePageSize", "100"
486    };
487    exampleMap.put(args,
488         "Identify all entries below dc=example,dc=com in which either the " +
489              "member or uniqueMember attribute references an entry that " +
490              "does not exist.");
491
492    return exampleMap;
493  }
494
495
496
497  /**
498   * Indicates that the provided search result entry has been returned by the
499   * server and may be processed by this search result listener.
500   *
501   * @param  searchEntry  The search result entry that has been returned by the
502   *                      server.
503   */
504  public void searchEntryReturned(final SearchResultEntry searchEntry)
505  {
506    try
507    {
508      // Find attributes which references to entries that do not exist.
509      for (final String attr : attributes)
510      {
511        final List<Attribute> attrList =
512             searchEntry.getAttributesWithOptions(attr, null);
513        for (final Attribute a : attrList)
514        {
515          for (final String value : a.getValues())
516          {
517            try
518            {
519              final SearchResultEntry e =
520                   getReferencedEntriesConnection.getEntry(value, "1.1");
521              if (e == null)
522              {
523                err("Entry '", searchEntry.getDN(), "' includes attribute ",
524                     a.getName(), " that references entry '", value,
525                     "' which does not exist.");
526                missingReferenceCounts.get(attr).incrementAndGet();
527              }
528            }
529            catch (final LDAPException le)
530            {
531              Debug.debugException(le);
532              err("An error occurred while attempting to determine whether " +
533                   "entry '" + value + "' referenced in attribute " +
534                   a.getName() + " of entry '" + searchEntry.getDN() +
535                   "' exists:  " + StaticUtils.getExceptionMessage(le));
536              missingReferenceCounts.get(attr).incrementAndGet();
537            }
538          }
539        }
540      }
541    }
542    finally
543    {
544      final long count = entriesExamined.incrementAndGet();
545      if ((count % 1000L) == 0L)
546      {
547        out(count, " entries examined");
548      }
549    }
550  }
551
552
553
554  /**
555   * Indicates that the provided search result reference has been returned by
556   * the server and may be processed by this search result listener.
557   *
558   * @param  searchReference  The search result reference that has been returned
559   *                          by the server.
560   */
561  public void searchReferenceReturned(
562                   final SearchResultReference searchReference)
563  {
564    // No implementation is required.  This tool will not follow referrals.
565  }
566}