New loplugin:faileddyncast

Offline discussion about tdf#96067 "Crash on undo row inserts" brought up the
idea to warn about cases where uses of dynamic_cast are statically knwon to
always fail.  Clang's clang::AST::CXXDynamicCastExpr::isAlwaysNull already
implements such a check, reporting true if the casted-from class is final, but
has two issues:

For one, it does not work for template code, when one of the involved types is a
template parameter type (so e.g., DestType->castAs<PointerType>() can crash).

For another, it misses the opportunity to report true if the casted-to type is
final and only derives from the casted-from type non-publicly.  My hope was that
this, after the "final" decorations in 548c4323
"Mark some classes as final," might turn up the culprit of tdf#96067 (with a
scenario similar to the failed dynamic_cast on private derivation in
63b67ab5 "Use public derivation, and remove
then-unnecessary downcasts")---but not so.

Change-Id: I962ee19820758f9c601f4a292da7f37fa9dff5ce

compilerplugins/clang/faileddyncast.cxx

+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <algorithm>
+
+#include "clang/AST/CXXInheritance.h"
+
+#include "plugin.hxx"
+
+namespace {
+
+// cf. Clang's clang::AST::CXXDynamicCastExpr::isAlwaysNull
+// (lib/AST/ExprCXX.cpp):
+bool isAlwaysNull(CXXDynamicCastExpr const * expr) {
+  QualType SrcType = expr->getSubExpr()->getType();
+  QualType DestType = expr->getType();
+
+  if (const PointerType *SrcPTy = SrcType->getAs<PointerType>()) {
+    SrcType = SrcPTy->getPointeeType();
+#if 0
+    DestType = DestType->castAs<PointerType>()->getPointeeType();
+#else
+    auto DstPTy = DestType->getAs<PointerType>();
+    if (!DstPTy)
+      return false;
+    DestType = DstPTy->getPointeeType();
+#endif
+  }
+
+  if (DestType->isVoidType())
+    return false;
+
+#if 0
+  const CXXRecordDecl *SrcRD =
+    cast<CXXRecordDecl>(SrcType->castAs<RecordType>()->getDecl());
+#else
+  auto SrcRT = SrcType->getAs<RecordType>();
+  if (!SrcRT)
+    return false;
+  const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(SrcRT->getDecl());
+#endif
+
+#if 0
+  if (!SrcRD->hasAttr<FinalAttr>())
+    return false;
+#endif
+
+#if 0
+  const CXXRecordDecl *DestRD =
+    cast<CXXRecordDecl>(DestType->castAs<RecordType>()->getDecl());
+#else
+  auto DestRT = DestType->getAs<RecordType>();
+  if (!DestRT)
+    return false;
+  const CXXRecordDecl *DestRD = cast<CXXRecordDecl>(DestRT->getDecl());
+#endif
+
+#if 1
+  if (!(SrcRD && DestRD))
+    return false;
+
+  if (DestRD->hasAttr<FinalAttr>()) {
+    CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true,
+                       /*DetectVirtual=*/false);
+    if (DestRD->isDerivedFrom(SrcRD, Paths) &&
+        std::all_of(Paths.begin(), Paths.end(),
+                    [](CXXBasePath const & Path) {
+                        return Path.Access != AS_public; }))
+      return true;
+  }
+
+  if (!SrcRD->hasAttr<FinalAttr>())
+    return false;
+#endif
+
+  return !DestRD->isDerivedFrom(SrcRD);
+}
+
+class FailedDynCast:
+    public RecursiveASTVisitor<FailedDynCast>, public loplugin::Plugin
+{
+public:
+    explicit FailedDynCast(InstantiationData const & data): Plugin(data) {}
+
+    bool shouldVisitTemplateInstantiations() const { return true; }
+
+    void run() override;
+
+    bool VisitCXXDynamicCastExpr(CXXDynamicCastExpr const * expr);
+};
+
+void FailedDynCast::run() {
+    if (compiler.getLangOpts().CPlusPlus) {
+        TraverseDecl(compiler.getASTContext().getTranslationUnitDecl());
+    }
+}
+
+bool FailedDynCast::VisitCXXDynamicCastExpr(CXXDynamicCastExpr const * expr) {
+    if (ignoreLocation(expr)) {
+        return true;
+    }
+    if (isAlwaysNull(expr)) {
+        report(
+            DiagnosticsEngine::Warning,
+            "dynamic_cast from %0 to %1 always fails", expr->getLocStart())
+            << expr->getSubExpr()->getType() << expr->getType()
+            << expr->getSourceRange();
+    }
+    return true;
+}
+
+loplugin::Plugin::Registration<FailedDynCast> X("faileddyncast");
+
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */