Python error handling

Summary:
This diff introduces a generic abstraction to convert a python error (set by a C-API call) into an informative C++ exception.

The main motivation for this is to improve the error handling behavior when dealing with failed cython api module imports.

This diff adds one callsite to the new abstraction in fiber.cpp  to demonstrate its usage.

Reviewed By: yfeldblum

Differential Revision: D25469594

fbshipit-source-id: e9f83c06460f35fa76030ceae9726297276db850

CMakeLists.txt

@@ -160,6 +160,7 @@ REMOVE_MATCHES_FROM_LISTS(files hfiles
 list(REMOVE_ITEM files
   ${FOLLY_DIR}/experimental/JSONSchemaTester.cpp
   ${FOLLY_DIR}/experimental/io/HugePageUtil.cpp
+  ${FOLLY_DIR}/python/error.cpp
   ${FOLLY_DIR}/python/executor.cpp
   ${FOLLY_DIR}/python/fibers.cpp
   ${FOLLY_DIR}/python/GILAwareManualExecutor.cpp

folly/python/error.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/python/error.h>
+#include <Python.h>
+#include <folly/Conv.h>
+#include <folly/ScopeGuard.h>
+#include <stdexcept>
+#include <string>
+
+namespace folly {
+namespace python {
+namespace {
+// Best effort c-api implementation of repr(obj)
+std::string pyObjectToString(PyObject* obj) {
+  constexpr StringPiece kConversionFail = "Error conversion failed";
+  PyObject *pyStr, *pyBytes;
+  SCOPE_EXIT {
+    Py_XDECREF(pyStr);
+    Py_XDECREF(pyBytes);
+    // Swallow any errors that arise in this function
+    PyErr_Clear();
+  };
+
+  pyStr = PyObject_Repr(obj);
+  if (pyStr == nullptr) {
+    return std::string(kConversionFail);
+  }
+
+  char* cStr = nullptr;
+#if PY_VERSION_HEX < 0x0300000
+  cStr = PyString_AsString(pyStr);
+#else
+  pyBytes = PyUnicode_AsEncodedString(pyStr, "utf-8", "strict");
+  if (pyBytes == nullptr) {
+    return std::string(kConversionFail);
+  }
+
+  cStr = PyBytes_AsString(pyBytes);
+#endif
+
+  if (cStr == nullptr) {
+    return std::string(kConversionFail);
+  }
+
+  return std::string(cStr);
+}
+
+} // namespace
+
+void handlePythonError(StringPiece errPrefix) {
+  PyObject *ptype, *pvalue, *ptraceback;
+  SCOPE_EXIT {
+    Py_XDECREF(ptype);
+    Py_XDECREF(pvalue);
+    Py_XDECREF(ptraceback);
+  };
+  /**
+   * PyErr_Fetch will clear the error indicator (which *should* be set here, but
+   * let's not assume it is)
+   */
+  PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+
+  if (ptype == nullptr) {
+    throw std::runtime_error(
+        to<std::string>(errPrefix, "No error indicator set"));
+  }
+
+  if (pvalue == nullptr) {
+    throw std::runtime_error(to<std::string>(
+        errPrefix, "Exception of type: ", pyObjectToString(ptype)));
+  }
+
+  throw std::runtime_error(
+      to<std::string>(errPrefix, pyObjectToString(pvalue)));
+}
+} // namespace python
+} // namespace folly

folly/python/error.h

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <folly/Range.h>
+
+namespace folly {
+namespace python {
+/**
+ * Should be called on C-API call failure to convert the python error into an
+ * informative C++ exception
+ */
+[[noreturn]] void handlePythonError(StringPiece errPrefix);
+
+} // namespace python
+} // namespace folly

folly/python/fibers.cpp

@@ -19,6 +19,7 @@
 #include <stdexcept>
 
 #include <folly/CppAttributes.h>
+#include <folly/python/error.h>
 #include <folly/python/fiber_manager_api.h>
 
 namespace folly {
@@ -27,7 +28,7 @@ namespace {
 
 void do_import() {
   if (0 != import_folly__fiber_manager()) {
-    throw std::runtime_error("import_folly__fiber_manager failed");
+    handlePythonError("import_folly__fiber_manager failed: ");
   }
 }
 

folly/python/test/ErrorTest.cpp

+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <folly/python/error.h>
+#include <Python.h>
+#include <folly/ScopeGuard.h>
+#include <folly/portability/GTest.h>
+#include <string>
+
+namespace folly {
+namespace python {
+namespace detail {
+namespace test {
+
+struct ErrorTest : public testing::Test {
+  void SetUp() override {
+    Py_Initialize();
+    gstate_ = PyGILState_Ensure();
+  }
+
+  void TearDown() override { PyGILState_Release(gstate_); }
+
+ protected:
+  template <typename F>
+  static void expectThrowsWithMessage(F&& func, std::string message) {
+    try {
+      func();
+      FAIL();
+    } catch (const std::runtime_error& e) {
+      EXPECT_STREQ(message.c_str(), e.what());
+    } catch (...) {
+      FAIL();
+    }
+
+    EXPECT_EQ(nullptr, PyErr_Occurred());
+  }
+
+ private:
+  PyGILState_STATE gstate_;
+};
+
+TEST_F(ErrorTest, testNoErrorIndicator) {
+  expectThrowsWithMessage(
+      []() { handlePythonError("fail: "); }, "fail: No error indicator set");
+}
+
+TEST_F(ErrorTest, testNullError) {
+  expectThrowsWithMessage(
+      []() {
+        PyErr_SetObject(PyExc_RuntimeError, NULL);
+        handlePythonError("fail: ");
+      },
+      "fail: Exception of type: <class 'RuntimeError'>");
+}
+
+TEST_F(ErrorTest, testStringError) {
+  expectThrowsWithMessage(
+      []() {
+        PyErr_SetString(PyExc_RuntimeError, "test error");
+        handlePythonError("fail: ");
+      },
+      "fail: 'test error'");
+}
+
+TEST_F(ErrorTest, testException) {
+  expectThrowsWithMessage(
+      []() {
+        PyObject *args, *exc;
+        SCOPE_EXIT {
+          Py_XDECREF(args);
+          Py_XDECREF(exc);
+        };
+
+        args = Py_BuildValue("(s)", "test error");
+        if (!args) {
+          handlePythonError("fail: ");
+        }
+
+        exc = PyObject_CallObject(PyExc_RuntimeError, args);
+        if (!exc) {
+          handlePythonError("fail: ");
+        }
+
+        PyErr_SetObject(PyExc_RuntimeError, exc);
+        handlePythonError("fail: ");
+      },
+      "fail: RuntimeError('test error')");
+}
+
+} // namespace test
+} // namespace detail
+} // namespace python
+} // namespace folly