Fixing fbthrift and folly cython bindings build

Summary:
This diff addresses a collection of issues with our thrift-py3 build story in open source.
- Cython 0.29 bug https://github.com/cython/cython/issues/2985 prevents us from loading the generated binary python extensions. I've tested 0.28.6 and it works perfectly.
- Adds ssl to the extensions (since we need SSLContext in get_client, we must have access to the module)
- No need to special case folly_pic. There is really no easy way to get the python bindings to work without using shared library libfolly.so. So, In our build (logdevice) we will build folly as a shared library, this also reduces the complexity everywhere.
- We also need iobuf as extension for thrift to work, added that to setup.py
- Refactored some of the CMake code that had hardcoded file names to use globbing to future-proof this as much as possible.
- Moved `python/lang/cast.pxd` to match the python module name `folly.cast`, so now it lives in `python/cast.pxd`. This simplifies the globbing as well.
- Moved `setup.py` to live in the `/python` directory.
- Added `*.py` in setup.py to pickup any raw python code (including `__init__.py`)

Reviewed By: yfeldblum

Differential Revision: D18685009

fbshipit-source-id: 749b30942a3e5e9e314b5248cc0aa90c6ac1581f

CMakeLists.txt

@@ -343,25 +343,10 @@ target_compile_definitions(folly_base
 
 set(FOLLY_INSTALL_TARGETS folly folly_deps)
 
-option(PYTHON_EXTENSIONS "Build Python Bindings for Folly, requires Cython" OFF)
-if (PYTHON_EXTENSIONS)
-  # Compile folly such that it can be linked in to a shared library
-  # required for linking to python extensions
-  add_library(folly_pic
-    $<TARGET_PROPERTY:folly_base,SOURCES>
-  )
-  apply_folly_compile_options_to_target(folly_pic)
-  target_include_directories(folly_pic
-    PUBLIC
-      $<TARGET_PROPERTY:folly_deps,INTERFACE_INCLUDE_DIRECTORIES>
-  )
-  target_compile_definitions(folly_pic
-    PUBLIC
-      $<TARGET_PROPERTY:folly_deps,INTERFACE_COMPILE_DEFINITIONS>
-  )
-  set_target_properties(folly_pic PROPERTIES POSITION_INDEPENDENT_CODE True)
-  list(APPEND FOLLY_INSTALL_TARGETS folly_pic)
-endif ()
+option(PYTHON_EXTENSIONS
+  "Build Python Bindings for Folly, requires Cython and (BUILD_SHARED_LIBS=ON)"
+  OFF
+)
 
 add_library(folly
   $<TARGET_OBJECTS:folly_base>

folly/CMakeLists.txt

@@ -35,53 +35,47 @@ if (PYTHON_EXTENSIONS)
 
   set(_cybld "${CMAKE_CURRENT_BINARY_DIR}/cybld")
 
-  file(MAKE_DIRECTORY "${_cybld}/folly")
-  foreach(_src
-    "executor.pxd"
-    "executor.pyx"
-    "futures.pxd"
-    "range.pxd"
-    "iobuf.pxd"
-    "iobuf.pyx"
-    "optional.pyx"
-    "__init__.pxd"
-    "AsyncioExecutor.h")
+  add_custom_target(create_binding_symlink ALL)
+  file(GLOB BindingFiles
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.pyx"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.pxd"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.h"
+    "${CMAKE_CURRENT_SOURCE_DIR}/python/*.cpp"
+  )
+  file(MAKE_DIRECTORY "${_cybld}/folly/")
+
+  foreach(_src ${BindingFiles})
+    get_filename_component(_target_file "${_src}" NAME)
+
   message(
     STATUS
-    "Linking ${CMAKE_CURRENT_SOURCE_DIR}/python/lang/cast.pxd "
-    "to ${_cybld}/folly/cast.pxd"
+    "Linking ${_src} "
+    "to ${_cybld}/folly/${_target_file}"
   )
-  add_custom_target("create_folly_link_cast.pxd" ALL
+  add_custom_command(TARGET create_binding_symlink PRE_BUILD
     COMMAND
       ${CMAKE_COMMAND} -E create_symlink
-      ${CMAKE_CURRENT_SOURCE_DIR}/python/lang/cast.pxd
-      "${_cybld}/folly/cast.pxd"
+      "${_src}"
+      "${_cybld}/folly/${_target_file}"
   )
-    message(
-      STATUS
-      "Linking ${CMAKE_CURRENT_SOURCE_DIR}/python/${_src} "
-      "to ${_cybld}/folly/${_src}"
-    )
-    add_custom_target("create_folly_link_${_src}" ALL
-      COMMAND
-        ${CMAKE_COMMAND} -E create_symlink
-        ${CMAKE_CURRENT_SOURCE_DIR}/python/${_src}
-        "${_cybld}/folly/${_src}"
-    )
   endforeach()
 
-  # Tell setup.py where to find includes and libfolly_pic.a
+  # Tell setup.py where to find includes and libfolly.so
   set(prop "$<TARGET_PROPERTY:folly_base,INCLUDE_DIRECTORIES>")
   set(incs "$<$<BOOL:${prop}>:-I$<JOIN:${prop},:>>")
   set(libs "-L${CMAKE_BINARY_DIR}")
 
   add_custom_target(folly_python_bindings ALL
+    DEPENDS folly create_binding_symlink
+    WORKING_DIRECTORY ${_cybld})
+
+  add_custom_command(TARGET folly_python_bindings POST_BUILD
     COMMAND
-      python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py
+      python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py
       build_ext -f ${incs} ${libs}
     BYPRODUCTS ${_cybld}/folly/executor_api.h
-    DEPENDS folly_pic
-    WORKING_DIRECTORY ${_cybld})
+    WORKING_DIRECTORY ${_cybld}
+    )
 
   install(
     FILES ${_cybld}/folly/executor_api.h
@@ -91,8 +85,7 @@ if (PYTHON_EXTENSIONS)
 
   # Install Folly Python Bindings
   install(CODE "
-    string(REGEX REPLACE \"^(..*)$\" \"--root=\\\\1\" PYROOT \"\$ENV{DESTDIR}\")
     execute_process(COMMAND
-    python3 ${CMAKE_CURRENT_SOURCE_DIR}/setup.py install \${PYROOT}
+    python3 ${CMAKE_CURRENT_SOURCE_DIR}/python/setup.py install
     WORKING_DIRECTORY ${_cybld})")
 endif ()

folly/setup.py → folly/python/setup.py

@@ -25,17 +25,25 @@ from setuptools import Extension, setup
 
 Options.fast_fail = True
 
-ext = Extension(
-    "folly.executor",
-    sources=["folly/executor.pyx", "folly/iobuf.pyx"],
-    libraries=["folly_pic", "glog", "double-conversion", "iberty"],
-)
+exts = [
+    Extension(
+        "folly.executor",
+        sources=["folly/executor.pyx"],
+        libraries=["folly", "glog", "double-conversion", "iberty"],
+    ),
+    Extension(
+        "folly.iobuf",
+        sources=["folly/iobuf.pyx"],
+        libraries=["folly", "glog", "double-conversion", "iberty"],
+    ),
+]
 
 setup(
     name="folly",
     version="0.0.1",
     packages=["folly"],
     package_data={"": ["*.pxd", "*.h"]},
+    setup_requires=["cython"],
     zip_safe=False,
-    ext_modules=cythonize([ext], compiler_directives={"language_level": 3}),
+    ext_modules=cythonize(exts, compiler_directives={"language_level": 3}),
 )