Skip to content

[3.10+][1/n] Add conditional archiving for libmpdec module #106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open

[3.10+][1/n] Add conditional archiving for libmpdec module #106

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
4d6a0b3
Make cpython version user specifiable at build time
anirbanr-fb-r2p Jul 29, 2022
abd3251
[Support cpython 3.10+][1/n] Add libmpdec mem module conditionally
anirbanr-fb-r2p Jul 30, 2022
31a45b6
[Support cpython 3.10+][1/n] Add libmpdec mem module conditionally
anirbanr-fb-r2p Jul 30, 2022
1dd266c
Undoing addl commits
anirbanr-fb-r2p Jul 30, 2022
65c41b8
[3.10+][1/n] Add conditional archiving for libmpdec module
anirbanr-fb-r2p Jul 30, 2022
6f21200
Merge pull request #1 from anirbanr-fb-r2p/3_10_support
anirbanr-fb-r2p Jul 30, 2022
2f322a4
Updates
anirbanr-fb-r2p Aug 10, 2022
0f3a97e
Updates
anirbanr-fb-r2p Aug 10, 2022
fb6f24c
Small refactor
anirbanr-fb-r2p Aug 11, 2022
439d99b
Small refactor
anirbanr-fb-r2p Aug 11, 2022
14a77e7
Updates
anirbanr-fb-r2p Aug 11, 2022
f833e4b
Cleanup
anirbanr-fb-r2p Aug 11, 2022
12269b6
Cleanup
anirbanr-fb-r2p Aug 11, 2022
5841622
Manual rebase.
anirbanr-fb-r2p Aug 11, 2022
abd2cfe
Merge branch '3_10_support' of github.com:anirbanr-fb-r2p/multipy int…
anirbanr-fb-r2p Aug 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions multipy/runtime/interpreter/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,19 @@ include_directories(BEFORE "${PYTORCH_ROOT}/torch/include/torch/csrc/api/include
SET(MULTIPY_UTILS "${CMAKE_CURRENT_SOURCE_DIR}/../../utils")

# Build cpython
SET(CPYTHON_VERSION 3.8 CACHE STRING "Default version of Cpython to bundle.")
SET(CPYTHON_VERSION_GIT_TAG 3.8.6 CACHE STRING "Git tag for default version of Cpython to bundle.")
SET(PYTHON_INSTALL_DIR "${INTERPRETER_DIR}/cpython")
SET(PYTHON_INC_DIR "${PYTHON_INSTALL_DIR}/include/python3.8")
SET(PYTHON_INC_DIR "${PYTHON_INSTALL_DIR}/include/python3.8" PARENT_SCOPE)
SET(PYTHON_LIB "${PYTHON_INSTALL_DIR}/lib/libpython3.8.a")
SET(PYTHON_INC_DIR "${PYTHON_INSTALL_DIR}/include/python{CPYTHON_VERSION}")
SET(PYTHON_INC_DIR "${PYTHON_INSTALL_DIR}/include/python{CPYTHON_VERSION}" PARENT_SCOPE)
SET(PYTHON_LIB "${PYTHON_INSTALL_DIR}/lib/libpython{CPYTHON_VERSION}.a")
SET(PYTHON_BIN "${PYTHON_INSTALL_DIR}/bin/python3")
include(ExternalProject)
ExternalProject_Add(
cpython
PREFIX cpython
GIT_REPOSITORY https://github.com/python/cpython.git
GIT_TAG v3.8.6
GIT_TAG v{CPYTHON_VERSION_GIT_TAG}
UPDATE_COMMAND ""
PATCH_COMMAND git apply ${CMAKE_CURRENT_SOURCE_DIR}/cpython_patch.diff
BUILD_IN_SOURCE True
Expand All @@ -47,17 +49,24 @@ include(GoogleTest)
# the modules in a strange nested path, and then that path is relative to the
# Cmake ExternalProject root in the cmake build dir.
ExternalProject_Get_property(cpython SOURCE_DIR)
SET(PYTHON_MODULE_DIR "${SOURCE_DIR}/build/temp.linux-x86_64-3.8/${SOURCE_DIR}/Modules")
SET(PYTHON_MODULE_DIR "${SOURCE_DIR}/build/temp.linux-x86_64-{CPYTHON_VERSION}/${SOURCE_DIR}/Modules")
SET(PYTHON_STDLIB_DIR "${SOURCE_DIR}/Lib")
SET(PYTHON_STDLIB "${PYTHON_INSTALL_DIR}/lib/libpython_stdlib3.8.a")
SET(PYTHON_STDLIB "${PYTHON_INSTALL_DIR}/lib/libpython_stdlib{CPYTHON_VERSION}.a")

if(${CPYTHON_VERSION} MATCHES "3\.[8-9]")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like later on versioning updates to python might be annoying, it may be a good idea to document the process or turn this logic into a function to make the process a bit easier. Otherwise, lgtm.

SET(PYTHON_MEM_MODULE "${PYTHON_MODULE_DIR}/_decimal/libmpdec/memory.o")
elseif(${CPYTHON_VERSION} MATCHES "3\.1[0-1]")
SET(PYTHON_MEM_MODULE "${PYTHON_MODULE_DIR}/_decimal/libmpdec/mpalloc.o")
endif()

# Then we use a hardcoded list of expected module names and include them in our lib
include("CMakePythonModules.txt")
ExternalProject_Add_Step(
cpython
archive_stdlib
DEPENDEES install
BYPRODUCTS ${PYTHON_STDLIB}
COMMAND ar -rc ${PYTHON_STDLIB} ${PYTHON_MODULES}
COMMAND ar -rc ${PYTHON_STDLIB} ${PYTHON_MODULES} ${PYTHON_MEM_MODULE}
VERBATIM
)
# Get python typing extension, needed by torch
Expand Down
2 changes: 1 addition & 1 deletion multipy/runtime/interpreter/CMakePythonModules.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ SET(PYTHON_MODULES
${PYTHON_MODULE_DIR}/_cursesmodule.o
${PYTHON_MODULE_DIR}/_curses_panel.o
${PYTHON_MODULE_DIR}/_datetimemodule.o
${PYTHON_MODULE_DIR}/_decimal/_decimal.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/basearith.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/constants.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/context.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/convolute.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/crt.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/difradix2.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/fnt.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/fourstep.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/io.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/memory.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/mpdecimal.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/numbertheory.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/sixstep.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/transpose.o
${PYTHON_MODULE_DIR}/_decimal/_decimal.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/basearith.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/constants.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/context.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/convolute.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/crt.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/difradix2.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/fnt.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/fourstep.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/io.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/mpdecimal.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/numbertheory.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/sixstep.o ${PYTHON_MODULE_DIR}/_decimal/libmpdec/transpose.o
${PYTHON_MODULE_DIR}/_elementtree.o
${PYTHON_MODULE_DIR}/fcntlmodule.o
${PYTHON_MODULE_DIR}/grpmodule.o
Expand Down