Enable Multi-python support extension for Hue

.circleci/config.yml

@@ -21,15 +21,15 @@ commands:
           command: |
             sudo ln -fs /usr/share/zoneinfo/UTC /etc/localtime
             export DEBIAN_FRONTEND=noninteractive
-            sudo apt-get update
-            sudo apt-get install -y gcc g++ build-essential python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev
+            sudo apt-get update -y
+            sudo apt-get install -y gcc g++ build-essential git sudo curl
+            sudo apt-get install -y curl python3.8-dev python3.8-venv python3.8-distutils asciidoc rsync curl libkrb5-dev libldap2-dev libsasl2-dev libxml2-dev libxslt-dev  libsasl2-modules-gssapi-mit libsnappy-dev libffi-dev libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
             curl -sL https://deb.nodesource.com/setup_20.x | sudo bash - && sudo apt-get install -y nodejs
+            unset PIP_FIND_LINKS
+            unset PIP_EXTRA_INDEX_URL
             curl -sL https://bootstrap.pypa.io/get-pip.py | sudo python3.8
-            sudo apt-get install -y libncursesw5-dev libgdbm-dev libc6-dev libssl-dev openssl
 
-            export PYTHON_VER=python3.8
-            export ROOT=$PWD
-            make test_prep
+            PYTHON_VER=python3.8 make apps docs test_prep
 
       - run:
           name: run documentation lints
@@ -52,12 +52,12 @@ commands:
       - run:
           name: run python lints
           command: |
-            PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+            PYTHONWARNINGS=always ./build/venvs/python3.8/bin/hue runruff check
 
       - run:
           name: run tests
           command: |
-            PYTHONWARNINGS=always ./build/env/bin/pytest 
+            PYTHONWARNINGS=always ./build/venvs/python3.8/bin/pytest 
 
       - store_artifacts:
           path: test-reports

.github/workflows/commitflow-py3.yml

@@ -57,15 +57,23 @@ jobs:
 
         export PYTHON_VER=python${{ matrix.python-version }}
         export ROOT=$PWD
-        make test_prep
+        make apps test_prep
 
     - name: Run python unit tests
       run: |
-        ./build/env/bin/pytest
+        if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
+          ./build/env/bin/pytest
+        else
+          ./build/venvs/python${{ matrix.python-version }}/bin/pytest
+        fi
 
     - name: Run python lints
       run: |
-        PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+        if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
+          PYTHONWARNINGS=always ./build/env/bin/hue runruff check
+        else
+          PYTHONWARNINGS=always ./build/venvs/python${{ matrix.python-version }}/bin/hue runruff check
+        fi
 
     - name: Run documentation lints
       run: |

.github/workflows/pr-comments.yml

@@ -46,7 +46,7 @@ jobs:
 
           export PYTHON_VER=python3.11
           export ROOT=$PWD
-          make test_prep
+          make apps test_prep
 
       - name: Run python unit tests
         run: |

Makefile

@@ -96,7 +96,7 @@ default:
 	@echo '  clean       : Remove desktop build products'
 	@echo '  distclean   : Remove desktop and thirdparty build products'
 # <<<< DEV ONLY
-	@echo '  doc 	      : Build documentation'
+	@echo '  doc 	     : Build documentation'
 	@echo '  prod        : Generate a tar file for production distribution'
 	@echo '  locales     : Extract strings and update dictionary of each locale'
 	@echo '  ace         : Builds the Ace Editor tool'
@@ -111,9 +111,8 @@ include Makefile.tarball
 ###################################
 # Build docs (unused)
 ###################################
-.PHONY: test_prep
-test_prep: apps
-	@$(ENV_PIP) install -r $(REQUIREMENT_TEST_FILE)
+test_prep:
+	@$(ENV_PYTHON) -m pip install -r $(REQUIREMENT_TEST_FILE)
 
 ###################################
 # Build docs (unused)
@@ -126,42 +125,46 @@ docs:
 ###################################
 # Install parent POM
 ###################################
+.PHONY: parent-pom
 parent-pom:
 ifneq (,$(BUILD_DB_PROXY))
 	cd $(ROOT)/maven && mvn install $(MAVEN_OPTIONS)
 endif
 
-.PHONY: parent-pom
-
+$(info "PYTHON_VER is $(PYTHON_VER)")
+$(info "SYS_PYTHON is $(SYS_PYTHON)")
+$(info "ENV_PYTHON is $(ENV_PYTHON)")
+$(info "SYS_PIP is $(SYS_PIP)")
+$(info "ENV_PIP is $(ENV_PIP)")
+$(info "PYTHON_INCLUDE_DIR is $(PYTHON_INCLUDE_DIR)")
+$(info "PYTHON_H is $(PYTHON_H)")
+$(info "PIP_VERSION is $(PIP_VERSION)")
+$(info "VIRTUAL_ENV_VERSION is $(VIRTUAL_ENV_VERSION)")
+$(info "RELOCATABLE is $(RELOCATABLE)")
+$(info "LOCAL_PY_BIN is $(LOCAL_PY_BIN)")
+$(info "BLD_DIR_ENV is $(BLD_DIR_ENV)")
+$(info "REQUIREMENT_FILE is $(REQUIREMENT_FILE)")
+$(info "INSTALL_DIR is $(INSTALL_DIR)")
+$(info "INST_DIR_ENV is $(INST_DIR_ENV)")
+$(info "PPC64LE is $(PPC64LE)")
 ###################################
 # virtual-env
+# Enhanced to support building and packaging Hue for multiple Python versions.
+# Adds per-Python virtual environment creation, improved logging and diagnostics,
+# Test push
 ###################################
-
-.PHONY: virtual-env
-virtual-env: $(BLD_DIR_ENV)/stamp
-$(BLD_DIR_ENV)/stamp:
-	@echo "--- Creating virtual environment at $(BLD_DIR_ENV) using $(PYTHON_VER)"
+virtual-env: $(BLD_DIR_ENV)/bin/python
+$(BLD_DIR_ENV)/bin/python:
+	@echo "--- Creating virtual environment for $(PYTHON_VER) ---"
+	@mkdir -p $(BLD_DIR_ENV)
 	@$(SYS_PYTHON) -m pip install --upgrade pip==$(PIP_VERSION)
-	$(SYS_PIP) install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  $(SYS_PYTHON) -m venv $(BLD_DIR_ENV); \
-	 fi
-	@virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV)
-	@echo "--- Virtual environment $(BLD_DIR_ENV) ready"
-	@touch $@
-	@echo '--- Installing PIP_MODULES in virtual-env'
-	@if [[ "ppc64le" == $(PPC64LE) ]]; then \
-	  echo '--- Installing $(REQUIREMENT_PPC64LE_FILE) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_PPC64LE_FILE); \
-	  echo '--- Finished $(REQUIREMENT_PPC64LE_FILE) into virtual-env'; \
-	 else \
-	  echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'; \
-	  $(ENV_PIP) install -r $(REQUIREMENT_FILE); \
-	  echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'; \
-         fi
+	@$(SYS_PYTHON) -m pip install virtualenv==$(VIRTUAL_ENV_VERSION) virtualenv-make-relocatable==$(VIRTUAL_ENV_RELOCATABLE_VERSION)
+	@$(SYS_PYTHON) -m virtualenv -p $(PYTHON_VER) $(BLD_DIR_ENV) --copies
+	@echo "REQUIREMENT_FILE is $(REQUIREMENT_FILE)"
+	@unset PIP_FIND_LINKS && unset PIP_EXTRA_INDEX_URL && $(ENV_PIP) install -r $(REQUIREMENT_FILE)
+	@echo "--- Virtual environment setup complete for $(PYTHON_VER) ---"
 	@$(ENV_PIP) install $(NAVOPTAPI_WHL)
-	@echo '--- Finished $(NAVOPTAPI_WHL) into virtual-env'
-	@touch $(REQUIREMENT_DOT_FILE)
+	@echo "--- Finished installing $(NAVOPTAPI_WHL) into virtual-env ---"
 ###################################
 # Build desktop
 ###################################
@@ -173,6 +176,16 @@ desktop: parent-pom
 desktop: virtual-env
 	@$(MAKE) -C desktop
 
+###################################
+# relocatable-env
+# Enhanced to support relocatable .pth files.
+###################################
+relocatable-env:
+	@$(MAKE) virtual-env
+	@echo "--- Running $(SYS_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)"
+	@$(SYS_PYTHON) $(RELOCATABLE) $(BLD_DIR_ENV)
+	@echo "--- Setting up pth files $(ENV_PYTHON) $(ROOT)/tools/relocatable.py"
+	@$(ENV_PYTHON) $(ROOT)/tools/relocatable.py
 
 ###################################
 # Build apps
@@ -190,17 +203,18 @@ INSTALL_CORE_FILES = \
 	Makefile* $(wildcard *.mk) \
 	ext \
 	tools/app_reg \
-	tools/virtual-bootstrap \
-	tools/relocatable.sh \
+	$(INSTALL_DIR) \
 	VERS* LICENSE* README*
 
 .PHONY: install
 install: virtual-env install-check install-core-structure install-desktop install-apps install-env
 
 .PHONY: install-check
 install-check:
-	@if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
-	  echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
+	@if [ -n '$(wildcard $(INST_DIR_ENV)/*)' ] ; then \
+	  echo "ERROR: $(INST_DIR_ENV) is not empty. Contents:" ; \
+	  ls -la "$(INST_DIR_ENV)" ; \
+	  echo 'ERROR: $(INST_DIR_ENV) not empty. Cowardly refusing to continue.' ; \
 	  false ; \
 	fi
 
@@ -249,7 +263,8 @@ install-env:
 ###################################
 
 .PHONY: npm-install
-npm-install:
+npm-install: .npm-install-lock
+.npm-install-lock:
 	npm --version
 	node --version
 	npm install
@@ -258,10 +273,11 @@ npm-install:
 	npm run webpack-workers
 	node_modules/.bin/removeNPMAbsolutePaths .
 	rm -rf node_modules
+	touch .npm-install-lock
 
 .PHONY: create-static
 create-static:
-	./build/env/bin/python ./build/env/bin/hue collectstatic --noinput
+	$(ENV_PYTHON) $(BLD_DIR_BIN)/hue collectstatic --noinput
 
 # <<<< DEV ONLY
 .PHONY: doc
@@ -366,6 +382,10 @@ ext-clean:
 # Misc (some used by automated test scripts)
 ###############################################
 
+huecheck:
+	@echo "Checking for release..."
+	DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue check
+
 test:
 	DESKTOP_DEBUG=1 $(ENV_PYTHON) $(BLD_DIR_BIN)/hue test fast --with-xunit
 

Makefile.sdk

@@ -114,8 +114,7 @@ $(EXT_PY_ENV_INSTALL_TARGETS): CUR_EXT_PY=$(notdir $(@D))
 #   	Pip install any 3rd party package that are based on requirements.txt
 #
 .PHONY: ext-env-pip-install
-ext-env-pip-install: $(REQUIREMENT_DOT_FILE)
-$(REQUIREMENT_DOT_FILE):
+ext-env-pip-install:
 	@echo '--- Installing $(REQUIREMENT_FILE) into virtual-env via $(ENV_PIP)'
 	@$(ENV_PIP) install -r $(REQUIREMENT_FILE)
 	@echo '--- Finished $(REQUIREMENT_FILE) into virtual-env'
@@ -222,7 +221,7 @@ sdist:
 	@mkdir -p $(SDIST_DIR)
 	@# Copy sources of our app
 	@rsync $(RSYNC_OPT) ./ $(SDIST_DIR)/ $(SDIST_EXCLUDES)
-	@$(MAKE) -C $(SDIST_DIR) clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $(SDIST_DIR) clean
 	@# Also make a tarball
 	@tar -C $(SDIST_DIR)/.. -czf $(SDIST_TGZ) $(APP_FULL_NAME)
 	@echo "--- Created $(SDIST_TGZ)"
@@ -255,8 +254,9 @@ bdist: ext-eggs compile
 install-bdist: bdist
 	@echo "--- Install built distribution for $(APP_NAME) at $(INSTALL_DIR)"
 	@# Check that INSTALL_DIR is empty
-	@if [ -n '$(wildcard $(INSTALL_DIR)/*)' ] ; then \
-	  echo 'ERROR: $(INSTALL_DIR) not empty. Cowardly refusing to continue.' ; \
+	@echo "Prakash $(BDIST_DIR)/ $(INSTALL_DIR)"
+	@if [ -n '$(wildcard $(INST_DIR_ENV)/*)' ] ; then \
+	  echo 'ERROR: $(INST_DIR_ENV) not empty. Cowardly refusing to continue.' ; \
 	  false ; \
 	fi
 	@mkdir -p $(INSTALL_DIR)

Makefile.tarball

@@ -17,7 +17,7 @@
 
 
 # <<<< DEV ONLY
-
+# Extended packaging rules to support Python version-specific virtualenv outputs.
 PROD_INCLUDES := \
 	apps \
 	desktop \
@@ -26,10 +26,15 @@ PROD_INCLUDES := \
 	NOTICE.txt \
 	Makefile* \
 	maven \
+	build \
+	app.reg \
+	desktop/conf \
+	desktop/desktop.db \
 	tools/app_reg \
 	tools/virtual-bootstrap \
-	tools/relocatable.sh \
+	tools/relocatable.py \
 	tools/load-balancer \
+	tools/scripts \
 	VERSION \
 	webpack-stats*.json \
 	package.json \
@@ -39,13 +44,13 @@ PROD_INCLUDES := \
 	tsconfig.json
 
 # Check for things in BDIST_EXCLUDES in various apps
+# Excludes more dev-only content and handles packaging across architectures.
 PROD_EXCLUDES := \
 	apps/beeswax/thrift \
 	apps/hello \
 	apps/jframegallery \
-	build \
-	desktop/conf \
-	desktop/desktop.db \
+	desktop/core/regenerate_thrift.sh \
+	desktop/core/ext-py3 \
 	desktop/devtools.mk \
 	desktop/libs/hadoop/regenerate-thrift.sh \
 	ext/thirdparty/js/manifest.json \
@@ -57,35 +62,58 @@ define remove_devtree_exclusions
 		      -name '.gitignore' -o \
 		      -name '.*~' -o \
 		      -name '.#*' -o \
-		      -name 'build' -o \
+		      -name '.pylintrc' -o \
 		      -name 'logs' -o \
 		      -name 'tags' -o \
 		      -name 'target' -o \
-		      -name 'desktop.db' -o \
-		      -name 'app.reg' -o \
-		      -name '*.py[co]' -o \
 		      -name '.*.sw?' \) -not -path '*/boto/*' -prune -exec rm -rf {} \;
 endef
 
 
 ###################################
 # Packaging for production release
 ###################################
+.PHONY: release
+release:
+	@echo "--- Preparing general distribution tree at $@"
+	@rm -rf $(BLD_DIR_PROD) $(BLD_DIR_PROD_TGZ)
+	@mkdir -p $(BLD_DIR_PROD)
+	@tar --exclude="build/release" -chf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
+
+	@echo "---- Removing exclusions"
+	@for i in $(PROD_EXCLUDES) ; do rm -rf $(BLD_DIR_PROD)/$$i ; done
+	@echo "---- Removing *.py[co] files"
+	@find $(BLD_DIR_PROD) -iname \*.py[co]  -exec rm -f {} \;
+	@echo "---- Performing remove_devtree_exclusions"
+	$(call remove_devtree_exclusions,$(BLD_DIR_PROD))
+
+	@echo "---- Copying misc files"
+	@cp $(ROOT)/dist/* $(BLD_DIR_PROD)
+	@mv $(BLD_DIR_PROD)/desktop/conf{.dist,}
+	@cp -r $(BLD_DIR_DOC) $(BLD_DIR_PROD)/docs
+
+	@echo "---- Stripping Makefiles"
+	@$(call STRIP_DEV, $(ROOT)/Makefile, $(BLD_DIR_PROD)/Makefile)
+	@$(call STRIP_DEV, $(ROOT)/Makefile.vars.priv, $(BLD_DIR_PROD)/Makefile.vars.priv)
+	@$(call STRIP_DEV, $(APPS_DIR)/Makefile, $(BLD_DIR_PROD)/apps/Makefile)
+	@$(call STRIP_DEV, $(DESKTOP_DIR)/Makefile, $(BLD_DIR_PROD)/desktop/Makefile)
+	@tar --exclude="build/release" -C $(BLD_DIR_PROD)/.. -chzf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
+	@echo "---- $(BLD_DIR_PROD_TGZ)"
 
 # Make a tarball
 .PHONY: prod
 prod: $(BLD_DIR_PROD_TGZ)
 
 $(BLD_DIR_PROD_TGZ): $(BLD_DIR_PROD)
-	@tar -C $(BLD_DIR_PROD)/.. -czf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
+	@tar --exclude="python" --exclude="build/release/prod" -C $(BLD_DIR_PROD)/.. -czf $(BLD_DIR_PROD_TGZ) hue-$(DESKTOP_VERSION)
 	@echo "--- Generated $(BLD_DIR_PROD_TGZ)"
 
 .PHONY: $(BLD_DIR_PROD)
 $(BLD_DIR_PROD): apps docs locales
 	@echo "--- Preparing general distribution tree at $@"
 	@rm -rf $@
 	@mkdir -p $@
-	tar -cf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
+	tar --exclude="python" --exclude="build/release/prod" -cf - $(PROD_INCLUDES) | tar -C $(BLD_DIR_PROD) -xf -
 
 	@echo "---- Removing exclusions"
 	@for i in $(PROD_EXCLUDES) ; do rm -rf $(BLD_DIR_PROD)/$$i ; done
@@ -103,6 +131,6 @@ $(BLD_DIR_PROD): apps docs locales
 	@$(call STRIP_DEV, $(DESKTOP_DIR)/Makefile, $(BLD_DIR_PROD)/desktop/Makefile)
 	@echo "---- Removing build directories"
 	find $(BLD_DIR_PROD) -name build -prune -exec rm -rf {} \;
-	@$(MAKE) -C $(BLD_DIR_PROD) ext-clean
+	@PYTHON_VER=$(PYTHON_VER) ENV_PYTHON=$(ENV_PYTHON) VIRTUAL_ENV=$(BLD_DIR_ENV) $(MAKE) -C $(BLD_DIR_PROD) ext-clean
 
 # END DEV ONLY >>>>