Skip to content

Commit df5aca8

Browse files
amitsrivastavaamitsrivastava
committed
[multi-python] scripts to simplify the selection of python version
1. `tools/scripts/hue.sh` is the wrapper script to call the hue script, that is currently in `build/env/bin/hue` but could be in various locations depending on python version in use. 2. `tools/scripts/python/python_helper.sh` contains helper functions - `find_latest_python` returns the latest py version (3.11, 3.9, 3.8) - `latest_venv_bin_path` returns the path to the env/bin of the latest python, relative to the top-level hue directory. 3. `tools/scripts/python/hue_py_shebang.sh` is a helper script that is supposed to be used in the shebang line of python scripts in Hue. I can direct the python script to the correct python to be used. 4. `HUE_PYTHON_VERSION` environment variable can be set to override the automatic selection of latest available python.
1 parent 5fdca09 commit df5aca8

File tree

3 files changed

+191
-0
lines changed

3 files changed

+191
-0
lines changed

‎tools/scripts/hue.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
# Licensed to Cloudera, Inc. under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. Cloudera, Inc. licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# wrapper script to identify the latest python and pass the execution
19+
# to the corresponding hue script in the `build` directory.
20+
21+
set -ex
22+
23+
# Time marker for both stderr and stdout
24+
date; date 1>&2
25+
26+
export SCRIPT_DIR=`dirname $0`
27+
export HUE_HOME_DIR=$(dirname $(dirname "$SCRIPT_DIR"))
28+
29+
source $SCRIPT_DIR/python/python_helper.sh
30+
31+
function stop_previous_hueprocs() {
32+
for p in $(cat /tmp/hue_${HUE_PORT}.pid); do
33+
if [[ $p -eq $(ps -p $p -ho pid=) ]]; then
34+
kill -9 $p
35+
fi
36+
done
37+
}
38+
39+
PYTHON_BIN="${HUE_HOME_DIR}/$(latest_venv_bin_path)/python"
40+
HUE="${HUE_HOME_DIR}/$(latest_venv_bin_path)/hue"
41+
HUE_LOGLISTENER="${HUE_HOME_DIR}/desktop/core/src/desktop/loglistener.py"
42+
43+
if [[ "dumpdata" == "$1" ]]; then
44+
umask 037
45+
"$HUE" "$1" --indent 2 > "$2"
46+
elif [[ "syncdb" == "$1" ]]; then
47+
run_syncdb_and_migrate_subcommands
48+
elif [[ "ldaptest" == "$1" ]]; then
49+
"$HUE" "$1"
50+
elif [[ "runcpserver" == "$1" ]]; then
51+
exec "$HUE" "runcpserver"
52+
elif [[ "rungunicornserver" == "$1" ]]; then
53+
stop_previous_hueprocs
54+
exec "$PYTHON_BIN" "$HUE_LOGLISTENER" &
55+
echo $! > /tmp/hue_${HUE_PORT}.pid
56+
exec "$HUE" "rungunicornserver"
57+
else
58+
exec "$HUE" "$@"
59+
fi

‎tools/scripts/python/hue_py_shebang.sh

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Licensed to Cloudera, Inc. under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. Cloudera, Inc. licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# This script is supposed to be used in the shebang line of python scripts
19+
# It will choose the correct python version and corresponding build dir
20+
set -ex
21+
22+
SCRIPT_DIR=`dirname $0`
23+
export HUE_HOME_DIR=$(dirname $(dirname $(dirname "$SCRIPT_DIR")))
24+
25+
source $SCRIPT_DIR/python_helper.sh
26+
27+
PYTHON_BIN="${HUE_HOME_DIR}/$(latest_venv_bin_path)/python"
28+
29+
exec "$PYTHON_BIN" "$@"

‎tools/scripts/python/python_helper.sh

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#!/bin/bash
2+
# Licensed to Cloudera, Inc. under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. Cloudera, Inc. licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# Some helper functions to find the latest python bin and venv path
19+
set -ex
20+
21+
# Find the latest Python binary in build/env/bin
22+
LATEST_PYTHON=$(find "$HUE_HOME_DIR/build/env/bin" -name "python3*" -exec basename {} \; | sort -V | tail -n 1)
23+
24+
# Extract version from the latest python binary (e.g., python3.11 → 3.11)
25+
LATEST_PYTHON_VERSION=$(echo "$LATEST_PYTHON" | grep -oP '\d+\.\d+')
26+
27+
# Find all supported python versions from build/venvs and include latest version
28+
readarray -t SUPPORTED_VERSIONS < <(
29+
(
30+
find "$HUE_HOME_DIR/build/venvs" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | grep -oP '\d+\.\d+';
31+
echo "$LATEST_PYTHON_VERSION"
32+
) | sort -Vr | uniq
33+
)
34+
35+
# Create array of supported versions
36+
SUPPORTED_PYTHON_VERSIONS=("${SUPPORTED_VERSIONS[@]}")
37+
38+
python_bin_path() {
39+
# Searches for the provided python version in various locations on the node
40+
PYTHON_VERSION="$1"
41+
PYTHON_VERSION_NO_DOT="${PYTHON_VERSION//./}"
42+
43+
# First check in PATH if it exists
44+
if [ -n "$PATH" ]; then
45+
IFS=: read -ra PATH_DIRS <<< "$PATH"
46+
for DIR in "${PATH_DIRS[@]}"; do
47+
# Skip cm-agent/bin directories
48+
if [[ "$DIR" == *"cm-agent/bin"* ]]; then
49+
continue
50+
fi
51+
PY_PATH="$DIR/python$PYTHON_VERSION"
52+
if [[ -x "$PY_PATH" ]]; then
53+
echo "$PY_PATH"
54+
return 0
55+
fi
56+
done
57+
fi
58+
59+
# Fall back to hardcoded directories
60+
SEARCH_DIRS=("/usr/local/bin" "/bin" "/usr/bin" "/opt/rh/rh-${PYTHON_VERSION_NO_DOT}/root/usr/bin")
61+
for DIR in "${SEARCH_DIRS[@]}"; do
62+
PY_PATH="$DIR/python$PYTHON_VERSION"
63+
if [[ -x "$PY_PATH" ]]; then
64+
echo "$PY_PATH"
65+
return 0
66+
fi
67+
done
68+
return 1
69+
}
70+
71+
find_latest_python() {
72+
# returns the latest py version, e.g., 3.11, 3.9 or 3.8
73+
for PYTHON_VERSION in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
74+
if python_bin_path "$PYTHON_VERSION" > /dev/null; then
75+
echo "$PYTHON_VERSION"
76+
return 0
77+
fi
78+
done
79+
80+
echo "No supported Python versions found in expected locations."
81+
return 1
82+
}
83+
84+
latest_venv_bin_path() {
85+
# returns the path to the env/bin of the latest python,
86+
# relative to the top-level hue directory.
87+
local version
88+
if [ -n "$HUE_PYTHON_VERSION" ]; then
89+
version=$(echo "$HUE_PYTHON_VERSION" | grep -oP '\d+\.\d+')
90+
else
91+
version="$(find_latest_python)"
92+
fi
93+
94+
if [ -z "$version" ]; then
95+
return 1 # Return error if no version provided/found
96+
fi
97+
98+
if [ "$version" = "$LATEST_PYTHON_VERSION" ]; then
99+
echo "build/env/bin"
100+
else
101+
echo "build/venvs/python${version}/bin"
102+
fi
103+
}

0 commit comments

Comments
 (0)