From 1d489e50cdfea8d3b836e25a2245547c6fa8717e Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 15:42:35 +0100 Subject: [PATCH 1/7] flow sdist->wheel --- .github/workflows/build.yml | 121 ++++++++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c2ca8e4bd..b165df011 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,13 +43,9 @@ jobs: packages/${{ matrix.package }}/dist/*.whl name: dist-${{ matrix.package }} - build_basemap: - name: Build basemap package (${{ matrix.os }}) - needs: [build_data] - strategy: - matrix: - os: [ubuntu-22.04, windows-2019, macos-13, macos-14] - runs-on: ${{ matrix.os }} + build_sdist: + name: Build basemap sdist + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -59,13 +55,83 @@ jobs: python-version: "3.9" - name: Build sdist - if: matrix.os == 'ubuntu-22.04' run: | cd packages/basemap python -m pip install build python -m build --sdist - - name: Build wheels + - uses: actions/upload-artifact@v4 + with: + path: packages/basemap/dist/*.tar.gz + name: basemap-sdist + + build_wheels: + name: Build wheels on ${{ matrix.os }} + needs: [build_data, build_sdist] + strategy: + matrix: + os: [ubuntu-22.04, windows-2019, macos-13, macos-14] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Download basemap sdist + uses: actions/download-artifact@v4 + with: + name: basemap-sdist + path: ./sdist/ + + - name: Extract sdist (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: | + # Create extraction directory in the workspace + mkdir -p ./sdist_extract + + # Extract using tar (Unix-style) + tar -xvf ./sdist/*.tar.gz -C ./sdist_extract + + # Get the extracted directory name + EXTRACTED_DIR=$(ls -d ./sdist_extract/*/ | head -1) + echo "SDIST_DIR=$(pwd)/${EXTRACTED_DIR}" >> $GITHUB_ENV + + # Verify contents + ls -la ${EXTRACTED_DIR} + + - name: Extract sdist (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + # Create extraction directory + New-Item -ItemType Directory -Force -Path "sdist_extract" + + # Find the tarball file (without using wildcards) + $tarball = Get-ChildItem -Path "sdist" -Filter "*.tar.gz" | Select-Object -First 1 + + # Debug - show what we found + Write-Host "Found tarball: $($tarball.FullName)" + + # Extract using the specific file path (not wildcard) + tar -xvf $tarball.FullName -C "sdist_extract" + + # Get the extracted directory name + $extractedDir = (Get-ChildItem -Path "sdist_extract" -Directory | Select-Object -First 1).FullName + + # Debug - show what we found + Write-Host "Extracted directory: $extractedDir" + + # Set the environment variable + echo "SDIST_DIR=$extractedDir" | Out-File -FilePath $env:GITHUB_ENV -Append + + # Verify contents + Get-ChildItem $extractedDir + + - name: Build wheels from sdist uses: pypa/cibuildwheel@v2.22.0 env: CIBW_ARCHS: "native" @@ -82,22 +148,18 @@ jobs: PIP_PREFER_BINARY=1 PYTHONUNBUFFERED=1 LD_LIBRARY_PATH="${GEOS_DIR}/lib" - # LD_LIBRARY_PATH in environment is needed by - # auditwheel (Linux) and delocate (MacOS). with: - package-dir: "packages/basemap" - output-dir: "packages/basemap/dist" + package-dir: ${{ env.SDIST_DIR }} # Use extracted sdist + output-dir: "dist" - uses: actions/upload-artifact@v4 with: - path: | - packages/basemap/dist/*.tar.gz - packages/basemap/dist/*.whl - name: dist-basemap-${{ matrix.os }} + path: dist/*.whl + name: dist-basemap-wheels-${{ matrix.os }} check: name: Check packages - needs: [build_data, build_basemap] + needs: [build_data, build_sdist, build_wheels] runs-on: ubuntu-22.04 steps: - uses: actions/download-artifact@v4 @@ -106,6 +168,11 @@ jobs: pattern: "dist-*" merge-multiple: true + - uses: actions/download-artifact@v4 + with: + path: dist + name: basemap-sdist + - name: Set up Python uses: actions/setup-python@v5 with: @@ -117,9 +184,20 @@ jobs: python -m twine check dist/*.tar.gz python -m twine check dist/*.whl + # Verification step to ensure sdist is complete + - name: Verify sdist can build wheel + run: | + python -m pip install build setuptools wheel + mkdir -p /tmp/sdist_test + tar -xvf dist/*.tar.gz -C /tmp/sdist_test + cd /tmp/sdist_test/*/ + python -m pip install -e . + python -m build --wheel + ls -la dist/*.whl + upload: name: Upload packages - needs: [build_data, build_basemap, check] + needs: [build_data, build_sdist, build_wheels, check] runs-on: ubuntu-22.04 environment: PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') @@ -130,6 +208,11 @@ jobs: pattern: "dist-*" merge-multiple: true + - uses: actions/download-artifact@v4 + with: + path: dist + name: basemap-sdist + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: From b86db26aa140d69f1f8b42ce8060d72e05d695bd Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 16:05:08 +0100 Subject: [PATCH 2/7] build data packages directly --- .github/workflows/build.yml | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b165df011..b1f8cc175 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,6 +86,57 @@ jobs: name: basemap-sdist path: ./sdist/ + - name: Download data packages + uses: actions/download-artifact@v4 + with: + pattern: dist-basemap_data* + path: ./data_packages/ + merge-multiple: true + + - name: Install data packages (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: | + # Debug - show what we downloaded + ls -la ./data_packages/ + + # Install the data packages + python -m pip install ./data_packages/*.whl + + # Verify they're installed + python -c "import mpl_toolkits.basemap_data; print('Data package installed')" + python -c "import mpl_toolkits.basemap_data_hires; print('Hires data package installed')" || echo "Optional hires package not installed" + + # Install the data packages (Windows) + - name: Install data packages (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + # Debug - show what we downloaded + Get-ChildItem -Path "./data_packages" -Recurse + + # Find all wheel files + $wheels = Get-ChildItem -Path "./data_packages" -Filter "*.whl" -Recurse + + # Install each wheel file + foreach ($wheel in $wheels) { + Write-Host "Installing $($wheel.FullName)" + python -m pip install $wheel.FullName + } + + # Verify they're installed + try { + python -c "import mpl_toolkits.basemap_data; print('Data package installed')" + } catch { + Write-Host "Error importing basemap_data" + } + + try { + python -c "import mpl_toolkits.basemap_data_hires; print('Hires data package installed')" + } catch { + Write-Host "Optional hires package not installed" + } + - name: Extract sdist (Linux/macOS) if: runner.os != 'Windows' shell: bash @@ -148,6 +199,8 @@ jobs: PIP_PREFER_BINARY=1 PYTHONUNBUFFERED=1 LD_LIBRARY_PATH="${GEOS_DIR}/lib" + # LD_LIBRARY_PATH in environment is needed by + # auditwheel (Linux) and delocate (MacOS). with: package-dir: ${{ env.SDIST_DIR }} # Use extracted sdist output-dir: "dist" From 8db731f3bfdb1f4353771a190fe0241c0d32f76e Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 16:10:00 +0100 Subject: [PATCH 3/7] attempt fix install windows --- .github/workflows/build.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b1f8cc175..cb13bdbf1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -105,7 +105,6 @@ jobs: # Verify they're installed python -c "import mpl_toolkits.basemap_data; print('Data package installed')" - python -c "import mpl_toolkits.basemap_data_hires; print('Hires data package installed')" || echo "Optional hires package not installed" # Install the data packages (Windows) - name: Install data packages (Windows) @@ -124,18 +123,12 @@ jobs: python -m pip install $wheel.FullName } - # Verify they're installed - try { - python -c "import mpl_toolkits.basemap_data; print('Data package installed')" - } catch { - Write-Host "Error importing basemap_data" - } + # Show installed packages + python -m pip list | Select-String "mpl_toolkits.basemap" - try { - python -c "import mpl_toolkits.basemap_data_hires; print('Hires data package installed')" - } catch { - Write-Host "Optional hires package not installed" - } + # Try different import paths + Write-Host "Trying to import basemap_data..." + python -c "import mpl_toolkits.basemap_data; print('mpl_toolkits.basemap_data imported successfully')" - name: Extract sdist (Linux/macOS) if: runner.os != 'Windows' From 026ceea20daff80f56446d2c5336dea45f5b4a18 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 16:38:02 +0100 Subject: [PATCH 4/7] update verification step --- .github/workflows/build.yml | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb13bdbf1..86a4ef736 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -235,11 +235,43 @@ jobs: run: | python -m pip install build setuptools wheel mkdir -p /tmp/sdist_test - tar -xvf dist/*.tar.gz -C /tmp/sdist_test + + # List all files in dist directory + echo "Files in dist directory:" + ls -la dist/ + + # Extract only the basemap sdist + BASEMAP_SDIST=$(ls dist/basemap-*.tar.gz 2>/dev/null || echo "") + + if [ -z "$BASEMAP_SDIST" ]; then + echo "Basemap sdist not found with pattern 'basemap-*.tar.gz', trying alternative patterns..." + BASEMAP_SDIST=$(ls dist/*basemap*.tar.gz 2>/dev/null | head -1 || echo "") + fi + + if [ -z "$BASEMAP_SDIST" ]; then + echo "ERROR: Could not find any basemap sdist" + exit 1 + fi + + echo "Using sdist: $BASEMAP_SDIST" + + # Extract just the one sdist file + tar -xvf "$BASEMAP_SDIST" -C /tmp/sdist_test + + # Enter extracted directory cd /tmp/sdist_test/*/ + + # Install and build python -m pip install -e . python -m build --wheel - ls -la dist/*.whl + + # Check for built wheel + if [ -d "dist" ]; then + ls -la dist/*.whl || echo "No wheels found in dist directory" + else + echo "No dist directory created" + exit 1 + fi upload: name: Upload packages From 76ed633aab944a4be56a551e05790f4b961c2f96 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 16:54:02 +0100 Subject: [PATCH 5/7] simplify verification --- .github/workflows/build.yml | 43 +++++++++---------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 86a4ef736..fcaf1336c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -230,46 +230,23 @@ jobs: python -m twine check dist/*.tar.gz python -m twine check dist/*.whl - # Verification step to ensure sdist is complete - - name: Verify sdist can build wheel + - name: Verify sdist content run: | - python -m pip install build setuptools wheel mkdir -p /tmp/sdist_test - # List all files in dist directory - echo "Files in dist directory:" - ls -la dist/ - - # Extract only the basemap sdist - BASEMAP_SDIST=$(ls dist/basemap-*.tar.gz 2>/dev/null || echo "") - - if [ -z "$BASEMAP_SDIST" ]; then - echo "Basemap sdist not found with pattern 'basemap-*.tar.gz', trying alternative patterns..." - BASEMAP_SDIST=$(ls dist/*basemap*.tar.gz 2>/dev/null | head -1 || echo "") - fi - - if [ -z "$BASEMAP_SDIST" ]; then - echo "ERROR: Could not find any basemap sdist" - exit 1 - fi - - echo "Using sdist: $BASEMAP_SDIST" - - # Extract just the one sdist file + # Find and extract basemap sdist + BASEMAP_SDIST=$(ls dist/basemap-*.tar.gz 2>/dev/null || ls dist/*basemap*.tar.gz 2>/dev/null | head -1) tar -xvf "$BASEMAP_SDIST" -C /tmp/sdist_test - # Enter extracted directory - cd /tmp/sdist_test/*/ - - # Install and build - python -m pip install -e . - python -m build --wheel + # Verify contents + echo "Files in extracted sdist:" + find /tmp/sdist_test -type f | grep -v "__pycache__" | sort - # Check for built wheel - if [ -d "dist" ]; then - ls -la dist/*.whl || echo "No wheels found in dist directory" + # Check for critical files + if [ -f "$(find /tmp/sdist_test -name "_geoslib.pyx")" ]; then + echo "✓ Source files verified in sdist" else - echo "No dist directory created" + echo "✗ Missing critical source files in sdist" exit 1 fi From 80476d7855213654d5c840bc41247e5f1165f5e2 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 17:19:04 +0100 Subject: [PATCH 6/7] add doc building --- .github/workflows/build.yml | 85 +++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fcaf1336c..f3549b87d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -250,6 +250,91 @@ jobs: exit 1 fi + docs: + name: Build documentation + needs: [build_wheels] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Download basemap wheel for Linux + uses: actions/download-artifact@v4 + with: + name: dist-basemap-wheels-ubuntu-22.04 + path: ./wheels/ + + - name: Download data packages + uses: actions/download-artifact@v4 + with: + pattern: dist-basemap_data* + path: ./data_packages/ + merge-multiple: true + + - name: Install packages + run: | + # Get Python version to find matching wheel + PYTHON_VERSION=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + echo "Using Python $PYTHON_VERSION" + + # Install matching wheel for Linux + MATCHING_WHEEL=$(find ./wheels -name "basemap-*-cp3${PYTHON_VERSION/./}*.whl" | head -1) + if [ -n "$MATCHING_WHEEL" ]; then + echo "Installing wheel: $MATCHING_WHEEL" + python -m pip install "$MATCHING_WHEEL" + else + echo "No matching wheel found for Python $PYTHON_VERSION. Available wheels:" + ls -la ./wheels/ + echo "Falling back to installing any available wheel..." + python -m pip install ./wheels/*.whl + fi + + # Install data packages + echo "Installing data packages..." + python -m pip install ./data_packages/*.whl + + - name: Install documentation requirements + run: | + cd packages/basemap + python -m pip install -r requirements-doc.txt + + - name: Build documentation + run: | + cd packages/basemap + python -m sphinx doc/source public + + - name: Upload docs artifacts + uses: actions/upload-artifact@v4 + with: + name: docs-artifact + path: packages/basemap/public + + - name: Upload github-pages artifact + uses: actions/upload-pages-artifact@v3 + with: + name: github-pages + path: packages/basemap/public + + pages: + name: Deploy documentation + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + needs: [docs, check] + runs-on: ubuntu-22.04 + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + permissions: + pages: write + id-token: write + steps: + - name: Deploy github-pages + uses: actions/deploy-pages@v3 + id: deployment + upload: name: Upload packages needs: [build_data, build_sdist, build_wheels, check] From db61f75c247f8da07c7380b032526c6a7b5d4616 Mon Sep 17 00:00:00 2001 From: cvanelteren Date: Wed, 26 Feb 2025 18:12:31 +0100 Subject: [PATCH 7/7] ensure correct version is used --- .github/workflows/build.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f3549b87d..03d0a5a47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -277,26 +277,33 @@ jobs: - name: Install packages run: | - # Get Python version to find matching wheel - PYTHON_VERSION=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") - echo "Using Python $PYTHON_VERSION" + # Get Python version + PY_VER=$(python -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')") + echo "Using Python $PY_VER" + + # List available wheels + echo "Available wheels:" + ls -la ./wheels/ + + # Find wheel matching current Python version + MATCHING_WHEEL=$(find ./wheels -name "*-cp${PY_VER}-cp${PY_VER}*" | head -1) - # Install matching wheel for Linux - MATCHING_WHEEL=$(find ./wheels -name "basemap-*-cp3${PYTHON_VERSION/./}*.whl" | head -1) if [ -n "$MATCHING_WHEEL" ]; then - echo "Installing wheel: $MATCHING_WHEEL" + echo "Installing matching wheel: $MATCHING_WHEEL" python -m pip install "$MATCHING_WHEEL" else - echo "No matching wheel found for Python $PYTHON_VERSION. Available wheels:" - ls -la ./wheels/ - echo "Falling back to installing any available wheel..." + echo "No matching wheel found for Python $PY_VER. Installing any available wheel..." python -m pip install ./wheels/*.whl fi # Install data packages - echo "Installing data packages..." + echo "Installing data packages:" + ls -la ./data_packages/ python -m pip install ./data_packages/*.whl + # Verify installation + python -c "import mpl_toolkits.basemap; print('Basemap version:', mpl_toolkits.basemap.__version__)" + - name: Install documentation requirements run: | cd packages/basemap