Appveyor wheel (#4)

pkittenis · web-flow · commit cea330ebdca0 · 2017-08-21T23:12:53.000+01:00
Add binary wheel building on Windows for all Python versions and PyPi upload on tags.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -1,46 +1,120 @@
 environment:
+  global:
+    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+    # /E:ON and /V:ON options are not enabled in the batch script intepreter
+    # See: http://stackoverflow.com/a/13751649/163740
+    CMD_IN_ENV: "cmd /E:ON /V:ON /C %APPVEYOR_BUILD_FOLDER%\\ci\\appveyor\\run_with_env.cmd"
+    PYTHONUNBUFFERED: 1
+    EMBEDDED_LIB: 1
+    PYPI_USER:
+      secure: 2m0jy6JD/R9RExIosOT6YA==
+    PYPI_PASS:
+      secure: x+dF0A8BZUf2IrPNRN1O0w==
   matrix:
-    - TARGET_ARCH: "x64"
-      CONDA_PY: "27"
-      PY_CONDITION: "python >=2.7,<3"
-      CONDA_INSTALL_LOCN: "C:\\Miniconda-x64"
-    - TARGET_ARCH: "x64"
-      CONDA_PY: "35"
-      PY_CONDITION: "python >=3.5,<3.6"
-      CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64"
-    - TARGET_ARCH: "x64"
-      CONDA_PY: "36"
-      PY_CONDITION: "python >=3.6"
-      CONDA_INSTALL_LOCN: "C:\\Miniconda36-x64"
-
-matrix:
-  fast_finish: true
-
-cache:
-  - "%TMP%\\py\\"
-
-build: false
+    - PYTHON: "C:\\Python27"
+      PYTHON_VERSION: "2.7"
+      PYTHON_ARCH: "32"
+      MSVC: "Visual Studio 9"
+
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7"
+      PYTHON_ARCH: "64"
+      MSVC: "Visual Studio 9"
+
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4"
+      PYTHON_ARCH: "32"
+      MSVC: "Visual Studio 10"
+
+    - PYTHON: "C:\\Python34-x64"
+      PYTHON_VERSION: "3.4"
+      PYTHON_ARCH: "64"
+      MSVC: "Visual Studio 10 Win64"
+
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5"
+      PYTHON_ARCH: "32"
+      MSVC: "Visual Studio 14"
+
+    - PYTHON: "C:\\Python35-x64"
+      PYTHON_VERSION: "3.5"
+      PYTHON_ARCH: "64"
+      MSVC: "Visual Studio 14 Win64"
+
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6"
+      PYTHON_ARCH: "32"
+      MSVC: "Visual Studio 14"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6"
+      PYTHON_ARCH: "64"
+      MSVC: "Visual Studio 14 Win64"
+
 install:
-  - set CONDA_NPY=19
-  # Remove cygwin (and therefore the git that comes with it).
-  - rmdir C:\cygwin /s /q
-  # Use the pre-installed Miniconda for the desired arch
-  - set PATH=%CONDA_INSTALL_LOCN%/Library/bin;%CONDA_INSTALL_LOCN%/Scripts;%PATH%
-  - conda update --yes --quiet conda
-  - call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
-  - conda config --add channels conda-forge
-  - conda config --set show_channel_urls true
-  - conda install --yes --quiet conda-build-all
-  - conda update --yes conda-build
-  - conda install --yes --quiet conda-forge-build-setup
-  - run_conda_forge_build_setup
-  - conda build conda-recipe
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  # credits: JuliaLang developers.
+  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
+        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
+        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
+          throw "There are newer queued builds for this pull request, failing early." }
+  - ECHO "Installed SDKs:"
+  - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\""
+
+  # Prepend newly installed Python to the PATH of this build (this cannot be
+  # done from inside the powershell script as it would require to restart
+  # the parent CMD process).
+  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+
+  # Check that we have the expected version and architecture for Python
+  - "python --version"
+  - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+  # Upgrade to the latest version of pip to avoid it displaying warnings
+  # about it being out of date.
+  - "pip install --disable-pip-version-check --user --upgrade pip"
+
+  # Install the build dependencies of the project. If some dependencies contain
+  # compiled extensions and are not provided as pre-built wheel packages,
+  # pip will build them from source using the MSVC compiler matching the
+  # target Python version and architecture
+  - "%CMD_IN_ENV% pip install -r requirements_dev.txt"
+  - "%CMD_IN_ENV% pip install -U wheel setuptools twine"
+  - git submodule update --init --recursive
+
+  # .c files need to be generated on Windows to handle platform
+  # specific code.
+  # Fix version used by versioneer to current git tag so the generated .c files
+  # do not cause a version change.
+  - python ci/appveyor/fix_version.py .
+  - mv -f .git .git.bak
+
+build_script:
+  # Build the compiled extension
+  - "%CMD_IN_ENV% ci\\appveyor\\build_ssh2.bat"
+  - rm -f ssh2/*.c
+  - "%CMD_IN_ENV% python setup.py build"
+  - "%CMD_IN_ENV% python setup.py build_ext -i"
 
 test_script:
-  - python ci\move-conda-package.py
+  - python -c "from ssh2.session import Session; Session()"
 
-platform:
-  - x64
+after_test:
+  # If tests are successful, create binary packages for the project.
+  - "%CMD_IN_ENV% python setup.py bdist_wheel"
+  - "%CMD_IN_ENV% python setup.py bdist_wininst"
+  - mv dist/* .
+  # Create 'none' ABI dist packages for use by older pip/wheel versions
+  - python ci/copy_abi_dist.py *.whl
 
 artifacts:
-  - path: '*.tar.bz2'
+  # Archive the generated packages in the ci.appveyor.com build report.
+  - path: "*.whl"
+  - path: "*.exe"
+
+deploy_script:
+  - python ci/appveyor/pypi_upload.py *.whl
+  - python ci/appveyor/pypi_upload.py *.exe
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "libssh2"]
+	path = libssh2
+	url = https://github.com/libssh2/libssh2.git
diff --git a/ci/appveyor/anaconda_upload.py b/ci/appveyor/anaconda_upload.py
@@ -0,0 +1,13 @@
+import sys
+import os
+from glob import glob
+import subprocess
+import traceback
+
+
+cmd = ['anaconda', '-t', os.environ['ANACONDA_TOKEN'], 'upload']
+cmd.extend(glob('*.tar.bz2'))
+try:
+    subprocess.check_call(cmd)
+except subprocess.CalledProcessError:
+    sys.exit(1)
diff --git a/ci/appveyor/build_ssh2.bat b/ci/appveyor/build_ssh2.bat
@@ -0,0 +1,22 @@
+mkdir src && cd src
+
+IF "%MSVC%" == "Visual Studio 9" (
+   ECHO "Building without platform set"
+   cmake ..\libssh2 -G "NMake Makefiles"        ^
+   	 -DCMAKE_BUILD_TYPE=Release             ^
+	 -DCRYPTO_BACKEND=WinCNG                ^
+	 -DBUILD_SHARED_LIBS=OFF
+) ELSE (
+   ECHO "Building with platform %MSVC%"
+   cmake ..\libssh2 -G "NMake Makefiles"        ^
+   	 -DCMAKE_BUILD_TYPE=Release             ^
+	 -DCRYPTO_BACKEND=WinCNG                ^
+	 -G"%MSVC%"                             ^
+	 -DBUILD_SHARED_LIBS=OFF
+)
+
+cmake --build . --config Release
+cd ..
+ls src/src
+cp src/src/libssh2.lib %PYTHON%/libs/ || cp src/src/Release/libssh2.lib %PYTHON%/libs/
+ls %PYTHON%/libs/
diff --git a/ci/appveyor/fix_version.py b/ci/appveyor/fix_version.py
@@ -0,0 +1,39 @@
+import os
+from datetime import datetime
+import subprocess
+import json
+import sys
+
+def get_describe_tag():
+    return subprocess.check_output(['git', 'describe', '--tags']).strip().decode('utf-8')
+
+def make_version_file(basedir):
+    # import ipdb; ipdb.set_trace()
+    rev = os.environ['APPVEYOR_REPO_COMMIT']
+    basedir = os.path.abspath(basedir)
+    git_desc = get_describe_tag()
+    version_json = {'date': datetime.now().isoformat(),
+                    'dirty': False,
+                    'error': None,
+                    'full-revisionid': rev,
+                    'version': git_desc}
+    data = """
+import json
+
+version_json = '''
+%s'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)
+
+""" % (json.dumps(version_json))
+    with open(os.path.join(basedir, 'ssh2', '_version.py'), 'w') as fh:
+        fh.write(data)
+
+
+if __name__ == "__main__":
+    if not len(sys.argv) > 1:
+        sys.stderr.write("Need basedir of repo" + os.linesep)
+        sys.exit(1)
+    make_version_file(sys.argv[1])
diff --git a/ci/appveyor/pypi_upload.py b/ci/appveyor/pypi_upload.py
@@ -0,0 +1,21 @@
+from __future__ import print_function
+
+import sys
+import subprocess
+import os
+
+
+def upload_pypi(files):
+    _user, _pass = os.environ['PYPI_USER'], os.environ['PYPI_PASS']
+    try:
+        subprocess.check_call(['twine', 'upload', '-u', _user,
+                               '-p', _pass, files])
+    except Exception:
+        sys.stderr.write("Error uploading to PyPi" + os.linesep)
+
+
+if __name__ == "__main__":
+    if not len(sys.argv) > 1:
+        sys.stderr.write("Need files to upload argument" + os.linesep)
+        sys.exit(1)
+    upload_pypi(os.path.abspath(sys.argv[1]))
diff --git a/ci/appveyor/run_with_env.cmd b/ci/appveyor/run_with_env.cmd
@@ -1,4 +1,11 @@
-:: To build extensions for 64 bit Python 3, we need to configure environment
+:: To build extensions for 64 bit Python 3.5 or later no special environment needs
+:: to be configured for the Python extension code alone, however, all dependent
+:: libraries also need to be compiled with the same SDK in order to be able to
+:: link them together. 
+::
+:: This script sets SDK version and environment for all commands.
+::
+:: To build extensions for 64 bit Python 3.4 or earlier, we need to configure environment
 :: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
 :: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
 ::
@@ -13,35 +20,52 @@
 ::
 :: More details at:
 :: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
-:: http://stackoverflow.com/a/13751649/163740
+:: https://stackoverflow.com/a/13751649/163740
 ::
-:: Author: Olivier Grisel
-:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
+:: Original Author: Olivier Grisel
+:: License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/
+:: This version based on updates for python 3.5 by Phil Elson at:
+::     https://github.com/pelson/Obvious-CI/tree/master/scripts
+:: Further updates to always correctly set SDK version and environment so
+:: that compiled library dependencies use same SDK as Python extension.
+
 @ECHO OFF
 
 SET COMMAND_TO_RUN=%*
 SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
 
 SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%"
+SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
 IF %MAJOR_PYTHON_VERSION% == "2" (
     SET WINDOWS_SDK_VERSION="v7.0"
+    SET SET_SDK_64=Y
 ) ELSE IF %MAJOR_PYTHON_VERSION% == "3" (
     SET WINDOWS_SDK_VERSION="v7.1"
+    IF %MINOR_PYTHON_VERSION% LEQ 4 (
+        SET SET_SDK_64=Y
+    ) ELSE (
+        SET SET_SDK_64=N
+    )
 ) ELSE (
     ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
     EXIT 1
 )
 
+"%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+
 IF "%PYTHON_ARCH%"=="64" (
-    ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
-    SET DISTUTILS_USE_SDK=1
-    SET MSSdk=1
-    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
+    IF %SET_SDK_64% == Y (
+        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
+        SET DISTUTILS_USE_SDK=1
+        SET MSSdk=1
+    )
+    ECHO Setting MSVC %WINDOWS_SDK_VERSION% build environment for 64 bit architecture
     "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
     ECHO Executing: %COMMAND_TO_RUN%
     call %COMMAND_TO_RUN% || EXIT 1
 ) ELSE (
-    ECHO Using default MSVC build environment for 32 bit architecture
+    ECHO Setting MSVC %WINDOWS_SDK_VERSION% build environment for 32 bit architecture
+    "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x86 /release
     ECHO Executing: %COMMAND_TO_RUN%
     call %COMMAND_TO_RUN% || EXIT 1
 )
diff --git a/ci/copy_abi_dist.py b/ci/copy_abi_dist.py
@@ -0,0 +1,25 @@
+from __future__ import print_function
+
+import os
+from glob import glob
+import re
+import sys
+import shutil
+
+
+re_c = re.compile(r'.+-.+-.+-(.+)-.+\.')
+
+
+def rename_dist_files(files):
+    for _file in glob(files):
+        match = re_c.match(_file)
+        abi = match.group(1)
+        new_file = _file.replace(abi, 'none')
+    print("Copying %s to new file %s" % (_file, new_file))
+    shutil.copy2(_file, new_file)
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        sys.stderr.write("Need files argument" + os.linesep)
+        sys.exit(1)
+    rename_dist_files(os.path.abspath(sys.argv[1]))
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
@@ -1,27 +1,22 @@
 {% set name = "ssh2-python" %}
 {% set version = GIT_DESCRIBE_TAG %}
-# {% set sha256 = "e5e77801a7a0d49799e288ecbd189b4d1c23eeadcf9c6f4ce80b31b7d397babe" %}
 
 package:
   name: {{ name|lower }}
   version: {{ version }}
 
 source:
-  path: ../
-  # fn: {{ name }}-{{ version }}.tar.gz
-  # url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz
-  # sha256: {{ sha256 }}
+  git_url: https://github.com/ParallelSSH/ssh2-python.git
 
 build:
   features:
     - vc9  # [win and py27]
     - vc10  # [win and py34]
     - vc14  # [win and py35]
-  # skip: true  # [win]
   number: 0
   script:
-    - python setup.py install --single-version-externally-managed --record record.txt  # [unix]
-    - rm -f ssh2/*.c  # [win]
+    - rm -f ssh2/*.c
+    - EMBEDDED_LIB=1 python setup.py install --single-version-externally-managed --record record.txt  # [unix]
     - python setup.py build_ext --compiler=msvc  # [win]
     - python setup.py install --single-version-externally-managed --record record.txt  # [win]
 
diff --git a/libssh2 b/libssh2
@@ -0,0 +1 @@
+Subproject commit 30e9c1347e3b8baa2951db612f05e6d87fc8e2f2
diff --git a/setup.py b/setup.py
diff --git a/ssh2/fileinfo.pyx b/ssh2/fileinfo.pyx

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[submodule "libssh2"]`
	`2`	`+ path = libssh2`
	`3`	`+ url = https://github.com/libssh2/libssh2.git`