Auto merge of #4675 - lzutao:improve-shellscript, r=phansch

build: improve script and travis config

* fix a diff failure on windows

See https://travis-ci.com/rust-lang/rust-clippy/jobs/245971932#L1625
for an example.

* use cmp instead of diff > /dev/null

* clone single branch instead of clone then checking out

* do not decrypt key if have no diff change

changelog: none

Author: bors

.github/deploy.sh

@@ -13,11 +13,7 @@ SSH_REPO=${REPO/https:\/\/github.com\//[email protected]:}
 SHA=$(git rev-parse --verify HEAD)
 
 # Clone the existing gh-pages for this repo into out/
-(
-    git clone "$REPO" out
-    cd out
-    git checkout $TARGET_BRANCH
-)
+git clone --quiet --single-branch --branch "$TARGET_BRANCH" "$REPO" out
 
 echo "Removing the current docs for master"
 rm -rf out/master/ || exit 0
@@ -27,35 +23,44 @@ mkdir out/master/
 cp util/gh-pages/index.html out/master
 python ./util/export.py out/master/lints.json
 
-if [ -n "$TRAVIS_TAG" ]; then
+if [[ -n "$TRAVIS_TAG" ]]; then
     echo "Save the doc for the current tag ($TRAVIS_TAG) and point current/ to it"
     cp -r out/master "out/$TRAVIS_TAG"
     rm -f out/current
     ln -s "$TRAVIS_TAG" out/current
 fi
 
 # Generate version index that is shown as root index page
-(
-    cp util/gh-pages/versions.html out/index.html
-
-    cd out
-    python -c '\
-        import os, json;\
-        print json.dumps([\
-            dir for dir in os.listdir(".")\
-            if not dir.startswith(".") and os.path.isdir(dir)\
-        ])' > versions.json
-)
+cp util/gh-pages/versions.html out/index.html
+pushd out
+
+cat <<-EOF | python - > versions.json
+import os, json
+print json.dumps([
+    dir for dir in os.listdir(".") if not dir.startswith(".") and os.path.isdir(dir)
+])
+EOF
+popd
 
 # Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
+if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] || [[ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]]; then
     # Tags should deploy
-    if [ -z "$TRAVIS_TAG" ]; then
+    if [[ -z "$TRAVIS_TAG" ]]; then
         echo "Generated, won't push"
         exit 0
     fi
 fi
 
+# Now let's go have some fun with the cloned repo
+cd out
+git config user.name "Travis CI"
+git config user.email "[email protected]"
+
+if git diff --exit-code --quiet; then
+    echo "No changes to the output on this push; exiting."
+    exit 0
+fi
+
 # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc
 ENCRYPTION_LABEL=e3a2d77100be
 ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
@@ -64,19 +69,9 @@ ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
 ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
 openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" -in .github/deploy_key.enc -out .github/deploy_key -d
 chmod 600 .github/deploy_key
-eval $(ssh-agent -s)
+eval "$(ssh-agent -s)"
 ssh-add .github/deploy_key
 
-# Now let's go have some fun with the cloned repo
-cd out
-git config user.name "Travis CI"
-git config user.email "[email protected]"
-
-if [ -z "$(git diff --exit-code)" ]; then
-    echo "No changes to the output on this push; exiting."
-    exit 0
-fi
-
 git add .
 git commit -m "Automatic deploy to GitHub Pages: ${SHA}"
 

.travis.yml

@@ -1,11 +1,6 @@
 dist: xenial
 language: bash
 
-os:
-  - linux
-  - osx
-  - windows
-
 branches:
   # Don't build these branches
   except:
@@ -34,13 +29,12 @@ install:
       if ! rustup component add rustfmt; then
         cargo install -Z install-upgrade --git https://github.com/rust-lang/rustfmt --bin rustfmt
       fi
-      if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+      if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
         . $HOME/.nvm/nvm.sh
         nvm install stable
         nvm use stable
         npm install remark-cli remark-lint
-      fi
-      if [ "$TRAVIS_OS_NAME" == "windows" ]; then
+      elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
         choco install windows-sdk-10.1
       fi
     fi
@@ -53,11 +47,9 @@ matrix:
   include:
     # Builds that are executed for every PR
     - os: osx # run base tests on both platforms
-      env: BASE_TESTS=true
     - os: linux
-      env: BASE_TESTS=true
     - os: windows
-      env: CARGO_INCREMENTAL=0 BASE_TESTS=true OS_WINDOWS=true
+      env: CARGO_INCREMENTAL=0 OS_WINDOWS=true
 
     # Builds that are only executed when a PR is r+ed or a try build is started
     # We don't want to run these always because they go towards
@@ -95,51 +87,51 @@ matrix:
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
   allow_failures:
     - os: windows
-      env: CARGO_INCREMENTAL=0 BASE_TESTS=true OS_WINDOWS=true
-  # prevent these jobs with default env vars
-  exclude:
-    - os: linux
-    - os: osx
-    - os: windows
+      env: CARGO_INCREMENTAL=0 OS_WINDOWS=true
 
 before_script:
   - |
-    if [ "$TRAVIS_BRANCH" == "auto" ] || [ "$TRAVIS_BRANCH" == "try" ]; then
-      pr=$(echo $TRAVIS_COMMIT_MESSAGE | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
-      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$pr" | \
+    if [[ "$TRAVIS_BRANCH" == "auto" ]] || [[ "$TRAVIS_BRANCH" == "try" ]]; then
+      PR=$(echo "$TRAVIS_COMMIT_MESSAGE" | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
+      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$PR" | \
         python -c "import sys, json; print(json.load(sys.stdin)['body'])" | \
         grep "^changelog: " | \
         sed "s/changelog: //g")
-      if [ -z "$output" ]; then
+      if [[ -z "$output" ]]; then
         echo "ERROR: PR body must contain 'changelog: ...'"
         exit 1
-      elif [ "$output" = "none" ]; then
+      elif [[ "$output" = "none" ]]; then
         echo "WARNING: changelog is 'none'"
       fi
     fi
   - |
     rm rust-toolchain
     ./setup-toolchain.sh
   - |
-    if [ "$TRAVIS_OS_NAME" == "windows" ]; then
-      export PATH=$PATH:$(rustc --print sysroot)/bin
-    else
-      export LD_LIBRARY_PATH=$(rustc --print sysroot)/lib
-    fi
+    SYSROOT=$(rustc --print sysroot)
+    case "$TRAVIS_OS_NAME" in
+      windows ) export PATH="${SYSROOT}/bin:${PATH}" ;;
+      linux ) export LD_LIBRARY_PATH="${SYSROOT}/lib" ;;
+      osx )
+        # See <https://github.com/nteract/nteract/issues/1523#issuecomment-301623519>
+        sudo mkdir -p /usr/local/lib
+        sudo find "$SYSROOT/lib" -maxdepth 1 -name '*.dylib' -exec ln -s {} /usr/local/lib \;
+        ;;
+    esac
 
 script:
   - |
-    if [ -z ${INTEGRATION} ]; then
-      travis_wait 30 ./ci/base-tests.sh && sleep 5
-    else
+    if [[ -n ${INTEGRATION} ]]; then
       ./ci/integration-tests.sh && sleep 5
+    else
+      ./ci/base-tests.sh && sleep 5
     fi
 
 after_success:
   - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
       set -e
-      if [ -z ${INTEGRATION} ]; then
+      if [[ -z ${INTEGRATION} ]]; then
         ./.github/deploy.sh
       else
         echo "Not deploying, because we're in an integration test run"

ci/base-tests.sh

@@ -1,16 +1,20 @@
+#!/usr/bin/env bash
 set -ex
 
+# for faster build, share target dir between subcrates
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
+
 echo "Running clippy base tests"
 
 PATH=$PATH:./node_modules/.bin
-if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-  remark -f *.md -f doc/*.md > /dev/null
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  remark -f ./*.md -f doc/*.md > /dev/null
 fi
 # build clippy in debug mode and run tests
 cargo build --features "debugging deny-warnings"
 cargo test --features "debugging deny-warnings"
-# for faster build, share target dir between subcrates
-export CARGO_TARGET_DIR=`pwd`/target/
+
 (cd clippy_lints && cargo test)
 (cd rustc_tools_util && cargo test)
 (cd clippy_dev && cargo test)
@@ -29,26 +33,27 @@ export CARGO_TARGET_DIR=`pwd`/target/
 (
   # Check sysroot handling
   sysroot=$(./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $(rustc --print sysroot)
+  test "$sysroot" = "$(rustc --print sysroot)"
 
-  if [ -z $OS_WINDOWS ]; then
+  if [[ -z "$OS_WINDOWS" ]]; then
     desired_sysroot=/tmp
   else
     desired_sysroot=C:/tmp
   fi
   sysroot=$(./target/debug/clippy-driver --sysroot $desired_sysroot --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot
 
   sysroot=$(SYSROOT=$desired_sysroot ./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot
 
   # Make sure this isn't set - clippy-driver should cope without it
   unset CARGO_MANIFEST_DIR
 
   # Run a lint and make sure it produces the expected output. It's also expected to exit with code 1
-  # XXX How to match the clippy invocation in compile-test.rs?
-  ! ./target/debug/clippy-driver -Dwarnings -Aunused -Zui-testing --emit metadata --crate-type bin tests/ui/cstring.rs 2> cstring.stderr
-  diff <(sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr) tests/ui/cstring.stderr
+  # FIXME: How to match the clippy invocation in compile-test.rs?
+  ./target/debug/clippy-driver -Dwarnings -Aunused -Zui-testing --emit metadata --crate-type bin tests/ui/cstring.rs 2> cstring.stderr && exit 1
+  sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr > normalized.stderr
+  diff normalized.stderr tests/ui/cstring.stderr
 
   # TODO: CLIPPY_CONF_DIR / CARGO_MANIFEST_DIR
 )

ci/integration-tests.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/env bash
-set -x
+set -ex
 
 if [[ -z "$INTEGRATION" ]]; then
     exit 0
 fi
 
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
+
 rm ~/.cargo/bin/cargo-clippy
 cargo install --force --debug --path .
 

tests/ui-toml/update-all-references.sh

@@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi
 
 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +

tests/ui-toml/update-references.sh

@@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
     echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi
 
-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")
 
 BUILD_DIR="$1"
 shift
@@ -25,16 +25,14 @@ while [[ "$1" != "" ]]; do
     STDERR_NAME="${1/%.rs/.stderr}"
     STDOUT_NAME="${1/%.rs/.stdout}"
     shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
     fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
     fi
 done
-
-

tests/ui/update-all-references.sh

@@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi
 
 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +

tests/ui/update-references.sh

@@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
     echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi
 
-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")
 
 BUILD_DIR="$1"
 shift
@@ -26,21 +26,19 @@ while [[ "$1" != "" ]]; do
     STDOUT_NAME="${1/%.rs/.stdout}"
     FIXED_NAME="${1/%.rs/.fixed}"
     shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
     fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
     fi
-    if [ -f $BUILD_DIR/$FIXED_NAME ] && \
-           ! (diff $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME >& /dev/null); then
-        echo updating $MYDIR/$FIXED_NAME
-        cp $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME
+    if [[ -f "$BUILD_DIR"/"$FIXED_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"); then
+        echo updating "$MYDIR"/"$FIXED_NAME"
+        cp "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"
     fi
 done
-
-

util/dev

@@ -1,3 +1,5 @@
 #!/bin/sh
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
 
-cd clippy_dev && cargo run -- $@
+cd clippy_dev && cargo run -- "$@"