diff --git a/package/ci/pybind11-system-includes.patch b/package/ci/pybind11-system-includes.patch
new file mode 100644
index 0000000..60d8954
--- /dev/null
+++ b/package/ci/pybind11-system-includes.patch
@@ -0,0 +1,13 @@
+diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake
+index 52a70c2..1e9d033 100644
+--- a/tools/pybind11Tools.cmake
++++ b/tools/pybind11Tools.cmake
+@@ -130,7 +130,7 @@ function(pybind11_add_module target_name)
+ 
+   add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
+ 
+-  target_include_directories(${target_name}
++  target_include_directories(${target_name} SYSTEM
+     PRIVATE ${PYBIND11_INCLUDE_DIR}  # from project CMakeLists.txt
+     PRIVATE ${pybind11_INCLUDE_DIR}  # from pybind11Config
+     PRIVATE ${PYTHON_INCLUDE_DIRS})
diff --git a/package/ci/setup-pybind11.sh b/package/ci/setup-pybind11.sh
new file mode 100755
index 0000000..dcf96da
--- /dev/null
+++ b/package/ci/setup-pybind11.sh
@@ -0,0 +1,20 @@
+set -e
+
+wget --no-clobber https://github.com/pybind/pybind11/archive/v2.2.4.tar.gz && tar -xzf v2.2.4.tar.gz
+
+cd pybind11-2.2.4
+
+# Patch the CMake macro to add include paths as SYSTEM because otherwise GCC
+# 4.8 gets very loud. This is already done in
+# https://github.com/pybind/pybind11/pull/1416 but it's opt-in and not part of
+# any release yet. I don't want to force users to use pybind11 git, so apply
+# a patch manually.
+patch -p1 < $TRAVIS_BUILD_DIR/package/ci/pybind11-system-includes.patch
+
+mkdir -p build && cd build
+cmake .. \
+    -DCMAKE_INSTALL_PREFIX=$HOME/pybind11 \
+    -DPYBIND11_PYTHON_VERSION=3.6 \
+    -DPYBIND11_TEST=OFF \
+    -G Ninja
+ninja install
diff --git a/package/ci/travis.yml b/package/ci/travis.yml
index 85d7ba7..eead4d5 100644
--- a/package/ci/travis.yml
+++ b/package/ci/travis.yml
@@ -48,7 +48,7 @@ install:
 - if [ "$TRAVIS_OS_NAME" == "linux" ] || [ "$TRAVIS_OS_NAME" == "osx" ]; then pip3 install coverage codecov; fi
 
 # pybind11 (cached)
-- if [ "$TRAVIS_OS_NAME" == "linux" ] && [ ! -e "$HOME/pybind11/bin" ]; then wget -nc https://github.com/pybind/pybind11/archive/v2.2.4.tar.gz && tar -xzf v2.2.4.tar.gz && cd pybind11-2.2.4 && mkdir -p build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=$HOME/pybind11 -DPYBIND11_PYTHON_VERSION=3.6 -DPYBIND11_TEST=OFF -G Ninja && ninja install && cd $TRAVIS_BUILD_DIR; fi
+- if [ "$TRAVIS_OS_NAME" == "linux" ] && [ ! -e "$HOME/pybind11/bin" ]; then $TRAVIS_BUILD_DIR/package/ci/setup-pybind11.sh; fi
 - if [ "$TRAVIS_OS_NAME" == "osx" ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew install pybind11; fi
 
 script: