jenkins: include local bin directory in PATH
[gnupg-doc.git] / misc / jenkins / bin / build.bash
1 #!/bin/bash
2 # Script used jenkins to run builds for GnuPG and related packages.
3
4 # Stop on error and be nice to other processes.
5 set -xe
6 renice -n 10 -p $$
7
8 # Configuration.
9 MAKE=make
10
11 XTARGET="${XTARGET:-native}"
12
13 # Platform-specific configuration.
14 case "$(uname)" in
15     OpenBSD)
16         MAKE=gmake
17         ;;
18 esac
19
20 if [ "$XTARGET" = w32 ]; then
21     CC=i686-w64-mingw32-gcc
22     CXX=i686-w64-mingw32-g++
23 fi
24
25 # Setup ccache if installed.
26 if ccache --version >/dev/null; then
27     export CCACHE_DIR="$HOME/cache/$JOB_NAME"
28     mkdir -p "$CCACHE_DIR"
29     export CC="ccache ${CC:-gcc}"
30     export CXX="ccache ${CXX:-g++}"
31 fi
32
33 # Include local bin directory in PATH.
34 if [ -e "$HOME/bin" ]; then
35     export PATH="$HOME/bin:$PATH"
36 fi
37
38 # Setup important envars
39 PREFIX=$HOME/prefix/$XTARGET
40 ORIGINAL_PREFIX=$HOME/prefix/$XTARGET
41
42 # hackhackhack
43 #
44 # Copy all *-config scripts into a separate directory and put that
45 # into PATH.  We want configure to pick them up, but we do not
46 # necessarily want to use all the other tools from $PREFIX/bin,
47 # because then we would have to point LD_LIBRARY_PATH to $PREFIX/lib,
48 # which we want to avoid at all costs.
49 mkdir -p $PREFIX/bin-config
50 cp $PREFIX/bin/*-config $PREFIX/bin-config
51 export PATH=$PREFIX/bin-config:$PATH
52 # kcahkcahkcah
53
54 # Tweak the prefix we're installing this project into.  For gnupg-1.4
55 # and friends.
56 case "$JOB_NAME" in
57     *-1.4*)
58         PREFIX=$PREFIX-1.4
59         ;;
60     *-2.0*)
61         PREFIX=$PREFIX-2.0
62         ;;
63     *-2.2*)
64         PREFIX=$PREFIX-2.2
65         ;;
66 esac
67 mkdir -p $PREFIX
68
69
70 fix_permissions()
71 {
72   find $1 -type d -exec chmod +w {} + || true
73 }
74
75 fix_permissions .
76
77 # Clean everything
78 git clean -fdx
79
80 # Run out autogen - note that --force is not required due to the git clean.
81 ./autogen.sh
82
83 # Parallel jobs.
84 MAKEFLAGS="-j6"
85
86 # Parallel tests with our test suite.
87 export TESTFLAGS="--parallel"
88
89 SCANBUILD=
90 if [ "$(uname)" = Linux ] \
91        && [ "$ROOT_BUILD_CAUSE_TIMERTRIGGER" = true ]; then
92     # We only do scan-builds (which are really slow) on nightly
93     # builds.
94     SCANBUILD="scan-build -o ${WORKSPACE}/clangScanBuildReports -v"
95 fi
96 CONFIGUREFLAGS=
97 SANFLAGS=""
98 if [ "$(uname)" = Linux ]; then
99     # XXX: We should check if the sanitizers are available.
100     SANFLAGS="-fsanitize=undefined -fsanitize=address"
101 fi
102
103 if [ "$(uname)" = Darwin ]; then
104     # XXX until we properly set this somewhere else
105     cversion="_DARWIN_C_SOURCE=900000L"
106     CFLAGS="$CFLAGS -D$cversion"
107     CXXFLAGS="$CXXFLAGS -D$cversion"
108 fi
109
110 # Tweak the build depending on the package.
111 case "$JOB_NAME" in
112     *tgpg*)
113         MAKEFLAGS="$MAKEFLAGS GPG=/usr/bin/gpg2"
114         ;;
115     *gpgme*)
116         # using libasan for python broke again, so disable the python
117         # bindings for the sanitizer build
118         if [ "$XTARGET" = sanitizer ]; then
119             CONFIGUREFLAGS_0="--enable-languages=cpp qt"
120         fi
121
122         # Disable Python bindings on macOS.  Something is not working
123         # there.
124         if [ "$NODE_NAME" = zygalski ]; then
125             CONFIGUREFLAGS_0="--enable-languages=cpp qt"
126         fi
127         ;;
128     *gnupg*)
129         # Common configure options.
130         CONFIGUREFLAGS="--enable-wks-tools --enable-gpg2-is-gpg"
131
132         # For Windows builds...
133         if [ "$XTARGET" = w32 ]; then
134             # ... we need to tweak it a little and we leave out some
135             # stuff...
136             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-zlib=$ORIGINAL_PREFIX --with-bzip2=$ORIGINAL_PREFIX"
137         else
138             # ... that we enable for all other builds.
139             CONFIGUREFLAGS="$CONFIGUREFLAGS --enable-g13 --enable-symcryptrun"
140         fi
141
142         if [ "$NODE_NAME" = zygalski ]; then
143             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-libiconv-prefix=$HOME/pkg"
144         fi
145         if [ "$NODE_NAME" = openbsd60 ]; then
146             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-libiconv-prefix=$HOME/compat --with-bzip2=$HOME/compat"
147         fi
148
149         # Disable NTBTLS for now until it is actually mature and used.
150         CONFIGUREFLAGS="$CONFIGUREFLAGS --disable-ntbtls"
151         ;;
152 esac
153
154 # See if we have a GPGME checkout for the tesets.
155 xtest_gpgme_srcdir="$HOME/src/gpgme-for-gnupgs-tests"
156 if [ -d "$xtest_gpgme_srcdir/obj-$XTARGET" ]; then
157     # Some targets, like the sanitizer target, require a custom
158     # version of GPGME.
159     export XTEST_GPGME_SRCDIR="$xtest_gpgme_srcdir"
160     export XTEST_GPGME_BUILDDIR="$xtest_gpgme_srcdir/obj-$XTARGET"
161 elif [ -d "$xtest_gpgme_srcdir/obj" ]; then
162     export XTEST_GPGME_SRCDIR="$xtest_gpgme_srcdir"
163     export XTEST_GPGME_BUILDDIR="$xtest_gpgme_srcdir/obj"
164 fi
165
166 # The libraries use RUNPATH when linking the tests, so they locate
167 # their dependencies that way.  GnuPG, however, does not.  Therefore,
168 # we set LD_LIBRARY_PATH.
169 test_environment="LD_LIBRARY_PATH=$ORIGINAL_PREFIX/lib"
170
171 # HACKHACKHACK:
172 #
173 # Because newer Debian toolchains prefer RUNPATH over RPATH, and
174 # RUNPATH has lower precedence than LD_LIBRARY_PATH, we need to
175 # explicitly add libtool's .libs directory:
176 case "$JOB_NAME" in
177   *gnupg*)
178     if [ "${XTEST_GPGME_BUILDDIR}" ]; then
179         test_environment="LD_LIBRARY_PATH=${XTEST_GPGME_BUILDDIR}/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/cpp/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
180     fi
181     ;;
182   *gpgme*)
183     test_environment="LD_LIBRARY_PATH=$(pwd)/obj/src/.libs:$(pwd)/obj/lang/cpp/src/.libs:$(pwd)/obj/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
184     ;;
185   *)
186     test_environment="LD_LIBRARY_PATH=$(pwd)/obj/src/.libs:$ORIGINAL_PREFIX/lib"
187     ;;
188 esac
189 #
190 # If we don't do this, the version tests fail because the runtime
191 # linker will pick up the library from LD_LIBRARY_PATH.  Also, testing
192 # the installed version is not what we want ofc.
193 #
194 # KCAHKCAHKCAH
195
196 # We build on the "obj" subdir.
197 abs_configure="$(pwd)/configure"
198 mkdir -p obj
199 cd obj
200
201
202 # Print the environment.
203 env
204 ulimit -a
205 set +x
206 for f in /etc/gcrypt/hwf.deny /etc/gcrypt/fips_enabled ; do
207   if [ -f "$f" ]; then
208     echo "=== $f ==="
209     cat -n "$f"
210   fi
211 done
212 set -x
213
214
215 # Switch on the different targets.
216 case "$XTARGET" in
217     native)
218         ../configure --prefix=$PREFIX --enable-maintainer-mode \
219                    $CONFIGUREFLAGS \
220                    "$CONFIGUREFLAGS_0" \
221                    CFLAGS="$CFLAGS" \
222                    CXXFLAGS="$CXXFLAGS -std=c++11"
223         $MAKE $MAKEFLAGS
224
225         env $test_environment $MAKE -k check verbose=2 \
226             || echo "FAIL: make check failed with status $?"
227         # Jenkins looks for "FAIL:" to mark a build unstable,
228         # hence || ... here
229
230         $MAKE install
231         ;;
232     in-tree)
233         cd ..
234         ./configure --prefix=$PREFIX --enable-maintainer-mode \
235                    $CONFIGUREFLAGS \
236                    "$CONFIGUREFLAGS_0" \
237                    CFLAGS="$CFLAGS" \
238                    CXXFLAGS="$CXXFLAGS -std=c++11"
239         $MAKE $MAKEFLAGS
240
241         # HACKHACKHACK: Fix the test_environment hack.
242         test_environment="$(echo $test_environment | sed -e 's#obj/##g')"
243         # KCAHKCAHKCAH
244
245         env $test_environment $MAKE -k check verbose=2 \
246             || echo "FAIL: make check failed with status $?"
247         # Jenkins looks for "FAIL:" to mark a build unstable,
248         # hence || ... here
249
250         $MAKE install
251         ;;
252     sanitizer)
253         # asan breaks the configure tests, so we disable it here.
254         ASAN_OPTIONS=detect_leaks=0 \
255         $SCANBUILD \
256             ../configure --prefix=$PREFIX --enable-maintainer-mode \
257                    $CONFIGUREFLAGS \
258                    "$CONFIGUREFLAGS_0" \
259                    CFLAGS="$CFLAGS $SANFLAGS -fPIC" \
260                    CXXFLAGS="$CXXFLAGS $SANFLAGS -fPIC -std=c++11"
261         $SCANBUILD $MAKE $MAKEFLAGS
262
263         env $test_environment $MAKE -k check verbose=2 \
264             || echo "FAIL: make check failed with status $?"
265         # Jenkins looks for "FAIL:" to mark a build unstable,
266         # hence || ... here
267
268         $MAKE install
269         ;;
270     w32)
271         export w32root=$PREFIX
272
273         # autogen.rc adds --with-gpg-error-prefix=@SYSROOT@, so we cannot
274         # install to a prefix that doesn't also contain all the dependencies,
275         # patch that out, so that the gpg-error-config and friends are located
276         # using PATH
277         if [ -f "/home/jenkins/bin/$(dirname $JOB_NAME)-w32.patch" ]; then
278           ( cd .. && patch -p1 <"/home/jenkins/bin/$(dirname $JOB_NAME)-w32.patch" )
279         fi
280         # We need to point it to npth then...
281         case "$JOB_NAME" in
282             gnupg/XTARGET=w32|gnupg-2.2/XTARGET=w32)
283                 CONFIGUREFLAGS="${CONFIGUREFLAGS} --with-npth-prefix=$ORIGINAL_PREFIX"
284                 ;;
285             gnupg-2.0/XTARGET=w32)
286                 CONFIGUREFLAGS="${CONFIGUREFLAGS} --with-pth-prefix=$ORIGINAL_PREFIX --with-adns=$ORIGINAL_PREFIX"
287                 ;;
288         esac
289
290         # gpg1's autogen.sh does not add --enable-maintainer-mode, so
291         # version.texi is not generated.  we add it here to be sure.
292         # likewise for --prefix
293         ../autogen.sh --build-w32 --enable-maintainer-mode --prefix=$PREFIX \
294           $CONFIGUREFLAGS
295         $MAKE $MAKEFLAGS
296         $MAKE install
297
298         case "$JOB_NAME" in
299                 gnupg/*|gnupg-2.2/*)
300                         bash /home/jenkins/bin/make-windows-cd.sh
301                         # We need to pass the absolute path of the iso.
302                         bash $HOME/bin/run-tests-w32.bash "$(readlink -f gnupg-test.iso)" || echo "Warning: error running tests on Windows."
303                         ;;
304         esac
305         ;;
306     distcheck)
307         CONFIGUREFLAGS=
308         WORKDIR="$(mktemp -d)"
309         cleanup()
310         {
311           cd /tmp
312           fix_permissions "$WORKDIR"
313           rm -rf -- "$WORKDIR" || true
314         }
315         trap cleanup EXIT
316
317           # We use a different WORKDIR to avoid problems with too long
318           # file names
319           cd "$WORKDIR"
320           $abs_configure --prefix=$PREFIX --enable-maintainer-mode \
321                    $CONFIGUREFLAGS
322
323           # Extract the directory / tarname from the package
324           tarname=$(awk <config.h '
325                      /^#define PACKAGE_TARNAME/ {gsub(/"/,"",$3);name=$3};
326                      /^#define PACKAGE_VERSION/ {gsub(/"/,"",$3);vers=$3};
327                      END {print name "-" vers}')
328
329           # HACKHACKHACK: Because newer Debian toolchains prefer
330           # RUNPATH over RPATH, and RUNPATH has lower precedence than
331           # LD_LIBRARY_PATH, we need to explicitly add libtool's .libs
332           # directory:
333           case "$JOB_NAME" in
334             *gnupg*)
335               if [ "${XTEST_GPGME_BUILDDIR}" ]; then
336                   test_environment="LD_LIBRARY_PATH=${XTEST_GPGME_BUILDDIR}/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/cpp/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
337               fi
338               ;;
339             *gpgme*)
340               test_environment="LD_LIBRARY_PATH=$(pwd)/${tarname}/_build/sub/src/.libs:$(pwd)/${tarname}/_build/sub/lang/cpp/src/.libs:$(pwd)/${tarname}/_build/sub/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
341               ;;
342             *)
343               test_environment="LD_LIBRARY_PATH=$(pwd)/${tarname}/_build/sub/src/.libs:$ORIGINAL_PREFIX/lib"
344               ;;
345           esac
346           # KCAHKCAHKCAH
347
348           if ! env $test_environment $MAKE $MAKEFLAGS distcheck ; then
349               # Jenkins looks for "FAIL:" to mark a build unstable,
350               # hence we ignore errors here.
351               echo "FAIL: make distcheck failed with status $?"
352               # Disable the cleanup so that we can investigate.
353               trap - EXIT
354               exit 0
355           fi
356
357           if [ -f "${tarname}.tar.bz2" ]; then
358              bzcat "${tarname}.tar.bz2" | tar xf -
359           elif [ -f "${tarname}.tar.gz" ]; then
360              zcat "${tarname}.tar.gz" | tar xf -
361           else
362              echo "No tarball named $tarname found - skipping installation" >&2
363              exit 0
364           fi
365           # And do a final build using the generated tarball
366           cd ${tarname}
367           ./configure --prefix=$PREFIX $CONFIGUREFLAGS
368           $MAKE $MAKEFLAGS
369           $MAKE $MAKEFLAGS install
370
371         ;;
372     *)
373         echo "Bad XTARGET: '$XTARGET'"
374         exit 2
375 esac