eb17834894b1ce17e536762335f6028c30860342
[gnupg-doc.git] / misc / jenkins / bin / build.bash
1 #!/bin/bash
2 # Script used jenkins to run builds for GnuPG and related packages.
3
4 # Stop on error and be nice to other processes.
5 set -xe
6 renice -n 10 -p $$
7
8 # Configuration.
9 MAKE=make
10
11 XTARGET="${XTARGET:-native}"
12
13 # Platform-specific configuration.
14 case "$(uname)" in
15     OpenBSD)
16         MAKE=gmake
17         ;;
18 esac
19
20 if [ "$XTARGET" = w32 ]; then
21     CC=i686-w64-mingw32-gcc
22     CXX=i686-w64-mingw32-g++
23 fi
24
25 # Setup ccache if installed.
26 if ccache --version >/dev/null; then
27     export CCACHE_DIR="$HOME/cache/$JOB_NAME"
28     mkdir -p "$CCACHE_DIR"
29     export CC="ccache ${CC:-gcc}"
30     export CXX="ccache ${CXX:-g++}"
31 fi
32
33 # Setup important envars
34 PREFIX=$HOME/prefix/$XTARGET
35 ORIGINAL_PREFIX=$HOME/prefix/$XTARGET
36
37 # hackhackhack
38 #
39 # Copy all *-config scripts into a separate directory and put that
40 # into PATH.  We want configure to pick them up, but we do not
41 # necessarily want to use all the other tools from $PREFIX/bin,
42 # because then we would have to point LD_LIBRARY_PATH to $PREFIX/lib,
43 # which we want to avoid at all costs.
44 mkdir -p $PREFIX/bin-config
45 cp $PREFIX/bin/*-config $PREFIX/bin-config
46 export PATH=$PREFIX/bin-config:$PATH
47 # kcahkcahkcah
48
49 # Tweak the prefix we're installing this project into.  For gnupg-1.4
50 # and friends.
51 case "$JOB_NAME" in
52     *-1.4*)
53         PREFIX=$PREFIX-1.4
54         ;;
55     *-2.0*)
56         PREFIX=$PREFIX-2.0
57         ;;
58     *-2.2*)
59         PREFIX=$PREFIX-2.2
60         ;;
61 esac
62 mkdir -p $PREFIX
63
64
65 fix_permissions()
66 {
67   find $1 -type d -exec chmod +w {} + || true
68 }
69
70 fix_permissions .
71
72 # Clean everything
73 git clean -fdx
74
75 # Run out autogen - note that --force is not required due to the git clean.
76 ./autogen.sh
77
78 # Parallel jobs.
79 MAKEFLAGS="-j6"
80
81 # Parallel tests with our test suite.
82 export TESTFLAGS="--parallel"
83
84 SCANBUILD=
85 if [ "$(uname)" = Linux ] \
86        && [ "$ROOT_BUILD_CAUSE_TIMERTRIGGER" = true ]; then
87     # We only do scan-builds (which are really slow) on nightly
88     # builds.
89     SCANBUILD="scan-build -o ${WORKSPACE}/clangScanBuildReports -v"
90 fi
91 CONFIGUREFLAGS=
92 SANFLAGS=""
93 if [ "$(uname)" = Linux ]; then
94     # XXX: We should check if the sanitizers are available.
95     SANFLAGS="-fsanitize=undefined -fsanitize=address"
96 fi
97
98 if [ "$(uname)" = Darwin ]; then
99     # XXX until we properly set this somewhere else
100     cversion="_DARWIN_C_SOURCE=900000L"
101     CFLAGS="$CFLAGS -D$cversion"
102     CXXFLAGS="$CXXFLAGS -D$cversion"
103 fi
104
105 # Tweak the build depending on the package.
106 case "$JOB_NAME" in
107     *tgpg*)
108         MAKEFLAGS="$MAKEFLAGS GPG=/usr/bin/gpg2"
109         ;;
110     *gpgme*)
111         # using libasan for python broke again, so disable the python
112         # bindings for the sanitizer build
113         if [ "$XTARGET" = sanitizer ]; then
114             CONFIGUREFLAGS_0="--enable-languages=cpp qt"
115         fi
116
117         # Disable Python bindings on macOS.  Something is not working
118         # there.
119         if [ "$NODE_NAME" = zygalski ]; then
120             CONFIGUREFLAGS_0="--enable-languages=cpp qt"
121         fi
122         ;;
123     *gnupg*)
124         # Common configure options.
125         CONFIGUREFLAGS="--enable-wks-tools --enable-gpg2-is-gpg"
126
127         # For Windows builds...
128         if [ "$XTARGET" = w32 ]; then
129             # ... we need to tweak it a little and we leave out some
130             # stuff...
131             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-zlib=$ORIGINAL_PREFIX --with-bzip2=$ORIGINAL_PREFIX"
132         else
133             # ... that we enable for all other builds.
134             CONFIGUREFLAGS="$CONFIGUREFLAGS --enable-g13 --enable-symcryptrun"
135         fi
136
137         if [ "$NODE_NAME" = zygalski ]; then
138             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-libiconv-prefix=$HOME/pkg"
139         fi
140         if [ "$NODE_NAME" = openbsd60 ]; then
141             CONFIGUREFLAGS="$CONFIGUREFLAGS --with-libiconv-prefix=$HOME/compat --with-bzip2=$HOME/compat"
142         fi
143
144         # Disable NTBTLS for now until it is actually mature and used.
145         CONFIGUREFLAGS="$CONFIGUREFLAGS --disable-ntbtls"
146         ;;
147 esac
148
149 # See if we have a GPGME checkout for the tesets.
150 xtest_gpgme_srcdir="$HOME/src/gpgme-for-gnupgs-tests"
151 if [ -d "$xtest_gpgme_srcdir/obj-$XTARGET" ]; then
152     # Some targets, like the sanitizer target, require a custom
153     # version of GPGME.
154     export XTEST_GPGME_SRCDIR="$xtest_gpgme_srcdir"
155     export XTEST_GPGME_BUILDDIR="$xtest_gpgme_srcdir/obj-$XTARGET"
156 elif [ -d "$xtest_gpgme_srcdir/obj" ]; then
157     export XTEST_GPGME_SRCDIR="$xtest_gpgme_srcdir"
158     export XTEST_GPGME_BUILDDIR="$xtest_gpgme_srcdir/obj"
159 fi
160
161 # The libraries use RUNPATH when linking the tests, so they locate
162 # their dependencies that way.  GnuPG, however, does not.  Therefore,
163 # we set LD_LIBRARY_PATH.
164 test_environment="LD_LIBRARY_PATH=$ORIGINAL_PREFIX/lib"
165
166 # HACKHACKHACK:
167 #
168 # Because newer Debian toolchains prefer RUNPATH over RPATH, and
169 # RUNPATH has lower precedence than LD_LIBRARY_PATH, we need to
170 # explicitly add libtool's .libs directory:
171 case "$JOB_NAME" in
172   *gnupg*)
173     if [ "${XTEST_GPGME_BUILDDIR}" ]; then
174         test_environment="LD_LIBRARY_PATH=${XTEST_GPGME_BUILDDIR}/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/cpp/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
175     fi
176     ;;
177   *gpgme*)
178     test_environment="LD_LIBRARY_PATH=$(pwd)/obj/src/.libs:$(pwd)/obj/lang/cpp/src/.libs:$(pwd)/obj/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
179     ;;
180   *)
181     test_environment="LD_LIBRARY_PATH=$(pwd)/obj/src/.libs:$ORIGINAL_PREFIX/lib"
182     ;;
183 esac
184 #
185 # If we don't do this, the version tests fail because the runtime
186 # linker will pick up the library from LD_LIBRARY_PATH.  Also, testing
187 # the installed version is not what we want ofc.
188 #
189 # KCAHKCAHKCAH
190
191 # We build on the "obj" subdir.
192 abs_configure="$(pwd)/configure"
193 mkdir -p obj
194 cd obj
195
196
197 # Print the environment.
198 env
199 ulimit -a
200 set +x
201 for f in /etc/gcrypt/hwf.deny /etc/gcrypt/fips_enabled ; do
202   if [ -f "$f" ]; then
203     echo "=== $f ==="
204     cat -n "$f"
205   fi
206 done
207 set -x
208
209
210 # Switch on the different targets.
211 case "$XTARGET" in
212     native)
213         ../configure --prefix=$PREFIX --enable-maintainer-mode \
214                    $CONFIGUREFLAGS \
215                    "$CONFIGUREFLAGS_0" \
216                    CFLAGS="$CFLAGS" \
217                    CXXFLAGS="$CXXFLAGS -std=c++11"
218         $MAKE $MAKEFLAGS
219
220         env $test_environment $MAKE -k check verbose=2 \
221             || echo "FAIL: make check failed with status $?"
222         # Jenkins looks for "FAIL:" to mark a build unstable,
223         # hence || ... here
224
225         $MAKE install
226         ;;
227     in-tree)
228         cd ..
229         ./configure --prefix=$PREFIX --enable-maintainer-mode \
230                    $CONFIGUREFLAGS \
231                    "$CONFIGUREFLAGS_0" \
232                    CFLAGS="$CFLAGS" \
233                    CXXFLAGS="$CXXFLAGS -std=c++11"
234         $MAKE $MAKEFLAGS
235
236         # HACKHACKHACK: Fix the test_environment hack.
237         test_environment="$(echo $test_environment | sed -e 's#obj/##g')"
238         # KCAHKCAHKCAH
239
240         env $test_environment $MAKE -k check verbose=2 \
241             || echo "FAIL: make check failed with status $?"
242         # Jenkins looks for "FAIL:" to mark a build unstable,
243         # hence || ... here
244
245         $MAKE install
246         ;;
247     sanitizer)
248         # asan breaks the configure tests, so we disable it here.
249         ASAN_OPTIONS=detect_leaks=0 \
250         $SCANBUILD \
251             ../configure --prefix=$PREFIX --enable-maintainer-mode \
252                    $CONFIGUREFLAGS \
253                    "$CONFIGUREFLAGS_0" \
254                    CFLAGS="$CFLAGS $SANFLAGS -fPIC" \
255                    CXXFLAGS="$CXXFLAGS $SANFLAGS -fPIC -std=c++11"
256         $SCANBUILD $MAKE $MAKEFLAGS
257
258         env $test_environment $MAKE -k check verbose=2 \
259             || echo "FAIL: make check failed with status $?"
260         # Jenkins looks for "FAIL:" to mark a build unstable,
261         # hence || ... here
262
263         $MAKE install
264         ;;
265     w32)
266         export w32root=$PREFIX
267
268         # autogen.rc adds --with-gpg-error-prefix=@SYSROOT@, so we cannot
269         # install to a prefix that doesn't also contain all the dependencies,
270         # patch that out, so that the gpg-error-config and friends are located
271         # using PATH
272         if [ -f "/home/jenkins/bin/$(dirname $JOB_NAME)-w32.patch" ]; then
273           ( cd .. && patch -p1 <"/home/jenkins/bin/$(dirname $JOB_NAME)-w32.patch" )
274         fi
275         # We need to point it to npth then...
276         case "$JOB_NAME" in
277             gnupg/XTARGET=w32|gnupg-2.2/XTARGET=w32)
278                 CONFIGUREFLAGS="${CONFIGUREFLAGS} --with-npth-prefix=$ORIGINAL_PREFIX"
279                 ;;
280             gnupg-2.0/XTARGET=w32)
281                 CONFIGUREFLAGS="${CONFIGUREFLAGS} --with-pth-prefix=$ORIGINAL_PREFIX --with-adns=$ORIGINAL_PREFIX"
282                 ;;
283         esac
284
285         # gpg1's autogen.sh does not add --enable-maintainer-mode, so
286         # version.texi is not generated.  we add it here to be sure.
287         # likewise for --prefix
288         ../autogen.sh --build-w32 --enable-maintainer-mode --prefix=$PREFIX \
289           $CONFIGUREFLAGS
290         $MAKE $MAKEFLAGS
291         $MAKE install
292
293         case "$JOB_NAME" in
294                 gnupg/*|gnupg-2.2/*)
295                         bash /home/jenkins/bin/make-windows-cd.sh
296                         # We need to pass the absolute path of the iso.
297                         bash $HOME/bin/run-tests-w32.bash "$(readlink -f gnupg-test.iso)" || echo "Warning: error running tests on Windows."
298                         ;;
299         esac
300         ;;
301     distcheck)
302         CONFIGUREFLAGS=
303         WORKDIR="$(mktemp -d)"
304         cleanup()
305         {
306           cd /tmp
307           fix_permissions "$WORKDIR"
308           rm -rf -- "$WORKDIR" || true
309         }
310         trap cleanup EXIT
311
312           # We use a different WORKDIR to avoid problems with too long
313           # file names
314           cd "$WORKDIR"
315           $abs_configure --prefix=$PREFIX --enable-maintainer-mode \
316                    $CONFIGUREFLAGS
317
318           # Extract the directory / tarname from the package
319           tarname=$(awk <config.h '
320                      /^#define PACKAGE_TARNAME/ {gsub(/"/,"",$3);name=$3};
321                      /^#define PACKAGE_VERSION/ {gsub(/"/,"",$3);vers=$3};
322                      END {print name "-" vers}')
323
324           # HACKHACKHACK: Because newer Debian toolchains prefer
325           # RUNPATH over RPATH, and RUNPATH has lower precedence than
326           # LD_LIBRARY_PATH, we need to explicitly add libtool's .libs
327           # directory:
328           case "$JOB_NAME" in
329             *gnupg*)
330               if [ "${XTEST_GPGME_BUILDDIR}" ]; then
331                   test_environment="LD_LIBRARY_PATH=${XTEST_GPGME_BUILDDIR}/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/cpp/src/.libs:${XTEST_GPGME_BUILDDIR}/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
332               fi
333               ;;
334             *gpgme*)
335               test_environment="LD_LIBRARY_PATH=$(pwd)/${tarname}/_build/sub/src/.libs:$(pwd)/${tarname}/_build/sub/lang/cpp/src/.libs:$(pwd)/${tarname}/_build/sub/lang/qt/src/.libs:$ORIGINAL_PREFIX/lib"
336               ;;
337             *)
338               test_environment="LD_LIBRARY_PATH=$(pwd)/${tarname}/_build/sub/src/.libs:$ORIGINAL_PREFIX/lib"
339               ;;
340           esac
341           # KCAHKCAHKCAH
342
343           if ! env $test_environment $MAKE $MAKEFLAGS distcheck ; then
344               # Jenkins looks for "FAIL:" to mark a build unstable,
345               # hence we ignore errors here.
346               echo "FAIL: make distcheck failed with status $?"
347               # Disable the cleanup so that we can investigate.
348               trap - EXIT
349               exit 0
350           fi
351
352           if [ -f "${tarname}.tar.bz2" ]; then
353              bzcat "${tarname}.tar.bz2" | tar xf -
354           elif [ -f "${tarname}.tar.gz" ]; then
355              zcat "${tarname}.tar.gz" | tar xf -
356           else
357              echo "No tarball named $tarname found - skipping installation" >&2
358              exit 0
359           fi
360           # And do a final build using the generated tarball
361           cd ${tarname}
362           ./configure --prefix=$PREFIX $CONFIGUREFLAGS
363           $MAKE $MAKEFLAGS
364           $MAKE $MAKEFLAGS install
365
366         ;;
367     *)
368         echo "Bad XTARGET: '$XTARGET'"
369         exit 2
370 esac