DCO tree on Ubuntu 20.04 does not build with OpenSSL 3.0.5

[Gert Döring]

Having OpenSSL 1.1.1 in the standard system library path and OpenSSL 3.0.x in a custom path fails, because pkg-config for libnl injects -L/usr/lib/x86_64-linux-gnu in the link path, so -lssl -lcrypto finds the wrong library:

...
cryptoapi.o -Wl,-rpath=/home/openssl-3.0.5/lib64  ../../src/compat/.libs/libcompat.a -L/usr/lib/x86_64-linux-gnu -lnsl -lresolv -llzo2 -L/usr/lib -llz4 -L/home/openssl-3.0.5/lib64 -lssl -lcrypto -ldl -lnl-genl-3 -lnl-3

Arguably the bug is in Ubuntu's .pc file

$ pkg-config --libs --print-errors "libnl-genl-3.0 >= 3.4.0"
-L/usr/lib/x86_64-linux-gnu -lnl-genl-3 -lnl-3 

but I think our configure could be a bit smarter about command line placement - so if we place "things that are not pkg-config" in front (OPENSSL_LIBS, LZO_LIBS, ...) - assuming that those will be "in that place, there is only this single library" - we'd be more robust against .pc stupidity.

We might want to file an ubuntu bug as well, but I think this is likely to hit us again - so if we can handle it, even better.

[David Sommerseth]

Can you try this patch and see how that works out for you?

diff --git a/configure.ac b/configure.ac
index bebed1ac..b693e63f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -781,8 +781,8 @@ dnl
                                          [AC_MSG_ERROR([libnl-genl-3.0 package not found or too old. Is the development package and pkg-config installed? Must be version 3.4.0 or newer])]
                        )
 
-                       CFLAGS="${CFLAGS} ${LIBNL_GENL_CFLAGS}"
-                       LIBS="${LIBS} ${LIBNL_GENL_LIBS}"
+                       AC_SUBST([LIBNL_GENL_CFLAGS])
+                       AC_SUBST([LIBNL_GENL_LIBS])
 
                        AC_DEFINE(ENABLE_DCO, 1, [Enable shared data channel offload])
                        AC_MSG_NOTICE([Enabled ovpn-dco support for Linux])
diff --git a/src/openvpn/Makefile.am b/src/openvpn/Makefile.am
index 8d0e66b4..1abd3e6d 100644
--- a/src/openvpn/Makefile.am
+++ b/src/openvpn/Makefile.am
@@ -26,6 +26,7 @@ AM_CPPFLAGS = \
 AM_CFLAGS = \
        $(TAP_CFLAGS) \
        $(OPTIONAL_CRYPTO_CFLAGS) \
+       $(LIBNL_GENL_CFLAGS) \
        $(OPTIONAL_LZO_CFLAGS) \
        $(OPTIONAL_LZ4_CFLAGS) \
        $(OPTIONAL_PKCS11_HELPER_CFLAGS) \
@@ -147,6 +148,7 @@ openvpn_LDADD = \
        $(OPTIONAL_LZ4_LIBS) \
        $(OPTIONAL_PKCS11_HELPER_LIBS) \
        $(OPTIONAL_CRYPTO_LIBS) \
+       $(LIBNL_GENL_LIBS) \
        $(OPTIONAL_SELINUX_LIBS) \
        $(OPTIONAL_SYSTEMD_LIBS) \
        $(OPTIONAL_DL_LIBS) \