svn commit: r335450 - in head: . share/mk tools/build/options

Wed Jun 20 16:10:18 UTC 2018

Author: bdrewery
Date: Wed Jun 20 16:10:14 2018
New Revision: 335450
URL: https://svnweb.freebsd.org/changeset/base/335450

Log:
  Add WITH_SYSTEM_LINKER, on by default, that avoids building lld when possible.
  
  This works similar to WITH_SYSTEM_COMPILER added in r300354.  It only
  supports lld via WITH_LLD_BOOTSTRAP.
  
  When both SYSTEM_COMPILER and SYSTEM_LINKER logic passes then libclang
  will not build in cross-tools.  If either check fails though then
  libclang is built.
  
  The .info is reworked to notify when libclang will be built since if
  either clang or lld needs to be rebuilt, but not the other, the
  notification can lead to confusion on why "clang is building".
  
  -fuse-ld= is not used with this method so some combinations of compiler
  and linker are expected to fail.
  
  A new 'make test-system-linker' target is added to see the logic results.
  
  Makefile.inc1:
    CROSS_BINUTILS_PREFIX support had to be moved higher up so that XLD
    could be set and MK_LLD_BOOTSTRAP disabled before checking SYSTEM_LINKER
    logic as done with SYSTEM_COMPILER.  This also required moving where
    bsd.linker.mk was read since XLD needs to be set before parsing it.  This
    creates a situation where src.opts.mk can not test LINKER_FEATURES or
    add LLD_BOOTSTAP to BROKEN_OPTIONS.
  
  Reviewed by:	emaste (earlier version)
  Sponsored by:	Dell EMC
  Differential Revision:	https://reviews.freebsd.org/D15894

Added:
  head/tools/build/options/WITHOUT_SYSTEM_LINKER   (contents, props changed)
  head/tools/build/options/WITH_SYSTEM_LINKER   (contents, props changed)
Modified:
  head/Makefile
  head/Makefile.inc1
  head/share/mk/bsd.compiler.mk
  head/share/mk/bsd.linker.mk
  head/share/mk/src.opts.mk

Modified: head/Makefile
==============================================================================

--- head/Makefile	Wed Jun 20 16:10:10 2018	(r335449)
+++ head/Makefile	Wed Jun 20 16:10:14 2018	(r335450)
@@ -143,7 +143,7 @@ TGTS=	all all-man buildenv buildenvvars buildkernel bu
 	stage-packages \
 	create-packages-world create-packages-kernel create-packages \
 	packages installconfig real-packages sign-packages package-pkg \
-	print-dir test-system-compiler
+	print-dir test-system-compiler test-system-linker
 
 # These targets require a TARGET and TARGET_ARCH be defined.
 XTGTS=	native-xtools native-xtools-install xdev xdev-build xdev-install \
@@ -168,7 +168,7 @@ META_TGT_WHITELIST+= \
 	_* build32 buildfiles buildincludes buildkernel buildsoft \
 	buildworld everything kernel-toolchain kernel-toolchains kernel \
 	kernels libraries native-xtools showconfig test-system-compiler \
-	tinderbox toolchain \
+	test-system-linker tinderbox toolchain \
 	toolchains universe world worlds xdev xdev-build
 
 .ORDER: buildworld installworld

Modified: head/Makefile.inc1
==============================================================================
--- head/Makefile.inc1	Wed Jun 20 16:10:10 2018	(r335449)
+++ head/Makefile.inc1	Wed Jun 20 16:10:14 2018	(r335450)
@@ -97,9 +97,74 @@ MK_GCC_BOOTSTRAP=	no
 # Pull in COMPILER_TYPE and COMPILER_FREEBSD_VERSION early. Pull it from the
 # tree to be friendlier to foreign OS builds. It's safe to do so unconditionally
 # here since we will always have the right make, unlike in src/Makefile
+# Don't include bsd.linker.mk yet until XBINUTILS is handled (after src.opts.mk)
+_NO_INCLUDE_LINKERMK=	t
 .include "share/mk/bsd.compiler.mk"
+.undef _NO_INCLUDE_LINKERMK
+# src.opts.mk depends on COMPILER_FEATURES
 .include "share/mk/src.opts.mk"
 
+.if ${TARGET} == ${MACHINE}
+TARGET_CPUTYPE?=${CPUTYPE}
+.else
+TARGET_CPUTYPE?=
+.endif
+.if !empty(TARGET_CPUTYPE)
+_TARGET_CPUTYPE=${TARGET_CPUTYPE}
+.else
+_TARGET_CPUTYPE=dummy
+.endif
+.if ${TARGET} == "arm"
+.if ${TARGET_ARCH:Marmv[67]*} != "" && ${TARGET_CPUTYPE:M*soft*} == ""
+TARGET_ABI=	gnueabihf
+.else
+TARGET_ABI=	gnueabi
+.endif
+.endif
+MACHINE_ABI?=	unknown
+MACHINE_TRIPLE?=${MACHINE_ARCH:S/amd64/x86_64/:C/hf$//:S/mipsn32/mips64/}-${MACHINE_ABI}-freebsd12.0
+TARGET_ABI?=	unknown
+TARGET_TRIPLE?=	${TARGET_ARCH:S/amd64/x86_64/:C/hf$//:S/mipsn32/mips64/}-${TARGET_ABI}-freebsd12.0
+
+# Handle external binutils.
+.if defined(CROSS_TOOLCHAIN_PREFIX)
+CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
+.endif
+# If we do not have a bootstrap binutils (because the in-tree one does not
+# support the target architecture), provide a default cross-binutils prefix.
+# This allows riscv64 builds, for example, to automatically use the
+# riscv64-binutils port or package.
+.if !make(showconfig)
+.if !empty(BROKEN_OPTIONS:MBINUTILS_BOOTSTRAP) && \
+    ${MK_LLD_BOOTSTRAP} == "no" && \
+    !defined(CROSS_BINUTILS_PREFIX)
+CROSS_BINUTILS_PREFIX=/usr/local/${TARGET_TRIPLE}/bin/
+.if !exists(${CROSS_BINUTILS_PREFIX})
+.error In-tree binutils does not support the ${TARGET_ARCH} architecture. Install the ${TARGET_ARCH}-binutils port or package or set CROSS_BINUTILS_PREFIX.
+.endif
+.endif
+.endif
+XBINUTILS=	AS AR LD NM OBJCOPY RANLIB SIZE STRINGS
+.for BINUTIL in ${XBINUTILS}
+.if defined(CROSS_BINUTILS_PREFIX) && \
+    exists(${CROSS_BINUTILS_PREFIX}/${${BINUTIL}})
+X${BINUTIL}?=	${CROSS_BINUTILS_PREFIX:C,/*$,,}/${${BINUTIL}}
+.else
+X${BINUTIL}?=	${${BINUTIL}}
+.endif
+.endfor
+
+# If a full path to an external linker is given, don't build lld.
+.if ${XLD:M/*}
+MK_LLD_BOOTSTRAP=	no
+.endif
+
+.include "share/mk/bsd.linker.mk"
+
+# Begin WITH_SYSTEM_COMPILER / WITH_SYSTEM_LD
+
+# WITH_SYSTEM_COMPILER - Pull in needed values and make a decision.
+
 # Check if there is a local compiler that can satisfy as an external compiler.
 # Which compiler is expected to be used?
 .if ${MK_CLANG_BOOTSTRAP} == "yes"
@@ -109,6 +174,7 @@ WANT_COMPILER_TYPE=	gcc
 .else
 WANT_COMPILER_TYPE=
 .endif
+
 .if !defined(WANT_COMPILER_FREEBSD_VERSION)
 .if ${WANT_COMPILER_TYPE} == "clang"
 WANT_COMPILER_FREEBSD_VERSION_FILE= lib/clang/freebsd_cc_version.h
@@ -131,6 +197,7 @@ WANT_COMPILER_VERSION!= \
 .endif
 .export WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_VERSION
 .endif	# !defined(WANT_COMPILER_FREEBSD_VERSION)
+
 # It needs to be the same revision as we would build for the bootstrap.
 # If the expected vs CC is different then we can't skip.
 # GCC cannot be used for cross-arch yet.  For clang we pass -target later if
@@ -147,7 +214,52 @@ MK_CLANG_BOOTSTRAP=	no
 MK_GCC_BOOTSTRAP=	no
 USING_SYSTEM_COMPILER=	yes
 .endif	# ${WANT_COMPILER_TYPE} == ${COMPILER_TYPE}
+
+# WITH_SYSTEM_LD - Pull in needed values and make a decision.
+
+# Check if there is a local linker that can satisfy as an external linker.
+# Which linker is expected to be used?
+.if ${MK_LLD_BOOTSTRAP} == "yes"
+WANT_LINKER_TYPE=		lld
+.elif ${MK_BINUTILS_BOOTSTRAP} == "yes"
+# Note that there's no support for bfd in WITH_SYSTEM_LINKER.
+WANT_LINKER_TYPE=	bfd
+.else
+WANT_LINKER_TYPE=
+.endif
+
+.if !defined(WANT_LINKER_FREEBSD_VERSION)
+.if ${WANT_LINKER_TYPE} == "lld"
+WANT_LINKER_FREEBSD_VERSION_FILE= lib/clang/include/lld/Common/Version.inc
+WANT_LINKER_FREEBSD_VERSION!= \
+	awk '$$2 == "LLD_REVISION_STRING" {gsub(/"/, "", $$3); print $$3}' \
+	${SRCDIR}/${WANT_LINKER_FREEBSD_VERSION_FILE} || echo unknown
+WANT_LINKER_VERSION_FILE= lib/clang/include/lld/Common/Version.inc
+WANT_LINKER_VERSION!= \
+	awk '$$2 == "LLD_VERSION" {split($$3, a, "."); print a[1] * 10000 + a[2] * 100 + a[3]}' \
+	${SRCDIR}/${WANT_LINKER_VERSION_FILE} || echo unknown
+.else
+WANT_LINKER_FREEBSD_VERSION_FILE=
+WANT_LINKER_FREEBSD_VERSION=
+.endif
+.export WANT_LINKER_FREEBSD_VERSION WANT_LINKER_VERSION
+.endif	# !defined(WANT_LINKER_FREEBSD_VERSION)
+
+.if ${MK_SYSTEM_LINKER} == "yes" && \
+    (${MK_LLD_BOOTSTRAP} == "yes") && \
+    !make(showconfig) && !make(xdev*) && \
+    ${X_LINKER_TYPE} == ${WANT_LINKER_TYPE} && \
+    ${X_LINKER_VERSION} == ${WANT_LINKER_VERSION} && \
+    ${X_LINKER_FREEBSD_VERSION} == ${WANT_LINKER_FREEBSD_VERSION}
+# Everything matches, disable the bootstrap linker.
+MK_LLD_BOOTSTRAP=	no
+USING_SYSTEM_LINKER=	yes
+.endif	# ${WANT_LINKER_TYPE} == ${LINKER_TYPE}
+
+# WITH_SYSTEM_COMPILER / WITH_SYSTEM_LINKER - Handle defaults and debug.
 USING_SYSTEM_COMPILER?=	no
+USING_SYSTEM_LINKER?=	no
+
 TEST_SYSTEM_COMPILER_VARS= \
 	USING_SYSTEM_COMPILER MK_SYSTEM_COMPILER \
 	MK_CROSS_COMPILER MK_CLANG_BOOTSTRAP MK_GCC_BOOTSTRAP \
@@ -155,20 +267,40 @@ TEST_SYSTEM_COMPILER_VARS= \
 	WANT_COMPILER_FREEBSD_VERSION WANT_COMPILER_FREEBSD_VERSION_FILE \
 	CC COMPILER_TYPE COMPILER_FEATURES COMPILER_VERSION \
 	COMPILER_FREEBSD_VERSION \
-	X_COMPILER_TYPE X_COMPILER_FEATURES X_COMPILER_VERSION \
-	X_COMPILER_FREEBSD_VERSION \
-	LINKER_TYPE LINKER_FEATURES LINKER_VERSION \
-	X_LINKER_TYPE X_LINKER_FEATURES X_LINKER_VERSION
-test-system-compiler: .PHONY
-.for v in ${TEST_SYSTEM_COMPILER_VARS}
+	XCC X_COMPILER_TYPE X_COMPILER_FEATURES X_COMPILER_VERSION \
+	X_COMPILER_FREEBSD_VERSION
+TEST_SYSTEM_LINKER_VARS= \
+	USING_SYSTEM_LINKER MK_SYSTEM_LINKER \
+	MK_LLD_BOOTSTRAP MK_BINUTILS_BOOTSTRAP \
+	WANT_LINKER_TYPE WANT_LINKER_VERSION WANT_LINKER_VERSION_FILE \
+	WANT_LINKER_FREEBSD_VERSION WANT_LINKER_FREEBSD_VERSION_FILE \
+	LD LINKER_TYPE LINKER_FEATURES LINKER_VERSION \
+	LINKER_FREEBSD_VERSION \
+	XLD X_LINKER_TYPE X_LINKER_FEATURES X_LINKER_VERSION \
+	X_LINKER_FREEBSD_VERSION
+
+.for _t in compiler linker
+test-system-${_t}: .PHONY
+.for v in ${TEST_SYSTEM_${_t:tu}_VARS}
 	${_+_}@printf "%-35s= %s\n" "${v}" "${${v}}"
 .endfor
-.if ${USING_SYSTEM_COMPILER} == "yes" && \
-    (make(buildworld) || make(buildkernel) || make(kernel-toolchain) || \
+.endfor
+.if (make(buildworld) || make(buildkernel) || make(kernel-toolchain) || \
     make(toolchain) || make(_cross-tools))
+.if ${USING_SYSTEM_COMPILER} == "yes"
 .info SYSTEM_COMPILER: Determined that CC=${CC} matches the source tree.  Not bootstrapping a cross-compiler.
+.elif ${MK_CLANG_BOOTSTRAP} == "yes"
+.info SYSTEM_COMPILER: libclang will be built for bootstrapping a cross-compiler.
 .endif
+.if ${USING_SYSTEM_LINKER} == "yes"
+.info SYSTEM_LINKER: Determined that LD=${LD} matches the source tree.  Not bootstrapping a cross-linker.
+.elif ${MK_LLD_BOOTSTRAP} == "yes"
+.info SYSTEM_LINKER: libclang will be built for bootstrapping a cross-linker.
+.endif
+.endif
 
+# End WITH_SYSTEM_COMPILER / WITH_SYSTEM_LD
+
 # Store some compiler metadata for use in installworld where we don't
 # want to invoke CC at all.
 _COMPILER_METADATA_VARS=	COMPILER_VERSION \
@@ -177,7 +309,8 @@ _COMPILER_METADATA_VARS=	COMPILER_VERSION \
 				COMPILER_FREEBSD_VERSION \
 				LINKER_VERSION \
 				LINKER_FEATURES \
-				LINKER_TYPE
+				LINKER_TYPE \
+				LINKER_FREEBSD_VERSION
 compiler-metadata.mk: .PHONY .META
 	@: > ${.TARGET}
 	@echo ".info Using cached compiler metadata from build at $$(hostname) on $$(date)" \
@@ -190,61 +323,7 @@ compiler-metadata.mk: .PHONY .META
 	@echo ".export ${_COMPILER_METADATA_VARS}" >> ${.TARGET}
 	@echo ".export ${_COMPILER_METADATA_VARS:C,^,X_,}" >> ${.TARGET}
 
-.if ${TARGET} == ${MACHINE}
-TARGET_CPUTYPE?=${CPUTYPE}
-.else
-TARGET_CPUTYPE?=
-.endif
-.if !empty(TARGET_CPUTYPE)
-_TARGET_CPUTYPE=${TARGET_CPUTYPE}
-.else
-_TARGET_CPUTYPE=dummy
-.endif
-.if ${TARGET} == "arm"
-.if ${TARGET_ARCH:Marmv[67]*} != "" && ${TARGET_CPUTYPE:M*soft*} == ""
-TARGET_ABI=	gnueabihf
-.else
-TARGET_ABI=	gnueabi
-.endif
-.endif
-MACHINE_ABI?=	unknown
-MACHINE_TRIPLE?=${MACHINE_ARCH:S/amd64/x86_64/:C/hf$//:S/mipsn32/mips64/}-${MACHINE_ABI}-freebsd12.0
-TARGET_ABI?=	unknown
-TARGET_TRIPLE?=	${TARGET_ARCH:S/amd64/x86_64/:C/hf$//:S/mipsn32/mips64/}-${TARGET_ABI}-freebsd12.0
 
-# Handle external binutils.
-.if defined(CROSS_TOOLCHAIN_PREFIX)
-CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
-.endif
-# If we do not have a bootstrap binutils (because the in-tree one does not
-# support the target architecture), provide a default cross-binutils prefix.
-# This allows riscv64 builds, for example, to automatically use the
-# riscv64-binutils port or package.
-.if !make(showconfig)
-.if !empty(BROKEN_OPTIONS:MBINUTILS_BOOTSTRAP) && \
-    ${MK_LLD_BOOTSTRAP} == "no" && \
-    !defined(CROSS_BINUTILS_PREFIX)
-CROSS_BINUTILS_PREFIX=/usr/local/${TARGET_TRIPLE}/bin/
-.if !exists(${CROSS_BINUTILS_PREFIX})
-.error In-tree binutils does not support the ${TARGET_ARCH} architecture. Install the ${TARGET_ARCH}-binutils port or package or set CROSS_BINUTILS_PREFIX.
-.endif
-.endif
-.endif
-XBINUTILS=	AS AR LD NM OBJCOPY RANLIB SIZE STRINGS
-.for BINUTIL in ${XBINUTILS}
-.if defined(CROSS_BINUTILS_PREFIX) && \
-    exists(${CROSS_BINUTILS_PREFIX}/${${BINUTIL}})
-X${BINUTIL}?=	${CROSS_BINUTILS_PREFIX:C,/*$,,}/${${BINUTIL}}
-.else
-X${BINUTIL}?=	${${BINUTIL}}
-.endif
-.endfor
-
-# If a full path to an external linker is given, don't build lld.
-.if ${XLD:M/*}
-MK_LLD_BOOTSTRAP=	no
-.endif
-
 # We must do lib/ and libexec/ before bin/ in case of a mid-install error to
 # keep the users system reasonably usable.  For static->dynamic root upgrades,
 # we don't want to install a dynamic binary without rtld and the needed
@@ -857,6 +936,9 @@ _worldtmp: .PHONY
 	fi
 .endfor
 .endif	# ${USING_SYSTEM_COMPILER} == "yes"
+.if ${USING_SYSTEM_LINKER} == "yes"
+	@rm -f ${WORLDTMP}/usr/bin/ld ${WORLDTMP}/usr/bin/ld.lld
+.endif	# ${USING_SYSTEM_LINKER} == "yes"
 .endif	# !defined(NO_CLEAN)
 	@mkdir -p ${WORLDTMP}
 	@touch ${WORLDTMP}/${.TARGET}

Modified: head/share/mk/bsd.compiler.mk
==============================================================================
--- head/share/mk/bsd.compiler.mk	Wed Jun 20 16:10:10 2018	(r335449)
+++ head/share/mk/bsd.compiler.mk	Wed Jun 20 16:10:14 2018	(r335450)
@@ -208,5 +208,7 @@ ${var}.${${X_}_cc_hash}:=	${${var}}
 .endif	# ${cc} == "CC" || !empty(XCC)
 .endfor	# .for cc in CC XCC
 
+.if !defined(_NO_INCLUDE_LINKERMK)
 .include <bsd.linker.mk>
+.endif
 .endif	# !target(__<bsd.compiler.mk>__)

Modified: head/share/mk/bsd.linker.mk
==============================================================================
--- head/share/mk/bsd.linker.mk	Wed Jun 20 16:10:10 2018	(r335449)
+++ head/share/mk/bsd.linker.mk	Wed Jun 20 16:10:14 2018	(r335450)
@@ -16,6 +16,8 @@
 # - retpoline: support for generating PLT with retpoline speculative
 #              execution vulnerability mitigation
 #
+# LINKER_FREEBSD_VERSION is the linker's internal source version.
+#
 # These variables with an X_ prefix will also be provided if XLD is set.
 #
 # This file may be included multiple times, but only has effect the first time.
@@ -29,7 +31,8 @@ __<bsd.linker.mk>__:
 # Try to import LINKER_TYPE and LINKER_VERSION from parent make.
 # The value is only used/exported for the same environment that impacts
 # LD and LINKER_* settings here.
-_exported_vars=	${X_}LINKER_TYPE ${X_}LINKER_VERSION ${X_}LINKER_FEATURES
+_exported_vars=	${X_}LINKER_TYPE ${X_}LINKER_VERSION ${X_}LINKER_FEATURES \
+		${X_}LINKER_FREEBSD_VERSION
 ${X_}_ld_hash=	${${ld}}${MACHINE}${PATH}
 ${X_}_ld_hash:=	${${X_}_ld_hash:hash}
 # Only import if none of the vars are set somehow else.
@@ -55,10 +58,14 @@ _ld_version!=	(${${ld}} --version || echo none) | sed 
 .endif
 .if ${_ld_version:[1..2]} == "GNU ld"
 ${X_}LINKER_TYPE=	bfd
+${X_}LINKER_FREEBSD_VERSION=	0
 _v=	${_ld_version:M[1-9].[0-9]*:[1]}
 .elif ${_ld_version:[1]} == "LLD"
 ${X_}LINKER_TYPE=	lld
 _v=	${_ld_version:[2]}
+${X_}LINKER_FREEBSD_VERSION!= \
+	${${ld}} --version | \
+	awk '$$3 ~ /FreeBSD/ {print substr($$4, 1, length($$4)-1)}'
 .else
 .warning Unknown linker from ${ld}=${${ld}}: ${_ld_version}, defaulting to bfd
 ${X_}LINKER_TYPE=	bfd
@@ -85,6 +92,7 @@ ${X_}LINKER_FEATURES+=	retpoline
 X_LINKER_TYPE=		${LINKER_TYPE}
 X_LINKER_VERSION=	${LINKER_VERSION}
 X_LINKER_FEATURES=	${LINKER_FEATURES}
+X_LINKER_FREEBSD_VERSION= ${LINKER_FREEBSD_VERSION}
 .endif	# ${ld} == "LD" || (${ld} == "XLD" && ${XLD} != ${LD})
 
 # Export the values so sub-makes don't have to look them up again, using the

Modified: head/share/mk/src.opts.mk
==============================================================================
--- head/share/mk/src.opts.mk	Wed Jun 20 16:10:10 2018	(r335449)
+++ head/share/mk/src.opts.mk	Wed Jun 20 16:10:14 2018	(r335450)
@@ -168,6 +168,7 @@ __DEFAULT_YES_OPTIONS = \
     SVNLITE \
     SYSCONS \
     SYSTEM_COMPILER \
+    SYSTEM_LINKER \
     TALK \
     TCP_WRAPPERS \
     TCSH \

Added: head/tools/build/options/WITHOUT_SYSTEM_LINKER
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/tools/build/options/WITHOUT_SYSTEM_LINKER	Wed Jun 20 16:10:14 2018	(r335450)
@@ -0,0 +1,16 @@
+.\" $FreeBSD$
+Set to not opportunistically skip building a cross-linker during the
+bootstrap phase of the build.
+Normally, if the currently installed linker matches the planned bootstrap
+linker type and revision, then it will not be built.
+This does not prevent a linker from being built for installation though,
+only for building one for the build itself.
+The
+.Va WITHOUT_LLD
+and
+.Va WITHOUT_BINUTILS
+options control those.
+.Pp
+This option is only relevant when
+.Va WITH_LLD_BOOTSTRAP
+is set.

Added: head/tools/build/options/WITH_SYSTEM_LINKER
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ head/tools/build/options/WITH_SYSTEM_LINKER	Wed Jun 20 16:10:14 2018	(r335450)
@@ -0,0 +1,16 @@
+.\" $FreeBSD$
+Set to opportunistically skip building a cross-linker during the
+bootstrap phase of the build.
+If the currently installed linker matches the planned bootstrap linker
+type and revision, then it will not be built.
+This does not prevent a linker from being built for installation though,
+only for building one for the build itself.
+The
+.Va WITHOUT_LLD
+and
+.Va WITHOUT_BINUTILS
+options control those.
+.Pp
+This option is only relevant when
+.Va WITH_LLD_BOOTSTRAP
+is set.
