svn commit: r365836 - head/share/mk

[Alex Richardson]

Author: arichardson
Date: Thu Sep 17 15:07:25 2020
New Revision: 365836
URL: https://svnweb.freebsd.org/changeset/base/365836

Log:
  Stop using lorder and ranlib when building libraries
  
  Use of ranlib or lorder is no longer necessary with current linkers
  (probably anything newer than ~1990) and ar's ability to create an object
  index and symbol table in the archive.
  Currently the build system uses lorder+tsort to sort the .o files in
  dependency order so that a single-pass linker can use them. However,
  we can use the -s flag to ar to add an index to the .a file which makes
  lorder unnecessary.
  Running ar -s is equivalent to running ranlib afterwards, so we can also
  skip the ranlib invocation.
  
  Similarly, we don't have to pass the .o files for shared libraries in
  dependency order since both ld.bfd and ld.lld will correctly resolve
  references between the .o files.
  
  This removes many fork()+execve calls for each library so should speed up
  builds a bit. Additionally lorder.sh uses a regular expression that is not
  supported by the macOS libc or glibc and results in many warnings when
  cross-building (see D25989).
  
  There is one functional change: lorder.sh removed duplicated .o files
  from the linker command line which now no longer happens. I fixed the duplicates
  in the base system in r364649. I also checked the ports tree for uses of
  bsd.lib.mk and found one duplicate source file which I fixed in r548168.
  Most ports use CMake/autotools rather than bsd.lib.mk but if this breaks any
  ports that I missed in my search please let me know.
  
  Avoiding the shell script actually speeds up the linking step noticeably: I
  measured how long it takes to rebuild the .a and .so files for lib/libc using a
  basic benchmark: `rm $LIBC_OBJDIR/*.so* $LIBC_OBJDIR/*.a* && /usr/bin/time make -DWITHOUT_TESTS -s > /dev/null`
  Without this change ~4.5 seconds and afterwards ~3.1 seconds.
  Looking at truss -cf output we can see that the number fork() system
  calls goes down from 27 to 12 (and the speedup while tracing is more
  noticeable: 81 seconds -> 65 seconds).
  
  See also https://www.gnu.org/software/coreutils/manual/html_node/tsort-background.html
  for some more background:
  This whole procedure has been obsolete since about 1980, because Unix
  archives now contain a symbol table (traditionally built by ranlib, now
  generally built by ar itself), and the Unix linker uses the symbol table
  to effectively make multiple passes over an archive file.
  
  Or alternatively https://www.unix.com/man-page/osf1/1/lorder/:
  The lorder command is essentially obsolete.  Use the following command in
  its place: % ar -ts file.a
  
  Reviewed By:	 emaste, imp, dim
  Differential Revision: https://reviews.freebsd.org/D26044

Modified:
  head/share/mk/bsd.lib.mk
  head/share/mk/sys.mk

Modified: head/share/mk/bsd.lib.mk
==============================================================================
--- head/share/mk/bsd.lib.mk	Thu Sep 17 14:58:30 2020	(r365835)
+++ head/share/mk/bsd.lib.mk	Thu Sep 17 15:07:25 2020	(r365836)
@@ -245,9 +245,7 @@ _LIBS=		lib${LIB_PRIVATE}${LIB}.a
 lib${LIB_PRIVATE}${LIB}.a: ${OBJS} ${STATICOBJS}
 	@${ECHO} building static ${LIB} library
 	@rm -f ${.TARGET}
-	${AR} ${ARFLAGS} ${.TARGET} `NM='${NM}' NMFLAGS='${NMFLAGS}' \
-	    ${LORDER} ${OBJS} ${STATICOBJS} | ${TSORT} ${TSORTFLAGS}` ${ARADD}
-	${RANLIB} ${RANLIBFLAGS} ${.TARGET}
+	${AR} ${ARFLAGS} ${.TARGET} ${OBJS} ${STATICOBJS} ${ARADD}
 .endif
 
 .if !defined(INTERNALLIB)
@@ -261,9 +259,7 @@ CLEANFILES+=	${POBJS}
 lib${LIB_PRIVATE}${LIB}_p.a: ${POBJS}
 	@${ECHO} building profiled ${LIB} library
 	@rm -f ${.TARGET}
-	${AR} ${ARFLAGS} ${.TARGET} `NM='${NM}' NMFLAGS='${NMFLAGS}' \
-	    ${LORDER} ${POBJS} | ${TSORT} ${TSORTFLAGS}` ${ARADD}
-	${RANLIB} ${RANLIBFLAGS} ${.TARGET}
+	${AR} ${ARFLAGS} ${.TARGET} ${POBJS} ${ARADD}
 .endif
 
 .if defined(LLVM_LINK)
@@ -319,9 +315,7 @@ ${SHLIB_NAME_FULL}: ${SOBJS}
 	@${INSTALL_LIBSYMLINK} ${TAG_ARGS:D${TAG_ARGS},dev} ${SHLIB_NAME} ${SHLIB_LINK}
 .endif
 	${_LD:N${CCACHE_BIN}} ${LDFLAGS} ${SSP_CFLAGS} ${SOLINKOPTS} \
-	    -o ${.TARGET} -Wl,-soname,${SONAME} \
-	    `NM='${NM}' NMFLAGS='${NMFLAGS}' ${LORDER} ${SOBJS} | \
-	    ${TSORT} ${TSORTFLAGS}` ${LDADD}
+	    -o ${.TARGET} -Wl,-soname,${SONAME} ${SOBJS} ${LDADD}
 .if ${MK_CTF} != "no"
 	${CTFMERGE} ${CTFFLAGS} -o ${.TARGET} ${SOBJS}
 .endif
@@ -347,7 +341,6 @@ lib${LIB_PRIVATE}${LIB}_pic.a: ${SOBJS}
 	@${ECHO} building special pic ${LIB} library
 	@rm -f ${.TARGET}
 	${AR} ${ARFLAGS} ${.TARGET} ${SOBJS} ${ARADD}
-	${RANLIB} ${RANLIBFLAGS} ${.TARGET}
 .endif
 
 .if defined(BUILD_NOSSP_PIC_ARCHIVE) && defined(LIB) && !empty(LIB)
@@ -360,7 +353,6 @@ lib${LIB_PRIVATE}${LIB}_nossp_pic.a: ${NOSSPSOBJS}
 	@${ECHO} building special nossp pic ${LIB} library
 	@rm -f ${.TARGET}
 	${AR} ${ARFLAGS} ${.TARGET} ${NOSSPSOBJS} ${ARADD}
-	${RANLIB} ${RANLIBFLAGS} ${.TARGET}
 .endif
 
 .endif # !defined(INTERNALLIB)
@@ -376,7 +368,6 @@ lib${LIB_PRIVATE}${LIB}_pie.a: ${PIEOBJS}
 	@${ECHO} building pie ${LIB} library
 	@rm -f ${.TARGET}
 	${AR} ${ARFLAGS} ${.TARGET} ${PIEOBJS} ${ARADD}
-	${RANLIB} ${RANLIBFLAGS} ${.TARGET}
 .endif
 
 .if defined(_SKIP_BUILD)

Modified: head/share/mk/sys.mk
==============================================================================
--- head/share/mk/sys.mk	Thu Sep 17 14:58:30 2020	(r365835)
+++ head/share/mk/sys.mk	Thu Sep 17 15:07:25 2020	(r365836)
@@ -150,7 +150,7 @@ AR		?=	ar
 .if defined(%POSIX)
 ARFLAGS		?=	-rv
 .else
-ARFLAGS		?=	-crD
+ARFLAGS		?=	-crsD
 .endif
 RANLIB		?=	ranlib
 .if !defined(%POSIX)