git: af386f70fa0c - stable/12 - libarchive: merge security fix from vendor branch

The branch stable/12 has been updated by mm:

URL: https://cgit.FreeBSD.org/src/commit/?id=af386f70fa0c8d0a605c50710283f6f59525bd05

commit af386f70fa0c8d0a605c50710283f6f59525bd05
Author:     Martin Matuska <mm@FreeBSD.org>
AuthorDate: 2023-09-07 15:18:12 +0000
Commit:     Martin Matuska <mm@FreeBSD.org>
CommitDate: 2023-09-11 07:03:43 +0000

    libarchive: merge security fix from vendor branch
    
    This commit fixes a couple of security vulnerabilities in the PAX writer:
    1. Heap overflow in url_encode() in archive_write_set_format_pax.c
    2. NULL dereference in archive_write_pax_header_xattrs()
    3. Another NULL dereference in archive_write_pax_header_xattrs()
    4. NULL dereference in archive_write_pax_header_xattr()
    
    Security:       No known reference yet
    Obtained from:  https://github.com/libarchive/libarchive/commit/1b4e0d0f9
    MFC after:      3 days
    
    (cherry picked from commit f10f65999fe56e92f00b5bc5d27ac342cfea5364)
---
 .../libarchive/archive_write_set_format_pax.c      | 35 +++++++++++++++-------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/contrib/libarchive/libarchive/archive_write_set_format_pax.c b/contrib/libarchive/libarchive/archive_write_set_format_pax.c
index a2e6d9ad1c53..2d3c7086e1f9 100644
--- a/contrib/libarchive/libarchive/archive_write_set_format_pax.c
+++ b/contrib/libarchive/libarchive/archive_write_set_format_pax.c
@@ -367,10 +367,12 @@ archive_write_pax_header_xattr(struct pax *pax, const char *encoded_name,
 	struct archive_string s;
 	char *encoded_value;
 
+	if (encoded_name == NULL)
+		return;
+
 	if (pax->flags & WRITE_LIBARCHIVE_XATTR) {
 		encoded_value = base64_encode((const char *)value, value_len);
-
-		if (encoded_name != NULL && encoded_value != NULL) {
+		if (encoded_value != NULL) {
 			archive_string_init(&s);
 			archive_strcpy(&s, "LIBARCHIVE.xattr.");
 			archive_strcat(&s, encoded_name);
@@ -403,17 +405,22 @@ archive_write_pax_header_xattrs(struct archive_write *a,
 
 		archive_entry_xattr_next(entry, &name, &value, &size);
 		url_encoded_name = url_encode(name);
-		if (url_encoded_name != NULL) {
+		if (url_encoded_name == NULL)
+			goto malloc_error;
+		else {
 			/* Convert narrow-character to UTF-8. */
 			r = archive_strcpy_l(&(pax->l_url_encoded_name),
 			    url_encoded_name, pax->sconv_utf8);
 			free(url_encoded_name); /* Done with this. */
 			if (r == 0)
 				encoded_name = pax->l_url_encoded_name.s;
-			else if (errno == ENOMEM) {
-				archive_set_error(&a->archive, ENOMEM,
-				    "Can't allocate memory for Linkname");
-				return (ARCHIVE_FATAL);
+			else if (r == -1)
+				goto malloc_error;
+			else {
+				archive_set_error(&a->archive,
+				    ARCHIVE_ERRNO_MISC,
+				    "Error encoding pax extended attribute");
+				return (ARCHIVE_FAILED);
 			}
 		}
 
@@ -422,6 +429,9 @@ archive_write_pax_header_xattrs(struct archive_write *a,
 
 	}
 	return (ARCHIVE_OK);
+malloc_error:
+	archive_set_error(&a->archive, ENOMEM, "Can't allocate memory");
+	return (ARCHIVE_FATAL);
 }
 
 static int
@@ -1904,14 +1914,19 @@ url_encode(const char *in)
 {
 	const char *s;
 	char *d;
-	int out_len = 0;
+	size_t out_len = 0;
 	char *out;
 
 	for (s = in; *s != '\0'; s++) {
-		if (*s < 33 || *s > 126 || *s == '%' || *s == '=')
+		if (*s < 33 || *s > 126 || *s == '%' || *s == '=') {
+			if (SIZE_MAX - out_len < 4)
+				return (NULL);
 			out_len += 3;
-		else
+		} else {
+			if (SIZE_MAX - out_len < 2)
+				return (NULL);
 			out_len++;
+		}
 	}
 
 	out = (char *)malloc(out_len + 1);