git: e3bad5f7aa86 - main - fetch(1): process truncated transfer as soft failure

The branch main has been updated by eugen:

URL: https://cgit.FreeBSD.org/src/commit/?id=e3bad5f7aa86a0911cf8d28395e7a29395739985

commit e3bad5f7aa86a0911cf8d28395e7a29395739985
Author:     Eugene Grosbein <eugen@FreeBSD.org>
AuthorDate: 2022-01-24 04:03:42 +0000
Commit:     Eugene Grosbein <eugen@FreeBSD.org>
CommitDate: 2022-01-24 04:09:37 +0000

    fetch(1): process truncated transfer as soft failure
    
    Let "fetch -a" resume truncated transfer automatically
    perform another attempt if it obtained some new data in previous one
    making progress.
    
    This makes it more robust against frequent but transient network failures.
    For example:
    
    => sqlite-src-3370200.zip doesn't seem to exist in /usr/ports/distfiles/.
    => Attempting to fetch https://www.sqlite.org/2022/sqlite-src-3370200.zip
    sqlite-src-3370200.zip                          3% of   12 MB   45 kBps 04m24s
    fetch: sqlite-src-3370200.zip appears to be truncated: 524288/13145234 bytes
    sqlite-src-3370200.zip                         10% of   12 MB   67 kBps 02m56s
    fetch: sqlite-src-3370200.zip appears to be truncated: 1327104/13145234 bytes
    sqlite-src-3370200.zip                         28% of   12 MB  123 kBps 01m14s
    fetch: sqlite-src-3370200.zip appears to be truncated: 3735552/13145234 bytes
    sqlite-src-3370200.zip                         54% of   12 MB  253 kBps    24s
    fetch: sqlite-src-3370200.zip appears to be truncated: 7176192/13145234 bytes
    sqlite-src-3370200.zip                         62% of   12 MB   90 kBps    55s
    fetch: sqlite-src-3370200.zip appears to be truncated: 8241152/13145234 bytes
    sqlite-src-3370200.zip                         82% of   12 MB  113 kBps    20s
    fetch: sqlite-src-3370200.zip appears to be truncated: 10862592/13145234 bytes
    sqlite-src-3370200.zip                                  12 MB  185 kBps    12s
    ===> Fetching all distfiles required by sqlite3-3.37.2,1 for building
    
    MFC after:      1 month
---
 usr.bin/fetch/fetch.c | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/usr.bin/fetch/fetch.c b/usr.bin/fetch/fetch.c
index 22f7a1d2950b..3526e5d6c5c1 100644
--- a/usr.bin/fetch/fetch.c
+++ b/usr.bin/fetch/fetch.c
@@ -433,11 +433,11 @@ fetch(char *URL, const char *path)
 	struct xferstat xs;
 	FILE *f, *of;
 	size_t size, readcnt, wr;
-	off_t count;
+	off_t count, size_prev;
 	char flags[8];
 	const char *slash;
 	char *tmppath;
-	int r;
+	int r, tries;
 	unsigned timeout;
 	char *ptr;
 
@@ -537,6 +537,9 @@ fetch(char *URL, const char *path)
 		goto success;
 	}
 
+	tries = 1;
+again:
+	r = 0;
 	/*
 	 * If the -r flag was specified, we have to compare the local
 	 * and remote files, so we should really do a fetchStat()
@@ -553,7 +556,7 @@ fetch(char *URL, const char *path)
 	sb.st_size = -1;
 	if (!o_stdout) {
 		r = stat(path, &sb);
-		if (r == 0 && r_flag && S_ISREG(sb.st_mode)) {
+		if (r == 0 && (r_flag || tries > 1) && S_ISREG(sb.st_mode)) {
 			url->offset = sb.st_size;
 		} else if (r == -1 || !S_ISREG(sb.st_mode)) {
 			/*
@@ -568,6 +571,7 @@ fetch(char *URL, const char *path)
 			goto failure;
 		}
 	}
+	size_prev = sb.st_size;
 
 	/* start the transfer */
 	if (timeout)
@@ -629,7 +633,7 @@ fetch(char *URL, const char *path)
 		of = stdout;
 	} else if (r_flag && sb.st_size != -1) {
 		/* resume mode, local file exists */
-		if (!F_flag && us.mtime && sb.st_mtime != us.mtime) {
+		if (!F_flag && us.mtime && sb.st_mtime != us.mtime && tries == 1) {
 			/* no match! have to refetch */
 			fclose(f);
 			/* if precious, warn the user and give up */
@@ -717,6 +721,8 @@ fetch(char *URL, const char *path)
 				slash = path;
 			else
 				++slash;
+			if(tmppath != NULL)
+				free(tmppath);
 			asprintf(&tmppath, "%.*s.fetch.XXXXXX.%s",
 			    (int)(slash - path), path, slash);
 			if (tmppath != NULL) {
@@ -829,6 +835,13 @@ fetch(char *URL, const char *path)
 	if (us.size != -1 && count < us.size) {
 		warnx("%s appears to be truncated: %jd/%jd bytes",
 		    path, (intmax_t)count, (intmax_t)us.size);
+		if(!o_stdout && a_flag && us.size > size_prev) {
+			fclose(f);
+			if (w_secs)
+				sleep(w_secs);
+			tries++;
+			goto again;
+		}
 		goto failure_keep;
 	}