git: 60382b4a04fa - main - libfetch: Reduce copying
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Mon, 13 Jul 2026 06:45:50 UTC
The branch main has been updated by des:
URL: https://cgit.FreeBSD.org/src/commit/?id=60382b4a04fa39e9bf65b964b1b7b4bed6eaa56a
commit 60382b4a04fa39e9bf65b964b1b7b4bed6eaa56a
Author: Dag-Erling Smørgrav <des@FreeBSD.org>
AuthorDate: 2026-07-13 06:43:56 +0000
Commit: Dag-Erling Smørgrav <des@FreeBSD.org>
CommitDate: 2026-07-13 06:43:56 +0000
libfetch: Reduce copying
Reduce the amount of copying we do when performing buffered reads.
MFC after: 1 week
Reviewed by: op
Differential Revision: https://reviews.freebsd.org/D58113
---
lib/libfetch/common.c | 45 +++++++++++++++++++++++++++++----------------
lib/libfetch/common.h | 2 ++
lib/libfetch/ftp.c | 23 ++++++++++++-----------
lib/libfetch/http.c | 41 ++++++++++++++++++-----------------------
4 files changed, 61 insertions(+), 50 deletions(-)
diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index 74748e5b7d17..dabc8c4b16b3 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -1437,12 +1437,23 @@ fetch_getln(conn_t *conn)
/* allocate initial buffer */
if (conn->buf == NULL) {
if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL)
- return (-1);
+ goto fail;
+ conn->line = conn->buf;
conn->bufsize = MIN_BUF_SIZE;
conn->buflen = 0;
conn->pos = 0;
}
+ /* look at the data we already have */
+ if (conn->pos < conn->buflen) {
+ conn->line = conn->buf + conn->pos;
+ while (conn->pos < conn->buflen)
+ if (conn->buf[conn->pos++] == '\n')
+ goto found;
+ /* reset for the upcoming memmove() */
+ conn->pos = conn->line - conn->buf;
+ }
+
/* move residual data up */
if (conn->buflen > 0) {
if (conn->pos < conn->buflen) {
@@ -1451,20 +1462,16 @@ fetch_getln(conn_t *conn)
}
conn->buflen -= conn->pos;
conn->pos -= conn->pos;
+ conn->line = conn->buf;
}
- /* do we have a complete line? */
- while (conn->pos < conn->buflen)
- if (conn->buf[conn->pos++] == '\n')
- goto found;
-
for (;;) {
/* read as much as we can right now */
rlen = fetch_read(conn, conn->buf + conn->buflen,
conn->bufsize - conn->buflen - 1);
/* error */
if (rlen < 0)
- return (-1);
+ goto fail;
/* advance and terminate */
conn->buflen += rlen;
conn->buf[conn->buflen] = '\0';
@@ -1480,21 +1487,25 @@ fetch_getln(conn_t *conn)
tmp = conn->buf;
tmpsize = conn->bufsize * 2;
if ((tmp = realloc(tmp, tmpsize)) == NULL)
- return (-1);
- conn->buf = tmp;
+ goto fail;
+ conn->line = conn->buf = tmp;
conn->bufsize = tmpsize;
}
}
/* connection closed, return what's left */
conn->pos = conn->buflen;
found:
- rlen = conn->pos;
- if (rlen > 0 && conn->buf[rlen - 1] == '\n')
- conn->buf[--rlen] = '\0';
- if (rlen > 0 && conn->buf[rlen - 1] == '\r')
- conn->buf[--rlen] = '\0';
- DEBUGF("<<< %.*s\n", (int)rlen, conn->buf);
- return (rlen);
+ conn->linelen = (conn->buf + conn->pos) - conn->line;
+ if (conn->linelen > 0 && conn->line[conn->linelen - 1] == '\n')
+ conn->line[--conn->linelen] = '\0';
+ if (conn->linelen > 0 && conn->line[conn->linelen - 1] == '\r')
+ conn->line[--conn->linelen] = '\0';
+ DEBUGF("<<< %.*s\n", (int)conn->linelen, conn->line);
+ return (conn->linelen);
+fail:
+ conn->line = NULL;
+ conn->linelen = 0;
+ return (-1);
}
@@ -1520,6 +1531,8 @@ fetch_bufread(conn_t *conn, void *buf, size_t len)
conn->buflen = 0;
conn->pos = 0;
}
+ conn->line = NULL;
+ conn->linelen = 0;
/* return residual data first */
if (conn->buflen > conn->pos) {
diff --git a/lib/libfetch/common.h b/lib/libfetch/common.h
index 143d4fe82945..ac6ec923050d 100644
--- a/lib/libfetch/common.h
+++ b/lib/libfetch/common.h
@@ -49,8 +49,10 @@ typedef struct fetchconn conn_t;
struct fetchconn {
int sd; /* socket descriptor */
char *buf; /* buffer */
+ char *line; /* line (within buffer) */
size_t bufsize; /* buffer size */
size_t buflen; /* length of buffer contents */
+ size_t linelen; /* length of line */
size_t pos; /* current position in buffer */
int err; /* last protocol reply code */
#ifdef WITH_SSL
diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index 5e4148e0fa2c..4c1a63d3da82 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -148,8 +148,9 @@ ftp_chkerr(conn_t *conn)
fetch_syserr();
return (-1);
}
- if (isftpinfo(conn->buf)) {
- while (conn->buflen && !isftpreply(conn->buf)) {
+
+ if (isftpinfo(conn->line)) {
+ while (rlen && !isftpreply(conn->line)) {
if ((rlen = fetch_getln(conn)) < 0) {
fetch_syserr();
return (-1);
@@ -157,14 +158,14 @@ ftp_chkerr(conn_t *conn)
}
}
- if (!isftpreply(conn->buf)) {
+ if (!isftpreply(conn->line)) {
ftp_seterr(FTP_PROTOCOL_ERROR);
return (-1);
}
- conn->err = (conn->buf[0] - '0') * 100
- + (conn->buf[1] - '0') * 10
- + (conn->buf[2] - '0');
+ conn->err = (conn->line[0] - '0') * 100
+ + (conn->line[1] - '0') * 10
+ + (conn->line[2] - '0');
return (conn->err);
}
@@ -236,8 +237,8 @@ ftp_pwd(conn_t *conn, char *pwd, size_t pwdlen)
if (conn->err != FTP_WORKING_DIRECTORY &&
conn->err != FTP_FILE_ACTION_OK)
return (FTP_PROTOCOL_ERROR);
- end = conn->buf + conn->buflen;
- src = conn->buf + 4;
+ end = conn->line + conn->linelen;
+ src = conn->line + 4;
if (src >= end || *src++ != '"')
return (FTP_PROTOCOL_ERROR);
for (q = 0, dst = pwd; src < end && pwdlen--; ++src) {
@@ -417,7 +418,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
ftp_seterr(e);
return (-1);
}
- for (ln = conn->buf + 4; *ln && isspace((unsigned char)*ln); ln++)
+ for (ln = conn->line + 4; *ln && isspace((unsigned char)*ln); ln++)
/* nothing */ ;
for (us->size = 0; *ln && isdigit((unsigned char)*ln); ln++)
us->size = us->size * 10 + *ln - '0';
@@ -435,7 +436,7 @@ ftp_stat(conn_t *conn, const char *file, struct url_stat *us)
ftp_seterr(e);
return (-1);
}
- for (ln = conn->buf + 4; *ln && isspace((unsigned char)*ln); ln++)
+ for (ln = conn->line + 4; *ln && isspace((unsigned char)*ln); ln++)
/* nothing */ ;
switch (strspn(ln, "0123456789")) {
case 14:
@@ -688,7 +689,7 @@ ftp_transfer(conn_t *conn, const char *oper, const char *file,
* Find address and port number. The reply to the PASV command
* is IMHO the one and only weak point in the FTP protocol.
*/
- ln = conn->buf;
+ ln = conn->line;
switch (e) {
case FTP_PASSIVE_MODE:
case FTP_LPASSIVE_MODE:
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 89f731c896dc..cefa8adbb8bd 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -126,8 +126,7 @@
* I/O functions for decoding chunked streams
*/
-struct httpio
-{
+struct httpio {
conn_t *conn; /* connection */
int chunked; /* chunked mode */
char *buf; /* chunk buffer */
@@ -148,17 +147,14 @@ struct httpio
static int
http_new_chunk(struct httpio *io)
{
- unsigned char *p, *eol;
+ char *p;
if (fetch_getln(io->conn) == -1)
return (-1);
- p = (unsigned char *)io->conn->buf;
- if (io->conn->pos < 2 || !isxdigit(*p))
- return (-1);
-
- eol = (unsigned char *)io->conn->buf + io->conn->pos;
- while (p < eol && *p && !isspace(*p) && *p != ';') {
+ for (p = io->conn->line;
+ *p != '\0' && !isspace((unsigned char)*p) && *p != ';';
+ p++) {
io->chunksize <<= 4;
if (*p >= '0' && *p <= '9')
io->chunksize += *p - '0';
@@ -168,7 +164,6 @@ http_new_chunk(struct httpio *io)
io->chunksize += 10 + *p - 'a';
else
return (-1);
- p++;
}
#ifndef NDEBUG
@@ -435,9 +430,9 @@ http_get_reply(conn_t *conn)
* on finding one, but if we do, insist on it being 1.0 or 1.1.
* We don't care about the reason phrase.
*/
- if (strncmp(conn->buf, "HTTP", 4) != 0)
+ if (conn->linelen < 4 || strncmp(conn->line, "HTTP", 4) != 0)
return (HTTP_PROTOCOL_ERROR);
- p = conn->buf + 4;
+ p = conn->line + 4;
if (*p == '/') {
if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1'))
return (HTTP_PROTOCOL_ERROR);
@@ -521,17 +516,17 @@ http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
{
unsigned int i, len;
- if (conn->pos == 0 || conn->buf[0] == '\0')
+ if (conn->linelen == 0)
return (hdr_end);
/* Copy the line to the headerbuf */
- if (hbuf->bufsize < conn->pos + 1) {
- if ((hbuf->buf = realloc(hbuf->buf, conn->pos + 1)) == NULL)
+ if (hbuf->bufsize <= conn->linelen) {
+ if ((hbuf->buf = realloc(hbuf->buf, conn->linelen + 1)) == NULL)
return (hdr_syserror);
- hbuf->bufsize = conn->pos + 1;
+ hbuf->bufsize = conn->linelen + 1;
}
- strcpy(hbuf->buf, conn->buf);
- hbuf->buflen = conn->buflen;
+ memcpy(hbuf->buf, conn->line, conn->linelen + 1);
+ hbuf->buflen = conn->linelen;
/*
* Fetch possible continuation lines. Stop at 1st non-continuation
@@ -545,19 +540,19 @@ http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
* Note: we previously considered a pure whitespace line
* equivalent to an empty one. This was incorrect.
*/
- if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
+ if (conn->line[0] != ' ' && conn->line[0] != '\t')
break;
/* Got a continuation line. Concatenate to previous */
- len = hbuf->buflen + conn->buflen;
- if (hbuf->bufsize < len + 1) {
+ len = hbuf->buflen + conn->linelen;
+ if (hbuf->bufsize <= len) {
len *= 2;
if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL)
return (hdr_syserror);
hbuf->bufsize = len + 1;
}
- strcpy(hbuf->buf + hbuf->buflen, conn->buf);
- hbuf->buflen += conn->buflen;
+ memcpy(hbuf->buf + hbuf->buflen, conn->line, conn->linelen + 1);
+ hbuf->buflen += conn->linelen;
}
/*