svn commit: r294776 - in stable/9: lib/libfetch usr.bin/fetch

Dag-Erling Smørgrav des at FreeBSD.org
Tue Jan 26 07:44:28 UTC 2016


Author: des
Date: Tue Jan 26 07:44:26 2016
New Revision: 294776
URL: https://svnweb.freebsd.org/changeset/base/294776

Log:
  MFH (r261233): cleanup
  MFH (r261234): increase buffer size
  MFH (r280630): remove all traces of SSLv2 support
  MFH (r285141): remove unused variable
  MFH (r288217): correctly check return value from getaddrinfo(3)
  MFH (r289419): fix bugs in HTTPS tunnelling
  MFH (r289420): use fopen()'s "e" mode instead of fcntl for close-on-exec
  MFH (r291453, r291461): use .netrc for http servers and proxies
  MFH (r292330, r292332): reset bufpos to 0 after refilling in chunked mode
  
  PR:		194483 199801 193740 204771

Modified:
  stable/9/lib/libfetch/common.c
  stable/9/lib/libfetch/fetch.3
  stable/9/lib/libfetch/file.c
  stable/9/lib/libfetch/http.c
  stable/9/usr.bin/fetch/fetch.1
  stable/9/usr.bin/fetch/fetch.c
Directory Properties:
  stable/9/   (props changed)
  stable/9/lib/   (props changed)
  stable/9/lib/libfetch/   (props changed)
  stable/9/usr.bin/   (props changed)
  stable/9/usr.bin/fetch/   (props changed)

Modified: stable/9/lib/libfetch/common.c
==============================================================================
--- stable/9/lib/libfetch/common.c	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/lib/libfetch/common.c	Tue Jan 26 07:44:26 2016	(r294776)
@@ -495,7 +495,8 @@ fetch_ssl_get_numeric_addrinfo(const cha
 	hints.ai_protocol = 0;
 	hints.ai_flags = AI_NUMERICHOST;
 	/* port is not relevant for this purpose */
-	getaddrinfo(host, "443", &hints, &res);
+	if (getaddrinfo(host, "443", &hints, &res) != 0)
+		res = NULL;
 	free(host);
 	return res;
 }
@@ -672,9 +673,7 @@ fetch_ssl_setup_transport_layer(SSL_CTX 
 {
 	long ssl_ctx_options;
 
-	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_TICKET;
-	if (getenv("SSL_ALLOW_SSL2") == NULL)
-		ssl_ctx_options |= SSL_OP_NO_SSLv2;
+	ssl_ctx_options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_TICKET;
 	if (getenv("SSL_NO_SSL3") != NULL)
 		ssl_ctx_options |= SSL_OP_NO_SSLv3;
 	if (getenv("SSL_NO_TLS1") != NULL)

Modified: stable/9/lib/libfetch/fetch.3
==============================================================================
--- stable/9/lib/libfetch/fetch.3	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/lib/libfetch/fetch.3	Tue Jan 26 07:44:26 2016	(r294776)
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 30, 2013
+.Dd November 29, 2015
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -440,13 +440,11 @@ By default
 .Nm libfetch
 allows SSLv3 and TLSv1 when negotiating the connecting with the remote
 peer.
-You can change this behavior by setting the environment variable
-.Ev SSL_ALLOW_SSL2
-to allow SSLv2 (not recommended) and
+You can change this behavior by setting the
 .Ev SSL_NO_SSL3
 or
 .Ev SSL_NO_TLS1
-to disable the respective methods.
+environment variables to disable the respective methods.
 .Sh AUTHENTICATION
 Apart from setting the appropriate environment variables and
 specifying the user name and password in the URL or the
@@ -631,11 +629,11 @@ If defined but empty, no User-Agent head
 .It Ev NETRC
 Specifies a file to use instead of
 .Pa ~/.netrc
-to look up login names and passwords for FTP sites.
+to look up login names and passwords for FTP and HTTP sites as well as
+HTTP proxies.
 See
 .Xr ftp 1
 for a description of the file format.
-This feature is experimental.
 .It Ev NO_PROXY
 Either a single asterisk, which disables the use of proxies
 altogether, or a comma- or whitespace-separated list of hosts for
@@ -644,8 +642,6 @@ which proxies should not be used.
 Same as
 .Ev NO_PROXY ,
 for compatibility.
-.It Ev SSL_ALLOW_SSL2
-Allow SSL version 2 when negotiating the connection (not recommended).
 .It Ev SSL_CA_CERT_FILE
 CA certificate bundle containing trusted CA certificates.
 Default value:

Modified: stable/9/lib/libfetch/file.c
==============================================================================
--- stable/9/lib/libfetch/file.c	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/lib/libfetch/file.c	Tue Jan 26 07:44:26 2016	(r294776)
@@ -48,7 +48,7 @@ fetchXGetFile(struct url *u, struct url_
 	if (us && fetchStatFile(u, us, flags) == -1)
 		return (NULL);
 
-	f = fopen(u->doc, "r");
+	f = fopen(u->doc, "re");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -61,7 +61,6 @@ fetchXGetFile(struct url *u, struct url_
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 
@@ -77,9 +76,9 @@ fetchPutFile(struct url *u, const char *
 	FILE *f;
 
 	if (CHECK_FLAG('a'))
-		f = fopen(u->doc, "a");
+		f = fopen(u->doc, "ae");
 	else
-		f = fopen(u->doc, "w+");
+		f = fopen(u->doc, "w+e");
 
 	if (f == NULL) {
 		fetch_syserr();
@@ -92,7 +91,6 @@ fetchPutFile(struct url *u, const char *
 		return (NULL);
 	}
 
-	fcntl(fileno(f), F_SETFD, FD_CLOEXEC);
 	return (f);
 }
 

Modified: stable/9/lib/libfetch/http.c
==============================================================================
--- stable/9/lib/libfetch/http.c	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/lib/libfetch/http.c	Tue Jan 26 07:44:26 2016	(r294776)
@@ -130,8 +130,8 @@ struct httpio
 	int		 chunked;	/* chunked mode */
 	char		*buf;		/* chunk buffer */
 	size_t		 bufsize;	/* size of chunk buffer */
-	ssize_t		 buflen;	/* amount of data currently in buffer */
-	int		 bufpos;	/* current read offset in buffer */
+	size_t		 buflen;	/* amount of data currently in buffer */
+	size_t		 bufpos;	/* current read offset in buffer */
 	int		 eof;		/* end-of-file flag */
 	int		 error;		/* error flag */
 	size_t		 chunksize;	/* remaining size of current chunk */
@@ -215,6 +215,7 @@ http_fillbuf(struct httpio *io, size_t l
 	if (io->eof)
 		return (0);
 
+	/* not chunked: just fetch the requested amount */
 	if (io->chunked == 0) {
 		if (http_growbuf(io, len) == -1)
 			return (-1);
@@ -227,6 +228,7 @@ http_fillbuf(struct httpio *io, size_t l
 		return (io->buflen);
 	}
 
+	/* chunked, but we ran out: get the next chunk header */
 	if (io->chunksize == 0) {
 		switch (http_new_chunk(io)) {
 		case -1:
@@ -238,6 +240,7 @@ http_fillbuf(struct httpio *io, size_t l
 		}
 	}
 
+	/* fetch the requested amount, but no more than the current chunk */
 	if (len > io->chunksize)
 		len = io->chunksize;
 	if (http_growbuf(io, len) == -1)
@@ -246,8 +249,9 @@ http_fillbuf(struct httpio *io, size_t l
 		io->error = errno;
 		return (-1);
 	}
+	io->bufpos = 0;
 	io->buflen = nbytes;
-	io->chunksize -= io->buflen;
+	io->chunksize -= nbytes;
 
 	if (io->chunksize == 0) {
 		if (fetch_read(io->conn, &ch, 1) != 1 || ch != '\r' ||
@@ -255,8 +259,6 @@ http_fillbuf(struct httpio *io, size_t l
 			return (-1);
 	}
 
-	io->bufpos = 0;
-
 	return (io->buflen);
 }
 
@@ -1330,7 +1332,6 @@ static int
 http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
 	       http_auth_params_t *parms, struct url *url)
 {
-	http_auth_challenge_t *basic = NULL;
 	http_auth_challenge_t *digest = NULL;
 	int i;
 
@@ -1340,10 +1341,8 @@ http_authorize(conn_t *conn, const char 
 		return (-1);
 	}
 
-	/* Look for a Digest and a Basic challenge */
+	/* Look for a Digest */
 	for (i = 0; i < cs->count; i++) {
-		if (cs->challenges[i]->scheme == HTTPAS_BASIC)
-			basic = cs->challenges[i];
 		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
 			digest = cs->challenges[i];
 	}
@@ -1379,8 +1378,12 @@ http_connect(struct url *URL, struct url
 {
 	struct url *curl;
 	conn_t *conn;
+	hdr_t h;
+	http_headerbuf_t headerbuf;
+	const char *p;
 	int verbose;
 	int af, val;
+	int serrno;
 
 #ifdef INET6
 	af = AF_UNSPEC;
@@ -1401,6 +1404,7 @@ http_connect(struct url *URL, struct url
 	if ((conn = fetch_connect(curl->host, curl->port, af, verbose)) == NULL)
 		/* fetch_connect() has already set an error code */
 		return (NULL);
+	init_http_headerbuf(&headerbuf);
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && purl) {
 		http_cmd(conn, "CONNECT %s:%d HTTP/1.1",
 		    URL->host, URL->port);
@@ -1408,10 +1412,26 @@ http_connect(struct url *URL, struct url
 		    URL->host, URL->port);
 		http_cmd(conn, "");
 		if (http_get_reply(conn) != HTTP_OK) {
-			fetch_close(conn);
-			return (NULL);
+			http_seterr(conn->err);
+			goto ouch;
+		}
+		/* Read and discard the rest of the proxy response */
+		if (fetch_getln(conn) < 0) {
+			fetch_syserr();
+			goto ouch;
 		}
-		http_get_reply(conn);
+		do {
+			switch ((h = http_next_header(conn, &headerbuf, &p))) {
+			case hdr_syserror:
+				fetch_syserr();
+				goto ouch;
+			case hdr_error:
+				http_seterr(HTTP_PROTOCOL_ERROR);
+				goto ouch;
+			default:
+				/* ignore */ ;
+			}
+		} while (h < hdr_end);
 	}
 	if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 &&
 	    fetch_ssl(conn, URL, verbose) == -1) {
@@ -1419,13 +1439,20 @@ http_connect(struct url *URL, struct url
 		/* grrr */
 		errno = EAUTH;
 		fetch_syserr();
-		return (NULL);
+		goto ouch;
 	}
 
 	val = 1;
 	setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val));
 
+	clean_http_headerbuf(&headerbuf);
 	return (conn);
+ouch:
+	serrno = errno;
+	clean_http_headerbuf(&headerbuf);
+	fetch_close(conn);
+	errno = serrno;
+	return (NULL);
 }
 
 static struct url *
@@ -1633,6 +1660,9 @@ http_request_body(struct url *URL, const
 					http_seterr(HTTP_NEED_PROXY_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(purl) == 0) {
+				aparams.user = strdup(purl->user);
+				aparams.password = strdup(purl->pwd);
 			}
 			http_authorize(conn, "Proxy-Authorization",
 				       &proxy_challenges, &aparams, url);
@@ -1660,6 +1690,9 @@ http_request_body(struct url *URL, const
 					http_seterr(HTTP_NEED_AUTH);
 					goto ouch;
 				}
+			} else if (fetch_netrc_auth(url) == 0) {
+				aparams.user = strdup(url->user);
+				aparams.password = strdup(url->pwd);
 			} else if (fetchAuthMethod &&
 				   fetchAuthMethod(url) == 0) {
 				aparams.user = strdup(url->user);

Modified: stable/9/usr.bin/fetch/fetch.1
==============================================================================
--- stable/9/usr.bin/fetch/fetch.1	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/usr.bin/fetch/fetch.1	Tue Jan 26 07:44:26 2016	(r294776)
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 28, 2014
+.Dd March 25, 2015
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -39,7 +39,6 @@
 .Sh SYNOPSIS
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
-.Op Fl -allow-sslv2
 .Op Fl B Ar bytes
 .Op Fl -bind-address= Ns Ar host
 .Op Fl -ca-cert= Ns Ar file
@@ -113,9 +112,6 @@ Some broken Web sites will return a redi
 error when the requested object does not exist.
 .It Fl a , -retry
 Automatically retry the transfer upon soft failures.
-.It Fl -allow-sslv2
-[SSL]
-Allow SSL version 2 when negotiating the connection.
 .It Fl B Ar bytes , Fl -buffer-size= Ns Ar bytes
 Specify the read buffer size in bytes.
 The default is 16,384 bytes.
@@ -350,7 +346,6 @@ for a description of additional environm
 .Ev NETRC ,
 .Ev NO_PROXY ,
 .Ev no_proxy ,
-.Ev SSL_ALLOW_SSL2 ,
 .Ev SSL_CA_CERT_FILE ,
 .Ev SSL_CA_CERT_PATH ,
 .Ev SSL_CLIENT_CERT_FILE ,

Modified: stable/9/usr.bin/fetch/fetch.c
==============================================================================
--- stable/9/usr.bin/fetch/fetch.c	Tue Jan 26 07:22:22 2016	(r294775)
+++ stable/9/usr.bin/fetch/fetch.c	Tue Jan 26 07:44:26 2016	(r294776)
@@ -102,7 +102,6 @@ enum options
 	OPTION_HTTP_REFERER,
 	OPTION_HTTP_USER_AGENT,
 	OPTION_NO_PROXY,
-	OPTION_SSL_ALLOW_SSL2,
 	OPTION_SSL_CA_CERT_FILE,
 	OPTION_SSL_CA_CERT_PATH,
 	OPTION_SSL_CLIENT_CERT_FILE,
@@ -154,7 +153,6 @@ static struct option longopts[] =
 	{ "referer", required_argument, NULL, OPTION_HTTP_REFERER },
 	{ "user-agent", required_argument, NULL, OPTION_HTTP_USER_AGENT },
 	{ "no-proxy", required_argument, NULL, OPTION_NO_PROXY },
-	{ "allow-sslv2", no_argument, NULL, OPTION_SSL_ALLOW_SSL2 },
 	{ "ca-cert", required_argument, NULL, OPTION_SSL_CA_CERT_FILE },
 	{ "ca-path", required_argument, NULL, OPTION_SSL_CA_CERT_PATH },
 	{ "cert", required_argument, NULL, OPTION_SSL_CLIENT_CERT_FILE },
@@ -845,17 +843,17 @@ static void
 usage(void)
 {
 	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
-"usage: fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
-"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
-"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
-"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
-"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
+"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
+"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
+"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
+"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
 "       [--user-agent=agent-string] [-w seconds] URL ...",
-"       fetch [-146AadFlMmnPpqRrsUv] [--allow-sslv2] [-B bytes]",
-"       [--bind-address=host] [--ca-cert=file] [--ca-path=dir] [--cert=file]",
-"       [--crl=file] [-i file] [--key=file] [-N file] [--no-passive]",
-"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
-"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
+"       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
+"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
+"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
+"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
 "       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
 }
 
@@ -1004,9 +1002,6 @@ main(int argc, char *argv[])
 		case OPTION_NO_PROXY:
 			setenv("NO_PROXY", optarg, 1);
 			break;
-		case OPTION_SSL_ALLOW_SSL2:
-			setenv("SSL_ALLOW_SSL2", "", 1);
-			break;
 		case OPTION_SSL_CA_CERT_FILE:
 			setenv("SSL_CA_CERT_FILE", optarg, 1);
 			break;


More information about the svn-src-stable-9 mailing list