kern/53506: Support gzipped modules. (partial patch)
Peter Edwards
peter.edwards at openet-telecom.com
Thu Jun 19 10:50:12 PDT 2003
>Number: 53506
>Category: kern
>Synopsis: Support gzipped modules. (partial patch)
>Confidential: no
>Severity: non-critical
>Priority: low
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: change-request
>Submitter-Id: current-users
>Arrival-Date: Thu Jun 19 10:50:09 PDT 2003
>Closed-Date:
>Last-Modified:
>Originator: Peter Edwards
>Release: FreeBSD 5.1-CURRENT i386
>Organization:
>Environment:
System: FreeBSD rocklobster 5.1-CURRENT FreeBSD 5.1-CURRENT #5: Thu Jun 19 10:20:09 IST 2003 nfs at archie:/pub/FreeBSD/obj/pub/FreeBSD/current/src/sys/ROCKLOBSTER i386
>Description:
A recent mail to -current suggested that "fixing up imgact_gzip"
would allow compressed kernel modules. No one really said much about
the idea, but I had recently been mucking about with "inflate.c"
and image activators for two unrelated hacks, so knew enough about
ELF and gzip to whip a very naiive implementation of this together.
This patch was originally posted to -current in Message-Id:
<20030611135721.B369E43FBD at mx1.FreeBSD.org>
(Which was in response to Message-Id
<20030606221137.44110.qmail at web13504.mail.yahoo.com>, although this
was not recorded by the mailer I used, which, unfortunately, I have
no control over.)
There are a couple of weaknesses with the patch:
1: It doesn't support pre-loaded files (that would require extra
support in the loader.) This, for example, means ACPI won't work
under it, because it needs to be loaded by the loader, rather than
the kernel.
2: It inflates the content of the gzipped image into memory. This
means that for the brief period of the loading of the module, the
memory usage could be quite high (most of the image will end up
loaded anyway, but that doesn't excuse the waste.). The kernel
linker could be modified to ensure it only ever reads forwards in
a file, which would allow a more optimal implementation.
3: Dependency loads aren't great. Eg, loading snd_maestro3.ko.gz
will probably fail unless you load snd_pcm.ko.gz first, because the
dependency resolver tries to load it as snd_pcm.ko. This is probably
trivial to fix.
I'm posting the patch in its incomplete state to record it for
posterity. If there's any interest in supporting gzipped modules,
I'd be more than happy to fix the above problems. I just don't fancy
working any further on it if it's just going to rot. I've got the
"I could do that" buzz out of the whole thing at this stage.
The patch splits the old "GZIP" option (which wasn't fully there,
anyway) into "INFLATE" and "COMPAT_GZAOUT". In order to allow a
config to pull in inflate.c without pulling in the a.out gzipped
executable loader. It also adds a "GZLOADER" option, to allow gzipped
klds to be loaded.
>How-To-Repeat:
Before patch.
> # gzip /boot/kernel/procfs.ko
> # kldload procfs.ko.gz
> kldload: can't load procfs.ko.gz: Exec format error
> #
After patch.
> # kldload procfs.ko.gz
> # mount /proc
> #
:-)
>Fix:
Apply this patch, and add INFLATE and GZLOADER options to your
kernel.
Index: conf/files
===================================================================
RCS file: /pub/FreeBSD/development/FreeBSD-CVS/src/sys/conf/files,v
retrieving revision 1.795
diff -u -r1.795 files
--- conf/files 18 Jun 2003 09:29:28 -0000 1.795
+++ conf/files 18 Jun 2003 14:43:58 -0000
@@ -1019,7 +1019,7 @@
isofs/cd9660/cd9660_vnops.c optional cd9660
kern/imgact_elf.c standard
kern/imgact_shell.c standard
-kern/inflate.c optional gzip
+kern/inflate.c optional inflate
kern/init_main.c standard
kern/init_sysent.c standard
kern/kern_acct.c standard
Index: conf/files.i386
===================================================================
RCS file: /pub/FreeBSD/development/FreeBSD-CVS/src/sys/conf/files.i386,v
retrieving revision 1.445
diff -u -r1.445 files.i386
--- conf/files.i386 31 May 2003 17:06:19 -0000 1.445
+++ conf/files.i386 11 Jun 2003 12:51:27 -0000
@@ -407,7 +407,7 @@
isa/syscons_isa.c optional sc
isa/vga_isa.c optional vga
kern/imgact_aout.c optional compat_aout
-kern/imgact_gzip.c optional gzip
+kern/imgact_gzip.c optional compat_gzaout
libkern/divdi3.c standard
libkern/moddi3.c standard
libkern/qdivrem.c standard
Index: conf/options
===================================================================
RCS file: /pub/FreeBSD/development/FreeBSD-CVS/src/sys/conf/options,v
retrieving revision 1.396
diff -u -r1.396 options
--- conf/options 18 Jun 2003 15:25:01 -0000 1.396
+++ conf/options 19 Jun 2003 17:20:55 -0000
@@ -638,3 +638,8 @@
KBD_MAXWAIT opt_kbd.h
KBD_RESETDELAY opt_kbd.h
KBDIO_DEBUG opt_kbd.h
+
+# options for gzip/"inflate" related functionality
+INFLATE opt_inflate.h
+COMPAT_GZAOUT opt_gzaout.h
+GZLOADER opt_gzloader.h
Index: kern/link_elf.c
===================================================================
RCS file: /pub/FreeBSD/development/FreeBSD-CVS/src/sys/kern/link_elf.c,v
retrieving revision 1.74
diff -u -r1.74 link_elf.c
--- kern/link_elf.c 11 Jun 2003 00:56:57 -0000 1.74
+++ kern/link_elf.c 19 Jun 2003 17:52:46 -0000
@@ -29,6 +29,7 @@
#include "opt_ddb.h"
#include "opt_mac.h"
+#include "opt_gzloader.h"
#include <sys/param.h>
#include <sys/systm.h>
@@ -43,6 +44,10 @@
#include <sys/vnode.h>
#include <sys/linker.h>
+#ifdef GZLOADER
+#include <sys/inflate.h>
+#endif
+
#include <machine/elf.h>
#ifdef GPROF
#include <machine/profile.h>
@@ -99,6 +104,40 @@
#endif
} *elf_file_t;
+struct vnreader {
+ struct vnode *vnodep;
+ struct thread *td;
+};
+
+#ifdef GZLOADER
+/* Allow modules up to 1MB (uncompressed) */
+#define MAXGZPAGES (1024 * 1024 / PAGE_SIZE)
+
+struct gzreader {
+ /* reading from gzipped file. */
+ int error;
+ struct vnode *vn;
+ unsigned char *inPage;
+ struct thread *td;
+ int inPageSize;
+ int inPageOffset;
+ off_t inFileOffset;
+ int inPageCount;
+
+ /* gzip context */
+ struct inflate inflator;
+
+ /*
+ * Writing to inflated output: number of inflated bytes =
+ * (outPageCount - 1) * PAGE_SIZE + outPageOffset
+ */
+
+ int outPageCount; /* Number of valid pages in "pages" array below. */
+ int outPageOffset; /* Size of last page. */
+ unsigned char *pages[MAXGZPAGES]; /* The inflated data. */
+};
+#endif
+
static int link_elf_link_common_finish(linker_file_t);
static int link_elf_link_preload(linker_class_t cls,
const char*, linker_file_t*);
@@ -118,6 +157,20 @@
int (*)(const char *, void *),
void *);
static void link_elf_reloc_local(linker_file_t);
+static int link_elf_load_object(void *,
+ int (*)(void *, unsigned char *, int, off_t, int *),
+ const char *filename, linker_file_t* result);
+static int vnreadfunc(void *, unsigned char *, int, off_t, int *);
+#ifdef GZLOADER
+static int link_gz_link_preload_finish(linker_file_t);
+static int link_gz_load_file(linker_class_t, const char*, linker_file_t*);
+static int link_gz_link_preload(linker_class_t cls,
+ const char*, linker_file_t*);
+static void release_gzreader(struct gzreader *zr);
+static int gzreadfunc(void *, unsigned char *, int, off_t, int *);
+static int gzin(void *vp);
+static int gzout(void *vp, unsigned char *data, unsigned long size);
+#endif
static kobj_method_t link_elf_methods[] = {
KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol),
@@ -141,6 +194,37 @@
link_elf_methods, sizeof(struct elf_file)
};
+#ifdef GZLOADER
+/*
+ * The gzip loader is almost the same as the ELF loader, only when it comes to
+ * reading the file from disk. Symbol lookups, unloading, etc, are all thesame
+ * as for ELF. For the moment, preloaded files aren't supported: some work in
+ * the boot loader is required.
+ */
+
+static kobj_method_t link_gz_methods[] = {
+ KOBJMETHOD(linker_lookup_symbol, link_elf_lookup_symbol),
+ KOBJMETHOD(linker_symbol_values, link_elf_symbol_values),
+ KOBJMETHOD(linker_search_symbol, link_elf_search_symbol),
+ KOBJMETHOD(linker_unload, link_elf_unload_file),
+ KOBJMETHOD(linker_load_file, link_gz_load_file),
+ KOBJMETHOD(linker_link_preload, link_gz_link_preload),
+ KOBJMETHOD(linker_link_preload_finish, link_gz_link_preload_finish),
+ KOBJMETHOD(linker_lookup_set, link_elf_lookup_set),
+ KOBJMETHOD(linker_each_function_name, link_elf_each_function_name),
+ { 0, 0 }
+};
+
+static struct linker_class link_gz_class = {
+#if ELF_TARG_CLASS == ELFCLASS32
+ "gzelf32",
+#else
+ "gzelf64",
+#endif
+ link_gz_methods, sizeof(struct elf_file)
+};
+#endif
+
static int parse_dynamic(elf_file_t ef);
static int relocate_file(elf_file_t ef);
static int link_elf_preload_parse_symbols(elf_file_t ef);
@@ -256,6 +340,9 @@
char *modname;
linker_add_class(&link_elf_class);
+#ifdef GZLOADER
+ linker_add_class(&link_gz_class);
+#endif
dp = (Elf_Dyn*) &_DYNAMIC;
modname = NULL;
@@ -525,11 +612,10 @@
}
static int
-link_elf_load_file(linker_class_t cls, const char* filename,
- linker_file_t* result)
+link_elf_load_object(void *readCookie,
+ int (*readfunc)(void *, unsigned char *data, int, off_t, int *),
+ const char *filename, linker_file_t* result)
{
- struct nameidata nd;
- struct thread* td = curthread; /* XXX */
Elf_Ehdr *hdr;
caddr_t firstpage;
int nbytes, i;
@@ -545,7 +631,7 @@
Elf_Addr base_vaddr;
Elf_Addr base_vlimit;
int error = 0;
- int resid, flags;
+ int resid;
elf_file_t ef;
linker_file_t lf;
Elf_Shdr *shdr;
@@ -559,20 +645,6 @@
shdr = NULL;
lf = NULL;
- NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
- flags = FREAD;
- error = vn_open(&nd, &flags, 0);
- if (error)
- return error;
- NDFREE(&nd, NDF_ONLY_PNBUF);
-#ifdef MAC
- error = mac_check_kld_load(curthread->td_ucred, nd.ni_vp);
- if (error) {
- firstpage = NULL;
- goto out;
- }
-#endif
-
/*
* Read the elf header from the file.
*/
@@ -582,9 +654,7 @@
goto out;
}
hdr = (Elf_Ehdr *)firstpage;
- error = vn_rdwr(UIO_READ, nd.ni_vp, firstpage, PAGE_SIZE, 0,
- UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
- &resid, td);
+ error = readfunc(readCookie, firstpage, PAGE_SIZE, 0, &resid);
nbytes = PAGE_SIZE - resid;
if (error)
goto out;
@@ -728,10 +798,8 @@
*/
for (i = 0; i < 2; i++) {
caddr_t segbase = mapbase + segs[i]->p_vaddr - base_vaddr;
- error = vn_rdwr(UIO_READ, nd.ni_vp,
- segbase, segs[i]->p_filesz, segs[i]->p_offset,
- UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
- &resid, td);
+ error = readfunc(readCookie,
+ segbase, segs[i]->p_filesz, segs[i]->p_offset, &resid);
if (error) {
goto out;
}
@@ -791,10 +859,7 @@
error = ENOMEM;
goto out;
}
- error = vn_rdwr(UIO_READ, nd.ni_vp,
- (caddr_t)shdr, nbytes, hdr->e_shoff,
- UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
- &resid, td);
+ error = readfunc(readCookie, (caddr_t)shdr, nbytes, hdr->e_shoff, &resid);
if (error)
goto out;
symtabindex = -1;
@@ -817,16 +882,12 @@
error = ENOMEM;
goto out;
}
- error = vn_rdwr(UIO_READ, nd.ni_vp,
- ef->symbase, symcnt, shdr[symtabindex].sh_offset,
- UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
- &resid, td);
+ error = readfunc(readCookie, ef->symbase, symcnt,
+ shdr[symtabindex].sh_offset, &resid);
if (error)
goto out;
- error = vn_rdwr(UIO_READ, nd.ni_vp,
- ef->strbase, strcnt, shdr[symstrindex].sh_offset,
- UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED,
- &resid, td);
+ error = readfunc(readCookie, ef->strbase, strcnt,
+ shdr[symstrindex].sh_offset, &resid);
if (error)
goto out;
@@ -850,6 +911,49 @@
free(shdr, M_LINKER);
if (firstpage)
free(firstpage, M_LINKER);
+
+ return error;
+}
+
+static int
+vnreadfunc(void *readCookie, unsigned char *data, int len, off_t offset, int *residp)
+{
+ struct vnreader *vnr = (struct vnreader *)readCookie;
+
+ return vn_rdwr(UIO_READ, vnr->vnodep, data, len, offset,
+ UIO_SYSSPACE, IO_NODELOCKED, vnr->td->td_ucred, NOCRED,
+ residp, vnr->td);
+}
+
+static int
+link_elf_load_file(linker_class_t cls, const char* filename,
+ linker_file_t* result)
+{
+ struct nameidata nd;
+ struct vnreader vr;
+ int error, flags;
+ struct thread *td = curthread;
+
+ NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
+ flags = FREAD;
+ error = vn_open(&nd, &flags, 0);
+ if (error)
+ return error;
+ NDFREE(&nd, NDF_ONLY_PNBUF);
+#ifdef MAC
+ error = mac_check_kld_load(td->td_ucred, nd.ni_vp);
+ if (error) {
+ firstpage = NULL;
+ goto out;
+ }
+#endif
+ vr.vnodep = nd.ni_vp;
+ vr.td = td;
+ error = link_elf_load_object(&vr, vnreadfunc, filename, result);
+
+#ifdef MAC /* Avoid unused label warning. */
+out:
+#endif
VOP_UNLOCK(nd.ni_vp, 0, td);
vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
@@ -1311,3 +1415,210 @@
}
}
}
+
+#ifdef GZLOADER
+static int
+gzreadfunc(void *readCookie, unsigned char *data, int len, off_t offset, int *residp)
+{
+ struct gzreader *zr = (struct gzreader *)readCookie;
+ int pageId, pageOff, pageSize, copySize;
+
+ while (len) {
+ pageId = offset / PAGE_SIZE;
+ /* Cannot read beyond last page. */
+ if (pageId < 0 || pageId >= zr->outPageCount)
+ break;
+ pageOff = offset % PAGE_SIZE;
+
+ if (pageId == zr->outPageCount - 1) {
+ pageSize = zr->outPageOffset;
+ /* Last page may not be a full page size */
+ if (pageOff >= zr->outPageOffset)
+ break;
+ } else {
+ pageSize = PAGE_SIZE;
+ }
+ copySize = MIN(pageSize - pageOff, len);
+ bcopy(zr->pages[pageId] + pageOff, data, copySize);
+ len -= copySize;
+ offset += copySize;
+ data += copySize;
+ }
+ if (residp)
+ *residp = len;
+ return 0;
+}
+
+static int
+link_gz_load_file(linker_class_t cls, const char* filename,
+ linker_file_t* result)
+{
+ int resid;
+ const unsigned char *p;
+ struct nameidata nd;
+ struct gzreader *zr = 0;
+ int error, flags;
+ struct thread *td = curthread;
+
+ NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
+ flags = FREAD;
+ error = vn_open(&nd, &flags, 0);
+ if (error)
+ return error;
+ NDFREE(&nd, NDF_ONLY_PNBUF);
+#ifdef MAC
+ error = mac_check_kld_load(td->td_ucred, nd.ni_vp);
+ if (error)
+ goto out;
+#endif
+
+ zr = malloc(sizeof *zr, M_LINKER, M_WAITOK);
+ if (zr == 0)
+ goto out;
+
+ bzero(zr, sizeof *zr);
+ zr->td = td;
+ zr->vn = nd.ni_vp;
+ zr->inflator.gz_private = zr;
+ zr->inflator.gz_input = gzin;
+ zr->inflator.gz_output = gzout;
+
+ /*
+ * XXX: Would it be better to map the VM pages of the vnode, rather than
+ * using malloc/vn_rdwr()?
+ */
+ zr->inPage = malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
+ if (zr->inPage == 0)
+ goto out;
+
+ error = vn_rdwr(UIO_READ, nd.ni_vp, zr->inPage, PAGE_SIZE, 0,
+ UIO_SYSSPACE, IO_NODELOCKED, td->td_ucred, NOCRED, &resid, td);
+ if (error)
+ goto out;
+ zr->inFileOffset = zr->inPageSize = PAGE_SIZE - resid;
+
+ p = zr->inPage;
+
+ /* Magic from kern/imgact_gzip.c */
+ if (p[0] != 0x1f || p[1] != 0x8b || p[2] != 0x08 /* gzip magic */
+ || p[9] != 0x03 /* Compression type */
+ || p[3] & ~0x18 /* Extra fields: support filename and comment */
+ ) {
+ error = ENOEXEC;
+ goto out;
+ }
+
+ zr->inPageOffset = 10;
+
+ /* Skip filename in gzip file if present */
+ if (p[3] & 0x8) {
+ while (p[zr->inPageOffset++]) {
+ if (zr->inPageOffset == zr->inPageSize) {
+ error = ENOEXEC;
+ goto out;
+ }
+ }
+ }
+
+ /* Skip comment in gzip file if present */
+ if (p[3] & 0x10) {
+ while (p[zr->inPageOffset++]) {
+ if (zr->inPageOffset == zr->inPageSize) {
+ error = ENOEXEC;
+ goto out;
+ }
+ }
+ }
+
+ error = inflate(&zr->inflator); /* inflate the entire file */
+
+ if (error)
+ goto out;
+ if (zr->error) {
+ error = zr->error;
+ goto out;
+ }
+ error = link_elf_load_object(zr, gzreadfunc, filename, result);
+
+out:
+ VOP_UNLOCK(nd.ni_vp, 0, td);
+ vn_close(nd.ni_vp, FREAD, td->td_ucred, td);
+ if (zr)
+ release_gzreader(zr);
+
+ return error;
+}
+
+static void
+release_gzreader(struct gzreader *zr)
+{
+ if (zr->inPage)
+ free(zr->inPage, M_LINKER);
+ while (zr->outPageCount--)
+ free(zr->pages[zr->outPageCount], M_LINKER);
+ free(zr, M_LINKER);
+}
+
+static int
+link_gz_link_preload(linker_class_t cls,
+ const char* filename, linker_file_t *result)
+{
+ return ENOENT;
+}
+
+static int
+link_gz_link_preload_finish(linker_file_t l)
+{
+ return ENOENT;
+}
+
+static int
+gzin(void *vp)
+{
+ struct gzreader *zr = (struct gzreader *) vp;
+ if (zr->inPageSize == zr->inPageOffset) {
+ int resid;
+ /* We have consumed the entire page. */
+ zr->error = vn_rdwr(UIO_READ, zr->vn, zr->inPage, PAGE_SIZE,
+ zr->inFileOffset, UIO_SYSSPACE, IO_NODELOCKED,
+ zr->td->td_ucred, NOCRED, &resid, zr->td);
+ if (zr->error)
+ return GZ_EOF;
+ if (resid == PAGE_SIZE)
+ return GZ_EOF;
+ zr->inFileOffset += PAGE_SIZE - resid;
+ zr->inPageOffset = 0;
+ }
+ return zr->inPage[zr->inPageOffset++];
+}
+
+static int
+gzout(void *vp, unsigned char *data, unsigned long size)
+{
+ unsigned char *page;
+ int space;
+
+ struct gzreader *zr = (struct gzreader *) vp;
+ while (size) {
+ if (zr->outPageCount == 0 || zr->outPageOffset == PAGE_SIZE) {
+ /* We need a new page to generate output into. */
+ if (zr->outPageCount == MAXGZPAGES)
+ return ENOEXEC;
+ zr->pages[zr->outPageCount] = malloc(PAGE_SIZE, M_LINKER, M_WAITOK);
+ if (zr->pages[zr->outPageCount] == 0)
+ return ENOEXEC;
+ zr->outPageOffset = 0;
+ zr->outPageCount++; /* That's one more to free. */
+ }
+
+ /* Copy inflated data into our image */
+ page = zr->pages[zr->outPageCount - 1];
+ space = MIN(PAGE_SIZE - zr->outPageOffset, size);
+ bcopy(data, page + zr->outPageOffset, space);
+ data += space;
+ zr->outPageOffset += space;
+ size -= space;
+ }
+ return 0;
+}
+#endif
>Release-Note:
>Audit-Trail:
>Unformatted:
More information about the freebsd-bugs
mailing list