From nobody Mon Nov 01 14:32:59 2021 X-Original-To: dev-commits-src-branches@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id 68C26182D476; Mon, 1 Nov 2021 14:33:00 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4Hjb9R3ktSz3s5Z; Mon, 1 Nov 2021 14:32:59 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4AE74130FF; Mon, 1 Nov 2021 14:32:59 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 1A1EWxmu021316; Mon, 1 Nov 2021 14:32:59 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 1A1EWxNW021315; Mon, 1 Nov 2021 14:32:59 GMT (envelope-from git) Date: Mon, 1 Nov 2021 14:32:59 GMT Message-Id: <202111011432.1A1EWxNW021315@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org From: Mark Johnston Subject: git: 82f3e32c3958 - stable/13 - uma: Add KASAN state transitions List-Id: Commits to the stable branches of the FreeBSD src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-branches List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-branches@freebsd.org X-BeenThere: dev-commits-src-branches@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: markj X-Git-Repository: src X-Git-Refname: refs/heads/stable/13 X-Git-Reftype: branch X-Git-Commit: 82f3e32c3958c1fc6e6ab1d7a4440a9be128f205 Auto-Submitted: auto-generated X-ThisMailContainsUnwantedMimeParts: N The branch stable/13 has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=82f3e32c3958c1fc6e6ab1d7a4440a9be128f205 commit 82f3e32c3958c1fc6e6ab1d7a4440a9be128f205 Author: Mark Johnston AuthorDate: 2021-04-13 21:39:50 +0000 Commit: Mark Johnston CommitDate: 2021-11-01 14:02:54 +0000 uma: Add KASAN state transitions - Add a UMA_ZONE_NOKASAN flag to indicate that items from a particular zone should not be sanitized. This is applied implicitly for NOFREE and cache zones. - Add KASAN call backs which get invoked: 1) when a slab is imported into a keg 2) when an item is allocated from a zone 3) when an item is freed to a zone 4) when a slab is freed back to the VM In state transitions 1 and 3, memory is poisoned so that accesses will trigger a panic. In state transitions 2 and 4, memory is marked valid. - Disable trashing if KASAN is enabled. It just adds extra CPU overhead to catch problems that are detected by KASAN. Sponsored by: The FreeBSD Foundation (cherry picked from commit 09c8cb717d214d03e51b3e4f8e9997b9f4e1624d) --- sys/vm/uma.h | 7 ++- sys/vm/uma_core.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 144 insertions(+), 21 deletions(-) diff --git a/sys/vm/uma.h b/sys/vm/uma.h index 6407a4f50fb8..5d473ba909b6 100644 --- a/sys/vm/uma.h +++ b/sys/vm/uma.h @@ -276,6 +276,11 @@ uma_zone_t uma_zcache_create(const char *name, int size, uma_ctor ctor, * * See sys/smr.h for more details. */ +#define UMA_ZONE_NOKASAN 0x80000 /* + * Disable KASAN verification. This is + * implied by NOFREE. Cache zones are + * not verified by default. + */ /* In use by UMA_ZFLAGs: 0xffe00000 */ /* @@ -286,7 +291,7 @@ uma_zone_t uma_zcache_create(const char *name, int size, uma_ctor ctor, #define UMA_ZONE_INHERIT \ (UMA_ZONE_NOTOUCH | UMA_ZONE_MALLOC | UMA_ZONE_NOFREE | \ UMA_ZONE_VM | UMA_ZONE_NOTPAGE | UMA_ZONE_PCPU | \ - UMA_ZONE_FIRSTTOUCH | UMA_ZONE_ROUNDROBIN) + UMA_ZONE_FIRSTTOUCH | UMA_ZONE_ROUNDROBIN | UMA_ZONE_NOKASAN) /* Definitions for align */ #define UMA_ALIGN_PTR (sizeof(void *) - 1) /* Alignment fit for ptr */ diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 5dfcd0c860e9..1398796ee2e7 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -323,6 +324,7 @@ static void bucket_zone_drain(int domain); static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); +static size_t slab_sizeof(int nitems); static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, int align, uint32_t flags); static int zone_import(void *, void **, int, int, int); @@ -537,6 +539,94 @@ bucket_zone_drain(int domain) domain); } +#ifdef KASAN +static void +kasan_mark_item_valid(uma_zone_t zone, void *item) +{ + void *pcpu_item; + size_t sz, rsz; + int i; + + if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) + return; + + sz = zone->uz_size; + rsz = roundup2(sz, KASAN_SHADOW_SCALE); + if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { + kasan_mark(item, sz, rsz, 0); + } else { + pcpu_item = zpcpu_base_to_offset(item); + for (i = 0; i <= mp_maxid; i++) + kasan_mark(zpcpu_get_cpu(pcpu_item, i), sz, rsz, 0); + } +} + +static void +kasan_mark_item_invalid(uma_zone_t zone, void *item) +{ + void *pcpu_item; + size_t sz; + int i; + + if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) + return; + + sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE); + if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { + kasan_mark(item, 0, sz, KASAN_UMA_FREED); + } else { + pcpu_item = zpcpu_base_to_offset(item); + for (i = 0; i <= mp_maxid; i++) + kasan_mark(zpcpu_get_cpu(pcpu_item, i), 0, sz, 0); + } +} + +static void +kasan_mark_slab_valid(uma_keg_t keg, void *mem) +{ + size_t sz; + + if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { + sz = keg->uk_ppera * PAGE_SIZE; + kasan_mark(mem, sz, sz, 0); + } +} + +static void +kasan_mark_slab_invalid(uma_keg_t keg, void *mem) +{ + size_t sz; + + if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { + if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) + sz = keg->uk_ppera * PAGE_SIZE; + else + sz = keg->uk_pgoff; + kasan_mark(mem, 0, sz, KASAN_UMA_FREED); + } +} +#else /* !KASAN */ +static void +kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused) +{ +} + +static void +kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused) +{ +} + +static void +kasan_mark_slab_valid(uma_keg_t keg __unused, void *mem __unused) +{ +} + +static void +kasan_mark_slab_invalid(uma_keg_t keg __unused, void *mem __unused) +{ +} +#endif /* KASAN */ + /* * Acquire the domain lock and record contention. */ @@ -1205,8 +1295,11 @@ bucket_drain(uma_zone_t zone, uma_bucket_t bucket) zone->uz_size, NULL, SKIP_NONE); } if (zone->uz_fini) - for (i = 0; i < bucket->ub_cnt; i++) + for (i = 0; i < bucket->ub_cnt; i++) { + kasan_mark_item_valid(zone, bucket->ub_bucket[i]); zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); + kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); + } zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); if (zone->uz_max_items > 0) zone_free_limit(zone, bucket->ub_cnt); @@ -1437,6 +1530,7 @@ static void keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) { uint8_t *mem; + size_t size; int i; uint8_t flags; @@ -1444,10 +1538,11 @@ keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); mem = slab_data(slab, keg); - flags = slab->us_flags; - i = start; + size = PAGE_SIZE * keg->uk_ppera; + + kasan_mark_slab_valid(keg, mem); if (keg->uk_fini != NULL) { - for (i--; i > -1; i--) + for (i = start - 1; i > -1; i--) #ifdef INVARIANTS /* * trash_fini implies that dtor was trash_dtor. trash_fini @@ -1462,11 +1557,13 @@ keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) #endif keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); } - if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) + flags = slab->us_flags; + if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), NULL, SKIP_NONE); - keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); - uma_total_dec(PAGE_SIZE * keg->uk_ppera); + } + keg->uk_freef(mem, size, flags); + uma_total_dec(size); } static void @@ -1612,7 +1709,6 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, int aflags) { uma_domain_t dom; - uma_alloc allocf; uma_slab_t slab; unsigned long size; uint8_t *mem; @@ -1622,7 +1718,6 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, KASSERT(domain >= 0 && domain < vm_ndomains, ("keg_alloc_slab: domain %d out of range", domain)); - allocf = keg->uk_allocf; slab = NULL; mem = NULL; if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { @@ -1651,7 +1746,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, /* zone is passed for legacy reasons. */ size = keg->uk_ppera * PAGE_SIZE; - mem = allocf(zone, size, domain, &sflags, aflags); + mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); if (mem == NULL) { if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) zone_free_item(slabzone(keg->uk_ipers), @@ -1666,7 +1761,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, /* Point the slab into the allocated memory */ if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) - slab = (uma_slab_t )(mem + keg->uk_pgoff); + slab = (uma_slab_t)(mem + keg->uk_pgoff); else slab_tohashslab(slab)->uhs_data = mem; @@ -1694,6 +1789,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, goto fail; } } + kasan_mark_slab_invalid(keg, mem); KEG_LOCK(keg, domain); CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", @@ -2693,9 +2789,12 @@ zone_ctor(void *mem, int size, void *udata, int flags) STAILQ_INIT(&zdom->uzd_buckets); } -#ifdef INVARIANTS +#if defined(INVARIANTS) && !defined(KASAN) if (arg->uminit == trash_init && arg->fini == trash_fini) zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; +#elif defined(KASAN) + if ((arg->flags & (UMA_ZONE_NOFREE | UMA_ZFLAG_CACHE)) != 0) + arg->flags |= UMA_ZONE_NOKASAN; #endif /* @@ -3118,7 +3217,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, args.dtor = dtor; args.uminit = uminit; args.fini = fini; -#ifdef INVARIANTS +#if defined(INVARIANTS) && !defined(KASAN) /* * Inject procedures which check for memory use after free if we are * allowed to scramble the memory while it is not allocated. This @@ -3275,12 +3374,17 @@ item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, { #ifdef INVARIANTS bool skipdbg; +#endif + + kasan_mark_item_valid(zone, item); +#ifdef INVARIANTS skipdbg = uma_dbg_zskip(zone, item); - if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && + if (!skipdbg && (uz_flags & UMA_ZFLAG_TRASH) != 0 && zone->uz_ctor != trash_ctor) trash_ctor(item, size, udata, flags); #endif + /* Check flags before loading ctor pointer. */ if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0) && __predict_false(zone->uz_ctor != NULL) && @@ -3323,6 +3427,7 @@ item_dtor(uma_zone_t zone, void *item, int size, void *udata, trash_dtor(item, size, udata); #endif } + kasan_mark_item_invalid(zone, item); } #ifdef NUMA @@ -4044,7 +4149,7 @@ static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) { uma_bucket_t bucket; - int maxbucket, cnt; + int error, maxbucket, cnt; CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, zone, domain); @@ -4079,10 +4184,15 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { int i; - for (i = 0; i < bucket->ub_cnt; i++) - if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, - flags) != 0) + for (i = 0; i < bucket->ub_cnt; i++) { + kasan_mark_item_valid(zone, bucket->ub_bucket[i]); + error = zone->uz_init(bucket->ub_bucket[i], + zone->uz_size, flags); + kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); + if (error != 0) break; + } + /* * If we couldn't initialize the whole bucket, put the * rest back onto the freelist. @@ -4149,7 +4259,12 @@ zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) * to be both zone-init'd as well as zone-ctor'd. */ if (zone->uz_init != NULL) { - if (zone->uz_init(item, zone->uz_size, flags) != 0) { + int error; + + kasan_mark_item_valid(zone, item); + error = zone->uz_init(item, zone->uz_size, flags); + kasan_mark_item_invalid(zone, item); + if (error != 0) { zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); goto fail_cnt; } @@ -4631,8 +4746,11 @@ zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) item_dtor(zone, item, zone->uz_size, udata, skip); - if (skip < SKIP_FINI && zone->uz_fini) + if (skip < SKIP_FINI && zone->uz_fini) { + kasan_mark_item_valid(zone, item); zone->uz_fini(item, zone->uz_size); + kasan_mark_item_invalid(zone, item); + } zone->uz_release(zone->uz_arg, &item, 1);