svn commit: r310650 - in head/sys/mips: include mips
Adrian Chadd
adrian.chadd at gmail.com
Wed Dec 28 05:50:36 UTC 2016
hiya,
so I dug into the mips24k definition of this. It says this:
"
3.4.3 Uncached accelerated writes
The 24K core permits memory regions to be marked as “uncached
accelerated”. This type of region is useful to hard-
ware which is “write only” - perhaps video frame buffers, or some
other hardware stream. Sequential word stores in
such regions are gathered into cache-line-sized chunks, before being
written with a single burst cycle on the CPU
interface.
Such regions are uncached for read, and partial-word or
out-of-sequence writes have “unpredictable” effects - don’t
do them. The burst write is normally performed when software writes to
the last location in the memory block or does
an uncached-accelerated write to some other block; but it can also be
triggered by a
sync instruction, a pref nudge, a matching load or any exception. If
the block is not completely written by the time it’s pushed out, it
will be written using a series of doubleword or smaller write cycles
over the 24K core’s 64-bit memory interface.
"
So, question is - is our write combining page attribute in the VM
suitable for this? Is it defined as "only do full sequential word
writes"? Or do we risk having some other platform use it in a less
"don't do this" way and then MIPS is the one at fault again? :)
-adrian
On 27 December 2016 at 18:55, Alexander Kabaev <kan at freebsd.org> wrote:
> Author: kan
> Date: Wed Dec 28 02:55:26 2016
> New Revision: 310650
> URL: https://svnweb.freebsd.org/changeset/base/310650
>
> Log:
> Implement pmap_change_attr and related APIs on MIPS
>
> On platforms that have uncached-accelerate cache attribute, map it
> to VM_MEMATTR_WRITE_COMBINING. Otherwise, leave write comining
> undefined.
>
> Reviewed by: adrian, jhb (glance)
> Differential Revision: https://reviews.freebsd.org/D8894
>
> Modified:
> head/sys/mips/include/cpuregs.h
> head/sys/mips/include/pmap.h
> head/sys/mips/include/pte.h
> head/sys/mips/include/vm.h
> head/sys/mips/mips/pmap.c
>
> Modified: head/sys/mips/include/cpuregs.h
> ==============================================================================
> --- head/sys/mips/include/cpuregs.h Tue Dec 27 23:56:46 2016 (r310649)
> +++ head/sys/mips/include/cpuregs.h Wed Dec 28 02:55:26 2016 (r310650)
> @@ -171,6 +171,10 @@
> #define MIPS_CCA_CACHED MIPS_CCA_CCS
> #endif
>
> +#if defined(CPU_XBURST)
> +#define MIPS_CCA_UA 0x01
> +#endif
> +
> #ifndef MIPS_CCA_UNCACHED
> #define MIPS_CCA_UNCACHED MIPS_CCA_UC
> #endif
> @@ -188,6 +192,16 @@
> #endif
> #endif
>
> +/*
> + * Use uncached-accelerated mode for write-combining maps, if one is defined,
> + * otherwise fall back to uncached
> + */
> +#ifndef MIPS_CCA_WC
> +#ifdef MIPS_CCA_UA
> +#define MIPS_CCA_WC MIPS_CCA_UA
> +#endif
> +#endif
> +
> #define MIPS_PHYS_TO_XKPHYS(cca,x) \
> ((0x2ULL << 62) | ((unsigned long long)(cca) << 59) | (x))
> #define MIPS_PHYS_TO_XKPHYS_CACHED(x) \
>
> Modified: head/sys/mips/include/pmap.h
> ==============================================================================
> --- head/sys/mips/include/pmap.h Tue Dec 27 23:56:46 2016 (r310649)
> +++ head/sys/mips/include/pmap.h Wed Dec 28 02:55:26 2016 (r310650)
> @@ -74,7 +74,8 @@ struct md_page {
> };
>
> #define PV_TABLE_REF 0x02 /* referenced */
> -#define PV_MEMATTR_UNCACHEABLE 0x04
> +#define PV_MEMATTR_MASK 0xf0 /* store vm_memattr_t here */
> +#define PV_MEMATTR_SHIFT 0x04
>
> #define ASID_BITS 8
> #define ASIDGEN_BITS (32 - ASID_BITS)
> @@ -163,22 +164,24 @@ extern vm_offset_t virtual_end;
>
> extern vm_paddr_t dump_avail[PHYS_AVAIL_ENTRIES + 2];
>
> -#define pmap_page_get_memattr(m) VM_MEMATTR_DEFAULT
> +#define pmap_page_get_memattr(m) (((m)->md.pv_flags & PV_MEMATTR_MASK) >> PV_MEMATTR_SHIFT)
> #define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list))
> #define pmap_page_is_write_mapped(m) (((m)->aflags & PGA_WRITEABLE) != 0)
>
> void pmap_bootstrap(void);
> void *pmap_mapdev(vm_paddr_t, vm_size_t);
> +void *pmap_mapdev_attr(vm_paddr_t, vm_size_t, vm_memattr_t);
> void pmap_unmapdev(vm_offset_t, vm_size_t);
> vm_offset_t pmap_steal_memory(vm_size_t size);
> void pmap_kenter(vm_offset_t va, vm_paddr_t pa);
> -void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr);
> +void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t attr);
> void pmap_kremove(vm_offset_t va);
> void *pmap_kenter_temporary(vm_paddr_t pa, int i);
> void pmap_kenter_temporary_free(vm_paddr_t pa);
> void pmap_flush_pvcache(vm_page_t m);
> int pmap_emulate_modified(pmap_t pmap, vm_offset_t va);
> void pmap_page_set_memattr(vm_page_t, vm_memattr_t);
> +int pmap_change_attr(vm_offset_t, vm_size_t, vm_memattr_t);
>
> #endif /* _KERNEL */
>
>
> Modified: head/sys/mips/include/pte.h
> ==============================================================================
> --- head/sys/mips/include/pte.h Tue Dec 27 23:56:46 2016 (r310649)
> +++ head/sys/mips/include/pte.h Wed Dec 28 02:55:26 2016 (r310650)
> @@ -132,8 +132,10 @@ typedef pt_entry_t *pd_entry_t;
> * it is matched.
> */
> #define PTE_C(attr) ((attr & 0x07) << 3)
> +#define PTE_C_MASK (PTE_C(0x07))
> #define PTE_C_UNCACHED (PTE_C(MIPS_CCA_UNCACHED))
> #define PTE_C_CACHE (PTE_C(MIPS_CCA_CACHED))
> +#define PTE_C_WC (PTE_C(MIPS_CCA_WC))
> #define PTE_D 0x04
> #define PTE_V 0x02
> #define PTE_G 0x01
> @@ -158,6 +160,7 @@ typedef pt_entry_t *pd_entry_t;
> #define pte_clear(pte, bit) (*(pte) &= ~(bit))
> #define pte_set(pte, bit) (*(pte) |= (bit))
> #define pte_test(pte, bit) ((*(pte) & (bit)) == (bit))
> +#define pte_cache_bits(pte) ((*(pte) >> 3) & 0x07)
>
> /* Assembly support for PTE access*/
> #ifdef LOCORE
>
> Modified: head/sys/mips/include/vm.h
> ==============================================================================
> --- head/sys/mips/include/vm.h Tue Dec 27 23:56:46 2016 (r310649)
> +++ head/sys/mips/include/vm.h Wed Dec 28 02:55:26 2016 (r310650)
> @@ -32,7 +32,11 @@
> #include <machine/pte.h>
>
> /* Memory attributes. */
> -#define VM_MEMATTR_UNCACHEABLE ((vm_memattr_t)PTE_C_UNCACHED)
> -#define VM_MEMATTR_DEFAULT ((vm_memattr_t)PTE_C_CACHE)
> +#define VM_MEMATTR_UNCACHEABLE ((vm_memattr_t)MIPS_CCA_UNCACHED)
> +#define VM_MEMATTR_WRITE_BACK ((vm_memattr_t)MIPS_CCA_CACHED)
> +#define VM_MEMATTR_DEFAULT VM_MEMATTR_WRITE_BACK
> +#ifdef MIPS_CCA_WC
> +#define VM_MEMATTR_WRITE_COMBINING ((vm_memattr_t)MIPS_CCA_WC)
> +#endif
>
> #endif /* !_MACHINE_VM_H_ */
>
> Modified: head/sys/mips/mips/pmap.c
> ==============================================================================
> --- head/sys/mips/mips/pmap.c Tue Dec 27 23:56:46 2016 (r310649)
> +++ head/sys/mips/mips/pmap.c Wed Dec 28 02:55:26 2016 (r310650)
> @@ -189,10 +189,10 @@ static void pmap_update_page_action(void
> * The highmem area does not have a KSEG0 mapping, and we need a mechanism to
> * do temporary per-CPU mappings for pmap_zero_page, pmap_copy_page etc.
> *
> - * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To
> + * At bootup, we reserve 2 virtual pages per CPU for mapping highmem pages. To
> * access a highmem physical address on a CPU, we map the physical address to
> - * the reserved virtual address for the CPU in the kernel pagetable. This is
> - * done with interrupts disabled(although a spinlock and sched_pin would be
> + * the reserved virtual address for the CPU in the kernel pagetable. This is
> + * done with interrupts disabled(although a spinlock and sched_pin would be
> * sufficient).
> */
> struct local_sysmaps {
> @@ -303,7 +303,7 @@ pmap_lmem_map2(vm_paddr_t phys1, vm_padd
> return (0);
> }
>
> -static __inline vm_offset_t
> +static __inline vm_offset_t
> pmap_lmem_unmap(void)
> {
>
> @@ -312,12 +312,18 @@ pmap_lmem_unmap(void)
> #endif /* !__mips_n64 */
>
> static __inline int
> -is_cacheable_page(vm_paddr_t pa, vm_page_t m)
> +pmap_pte_cache_bits(vm_paddr_t pa, vm_page_t m)
> {
> + vm_memattr_t ma;
>
> - return ((m->md.pv_flags & PV_MEMATTR_UNCACHEABLE) == 0 &&
> - is_cacheable_mem(pa));
> -
> + ma = pmap_page_get_memattr(m);
> + if (ma == VM_MEMATTR_WRITE_BACK && !is_cacheable_mem(pa))
> + ma = VM_MEMATTR_UNCACHEABLE;
> + return PTE_C(ma);
> +}
> +#define PMAP_PTE_SET_CACHE_BITS(pte, ps, m) { \
> + pte &= ~PTE_C_MASK; \
> + pte |= pmap_pte_cache_bits(pa, m); \
> }
>
> /*
> @@ -359,7 +365,7 @@ pmap_pdpe_to_pde(pd_entry_t *pdpe, vm_of
> return (pdpe);
> }
>
> -static __inline
> +static __inline
> pd_entry_t *pmap_pde(pmap_t pmap, vm_offset_t va)
> {
>
> @@ -423,7 +429,7 @@ pmap_steal_memory(vm_size_t size)
> * Bootstrap the system enough to run with virtual memory. This
> * assumes that the phys_avail array has been initialized.
> */
> -static void
> +static void
> pmap_create_kernel_pagetable(void)
> {
> int i, j;
> @@ -486,7 +492,7 @@ void
> pmap_bootstrap(void)
> {
> int i;
> - int need_local_mappings = 0;
> + int need_local_mappings = 0;
>
> /* Sort. */
> again:
> @@ -600,7 +606,7 @@ pmap_page_init(vm_page_t m)
> {
>
> TAILQ_INIT(&m->md.pv_list);
> - m->md.pv_flags = 0;
> + m->md.pv_flags = VM_MEMATTR_DEFAULT << PV_MEMATTR_SHIFT;
> }
>
> /*
> @@ -635,8 +641,8 @@ pmap_call_on_active_cpus(pmap_t pmap, vo
> pmap->pm_asid[cpu].gen = 0;
> }
> cpuid = PCPU_GET(cpuid);
> - /*
> - * XXX: barrier/locking for active?
> + /*
> + * XXX: barrier/locking for active?
> *
> * Take a snapshot of active here, any further changes are ignored.
> * tlb update/invalidate should be harmless on inactive CPUs
> @@ -819,7 +825,7 @@ retry:
> * add a wired page to the kva
> */
> void
> -pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int attr)
> +pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)
> {
> pt_entry_t *pte;
> pt_entry_t opte, npte;
> @@ -830,7 +836,7 @@ pmap_kenter_attr(vm_offset_t va, vm_padd
>
> pte = pmap_pte(kernel_pmap, va);
> opte = *pte;
> - npte = TLBLO_PA_TO_PFN(pa) | attr | PTE_D | PTE_V | PTE_G;
> + npte = TLBLO_PA_TO_PFN(pa) | PTE_C(ma) | PTE_D | PTE_V | PTE_G;
> *pte = npte;
> if (pte_test(&opte, PTE_V) && opte != npte)
> pmap_update_page(kernel_pmap, va, npte);
> @@ -843,7 +849,7 @@ pmap_kenter(vm_offset_t va, vm_paddr_t p
> KASSERT(is_cacheable_mem(pa),
> ("pmap_kenter: memory at 0x%lx is not cacheable", (u_long)pa));
>
> - pmap_kenter_attr(va, pa, PTE_C_CACHE);
> + pmap_kenter_attr(va, pa, VM_MEMATTR_DEFAULT);
> }
>
> /*
> @@ -1144,11 +1150,11 @@ _pmap_allocpte(pmap_t pmap, unsigned pte
> int segindex = ptepindex >> (SEGSHIFT - PDRSHIFT);
> int pdeindex = ptepindex & (NPDEPG - 1);
> vm_page_t pg;
> -
> +
> pdep = &pmap->pm_segtab[segindex];
> - if (*pdep == NULL) {
> + if (*pdep == NULL) {
> /* recurse for allocating page dir */
> - if (_pmap_allocpte(pmap, NUPDE + segindex,
> + if (_pmap_allocpte(pmap, NUPDE + segindex,
> flags) == NULL) {
> /* alloc failed, release current */
> --m->wire_count;
> @@ -1680,7 +1686,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm
> * pmap_remove_pte: do the things to unmap a page in a process
> */
> static int
> -pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
> +pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va,
> pd_entry_t pde)
> {
> pt_entry_t oldpte;
> @@ -1864,7 +1870,7 @@ pmap_remove_all(vm_page_t m)
> PMAP_LOCK(pmap);
>
> /*
> - * If it's last mapping writeback all caches from
> + * If it's last mapping writeback all caches from
> * the page being destroyed
> */
> if (TAILQ_NEXT(pv, pv_list) == NULL)
> @@ -2030,10 +2036,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
> newpte |= PTE_W;
> if (is_kernel_pmap(pmap))
> newpte |= PTE_G;
> - if (is_cacheable_page(pa, m))
> - newpte |= PTE_C_CACHE;
> - else
> - newpte |= PTE_C_UNCACHED;
> + PMAP_PTE_SET_CACHE_BITS(newpte, pa, m);
>
> mpte = NULL;
>
> @@ -2218,7 +2221,7 @@ static vm_page_t
> pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
> vm_prot_t prot, vm_page_t mpte)
> {
> - pt_entry_t *pte;
> + pt_entry_t *pte, npte;
> vm_paddr_t pa;
>
> KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
> @@ -2297,18 +2300,16 @@ pmap_enter_quick_locked(pmap_t pmap, vm_
> /*
> * Now validate mapping with RO protection
> */
> - *pte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V;
> + npte = PTE_RO | TLBLO_PA_TO_PFN(pa) | PTE_V;
> if ((m->oflags & VPO_UNMANAGED) == 0)
> - *pte |= PTE_MANAGED;
> + npte |= PTE_MANAGED;
>
> - if (is_cacheable_page(pa, m))
> - *pte |= PTE_C_CACHE;
> - else
> - *pte |= PTE_C_UNCACHED;
> + PMAP_PTE_SET_CACHE_BITS(npte, pa, m);
>
> if (is_kernel_pmap(pmap))
> - *pte |= PTE_G;
> + *pte = npte | PTE_G;
> else {
> + *pte = npte;
> /*
> * Sync I & D caches. Do this only if the target pmap
> * belongs to the current process. Otherwise, an
> @@ -2649,12 +2650,12 @@ pmap_quick_enter_page(vm_page_t m)
> #else
> vm_paddr_t pa;
> struct local_sysmaps *sysm;
> - pt_entry_t *pte;
> + pt_entry_t *pte, npte;
>
> pa = VM_PAGE_TO_PHYS(m);
>
> if (MIPS_DIRECT_MAPPABLE(pa)) {
> - if (m->md.pv_flags & PV_MEMATTR_UNCACHEABLE)
> + if (pmap_page_get_memattr(m) != VM_MEMATTR_WRITE_BACK)
> return (MIPS_PHYS_TO_DIRECT_UNCACHED(pa));
> else
> return (MIPS_PHYS_TO_DIRECT(pa));
> @@ -2665,8 +2666,9 @@ pmap_quick_enter_page(vm_page_t m)
> KASSERT(sysm->valid1 == 0, ("pmap_quick_enter_page: PTE busy"));
>
> pte = pmap_pte(kernel_pmap, sysm->base);
> - *pte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G |
> - (is_cacheable_page(pa, m) ? PTE_C_CACHE : PTE_C_UNCACHED);
> + npte = TLBLO_PA_TO_PFN(pa) | PTE_D | PTE_V | PTE_G;
> + PMAP_PTE_SET_CACHE_BITS(npte, pa, m);
> + *pte = npte;
> sysm->valid1 = 1;
>
> return (sysm->base);
> @@ -3146,26 +3148,26 @@ pmap_is_referenced(vm_page_t m)
> * Use XKPHYS uncached for 64 bit, and KSEG1 where possible for 32 bit.
> */
> void *
> -pmap_mapdev(vm_paddr_t pa, vm_size_t size)
> +pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma)
> {
> vm_offset_t va, tmpva, offset;
>
> - /*
> - * KSEG1 maps only first 512M of phys address space. For
> + /*
> + * KSEG1 maps only first 512M of phys address space. For
> * pa > 0x20000000 we should make proper mapping * using pmap_kenter.
> */
> - if (MIPS_DIRECT_MAPPABLE(pa + size - 1))
> + if (MIPS_DIRECT_MAPPABLE(pa + size - 1) && ma == VM_MEMATTR_UNCACHEABLE)
> return ((void *)MIPS_PHYS_TO_DIRECT_UNCACHED(pa));
> else {
> offset = pa & PAGE_MASK;
> size = roundup(size + offset, PAGE_SIZE);
> -
> +
> va = kva_alloc(size);
> if (!va)
> panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
> pa = trunc_page(pa);
> for (tmpva = va; size > 0;) {
> - pmap_kenter_attr(tmpva, pa, PTE_C_UNCACHED);
> + pmap_kenter_attr(tmpva, pa, ma);
> size -= PAGE_SIZE;
> tmpva += PAGE_SIZE;
> pa += PAGE_SIZE;
> @@ -3175,6 +3177,12 @@ pmap_mapdev(vm_paddr_t pa, vm_size_t siz
> return ((void *)(va + offset));
> }
>
> +void *
> +pmap_mapdev(vm_paddr_t pa, vm_size_t size)
> +{
> + return pmap_mapdev_attr(pa, size, VM_MEMATTR_UNCACHEABLE);
> +}
> +
> void
> pmap_unmapdev(vm_offset_t va, vm_size_t size)
> {
> @@ -3220,7 +3228,7 @@ retry:
> * This may falsely report the given address as
> * MINCORE_REFERENCED. Unfortunately, due to the lack of
> * per-PTE reference information, it is impossible to
> - * determine if the address is MINCORE_REFERENCED.
> + * determine if the address is MINCORE_REFERENCED.
> */
> m = PHYS_TO_VM_PAGE(pa);
> if ((m->aflags & PGA_REFERENCED) != 0)
> @@ -3500,7 +3508,7 @@ pmap_kextract(vm_offset_t va)
> mapped = (va >= MIPS_KSEG2_START || va < MIPS_KSEG2_END);
> #if defined(__mips_n64)
> mapped = mapped || (va >= MIPS_XKSEG_START || va < MIPS_XKSEG_END);
> -#endif
> +#endif
> /*
> * Kernel virtual.
> */
> @@ -3524,7 +3532,7 @@ pmap_kextract(vm_offset_t va)
> }
>
>
> -void
> +void
> pmap_flush_pvcache(vm_page_t m)
> {
> pv_entry_t pv;
> @@ -3551,12 +3559,85 @@ pmap_page_set_memattr(vm_page_t m, vm_me
> if (TAILQ_FIRST(&m->md.pv_list) != NULL)
> panic("Can't change memattr on page with existing mappings");
>
> - /*
> - * The only memattr we support is UNCACHEABLE, translate the (semi-)MI
> - * representation of that into our internal flag in the page MD struct.
> - */
> - if (ma == VM_MEMATTR_UNCACHEABLE)
> - m->md.pv_flags |= PV_MEMATTR_UNCACHEABLE;
> - else
> - m->md.pv_flags &= ~PV_MEMATTR_UNCACHEABLE;
> + /* Clean memattr portion of pv_flags */
> + m->md.pv_flags &= ~PV_MEMATTR_MASK;
> + m->md.pv_flags |= (ma << PV_MEMATTR_SHIFT) & PV_MEMATTR_MASK;
> +}
> +
> +static __inline void
> +pmap_pte_attr(pt_entry_t *pte, vm_memattr_t ma)
> +{
> + u_int npte;
> +
> + npte = *(u_int *)pte;
> + npte &= ~PTE_C_MASK;
> + npte |= PTE_C(ma);
> + *pte = npte;
> +}
> +
> +int
> +pmap_change_attr(vm_offset_t sva, vm_size_t size, vm_memattr_t ma)
> +{
> + pd_entry_t *pde, *pdpe;
> + pt_entry_t *pte;
> + vm_offset_t ova, eva, va, va_next;
> + pmap_t pmap;
> +
> + ova = sva;
> + eva = sva + size;
> + if (eva < sva)
> + return (EINVAL);
> +
> + pmap = kernel_pmap;
> + PMAP_LOCK(pmap);
> +
> + for (; sva < eva; sva = va_next) {
> + pdpe = pmap_segmap(pmap, sva);
> +#ifdef __mips_n64
> + if (*pdpe == 0) {
> + va_next = (sva + NBSEG) & ~SEGMASK;
> + if (va_next < sva)
> + va_next = eva;
> + continue;
> + }
> +#endif
> + va_next = (sva + NBPDR) & ~PDRMASK;
> + if (va_next < sva)
> + va_next = eva;
> +
> + pde = pmap_pdpe_to_pde(pdpe, sva);
> + if (*pde == NULL)
> + continue;
> +
> + /*
> + * Limit our scan to either the end of the va represented
> + * by the current page table page, or to the end of the
> + * range being removed.
> + */
> + if (va_next > eva)
> + va_next = eva;
> +
> + va = va_next;
> + for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++,
> + sva += PAGE_SIZE) {
> + if (!pte_test(pte, PTE_V) || pte_cache_bits(pte) == ma) {
> + if (va != va_next) {
> + pmap_invalidate_range(pmap, va, sva);
> + va = va_next;
> + }
> + continue;
> + }
> + if (va == va_next)
> + va = sva;
> +
> + pmap_pte_attr(pte, ma);
> + }
> + if (va != va_next)
> + pmap_invalidate_range(pmap, va, sva);
> + }
> + PMAP_UNLOCK(pmap);
> +
> + /* Flush caches to be in the safe side */
> + mips_dcache_wbinv_range(ova, size);
> + return 0;
> }
>
More information about the svn-src-all
mailing list