From 47a9fbf7962287d656bbaf14058c78baf6304beb Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Mon, 29 Nov 2021 09:40:07 +0800 Subject: [PATCH 09/12] riscv: add RISC-V Svpbmt extension supports This patch follows the standard pure RISC-V Svpbmt extension in privilege spec to solve the non-coherent SOC dma synchronization issues. Here is the svpbmt PTE format: | 63 | 62-61 | 60-8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 N MT RSW D A G U X W R V ^ Of the Reserved bits [63:54] in a leaf PTE, the high bit is already allocated (as the N bit), so bits [62:61] are used as the MT (aka MemType) field. This field specifies one of three memory types that are close equivalents (or equivalent in effect) to the three main x86 and ARMv8 memory types - as shown in the following table. RISC-V Encoding & MemType RISC-V Description ---------- ------------------------------------------------ 00 - PMA Normal Cacheable, No change to implied PMA memory type 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory 11 - Rsvd Reserved for future standard use The standard protection_map[] needn't be modified because the "PMA" type keeps the highest bits zero. The implementation is limited to arch/riscv/* as it uses an inline function and the alternatives mechanism to generate the _PAGE_MTMASK/IO/NOCACHE values for pgprot_noncached (& writecombine) in pgtable.h on both svpbmt and non-svpbmt-capable chips. Signed-off-by: Heiko Stuebner --- arch/riscv/include/asm/alternative.h | 2 + arch/riscv/include/asm/errata_list.h | 13 ++++- arch/riscv/include/asm/pgtable-32.h | 10 ++++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++++++ arch/riscv/include/asm/pgtable-bits.h | 4 -- arch/riscv/include/asm/pgtable.h | 36 ++++++++++--- arch/riscv/kernel/alternative.c | 9 ++++ arch/riscv/kernel/cpufeature.c | 76 +++++++++++++++++++++++++++ arch/riscv/kernel/head.S | 2 + 9 files changed, 198 insertions(+), 13 deletions(-) diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 3ed027113526..640e34d7a4b8 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -19,8 +19,10 @@ #define RISCV_ALTERNATIVES_BOOT 0 #define RISCV_ALTERNATIVES_MODULE 1 +#define RISCV_ALTERNATIVES_EARLY 2 void __init apply_boot_alternatives(void); +void __init apply_early_alternatives(void); void apply_module_alternatives(void *start, size_t length); struct alt_entry { diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 6b95bd9aee82..da1373783658 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -14,7 +14,8 @@ #define ERRATA_SIFIVE_NUMBER 2 #endif -#define CPUFEATURE_NUMBER 0 +#define CPUFEATURE_SVPBMT 0 +#define CPUFEATURE_NUMBER 1 #ifdef __ASSEMBLY__ @@ -36,6 +37,16 @@ asm(ALTERNATIVE("sfence.vma %0", "sfence.vma", SIFIVE_VENDOR_ID, \ ERRATA_SIFIVE_CIP_1200, CONFIG_ERRATA_SIFIVE_CIP_1200) \ : : "r" (addr) : "memory") +/* + * _val is marked as "will be overwritten", so need to set it to 0 + * in the default case. + */ +#define ALT_SVPBMT_SHIFT 59 +#define ALT_SVPBMT(_val, prot) \ +asm(ALTERNATIVE("li %0, 0\t\nnop", "li %0, %1\t\nslli %0,%0,%2", 0, \ + CPUFEATURE_SVPBMT, CONFIG_64BIT) \ + : "=r"(_val) : "I"( prot##_SVPBMT >> ALT_SVPBMT_SHIFT), "I"(ALT_SVPBMT_SHIFT)) + #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h index 5b2e79e5bfa5..f4eac23fb300 100644 --- a/arch/riscv/include/asm/pgtable-32.h +++ b/arch/riscv/include/asm/pgtable-32.h @@ -16,4 +16,14 @@ #define MAX_POSSIBLE_PHYSMEM_BITS 34 +#define _PAGE_PMA 0 +#define _PAGE_NOCACHE 0 +#define _PAGE_IO 0 +#define _PAGE_MTMASK 0 + +/* Set of bits to preserve across pte_modify() */ +#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_USER | _PAGE_GLOBAL)) + #endif /* _ASM_RISCV_PGTABLE_32_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 6df493b55dde..7a3438be1c26 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -7,6 +7,7 @@ #define _ASM_RISCV_PGTABLE_64_H #include +#include #define PGDIR_SHIFT 30 /* Size of region mapped by a page global directory */ @@ -28,6 +29,64 @@ typedef struct { #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) +/* + * rv64 PTE format: + * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 + * N MT RSV PFN reserved for SW D A G U X W R V + * [62:61] Memory Type definitions: + * 00 - PMA Normal Cacheable, No change to implied PMA memory type + * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory + * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory + * 11 - Rsvd Reserved for future standard use + */ +#define _PAGE_PMA_SVPBMT 0UL +#define _PAGE_NOCACHE_SVPBMT (1UL << 61) +#define _PAGE_IO_SVPBMT (1UL << 62) +#define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT) + +static inline u64 riscv_page_mtmask(void) +{ + u64 val; + + ALT_SVPBMT(val, _PAGE_MTMASK); + return val; +} + +static inline u64 riscv_page_pma(void) +{ + u64 val; + + ALT_SVPBMT(val, _PAGE_PMA); + return val; +} + +static inline u64 riscv_page_nocache(void) +{ + u64 val; + + ALT_SVPBMT(val, _PAGE_NOCACHE); + return val; +} + +static inline u64 riscv_page_io(void) +{ + u64 val; + + ALT_SVPBMT(val, _PAGE_IO); + return val; +} + +#define _PAGE_PMA riscv_page_pma() +#define _PAGE_NOCACHE riscv_page_nocache() +#define _PAGE_IO riscv_page_io() +#define _PAGE_MTMASK riscv_page_mtmask() + +/* Set of bits to preserve across pte_modify() */ +#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_USER | _PAGE_GLOBAL | \ + _PAGE_MTMASK)) + static inline int pud_present(pud_t pud) { return (pud_val(pud) & _PAGE_PRESENT); diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index 2ee413912926..c639c563ac2c 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -35,10 +35,6 @@ #define _PAGE_PFN_SHIFT 10 -/* Set of bits to preserve across pte_modify() */ -#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ - _PAGE_WRITE | _PAGE_EXEC | \ - _PAGE_USER | _PAGE_GLOBAL)) /* * when all of R/W/X are zero, the PTE is a pointer to the next level * of the page table; otherwise, it is a leaf PTE. diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 450804c263b5..c6be32ba74d7 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -117,9 +117,9 @@ #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) /* Page protection bits */ -#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER) +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_USER | _PAGE_PMA) -#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE | _PAGE_PMA) #define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ) #define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE) #define PAGE_EXEC __pgprot(_PAGE_BASE | _PAGE_EXEC) @@ -138,7 +138,8 @@ | _PAGE_PRESENT \ | _PAGE_ACCESSED \ | _PAGE_DIRTY \ - | _PAGE_GLOBAL) + | _PAGE_GLOBAL \ + | _PAGE_PMA) #define PAGE_KERNEL __pgprot(_PAGE_KERNEL) #define PAGE_KERNEL_READ __pgprot(_PAGE_KERNEL & ~_PAGE_WRITE) @@ -148,11 +149,8 @@ #define PAGE_TABLE __pgprot(_PAGE_TABLE) -/* - * The RISC-V ISA doesn't yet specify how to query or modify PMAs, so we can't - * change the properties of memory regions. - */ -#define _PAGE_IOREMAP _PAGE_KERNEL +#define _PAGE_IOREMAP ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO) +#define PAGE_KERNEL_IO __pgprot(_PAGE_IOREMAP) extern pgd_t swapper_pg_dir[]; @@ -492,6 +490,28 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma, return ptep_test_and_clear_young(vma, address, ptep); } +#define pgprot_noncached pgprot_noncached +static inline pgprot_t pgprot_noncached(pgprot_t _prot) +{ + unsigned long prot = pgprot_val(_prot); + + prot &= ~_PAGE_MTMASK; + prot |= _PAGE_IO; + + return __pgprot(prot); +} + +#define pgprot_writecombine pgprot_writecombine +static inline pgprot_t pgprot_writecombine(pgprot_t _prot) +{ + unsigned long prot = pgprot_val(_prot); + + prot &= ~_PAGE_MTMASK; + prot |= _PAGE_NOCACHE; + + return __pgprot(prot); +} + /* * THP functions */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index 6acb324f4815..136058e17b86 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -84,6 +84,15 @@ void __init apply_boot_alternatives(void) RISCV_ALTERNATIVES_BOOT); } +void __init apply_early_alternatives(void) +{ + init_alternative(); + + _apply_alternatives((struct alt_entry *)__alt_start, + (struct alt_entry *)__alt_end, + RISCV_ALTERNATIVES_EARLY); +} + #ifdef CONFIG_MODULES void apply_module_alternatives(void *start, size_t length) { diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index dba8fc688303..cbeb3abd975e 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -160,7 +161,82 @@ struct cpufeature_info { bool (*check_func)(unsigned int stage); }; +static bool cpufeature_svpbmt_check_fdt(void) +{ + const void *fdt = dtb_early_va; + const char *str; + int offset; + + offset = fdt_path_offset(fdt, "/cpus"); + if (offset < 0) + return false; + + for (offset = fdt_next_node(fdt, offset, NULL); offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + str = fdt_getprop(fdt, offset, "device_type", NULL); + if (!str || strcmp(str, "cpu")) + break; + + str = fdt_getprop(fdt, offset, "mmu-type", NULL); + if (!str) + continue; + + if (!strncmp(str + 6, "none", 4)) + continue; + + str = fdt_getprop(fdt, offset, "mmu", NULL); + if (!str) + continue; + + if (!strncmp(str + 6, "svpbmt", 6)) + return true; + } + + return false; +} + +static bool cpufeature_svpbmt_check_of(void) +{ + struct device_node *node; + const char *str; + + for_each_of_cpu_node(node) { + if (of_property_read_string(node, "mmu-type", &str)) + continue; + + if (!strncmp(str + 6, "none", 4)) + continue; + + if (of_property_read_string(node, "mmu", &str)) + continue; + + if (!strncmp(str + 6, "svpbmt", 6)) + return true; + } + + return false; +} + +static bool cpufeature_svpbmt_check_func(unsigned int stage) +{ + bool ret = false; + +#if defined(CONFIG_MMU) && defined(CONFIG_64BIT) + if (stage == RISCV_ALTERNATIVES_EARLY) + return cpufeature_svpbmt_check_fdt(); + else + return cpufeature_svpbmt_check_of(); +#endif + + return ret; +} + static const struct cpufeature_info cpufeature_list[CPUFEATURE_NUMBER] = { + { + .name = "svpbmt", + .stage = RISCV_ALTERNATIVES_EARLY, + .check_func = cpufeature_svpbmt_check_func + }, }; static u32 __init cpufeature_probe(unsigned int stage) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index f52f01ecbeea..1e5224c19d4f 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -332,6 +332,8 @@ clear_bss_done: la tp, init_task la sp, init_thread_union + THREAD_SIZE + call apply_early_alternatives + #ifdef CONFIG_KASAN call kasan_early_init #endif -- 2.30.2