[arch][arm64] add cache flush by way/set

Pulled code from Fuchsia to implement way/set cache flush for arm64.

Issue #307

arch/arm64/cache-ops.S

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2014, Google Inc. All rights reserved
+ * Copyright 2016 The Fuchsia Authors
  *
  * Use of this source code is governed by a MIT-style
  * license that can be found in the LICENSE file or at
@@ -9,6 +10,7 @@
 #include <lk/asm.h>
 #include <arch/ops.h>
 #include <arch/defines.h>
+#include <arch/arm64/cache_loop.h>
 
 .text
 
@@ -27,19 +29,60 @@
 FUNCTION(arch_clean_cache_range)
     cache_range_op dc cvac         // clean cache to PoC by MVA
     ret
+END_FUNCTION(arch_clean_cache_range)
 
     /* void arch_flush_invalidate_cache_range(addr_t start, size_t len); */
 FUNCTION(arch_clean_invalidate_cache_range)
     cache_range_op dc civac        // clean & invalidate dcache to PoC by MVA
     ret
+END_FUNCTION(arch_clean_invalidate_cache_range)
 
     /* void arch_invalidate_cache_range(addr_t start, size_t len); */
 FUNCTION(arch_invalidate_cache_range)
     cache_range_op dc ivac         // invalidate dcache to PoC by MVA
     ret
+END_FUNCTION(arch_invalidate_cache_range)
 
     /* void arch_sync_cache_range(addr_t start, size_t len); */
 FUNCTION(arch_sync_cache_range)
     cache_range_op dc cvau         // clean dcache to PoU by MVA
     cache_range_op ic ivau         // invalidate icache to PoU by MVA
     ret
+END_FUNCTION(arch_sync_cache_range)
+
+// Below are 3 variants of cache flushing routines by way/set for
+// an individual cpu.
+
+    // void arm64_local_invalidate_cache_all()
+FUNCTION(arm64_local_invalidate_cache_all)
+    cache_way_set_op isw, invalidate
+
+    // dump the instruction cache as well
+    ic      iallu
+    isb
+
+    ret
+END_FUNCTION(arm64_local_invalidate_cache_all)
+
+    // void arm64_local_clean_cache_all()
+FUNCTION(arm64_local_clean_cache_all)
+    cache_way_set_op csw, clean
+
+    // dump the instruction cache as well
+    ic      iallu
+    isb
+
+    ret
+END_FUNCTION(arm64_local_clean_cache_all)
+
+    // void arm64_local_clean_invalidate_cache_all()
+FUNCTION(arm64_local_clean_invalidate_cache_all)
+    cache_way_set_op cisw, clean_invalidate
+
+    // dump the instruction cache as well
+    ic      iallu
+    isb
+
+    ret
+END_FUNCTION(arm64_local_clean_invalidate_cache_all)
+

arch/arm64/include/arch/arm64.h

@@ -74,5 +74,13 @@ static inline void arm64_fpu_pre_context_switch(struct thread *thread) {
 /* overridable syscall handler */
 void arm64_syscall(struct arm64_iframe_long *iframe, bool is_64bit);
 
+/* Local per-cpu cache flush routines.
+ * These routines clean or invalidate the cache from the point of view
+ * of a single cpu to the point of coherence.
+ */
+void arm64_local_invalidate_cache_all(void);
+void arm64_local_clean_invalidate_cache_all(void);
+void arm64_local_clean_cache_all(void);
+
 __END_CDECLS
 

arch/arm64/include/arch/arm64/cache_loop.h

@@ -0,0 +1,61 @@
+// Copyright 2020 The Fuchsia Authors
+//
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT
+
+#pragma once
+
+// clang-format off
+
+#ifdef __ASSEMBLER__
+
+// Routine to iterate over all ways/sets across all levels of data caches
+// from level 0 to the point of coherence.
+//
+// Adapted from example code in the ARM Architecture Reference Manual ARMv8.
+.macro cache_way_set_op, op name
+    mrs     x0, clidr_el1
+    and     w3, w0, #0x07000000     // get 2x level of coherence
+    lsr     w3, w3, #23
+    cbz     w3, .Lfinished_\name
+    mov     w10, #0                 // w10 = 2x cache level
+    mov     w8, #1                  // w8 = constant 1
+.Lloop1_\name:
+    add     w2, w10, w10, lsr #1    // calculate 3x cache level
+    lsr     w1, w0, w2              // extract 3 bit cache type for this level
+    and     w1, w1, #0x7
+    cmp     w1, #2
+    b.lt    .Lskip_\name            // no data or unified cache at this level
+    msr     csselr_el1, x10         // select this cache level
+    isb                             // synchronize change to csselr
+    mrs     x1, ccsidr_el1          // w1 = ccsidr
+    and     w2, w1, #7              // w2 = log2(line len) - 4
+    add     w2, w2, #4              // w2 = log2(line len)
+    ubfx    w4, w1, #3, #10         // w4 = max way number, right aligned
+    clz     w5, w4                  // w5 = 32 - log2(ways), bit position of way in DC operand
+    lsl     w9, w4, w5              // w9 = max way number, aligned to position in DC operand
+    lsl     w12, w8, w5             // w12 = amount to decrement way number per iteration
+
+.Lloop2_\name:
+    ubfx    w7, w1, #13, #15        // w7 = max set number, right aligned
+    lsl     w7, w7, w2              // w7 = max set number, aligned to position in DC operand
+    lsl     w13, w8, w2             // w13 = amount to decrement set number per iteration
+.Lloop3_\name:
+    orr     w11, w10, w9            // w11 = combine way number and cache number
+    orr     w11, w11, w7            //       and set number for DC operand
+    dc      \op, x11                // data cache op
+    subs    w7, w7, w13             // decrement set number
+    b.ge    .Lloop3_\name
+
+    subs    x9, x9, x12             // decrement way number
+    b.ge    .Lloop2_\name
+.Lskip_\name:
+    add     w10, w10, #2            // increment 2x cache level
+    cmp     w3, w10
+    dsb     sy                      // ensure completetion of previous cache maintainance instructions
+    b.gt    .Lloop1_\name
+.Lfinished_\name:
+.endm
+
+#endif