[bus][pci] add routines to pass in PCI bus resources prior to starting the pci bus manager

Wire them up on arm and riscv which need them. x86-pc does not, so dont
call it.

Also fix a few miscellaneous bugs, notably PCI not detecting 64bit bars
properly due to an off by one bit error.

dev/bus/pci/bus_mgr/bus_mgr.cpp

@@ -129,6 +129,12 @@ status_t pci_bus_mgr_init() {
     return NO_ERROR;
 }
 
+status_t pci_bus_mgr_add_resource(enum pci_resource_type type, uint64_t mmio_base, uint64_t aux_base, uint64_t len) {
+    TRACEF("type %d: mmio base %#llx aux base %#llx len %#llx\n", type, mmio_base, aux_base, len);
+
+    return NO_ERROR;
+}
+
 // for every bus in the system, pass the visit routine to the device
 status_t pci_bus_mgr_visit_devices(pci_visit_routine routine, void *cookie) {
     auto v = [&](device *d) -> status_t {
@@ -261,11 +267,15 @@ status_t pci_bus_mgr_allocate_irq(const pci_location_t loc, uint *irqbase) {
     return d->allocate_irq(irqbase);
 }
 
+void pci_dump_bar(const pci_bar_t *bar, int index) {
+    printf("BAR %d: addr %-#16llx size %-#16zx io %d 64bit %d prefetch %d\n",
+            index, bar->addr, bar->size, bar->io, bar->size_64, bar->prefetchable);
+}
+
 void pci_dump_bars(pci_bar_t bar[6], size_t count) {
     for (size_t i = 0; i < count; i++) {
         if (bar[i].valid) {
-            printf("BAR %zu: addr %#16llx size %#16zx io %d\n",
+            pci_dump_bar(bar + i, i);
-                    i, bar[i].addr, bar[i].size, bar[i].io);
         }
     }
 }

dev/bus/pci/bus_mgr/device.cpp

@@ -123,7 +123,7 @@ void device::dump(size_t indent) {
             for (size_t i = 0; i < indent + 1; i++) {
                 printf(" ");
             }
-            printf("BAR %zu: addr %#llx size %#zx io %d valid %d\n", b, bars_[b].addr, bars_[b].size, bars_[b].io, bars_[b].valid);
+            pci_dump_bar(bars_ + b, b);
         }
     }
 }
@@ -316,11 +316,13 @@ status_t device::load_bars() {
     }
 
     for (size_t i=0; i < num_bars; i++) {
-        bars_[i].valid = false;
+        bars_[i] = {};
         uint64_t bar_addr = config_.type0.base_addresses[i];
         if (bar_addr & 0x1) {
             // io address
             bars_[i].io = true;
+            bars_[i].prefetchable = false;
+            bars_[i].size_64 = false;
             bars_[i].addr = bar_addr & ~0x3;
 
             // probe size by writing all 1s and seeing what bits are masked
@@ -333,9 +335,11 @@ status_t device::load_bars() {
             bars_[i].size = ((size & ~0b11) ^ 0xffff) + 1;
 
             bars_[i].valid = (bars_[i].size != 0);
-        } else if ((bar_addr & 0b110) == 0) {
+        } else if ((bar_addr & 0b110) == 0b000) {
             // 32bit memory address
             bars_[i].io = false;
+            bars_[i].prefetchable = bar_addr & (1<<3);
+            bars_[i].size_64 = false;
             bars_[i].addr = bar_addr & ~0xf;
 
             // probe size by writing all 1s and seeing what bits are masked
@@ -348,13 +352,16 @@ status_t device::load_bars() {
             bars_[i].size = (~(size & ~0b1111)) + 1;
 
             bars_[i].valid = (bars_[i].size != 0);
-        } else if ((bar_addr & 0b110) == 2) {
+        } else if ((bar_addr & 0b110) == 0b100) {
             // 64bit memory address
-            if (i % 2) {
+            if (i >= num_bars - 1) {
-                // root of 64bit memory range can only be on 0, 2, 4 slot
+                // root of 64bit memory range will use up two slots, so cant
+                // start at the last bar
                 continue;
             }
             bars_[i].io = false;
+            bars_[i].prefetchable = bar_addr & (1<<3);
+            bars_[i].size_64 = true;
             bars_[i].addr = bar_addr & ~0xf;
             bars_[i].addr |= (uint64_t)config_.type0.base_addresses[i + 1] << 32;
 
@@ -364,11 +371,11 @@ status_t device::load_bars() {
             pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4, 0xffffffff);
             pci_read_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4, &size32);
             size = size32;
-            pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 1, 0xffffffff);
+            pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 4, 0xffffffff);
-            pci_read_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 1, &size32);
+            pci_read_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 4, &size32);
             size |= (uint64_t)size32 << 32;
             pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4, bars_[i].addr);
-            pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 1, bars_[i].addr >> 32);
+            pci_write_config_word(loc_, PCI_CONFIG_BASE_ADDRESSES + i * 4 + 4, bars_[i].addr >> 32);
 
             // mask out bottom bits, invert and add 1 to compute size
             bars_[i].size = (~(size & ~(uint64_t)0b1111)) + 1;
@@ -377,7 +384,7 @@ status_t device::load_bars() {
 
             // mark the next entry as invalid
             i++;
-            bars_[i].valid = false;
+            bars_[i] = {}; // clears the valid bit
         }
     }
 

dev/bus/pci/include/dev/bus/pci.h

@@ -31,6 +31,8 @@ typedef struct {
     uint64_t addr;
     size_t size;
     bool io;
+    bool prefetchable;
+    bool size_64;
     bool valid;
 } pci_bar_t;
 
@@ -83,6 +85,14 @@ status_t pci_write_config_word(pci_location_t state, uint32_t reg, uint32_t valu
 typedef void(*pci_visit_routine)(pci_location_t loc, void *cookie);
 status_t pci_bus_mgr_visit_devices(pci_visit_routine routine, void *cookie);
 
+// must be called before pci_bus_mgr_init if available
+enum pci_resource_type {
+    PCI_RESOURCE_IO_RANGE, // aux base is base of io range, mmio base is if memory mapped
+    PCI_RESOURCE_MMIO_RANGE,
+    PCI_RESOURCE_MMIO64_RANGE,
+};
+status_t pci_bus_mgr_add_resource(enum pci_resource_type, uint64_t mmio_base, uint64_t aux_base, uint64_t len);
+
 // must be called after pci_init_*();
 status_t pci_bus_mgr_init(void);
 
@@ -109,6 +119,7 @@ status_t pci_bus_mgr_allocate_irq(const pci_location_t loc, uint *irqbase);
 
 // return a pointer to a formatted string
 const char *pci_loc_string(pci_location_t loc, char out_str[14]);
+void pci_dump_bar(const pci_bar_t *bar, int index);
 void pci_dump_bars(pci_bar_t bar[6], size_t count);
 
 __END_CDECLS

dev/bus/pci/pci.cpp

@@ -290,7 +290,6 @@ status_t pci_init_ecam(paddr_t ecam_base, uint16_t segment, uint8_t start_bus, u
     if ((pcib = pci_ecam::detect(ecam_base, segment, start_bus, end_bus))) {
         dprintf(INFO, "PCI: pci ecam functions installed\n");
         dprintf(INFO, "PCI: last pci bus is %d\n", pcib->get_last_bus());
-        pci_bus_mgr_init();
         return NO_ERROR;
     }
 

platform/pc/platform.c

@@ -283,6 +283,7 @@ void platform_init(void) {
                 // try to initialize pci based on the MCFG ecam aperture
                 status_t err = pci_init_ecam(entry->base_address, entry->segment, entry->start_bus, entry->end_bus);
                 if (err == NO_ERROR) {
+                    pci_bus_mgr_init();
                     pci_initted = true;
                 }
             }
@@ -291,7 +292,10 @@ void platform_init(void) {
 
     // fall back to legacy pci if we couldn't find the pcie aperture
     if (!pci_initted) {
-        pci_init_legacy();
+        status_t err = pci_init_legacy();
+        if (err == NO_ERROR) {
+            pci_bus_mgr_init();
+        }
     }
 #endif
 

platform/qemu-virt-arm/platform.c

@@ -177,15 +177,30 @@ void platform_init(void) {
 
     /* detect pci */
 #if ARCH_ARM
-    if (pcie_state.ecam_base > (1ULL << 32)) {
+    if (pcie_state.info.ecam_base > (1ULL << 32)) {
         // dont try to configure this since we dont have LPAE support
         printf("PCIE: skipping pci initialization due to high memory ECAM\n");
-        pcie_state.ecam_len = 0;
+        pcie_state.info.ecam_len = 0;
     }
 #endif
-    if (pcie_state.ecam_len > 0) {
+    if (pcie_state.info.ecam_len > 0) {
-        printf("PCIE: initializing pcie with ecam at %#" PRIx64 " found in FDT\n", pcie_state.ecam_base);
+        printf("PCIE: initializing pcie with ecam at %#" PRIx64 " found in FDT\n", pcie_state.info.ecam_base);
-        pci_init_ecam(pcie_state.ecam_base, pcie_state.ecam_len, pcie_state.bus_start, pcie_state.bus_end);
+        err = pci_init_ecam(pcie_state.info.ecam_base, pcie_state.info.ecam_len, pcie_state.info.bus_start, pcie_state.info.bus_end);
+        if (err == NO_ERROR) {
+            // add some additional resources to the pci bus manager in case it needs to configure
+            if (pcie_state.info.io_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_IO_RANGE, pcie_state.info.io_base_mmio, pcie_state.info.io_base, pcie_state.info.io_len);
+            }
+            if (pcie_state.info.mmio_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_MMIO_RANGE, pcie_state.info.mmio_base, 0, pcie_state.info.mmio_len);
+            }
+            if (pcie_state.info.mmio64_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_MMIO64_RANGE, pcie_state.info.mmio64_base, 0, pcie_state.info.mmio64_len);
+            }
+
+            // start the bus manager
+            pci_bus_mgr_init();
+        }
     }
 
     /* detect any virtio devices */

platform/qemu-virt-riscv/platform.c

@@ -155,9 +155,24 @@ void platform_init(void) {
     uart_init();
 
     /* detect pci */
-    if (pcie_state.ecam_len > 0) {
+    if (pcie_state.info.ecam_len > 0) {
-        printf("PCIE: initializing pcie with ecam at %#" PRIx64 " found in FDT\n", pcie_state.ecam_base);
+        printf("PCIE: initializing pcie with ecam at %#" PRIx64 " found in FDT\n", pcie_state.info.ecam_base);
-        pci_init_ecam(pcie_state.ecam_base, pcie_state.ecam_len, pcie_state.bus_start, pcie_state.bus_end);
+        status_t err = pci_init_ecam(pcie_state.info.ecam_base, pcie_state.info.ecam_len, pcie_state.info.bus_start, pcie_state.info.bus_end);
+        if (err == NO_ERROR) {
+            // add some additional resources to the pci bus manager in case it needs to configure
+            if (pcie_state.info.io_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_IO_RANGE, pcie_state.info.io_base_mmio, pcie_state.info.io_base, pcie_state.info.io_len);
+            }
+            if (pcie_state.info.mmio_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_MMIO_RANGE, pcie_state.info.mmio_base, 0, pcie_state.info.mmio_len);
+            }
+            if (pcie_state.info.mmio64_len > 0) {
+                pci_bus_mgr_add_resource(PCI_RESOURCE_MMIO64_RANGE, pcie_state.info.mmio64_base, 0, pcie_state.info.mmio64_len);
+            }
+
+            // start the bus manager
+            pci_bus_mgr_init();
+        };
     }
 
     /* detect any virtio devices */

scripts/do-qemuarm

@@ -68,6 +68,7 @@ else
     QEMU="qemu-system-arm"
     CPU="cortex-a15"
     MACHINE="virt"
+    MACHINE+=",highmem=off" # disable the high PCI ECAM, since we dont support LPAE to map it
     _PROJECT="qemu-virt-arm32-test"
 fi
 if [ "$PROJECT" == "" ]; then