Re: [PATCH v8 13/19] hvf: Add Apple Silicon support

[Peter Maydell <peter.maydell@linaro.org>]

On Wed, 19 May 2021 at 21:23, Alexander Graf <agraf@csgraf.de> wrote:
>
> With Apple Silicon available to the masses, it's a good time to add support
> for driving its virtualization extensions from QEMU.
>
> This patch adds all necessary architecture specific code to get basic VMs
> working. It's still pretty raw, but definitely functional.
>
> Known limitations:
>
>   - Vtimer acknowledgement is hacky
>   - Should implement more sysregs and fault on invalid ones then
>   - WFI handling is missing, need to marry it with vtimer
>
> Signed-off-by: Alexander Graf <agraf@csgraf.de>
> Reviewed-by: Roman Bolshakov <r.bolshakov@yadro.com>
> @@ -446,11 +454,17 @@ static void hvf_start_vcpu_thread(CPUState *cpu)
>                         cpu, QEMU_THREAD_JOINABLE);
>  }
>
> +__attribute__((weak)) void hvf_kick_vcpu_thread(CPUState *cpu)
> +{
> +    cpus_kick_thread(cpu);
> +}

Why is this marked 'weak' ? If there's a reason for it then
it ought to have a comment describing the reason. If we can avoid
it then we should do so -- past experience is that 'weak' refs
are rather non-portable, though at least this one is in a
host-OS-specific file.

> +static void hvf_raise_exception(CPUARMState *env, uint32_t excp,
> +                                uint32_t syndrome)
> +{
> +    unsigned int new_el = 1;
> +    unsigned int old_mode = pstate_read(env);
> +    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
> +    target_ulong addr = env->cp15.vbar_el[new_el];
> +
> +    env->cp15.esr_el[new_el] = syndrome;
> +    aarch64_save_sp(env, arm_current_el(env));
> +    env->elr_el[new_el] = env->pc;
> +    env->banked_spsr[aarch64_banked_spsr_index(new_el)] = old_mode;
> +    pstate_write(env, PSTATE_DAIF | new_mode);
> +    aarch64_restore_sp(env, new_el);
> +    env->pc = addr;
> +}

KVM does "raise an exception" by calling arm_cpu_do_interrupt()
to do the "set ESR_ELx, save SPSR, etc etc" work (see eg
kvm_arm_handle_debug()". Does that not work here ?

> +
> +static uint64_t hvf_sysreg_read(CPUState *cpu, uint32_t reg)
> +{
> +    ARMCPU *arm_cpu = ARM_CPU(cpu);
> +    uint64_t val = 0;
> +
> +    switch (reg) {
> +    case SYSREG_CNTPCT_EL0:
> +        val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
> +              gt_cntfrq_period_ns(arm_cpu);
> +        break;

Does hvf handle the "EL0 access which should be denied because
CNTKCTL_EL1.EL0PCTEN is 0" case for us, or should we have
an access-check here ?

> +    case SYSREG_PMCCNTR_EL0:
> +        val = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);

This is supposed to be a cycle counter, not a timestamp...

> +        break;
> +    default:
> +        trace_hvf_unhandled_sysreg_read(reg,
> +                                        (reg >> 20) & 0x3,
> +                                        (reg >> 14) & 0x7,
> +                                        (reg >> 10) & 0xf,
> +                                        (reg >> 1) & 0xf,
> +                                        (reg >> 17) & 0x7);
> +        break;
> +    }
> +
> +    return val;
> +}
> +
> +static void hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val)
> +{
> +    switch (reg) {
> +    case SYSREG_CNTPCT_EL0:
> +        break;

CNTPCT_EL0 is read-only (ie writes should fault) but this
makes it writes-ignored, doesn't it ?

> +    default:
> +        trace_hvf_unhandled_sysreg_write(reg,
> +                                         (reg >> 20) & 0x3,
> +                                         (reg >> 14) & 0x7,
> +                                         (reg >> 10) & 0xf,
> +                                         (reg >> 1) & 0xf,
> +                                         (reg >> 17) & 0x7);
> +        break;
> +    }
> +}

> +    switch (ec) {
> +    case EC_DATAABORT: {
> +        bool isv = syndrome & ARM_EL_ISV;
> +        bool iswrite = (syndrome >> 6) & 1;
> +        bool s1ptw = (syndrome >> 7) & 1;
> +        uint32_t sas = (syndrome >> 22) & 3;
> +        uint32_t len = 1 << sas;
> +        uint32_t srt = (syndrome >> 16) & 0x1f;
> +        uint64_t val = 0;
> +
> +        trace_hvf_data_abort(env->pc, hvf_exit->exception.virtual_address,
> +                             hvf_exit->exception.physical_address, isv,
> +                             iswrite, s1ptw, len, srt);
> +
> +        assert(isv);

This seems dubious -- won't we just crash if the guest does
a data access to a device or to unmapped memory with an insn that
doesn't set ISV ? With KVM we feed this back to the guest as an
external data abort (see the KVM_EXIT_ARM_NISV handling).

> +
> +        if (iswrite) {
> +            val = hvf_get_reg(cpu, srt);
> +            address_space_write(&address_space_memory,
> +                                hvf_exit->exception.physical_address,
> +                                MEMTXATTRS_UNSPECIFIED, &val, len);
> +        } else {
> +            address_space_read(&address_space_memory,
> +                               hvf_exit->exception.physical_address,
> +                               MEMTXATTRS_UNSPECIFIED, &val, len);
> +            hvf_set_reg(cpu, srt, val);
> +        }
> +
> +        advance_pc = true;
> +        break;
> +    }
> +    case EC_SYSTEMREGISTERTRAP: {
> +        bool isread = (syndrome >> 0) & 1;
> +        uint32_t rt = (syndrome >> 5) & 0x1f;
> +        uint32_t reg = syndrome & SYSREG_MASK;
> +        uint64_t val = 0;
> +
> +        if (isread) {
> +            val = hvf_sysreg_read(cpu, reg);
> +            trace_hvf_sysreg_read(reg,
> +                                  (reg >> 20) & 0x3,
> +                                  (reg >> 14) & 0x7,
> +                                  (reg >> 10) & 0xf,
> +                                  (reg >> 1) & 0xf,
> +                                  (reg >> 17) & 0x7,
> +                                  val);
> +            hvf_set_reg(cpu, rt, val);
> +        } else {
> +            val = hvf_get_reg(cpu, rt);
> +            trace_hvf_sysreg_write(reg,
> +                                   (reg >> 20) & 0x3,
> +                                   (reg >> 14) & 0x7,
> +                                   (reg >> 10) & 0xf,
> +                                   (reg >> 1) & 0xf,
> +                                   (reg >> 17) & 0x7,
> +                                   val);
> +            hvf_sysreg_write(cpu, reg, val);
> +        }

This needs support for "this really was a bogus system register
access, feed the guest an exception".

> +
> +        advance_pc = true;
> +        break;
> +    }
> +    case EC_WFX_TRAP:
> +        advance_pc = true;
> +        break;
> +    case EC_AA64_HVC:
> +        cpu_synchronize_state(cpu);
> +        trace_hvf_unknown_hvf(env->xregs[0]);
> +        hvf_raise_exception(env, EXCP_UDEF, syn_uncategorized());
> +        break;
> +    case EC_AA64_SMC:
> +        cpu_synchronize_state(cpu);
> +        trace_hvf_unknown_smc(env->xregs[0]);
> +        hvf_raise_exception(env, EXCP_UDEF, syn_uncategorized());
> +        break;
> +    default:
> +        cpu_synchronize_state(cpu);
> +        trace_hvf_exit(syndrome, ec, env->pc);
> +        error_report("0x%llx: unhandled exit 0x%llx", env->pc, exit_reason);
> +    }
> +
> +    if (advance_pc) {
> +        uint64_t pc;
> +
> +        flush_cpu_state(cpu);
> +
> +        r = hv_vcpu_get_reg(cpu->hvf->fd, HV_REG_PC, &pc);
> +        assert_hvf_ok(r);
> +        pc += 4;
> +        r = hv_vcpu_set_reg(cpu->hvf->fd, HV_REG_PC, pc);
> +        assert_hvf_ok(r);
> +    }
> +
> +    return 0;
> +}

thanks
-- PMM