On 2019.02.19 02:46:32 -0500, Yan Zhao wrote:
> The patch implments the gvt interface intel_gvt_save_restore to
> save/restore vGPU's device config data for live migration.
> 
> vGPU device config data includes vreg, vggtt, vcfg space, workloads, ppgtt,
> execlist.
> It does not include dirty pages in system memory produced by vGPU.
> 
> Signed-off-by: Yulei Zhang <yulei.zhang@intel.com>
> Signed-off-by: Xiao Zheng <xiao.zheng@intel.com>
> Signed-off-by: Zhenyu Wang <zhenyuw@linux.intel.com>
> Signed-off-by: Yan Zhao <yan.y.zhao@intel.com>

...

> +
> +#ifndef __GVT_MIGRATE_H__
> +#define __GVT_MIGRATE_H__
> +
> +#define MIGRATION_DIRTY_BITMAP_SIZE (16*1024UL)
> +
> +/* Assume 9MB is enough to descript VM kernel state */
> +#define MIGRATION_IMG_MAX_SIZE (9*1024UL*1024UL)
> +#define GVT_MMIO_SIZE (2*1024UL*1024UL)
> +#define GVT_MIGRATION_VERSION	0
> +
> +enum gvt_migration_type_t {
> +	GVT_MIGRATION_NONE,
> +	GVT_MIGRATION_HEAD,
> +	GVT_MIGRATION_CFG_SPACE,
> +	GVT_MIGRATION_VREG,
> +	GVT_MIGRATION_SREG,
> +	GVT_MIGRATION_GTT,
> +	GVT_MIGRATION_PPGTT,
> +	GVT_MIGRATION_WORKLOAD,
> +	GVT_MIGRATION_EXECLIST,
> +};
> +
> +struct gvt_ppgtt_entry_t {
> +	int page_table_level;
> +	u64 pdp[4];
> +};
> +
> +struct gvt_pending_workload_t {
> +	int ring_id;
> +	bool emulate_schedule_in;
> +	struct execlist_ctx_descriptor_format ctx_desc;
> +	struct intel_vgpu_elsp_dwords elsp_dwords;
> +};
> +
> +struct gvt_region_t {
> +	enum gvt_migration_type_t type;
> +	u32 size;		/* obj size of bytes to read/write */
> +};
> +
> +struct gvt_migration_obj_t {
> +	void *img;
> +	void *vgpu;
> +	u32 offset;
> +	struct gvt_region_t region;
> +	/* operation func defines how data save-restore */
> +	struct gvt_migration_operation_t *ops;
> +	char *name;
> +};
> +
> +struct gvt_migration_operation_t {
> +	/* called during pre-copy stage, VM is still alive */
> +	int (*pre_copy)(const struct gvt_migration_obj_t *obj);
> +	/* called before when VM was paused,
> +	 * return bytes transferred
> +	 */
> +	int (*pre_save)(const struct gvt_migration_obj_t *obj);
> +	/* called before load the state of device */
> +	int (*pre_load)(const struct gvt_migration_obj_t *obj, u32 size);
> +	/* called after load the state of device, VM already alive */
> +	int (*post_load)(const struct gvt_migration_obj_t *obj, u32 size);
> +};
> +
> +struct gvt_image_header_t {
> +	int version;
> +	int data_size;
> +	u64 crc_check;
> +	u64 global_data[64];
> +};

I think this misses device info that should ship with the image,
currently what I can think is that each platform should have seperate
type, e.g BDW, SKL, KBL, etc. We won't allow to restore onto different
platform than the source.

> +
> +#endif
> diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
> index 43f65848ecd6..6221d2f274fc 100644
> --- a/drivers/gpu/drm/i915/gvt/mmio.c
> +++ b/drivers/gpu/drm/i915/gvt/mmio.c
> @@ -50,6 +50,19 @@ int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa)
>  	return gpa - gttmmio_gpa;
>  }
>  
> +/**
> + * intel_vgpu_mmio_offset_to_GPA - translate a MMIO offset to GPA
> + * @vgpu: a vGPU
> + *
> + * Returns:
> + * Zero on success, negative error code if failed
> + */
> +int intel_vgpu_mmio_offset_to_gpa(struct intel_vgpu *vgpu, u64 offset)
> +{
> +	return offset + ((*(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_0)) &
> +		~GENMASK(3, 0));
> +}
> +
>  #define reg_is_mmio(gvt, reg)  \
>  	(reg >= 0 && reg < gvt->device_info.mmio_size)
>  
> diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
> index 1ffc69eba30e..a2bddb0257cf 100644
> --- a/drivers/gpu/drm/i915/gvt/mmio.h
> +++ b/drivers/gpu/drm/i915/gvt/mmio.h
> @@ -82,6 +82,7 @@ void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr);
>  void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu);
>  
>  int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa);
> +int intel_vgpu_mmio_offset_to_gpa(struct intel_vgpu *vgpu, u64 offset);
>  
>  int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, u64 pa,
>  				void *p_data, unsigned int bytes);
> diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
> index fcccda35a456..7676dcfdca09 100644
> --- a/drivers/gpu/drm/i915/gvt/vgpu.c
> +++ b/drivers/gpu/drm/i915/gvt/vgpu.c
> @@ -213,6 +213,7 @@ void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu)
>  {
>  	mutex_lock(&vgpu->gvt->lock);
>  	vgpu->active = true;
> +	intel_vgpu_start_schedule(vgpu);
>  	mutex_unlock(&vgpu->gvt->lock);
>  }
>  
> -- 
> 2.17.1
> 
> _______________________________________________
> intel-gvt-dev mailing list
> intel-gvt-dev@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/intel-gvt-dev

-- 
Open Source Technology Center, Intel ltd.

$gpg --keyserver wwwkeys.pgp.net --recv-keys 4D781827