From: Akhil P Oommen <quic_akhilpo@quicinc.com> To: freedreno <freedreno@lists.freedesktop.org>, <dri-devel@lists.freedesktop.org>, <linux-arm-msm@vger.kernel.org>, Rob Clark <robdclark@gmail.com>, Bjorn Andersson <bjorn.andersson@linaro.org> Cc: "Jonathan Marek" <jonathan@marek.ca>, "Jordan Crouse" <jordan@cosmicpenguin.net>, "Matthias Kaehlcke" <mka@chromium.org>, "Douglas Anderson" <dianders@chromium.org>, "Akhil P Oommen" <quic_akhilpo@quicinc.com>, "Abhinav Kumar" <quic_abhinavk@quicinc.com>, "Chia-I Wu" <olvaffe@gmail.com>, "Christian König" <christian.koenig@amd.com>, "Daniel Vetter" <daniel@ffwll.ch>, "David Airlie" <airlied@linux.ie>, "Dmitry Baryshkov" <dmitry.baryshkov@linaro.org>, "Geert Uytterhoeven" <geert@linux-m68k.org>, "Konrad Dybcio" <konrad.dybcio@somainline.org>, "Sean Paul" <sean@poorly.run>, "Wang Qing" <wangqing@vivo.com>, linux-kernel@vger.kernel.org Subject: [PATCH v2 6/7] drm/msm/a6xx: Improve gpu recovery sequence Date: Sat, 9 Jul 2022 11:29:34 +0530 [thread overview] Message-ID: <20220709112837.v2.6.Idf2ba51078e87ae7ceb75cc77a5bd4ff2bd31eab@changeid> (raw) In-Reply-To: <1657346375-1461-1-git-send-email-quic_akhilpo@quicinc.com> We can do a few more things to improve our chance at a successful gpu recovery, especially during a hangcheck timeout: 1. Halt CP and GMU core 2. Do RBBM GBIF HALT sequence 3. Do a soft reset of GPU core Signed-off-by: Akhil P Oommen <quic_akhilpo@quicinc.com> --- (no changes since v1) drivers/gpu/drm/msm/adreno/a6xx.xml.h | 4 ++ drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 77 +++++++++++++++++++++-------------- drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 7 ++++ 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h index b03e2c4..beea4a7 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h @@ -1413,6 +1413,10 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00 #define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00000011 +#define REG_A6XX_RBBM_GBIF_HALT 0x00000016 + +#define REG_A6XX_RBBM_GBIF_HALT_ACK 0x00000017 + #define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD 0x0000001c #define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE 0x00000001 diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 310a317..ec25f19 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -873,9 +873,47 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) (val & 1), 100, 1000); } +#define GBIF_CLIENT_HALT_MASK BIT(0) +#define GBIF_ARB_HALT_MASK BIT(1) + +static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) +{ + struct msm_gpu *gpu = &adreno_gpu->base; + + if (!a6xx_has_gbif(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); + spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & + 0xf) == 0xf); + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); + + return; + } + + /* Halt the gx side of GBIF */ + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1); + spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1); + + /* Halt new client requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); + + /* Halt all AXI requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); + + /* The GBIF halt needs to be explicitly cleared */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); +} + /* Force the GMU off in case it isn't responsive */ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) { + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + /* Flush all the queues */ a6xx_hfi_stop(gmu); @@ -887,6 +925,15 @@ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) /* Make sure there are no outstanding RPMh votes */ a6xx_gmu_rpmh_off(gmu); + + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + + a6xx_bus_clear_pending_transactions(adreno_gpu); + + /* Reset GPU core blocks */ + gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, 1); + udelay(100); } static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) @@ -1014,36 +1061,6 @@ bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) return true; } -#define GBIF_CLIENT_HALT_MASK BIT(0) -#define GBIF_ARB_HALT_MASK BIT(1) - -static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) -{ - struct msm_gpu *gpu = &adreno_gpu->base; - - if (!a6xx_has_gbif(adreno_gpu)) { - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); - spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & - 0xf) == 0xf); - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); - - return; - } - - /* Halt new client requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); - - /* Halt all AXI requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); - - /* The GBIF halt needs to be explicitly cleared */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); -} - /* Gracefully try to shut down the GMU and by extension the GPU */ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) { diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 9aaa469..60c3033 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -987,6 +987,10 @@ static int hw_init(struct msm_gpu *gpu) /* Make sure the GMU keeps the GPU on while we set it up */ a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + /* Clear GBIF halt in case GX domain was not collapsed */ + if (a6xx_has_gbif(adreno_gpu)) + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0); + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); /* @@ -1276,6 +1280,9 @@ static void a6xx_recover(struct msm_gpu *gpu) if (hang_debug) a6xx_dump(gpu); + /* Halt SQE first */ + gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3); + /* * Turn off keep alive that might have been enabled by the hang * interrupt -- 2.7.4
WARNING: multiple messages have this Message-ID (diff)
From: Akhil P Oommen <quic_akhilpo@quicinc.com> To: freedreno <freedreno@lists.freedesktop.org>, <dri-devel@lists.freedesktop.org>, <linux-arm-msm@vger.kernel.org>, Rob Clark <robdclark@gmail.com>, Bjorn Andersson <bjorn.andersson@linaro.org> Cc: "Jonathan Marek" <jonathan@marek.ca>, "Christian König" <christian.koenig@amd.com>, "Akhil P Oommen" <quic_akhilpo@quicinc.com>, linux-kernel@vger.kernel.org, "Konrad Dybcio" <konrad.dybcio@somainline.org>, "Abhinav Kumar" <quic_abhinavk@quicinc.com>, "Douglas Anderson" <dianders@chromium.org>, "Wang Qing" <wangqing@vivo.com>, "David Airlie" <airlied@linux.ie>, "Matthias Kaehlcke" <mka@chromium.org>, "Geert Uytterhoeven" <geert@linux-m68k.org>, "Dmitry Baryshkov" <dmitry.baryshkov@linaro.org>, "Jordan Crouse" <jordan@cosmicpenguin.net>, "Sean Paul" <sean@poorly.run> Subject: [PATCH v2 6/7] drm/msm/a6xx: Improve gpu recovery sequence Date: Sat, 9 Jul 2022 11:29:34 +0530 [thread overview] Message-ID: <20220709112837.v2.6.Idf2ba51078e87ae7ceb75cc77a5bd4ff2bd31eab@changeid> (raw) In-Reply-To: <1657346375-1461-1-git-send-email-quic_akhilpo@quicinc.com> We can do a few more things to improve our chance at a successful gpu recovery, especially during a hangcheck timeout: 1. Halt CP and GMU core 2. Do RBBM GBIF HALT sequence 3. Do a soft reset of GPU core Signed-off-by: Akhil P Oommen <quic_akhilpo@quicinc.com> --- (no changes since v1) drivers/gpu/drm/msm/adreno/a6xx.xml.h | 4 ++ drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 77 +++++++++++++++++++++-------------- drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 7 ++++ 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h index b03e2c4..beea4a7 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h @@ -1413,6 +1413,10 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00 #define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL 0x00000011 +#define REG_A6XX_RBBM_GBIF_HALT 0x00000016 + +#define REG_A6XX_RBBM_GBIF_HALT_ACK 0x00000017 + #define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD 0x0000001c #define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE 0x00000001 diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 310a317..ec25f19 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -873,9 +873,47 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu) (val & 1), 100, 1000); } +#define GBIF_CLIENT_HALT_MASK BIT(0) +#define GBIF_ARB_HALT_MASK BIT(1) + +static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) +{ + struct msm_gpu *gpu = &adreno_gpu->base; + + if (!a6xx_has_gbif(adreno_gpu)) { + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); + spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & + 0xf) == 0xf); + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); + + return; + } + + /* Halt the gx side of GBIF */ + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1); + spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1); + + /* Halt new client requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); + + /* Halt all AXI requests on GBIF */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); + spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & + (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); + + /* The GBIF halt needs to be explicitly cleared */ + gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); +} + /* Force the GMU off in case it isn't responsive */ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) { + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + /* Flush all the queues */ a6xx_hfi_stop(gmu); @@ -887,6 +925,15 @@ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu) /* Make sure there are no outstanding RPMh votes */ a6xx_gmu_rpmh_off(gmu); + + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + + a6xx_bus_clear_pending_transactions(adreno_gpu); + + /* Reset GPU core blocks */ + gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, 1); + udelay(100); } static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu) @@ -1014,36 +1061,6 @@ bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) return true; } -#define GBIF_CLIENT_HALT_MASK BIT(0) -#define GBIF_ARB_HALT_MASK BIT(1) - -static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu) -{ - struct msm_gpu *gpu = &adreno_gpu->base; - - if (!a6xx_has_gbif(adreno_gpu)) { - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); - spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & - 0xf) == 0xf); - gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); - - return; - } - - /* Halt new client requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK); - - /* Halt all AXI requests on GBIF */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK); - spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) & - (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK); - - /* The GBIF halt needs to be explicitly cleared */ - gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); -} - /* Gracefully try to shut down the GMU and by extension the GPU */ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) { diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index 9aaa469..60c3033 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -987,6 +987,10 @@ static int hw_init(struct msm_gpu *gpu) /* Make sure the GMU keeps the GPU on while we set it up */ a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + /* Clear GBIF halt in case GX domain was not collapsed */ + if (a6xx_has_gbif(adreno_gpu)) + gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0); + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); /* @@ -1276,6 +1280,9 @@ static void a6xx_recover(struct msm_gpu *gpu) if (hang_debug) a6xx_dump(gpu); + /* Halt SQE first */ + gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3); + /* * Turn off keep alive that might have been enabled by the hang * interrupt -- 2.7.4
next prev parent reply other threads:[~2022-07-09 6:00 UTC|newest] Thread overview: 50+ messages / expand[flat|nested] mbox.gz Atom feed top 2022-07-09 5:59 [PATCH v2 0/7] Improve GPU Recovery Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 1/7] drm/msm: Remove unnecessary pm_runtime_get/put Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 2/7] drm/msm: Correct pm_runtime votes in recover worker Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 3/7] drm/msm: Fix cx collapse issue during recovery Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-11 23:22 ` Doug Anderson 2022-07-11 23:22 ` Doug Anderson 2022-07-12 5:04 ` [Freedreno] " Akhil P Oommen 2022-07-12 5:04 ` Akhil P Oommen 2022-07-12 16:44 ` Rob Clark 2022-07-12 16:44 ` Rob Clark 2022-07-12 19:15 ` Akhil P Oommen 2022-07-12 19:15 ` Akhil P Oommen 2022-07-20 18:06 ` Rob Clark 2022-07-20 18:06 ` Rob Clark 2022-07-20 20:38 ` Akhil P Oommen 2022-07-20 20:38 ` Akhil P Oommen 2022-07-22 17:25 ` Akhil P Oommen 2022-07-22 17:25 ` Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 4/7] drm/msm: Ensure cx gdsc collapse " Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 5/7] arm64: dts: qcom: sc7280: Update gpu register list Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen 2022-07-11 23:27 ` Doug Anderson 2022-07-11 23:27 ` Doug Anderson 2022-07-14 5:40 ` Akhil P Oommen 2022-07-14 5:40 ` Akhil P Oommen 2022-07-19 4:07 ` [Freedreno] " Akhil P Oommen 2022-07-19 4:07 ` Akhil P Oommen 2022-07-19 5:49 ` Stephen Boyd 2022-07-19 5:49 ` Stephen Boyd 2022-07-19 6:37 ` Akhil P Oommen 2022-07-19 6:37 ` Akhil P Oommen 2022-07-19 7:19 ` Stephen Boyd 2022-07-19 7:19 ` Stephen Boyd 2022-07-19 9:56 ` Rajendra Nayak 2022-07-19 9:56 ` Rajendra Nayak 2022-07-20 6:04 ` Akhil P Oommen 2022-07-20 6:04 ` Akhil P Oommen 2022-07-21 16:04 ` Akhil P Oommen 2022-07-21 16:04 ` Akhil P Oommen 2022-07-22 15:28 ` Rob Clark 2022-07-22 15:28 ` Rob Clark 2022-07-09 5:59 ` Akhil P Oommen [this message] 2022-07-09 5:59 ` [PATCH v2 6/7] drm/msm/a6xx: Improve gpu recovery sequence Akhil P Oommen 2022-07-09 5:59 ` [PATCH v2 7/7] drm/msm/a6xx: Handle GMU prepare-slumber hfi failure Akhil P Oommen 2022-07-09 5:59 ` Akhil P Oommen
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20220709112837.v2.6.Idf2ba51078e87ae7ceb75cc77a5bd4ff2bd31eab@changeid \ --to=quic_akhilpo@quicinc.com \ --cc=airlied@linux.ie \ --cc=bjorn.andersson@linaro.org \ --cc=christian.koenig@amd.com \ --cc=daniel@ffwll.ch \ --cc=dianders@chromium.org \ --cc=dmitry.baryshkov@linaro.org \ --cc=dri-devel@lists.freedesktop.org \ --cc=freedreno@lists.freedesktop.org \ --cc=geert@linux-m68k.org \ --cc=jonathan@marek.ca \ --cc=jordan@cosmicpenguin.net \ --cc=konrad.dybcio@somainline.org \ --cc=linux-arm-msm@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=mka@chromium.org \ --cc=olvaffe@gmail.com \ --cc=quic_abhinavk@quicinc.com \ --cc=robdclark@gmail.com \ --cc=sean@poorly.run \ --cc=wangqing@vivo.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.