Re: [PATCH 5/8] memory: tegra: Add EMC scaling sequence code for Tegra210

[Joseph Lo <josephl@nvidia.com>]

On 4/2/19 7:36 PM, Dmitry Osipenko wrote:
> 25.03.2019 10:45, Joseph Lo пишет:
>> This patch includes the sequence for controlling the rate changing for
>> EMC frequency and the dynamic training mechanism when the rate reaches
>> the higher rates of EMC rate.
>>
>> And historically there have been different sequences to change the EMC
>> clock. The sequence to be used is specified in the scaling data.
>> However, for the currently supported upstreaming platform, only the most
>> recent sequence is used. So only support that in this patch.
>>
>> Based on the work of Peter De Schrijver <pdeschrijver@nvidia.com>.
>>
>> Signed-off-by: Joseph Lo <josephl@nvidia.com>
>> ---
>>   drivers/memory/tegra/Makefile                 |    2 +-
>>   drivers/memory/tegra/tegra210-emc-cc-r21021.c | 1962 +++++++++++++++++
>>   drivers/memory/tegra/tegra210-emc-reg.h       |  134 ++
>>   drivers/memory/tegra/tegra210-emc.c           |    5 +
>>   4 files changed, 2102 insertions(+), 1 deletion(-)
>>   create mode 100644 drivers/memory/tegra/tegra210-emc-cc-r21021.c
>>
>> diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
>> index 36a835620bbd..dcc245b2ef45 100644
>> --- a/drivers/memory/tegra/Makefile
>> +++ b/drivers/memory/tegra/Makefile
>> @@ -12,5 +12,5 @@ obj-$(CONFIG_TEGRA_MC) += tegra-mc.o
>>   
>>   obj-$(CONFIG_TEGRA20_EMC)  += tegra20-emc.o
>>   obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o
>> -obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o tegra210-dt-parse.o
>> +obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o tegra210-dt-parse.o tegra210-emc-cc-r21021.o
>>   obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o
>> diff --git a/drivers/memory/tegra/tegra210-emc-cc-r21021.c b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
>> new file mode 100644
>> index 000000000000..f577a8c3aa95
>> --- /dev/null
>> +++ b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
>> @@ -0,0 +1,1962 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +/*
>> + * Copyright (c) 2014-2019, NVIDIA CORPORATION.  All rights reserved.
>> + */
>> +
>> +#include <linux/kernel.h>
>> +#include <linux/io.h>
>> +#include <linux/clk.h>
>> +#include <linux/delay.h>
>> +#include <linux/of.h>
>> +#include <soc/tegra/mc.h>
>> +
>> +#include "mc.h"
>> +#include "tegra210-emc-reg.h"
>> +
>> +#define DVFS_CLOCK_CHANGE_VERSION	21021
>> +#define EMC_PRELOCK_VERSION		2101
>> +
>> +#define emc_cc_dbg(t, ...) pr_debug(__VA_ARGS__)
>> +
>> +/*
>> + * Enable flags for specifying verbosity.
>> + */
>> +#define INFO            (1 << 0)
>> +#define STEPS           (1 << 1)
>> +#define SUB_STEPS       (1 << 2)
>> +#define PRELOCK         (1 << 3)
>> +#define PRELOCK_STEPS   (1 << 4)
>> +#define ACTIVE_EN       (1 << 5)
>> +#define PRAMP_UP        (1 << 6)
>> +#define PRAMP_DN        (1 << 7)
>> +#define EMA_WRITES      (1 << 10)
>> +#define EMA_UPDATES     (1 << 11)
>> +#define PER_TRAIN       (1 << 16)
>> +#define CC_PRINT        (1 << 17)
>> +#define CCFIFO          (1 << 29)
>> +#define REGS            (1 << 30)
>> +#define REG_LISTS       (1 << 31)
>> +
>> +enum {
>> +	DVFS_SEQUENCE = 1,
>> +	WRITE_TRAINING_SEQUENCE = 2,
>> +	PERIODIC_TRAINING_SEQUENCE = 3,
>> +	DVFS_PT1 = 10,
>> +	DVFS_UPDATE = 11,
>> +	TRAINING_PT1 = 12,
>> +	TRAINING_UPDATE = 13,
>> +	PERIODIC_TRAINING_UPDATE = 14
>> +};
>> +
>> +/*
>> + * PTFV defines - basically just indexes into the per table PTFV array.
>> + */
>> +#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX		0
>> +#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX		1
>> +#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX		2
>> +#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX		3
>> +#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX		4
>> +#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX		5
>> +#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX		6
>> +#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX		7
>> +#define PTFV_DVFS_SAMPLES_INDEX			9
>> +#define PTFV_MOVAVG_WEIGHT_INDEX		10
>> +#define PTFV_CONFIG_CTRL_INDEX			11
>> +
>> +#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA	(1 << 0)
>> +
>> +/*
>> + * Do arithmetic in fixed point.
>> + */
>> +#define MOVAVG_PRECISION_FACTOR		100
>> +
>> +/*
>> + * The division portion of the average operation.
>> + */
>> +#define __AVERAGE_PTFV(dev)						\
>> +	({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
>> +	   next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
>> +	   next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
>> +
>> +/*
>> + * Convert val to fixed point and add it to the temporary average.
>> + */
>> +#define __INCREMENT_PTFV(dev, val)					\
>> +	({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
>> +	   ((val) * MOVAVG_PRECISION_FACTOR); })
>> +
>> +/*
>> + * Convert a moving average back to integral form and return the value.
>> + */
>> +#define __MOVAVG_AC(timing, dev)					\
>> +	((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] /	\
>> +	 MOVAVG_PRECISION_FACTOR)
>> +
>> +/* Weighted update. */
>> +#define __WEIGHTED_UPDATE_PTFV(dev, nval)				\
>> +	do {								\
>> +		int w = PTFV_MOVAVG_WEIGHT_INDEX;			\
>> +		int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX;		\
>> +									\
>> +		next_timing->ptfv_list[dqs] =				\
>> +			((nval * MOVAVG_PRECISION_FACTOR) +		\
>> +			 (next_timing->ptfv_list[dqs] *			\
>> +			  next_timing->ptfv_list[w])) /			\
>> +			(next_timing->ptfv_list[w] + 1);		\
>> +									\
>> +		emc_cc_dbg(EMA_UPDATES, "%s: (s=%u) EMA: %u\n",		\
>> +			   __stringify(dev), nval,			\
>> +			   next_timing->ptfv_list[dqs]);		\
>> +	} while (0)
>> +
>> +/* Access a particular average. */
>> +#define __MOVAVG(timing, dev)                      \
>> +	((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
>> +
>> +static u32 update_clock_tree_delay(struct tegra_emc *emc,
>> +				   u32 dram_dev_num, u32 channel_mode, int type)
>> +{
>> +	u32 mrr_req = 0, mrr_data = 0;
>> +	u32 temp0_0 = 0, temp0_1 = 0, temp1_0 = 0, temp1_1 = 0;
>> +	s32 tdel = 0, tmdel = 0, adel = 0;
>> +	u32 cval = 0;
>> +	struct emc_table *last_timing = emc->current_timing;
>> +	struct emc_table *next_timing = emc->next_timing;
>> +	u32 last_timing_rate_mhz = last_timing->rate / 1000;
>> +	u32 next_timing_rate_mhz = next_timing->rate / 1000;
>> +	int dvfs_pt1 = type == DVFS_PT1;
>> +	int dvfs_update = type == DVFS_UPDATE;
>> +	int periodic_training_update = type == PERIODIC_TRAINING_UPDATE;
>> +
>> +	/*
>> +	 * Dev0 MSB.
>> +	 */
>> +	if (dvfs_pt1 || periodic_training_update) {
>> +		mrr_req = (2 << EMC_MRR_DEV_SEL_SHIFT) |
>> +			(19 << EMC_MRR_MA_SHIFT);
>> +		emc_writel(emc, mrr_req, EMC_MRR);
>> +
>> +		WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +				     EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
>> +		     "Timed out waiting for MRR 19 (ch=0)\n");
>> +		if (channel_mode == DUAL_CHANNEL)
>> +			WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +					     EMC_EMC_STATUS_MRR_DIVLD, 1,
>> +					     REG_EMC1),
>> +			     "Timed out waiting for MRR 19 (ch=1)\n");
>> +
>> +		mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
>> +			   EMC_MRR_DATA_SHIFT;
>> +
>> +		temp0_0 = (mrr_data & 0xff) << 8;
>> +		temp0_1 = mrr_data & 0xff00;
>> +
>> +		if (channel_mode == DUAL_CHANNEL) {
>> +			mrr_data = (emc1_readl(emc, EMC_MRR) &
>> +				    EMC_MRR_DATA_MASK) <<
>> +				   EMC_MRR_DATA_SHIFT;
>> +			temp1_0 = (mrr_data & 0xff) << 8;
>> +			temp1_1 = mrr_data & 0xff00;
>> +		}
>> +
>> +		/*
>> +		 * Dev0 LSB.
>> +		 */
>> +		mrr_req = (mrr_req & ~EMC_MRR_MA_MASK) |
>> +			  (18 << EMC_MRR_MA_SHIFT);
>> +		emc_writel(emc, mrr_req, EMC_MRR);
>> +
>> +		WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +				     EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
>> +		     "Timed out waiting for MRR 18 (ch=0)\n");
>> +		if (channel_mode == DUAL_CHANNEL)
>> +			WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +					     EMC_EMC_STATUS_MRR_DIVLD, 1,
>> +					     REG_EMC1),
>> +			     "Timed out waiting for MRR 18 (ch=1)\n");
>> +
>> +		mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
>> +			   EMC_MRR_DATA_SHIFT;
>> +
>> +		temp0_0 |= mrr_data & 0xff;
>> +		temp0_1 |= (mrr_data & 0xff00) >> 8;
>> +
>> +		if (channel_mode == DUAL_CHANNEL) {
>> +			mrr_data = (emc1_readl(emc, EMC_MRR) &
>> +				    EMC_MRR_DATA_MASK) <<
>> +				   EMC_MRR_DATA_SHIFT;
>> +			temp1_0 |= (mrr_data & 0xff);
>> +			temp1_1 |= (mrr_data & 0xff00) >> 8;
>> +		}
>> +	}
>> +
>> +	if (dvfs_pt1 || periodic_training_update)
>> +		cval = (1000000 * tegra210_actual_osc_clocks(
>> +					last_timing->run_clocks)) /
>> +		       (last_timing_rate_mhz * 2 * temp0_0);
>> +
>> +	if (dvfs_pt1)
>> +		__INCREMENT_PTFV(C0D0U0, cval);
>> +	else if (dvfs_update)
>> +		__AVERAGE_PTFV(C0D0U0);
>> +	else if (periodic_training_update)
>> +		__WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
>> +
>> +	if (dvfs_update || periodic_training_update) {
>> +		tdel = next_timing->current_dram_clktree_c0d0u0 -
>> +			__MOVAVG_AC(next_timing, C0D0U0);
>> +		tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +		adel = tmdel;
>> +		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +		    next_timing->tree_margin)
>> +			next_timing->current_dram_clktree_c0d0u0 =
>> +				__MOVAVG_AC(next_timing, C0D0U0);
>> +	}
>> +
>> +	if (dvfs_pt1 || periodic_training_update)
>> +		cval = (1000000 * tegra210_actual_osc_clocks(
>> +					last_timing->run_clocks)) /
>> +		       (last_timing_rate_mhz * 2 * temp0_1);
>> +
>> +	if (dvfs_pt1)
>> +		__INCREMENT_PTFV(C0D0U1, cval);
>> +	else if (dvfs_update)
>> +		__AVERAGE_PTFV(C0D0U1);
>> +	else if (periodic_training_update)
>> +		__WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
>> +
>> +	if (dvfs_update || periodic_training_update) {
>> +		tdel = next_timing->current_dram_clktree_c0d0u1 -
>> +			__MOVAVG_AC(next_timing, C0D0U1);
>> +		tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +
>> +		if (tmdel > adel)
>> +			adel = tmdel;
>> +
>> +		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +		    next_timing->tree_margin)
>> +			next_timing->current_dram_clktree_c0d0u1 =
>> +				__MOVAVG_AC(next_timing, C0D0U1);
>> +	}
>> +
>> +	if (channel_mode == DUAL_CHANNEL) {
>> +		if (dvfs_pt1 || periodic_training_update)
>> +			cval = (1000000 * tegra210_actual_osc_clocks(
>> +						last_timing->run_clocks)) /
>> +			       (last_timing_rate_mhz * 2 * temp1_0);
>> +		if (dvfs_pt1)
>> +			__INCREMENT_PTFV(C1D0U0, cval);
>> +		else if (dvfs_update)
>> +			__AVERAGE_PTFV(C1D0U0);
>> +		else if (periodic_training_update)
>> +			__WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
>> +
>> +		if (dvfs_update || periodic_training_update) {
>> +			tdel = next_timing->current_dram_clktree_c1d0u0 -
>> +				__MOVAVG_AC(next_timing, C1D0U0);
>> +			tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +
>> +			if (tmdel > adel)
>> +				adel = tmdel;
>> +
>> +			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +			    next_timing->tree_margin)
>> +				next_timing->current_dram_clktree_c1d0u0 =
>> +					__MOVAVG_AC(next_timing, C1D0U0);
>> +		}
>> +
>> +		if (dvfs_pt1 || periodic_training_update)
>> +			cval = (1000000 * tegra210_actual_osc_clocks(
>> +						last_timing->run_clocks)) /
>> +			       (last_timing_rate_mhz * 2 * temp1_1);
>> +		if (dvfs_pt1)
>> +			__INCREMENT_PTFV(C1D0U1, cval);
>> +		else if (dvfs_update)
>> +			__AVERAGE_PTFV(C1D0U1);
>> +		else if (periodic_training_update)
>> +			__WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
>> +
>> +		if (dvfs_update || periodic_training_update) {
>> +			tdel = next_timing->current_dram_clktree_c1d0u1 -
>> +				__MOVAVG_AC(next_timing, C1D0U1);
>> +			tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +
>> +			if (tmdel > adel)
>> +				adel = tmdel;
>> +
>> +			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +			    next_timing->tree_margin)
>> +				next_timing->current_dram_clktree_c1d0u1 =
>> +					__MOVAVG_AC(next_timing, C1D0U1);
>> +		}
>> +	}
>> +
>> +	if (emc->dram_dev_num != TWO_RANK)
>> +		goto done;
>> +
>> +	/*
>> +	 * Dev1 MSB.
>> +	 */
>> +	if (dvfs_pt1 || periodic_training_update) {
>> +		mrr_req = (1 << EMC_MRR_DEV_SEL_SHIFT) |
>> +			  (19 << EMC_MRR_MA_SHIFT);
>> +		emc_writel(emc, mrr_req, EMC_MRR);
>> +
>> +		WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +				     EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
>> +		     "Timed out waiting for MRR 19 (ch=0)\n");
>> +		if (channel_mode == DUAL_CHANNEL)
>> +			WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +					     EMC_EMC_STATUS_MRR_DIVLD, 1,
>> +					     REG_EMC1),
>> +			     "Timed out waiting for MRR 19 (ch=1)\n");
>> +
>> +		mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
>> +			   EMC_MRR_DATA_SHIFT;
>> +
>> +		temp0_0 = (mrr_data & 0xff) << 8;
>> +		temp0_1 = mrr_data & 0xff00;
>> +
>> +		if (channel_mode == DUAL_CHANNEL) {
>> +			mrr_data = (emc1_readl(emc, EMC_MRR) &
>> +				    EMC_MRR_DATA_MASK) <<
>> +				   EMC_MRR_DATA_SHIFT;
>> +			temp1_0 = (mrr_data & 0xff) << 8;
>> +			temp1_1 = mrr_data & 0xff00;
>> +		}
>> +
>> +		/*
>> +		 * Dev1 LSB.
>> +		 */
>> +		mrr_req = (mrr_req & ~EMC_MRR_MA_MASK) |
>> +			  (18 << EMC_MRR_MA_SHIFT);
>> +		emc_writel(emc, mrr_req, EMC_MRR);
>> +
>> +		WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +				     EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
>> +		     "Timed out waiting for MRR 18 (ch=0)\n");
>> +		if (channel_mode == DUAL_CHANNEL)
>> +			WARN(wait_for_update(emc, EMC_EMC_STATUS,
>> +					     EMC_EMC_STATUS_MRR_DIVLD, 1,
>> +					     REG_EMC1),
>> +			     "Timed out waiting for MRR 18 (ch=1)\n");
>> +
>> +		mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
>> +			   EMC_MRR_DATA_SHIFT;
>> +
>> +		temp0_0 |= mrr_data & 0xff;
>> +		temp0_1 |= (mrr_data & 0xff00) >> 8;
>> +
>> +		if (channel_mode == DUAL_CHANNEL) {
>> +			mrr_data = (emc1_readl(emc, EMC_MRR) &
>> +				    EMC_MRR_DATA_MASK) <<
>> +				   EMC_MRR_DATA_SHIFT;
>> +			temp1_0 |= (mrr_data & 0xff);
>> +			temp1_1 |= (mrr_data & 0xff00) >> 8;
>> +		}
>> +	}
>> +
>> +	if (dvfs_pt1 || periodic_training_update)
>> +		cval = (1000000 * tegra210_actual_osc_clocks(
>> +					last_timing->run_clocks)) /
>> +		       (last_timing_rate_mhz * 2 * temp0_0);
>> +
>> +	if (dvfs_pt1)
>> +		__INCREMENT_PTFV(C0D1U0, cval);
>> +	else if (dvfs_update)
>> +		__AVERAGE_PTFV(C0D1U0);
>> +	else if (periodic_training_update)
>> +		__WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
>> +
>> +	if (dvfs_update || periodic_training_update) {
>> +		tdel = next_timing->current_dram_clktree_c0d1u0 -
>> +		       __MOVAVG_AC(next_timing, C0D1U0);
>> +		tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +		if (tmdel > adel)
>> +			adel = tmdel;
>> +
>> +		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +		    next_timing->tree_margin)
>> +			next_timing->current_dram_clktree_c0d1u0 =
>> +				__MOVAVG_AC(next_timing, C0D1U0);
>> +	}
>> +
>> +	if (dvfs_pt1 || periodic_training_update)
>> +		cval = (1000000 * tegra210_actual_osc_clocks(
>> +					last_timing->run_clocks)) /
>> +		       (last_timing_rate_mhz * 2 * temp0_1);
>> +
>> +	if (dvfs_pt1)
>> +		__INCREMENT_PTFV(C0D1U1, cval);
>> +	else if (dvfs_update)
>> +		__AVERAGE_PTFV(C0D1U1);
>> +	else if (periodic_training_update)
>> +		__WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
>> +
>> +	if (dvfs_update || periodic_training_update) {
>> +		tdel = next_timing->current_dram_clktree_c0d1u1 -
>> +		       __MOVAVG_AC(next_timing, C0D1U1);
>> +		tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +		if (tmdel > adel)
>> +			adel = tmdel;
>> +
>> +		if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +		    next_timing->tree_margin)
>> +			next_timing->current_dram_clktree_c0d1u1 =
>> +				__MOVAVG_AC(next_timing, C0D1U1);
>> +	}
>> +
>> +	if (channel_mode == DUAL_CHANNEL) {
>> +		if (dvfs_pt1 || periodic_training_update)
>> +			cval = (1000000 * tegra210_actual_osc_clocks(
>> +						last_timing->run_clocks)) /
>> +			       (last_timing_rate_mhz * 2 * temp1_0);
>> +
>> +		if (dvfs_pt1)
>> +			__INCREMENT_PTFV(C1D1U0, cval);
>> +		else if (dvfs_update)
>> +			__AVERAGE_PTFV(C1D1U0);
>> +		else if (periodic_training_update)
>> +			__WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
>> +
>> +		if (dvfs_update || periodic_training_update) {
>> +			tdel = next_timing->current_dram_clktree_c1d1u0 -
>> +			       __MOVAVG_AC(next_timing, C1D1U0);
>> +			tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +			if (tmdel > adel)
>> +				adel = tmdel;
>> +
>> +			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +			    next_timing->tree_margin)
>> +				next_timing->current_dram_clktree_c1d1u0 =
>> +					__MOVAVG_AC(next_timing, C1D1U0);
>> +		}
>> +
>> +		if (dvfs_pt1 || periodic_training_update)
>> +			cval = (1000000 * tegra210_actual_osc_clocks(
>> +						last_timing->run_clocks)) /
>> +			       (last_timing_rate_mhz * 2 * temp1_1);
>> +
>> +		if (dvfs_pt1)
>> +			__INCREMENT_PTFV(C1D1U1, cval);
>> +		else if (dvfs_update)
>> +			__AVERAGE_PTFV(C1D1U1);
>> +		else if (periodic_training_update)
>> +			__WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
>> +
>> +		if (dvfs_update || periodic_training_update) {
>> +			tdel = next_timing->current_dram_clktree_c1d1u1 -
>> +			       __MOVAVG_AC(next_timing, C1D1U1);
>> +			tmdel = (tdel < 0) ? -1 * tdel : tdel;
>> +			if (tmdel > adel)
>> +				adel = tmdel;
>> +
>> +			if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
>> +			    next_timing->tree_margin)
>> +				next_timing->current_dram_clktree_c1d1u1 =
>> +					__MOVAVG_AC(next_timing, C1D1U1);
>> +		}
>> +	}
>> +
>> +done:
>> +	return adel;
>> +}
>> +
>> +static u32 periodic_compensation_handler(struct tegra_emc *emc, u32 type,
>> +					 u32 dram_dev_num,
>> +					 u32 channel_mode,
>> +					 struct emc_table *last_timing,
>> +					 struct emc_table *next_timing)
>> +{
>> +#define __COPY_EMA(nt, lt, dev)						\
>> +	({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) *			\
>> +	   (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
>> +
>> +	u32 i;
>> +	u32 adel = 0;
>> +	u32 samples = next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
>> +	u32 delay = 2 +
>> +		(1000 * tegra210_actual_osc_clocks(last_timing->run_clocks) /
>> +		last_timing->rate);
>> +
>> +	if (!next_timing->periodic_training)
>> +		return 0;
>> +
>> +	if (type == DVFS_SEQUENCE) {
>> +		if (last_timing->periodic_training &&
>> +		    (next_timing->ptfv_list[PTFV_CONFIG_CTRL_INDEX] &
>> +		     PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
>> +			/*
>> +			 * If the previous frequency was using periodic
>> +			 * calibration then we can reuse the previous
>> +			 * frequencies EMA data.
>> +			 */
>> +			__COPY_EMA(next_timing, last_timing, C0D0U0);
>> +			__COPY_EMA(next_timing, last_timing, C0D0U1);
>> +			__COPY_EMA(next_timing, last_timing, C1D0U0);
>> +			__COPY_EMA(next_timing, last_timing, C1D0U1);
>> +			__COPY_EMA(next_timing, last_timing, C0D1U0);
>> +			__COPY_EMA(next_timing, last_timing, C0D1U1);
>> +			__COPY_EMA(next_timing, last_timing, C1D1U0);
>> +			__COPY_EMA(next_timing, last_timing, C1D1U1);
>> +		} else {
>> +			/* Reset the EMA.*/
>> +			__MOVAVG(next_timing, C0D0U0) = 0;
>> +			__MOVAVG(next_timing, C0D0U1) = 0;
>> +			__MOVAVG(next_timing, C1D0U0) = 0;
>> +			__MOVAVG(next_timing, C1D0U1) = 0;
>> +			__MOVAVG(next_timing, C0D1U0) = 0;
>> +			__MOVAVG(next_timing, C0D1U1) = 0;
>> +			__MOVAVG(next_timing, C1D1U0) = 0;
>> +			__MOVAVG(next_timing, C1D1U1) = 0;
>> +
>> +			for (i = 0; i < samples; i++) {
>> +				tegra210_start_periodic_compensation(emc);
>> +				udelay(delay);
>> +
>> +				/*
>> +				 * Generate next sample of data.
>> +				 */
>> +				adel = update_clock_tree_delay(emc,
>> +							emc->dram_dev_num,
>> +							channel_mode,
>> +							DVFS_PT1);
>> +			}
>> +		}
>> +
>> +		/*
>> +		 * Seems like it should be part of the
>> +		 * 'if (last_timing->periodic_training)' conditional
>> +		 * since is already done for the else clause.
>> +		 */
>> +		adel = update_clock_tree_delay(emc,
>> +					       emc->dram_dev_num,
>> +					       channel_mode,
>> +					       DVFS_UPDATE);
>> +	}
>> +
>> +	if (type == PERIODIC_TRAINING_SEQUENCE) {
>> +		tegra210_start_periodic_compensation(emc);
>> +		udelay(delay);
>> +
>> +		adel = update_clock_tree_delay(emc,
>> +					       emc->dram_dev_num,
>> +					       channel_mode,
>> +					       PERIODIC_TRAINING_UPDATE);
>> +	}
>> +
>> +	return adel;
>> +}
>> +
>> +u32 __do_periodic_emc_compensation_r21021(struct tegra_emc *emc)
>> +{
>> +	u32 dram_dev_num;
>> +	u32 channel_mode;
>> +	u32 emc_cfg, emc_cfg_o;
>> +	u32 emc_dbg_o;
>> +	u32 del, i;
>> +	u32 list[] = {
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
>> +		EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
>> +		EMC_DATA_BRLSHFT_0,
>> +		EMC_DATA_BRLSHFT_1
>> +	};
>> +	u32 items = ARRAY_SIZE(list);
>> +	u32 emc_cfg_update;
>> +	struct emc_table *current_timing = emc->current_timing;
>> +
>> +	if (current_timing->periodic_training) {
>> +		channel_mode =
>> +			!!(current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] &
>> +			(1 << 2));
>> +		dram_dev_num = 1 + (mc_readl(emc->mc, MC_EMEM_ADR_CFG) & 0x1);
>> +
>> +		emc_cc_dbg(PER_TRAIN, "Periodic training starting\n");
>> +
>> +		emc_dbg_o = emc_readl(emc, EMC_DBG);
>> +		emc_cfg_o = emc_readl(emc, EMC_CFG);
>> +		emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF |
>> +					EMC_CFG_DRAM_ACPD |
>> +					EMC_CFG_DRAM_CLKSTOP_PD |
>> +					EMC_CFG_DRAM_CLKSTOP_PD);
>> +
>> +
>> +		/*
>> +		 * 1. Power optimizations should be off.
>> +		 */
>> +		emc_writel(emc, emc_cfg, EMC_CFG);
>> +
>> +		/* Does emc_timing_update() for above changes. */
>> +		tegra210_dll_disable(emc, channel_mode);
>> +
>> +		wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 0,
>> +				REG_EMC);
>> +		if (channel_mode)
>> +			wait_for_update(emc, EMC_EMC_STATUS,
>> +					EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
>> +					0, REG_EMC1);
>> +
>> +		wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
>> +				REG_EMC);
>> +		if (channel_mode)
>> +			wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
>> +				REG_EMC1);
>> +
>> +		emc_cfg_update = emc_readl(emc, EMC_CFG_UPDATE);
>> +		emc_writel(emc, (emc_cfg_update &
>> +			    ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK) |
>> +			   (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT),
>> +			   EMC_CFG_UPDATE);
>> +
>> +		/*
>> +		 * 2. osc kick off - this assumes training and dvfs have set
>> +		 *    correct MR23.
>> +		 */
>> +		tegra210_start_periodic_compensation(emc);
>> +
>> +		/*
>> +		 * 3. Let dram capture its clock tree delays.
>> +		 */
>> +		udelay((tegra210_actual_osc_clocks(current_timing->run_clocks) *
>> +			1000) /
>> +		       current_timing->rate + 1);
>> +
>> +		/*
>> +		 * 4. Check delta wrt previous values (save value if margin
>> +		 *    exceeds what is set in table).
>> +		 */
>> +		del = periodic_compensation_handler(emc,
>> +						    PERIODIC_TRAINING_SEQUENCE,
>> +						    dram_dev_num,
>> +						    channel_mode,
>> +						    current_timing,
>> +						    current_timing);
>> +
>> +		/*
>> +		 * 5. Apply compensation w.r.t. trained values (if clock tree
>> +		 *    has drifted more than the set margin).
>> +		 */
>> +		if (current_timing->tree_margin <
>> +		    ((del * 128 * (current_timing->rate / 1000)) / 1000000)) {
>> +			for (i = 0; i < items; i++) {
>> +				u32 tmp =
>> +				tegra210_apply_periodic_compensation_trimmer(
>> +				current_timing, list[i]);
>> +
>> +				emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
>> +					   list[i], tmp);
>> +				emc_writel(emc, tmp, list[i]);
>> +			}
>> +		}
>> +
>> +		emc_writel(emc, emc_cfg_o, EMC_CFG);
>> +
>> +		/*
>> +		 * 6. Timing update actally applies the new trimmers.
>> +		 */
>> +		emc_timing_update(emc, channel_mode);
>> +
>> +		/* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
>> +		emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
>> +
>> +		/* 6.2. Restore the DLL. */
>> +		tegra210_dll_enable(emc, channel_mode);
>> +
>> +		/*
>> +		 * 7. Copy over the periodic training registers that we updated
>> +		 *    here to the corresponding derated/non-derated table.
>> +		 */
>> +		tegra210_update_emc_alt_timing(emc, current_timing);
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +/*
>> + * Do the clock change sequence.
>> + */
>> +void emc_set_clock_r21021(struct tegra_emc *emc, u32 clksrc)
>> +{
>> +	/*
>> +	 * This is the timing table for the source frequency. It does _not_
>> +	 * necessarily correspond to the actual timing values in the EMC at the
>> +	 * moment. If the boot BCT differs from the table then this can happen.
>> +	 * However, we need it for accessing the dram_timings (which are not
>> +	 * really registers) array for the current frequency.
>> +	 */
>> +	struct emc_table *fake_timing;
>> +	struct emc_table *last_timing = emc->current_timing;
>> +	struct emc_table *next_timing = emc->next_timing;
>> +
>> +	u32 i, tmp;
>> +
>> +	u32 cya_allow_ref_cc = 0, ref_b4_sref_en = 0, cya_issue_pc_ref = 0;
>> +
>> +	u32 zqcal_before_cc_cutoff = 2400; /* In picoseconds */
>> +	u32 ref_delay_mult;
>> +	u32 ref_delay;
>> +	s32 zq_latch_dvfs_wait_time;
>> +	s32 tZQCAL_lpddr4_fc_adj;
>> +	/* Scaled by x1000 */
>> +	u32 tFC_lpddr4 = 1000 * next_timing->dram_timings[T_FC_LPDDR4];
>> +	u32 tZQCAL_lpddr4 = 1000000;
>> +
>> +	u32 dram_type, dram_dev_num, shared_zq_resistor;
>> +	u32 channel_mode;
>> +	u32 is_lpddr3;
>> +
>> +	u32 emc_cfg, emc_sel_dpd_ctrl, emc_cfg_reg;
>> +
>> +	u32 emc_dbg;
>> +	u32 emc_zcal_interval;
>> +	u32 emc_zcal_wait_cnt_old;
>> +	u32 emc_zcal_wait_cnt_new;
>> +	u32 emc_dbg_active;
>> +	u32 zq_op;
>> +	u32 zcal_wait_time_clocks;
>> +	u32 zcal_wait_time_ps;
>> +
>> +	u32 emc_auto_cal_config;
>> +	u32 auto_cal_en;
>> +
>> +	u32 mr13_catr_enable;
>> +
>> +	u32 ramp_up_wait = 0, ramp_down_wait = 0;
>> +
>> +	/* In picoseconds. */
>> +	u32 source_clock_period;
>> +	u32 destination_clock_period;
>> +
>> +	u32 emc_dbg_o;
>> +	u32 emc_cfg_pipe_clk_o;
>> +	u32 emc_pin_o;
>> +
>> +	u32 mr13_flip_fspwr;
>> +	u32 mr13_flip_fspop;
>> +
>> +	u32 opt_zcal_en_cc;
>> +	u32 opt_do_sw_qrst = 1;
>> +	u32 opt_dvfs_mode;
>> +	u32 opt_dll_mode;
>> +	u32 opt_cc_short_zcal = 1;
>> +	u32 opt_short_zcal = 1;
>> +	u32 save_restore_clkstop_pd = 1;
>> +
>> +	u32 prelock_dll_en = 0, dll_out;
>> +
>> +	int next_push, next_dq_e_ivref, next_dqs_e_ivref;
>> +
>> +	u32 opt_war_200024907;
>> +	u32 zq_wait_long;
>> +	u32 zq_wait_short;
>> +
>> +	u32 bg_regulator_switch_complete_wait_clks;
>> +	u32 bg_regulator_mode_change;
>> +	u32 enable_bglp_regulator;
>> +	u32 enable_bg_regulator;
>> +
>> +	u32 tRTM;
>> +	u32 RP_war;
>> +	u32 R2P_war;
>> +	u32 TRPab_war;
>> +	s32 nRTP;
>> +	u32 deltaTWATM;
>> +	u32 W2P_war;
>> +	u32 tRPST;
>> +
>> +	u32 mrw_req;
>> +	u32 adel = 0, compensate_trimmer_applicable = 0;
>> +	u32 next_timing_rate_mhz = next_timing->rate / 1000;
>> +
>> +	static u32 fsp_for_next_freq;
>> +
>> +	emc_cc_dbg(INFO, "Running clock change.\n");
>> +
>> +	fake_timing = get_timing_from_freq(emc, last_timing->rate);
>> +
>> +	fsp_for_next_freq = !fsp_for_next_freq;
>> +
>> +	dram_type = emc_readl(emc, EMC_FBIO_CFG5) &
>> +		    EMC_FBIO_CFG5_DRAM_TYPE_MASK >>
>> +		    EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
>> +	shared_zq_resistor = last_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] &
>> +			     1 << 31;
>> +	channel_mode = !!(last_timing->burst_regs[EMC_FBIO_CFG7_INDEX] &
>> +			  1 << 2);
>> +	opt_zcal_en_cc = (next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX] &&
>> +			  !last_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX]) ||
>> +			  dram_type == DRAM_TYPE_LPDDR4;
>> +	opt_dll_mode = (dram_type == DRAM_TYPE_DDR3) ?
>> +		       get_dll_state(next_timing) : DLL_OFF;
>> +	is_lpddr3 = (dram_type == DRAM_TYPE_LPDDR2) &&
>> +		    next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] &
>> +		    1 << 25;
>> +	opt_war_200024907 = (dram_type == DRAM_TYPE_LPDDR4);
>> +	opt_dvfs_mode = MAN_SR;
>> +	dram_dev_num = (mc_readl(emc->mc, MC_EMEM_ADR_CFG) & 0x1) + 1;
>> +
>> +	emc_cfg_reg = emc_readl(emc, EMC_CFG);
>> +	emc_auto_cal_config = emc_readl(emc, EMC_AUTO_CAL_CONFIG);
>> +
>> +	source_clock_period = 1000000000 / last_timing->rate;
>> +	destination_clock_period = 1000000000 / next_timing->rate;
>> +
>> +	tZQCAL_lpddr4_fc_adj = (destination_clock_period >
>> +				zqcal_before_cc_cutoff) ?
>> +		tZQCAL_lpddr4 / destination_clock_period :
>> +		(tZQCAL_lpddr4 - tFC_lpddr4) / destination_clock_period;
>> +	emc_dbg_o = emc_readl(emc, EMC_DBG);
>> +	emc_pin_o = emc_readl(emc, EMC_PIN);
>> +	emc_cfg_pipe_clk_o = emc_readl(emc, EMC_CFG_PIPE_CLK);
>> +	emc_dbg = emc_dbg_o;
>> +
>> +	emc_cfg = next_timing->burst_regs[EMC_CFG_INDEX];
>> +	emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD |
>> +		     EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
>> +	emc_sel_dpd_ctrl = next_timing->emc_sel_dpd_ctrl;
>> +	emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN |
>> +			      EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN |
>> +			      EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN |
>> +			      EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN |
>> +			      EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
>> +
>> +	emc_cc_dbg(INFO, "Clock change version: %d\n",
>> +		   DVFS_CLOCK_CHANGE_VERSION);
>> +	emc_cc_dbg(INFO, "DRAM type = %d\n", emc->dram_type);
>> +	emc_cc_dbg(INFO, "DRAM dev #: %d\n", dram_dev_num);
>> +	emc_cc_dbg(INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
>> +	emc_cc_dbg(INFO, "DLL clksrc:      0x%08x\n", next_timing->dll_clk_src);
>> +	emc_cc_dbg(INFO, "last rate: %u, next rate %u\n", last_timing->rate,
>> +		   next_timing->rate);
>> +	emc_cc_dbg(INFO, "last period: %u, next period: %u\n",
>> +		   source_clock_period, destination_clock_period);
>> +	emc_cc_dbg(INFO, "  shared_zq_resistor: %d\n", !!shared_zq_resistor);
>> +	emc_cc_dbg(INFO, "  channel_mode: %d\n", channel_mode);
>> +	emc_cc_dbg(INFO, "  opt_dll_mode: %d\n", opt_dll_mode);
>> +
>> +	/*
>> +	 * Step 1:
>> +	 *   Pre DVFS SW sequence.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 1\n");
>> +	emc_cc_dbg(STEPS, "Step 1.1: Disable DLL temporarily.\n");
>> +	tmp = emc_readl(emc, EMC_CFG_DIG_DLL);
>> +	tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
>> +	emc_writel(emc, tmp, EMC_CFG_DIG_DLL);
>> +
>> +	emc_timing_update(emc, channel_mode);
>> +	wait_for_update(emc, EMC_CFG_DIG_DLL,
>> +			EMC_CFG_DIG_DLL_CFG_DLL_EN, 0, REG_EMC);
>> +	if (channel_mode)
>> +		wait_for_update(emc, EMC_CFG_DIG_DLL,
>> +				EMC_CFG_DIG_DLL_CFG_DLL_EN, 0, REG_EMC1);
>> +
>> +	emc_cc_dbg(STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
>> +	emc_auto_cal_config = next_timing->emc_auto_cal_config;
>> +	auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
>> +	emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
>> +	emc_auto_cal_config |=  EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
>> +	emc_auto_cal_config |=  EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
>> +	emc_auto_cal_config |=  auto_cal_en;
>> +	emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
>> +	emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
>> +
>> +	emc_cc_dbg(STEPS, "Step 1.3: Disable other power features.\n");
>> +	emc_set_shadow_bypass(emc, ACTIVE);
>> +	emc_writel(emc, emc_cfg, EMC_CFG);
>> +	emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
>> +	emc_set_shadow_bypass(emc, ASSEMBLY);
>> +
>> +	if (next_timing->periodic_training) {
>> +		tegra210_reset_dram_clktree_values(next_timing);
>> +
>> +		wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 0,
>> +				REG_EMC);
>> +		if (channel_mode)
>> +			wait_for_update(emc, EMC_EMC_STATUS,
>> +					EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
>> +					0, REG_EMC1);
>> +
>> +		wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
>> +				REG_EMC);
>> +		if (channel_mode)
>> +			wait_for_update(emc, EMC_EMC_STATUS,
>> +				EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
>> +				REG_EMC1);
>> +
>> +		tegra210_start_periodic_compensation(emc);
>> +
>> +		udelay(((1000 *
>> +			 tegra210_actual_osc_clocks(last_timing->run_clocks)) /
>> +			last_timing->rate) + 2);
>> +		adel = periodic_compensation_handler(emc, DVFS_SEQUENCE,
>> +						     dram_dev_num,
>> +						     channel_mode,
>> +						     fake_timing, next_timing);
>> +		compensate_trimmer_applicable =
>> +			next_timing->periodic_training &&
>> +			((adel * 128 * next_timing_rate_mhz) / 1000000) >
>> +			next_timing->tree_margin;
>> +	}
>> +
>> +	emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
>> +	emc_set_shadow_bypass(emc, ACTIVE);
>> +	emc_writel(emc, emc_cfg, EMC_CFG);
>> +	emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
>> +	emc_writel(emc, emc_cfg_pipe_clk_o | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
>> +		   EMC_CFG_PIPE_CLK);
>> +	emc_writel(emc, next_timing->emc_fdpd_ctrl_cmd_no_ramp &
>> +		   ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE,
>> +		   EMC_FDPD_CTRL_CMD_NO_RAMP);
>> +
>> +	bg_regulator_mode_change =
>> +		((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		  EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^
>> +		 (last_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		  EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) ||
>> +		((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		  EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^
>> +		 (last_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		  EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
>> +	enable_bglp_regulator =
>> +		(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		 EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0;
>> +	enable_bg_regulator =
>> +		(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +		 EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
>> +
>> +	if (bg_regulator_mode_change) {
>> +		if (enable_bg_regulator)
>> +			emc_writel(emc, last_timing->burst_regs
>> +				   [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +				   ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
>> +				   EMC_PMACRO_BG_BIAS_CTRL_0);
>> +		else
>> +			emc_writel(emc, last_timing->burst_regs
>> +				   [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
>> +				   ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
>> +				   EMC_PMACRO_BG_BIAS_CTRL_0);
>> +	}
>> +
>> +	/* Check if we need to turn on VREF generator. */
>> +	if ((((last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
>> +	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) &&
>> +	     ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
>> +	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) ||
>> +	    (((last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
>> +	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) &&
>> +	     ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
>> +	       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 1))) {
>> +		u32 pad_tx_ctrl =
>> +		    next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
>> +		u32 last_pad_tx_ctrl =
>> +		    last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
>> +
>> +		next_dqs_e_ivref = pad_tx_ctrl &
>> +				   EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
>> +		next_dq_e_ivref = pad_tx_ctrl &
>> +				  EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
>> +		next_push = (last_pad_tx_ctrl &
>> +			     ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF &
>> +			     ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) |
>> +			    next_dq_e_ivref | next_dqs_e_ivref;
>> +		emc_writel(emc, next_push, EMC_PMACRO_DATA_PAD_TX_CTRL);
>> +		udelay(1);
>> +	} else if (bg_regulator_mode_change) {
>> +		udelay(1);
>> +	}
>> +
>> +	emc_set_shadow_bypass(emc, ASSEMBLY);
>> +
>> +	/*
>> +	 * Step 2:
>> +	 *   Prelock the DLL.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 2\n");
>> +	if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] &
>> +	    EMC_CFG_DIG_DLL_CFG_DLL_EN) {
>> +		emc_cc_dbg(INFO, "Prelock enabled for target frequency.\n");
>> +		dll_out = tegra210_dll_prelock(emc, 0, clksrc);
>> +		emc_cc_dbg(INFO, "DLL out: 0x%03x\n", dll_out);
>> +		prelock_dll_en = 1;
>> +	} else {
>> +		emc_cc_dbg(INFO, "Disabling DLL for target frequency.\n");
>> +		tegra210_dll_disable(emc, channel_mode);
>> +	}
>> +
>> +	/*
>> +	 * Step 3:
>> +	 *   Prepare autocal for the clock change.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 3\n");
>> +	emc_set_shadow_bypass(emc, ACTIVE);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config2,
>> +		   EMC_AUTO_CAL_CONFIG2);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config3,
>> +		   EMC_AUTO_CAL_CONFIG3);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config4,
>> +		   EMC_AUTO_CAL_CONFIG4);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config5,
>> +		   EMC_AUTO_CAL_CONFIG5);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config6,
>> +		   EMC_AUTO_CAL_CONFIG6);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config7,
>> +		   EMC_AUTO_CAL_CONFIG7);
>> +	emc_writel(emc, next_timing->emc_auto_cal_config8,
>> +		   EMC_AUTO_CAL_CONFIG8);
>> +	emc_set_shadow_bypass(emc, ASSEMBLY);
>> +
>> +	emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START |
>> +				auto_cal_en);
>> +	emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
>> +
>> +	/*
>> +	 * Step 4:
>> +	 *   Update EMC_CFG. (??)
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 4\n");
>> +	if (source_clock_period > 50000 && dram_type == DRAM_TYPE_LPDDR4)
>> +		ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
>> +	else
>> +		emc_writel(emc, next_timing->emc_cfg_2, EMC_CFG_2);
>> +
>> +	/*
>> +	 * Step 5:
>> +	 *   Prepare reference variables for ZQCAL regs.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 5\n");
>> +	emc_zcal_interval = 0;
>> +	emc_zcal_wait_cnt_old =
>> +		last_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
>> +	emc_zcal_wait_cnt_new =
>> +		next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
>> +	emc_zcal_wait_cnt_old &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
>> +	emc_zcal_wait_cnt_new &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
>> +
>> +	if (dram_type == DRAM_TYPE_LPDDR4)
>> +		zq_wait_long = max((u32)1,
>> +				   div_o3(1000000, destination_clock_period));
>> +	else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
>> +		zq_wait_long = max(next_timing->min_mrs_wait,
>> +				   div_o3(360000, destination_clock_period)) +
>> +			       4;
>> +	else if (dram_type == DRAM_TYPE_DDR3)
>> +		zq_wait_long = max((u32)256,
>> +				   div_o3(320000, destination_clock_period) +
>> +				   2);
>> +	else
>> +		zq_wait_long = 0;
>> +
>> +	if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
>> +		zq_wait_short = max(max(next_timing->min_mrs_wait, (u32)6),
>> +				    div_o3(90000, destination_clock_period)) +
>> +				4;
>> +	else if (dram_type == DRAM_TYPE_DDR3)
>> +		zq_wait_short = max((u32)64,
>> +				    div_o3(80000, destination_clock_period)) +
>> +				2;
>> +	else
>> +		zq_wait_short = 0;
>> +
>> +	/*
>> +	 * Step 6:
>> +	 *   Training code - removed.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 6\n");
>> +
>> +	/*
>> +	 * Step 7:
>> +	 *   Program FSP reference registers and send MRWs to new FSPWR.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 7\n");
>> +	emc_cc_dbg(SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
>> +	if (opt_war_200024907) {
>> +		nRTP = 16;
>> +		if (source_clock_period >= 1000000/1866) /* 535.91 ps */
>> +			nRTP = 14;
>> +		if (source_clock_period >= 1000000/1600) /* 625.00 ps */
>> +			nRTP = 12;
>> +		if (source_clock_period >= 1000000/1333) /* 750.19 ps */
>> +			nRTP = 10;
>> +		if (source_clock_period >= 1000000/1066) /* 938.09 ps */
>> +			nRTP = 8;
>> +
>> +		deltaTWATM = max_t(u32, div_o3(7500, source_clock_period), 8);
>> +
>> +		/*
>> +		 * Originally there was a + .5 in the tRPST calculation.
>> +		 * However since we can't do FP in the kernel and the tRTM
>> +		 * computation was in a floating point ceiling function, adding
>> +		 * one to tRTP should be ok. There is no other source of non
>> +		 * integer values, so the result was always going to be
>> +		 * something for the form: f_ceil(N + .5) = N + 1;
>> +		 */
>> +		tRPST = ((last_timing->emc_mrw & 0x80) >> 7);
>> +		tRTM = fake_timing->dram_timings[RL] +
>> +		       div_o3(3600, source_clock_period) +
>> +		       max_t(u32, div_o3(7500, source_clock_period), 8) +
>> +		       tRPST + 1 + nRTP;
>> +
>> +		emc_cc_dbg(INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
>> +			   next_timing->burst_regs[EMC_RP_INDEX]);
>> +
>> +		if (last_timing->burst_regs[EMC_RP_INDEX] < tRTM) {
>> +			if (tRTM > (last_timing->burst_regs[EMC_R2P_INDEX] +
>> +				    last_timing->burst_regs[EMC_RP_INDEX])) {
>> +				R2P_war = tRTM -
>> +					  last_timing->burst_regs[EMC_RP_INDEX];
>> +				RP_war = last_timing->burst_regs[EMC_RP_INDEX];
>> +				TRPab_war = last_timing->burst_regs[
>> +					    EMC_TRPAB_INDEX];
>> +				if (R2P_war > 63) {
>> +					RP_war = R2P_war +
>> +						 last_timing->burst_regs[
>> +						 EMC_RP_INDEX] - 63;
>> +					if (TRPab_war < RP_war)
>> +						TRPab_war = RP_war;
>> +					R2P_war = 63;
>> +				}
>> +			} else {
>> +				R2P_war = last_timing->burst_regs[
>> +					  EMC_R2P_INDEX];
>> +				RP_war = last_timing->burst_regs[EMC_RP_INDEX];
>> +				TRPab_war = last_timing->burst_regs[
>> +					    EMC_TRPAB_INDEX];
>> +			}
>> +
>> +			if (RP_war < deltaTWATM) {
>> +				W2P_war = last_timing->burst_regs[EMC_W2P_INDEX]
>> +					  + deltaTWATM - RP_war;
>> +				if (W2P_war > 63) {
>> +					RP_war = RP_war + W2P_war - 63;
>> +					if (TRPab_war < RP_war)
>> +						TRPab_war = RP_war;
>> +					W2P_war = 63;
>> +				}
>> +			} else {
>> +				W2P_war = last_timing->burst_regs[
>> +					  EMC_W2P_INDEX];
>> +			}
>> +
>> +			if ((last_timing->burst_regs[EMC_W2P_INDEX] ^
>> +			     W2P_war) ||
>> +			    (last_timing->burst_regs[EMC_R2P_INDEX] ^
>> +			     R2P_war) ||
>> +			    (last_timing->burst_regs[EMC_RP_INDEX] ^
>> +			     RP_war) ||
>> +			    (last_timing->burst_regs[EMC_TRPAB_INDEX] ^
>> +			     TRPab_war)) {
>> +				emc_writel(emc, RP_war, EMC_RP);
>> +				emc_writel(emc, R2P_war, EMC_R2P);
>> +				emc_writel(emc, W2P_war, EMC_W2P);
>> +				emc_writel(emc, TRPab_war, EMC_TRPAB);
>> +			}
>> +			emc_timing_update(emc, DUAL_CHANNEL);
>> +		} else {
>> +			emc_cc_dbg(INFO, "Skipped WAR\n");
>> +		}
>> +	}
>> +
>> +	if (!fsp_for_next_freq) {
>> +		mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x80;
>> +		mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0x00;
>> +	} else {
>> +		mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x40;
>> +		mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0xc0;
>> +	}
>> +
>> +	mr13_catr_enable = (mr13_flip_fspwr & 0xFFFFFFFE) | 0x01;
>> +	if (dram_dev_num == TWO_RANK)
>> +		mr13_catr_enable = (mr13_catr_enable & 0x3fffffff) | 0x80000000;
>> +
>> +	if (dram_type == DRAM_TYPE_LPDDR4) {
>> +		emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
>> +		emc_writel(emc, next_timing->emc_mrw, EMC_MRW);
>> +		emc_writel(emc, next_timing->emc_mrw2, EMC_MRW2);
>> +	}
>> +
>> +	/*
>> +	 * Step 8:
>> +	 *   Program the shadow registers.
>> +	 */
>> +	emc_cc_dbg(STEPS, "Step 8\n");
>> +	emc_cc_dbg(SUB_STEPS, "Writing burst_regs\n");
>> +	for (i = 0; i < next_timing->num_burst; i++) {
>> +		u32 var;
>> +		u32 wval;
>> +
>> +		if (!burst_regs_off[i])
>> +			continue;
>> +
>> +		var = burst_regs_off[i];
>> +		wval = next_timing->burst_regs[i];
>> +
>> +		if (dram_type != DRAM_TYPE_LPDDR4 &&
>> +		    (var == EMC_MRW6      || var == EMC_MRW7 ||
>> +		     var == EMC_MRW8      || var == EMC_MRW9 ||
>> +		     var == EMC_MRW10     || var == EMC_MRW11 ||
>> +		     var == EMC_MRW12     || var == EMC_MRW13 ||
>> +		     var == EMC_MRW14     || var == EMC_MRW15 ||
>> +		     var == EMC_TRAINING_CTRL))
>> +			continue;
>> +
>> +		/* Pain... And suffering. */
>> +		if (var == EMC_CFG) {
>> +			wval &= ~EMC_CFG_DRAM_ACPD;
>> +			wval &= ~EMC_CFG_DYN_SELF_REF;
>> +			if (dram_type == DRAM_TYPE_LPDDR4) {
>> +				wval &= ~EMC_CFG_DRAM_CLKSTOP_SR;
>> +				wval &= ~EMC_CFG_DRAM_CLKSTOP_PD;
>> +			}
>> +		} else if (var == EMC_MRS_WAIT_CNT &&
>> +			   dram_type == DRAM_TYPE_LPDDR2 &&
>> +			   opt_zcal_en_cc && !opt_cc_short_zcal &&
>> +			   opt_short_zcal) {
>> +			wval = (wval & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
>> +					 EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
>> +			   ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
>> +			    EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
>> +		} else if (var == EMC_ZCAL_WAIT_CNT &&
>> +			   dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc &&
>> +			   !opt_cc_short_zcal && opt_short_zcal) {
>> +			wval = (wval & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
>> +					 EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT))
>> +			       | ((zq_wait_long &
>> +				   EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
>> +				  EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
>> +		} else if (var == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) {
>> +			wval = 0; /* EMC_ZCAL_INTERVAL reset value. */
>> +		} else if (var == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
>> +			wval |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
>> +		} else if (var == EMC_PMACRO_DATA_PAD_TX_CTRL) {
>> +			wval &=
>> +			     ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
>> +			       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
>> +			       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
>> +			       EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
>> +		} else if (var == EMC_PMACRO_CMD_PAD_TX_CTRL) {
>> +			wval |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
>> +			wval &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
>> +				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
>> +				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
>> +				  EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
>> +		} else if (var == EMC_PMACRO_BRICK_CTRL_RFU1) {
>> +			wval &= 0xf800f800;
>> +		} else if (var == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
>> +			wval &= 0xfffffff0;
>> +		}
>> +
>> +		emc_writel(emc, wval, var);
>> +	}
>> +
>> +	/* SW addition: do EMC refresh adjustment here. */
>> +	set_over_temp_timing(emc, next_timing, dram_over_temp_state);
>> +
>> +	if (dram_type == DRAM_TYPE_LPDDR4) {
>> +		mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) |
>> +			  (next_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
>> +		emc_writel(emc, mrw_req, EMC_MRW);
>> +	}
>> +
>> +	/* Per channel burst registers. */
>> +	emc_cc_dbg(SUB_STEPS, "Writing burst_regs_per_ch\n");
>> +	for (i = 0; i < next_timing->num_burst_per_ch; i++) {
>> +		if (!burst_regs_per_ch_off[i])
>> +			continue;
>> +
>> +		if (dram_type != DRAM_TYPE_LPDDR4 &&
>> +		    (burst_regs_per_ch_off[i] == EMC_MRW6 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW7 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW8 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW9 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW10 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW11 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW12 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW13 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW14 ||
>> +		     burst_regs_per_ch_off[i] == EMC_MRW15))
>> +			continue;
>> +
>> +		/* Filter out second channel if not in DUAL_CHANNEL mode. */
>> +		if (channel_mode != DUAL_CHANNEL &&
>> +		    burst_regs_per_ch_type[i] >= REG_EMC1)
>> +			continue;
>> +
>> +		emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +			   i, next_timing->burst_reg_per_ch[i],
>> +			   burst_regs_per_ch_off[i]);
>> +		emc_writel_per_ch(emc, next_timing->burst_reg_per_ch[i],
>> +				  burst_regs_per_ch_type[i],
>> +				  burst_regs_per_ch_off[i]);
>> +	}
>> +
>> +	/* Vref regs. */
>> +	emc_cc_dbg(SUB_STEPS, "Writing vref_regs\n");
>> +	for (i = 0; i < next_timing->vref_num; i++) {
>> +		if (!vref_regs_per_ch_off[i])
>> +			continue;
>> +
>> +		if (channel_mode != DUAL_CHANNEL &&
>> +			vref_regs_per_ch_type[i] >= REG_EMC1)
>> +			continue;
>> +
>> +		emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +			   i, next_timing->vref_perch_regs[i],
>> +			   vref_regs_per_ch_off[i]);
>> +		emc_writel_per_ch(emc, next_timing->vref_perch_regs[i],
>> +				  vref_regs_per_ch_type[i],
>> +				  vref_regs_per_ch_off[i]);
>> +	}
>> +
>> +	/* Trimmers. */
>> +	emc_cc_dbg(SUB_STEPS, "Writing trim_regs\n");
>> +	for (i = 0; i < next_timing->num_trim; i++) {
>> +		u64 trim_reg;
>> +
>> +		if (!trim_regs_off[i])
>> +			continue;
>> +
>> +		trim_reg = trim_regs_off[i];
>> +		if (compensate_trimmer_applicable &&
>> +		    (trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
>> +		     trim_reg == EMC_DATA_BRLSHFT_0 ||
>> +		     trim_reg == EMC_DATA_BRLSHFT_1)) {
>> +			u32 reg = tegra210_apply_periodic_compensation_trimmer(
>> +				  next_timing, trim_reg);
>> +			emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, reg,
>> +				   trim_regs_off[i]);
>> +			emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
>> +				   (u32)(u64)trim_regs_off[i], reg);
>> +			emc_writel(emc, reg, trim_regs_off[i]);
>> +		} else {
>> +			emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +				   i, next_timing->trim_regs[i],
>> +				   trim_regs_off[i]);
>> +			emc_writel(emc, next_timing->trim_regs[i],
>> +				   trim_regs_off[i]);
>> +		}
>> +	}
>> +
>> +	/* Per channel trimmers. */
>> +	emc_cc_dbg(SUB_STEPS, "Writing trim_regs_per_ch\n");
>> +	for (i = 0; i < next_timing->num_trim_per_ch; i++) {
>> +		u32 trim_reg;
>> +
>> +		if (!trim_regs_per_ch_off[i])
>> +			continue;
>> +
>> +		if (channel_mode != DUAL_CHANNEL &&
>> +			trim_regs_per_ch_type[i] >= REG_EMC1)
>> +			continue;
>> +
>> +		trim_reg = trim_regs_per_ch_off[i];
>> +		if (compensate_trimmer_applicable &&
>> +		    (trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
>> +		     trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
>> +		     trim_reg == EMC_DATA_BRLSHFT_0 ||
>> +		     trim_reg == EMC_DATA_BRLSHFT_1)) {
>> +			u32 reg =
>> +				tegra210_apply_periodic_compensation_trimmer(
>> +						next_timing, trim_reg);
>> +			emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +				   i, reg, trim_regs_per_ch_off[i]);
>> +			emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
>> +				   trim_regs_per_ch_off[i], reg);
>> +			emc_writel_per_ch(emc, reg, trim_regs_per_ch_type[i],
>> +					  trim_regs_per_ch_off[i]);
>> +		} else {
>> +			emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +				   i, next_timing->trim_perch_regs[i],
>> +				   trim_regs_per_ch_off[i]);
>> +			emc_writel_per_ch(emc, next_timing->trim_perch_regs[i],
>> +					  trim_regs_per_ch_type[i],
>> +					  trim_regs_per_ch_off[i]);
>> +		}
>> +	}
>> +
>> +	emc_cc_dbg(SUB_STEPS, "Writing burst_mc_regs\n");
>> +	for (i = 0; i < next_timing->num_mc_regs; i++) {
>> +		emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +			   i, next_timing->burst_mc_regs[i],
>> +			   burst_mc_regs_off[i]);
>> +		mc_writel(emc->mc, next_timing->burst_mc_regs[i],
>> +			  burst_mc_regs_off[i]);
>> +	}
>> +
>> +	/* Registers to be programmed on the faster clock. */
>> +	if (next_timing->rate < last_timing->rate) {
>> +		emc_cc_dbg(SUB_STEPS, "Writing la_scale_regs\n");
>> +		for (i = 0; i < next_timing->num_up_down; i++) {
>> +			emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
>> +				   i, next_timing->la_scale_regs[i],
>> +				   la_scale_regs_off[i]);
>> +			mc_writel(emc->mc, next_timing->la_scale_regs[i],
>> +				  la_scale_regs_off[i]);
>> +		}
>> +	}
>> +
>> +	/* Flush all the burst register writes. */
>> +	wmb();
> 
> Won't it be a bit more optimal to just read back the lastly written register rather than to flush all of memory writes?
> 
Yes, should be fine. I'll give it a try.

Thanks,
Joseph