Haswell NRI: Use final timings after basic training

For "basic" (initial) memory training, we use larger-than-usual timings
for a few things, namely tCMD and tXP. After basic training is done, we
can switch to the final timings for the training steps that follow (not
many at the moment). Without this, NRI keeps using the training timings
at all times, which results in slightly lower performance (likely to go
unnoticed unless benchmarking the system).

Tested on Asrock B85M Pro4, still boots to Arch Linux.

Change-Id: I625f35adb02b36b1087cd758f983118d0a60b815
Signed-off-by: Angel Pons <th3fanbus@gmail.com>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/87831
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Benjamin Doron <benjamin.doron00@gmail.com>
Reviewed-by: David Hendricks <david.hendricks@gmail.com>

src/northbridge/intel/haswell/native_raminit/raminit_main.c

@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 
 #include <assert.h>
+#include <commonlib/bsd/clamp.h>
 #include <console/console.h>
 #include <cpu/intel/haswell/haswell.h>
 #include <delay.h>
@@ -48,6 +49,68 @@ static enum raminit_status pre_training(struct sysinfo *ctrl)
 	return RAMINIT_STATUS_SUCCESS;
 }
 
+
+static uint8_t get_cmd_stretch(const uint8_t tCMD)
+{
+	switch (tCMD) {
+	case 1:
+		return 0;
+	case 2:
+		return 2;
+	case 3:
+		return 3;
+	default:
+		return 2;
+	}
+}
+
+/** TODO: Why do we re-read the MCHBAR registers? Is the cached value outdated? **/
+static void update_command_rate(struct sysinfo *const ctrl, const uint8_t channel)
+{
+	const uint8_t old_n = clamp_s32(1, ctrl->tc_bankrank_a[channel].cmd_stretch, 3);
+	const uint8_t new_n = ctrl->tCMD;
+
+	/* Update CmdN timing */
+	ctrl->tc_bankrank_a[channel].raw = mchbar_read32(TC_BANK_RANK_A_ch(channel));
+	ctrl->tc_bankrank_a[channel].cmd_stretch = get_cmd_stretch(ctrl->tCMD);
+	mchbar_write32(TC_BANK_RANK_A_ch(channel), ctrl->tc_bankrank_a[channel].raw);
+
+	/* Switch to regular tXP value */
+	ctrl->tc_bankrank_c[channel].raw = mchbar_read32(TC_BANK_RANK_C_ch(channel));
+	ctrl->tc_bankrank_c[channel].tXP = ctrl->tXP;
+	mchbar_write32(TC_BANK_RANK_C_ch(channel), ctrl->tc_bankrank_c[channel].raw);
+
+	/* Adjust RT values to compensate */
+	const int8_t delta = new_n - old_n;
+	for (uint8_t rank = 0; rank < NUM_SLOTRANKS; rank++) {
+		if (!rank_in_ch(ctrl, rank, channel))
+			continue;
+
+		/*
+		 * RT (roundtrip) latency is the time it takes for signals to go
+		 * from the memory controller to the DRAM and back to the memory
+		 * controller. It's likely the delta is doubled for this reason.
+		 */
+		ctrl->rt_latency[channel].rank[rank] += delta * 2;
+	}
+	mchbar_write32(SC_ROUNDT_LAT_ch(channel), ctrl->rt_latency[channel].raw);
+}
+
+static enum raminit_status post_training(struct sysinfo *const ctrl)
+{
+	/* Command rate is always 1N for LPDDR3 */
+	if (ctrl->lpddr) {
+		return RAMINIT_STATUS_SUCCESS;
+	}
+	for (uint8_t channel = 0; channel < NUM_CHANNELS; channel++) {
+		if (!does_ch_exist(ctrl, channel))
+			continue;
+
+		update_command_rate(ctrl, channel);
+	}
+	return RAMINIT_STATUS_SUCCESS;
+}
+
 struct task_entry {
 	enum raminit_status (*task)(struct sysinfo *);
 	bool is_enabled;
@@ -66,6 +129,7 @@ static const struct task_entry cold_boot[] = {
 	{ train_receive_enable,                                   true, "RCVET",      },
 	{ train_read_mpr,                                         true, "RDMPRT",     },
 	{ train_jedec_write_leveling,                             true, "JWRL",       },
+	{ post_training,                                          true, "POSTTRAIN",  },
 	{ activate_mc,                                            true, "ACTIVATE",   },
 	{ save_training_values,                                   true, "SAVE_TRAIN", },
 	{ save_non_training,                                      true, "SAVE_NONT",  },

src/northbridge/intel/haswell/native_raminit/raminit_native.h

@@ -306,8 +306,9 @@ struct sysinfo {
 
 	union ddr_data_vref_adjust_reg dimm_vref;
 
+	union sc_roundt_lat_reg rt_latency[NUM_CHANNELS];
+
 	uint8_t io_latency[NUM_CHANNELS][NUM_SLOTRANKS];
-	uint8_t rt_latency[NUM_CHANNELS][NUM_SLOTRANKS];
 	uint32_t rt_io_comp[NUM_CHANNELS];
 
 	uint32_t data_offset_train[NUM_CHANNELS][NUM_LANES];

src/northbridge/intel/haswell/native_raminit/reg_structs.h

@@ -321,6 +321,21 @@ union ddr_scram_misc_control_reg {
 	uint32_t raw;
 };
 
+union sc_roundt_lat_reg {
+	struct __packed {
+		uint32_t R0D0 : 6; // Bits  5:0
+		uint32_t      : 2; // Bits  7:6
+		uint32_t R1D0 : 6; // Bits 13:8
+		uint32_t      : 2; // Bits 15:14
+		uint32_t R0D1 : 6; // Bits 21:16
+		uint32_t      : 2; // Bits 23:22
+		uint32_t R1D1 : 6; // Bits 29:24
+		uint32_t      : 2; // Bits 31:30
+	};
+	uint8_t rank[4];
+	uint32_t raw;
+};
+
 union sc_io_latency_reg {
 	struct __packed {
 		uint32_t iolat_rank0     : 4; // Bits  3:0

src/northbridge/intel/haswell/native_raminit/train_receive_enable.c

@@ -254,7 +254,7 @@ enum raminit_status train_receive_enable(struct sysinfo *ctrl)
 
 			ctrl->io_latency[channel][rank] = 0;
 			mchbar_write8(SC_ROUNDT_LAT_ch(channel) + rank, initial_rt_latency);
-			ctrl->rt_latency[channel][rank] = initial_rt_latency;
+			ctrl->rt_latency[channel].rank[rank] = initial_rt_latency;
 		}
 
 		printk(BIOS_DEBUG, "Rank %u\n", rank);