From: Sebastien Bourdeauducq <sb@m-labs.hk>
Date: Wed, 3 Sep 2014 06:25:26 +0000 (+0800)
Subject: bios: support DDR3 write leveling and read calibration. This makes the full DDR3... 
X-Git-Tag: 24jan2021_ls180~2643
X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=2388bfabc3b3496340ec51ad836a92e41352a1f9;p=litex.git

bios: support DDR3 write leveling and read calibration. This makes the full DDR3 SODIMM work on the KC705.
---

diff --git a/software/bios/main.c b/software/bios/main.c
index dfefe01b..29b141ce 100644
--- a/software/bios/main.c
+++ b/software/bios/main.c
@@ -382,6 +382,11 @@ static void do_command(char *c)
 	else if(strcmp(token, "sdrrd") == 0) sdrrd(get_token(&c), get_token(&c));
 	else if(strcmp(token, "sdrrderr") == 0) sdrrderr(get_token(&c));
 	else if(strcmp(token, "sdrwr") == 0) sdrwr(get_token(&c));
+#ifdef DDRPHY_BASE
+	else if(strcmp(token, "sdrwlon") == 0) sdrwlon();
+	else if(strcmp(token, "sdrwloff") == 0) sdrwloff();
+	else if(strcmp(token, "sdrlevel") == 0) sdrlevel();
+#endif
 	else if(strcmp(token, "memtest") == 0) memtest();
 	else if(strcmp(token, "sdrinit") == 0) sdrinit();
 #endif
diff --git a/software/bios/sdram.c b/software/bios/sdram.c
index 59215c36..24482ddf 100644
--- a/software/bios/sdram.c
+++ b/software/bios/sdram.c
@@ -114,11 +114,12 @@ void sdrrd(char *startaddr, char *dq)
 
 void sdrrderr(char *count)
 {
+	int addr;
 	char *c;
 	int _count;
 	int i, j, p;
 	unsigned char prev_data[DFII_NPHASES*DFII_PIX_RDDATA_SIZE];
-	unsigned char errs[DFII_PIX_RDDATA_SIZE/2];
+	unsigned char errs[DFII_NPHASES*DFII_PIX_RDDATA_SIZE];
 
 	if(*count == 0) {
 		printf("sdrrderr <count>\n");
@@ -130,30 +131,37 @@ void sdrrderr(char *count)
 		return;
 	}
 
-	dfii_pird_address_write(0);
-	dfii_pird_baddress_write(0);
-	command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
-	cdelay(15);
-	for(p=0;p<DFII_NPHASES;p++)
-		for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
-			prev_data[p*DFII_PIX_RDDATA_SIZE+i] = MMPTR(dfii_pix_rddata_addr[p]+4*i);
-	for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
-		errs[i] = 0;
-
-	for(j=0;j<_count;j++) {
+	for(i=0;i<DFII_NPHASES*DFII_PIX_RDDATA_SIZE;i++)
+			errs[i] = 0;
+	for(addr=0;addr<16;addr++) {
+		dfii_pird_address_write(addr*8);
+		dfii_pird_baddress_write(0);
 		command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 		cdelay(15);
 		for(p=0;p<DFII_NPHASES;p++)
-			for(i=0;i<DFII_PIX_RDDATA_SIZE;i++) {
-				unsigned char new_data;
+			for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
+				prev_data[p*DFII_PIX_RDDATA_SIZE+i] = MMPTR(dfii_pix_rddata_addr[p]+4*i);
 
-				new_data = MMPTR(dfii_pix_rddata_addr[p]+4*i);
-				errs[i%(DFII_PIX_RDDATA_SIZE/2)] |= prev_data[p*DFII_PIX_RDDATA_SIZE+i] ^ new_data;
-				prev_data[p*DFII_PIX_RDDATA_SIZE+i] = new_data;
-			}
+		for(j=0;j<_count;j++) {
+			command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+			cdelay(15);
+			for(p=0;p<DFII_NPHASES;p++)
+				for(i=0;i<DFII_PIX_RDDATA_SIZE;i++) {
+					unsigned char new_data;
+
+					new_data = MMPTR(dfii_pix_rddata_addr[p]+4*i);
+					errs[p*DFII_PIX_RDDATA_SIZE+i] |= prev_data[p*DFII_PIX_RDDATA_SIZE+i] ^ new_data;
+					prev_data[p*DFII_PIX_RDDATA_SIZE+i] = new_data;
+				}
+		}
 	}
-	for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
-		printf("%02x ", errs[i]);
+
+	for(i=0;i<DFII_NPHASES*DFII_PIX_RDDATA_SIZE;i++)
+		printf("%02x", errs[i]);
+	printf("\n");
+	for(p=0;p<DFII_NPHASES;p++)
+		for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
+			printf("%2x", DFII_PIX_RDDATA_SIZE/2 - 1 - (i % (DFII_PIX_RDDATA_SIZE/2)));
 	printf("\n");
 }
 
@@ -183,6 +191,237 @@ void sdrwr(char *startaddr)
 	command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 }
 
+#ifdef DDRPHY_BASE
+
+void sdrwlon(void)
+{
+	dfii_pi0_address_write(DDR3_MR1 | (1 << 7));
+	dfii_pi0_baddress_write(1);
+	command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+	ddrphy_wlevel_en_write(1);
+}
+
+void sdrwloff(void)
+{
+	dfii_pi0_address_write(DDR3_MR1);
+	dfii_pi0_baddress_write(1);
+	command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+	ddrphy_wlevel_en_write(0);
+}
+
+#define ERR_DDRPHY_DELAY 32
+
+static int write_level(int *delay, int *high_skew)
+{
+	int i;
+	int dq_address;
+	unsigned char dq;
+	int ok;
+
+	printf("Write leveling: ");
+
+	sdrwlon();
+	cdelay(100);
+	for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++) {
+		dq_address = dfii_pix_rddata_addr[0]+4*(DFII_PIX_RDDATA_SIZE/2-1-i);
+		ddrphy_dly_sel_write(1 << i);
+		ddrphy_wdly_dq_rst_write(1);
+		ddrphy_wdly_dqs_rst_write(1);
+
+		delay[i] = 0;
+
+		ddrphy_wlevel_strobe_write(1);
+		cdelay(10);
+		dq = MMPTR(dq_address);
+		if(dq != 0) {
+			/*
+			 * Assume this DQ group has between 1 and 2 bit times of skew.
+			 * Bring DQS into the CK=0 zone before continuing leveling.
+			 */
+			high_skew[i] = 1;
+			while(dq != 0) {
+				delay[i]++;
+				if(delay[i] >= ERR_DDRPHY_DELAY)
+					break;
+				ddrphy_wdly_dq_inc_write(1);
+				ddrphy_wdly_dqs_inc_write(1);
+				ddrphy_wlevel_strobe_write(1);
+				cdelay(10);
+				dq = MMPTR(dq_address);
+			 }
+		} else
+			high_skew[i] = 0;
+
+		while(dq == 0) {
+			delay[i]++;
+			if(delay[i] >= ERR_DDRPHY_DELAY)
+				break;
+			ddrphy_wdly_dq_inc_write(1);
+			ddrphy_wdly_dqs_inc_write(1);
+
+			ddrphy_wlevel_strobe_write(1);
+			cdelay(10);
+			dq = MMPTR(dq_address);
+		}
+	}
+	sdrwloff();
+
+	ok = 1;
+	for(i=DFII_PIX_RDDATA_SIZE/2-1;i>=0;i--) {
+		printf("%2d%c ", delay[i], high_skew[i] ? '*' : ' ');
+		if(delay[i] >= ERR_DDRPHY_DELAY)
+			ok = 0;
+	}
+
+	if(ok)
+		printf("completed\n");
+	else
+		printf("failed\n");
+
+	return ok;
+}
+
+static void read_bitslip(int *delay, int *high_skew)
+{
+	int bitslip_thr;
+	int i;
+
+	bitslip_thr = 0x7fffffff;
+	for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
+		if(high_skew[i] && (delay[i] < bitslip_thr))
+			bitslip_thr = delay[i];
+	if(bitslip_thr == 0x7fffffff)
+		return;
+	bitslip_thr = bitslip_thr/2;
+
+	printf("Read bitslip: ");
+	for(i=DFII_PIX_RDDATA_SIZE/2-1;i>=0;i--)
+		if(delay[i] > bitslip_thr) {
+			ddrphy_dly_sel_write(1 << i);
+			/* 7-series SERDES in DDR mode needs 3 pulses for 1 bitslip */
+			ddrphy_rdly_dq_bitslip_write(1);
+			ddrphy_rdly_dq_bitslip_write(1);
+			ddrphy_rdly_dq_bitslip_write(1);
+			printf("%d ", i);
+		}
+	printf("\n");
+}
+
+static void read_delays(void)
+{
+	unsigned int prv;
+	unsigned char prs[DFII_NPHASES*DFII_PIX_WRDATA_SIZE];
+	int p, i, j;
+	int working;
+	int delay, delay_min, delay_max;
+
+	printf("Read delays: ");
+
+	/* Generate pseudo-random sequence */
+	prv = 42;
+	for(i=0;i<DFII_NPHASES*DFII_PIX_WRDATA_SIZE;i++) {
+		prv = 1664525*prv + 1013904223;
+		prs[i] = prv;
+	}
+
+	/* Activate */
+	dfii_pi0_address_write(0);
+	dfii_pi0_baddress_write(0);
+	command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
+	cdelay(15);
+
+	/* Write test pattern */
+	for(p=0;p<DFII_NPHASES;p++)
+		for(i=0;i<DFII_PIX_WRDATA_SIZE;i++)
+			MMPTR(dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_WRDATA_SIZE*p+i];
+	dfii_piwr_address_write(0);
+	dfii_piwr_baddress_write(0);
+	command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
+
+	/* Calibrate each DQ in turn */
+	dfii_pird_address_write(0);
+	dfii_pird_baddress_write(0);
+	for(i=0;i<DFII_PIX_WRDATA_SIZE/2;i++) {
+		ddrphy_dly_sel_write(1 << (DFII_PIX_WRDATA_SIZE/2-i-1));
+		delay = 0;
+
+		/* Find smallest working delay */
+		ddrphy_rdly_dq_rst_write(1);
+		while(1) {
+			command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+			cdelay(15);
+			working = 1;
+			for(p=0;p<DFII_NPHASES;p++) {
+				if(MMPTR(dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_WRDATA_SIZE*p+i])
+					working = 0;
+				if(MMPTR(dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_WRDATA_SIZE/2)) != prs[DFII_PIX_WRDATA_SIZE*p+i+DFII_PIX_WRDATA_SIZE/2])
+					working = 0;
+			}
+			if(working)
+				break;
+			delay++;
+			if(delay >= ERR_DDRPHY_DELAY)
+				break;
+			ddrphy_rdly_dq_inc_write(1);
+		}
+		delay_min = delay;
+
+		/* Get a bit further into the working zone */
+		delay++;
+		ddrphy_rdly_dq_inc_write(1);
+
+		/* Find largest working delay */
+		while(1) {
+			command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+			cdelay(15);
+			working = 1;
+			for(p=0;p<DFII_NPHASES;p++) {
+				if(MMPTR(dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_WRDATA_SIZE*p+i])
+					working = 0;
+				if(MMPTR(dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_WRDATA_SIZE/2)) != prs[DFII_PIX_WRDATA_SIZE*p+i+DFII_PIX_WRDATA_SIZE/2])
+					working = 0;
+			}
+			if(!working)
+				break;
+			delay++;
+			if(delay >= ERR_DDRPHY_DELAY)
+				break;
+			ddrphy_rdly_dq_inc_write(1);
+		}
+		delay_max = delay;
+
+		printf("%d:%02d-%02d  ", DFII_PIX_WRDATA_SIZE/2-i-1, delay_min, delay_max);
+
+		/* Set delay to the middle */
+		ddrphy_rdly_dq_rst_write(1);
+		for(j=0;j<(delay_min+delay_max)/2;j++)
+			ddrphy_rdly_dq_inc_write(1);
+	}
+
+	/* Precharge */
+	dfii_pi0_address_write(0);
+	dfii_pi0_baddress_write(0);
+	command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+	cdelay(15);
+
+	printf("completed\n");
+}
+
+int sdrlevel(void)
+{
+	int delay[DFII_PIX_RDDATA_SIZE/2];
+	int high_skew[DFII_PIX_RDDATA_SIZE/2];
+
+	if(!write_level(delay, high_skew))
+		return 0;
+	read_bitslip(delay, high_skew);
+	read_delays();
+
+	return 1;
+}
+
+#endif /* DDRPHY_BASE */
+
 #define TEST_SIZE (4*1024*1024)
 
 int memtest_silent(void)
@@ -227,6 +466,10 @@ int sdrinit(void)
 	printf("Initializing SDRAM...\n");
 	
 	init_sequence();
+#ifdef DDRPHY_BASE
+	if(!sdrlevel())
+		return 0;
+#endif
 	dfii_control_write(DFII_CONTROL_SEL);
 	if(!memtest())
 		return 0;
diff --git a/software/bios/sdram.h b/software/bios/sdram.h
index 45c9adb7..aea70982 100644
--- a/software/bios/sdram.h
+++ b/software/bios/sdram.h
@@ -1,6 +1,8 @@
 #ifndef __SDRAM_H
 #define __SDRAM_H
 
+#include <generated/csr.h>
+
 void sdrsw(void);
 void sdrhw(void);
 void sdrrow(char *_row);
@@ -8,6 +10,13 @@ void sdrrdbuf(int dq);
 void sdrrd(char *startaddr, char *dq);
 void sdrrderr(char *count);
 void sdrwr(char *startaddr);
+
+#ifdef DDRPHY_BASE
+void sdrwlon(void);
+void sdrwloff(void);
+int sdrlevel(void);
+#endif
+
 int memtest_silent(void);
 int memtest(void);
 int sdrinit(void);