bios: support DDR3 write leveling and read calibration. This makes the full DDR3...

diff --git a/software/bios/main.c b/software/bios/main.c
index dfefe01b8e12897f8f9aa6f48c1be6774c369346..29b141ce7aeba4e3a555ad0ffeeee0856a6e4981 100644 (file)
--- a/software/bios/main.c
+++ b/software/bios/main.c
@@ -382,6 +382,11 @@ static void do_command(char *c)
        else if(strcmp(token, "sdrrd") == 0) sdrrd(get_token(&c), get_token(&c));
        else if(strcmp(token, "sdrrderr") == 0) sdrrderr(get_token(&c));
        else if(strcmp(token, "sdrwr") == 0) sdrwr(get_token(&c));
+#ifdef DDRPHY_BASE
+       else if(strcmp(token, "sdrwlon") == 0) sdrwlon();
+       else if(strcmp(token, "sdrwloff") == 0) sdrwloff();
+       else if(strcmp(token, "sdrlevel") == 0) sdrlevel();
+#endif
        else if(strcmp(token, "memtest") == 0) memtest();
        else if(strcmp(token, "sdrinit") == 0) sdrinit();
 #endif

diff --git a/software/bios/sdram.c b/software/bios/sdram.c
index 59215c3645969f9c45b2e99f443b72dc0945b82a..24482ddfd1e3fc3fd907f762c4bfb75e831b4d3e 100644 (file)
--- a/software/bios/sdram.c
+++ b/software/bios/sdram.c
@@ -114,11 +114,12 @@ void sdrrd(char *startaddr, char *dq)
 
 void sdrrderr(char *count)
 {
+       int addr;
        char *c;
        int _count;
        int i, j, p;
        unsigned char prev_data[DFII_NPHASES*DFII_PIX_RDDATA_SIZE];
-       unsigned char errs[DFII_PIX_RDDATA_SIZE/2];
+       unsigned char errs[DFII_NPHASES*DFII_PIX_RDDATA_SIZE];
 
        if(*count == 0) {
                printf("sdrrderr <count>\n");
@@ -130,30 +131,37 @@ void sdrrderr(char *count)
                return;
        }
 
-       dfii_pird_address_write(0);
-       dfii_pird_baddress_write(0);
-       command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
-       cdelay(15);
-       for(p=0;p<DFII_NPHASES;p++)
-               for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
-                       prev_data[p*DFII_PIX_RDDATA_SIZE+i] = MMPTR(dfii_pix_rddata_addr[p]+4*i);
-       for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
-               errs[i] = 0;
-
-       for(j=0;j<_count;j++) {
+       for(i=0;i<DFII_NPHASES*DFII_PIX_RDDATA_SIZE;i++)
+                       errs[i] = 0;
+       for(addr=0;addr<16;addr++) {
+               dfii_pird_address_write(addr*8);
+               dfii_pird_baddress_write(0);
                command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
                cdelay(15);
                for(p=0;p<DFII_NPHASES;p++)
-                       for(i=0;i<DFII_PIX_RDDATA_SIZE;i++) {
-                               unsigned char new_data;
+                       for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
+                               prev_data[p*DFII_PIX_RDDATA_SIZE+i] = MMPTR(dfii_pix_rddata_addr[p]+4*i);
 
-                               new_data = MMPTR(dfii_pix_rddata_addr[p]+4*i);
-                               errs[i%(DFII_PIX_RDDATA_SIZE/2)] |= prev_data[p*DFII_PIX_RDDATA_SIZE+i] ^ new_data;
-                               prev_data[p*DFII_PIX_RDDATA_SIZE+i] = new_data;
-                       }
+               for(j=0;j<_count;j++) {
+                       command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+                       cdelay(15);
+                       for(p=0;p<DFII_NPHASES;p++)
+                               for(i=0;i<DFII_PIX_RDDATA_SIZE;i++) {
+                                       unsigned char new_data;
+
+                                       new_data = MMPTR(dfii_pix_rddata_addr[p]+4*i);
+                                       errs[p*DFII_PIX_RDDATA_SIZE+i] |= prev_data[p*DFII_PIX_RDDATA_SIZE+i] ^ new_data;
+                                       prev_data[p*DFII_PIX_RDDATA_SIZE+i] = new_data;
+                               }
+               }
        }
-       for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
-               printf("%02x ", errs[i]);
+
+       for(i=0;i<DFII_NPHASES*DFII_PIX_RDDATA_SIZE;i++)
+               printf("%02x", errs[i]);
+       printf("\n");
+       for(p=0;p<DFII_NPHASES;p++)
+               for(i=0;i<DFII_PIX_RDDATA_SIZE;i++)
+                       printf("%2x", DFII_PIX_RDDATA_SIZE/2 - 1 - (i % (DFII_PIX_RDDATA_SIZE/2)));
        printf("\n");
 }
 
@@ -183,6 +191,237 @@ void sdrwr(char *startaddr)
        command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 }
 
+#ifdef DDRPHY_BASE
+
+void sdrwlon(void)
+{
+       dfii_pi0_address_write(DDR3_MR1 | (1 << 7));
+       dfii_pi0_baddress_write(1);
+       command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+       ddrphy_wlevel_en_write(1);
+}
+
+void sdrwloff(void)
+{
+       dfii_pi0_address_write(DDR3_MR1);
+       dfii_pi0_baddress_write(1);
+       command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+       ddrphy_wlevel_en_write(0);
+}
+
+#define ERR_DDRPHY_DELAY 32
+
+static int write_level(int *delay, int *high_skew)
+{
+       int i;
+       int dq_address;
+       unsigned char dq;
+       int ok;
+
+       printf("Write leveling: ");
+
+       sdrwlon();
+       cdelay(100);
+       for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++) {
+               dq_address = dfii_pix_rddata_addr[0]+4*(DFII_PIX_RDDATA_SIZE/2-1-i);
+               ddrphy_dly_sel_write(1 << i);
+               ddrphy_wdly_dq_rst_write(1);
+               ddrphy_wdly_dqs_rst_write(1);
+
+               delay[i] = 0;
+
+               ddrphy_wlevel_strobe_write(1);
+               cdelay(10);
+               dq = MMPTR(dq_address);
+               if(dq != 0) {
+                       /*
+                        * Assume this DQ group has between 1 and 2 bit times of skew.
+                        * Bring DQS into the CK=0 zone before continuing leveling.
+                        */
+                       high_skew[i] = 1;
+                       while(dq != 0) {
+                               delay[i]++;
+                               if(delay[i] >= ERR_DDRPHY_DELAY)
+                                       break;
+                               ddrphy_wdly_dq_inc_write(1);
+                               ddrphy_wdly_dqs_inc_write(1);
+                               ddrphy_wlevel_strobe_write(1);
+                               cdelay(10);
+                               dq = MMPTR(dq_address);
+                        }
+               } else
+                       high_skew[i] = 0;
+
+               while(dq == 0) {
+                       delay[i]++;
+                       if(delay[i] >= ERR_DDRPHY_DELAY)
+                               break;
+                       ddrphy_wdly_dq_inc_write(1);
+                       ddrphy_wdly_dqs_inc_write(1);
+
+                       ddrphy_wlevel_strobe_write(1);
+                       cdelay(10);
+                       dq = MMPTR(dq_address);
+               }
+       }
+       sdrwloff();
+
+       ok = 1;
+       for(i=DFII_PIX_RDDATA_SIZE/2-1;i>=0;i--) {
+               printf("%2d%c ", delay[i], high_skew[i] ? '*' : ' ');
+               if(delay[i] >= ERR_DDRPHY_DELAY)
+                       ok = 0;
+       }
+
+       if(ok)
+               printf("completed\n");
+       else
+               printf("failed\n");
+
+       return ok;
+}
+
+static void read_bitslip(int *delay, int *high_skew)
+{
+       int bitslip_thr;
+       int i;
+
+       bitslip_thr = 0x7fffffff;
+       for(i=0;i<DFII_PIX_RDDATA_SIZE/2;i++)
+               if(high_skew[i] && (delay[i] < bitslip_thr))
+                       bitslip_thr = delay[i];
+       if(bitslip_thr == 0x7fffffff)
+               return;
+       bitslip_thr = bitslip_thr/2;
+
+       printf("Read bitslip: ");
+       for(i=DFII_PIX_RDDATA_SIZE/2-1;i>=0;i--)
+               if(delay[i] > bitslip_thr) {
+                       ddrphy_dly_sel_write(1 << i);
+                       /* 7-series SERDES in DDR mode needs 3 pulses for 1 bitslip */
+                       ddrphy_rdly_dq_bitslip_write(1);
+                       ddrphy_rdly_dq_bitslip_write(1);
+                       ddrphy_rdly_dq_bitslip_write(1);
+                       printf("%d ", i);
+               }
+       printf("\n");
+}
+
+static void read_delays(void)
+{
+       unsigned int prv;
+       unsigned char prs[DFII_NPHASES*DFII_PIX_WRDATA_SIZE];
+       int p, i, j;
+       int working;
+       int delay, delay_min, delay_max;
+
+       printf("Read delays: ");
+
+       /* Generate pseudo-random sequence */
+       prv = 42;
+       for(i=0;i<DFII_NPHASES*DFII_PIX_WRDATA_SIZE;i++) {
+               prv = 1664525*prv + 1013904223;
+               prs[i] = prv;
+       }
+
+       /* Activate */
+       dfii_pi0_address_write(0);
+       dfii_pi0_baddress_write(0);
+       command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
+       cdelay(15);
+
+       /* Write test pattern */
+       for(p=0;p<DFII_NPHASES;p++)
+               for(i=0;i<DFII_PIX_WRDATA_SIZE;i++)
+                       MMPTR(dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_WRDATA_SIZE*p+i];
+       dfii_piwr_address_write(0);
+       dfii_piwr_baddress_write(0);
+       command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
+
+       /* Calibrate each DQ in turn */
+       dfii_pird_address_write(0);
+       dfii_pird_baddress_write(0);
+       for(i=0;i<DFII_PIX_WRDATA_SIZE/2;i++) {
+               ddrphy_dly_sel_write(1 << (DFII_PIX_WRDATA_SIZE/2-i-1));
+               delay = 0;
+
+               /* Find smallest working delay */
+               ddrphy_rdly_dq_rst_write(1);
+               while(1) {
+                       command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+                       cdelay(15);
+                       working = 1;
+                       for(p=0;p<DFII_NPHASES;p++) {
+                               if(MMPTR(dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_WRDATA_SIZE*p+i])
+                                       working = 0;
+                               if(MMPTR(dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_WRDATA_SIZE/2)) != prs[DFII_PIX_WRDATA_SIZE*p+i+DFII_PIX_WRDATA_SIZE/2])
+                                       working = 0;
+                       }
+                       if(working)
+                               break;
+                       delay++;
+                       if(delay >= ERR_DDRPHY_DELAY)
+                               break;
+                       ddrphy_rdly_dq_inc_write(1);
+               }
+               delay_min = delay;
+
+               /* Get a bit further into the working zone */
+               delay++;
+               ddrphy_rdly_dq_inc_write(1);
+
+               /* Find largest working delay */
+               while(1) {
+                       command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
+                       cdelay(15);
+                       working = 1;
+                       for(p=0;p<DFII_NPHASES;p++) {
+                               if(MMPTR(dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_WRDATA_SIZE*p+i])
+                                       working = 0;
+                               if(MMPTR(dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_WRDATA_SIZE/2)) != prs[DFII_PIX_WRDATA_SIZE*p+i+DFII_PIX_WRDATA_SIZE/2])
+                                       working = 0;
+                       }
+                       if(!working)
+                               break;
+                       delay++;
+                       if(delay >= ERR_DDRPHY_DELAY)
+                               break;
+                       ddrphy_rdly_dq_inc_write(1);
+               }
+               delay_max = delay;
+
+               printf("%d:%02d-%02d  ", DFII_PIX_WRDATA_SIZE/2-i-1, delay_min, delay_max);
+
+               /* Set delay to the middle */
+               ddrphy_rdly_dq_rst_write(1);
+               for(j=0;j<(delay_min+delay_max)/2;j++)
+                       ddrphy_rdly_dq_inc_write(1);
+       }
+
+       /* Precharge */
+       dfii_pi0_address_write(0);
+       dfii_pi0_baddress_write(0);
+       command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
+       cdelay(15);
+
+       printf("completed\n");
+}
+
+int sdrlevel(void)
+{
+       int delay[DFII_PIX_RDDATA_SIZE/2];
+       int high_skew[DFII_PIX_RDDATA_SIZE/2];
+
+       if(!write_level(delay, high_skew))
+               return 0;
+       read_bitslip(delay, high_skew);
+       read_delays();
+
+       return 1;
+}
+
+#endif /* DDRPHY_BASE */
+
 #define TEST_SIZE (4*1024*1024)
 
 int memtest_silent(void)
@@ -227,6 +466,10 @@ int sdrinit(void)
        printf("Initializing SDRAM...\n");
        
        init_sequence();
+#ifdef DDRPHY_BASE
+       if(!sdrlevel())
+               return 0;
+#endif
        dfii_control_write(DFII_CONTROL_SEL);
        if(!memtest())
                return 0;

diff --git a/software/bios/sdram.h b/software/bios/sdram.h
index 45c9adb7f836483137bf3421396c342e842ef772..aea709822316f5198bfbcbc8d19a67871f820c82 100644 (file)
--- a/software/bios/sdram.h
+++ b/software/bios/sdram.h
@@ -1,6 +1,8 @@
 #ifndef __SDRAM_H
 #define __SDRAM_H
 
+#include <generated/csr.h>
+
 void sdrsw(void);
 void sdrhw(void);
 void sdrrow(char *_row);
@@ -8,6 +10,13 @@ void sdrrdbuf(int dq);
 void sdrrd(char *startaddr, char *dq);
 void sdrrderr(char *count);
 void sdrwr(char *startaddr);
+
+#ifdef DDRPHY_BASE
+void sdrwlon(void);
+void sdrwloff(void);
+int sdrlevel(void);
+#endif
+
 int memtest_silent(void);
 int memtest(void);
 int sdrinit(void);