litex/soc/software/bios/sdram.c

   1 #include <generated/csr.h>
   2 #ifdef CSR_SDRAM_BASE
   3
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6
   7 #include <generated/sdram_phy.h>
   8 #include <generated/mem.h>
   9 #include <hw/flags.h>
  10 #include <system.h>
  11
  12 #include "sdram.h"
  13
  14 static void cdelay(int i)
  15 {
  16         while(i > 0) {
  17 #if defined (__lm32__)
  18                 __asm__ volatile("nop");
  19 #elif defined (__or1k__)
  20                 __asm__ volatile("l.nop");
  21 #elif defined (__riscv__)
  22                 __asm__ volatile("nop");
  23 #else
  24 #error Unsupported architecture
  25 #endif
  26                 i--;
  27         }
  28 }
  29
  30 void sdrsw(void)
  31 {
  32         sdram_dfii_control_write(DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N);
  33         printf("SDRAM now under software control\n");
  34 }
  35
  36 void sdrhw(void)
  37 {
  38         sdram_dfii_control_write(DFII_CONTROL_SEL);
  39         printf("SDRAM now under hardware control\n");
  40 }
  41
  42 void sdrrow(char *_row)
  43 {
  44         char *c;
  45         unsigned int row;
  46
  47         if(*_row == 0) {
  48                 sdram_dfii_pi0_address_write(0x0000);
  49                 sdram_dfii_pi0_baddress_write(0);
  50                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
  51                 cdelay(15);
  52                 printf("Precharged\n");
  53         } else {
  54                 row = strtoul(_row, &c, 0);
  55                 if(*c != 0) {
  56                         printf("incorrect row\n");
  57                         return;
  58                 }
  59                 sdram_dfii_pi0_address_write(row);
  60                 sdram_dfii_pi0_baddress_write(0);
  61                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
  62                 cdelay(15);
  63                 printf("Activated row %d\n", row);
  64         }
  65 }
  66
  67 void sdrrdbuf(int dq)
  68 {
  69         int i, p;
  70         int first_byte, step;
  71
  72         if(dq < 0) {
  73                 first_byte = 0;
  74                 step = 1;
  75         } else {
  76                 first_byte = DFII_PIX_DATA_SIZE/2 - 1 - dq;
  77                 step = DFII_PIX_DATA_SIZE/2;
  78         }
  79
  80         for(p=0;p<DFII_NPHASES;p++)
  81                 for(i=first_byte;i<DFII_PIX_DATA_SIZE;i+=step)
  82                         printf("%02x", MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i));
  83         printf("\n");
  84 }
  85
  86 void sdrrd(char *startaddr, char *dq)
  87 {
  88         char *c;
  89         unsigned int addr;
  90         int _dq;
  91
  92         if(*startaddr == 0) {
  93                 printf("sdrrd <address>\n");
  94                 return;
  95         }
  96         addr = strtoul(startaddr, &c, 0);
  97         if(*c != 0) {
  98                 printf("incorrect address\n");
  99                 return;
 100         }
 101         if(*dq == 0)
 102                 _dq = -1;
 103         else {
 104                 _dq = strtoul(dq, &c, 0);
 105                 if(*c != 0) {
 106                         printf("incorrect DQ\n");
 107                         return;
 108                 }
 109         }
 110
 111         sdram_dfii_pird_address_write(addr);
 112         sdram_dfii_pird_baddress_write(0);
 113         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 114         cdelay(15);
 115         sdrrdbuf(_dq);
 116 }
 117
 118 void sdrrderr(char *count)
 119 {
 120         int addr;
 121         char *c;
 122         int _count;
 123         int i, j, p;
 124         unsigned char prev_data[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 125         unsigned char errs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 126
 127         if(*count == 0) {
 128                 printf("sdrrderr <count>\n");
 129                 return;
 130         }
 131         _count = strtoul(count, &c, 0);
 132         if(*c != 0) {
 133                 printf("incorrect count\n");
 134                 return;
 135         }
 136
 137         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 138                         errs[i] = 0;
 139         for(addr=0;addr<16;addr++) {
 140                 sdram_dfii_pird_address_write(addr*8);
 141                 sdram_dfii_pird_baddress_write(0);
 142                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 143                 cdelay(15);
 144                 for(p=0;p<DFII_NPHASES;p++)
 145                         for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 146                                 prev_data[p*DFII_PIX_DATA_SIZE+i] = MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i);
 147
 148                 for(j=0;j<_count;j++) {
 149                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 150                         cdelay(15);
 151                         for(p=0;p<DFII_NPHASES;p++)
 152                                 for(i=0;i<DFII_PIX_DATA_SIZE;i++) {
 153                                         unsigned char new_data;
 154
 155                                         new_data = MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i);
 156                                         errs[p*DFII_PIX_DATA_SIZE+i] |= prev_data[p*DFII_PIX_DATA_SIZE+i] ^ new_data;
 157                                         prev_data[p*DFII_PIX_DATA_SIZE+i] = new_data;
 158                                 }
 159                 }
 160         }
 161
 162         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 163                 printf("%02x", errs[i]);
 164         printf("\n");
 165         for(p=0;p<DFII_NPHASES;p++)
 166                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 167                         printf("%2x", DFII_PIX_DATA_SIZE/2 - 1 - (i % (DFII_PIX_DATA_SIZE/2)));
 168         printf("\n");
 169 }
 170
 171 void sdrwr(char *startaddr)
 172 {
 173         char *c;
 174         unsigned int addr;
 175         int i;
 176         int p;
 177
 178         if(*startaddr == 0) {
 179                 printf("sdrrd <address>\n");
 180                 return;
 181         }
 182         addr = strtoul(startaddr, &c, 0);
 183         if(*c != 0) {
 184                 printf("incorrect address\n");
 185                 return;
 186         }
 187
 188         for(p=0;p<DFII_NPHASES;p++)
 189                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 190                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = 0x10*p + i;
 191
 192         sdram_dfii_piwr_address_write(addr);
 193         sdram_dfii_piwr_baddress_write(0);
 194         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 195 }
 196
 197 #ifdef CSR_DDRPHY_BASE
 198
 199 void sdrwlon(void)
 200 {
 201         sdram_dfii_pi0_address_write(DDR3_MR1 | (1 << 7));
 202         sdram_dfii_pi0_baddress_write(1);
 203         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 204         ddrphy_wlevel_en_write(1);
 205 }
 206
 207 void sdrwloff(void)
 208 {
 209         sdram_dfii_pi0_address_write(DDR3_MR1);
 210         sdram_dfii_pi0_baddress_write(1);
 211         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 212         ddrphy_wlevel_en_write(0);
 213 }
 214
 215 #define ERR_DDRPHY_DELAY 32
 216
 217 static int write_level(int *delay, int *high_skew)
 218 {
 219         int i;
 220         int dq_address;
 221         unsigned char dq;
 222         int ok;
 223
 224         printf("Write leveling: ");
 225
 226         sdrwlon();
 227         cdelay(100);
 228         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++) {
 229                 dq_address = sdram_dfii_pix_rddata_addr[0]+4*(DFII_PIX_DATA_SIZE/2-1-i);
 230                 ddrphy_dly_sel_write(1 << i);
 231                 ddrphy_wdly_dq_rst_write(1);
 232                 ddrphy_wdly_dqs_rst_write(1);
 233
 234                 delay[i] = 0;
 235
 236                 ddrphy_wlevel_strobe_write(1);
 237                 cdelay(10);
 238                 dq = MMPTR(dq_address);
 239                 if(dq != 0) {
 240                         /*
 241                          * Assume this DQ group has between 1 and 2 bit times of skew.
 242                          * Bring DQS into the CK=0 zone before continuing leveling.
 243                          */
 244                         high_skew[i] = 1;
 245                         while(dq != 0) {
 246                                 delay[i]++;
 247                                 if(delay[i] >= ERR_DDRPHY_DELAY)
 248                                         break;
 249                                 ddrphy_wdly_dq_inc_write(1);
 250                                 ddrphy_wdly_dqs_inc_write(1);
 251                                 ddrphy_wlevel_strobe_write(1);
 252                                 cdelay(10);
 253                                 dq = MMPTR(dq_address);
 254                          }
 255                 } else
 256                         high_skew[i] = 0;
 257
 258                 while(dq == 0) {
 259                         delay[i]++;
 260                         if(delay[i] >= ERR_DDRPHY_DELAY)
 261                                 break;
 262                         ddrphy_wdly_dq_inc_write(1);
 263                         ddrphy_wdly_dqs_inc_write(1);
 264
 265                         ddrphy_wlevel_strobe_write(1);
 266                         cdelay(10);
 267                         dq = MMPTR(dq_address);
 268                 }
 269         }
 270         sdrwloff();
 271
 272         ok = 1;
 273         for(i=DFII_PIX_DATA_SIZE/2-1;i>=0;i--) {
 274                 printf("%2d%c ", delay[i], high_skew[i] ? '*' : ' ');
 275                 if(delay[i] >= ERR_DDRPHY_DELAY)
 276                         ok = 0;
 277         }
 278
 279         if(ok)
 280                 printf("completed\n");
 281         else
 282                 printf("failed\n");
 283
 284         return ok;
 285 }
 286
 287 static void read_bitslip(int *delay, int *high_skew)
 288 {
 289         int bitslip_thr;
 290         int i;
 291
 292         bitslip_thr = 0x7fffffff;
 293         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++)
 294                 if(high_skew[i] && (delay[i] < bitslip_thr))
 295                         bitslip_thr = delay[i];
 296         if(bitslip_thr == 0x7fffffff)
 297                 return;
 298         bitslip_thr = bitslip_thr/2;
 299
 300         printf("Read bitslip: ");
 301         for(i=DFII_PIX_DATA_SIZE/2-1;i>=0;i--)
 302                 if(delay[i] > bitslip_thr) {
 303                         ddrphy_dly_sel_write(1 << i);
 304                         /* 7-series SERDES in DDR mode needs 3 pulses for 1 bitslip */
 305                         ddrphy_rdly_dq_bitslip_write(1);
 306                         ddrphy_rdly_dq_bitslip_write(1);
 307                         ddrphy_rdly_dq_bitslip_write(1);
 308                         printf("%d ", i);
 309                 }
 310         printf("\n");
 311 }
 312
 313 static void read_delays(void)
 314 {
 315         unsigned int prv;
 316         unsigned char prs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 317         int p, i, j;
 318         int working;
 319         int delay, delay_min, delay_max;
 320
 321         printf("Read delays: ");
 322
 323         /* Generate pseudo-random sequence */
 324         prv = 42;
 325         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++) {
 326                 prv = 1664525*prv + 1013904223;
 327                 prs[i] = prv;
 328         }
 329
 330         /* Activate */
 331         sdram_dfii_pi0_address_write(0);
 332         sdram_dfii_pi0_baddress_write(0);
 333         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
 334         cdelay(15);
 335
 336         /* Write test pattern */
 337         for(p=0;p<DFII_NPHASES;p++)
 338                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 339                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_DATA_SIZE*p+i];
 340         sdram_dfii_piwr_address_write(0);
 341         sdram_dfii_piwr_baddress_write(0);
 342         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 343
 344         /* Calibrate each DQ in turn */
 345         sdram_dfii_pird_address_write(0);
 346         sdram_dfii_pird_baddress_write(0);
 347         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++) {
 348                 ddrphy_dly_sel_write(1 << (DFII_PIX_DATA_SIZE/2-i-1));
 349                 delay = 0;
 350
 351                 /* Find smallest working delay */
 352                 ddrphy_rdly_dq_rst_write(1);
 353                 while(1) {
 354                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 355                         cdelay(15);
 356                         working = 1;
 357                         for(p=0;p<DFII_NPHASES;p++) {
 358                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_DATA_SIZE*p+i])
 359                                         working = 0;
 360                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_DATA_SIZE/2)) != prs[DFII_PIX_DATA_SIZE*p+i+DFII_PIX_DATA_SIZE/2])
 361                                         working = 0;
 362                         }
 363                         if(working)
 364                                 break;
 365                         delay++;
 366                         if(delay >= ERR_DDRPHY_DELAY)
 367                                 break;
 368                         ddrphy_rdly_dq_inc_write(1);
 369                 }
 370                 delay_min = delay;
 371
 372                 /* Get a bit further into the working zone */
 373                 delay++;
 374                 ddrphy_rdly_dq_inc_write(1);
 375
 376                 /* Find largest working delay */
 377                 while(1) {
 378                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 379                         cdelay(15);
 380                         working = 1;
 381                         for(p=0;p<DFII_NPHASES;p++) {
 382                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_DATA_SIZE*p+i])
 383                                         working = 0;
 384                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_DATA_SIZE/2)) != prs[DFII_PIX_DATA_SIZE*p+i+DFII_PIX_DATA_SIZE/2])
 385                                         working = 0;
 386                         }
 387                         if(!working)
 388                                 break;
 389                         delay++;
 390                         if(delay >= ERR_DDRPHY_DELAY)
 391                                 break;
 392                         ddrphy_rdly_dq_inc_write(1);
 393                 }
 394                 delay_max = delay;
 395
 396                 printf("%d:%02d-%02d  ", DFII_PIX_DATA_SIZE/2-i-1, delay_min, delay_max);
 397
 398                 /* Set delay to the middle */
 399                 ddrphy_rdly_dq_rst_write(1);
 400                 for(j=0;j<(delay_min+delay_max)/2;j++)
 401                         ddrphy_rdly_dq_inc_write(1);
 402         }
 403
 404         /* Precharge */
 405         sdram_dfii_pi0_address_write(0);
 406         sdram_dfii_pi0_baddress_write(0);
 407         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 408         cdelay(15);
 409
 410         printf("completed\n");
 411 }
 412
 413 int sdrlevel(void)
 414 {
 415         int delay[DFII_PIX_DATA_SIZE/2];
 416         int high_skew[DFII_PIX_DATA_SIZE/2];
 417
 418         if(!write_level(delay, high_skew))
 419                 return 0;
 420         read_bitslip(delay, high_skew);
 421         read_delays();
 422
 423         return 1;
 424 }
 425
 426 #endif /* CSR_DDRPHY_BASE */
 427
 428 static unsigned int seed_to_data_32(unsigned int seed, int random)
 429 {
 430         if (random)
 431                 return 1664525*seed + 1013904223;
 432         else
 433                 return seed + 1;
 434 }
 435
 436 static unsigned short seed_to_data_16(unsigned short seed, int random)
 437 {
 438         if (random)
 439                 return 25173*seed + 13849;
 440         else
 441                 return seed + 1;
 442 }
 443
 444 #define ONEZERO 0xAAAAAAAA
 445 #define ZEROONE 0x55555555
 446
 447 #ifndef MEMTEST_BUS_SIZE
 448 #define MEMTEST_BUS_SIZE (512)
 449 #endif
 450
 451 static int memtest_bus(void)
 452 {
 453         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 454         int i, errors;
 455
 456         errors = 0;
 457
 458         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 459                 array[i] = ONEZERO;
 460         }
 461         flush_cpu_dcache();
 462         flush_l2_cache();
 463         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 464                 if(array[i] != ONEZERO)
 465                         errors++;
 466         }
 467
 468         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 469                 array[i] = ZEROONE;
 470         }
 471         flush_cpu_dcache();
 472         flush_l2_cache();
 473         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 474                 if(array[i] != ZEROONE)
 475                         errors++;
 476         }
 477
 478         return errors;
 479 }
 480
 481 #ifndef MEMTEST_DATA_SIZE
 482 #define MEMTEST_DATA_SIZE (2*1024*1024)
 483 #endif
 484 #define MEMTEST_DATA_RANDOM 1
 485
 486 static int memtest_data(void)
 487 {
 488         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 489         int i, errors;
 490         unsigned int seed_32;
 491
 492         errors = 0;
 493         seed_32 = 0;
 494
 495         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 496                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 497                 array[i] = seed_32;
 498         }
 499
 500         seed_32 = 0;
 501         flush_cpu_dcache();
 502         flush_l2_cache();
 503         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 504                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 505                 if(array[i] != seed_32)
 506                         errors++;
 507         }
 508
 509         return errors;
 510 }
 511 #ifndef MEMTEST_ADDR_SIZE
 512 #define MEMTEST_ADDR_SIZE (32*1024)
 513 #endif
 514 #define MEMTEST_ADDR_RANDOM 0
 515
 516 static int memtest_addr(void)
 517 {
 518         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 519         int i, errors;
 520         unsigned short seed_16;
 521
 522         errors = 0;
 523         seed_16 = 0;
 524
 525         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 526                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 527                 array[(unsigned int) seed_16] = i;
 528         }
 529
 530         seed_16 = 0;
 531         flush_cpu_dcache();
 532         flush_l2_cache();
 533         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 534                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 535                 if(array[(unsigned int) seed_16] != i)
 536                         errors++;
 537         }
 538
 539         return errors;
 540 }
 541
 542 int memtest(void)
 543 {
 544         int bus_errors, data_errors, addr_errors;
 545
 546         bus_errors = memtest_bus();
 547         if(bus_errors != 0)
 548                 printf("Memtest bus failed: %d/%d errors\n", bus_errors, 2*128);
 549
 550         data_errors = memtest_data();
 551         if(bus_errors != 0)
 552                 printf("Memtest data failed: %d/%d errors\n", data_errors, MEMTEST_DATA_SIZE/4);
 553
 554         addr_errors = memtest_addr();
 555         if(addr_errors != 0)
 556                 printf("Memtest addr failed: %d/%d errors\n", addr_errors, MEMTEST_ADDR_SIZE/4);
 557
 558         if(bus_errors + data_errors + addr_errors != 0)
 559                 return 0;
 560         else {
 561                 printf("Memtest OK\n");
 562                 return 1;
 563         }
 564 }
 565
 566 int sdrinit(void)
 567 {
 568         printf("Initializing SDRAM...\n");
 569
 570         init_sequence();
 571 #ifdef CSR_DDRPHY_BASE
 572         if(!sdrlevel())
 573                 return 0;
 574 #endif
 575         sdram_dfii_control_write(DFII_CONTROL_SEL);
 576         if(!memtest())
 577                 return 0;
 578
 579         return 1;
 580 }
 581
 582 #endif