litex/soc/software/bios/sdram.c

   1 // This file is Copyright (c) 2013-2014 Sebastien Bourdeauducq <sb@m-labs.hk>
   2 // This file is Copyright (c) 2013-2019 Florent Kermarrec <florent@enjoy-digital.fr>
   3 // This file is Copyright (c) 2018 Chris Ballance <chris.ballance@physics.ox.ac.uk>
   4 // This file is Copyright (c) 2018 Dolu1990 <charles.papon.90@gmail.com>
   5 // This file is Copyright (c) 2019 Gabriel L. Somlo <gsomlo@gmail.com>
   6 // This file is Copyright (c) 2018 Jean-François Nguyen <jf@lambdaconcept.fr>
   7 // This file is Copyright (c) 2018 Sergiusz Bazanski <q3k@q3k.org>
   8 // This file is Copyright (c) 2018 Tim 'mithro' Ansell <me@mith.ro>
   9 // License: BSD
  10
  11 #include <generated/csr.h>
  12
  13 #include <stdio.h>
  14 #include <stdlib.h>
  15
  16 #ifdef CSR_SDRAM_BASE
  17 #include <generated/sdram_phy.h>
  18 #endif
  19 #include <generated/mem.h>
  20 #include <hw/flags.h>
  21 #include <system.h>
  22
  23 #include "sdram.h"
  24
  25 // FIXME(hack): If we don't have main ram, just target the sram instead.
  26 #ifndef MAIN_RAM_BASE
  27 #define MAIN_RAM_BASE SRAM_BASE
  28 #endif
  29
  30 __attribute__((unused)) static void cdelay(int i)
  31 {
  32         while(i > 0) {
  33 #if defined (__lm32__)
  34                 __asm__ volatile("nop");
  35 #elif defined (__or1k__)
  36                 __asm__ volatile("l.nop");
  37 #elif defined (__picorv32__)
  38                 __asm__ volatile("nop");
  39 #elif defined (__vexriscv__)
  40                 __asm__ volatile("nop");
  41 #elif defined (__minerva__)
  42                 __asm__ volatile("nop");
  43 #elif defined (__rocket__)
  44                 __asm__ volatile("nop");
  45 #else
  46 #error Unsupported architecture
  47 #endif
  48                 i--;
  49         }
  50 }
  51
  52 #ifdef CSR_SDRAM_BASE
  53
  54 #define DFII_ADDR_SHIFT CONFIG_CSR_ALIGNMENT/8
  55
  56 void sdrsw(void)
  57 {
  58         sdram_dfii_control_write(DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N);
  59         printf("SDRAM now under software control\n");
  60 }
  61
  62 void sdrhw(void)
  63 {
  64         sdram_dfii_control_write(DFII_CONTROL_SEL);
  65         printf("SDRAM now under hardware control\n");
  66 }
  67
  68 void sdrrow(char *_row)
  69 {
  70         char *c;
  71         unsigned int row;
  72
  73         if(*_row == 0) {
  74                 sdram_dfii_pi0_address_write(0x0000);
  75                 sdram_dfii_pi0_baddress_write(0);
  76                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
  77                 cdelay(15);
  78                 printf("Precharged\n");
  79         } else {
  80                 row = strtoul(_row, &c, 0);
  81                 if(*c != 0) {
  82                         printf("incorrect row\n");
  83                         return;
  84                 }
  85                 sdram_dfii_pi0_address_write(row);
  86                 sdram_dfii_pi0_baddress_write(0);
  87                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
  88                 cdelay(15);
  89                 printf("Activated row %d\n", row);
  90         }
  91 }
  92
  93 void sdrrdbuf(int dq)
  94 {
  95         int i, p;
  96         int first_byte, step;
  97
  98         if(dq < 0) {
  99                 first_byte = 0;
 100                 step = 1;
 101         } else {
 102                 first_byte = DFII_PIX_DATA_SIZE/2 - 1 - dq;
 103                 step = DFII_PIX_DATA_SIZE/2;
 104         }
 105
 106         for(p=0;p<DFII_NPHASES;p++)
 107                 for(i=first_byte;i<DFII_PIX_DATA_SIZE;i+=step)
 108                         printf("%02x", MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*i));
 109         printf("\n");
 110 }
 111
 112 void sdrrd(char *startaddr, char *dq)
 113 {
 114         char *c;
 115         unsigned int addr;
 116         int _dq;
 117
 118         if(*startaddr == 0) {
 119                 printf("sdrrd <address>\n");
 120                 return;
 121         }
 122         addr = strtoul(startaddr, &c, 0);
 123         if(*c != 0) {
 124                 printf("incorrect address\n");
 125                 return;
 126         }
 127         if(*dq == 0)
 128                 _dq = -1;
 129         else {
 130                 _dq = strtoul(dq, &c, 0);
 131                 if(*c != 0) {
 132                         printf("incorrect DQ\n");
 133                         return;
 134                 }
 135         }
 136
 137         sdram_dfii_pird_address_write(addr);
 138         sdram_dfii_pird_baddress_write(0);
 139         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 140         cdelay(15);
 141         sdrrdbuf(_dq);
 142 }
 143
 144 void sdrrderr(char *count)
 145 {
 146         int addr;
 147         char *c;
 148         int _count;
 149         int i, j, p;
 150         unsigned char prev_data[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 151         unsigned char errs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 152
 153         if(*count == 0) {
 154                 printf("sdrrderr <count>\n");
 155                 return;
 156         }
 157         _count = strtoul(count, &c, 0);
 158         if(*c != 0) {
 159                 printf("incorrect count\n");
 160                 return;
 161         }
 162
 163         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 164                         errs[i] = 0;
 165         for(addr=0;addr<16;addr++) {
 166                 sdram_dfii_pird_address_write(addr*8);
 167                 sdram_dfii_pird_baddress_write(0);
 168                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 169                 cdelay(15);
 170                 for(p=0;p<DFII_NPHASES;p++)
 171                         for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 172                                 prev_data[p*DFII_PIX_DATA_SIZE+i] = MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*i);
 173
 174                 for(j=0;j<_count;j++) {
 175                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 176                         cdelay(15);
 177                         for(p=0;p<DFII_NPHASES;p++)
 178                                 for(i=0;i<DFII_PIX_DATA_SIZE;i++) {
 179                                         unsigned char new_data;
 180
 181                                         new_data = MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*i);
 182                                         errs[p*DFII_PIX_DATA_SIZE+i] |= prev_data[p*DFII_PIX_DATA_SIZE+i] ^ new_data;
 183                                         prev_data[p*DFII_PIX_DATA_SIZE+i] = new_data;
 184                                 }
 185                 }
 186         }
 187
 188         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 189                 printf("%02x", errs[i]);
 190         printf("\n");
 191         for(p=0;p<DFII_NPHASES;p++)
 192                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 193                         printf("%2x", DFII_PIX_DATA_SIZE/2 - 1 - (i % (DFII_PIX_DATA_SIZE/2)));
 194         printf("\n");
 195 }
 196
 197 void sdrwr(char *startaddr)
 198 {
 199         char *c;
 200         unsigned int addr;
 201         int i;
 202         int p;
 203
 204         if(*startaddr == 0) {
 205                 printf("sdrrd <address>\n");
 206                 return;
 207         }
 208         addr = strtoul(startaddr, &c, 0);
 209         if(*c != 0) {
 210                 printf("incorrect address\n");
 211                 return;
 212         }
 213
 214         for(p=0;p<DFII_NPHASES;p++)
 215                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 216                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+DFII_ADDR_SHIFT*i) = 0x10*p + i;
 217
 218         sdram_dfii_piwr_address_write(addr);
 219         sdram_dfii_piwr_baddress_write(0);
 220         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 221 }
 222
 223 #ifdef CSR_DDRPHY_BASE
 224
 225 #if defined (USDDRPHY)
 226 #define ERR_DDRPHY_DELAY 512
 227 #define ERR_DDRPHY_BITSLIP 8
 228 #define NBMODULES DFII_PIX_DATA_SIZE/2
 229 #elif defined (ECP5DDRPHY)
 230 #define ERR_DDRPHY_DELAY 8
 231 #define ERR_DDRPHY_BITSLIP 1
 232 #define NBMODULES DFII_PIX_DATA_SIZE/4
 233 #else
 234 #define ERR_DDRPHY_DELAY 32
 235 #define ERR_DDRPHY_BITSLIP 8
 236 #define NBMODULES DFII_PIX_DATA_SIZE/2
 237 #endif
 238
 239 #ifdef CSR_DDRPHY_WLEVEL_EN_ADDR
 240
 241 void sdrwlon(void)
 242 {
 243         sdram_dfii_pi0_address_write(DDRX_MR1 | (1 << 7));
 244         sdram_dfii_pi0_baddress_write(1);
 245         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 246         ddrphy_wlevel_en_write(1);
 247 }
 248
 249 void sdrwloff(void)
 250 {
 251         sdram_dfii_pi0_address_write(DDRX_MR1);
 252         sdram_dfii_pi0_baddress_write(1);
 253         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 254         ddrphy_wlevel_en_write(0);
 255 }
 256
 257 static void write_delay_rst(int module) {
 258         int i;
 259
 260         /* sel module */
 261         ddrphy_dly_sel_write(1 << module);
 262
 263         /* rst delay */
 264         ddrphy_wdly_dq_rst_write(1);
 265         ddrphy_wdly_dqs_rst_write(1);
 266 #ifdef USDDRPHY /* need to init manually on Ultrascale */
 267         for(i=0; i<ddrphy_half_sys8x_taps_read(); i++)
 268                 ddrphy_wdly_dqs_inc_write(1);
 269 #endif
 270
 271         /* unsel module */
 272         ddrphy_dly_sel_write(0);
 273 }
 274
 275 static void write_delay_inc(int module) {
 276         /* sel module */
 277         ddrphy_dly_sel_write(1 << module);
 278
 279         /* inc delay */
 280         ddrphy_wdly_dq_inc_write(1);
 281         ddrphy_wdly_dqs_inc_write(1);
 282
 283         /* unsel module */
 284         ddrphy_dly_sel_write(0);
 285 }
 286
 287 int write_level(void)
 288 {
 289         int i, j, k;
 290
 291         int dq_address;
 292         unsigned char dq;
 293
 294         int err_ddrphy_wdly;
 295
 296         unsigned char taps_scan[ERR_DDRPHY_DELAY];
 297
 298         int one_window_active;
 299         int one_window_start, one_window_best_start;
 300         int one_window_count, one_window_best_count;
 301
 302         int delays[NBMODULES];
 303
 304         int ok;
 305
 306         err_ddrphy_wdly = ERR_DDRPHY_DELAY - ddrphy_half_sys8x_taps_read();
 307
 308         printf("Write leveling:\n");
 309
 310         sdrwlon();
 311         cdelay(100);
 312         for(i=0;i<NBMODULES;i++) {
 313                 printf("m%d: |", i);
 314                 dq_address = sdram_dfii_pix_rddata_addr[0]+DFII_ADDR_SHIFT*(NBMODULES-1-i);
 315
 316                 /* rst delay */
 317                 write_delay_rst(i);
 318
 319                 /* scan write delay taps */
 320                 for(j=0;j<err_ddrphy_wdly;j++) {
 321                         int zero_count = 0;
 322                         int one_count = 0;
 323                         int show = 1;
 324 #ifdef USDDRPHY
 325                         show = (j%16 == 0);
 326 #endif
 327                         for (k=0; k<128; k++) {
 328                                 ddrphy_wlevel_strobe_write(1);
 329                                 cdelay(10);
 330                                 dq = MMPTR(dq_address);
 331                                 if (dq != 0)
 332                                         one_count++;
 333                                 else
 334                                         zero_count++;
 335                         }
 336                         if (one_count > zero_count)
 337                                 taps_scan[j] = 1;
 338                         else
 339                                 taps_scan[j] = 0;
 340                         if (show)
 341                                 printf("%d", taps_scan[j]);
 342                         write_delay_inc(i);
 343                         cdelay(10);
 344                 }
 345                 printf("|");
 346
 347                 /* find longer 1 window and set delay at the 0/1 transition */
 348                 one_window_active = 0;
 349                 one_window_start = 0;
 350                 one_window_count = 0;
 351                 one_window_best_start = 0;
 352                 one_window_best_count = 0;
 353                 delays[i] = -1;
 354                 for(j=0;j<err_ddrphy_wdly;j++) {
 355                         if (one_window_active) {
 356                                 if ((taps_scan[j] == 0) | (j == err_ddrphy_wdly - 1)) {
 357                                         one_window_active = 0;
 358                                         one_window_count = j - one_window_start;
 359                                         if (one_window_count > one_window_best_count) {
 360                                                 one_window_best_start = one_window_start;
 361                                                 one_window_best_count = one_window_count;
 362                                         }
 363                                 }
 364                         } else {
 365                                 if (taps_scan[j]) {
 366                                         one_window_active = 1;
 367                                         one_window_start = j;
 368                                 }
 369                         }
 370                 }
 371                 delays[i] = one_window_best_start;
 372
 373                 /* configure write delay */
 374                 write_delay_rst(i);
 375                 for(j=0; j<delays[i]; j++)
 376                         write_delay_inc(i);
 377                 printf(" delay: %02d\n", delays[i]);
 378         }
 379
 380         sdrwloff();
 381
 382         ok = 1;
 383         for(i=NBMODULES-1;i>=0;i--) {
 384                 if(delays[i] < 0)
 385                         ok = 0;
 386         }
 387
 388         return ok;
 389 }
 390
 391 #endif /* CSR_DDRPHY_WLEVEL_EN_ADDR */
 392
 393 static void read_delay_rst(int module) {
 394         /* sel module */
 395         ddrphy_dly_sel_write(1 << module);
 396
 397         /* rst delay */
 398         ddrphy_rdly_dq_rst_write(1);
 399
 400         /* unsel module */
 401         ddrphy_dly_sel_write(0);
 402 }
 403
 404 static void read_delay_inc(int module) {
 405         /* sel module */
 406         ddrphy_dly_sel_write(1 << module);
 407
 408         /* inc delay */
 409         ddrphy_rdly_dq_inc_write(1);
 410
 411         /* unsel module */
 412         ddrphy_dly_sel_write(0);
 413 }
 414
 415 static void read_bitslip_rst(char m)
 416 {
 417         /* sel module */
 418         ddrphy_dly_sel_write(1 << m);
 419
 420         /* inc delay */
 421         ddrphy_rdly_dq_bitslip_rst_write(1);
 422
 423         /* unsel module */
 424         ddrphy_dly_sel_write(0);
 425 }
 426
 427
 428 static void read_bitslip_inc(char m)
 429 {
 430         /* sel module */
 431         ddrphy_dly_sel_write(1 << m);
 432
 433         /* inc delay */
 434         ddrphy_rdly_dq_bitslip_write(1);
 435
 436         /* unsel module */
 437         ddrphy_dly_sel_write(0);
 438 }
 439
 440 static int read_level_scan(int module, int bitslip)
 441 {
 442         unsigned int prv;
 443         unsigned char prs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 444         int p, i, j;
 445         int score;
 446
 447         /* Generate pseudo-random sequence */
 448         prv = 42;
 449         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++) {
 450                 prv = 1664525*prv + 1013904223;
 451                 prs[i] = prv;
 452         }
 453
 454         /* Activate */
 455         sdram_dfii_pi0_address_write(0);
 456         sdram_dfii_pi0_baddress_write(0);
 457         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
 458         cdelay(15);
 459
 460         /* Write test pattern */
 461         for(p=0;p<DFII_NPHASES;p++)
 462                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 463                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+DFII_ADDR_SHIFT*i) = prs[DFII_PIX_DATA_SIZE*p+i];
 464         sdram_dfii_piwr_address_write(0);
 465         sdram_dfii_piwr_baddress_write(0);
 466         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 467
 468         /* Calibrate each DQ in turn */
 469         sdram_dfii_pird_address_write(0);
 470         sdram_dfii_pird_baddress_write(0);
 471         score = 0;
 472
 473         printf("m%d, b%d: |", module, bitslip);
 474         read_delay_rst(module);
 475         for(j=0; j<ERR_DDRPHY_DELAY;j++) {
 476                 int working;
 477                 int show = 1;
 478 #ifdef USDDRPHY
 479                 show = (j%16 == 0);
 480 #endif
 481 #ifdef ECP5DDRPHY
 482                 ddrphy_burstdet_clr_write(1);
 483 #endif
 484                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 485                 cdelay(15);
 486                 working = 1;
 487                 for(p=0;p<DFII_NPHASES;p++) {
 488                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+(NBMODULES-module-1)])
 489                                 working = 0;
 490                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(2*NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+2*NBMODULES-module-1])
 491                                 working = 0;
 492                 }
 493 #ifdef ECP5DDRPHY
 494                 if (((ddrphy_burstdet_seen_read() >> module) & 0x1) != 1)
 495                         working = 0;
 496 #endif
 497                 if (show)
 498                         printf("%d", working);
 499                 score += working;
 500                 read_delay_inc(module);
 501         }
 502         printf("| ");
 503
 504         /* Precharge */
 505         sdram_dfii_pi0_address_write(0);
 506         sdram_dfii_pi0_baddress_write(0);
 507         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 508         cdelay(15);
 509
 510         return score;
 511 }
 512
 513 static void read_level(int module)
 514 {
 515         unsigned int prv;
 516         unsigned char prs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 517         int p, i, j;
 518         int working;
 519         int delay, delay_min, delay_max;
 520
 521         printf("delays: ");
 522
 523         /* Generate pseudo-random sequence */
 524         prv = 42;
 525         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++) {
 526                 prv = 1664525*prv + 1013904223;
 527                 prs[i] = prv;
 528         }
 529
 530         /* Activate */
 531         sdram_dfii_pi0_address_write(0);
 532         sdram_dfii_pi0_baddress_write(0);
 533         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
 534         cdelay(15);
 535
 536         /* Write test pattern */
 537         for(p=0;p<DFII_NPHASES;p++)
 538                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 539                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_DATA_SIZE*p+i];
 540         sdram_dfii_piwr_address_write(0);
 541         sdram_dfii_piwr_baddress_write(0);
 542         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 543
 544         /* Calibrate each DQ in turn */
 545         sdram_dfii_pird_address_write(0);
 546         sdram_dfii_pird_baddress_write(0);
 547
 548         /* Find smallest working delay */
 549         delay = 0;
 550         read_delay_rst(module);
 551         while(1) {
 552 #ifdef ECP5DDRPHY
 553                 ddrphy_burstdet_clr_write(1);
 554 #endif
 555                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 556                 cdelay(15);
 557                 working = 1;
 558                 for(p=0;p<DFII_NPHASES;p++) {
 559                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+(NBMODULES-module-1)])
 560                                 working = 0;
 561                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(2*NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+2*NBMODULES-module-1])
 562                                 working = 0;
 563                 }
 564 #ifdef ECP5DDRPHY
 565                 if (((ddrphy_burstdet_seen_read() >> module) & 0x1) != 1)
 566                         working = 0;
 567 #endif
 568                 if(working)
 569                         break;
 570                 delay++;
 571                 if(delay >= ERR_DDRPHY_DELAY)
 572                         break;
 573                 read_delay_inc(module);
 574         }
 575         delay_min = delay;
 576
 577         /* Get a bit further into the working zone */
 578 #ifdef USDDRPHY
 579         for(j=0;j<16;j++) {
 580                 delay += 1;
 581                 read_delay_inc(module);
 582         }
 583 #else
 584         delay++;
 585         read_delay_inc(module);
 586 #endif
 587
 588         /* Find largest working delay */
 589         while(1) {
 590 #ifdef ECP5DDRPHY
 591                 ddrphy_burstdet_clr_write(1);
 592 #endif
 593                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 594                 cdelay(15);
 595                 working = 1;
 596                 for(p=0;p<DFII_NPHASES;p++) {
 597                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+(NBMODULES-module-1)])
 598                                 working = 0;
 599                         if(MMPTR(sdram_dfii_pix_rddata_addr[p]+DFII_ADDR_SHIFT*(2*NBMODULES-module-1)) != prs[DFII_PIX_DATA_SIZE*p+2*NBMODULES-module-1])
 600                                 working = 0;
 601                 }
 602 #ifdef ECP5DDRPHY
 603                 if (((ddrphy_burstdet_seen_read() >> module) & 0x1) != 1)
 604                         working = 0;
 605 #endif
 606                 if(!working)
 607                         break;
 608                 delay++;
 609                 if(delay >= ERR_DDRPHY_DELAY)
 610                         break;
 611                 read_delay_inc(module);
 612         }
 613         delay_max = delay;
 614
 615         if (delay_min >= ERR_DDRPHY_DELAY)
 616                 printf("-");
 617         else
 618                 printf("%02d+-%02d", (delay_min+delay_max)/2, (delay_max-delay_min)/2);
 619
 620         /* Set delay to the middle */
 621         read_delay_rst(module);
 622         for(j=0;j<(delay_min+delay_max)/2;j++)
 623                 read_delay_inc(module);
 624
 625         /* Precharge */
 626         sdram_dfii_pi0_address_write(0);
 627         sdram_dfii_pi0_baddress_write(0);
 628         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 629         cdelay(15);
 630 }
 631 #endif /* CSR_DDRPHY_BASE */
 632
 633 #endif /* CSR_SDRAM_BASE */
 634
 635 static unsigned int seed_to_data_32(unsigned int seed, int random)
 636 {
 637         if (random)
 638                 return 1664525*seed + 1013904223;
 639         else
 640                 return seed + 1;
 641 }
 642
 643 static unsigned short seed_to_data_16(unsigned short seed, int random)
 644 {
 645         if (random)
 646                 return 25173*seed + 13849;
 647         else
 648                 return seed + 1;
 649 }
 650
 651 #define ONEZERO 0xAAAAAAAA
 652 #define ZEROONE 0x55555555
 653
 654 #ifndef MEMTEST_BUS_SIZE
 655 #define MEMTEST_BUS_SIZE (512)
 656 #endif
 657
 658 //#define MEMTEST_BUS_DEBUG
 659
 660 static int memtest_bus(void)
 661 {
 662         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 663         int i, errors;
 664         unsigned int rdata;
 665
 666         errors = 0;
 667
 668         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 669                 array[i] = ONEZERO;
 670         }
 671         flush_cpu_dcache();
 672 #ifdef CONFIG_L2_SIZE
 673         flush_l2_cache();
 674 #endif
 675         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 676                 rdata = array[i];
 677                 if(rdata != ONEZERO) {
 678                         errors++;
 679 #ifdef MEMTEST_BUS_DEBUG
 680                         printf("[bus: 0x%0x]: 0x%08x vs 0x%08x\n", i, rdata, ONEZERO);
 681 #endif
 682                 }
 683         }
 684
 685         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 686                 array[i] = ZEROONE;
 687         }
 688         flush_cpu_dcache();
 689 #ifdef CONFIG_L2_SIZE
 690         flush_l2_cache();
 691 #endif
 692         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 693                 rdata = array[i];
 694                 if(rdata != ZEROONE) {
 695                         errors++;
 696 #ifdef MEMTEST_BUS_DEBUG
 697                         printf("[bus 0x%0x]: 0x%08x vs 0x%08x\n", i, rdata, ZEROONE);
 698 #endif
 699                 }
 700         }
 701
 702         return errors;
 703 }
 704
 705 #ifndef MEMTEST_DATA_SIZE
 706 #define MEMTEST_DATA_SIZE (2*1024*1024)
 707 #endif
 708 #define MEMTEST_DATA_RANDOM 1
 709
 710 //#define MEMTEST_DATA_DEBUG
 711
 712 static int memtest_data(void)
 713 {
 714         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 715         int i, errors;
 716         unsigned int seed_32;
 717         unsigned int rdata;
 718
 719         errors = 0;
 720         seed_32 = 0;
 721
 722         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 723                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 724                 array[i] = seed_32;
 725         }
 726
 727         seed_32 = 0;
 728         flush_cpu_dcache();
 729 #ifdef CONFIG_L2_SIZE
 730         flush_l2_cache();
 731 #endif
 732         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 733                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 734                 rdata = array[i];
 735                 if(rdata != seed_32) {
 736                         errors++;
 737 #ifdef MEMTEST_DATA_DEBUG
 738                         printf("[data 0x%0x]: 0x%08x vs 0x%08x\n", i, rdata, seed_32);
 739 #endif
 740                 }
 741         }
 742
 743         return errors;
 744 }
 745 #ifndef MEMTEST_ADDR_SIZE
 746 #define MEMTEST_ADDR_SIZE (32*1024)
 747 #endif
 748 #define MEMTEST_ADDR_RANDOM 0
 749
 750 //#define MEMTEST_ADDR_DEBUG
 751
 752 static int memtest_addr(void)
 753 {
 754         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 755         int i, errors;
 756         unsigned short seed_16;
 757         unsigned short rdata;
 758
 759         errors = 0;
 760         seed_16 = 0;
 761
 762         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 763                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 764                 array[(unsigned int) seed_16] = i;
 765         }
 766
 767         seed_16 = 0;
 768         flush_cpu_dcache();
 769 #ifdef CONFIG_L2_SIZE
 770         flush_l2_cache();
 771 #endif
 772         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 773                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 774                 rdata = array[(unsigned int) seed_16];
 775                 if(rdata != i) {
 776                         errors++;
 777 #ifdef MEMTEST_ADDR_DEBUG
 778                         printf("[addr 0x%0x]: 0x%08x vs 0x%08x\n", i, rdata, i);
 779 #endif
 780                 }
 781         }
 782
 783         return errors;
 784 }
 785
 786 int memtest(void)
 787 {
 788         int bus_errors, data_errors, addr_errors;
 789
 790         bus_errors = memtest_bus();
 791         if(bus_errors != 0)
 792                 printf("Memtest bus failed: %d/%d errors\n", bus_errors, 2*128);
 793
 794         data_errors = memtest_data();
 795         if(data_errors != 0)
 796                 printf("Memtest data failed: %d/%d errors\n", data_errors, MEMTEST_DATA_SIZE/4);
 797
 798         addr_errors = memtest_addr();
 799         if(addr_errors != 0)
 800                 printf("Memtest addr failed: %d/%d errors\n", addr_errors, MEMTEST_ADDR_SIZE/4);
 801
 802         if(bus_errors + data_errors + addr_errors != 0)
 803                 return 0;
 804         else {
 805                 printf("Memtest OK\n");
 806                 return 1;
 807         }
 808 }
 809
 810 #ifdef CSR_SDRAM_BASE
 811
 812 #ifdef CSR_DDRPHY_BASE
 813 int sdrlevel(void)
 814 {
 815         int module;
 816         int bitslip;
 817         int score;
 818         int best_score;
 819         int best_bitslip;
 820
 821         sdrsw();
 822
 823         for(module=0; module<NBMODULES; module++) {
 824 #ifdef CSR_DDRPHY_WLEVEL_EN_ADDR
 825                 write_delay_rst(module);
 826 #endif
 827                 read_delay_rst(module);
 828                 read_bitslip_rst(module);
 829         }
 830
 831 #ifdef CSR_DDRPHY_WLEVEL_EN_ADDR
 832         if(!write_level())
 833                 return 0;
 834 #endif
 835
 836         printf("Read leveling:\n");
 837         for(module=0; module<NBMODULES; module++) {
 838                 /* scan possible read windows */
 839                 best_score = 0;
 840                 best_bitslip = 0;
 841                 for(bitslip=0; bitslip<ERR_DDRPHY_BITSLIP; bitslip++) {
 842                         /* compute score */
 843                         score = read_level_scan(module, bitslip);
 844                         read_level(module);
 845                         printf("\n");
 846                         if (score > best_score) {
 847                                 best_bitslip = bitslip;
 848                                 best_score = score;
 849                         }
 850                         /* exit */
 851                         if (bitslip == ERR_DDRPHY_BITSLIP-1)
 852                                 break;
 853                         /* increment bitslip */
 854                         read_bitslip_inc(module);
 855                 }
 856
 857                 /* select best read window */
 858                 printf("best: m%d, b%d ", module, best_bitslip);
 859                 read_bitslip_rst(module);
 860                 for (bitslip=0; bitslip<best_bitslip; bitslip++)
 861                         read_bitslip_inc(module);
 862
 863                 /* re-do leveling on best read window*/
 864                 read_level(module);
 865                 printf("\n");
 866         }
 867
 868         return 1;
 869 }
 870 #endif
 871
 872 int sdrinit(void)
 873 {
 874         printf("Initializing SDRAM...\n");
 875
 876 #ifdef CSR_DDRCTRL_BASE
 877         ddrctrl_init_done_write(0);
 878         ddrctrl_init_error_write(0);
 879 #endif
 880
 881         init_sequence();
 882 #ifdef CSR_DDRPHY_BASE
 883 #if CSR_DDRPHY_EN_VTC_ADDR
 884         ddrphy_en_vtc_write(0);
 885 #endif
 886         sdrlevel();
 887 #if CSR_DDRPHY_EN_VTC_ADDR
 888         ddrphy_en_vtc_write(1);
 889 #endif
 890 #endif
 891         sdrhw();
 892 #ifdef CSR_DDRCTRL_BASE
 893         ddrctrl_init_done_write(1);
 894 #endif
 895         if(!memtest()) {
 896 #ifdef CSR_DDRCTRL_BASE
 897                 ddrctrl_init_error_write(1);
 898 #endif
 899                 return 0;
 900         }
 901
 902         return 1;
 903 }
 904
 905 #endif