litex/soc/software/bios/sdram.c

   1 #include <generated/csr.h>
   2 #ifdef CSR_SDRAM_BASE
   3
   4 #include <stdio.h>
   5 #include <stdlib.h>
   6
   7 #include <generated/sdram_phy.h>
   8 #include <generated/mem.h>
   9 #include <hw/flags.h>
  10 #include <system.h>
  11
  12 #include "sdram.h"
  13
  14 static void cdelay(int i)
  15 {
  16         while(i > 0) {
  17 #if defined (__lm32__)
  18                 __asm__ volatile("nop");
  19 #elif defined (__or1k__)
  20                 __asm__ volatile("l.nop");
  21 #elif defined (__picorv32__)
  22                 __asm__ volatile("nop");
  23 #elif defined (__vexriscv__)
  24                 __asm__ volatile("nop");
  25 #elif defined (__minerva__)
  26                 __asm__ volatile("nop");
  27 #else
  28 #error Unsupported architecture
  29 #endif
  30                 i--;
  31         }
  32 }
  33
  34 void sdrsw(void)
  35 {
  36         sdram_dfii_control_write(DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N);
  37         printf("SDRAM now under software control\n");
  38 }
  39
  40 void sdrhw(void)
  41 {
  42         sdram_dfii_control_write(DFII_CONTROL_SEL);
  43         printf("SDRAM now under hardware control\n");
  44 }
  45
  46 void sdrrow(char *_row)
  47 {
  48         char *c;
  49         unsigned int row;
  50
  51         if(*_row == 0) {
  52                 sdram_dfii_pi0_address_write(0x0000);
  53                 sdram_dfii_pi0_baddress_write(0);
  54                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
  55                 cdelay(15);
  56                 printf("Precharged\n");
  57         } else {
  58                 row = strtoul(_row, &c, 0);
  59                 if(*c != 0) {
  60                         printf("incorrect row\n");
  61                         return;
  62                 }
  63                 sdram_dfii_pi0_address_write(row);
  64                 sdram_dfii_pi0_baddress_write(0);
  65                 command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
  66                 cdelay(15);
  67                 printf("Activated row %d\n", row);
  68         }
  69 }
  70
  71 void sdrrdbuf(int dq)
  72 {
  73         int i, p;
  74         int first_byte, step;
  75
  76         if(dq < 0) {
  77                 first_byte = 0;
  78                 step = 1;
  79         } else {
  80                 first_byte = DFII_PIX_DATA_SIZE/2 - 1 - dq;
  81                 step = DFII_PIX_DATA_SIZE/2;
  82         }
  83
  84         for(p=0;p<DFII_NPHASES;p++)
  85                 for(i=first_byte;i<DFII_PIX_DATA_SIZE;i+=step)
  86                         printf("%02x", MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i));
  87         printf("\n");
  88 }
  89
  90 void sdrrd(char *startaddr, char *dq)
  91 {
  92         char *c;
  93         unsigned int addr;
  94         int _dq;
  95
  96         if(*startaddr == 0) {
  97                 printf("sdrrd <address>\n");
  98                 return;
  99         }
 100         addr = strtoul(startaddr, &c, 0);
 101         if(*c != 0) {
 102                 printf("incorrect address\n");
 103                 return;
 104         }
 105         if(*dq == 0)
 106                 _dq = -1;
 107         else {
 108                 _dq = strtoul(dq, &c, 0);
 109                 if(*c != 0) {
 110                         printf("incorrect DQ\n");
 111                         return;
 112                 }
 113         }
 114
 115         sdram_dfii_pird_address_write(addr);
 116         sdram_dfii_pird_baddress_write(0);
 117         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 118         cdelay(15);
 119         sdrrdbuf(_dq);
 120 }
 121
 122 void sdrrderr(char *count)
 123 {
 124         int addr;
 125         char *c;
 126         int _count;
 127         int i, j, p;
 128         unsigned char prev_data[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 129         unsigned char errs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 130
 131         if(*count == 0) {
 132                 printf("sdrrderr <count>\n");
 133                 return;
 134         }
 135         _count = strtoul(count, &c, 0);
 136         if(*c != 0) {
 137                 printf("incorrect count\n");
 138                 return;
 139         }
 140
 141         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 142                         errs[i] = 0;
 143         for(addr=0;addr<16;addr++) {
 144                 sdram_dfii_pird_address_write(addr*8);
 145                 sdram_dfii_pird_baddress_write(0);
 146                 command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 147                 cdelay(15);
 148                 for(p=0;p<DFII_NPHASES;p++)
 149                         for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 150                                 prev_data[p*DFII_PIX_DATA_SIZE+i] = MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i);
 151
 152                 for(j=0;j<_count;j++) {
 153                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 154                         cdelay(15);
 155                         for(p=0;p<DFII_NPHASES;p++)
 156                                 for(i=0;i<DFII_PIX_DATA_SIZE;i++) {
 157                                         unsigned char new_data;
 158
 159                                         new_data = MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i);
 160                                         errs[p*DFII_PIX_DATA_SIZE+i] |= prev_data[p*DFII_PIX_DATA_SIZE+i] ^ new_data;
 161                                         prev_data[p*DFII_PIX_DATA_SIZE+i] = new_data;
 162                                 }
 163                 }
 164         }
 165
 166         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++)
 167                 printf("%02x", errs[i]);
 168         printf("\n");
 169         for(p=0;p<DFII_NPHASES;p++)
 170                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 171                         printf("%2x", DFII_PIX_DATA_SIZE/2 - 1 - (i % (DFII_PIX_DATA_SIZE/2)));
 172         printf("\n");
 173 }
 174
 175 void sdrwr(char *startaddr)
 176 {
 177         char *c;
 178         unsigned int addr;
 179         int i;
 180         int p;
 181
 182         if(*startaddr == 0) {
 183                 printf("sdrrd <address>\n");
 184                 return;
 185         }
 186         addr = strtoul(startaddr, &c, 0);
 187         if(*c != 0) {
 188                 printf("incorrect address\n");
 189                 return;
 190         }
 191
 192         for(p=0;p<DFII_NPHASES;p++)
 193                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 194                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = 0x10*p + i;
 195
 196         sdram_dfii_piwr_address_write(addr);
 197         sdram_dfii_piwr_baddress_write(0);
 198         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 199 }
 200
 201 #ifdef CSR_DDRPHY_BASE
 202
 203 #ifdef KUSDDRPHY
 204 #define ERR_DDRPHY_DELAY 512
 205 #else
 206 #define ERR_DDRPHY_DELAY 32
 207 #endif
 208 #define ERR_DDRPHY_BITSLIP 8
 209
 210 #ifdef CSR_DDRPHY_WLEVEL_EN_ADDR
 211
 212 void sdrwlon(void)
 213 {
 214         sdram_dfii_pi0_address_write(DDR3_MR1 | (1 << 7));
 215         sdram_dfii_pi0_baddress_write(1);
 216         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 217         ddrphy_wlevel_en_write(1);
 218 }
 219
 220 void sdrwloff(void)
 221 {
 222         sdram_dfii_pi0_address_write(DDR3_MR1);
 223         sdram_dfii_pi0_baddress_write(1);
 224         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 225         ddrphy_wlevel_en_write(0);
 226 }
 227
 228 static void write_level_scan(void)
 229 {
 230         int i, j;
 231         int dq_address;
 232         unsigned char dq;
 233
 234         printf("Write leveling scan:\n");
 235
 236         sdrwlon();
 237         cdelay(100);
 238         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++) {
 239                 printf("m%d: ", i);
 240             dq_address = sdram_dfii_pix_rddata_addr[0]+4*(DFII_PIX_DATA_SIZE/2-1-i);
 241                 ddrphy_dly_sel_write(1 << i);
 242                 ddrphy_wdly_dq_rst_write(1);
 243                 ddrphy_wdly_dqs_rst_write(1);
 244                 for(j=0;j<ERR_DDRPHY_DELAY - ddrphy_half_sys8x_taps_read();j++) {
 245                         ddrphy_wlevel_strobe_write(1);
 246                         cdelay(10);
 247                         dq = MMPTR(dq_address);
 248                         printf("%d", dq != 0);
 249                         ddrphy_wdly_dq_inc_write(1);
 250                         ddrphy_wdly_dqs_inc_write(1);
 251                         cdelay(10);
 252                 }
 253                 printf("\n");
 254         }
 255         sdrwloff();
 256 }
 257
 258 static int write_level(int *delay, int *high_skew)
 259 {
 260         int i;
 261         int dq_address;
 262         unsigned char dq;
 263         int err_ddrphy_wdly;
 264         int ok;
 265
 266         err_ddrphy_wdly = ERR_DDRPHY_DELAY - ddrphy_half_sys8x_taps_read();
 267
 268         printf("Write leveling: ");
 269
 270         sdrwlon();
 271         cdelay(100);
 272         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++) {
 273                 dq_address = sdram_dfii_pix_rddata_addr[0]+4*(DFII_PIX_DATA_SIZE/2-1-i);
 274                 ddrphy_dly_sel_write(1 << i);
 275                 ddrphy_wdly_dq_rst_write(1);
 276                 ddrphy_wdly_dqs_rst_write(1);
 277 #ifdef KUSDDRPHY /* Need to init manually on Ultrascale */
 278                 int j;
 279                 for(j=0; j<ddrphy_wdly_dqs_taps_read(); j++)
 280                         ddrphy_wdly_dqs_inc_write(1);
 281 #endif
 282
 283                 delay[i] = 0;
 284
 285                 ddrphy_wlevel_strobe_write(1);
 286                 cdelay(10);
 287                 dq = MMPTR(dq_address);
 288                 if(dq != 0) {
 289                         /*
 290                          * Assume this DQ group has between 1 and 2 bit times of skew.
 291                          * Bring DQS into the CK=0 zone before continuing leveling.
 292                          */
 293 #ifndef DDRPHY_HIGH_SKEW_DISABLE
 294                         high_skew[i] = 1;
 295                         while(dq != 0) {
 296                                 delay[i]++;
 297                                 if(delay[i] >= err_ddrphy_wdly)
 298                                         break;
 299                                 ddrphy_wdly_dq_inc_write(1);
 300                                 ddrphy_wdly_dqs_inc_write(1);
 301                                 ddrphy_wlevel_strobe_write(1);
 302                                 cdelay(10);
 303                                 dq = MMPTR(dq_address);
 304                          }
 305 #else
 306                         high_skew[i] = 0;
 307 #endif
 308                 } else
 309                         high_skew[i] = 0;
 310
 311                 while(dq == 0) {
 312                         delay[i]++;
 313                         if(delay[i] >= err_ddrphy_wdly)
 314                                 break;
 315                         ddrphy_wdly_dq_inc_write(1);
 316                         ddrphy_wdly_dqs_inc_write(1);
 317
 318                         ddrphy_wlevel_strobe_write(1);
 319                         cdelay(10);
 320                         dq = MMPTR(dq_address);
 321                 }
 322         }
 323         sdrwloff();
 324
 325         ok = 1;
 326         for(i=DFII_PIX_DATA_SIZE/2-1;i>=0;i--) {
 327                 printf("%2d%c ", delay[i], high_skew[i] ? '*' : ' ');
 328                 if(delay[i] >= err_ddrphy_wdly)
 329                         ok = 0;
 330         }
 331
 332         if(ok)
 333                 printf("completed\n");
 334         else
 335                 printf("failed\n");
 336
 337         return ok;
 338 }
 339
 340 #endif /* CSR_DDRPHY_WLEVEL_EN_ADDR */
 341
 342 static void read_bitslip_inc(char m)
 343 {
 344                 ddrphy_dly_sel_write(1 << m);
 345 #ifdef KUSDDRPHY
 346                 ddrphy_rdly_dq_bitslip_write(1);
 347 #else
 348                 /* 7-series SERDES in DDR mode needs 3 pulses for 1 bitslip */
 349                 ddrphy_rdly_dq_bitslip_write(1);
 350                 ddrphy_rdly_dq_bitslip_write(1);
 351                 ddrphy_rdly_dq_bitslip_write(1);
 352 #endif
 353 }
 354
 355 static void read_bitslip(int *delay, int *high_skew)
 356 {
 357         int bitslip_thr;
 358         int i;
 359
 360         bitslip_thr = 0x7fffffff;
 361         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++)
 362                 if(high_skew[i] && (delay[i] < bitslip_thr))
 363                         bitslip_thr = delay[i];
 364         if(bitslip_thr == 0x7fffffff)
 365                 return;
 366         bitslip_thr = bitslip_thr/2;
 367
 368         printf("Read bitslip: ");
 369         for(i=DFII_PIX_DATA_SIZE/2-1;i>=0;i--)
 370                 if(delay[i] > bitslip_thr) {
 371                         read_bitslip_inc(i);
 372                         printf("%d ", i);
 373                 }
 374         printf("\n");
 375 }
 376
 377 static int read_level_scan(int silent)
 378 {
 379         unsigned int prv;
 380         unsigned char prs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 381         int p, i, j;
 382         int score;
 383
 384         if (!silent)
 385                 printf("Read delays scan:\n");
 386
 387         /* Generate pseudo-random sequence */
 388         prv = 42;
 389         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++) {
 390                 prv = 1664525*prv + 1013904223;
 391                 prs[i] = prv;
 392         }
 393
 394         /* Activate */
 395         sdram_dfii_pi0_address_write(0);
 396         sdram_dfii_pi0_baddress_write(0);
 397         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
 398         cdelay(15);
 399
 400         /* Write test pattern */
 401         for(p=0;p<DFII_NPHASES;p++)
 402                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 403                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_DATA_SIZE*p+i];
 404         sdram_dfii_piwr_address_write(0);
 405         sdram_dfii_piwr_baddress_write(0);
 406         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 407
 408         /* Calibrate each DQ in turn */
 409         sdram_dfii_pird_address_write(0);
 410         sdram_dfii_pird_baddress_write(0);
 411         score = 0;
 412         for(i=DFII_PIX_DATA_SIZE/2-1;i>=0;i--) {
 413                 if (!silent)
 414                         printf("m%d: ", (DFII_PIX_DATA_SIZE/2-i-1));
 415                 ddrphy_dly_sel_write(1 << (DFII_PIX_DATA_SIZE/2-i-1));
 416                 ddrphy_rdly_dq_rst_write(1);
 417                 for(j=0; j<ERR_DDRPHY_DELAY;j++) {
 418                         int working;
 419                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 420                         cdelay(15);
 421                         working = 1;
 422                         for(p=0;p<DFII_NPHASES;p++) {
 423                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_DATA_SIZE*p+i])
 424                                         working = 0;
 425                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_DATA_SIZE/2)) != prs[DFII_PIX_DATA_SIZE*p+i+DFII_PIX_DATA_SIZE/2])
 426                                         working = 0;
 427                         }
 428                         if (!silent)
 429                                 printf("%d", working);
 430                         score += working;
 431                         ddrphy_rdly_dq_inc_write(1);
 432                 }
 433                 if (!silent)
 434                         printf("\n");
 435         }
 436
 437         /* Precharge */
 438         sdram_dfii_pi0_address_write(0);
 439         sdram_dfii_pi0_baddress_write(0);
 440         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 441         cdelay(15);
 442
 443         return score;
 444 }
 445
 446 static void read_level(void)
 447 {
 448         unsigned int prv;
 449         unsigned char prs[DFII_NPHASES*DFII_PIX_DATA_SIZE];
 450         int p, i, j;
 451         int working;
 452         int delay, delay_min, delay_max;
 453
 454         printf("Read delays: ");
 455
 456         /* Generate pseudo-random sequence */
 457         prv = 42;
 458         for(i=0;i<DFII_NPHASES*DFII_PIX_DATA_SIZE;i++) {
 459                 prv = 1664525*prv + 1013904223;
 460                 prs[i] = prv;
 461         }
 462
 463         /* Activate */
 464         sdram_dfii_pi0_address_write(0);
 465         sdram_dfii_pi0_baddress_write(0);
 466         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS);
 467         cdelay(15);
 468
 469         /* Write test pattern */
 470         for(p=0;p<DFII_NPHASES;p++)
 471                 for(i=0;i<DFII_PIX_DATA_SIZE;i++)
 472                         MMPTR(sdram_dfii_pix_wrdata_addr[p]+4*i) = prs[DFII_PIX_DATA_SIZE*p+i];
 473         sdram_dfii_piwr_address_write(0);
 474         sdram_dfii_piwr_baddress_write(0);
 475         command_pwr(DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS|DFII_COMMAND_WRDATA);
 476
 477         /* Calibrate each DQ in turn */
 478         sdram_dfii_pird_address_write(0);
 479         sdram_dfii_pird_baddress_write(0);
 480         for(i=0;i<DFII_PIX_DATA_SIZE/2;i++) {
 481                 ddrphy_dly_sel_write(1 << (DFII_PIX_DATA_SIZE/2-i-1));
 482                 delay = 0;
 483
 484                 /* Find smallest working delay */
 485                 ddrphy_rdly_dq_rst_write(1);
 486                 while(1) {
 487                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 488                         cdelay(15);
 489                         working = 1;
 490                         for(p=0;p<DFII_NPHASES;p++) {
 491                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_DATA_SIZE*p+i])
 492                                         working = 0;
 493                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_DATA_SIZE/2)) != prs[DFII_PIX_DATA_SIZE*p+i+DFII_PIX_DATA_SIZE/2])
 494                                         working = 0;
 495                         }
 496                         if(working)
 497                                 break;
 498                         delay++;
 499                         if(delay >= ERR_DDRPHY_DELAY)
 500                                 break;
 501                         ddrphy_rdly_dq_inc_write(1);
 502                 }
 503                 delay_min = delay;
 504
 505                 /* Get a bit further into the working zone */
 506 #ifdef KUSDDRPHY
 507                 for(j=0;j<16;j++) {
 508                         delay += 1;
 509                         ddrphy_rdly_dq_inc_write(1);
 510                 }
 511 #else
 512                 delay++;
 513                 ddrphy_rdly_dq_inc_write(1);
 514 #endif
 515
 516                 /* Find largest working delay */
 517                 while(1) {
 518                         command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA);
 519                         cdelay(15);
 520                         working = 1;
 521                         for(p=0;p<DFII_NPHASES;p++) {
 522                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*i) != prs[DFII_PIX_DATA_SIZE*p+i])
 523                                         working = 0;
 524                                 if(MMPTR(sdram_dfii_pix_rddata_addr[p]+4*(i+DFII_PIX_DATA_SIZE/2)) != prs[DFII_PIX_DATA_SIZE*p+i+DFII_PIX_DATA_SIZE/2])
 525                                         working = 0;
 526                         }
 527                         if(!working)
 528                                 break;
 529                         delay++;
 530                         if(delay >= ERR_DDRPHY_DELAY)
 531                                 break;
 532                         ddrphy_rdly_dq_inc_write(1);
 533                 }
 534                 delay_max = delay;
 535
 536                 printf("%d:%02d-%02d  ", DFII_PIX_DATA_SIZE/2-i-1, delay_min, delay_max);
 537
 538                 /* Set delay to the middle */
 539                 ddrphy_rdly_dq_rst_write(1);
 540                 for(j=0;j<(delay_min+delay_max)/2;j++)
 541                         ddrphy_rdly_dq_inc_write(1);
 542         }
 543
 544         /* Precharge */
 545         sdram_dfii_pi0_address_write(0);
 546         sdram_dfii_pi0_baddress_write(0);
 547         command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS);
 548         cdelay(15);
 549
 550         printf("completed\n");
 551 }
 552 #endif /* CSR_DDRPHY_BASE */
 553
 554 static unsigned int seed_to_data_32(unsigned int seed, int random)
 555 {
 556         if (random)
 557                 return 1664525*seed + 1013904223;
 558         else
 559                 return seed + 1;
 560 }
 561
 562 static unsigned short seed_to_data_16(unsigned short seed, int random)
 563 {
 564         if (random)
 565                 return 25173*seed + 13849;
 566         else
 567                 return seed + 1;
 568 }
 569
 570 #define ONEZERO 0xAAAAAAAA
 571 #define ZEROONE 0x55555555
 572
 573 #ifndef MEMTEST_BUS_SIZE
 574 #define MEMTEST_BUS_SIZE (512)
 575 #endif
 576
 577 //#define MEMTEST_BUS_DEBUG
 578
 579 static int memtest_bus(void)
 580 {
 581         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 582         int i, errors;
 583         unsigned int rdata;
 584
 585         errors = 0;
 586
 587         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 588                 array[i] = ONEZERO;
 589         }
 590         flush_cpu_dcache();
 591         flush_l2_cache();
 592         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 593                 rdata = array[i];
 594                 if(rdata != ONEZERO) {
 595                         errors++;
 596 #ifdef MEMTEST_BUS_DEBUG
 597                         printf("[bus: %0x]: %08x vs %08x\n", i, rdata, ONEZERO);
 598 #endif
 599                 }
 600         }
 601
 602         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 603                 array[i] = ZEROONE;
 604         }
 605         flush_cpu_dcache();
 606         flush_l2_cache();
 607         for(i=0;i<MEMTEST_BUS_SIZE/4;i++) {
 608                 rdata = array[i];
 609                 if(rdata != ZEROONE) {
 610                         errors++;
 611 #ifdef MEMTEST_BUS_DEBUG
 612                         printf("[bus %0x]: %08x vs %08x\n", i, rdata, ZEROONE);
 613 #endif
 614                 }
 615         }
 616
 617         return errors;
 618 }
 619
 620 #ifndef MEMTEST_DATA_SIZE
 621 #define MEMTEST_DATA_SIZE (2*1024*1024)
 622 #endif
 623 #define MEMTEST_DATA_RANDOM 1
 624
 625 //#define MEMTEST_DATA_DEBUG
 626
 627 static int memtest_data(void)
 628 {
 629         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 630         int i, errors;
 631         unsigned int seed_32;
 632         unsigned int rdata;
 633
 634         errors = 0;
 635         seed_32 = 0;
 636
 637         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 638                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 639                 array[i] = seed_32;
 640         }
 641
 642         seed_32 = 0;
 643         flush_cpu_dcache();
 644         flush_l2_cache();
 645         for(i=0;i<MEMTEST_DATA_SIZE/4;i++) {
 646                 seed_32 = seed_to_data_32(seed_32, MEMTEST_DATA_RANDOM);
 647                 rdata = array[i];
 648                 if(rdata != seed_32) {
 649                         errors++;
 650 #ifdef MEMTEST_DATA_DEBUG
 651                         printf("[data %0x]: %08x vs %08x\n", i, rdata, seed_32);
 652 #endif
 653                 }
 654         }
 655
 656         return errors;
 657 }
 658 #ifndef MEMTEST_ADDR_SIZE
 659 #define MEMTEST_ADDR_SIZE (32*1024)
 660 #endif
 661 #define MEMTEST_ADDR_RANDOM 0
 662
 663 //#define MEMTEST_ADDR_DEBUG
 664
 665 static int memtest_addr(void)
 666 {
 667         volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE;
 668         int i, errors;
 669         unsigned short seed_16;
 670         unsigned short rdata;
 671
 672         errors = 0;
 673         seed_16 = 0;
 674
 675         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 676                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 677                 array[(unsigned int) seed_16] = i;
 678         }
 679
 680         seed_16 = 0;
 681         flush_cpu_dcache();
 682         flush_l2_cache();
 683         for(i=0;i<MEMTEST_ADDR_SIZE/4;i++) {
 684                 seed_16 = seed_to_data_16(seed_16, MEMTEST_ADDR_RANDOM);
 685                 rdata = array[(unsigned int) seed_16];
 686                 if(rdata != i) {
 687                         errors++;
 688 #ifdef MEMTEST_ADDR_DEBUG
 689                         printf("[addr %0x]: %08x vs %08x\n", i, rdata, i);
 690 #endif
 691                 }
 692         }
 693
 694         return errors;
 695 }
 696
 697 int memtest(void)
 698 {
 699         int bus_errors, data_errors, addr_errors;
 700
 701         bus_errors = memtest_bus();
 702         if(bus_errors != 0)
 703                 printf("Memtest bus failed: %d/%d errors\n", bus_errors, 2*128);
 704
 705         data_errors = memtest_data();
 706         if(data_errors != 0)
 707                 printf("Memtest data failed: %d/%d errors\n", data_errors, MEMTEST_DATA_SIZE/4);
 708
 709         addr_errors = memtest_addr();
 710         if(addr_errors != 0)
 711                 printf("Memtest addr failed: %d/%d errors\n", addr_errors, MEMTEST_ADDR_SIZE/4);
 712
 713         if(bus_errors + data_errors + addr_errors != 0)
 714                 return 0;
 715         else {
 716                 printf("Memtest OK\n");
 717                 return 1;
 718         }
 719 }
 720
 721 #ifdef CSR_DDRPHY_BASE
 722 int sdrlevel(int silent)
 723 {
 724         int delay[DFII_PIX_DATA_SIZE/2];
 725         int high_skew[DFII_PIX_DATA_SIZE/2];
 726         int i, j;
 727         int bitslip;
 728         int score;
 729         int best_score;
 730         int best_bitslip;
 731
 732         sdrsw();
 733
 734         for(i=0; i<DFII_PIX_DATA_SIZE/2; i++) {
 735                 ddrphy_dly_sel_write(1<<i);
 736                 ddrphy_rdly_dq_rst_write(1);
 737                 ddrphy_rdly_dq_bitslip_rst_write(1);
 738         }
 739
 740 #ifndef CSR_DDRPHY_WLEVEL_EN_ADDR
 741         for(i=0; i<DFII_PIX_DATA_SIZE/2; i++) {
 742                 delay[i] = 0;
 743                 high_skew[i] = 0;
 744         }
 745 #else
 746         write_level_scan();
 747         if(!write_level(delay, high_skew))
 748                 return 0;
 749 #endif
 750         /* scan possible read windows */
 751         best_score = 0;
 752         best_bitslip = 0;
 753         for(bitslip=0; bitslip<ERR_DDRPHY_BITSLIP; bitslip++) {
 754                 if (!silent)
 755                         printf("Read bitslip: %d\n", bitslip);
 756                 /* compute score */
 757                 score = read_level_scan(silent);
 758                 if (score > best_score) {
 759                         best_bitslip = bitslip;
 760                         best_score = score;
 761                 }
 762                 /* exit */
 763                 if (bitslip == ERR_DDRPHY_BITSLIP-1)
 764                         break;
 765                 /* increment bitslip */
 766                 for(i=0; i<DFII_PIX_DATA_SIZE/2; i++)
 767                         read_bitslip_inc(i);
 768         }
 769
 770         /* select best read window */
 771         printf("Best read bitslip: %d\n", best_bitslip);
 772         for(i=0; i<DFII_PIX_DATA_SIZE/2; i++) {
 773                 ddrphy_dly_sel_write(1<<i);
 774                 ddrphy_rdly_dq_bitslip_rst_write(1);
 775                 for (j=0; j<best_bitslip; j++)
 776                         read_bitslip_inc(i);
 777         }
 778
 779         /* show scan and do leveling */
 780     read_level_scan(0);
 781         read_level();
 782
 783         return 1;
 784 }
 785 #endif
 786
 787 int sdrinit(void)
 788 {
 789         printf("Initializing SDRAM...\n");
 790
 791         init_sequence();
 792 #ifdef CSR_DDRPHY_BASE
 793 #if CSR_DDRPHY_EN_VTC_ADDR
 794         ddrphy_en_vtc_write(0);
 795 #endif
 796         sdrlevel(1);
 797 #if CSR_DDRPHY_EN_VTC_ADDR
 798         ddrphy_en_vtc_write(1);
 799 #endif
 800 #endif
 801         sdrhw();
 802         if(!memtest()) {
 803 #ifdef CSR_DDRPHY_BASE
 804                 /* show scans */
 805                 sdrlevel(0);
 806 #endif
 807                 return 0;
 808         }
 809
 810         return 1;
 811 }
 812
 813 #endif