From: Florent Kermarrec Date: Mon, 18 May 2020 20:39:59 +0000 (+0200) Subject: software: create liblitedram and move sdram init/test code to it. X-Git-Tag: 24jan2021_ls180~328^2~5 X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=920d0ee53683e59a3dc1cc65e8ab7bcf4f388076;p=litex.git software: create liblitedram and move sdram init/test code to it. --- diff --git a/litex/soc/integration/builder.py b/litex/soc/integration/builder.py index 79c6b95d..64dca9d6 100644 --- a/litex/soc/integration/builder.py +++ b/litex/soc/integration/builder.py @@ -25,6 +25,7 @@ __all__ = ["soc_software_packages", "soc_directory", soc_software_packages = [ "libcompiler_rt", "libbase", + "liblitedram", "libliteeth", "bios" ] diff --git a/litex/soc/software/bios/Makefile b/litex/soc/software/bios/Makefile index 24d33a50..73d9ed0a 100755 --- a/litex/soc/software/bios/Makefile +++ b/litex/soc/software/bios/Makefile @@ -11,7 +11,6 @@ CFLAGS += -DTFTP_SERVER_PORT=$(TFTP_SERVER_PORT) endif OBJECTS = isr.o \ - sdram.o \ sdcard.o \ main.o \ boot-helper.o \ @@ -59,15 +58,20 @@ endif bios.elf: $(BIOS_DIRECTORY)/linker.ld $(OBJECTS) -%.elf: ../libbase/crt0-ctr.o ../libliteeth/libliteeth.a ../libbase/libbase-nofloat.a ../libcompiler_rt/libcompiler_rt.a +%.elf: ../libbase/crt0-ctr.o \ + ../libcompiler_rt/libcompiler_rt.a \ + ../libbase/libbase-nofloat.a \ + ../liblitedram/liblitedram.a \ + ../libliteeth/libliteeth.a $(LD) $(LDFLAGS) -T $(BIOS_DIRECTORY)/linker.ld -N -o $@ \ ../libbase/crt0-ctr.o \ $(OBJECTS) \ - -L../libliteeth \ - -L../libbase \ -L../libcompiler_rt \ + -L../libbase \ + -L../liblitedram \ + -L../libliteeth \ $(BP_LIBS) \ - -lliteeth -lbase-nofloat -lcompiler_rt \ + -lcompiler_rt -lbase-nofloat -llitedram -lliteeth \ $(BP_FLAGS) ifneq ($(OS),Windows_NT) diff --git a/litex/soc/software/bios/lfsr.h b/litex/soc/software/bios/lfsr.h deleted file mode 100644 index 50dfccfc..00000000 --- a/litex/soc/software/bios/lfsr.h +++ /dev/null @@ -1,109 +0,0 @@ -#include - -/* - * Copyright (C) 2020, Anton Blanchard , IBM - * - * Redistribution and use in source and binary forms, with or without modification, - * are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * Galois LFSR - * - * Polynomials verified with https://bitbucket.org/gallen/mlpolygen/ - */ -static inline unsigned long lfsr(unsigned long bits, unsigned long prev) -{ - static const unsigned long lfsr_taps[] = { - 0x0, - 0x0, - 0x3, - 0x6, - 0xc, - 0x14, - 0x30, - 0x60, - 0xb8, - 0x110, - 0x240, - 0x500, - 0x829, - 0x100d, - 0x2015, - 0x6000, - 0xd008, - 0x12000, - 0x20400, - 0x40023, - 0x90000, - 0x140000, - 0x300000, - 0x420000, - 0xe10000, - 0x1200000, - 0x2000023, - 0x4000013, - 0x9000000, - 0x14000000, - 0x20000029, - 0x48000000, - 0x80200003, -#if __WORDSIZE == 64 - 0x100080000, - 0x204000003, - 0x500000000, - 0x801000000, - 0x100000001f, - 0x2000000031, - 0x4400000000, - 0xa000140000, - 0x12000000000, - 0x300000c0000, - 0x63000000000, - 0xc0000030000, - 0x1b0000000000, - 0x300003000000, - 0x420000000000, - 0xc00000180000, - 0x1008000000000, - 0x3000000c00000, - 0x6000c00000000, - 0x9000000000000, - 0x18003000000000, - 0x30000000030000, - 0x40000040000000, - 0xc0000600000000, - 0x102000000000000, - 0x200004000000000, - 0x600003000000000, - 0xc00000000000000, - 0x1800300000000000, - 0x3000000000000030, - 0x6000000000000000, - 0x800000000000000d -#endif - }; - unsigned long lsb = prev & 1; - - prev >>= 1; - prev ^= (-lsb) & lfsr_taps[bits]; - - return prev; -} diff --git a/litex/soc/software/bios/sdram.c b/litex/soc/software/bios/sdram.c deleted file mode 100644 index 81b169cf..00000000 --- a/litex/soc/software/bios/sdram.c +++ /dev/null @@ -1,1066 +0,0 @@ -// This file is Copyright (c) 2013-2014 Sebastien Bourdeauducq -// This file is Copyright (c) 2013-2020 Florent Kermarrec -// This file is Copyright (c) 2018 Chris Ballance -// This file is Copyright (c) 2018 Dolu1990 -// This file is Copyright (c) 2019 Gabriel L. Somlo -// This file is Copyright (c) 2018 Jean-François Nguyen -// This file is Copyright (c) 2018 Sergiusz Bazanski -// This file is Copyright (c) 2018 Tim 'mithro' Ansell -// License: BSD - -#include - -#include -#include - -#ifdef CSR_SDRAM_BASE -#include -#endif -#include -#include -#include - -#include "sdram.h" -#include "lfsr.h" - -// FIXME(hack): If we don't have main ram, just target the sram instead. -#ifndef MAIN_RAM_BASE -#define MAIN_RAM_BASE SRAM_BASE -#endif - -__attribute__((unused)) static void cdelay(int i) -{ - while(i > 0) { - __asm__ volatile(CONFIG_CPU_NOP); - i--; - } -} - -#ifdef CSR_SDRAM_BASE - -#define DFII_ADDR_SHIFT CONFIG_CSR_ALIGNMENT/8 - -#define CSR_DATA_BYTES CONFIG_CSR_DATA_WIDTH/8 - -#define DFII_PIX_DATA_BYTES DFII_PIX_DATA_SIZE*CSR_DATA_BYTES - -void sdrsw(void) -{ - sdram_dfii_control_write(DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N); - printf("SDRAM now under software control\n"); -} - -void sdrhw(void) -{ - sdram_dfii_control_write(DFII_CONTROL_SEL); - printf("SDRAM now under hardware control\n"); -} - -void sdrrow(unsigned int row) -{ - if(row == 0) { - sdram_dfii_pi0_address_write(0x0000); - sdram_dfii_pi0_baddress_write(0); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); - cdelay(15); - } else { - sdram_dfii_pi0_address_write(row); - sdram_dfii_pi0_baddress_write(0); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS); - cdelay(15); - } -} - -void sdrrdbuf(int dq) -{ - int i, p; - int first_byte, step; - unsigned char buf[DFII_PIX_DATA_BYTES]; - - if(dq < 0) { - first_byte = 0; - step = 1; - } else { - first_byte = DFII_PIX_DATA_BYTES/2 - 1 - dq; - step = DFII_PIX_DATA_BYTES/2; - } - - for(p=0;p 0) { - ddrphy_cdly_inc_write(1); - cdelay(1000); - delay--; - } -#if CSR_DDRPHY_EN_VTC_ADDR - ddrphy_en_vtc_write(1); -#endif -} -#endif - -#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE -void sdrwlon(void) -{ - sdram_dfii_pi0_address_write(DDRX_MR1 | (1 << 7)); - sdram_dfii_pi0_baddress_write(1); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); - -#ifdef SDRAM_PHY_DDR4_RDIMM - sdram_dfii_pi0_address_write((DDRX_MR1 | (1 << 7)) ^ 0x2BF8) ; - sdram_dfii_pi0_baddress_write(1 ^ 0xF); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); -#endif - - ddrphy_wlevel_en_write(1); -} - -void sdrwloff(void) -{ - sdram_dfii_pi0_address_write(DDRX_MR1); - sdram_dfii_pi0_baddress_write(1); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); - -#ifdef SDRAM_PHY_DDR4_RDIMM - sdram_dfii_pi0_address_write(DDRX_MR1 ^ 0x2BF8); - sdram_dfii_pi0_baddress_write(1 ^ 0xF); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); -#endif - - ddrphy_wlevel_en_write(0); -} - -static void write_delay_rst(int module) { -#ifdef SDRAM_PHY_WRITE_LEVELING_REINIT - int i; -#endif - - /* sel module */ - ddrphy_dly_sel_write(1 << module); - - /* rst delay */ - ddrphy_wdly_dq_rst_write(1); - ddrphy_wdly_dqs_rst_write(1); -#ifdef SDRAM_PHY_WRITE_LEVELING_REINIT - for(i=0; i 32 - show_iter = (j%16 == 0) && show; -#endif - for (k=0; k zero_count) - taps_scan[j] = 1; - else - taps_scan[j] = 0; - if (show_iter) - printf("%d", taps_scan[j]); - write_delay_inc(i); - cdelay(10); - } - if (show) - printf("|"); - - /* find longer 1 window and set delay at the 0/1 transition */ - one_window_active = 0; - one_window_start = 0; - one_window_count = 0; - one_window_best_start = 0; - one_window_best_count = -1; - delays[i] = -1; - for(j=0;j one_window_best_count) { - one_window_best_start = one_window_start; - one_window_best_count = one_window_count; - } - } - } else { - if (taps_scan[j]) { - one_window_active = 1; - one_window_start = j; - } - } - } - /* succeed only if the start of a 1s window has been found */ - if (one_window_best_count > 0 && one_window_best_start > 0) { - delays[i] = one_window_best_start; - - /* configure write delay */ - write_delay_rst(i); - for(j=0; j=0;i--) { - if(delays[i] < 0) - ok = 0; - } - - return ok; -} - -static void write_level_cdly_range(unsigned int *best_error, int *best_cdly, - int cdly_start, int cdly_stop, int cdly_step) -{ - int cdly; - int cdly_actual = 0; - int delays[SDRAM_PHY_MODULES]; - - /* scan through the range */ - ddrphy_cdly_rst_write(1); - for (cdly = cdly_start; cdly < cdly_stop; cdly += cdly_step) { - /* increment cdly to current value */ - while (cdly_actual < cdly) { - ddrphy_cdly_inc_write(1); - cdelay(10); - cdly_actual++; - } - - /* write level using this delay */ - if (write_level_scan(delays, 8, 0)) { - /* use the mean of delays for error calulation */ - int delay_mean = 0; - for (int i=0; i < SDRAM_PHY_MODULES; ++i) { - delay_mean += delays[i]; - } - delay_mean /= SDRAM_PHY_MODULES; - - /* we want it to be at the start */ - int ideal_delay = 4*SDRAM_PHY_DELAYS/32; - int error = ideal_delay - delay_mean; - if (error < 0) - error *= -1; - - if (error < *best_error) { - *best_cdly = cdly; - *best_error = error; - } - printf("1"); - } else { - printf("0"); - } - } -} - -int write_level(void) -{ - int delays[SDRAM_PHY_MODULES]; - unsigned int best_error = ~0u; - int best_cdly = -1; - int cdly_range_start; - int cdly_range_end; - int cdly_range_step; - - printf("Command/Clk scan:\n"); - - /* Center write leveling by varying cdly. Searching through all possible - * values is slow, but we can use a simple optimization method of iterativly - * scanning smaller ranges with decreasing step */ - cdly_range_start = 0; - cdly_range_end = SDRAM_PHY_DELAYS; - if (SDRAM_PHY_DELAYS > 32) - cdly_range_step = SDRAM_PHY_DELAYS/8; - else - cdly_range_step = 1; - while (cdly_range_step > 0) { - printf("|"); - write_level_cdly_range(&best_error, &best_cdly, - cdly_range_start, cdly_range_end, cdly_range_step); - - /* small optimization - stop if we have zero error */ - if (best_error == 0) - break; - - /* use best result as the middle of next range */ - cdly_range_start = best_cdly - cdly_range_step; - cdly_range_end = best_cdly + cdly_range_step + 1; - if (cdly_range_start < 0) - cdly_range_start = 0; - if (cdly_range_end > 512) - cdly_range_end = 512; - - cdly_range_step /= 4; - } - printf("| best: %d\n", best_cdly); - - /* if we found any working delay then set it */ - if (best_cdly >= 0) { - ddrphy_cdly_rst_write(1); - for (int i = 0; i < best_cdly; ++i) { - ddrphy_cdly_inc_write(1); - cdelay(10); - } - } - - printf("Data scan:\n"); - - /* re-run write leveling the final time */ - if (!write_level_scan(delays, 128, 1)) - return 0; - - return best_cdly >= 0; -} - - -#endif /* SDRAM_PHY_WRITE_LEVELING_CAPABLE */ - -static void read_delay_rst(int module) { - /* sel module */ - ddrphy_dly_sel_write(1 << module); - - /* rst delay */ - ddrphy_rdly_dq_rst_write(1); - - /* unsel module */ - ddrphy_dly_sel_write(0); - -#ifdef SDRAM_PHY_ECP5DDRPHY - /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ - ddrphy_dly_sel_write(0xFF); - ddrphy_dly_sel_write(0); -#endif -} - -static void read_delay_inc(int module) { - /* sel module */ - ddrphy_dly_sel_write(1 << module); - - /* inc delay */ - ddrphy_rdly_dq_inc_write(1); - - /* unsel module */ - ddrphy_dly_sel_write(0); - -#ifdef SDRAM_PHY_ECP5DDRPHY - /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ - ddrphy_dly_sel_write(0xFF); - ddrphy_dly_sel_write(0); -#endif -} - -static void read_bitslip_rst(char m) -{ - /* sel module */ - ddrphy_dly_sel_write(1 << m); - - /* inc delay */ - ddrphy_rdly_dq_bitslip_rst_write(1); - - /* unsel module */ - ddrphy_dly_sel_write(0); -} - - -static void read_bitslip_inc(char m) -{ - /* sel module */ - ddrphy_dly_sel_write(1 << m); - - /* inc delay */ - ddrphy_rdly_dq_bitslip_write(1); - - /* unsel module */ - ddrphy_dly_sel_write(0); -} - -static int read_level_scan(int module, int bitslip) -{ - unsigned int prv; - unsigned char prs[SDRAM_PHY_PHASES][DFII_PIX_DATA_BYTES]; - unsigned char tst[DFII_PIX_DATA_BYTES]; - int p, i; - int score; - - /* Generate pseudo-random sequence */ - prv = 42; - for(p=0;p 32 - show = (i%16 == 0); -#endif -#ifdef SDRAM_PHY_ECP5DDRPHY - ddrphy_burstdet_clr_write(1); -#endif - command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA); - cdelay(15); - for(p=0;p> module) & 0x1) != 1) - working = 0; -#endif - if (show) - printf("%d", working); - score += working; - read_delay_inc(module); - } - printf("| "); - - /* Precharge */ - sdram_dfii_pi0_address_write(0); - sdram_dfii_pi0_baddress_write(0); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); - cdelay(15); - - return score; -} - -static void read_level(int module) -{ - unsigned int prv; - unsigned char prs[SDRAM_PHY_PHASES][DFII_PIX_DATA_BYTES]; - unsigned char tst[DFII_PIX_DATA_BYTES]; - int p, i; - int working; - int delay, delay_min, delay_max; - - printf("delays: "); - - /* Generate pseudo-random sequence */ - prv = 42; - for(p=0;p> module) & 0x1) != 1) - working = 0; -#endif - if(working) - break; - delay++; - if(delay >= SDRAM_PHY_DELAYS) - break; - read_delay_inc(module); - } - delay_min = delay; - - /* Get a bit further into the working zone */ -#if SDRAM_PHY_DELAYS > 32 - for(i=0;i<16;i++) { - delay += 1; - read_delay_inc(module); - } -#else - delay++; - read_delay_inc(module); -#endif - - /* Find largest working delay */ - while(1) { -#ifdef SDRAM_PHY_ECP5DDRPHY - ddrphy_burstdet_clr_write(1); -#endif - command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA); - cdelay(15); - working = 1; - for(p=0;p> module) & 0x1) != 1) - working = 0; -#endif - if(!working) - break; - delay++; - if(delay >= SDRAM_PHY_DELAYS) - break; - read_delay_inc(module); - } - delay_max = delay; - - if (delay_min >= SDRAM_PHY_DELAYS) - printf("-"); - else - printf("%02d+-%02d", (delay_min+delay_max)/2, (delay_max-delay_min)/2); - - /* Set delay to the middle */ - read_delay_rst(module); - for(i=0;i<(delay_min+delay_max)/2;i++) - read_delay_inc(module); - - /* Precharge */ - sdram_dfii_pi0_address_write(0); - sdram_dfii_pi0_baddress_write(0); - command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); - cdelay(15); -} -#endif /* CSR_DDRPHY_BASE */ - -#endif /* CSR_SDRAM_BASE */ - -static unsigned int seed_to_data_32(unsigned int seed, int random) -{ - if (random) - return lfsr(32, seed); - else - return seed + 1; -} - -static unsigned short seed_to_data_16(unsigned short seed, int random) -{ - if (random) - return lfsr(16, seed); - else - return seed + 1; -} - -#define ONEZERO 0xAAAAAAAA -#define ZEROONE 0x55555555 - -#ifndef MEMTEST_BUS_SIZE -#define MEMTEST_BUS_SIZE (512) -#endif - -//#define MEMTEST_BUS_DEBUG - -static int memtest_bus(void) -{ - volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE; - int i, errors; - unsigned int rdata; - - errors = 0; - - for(i=0;i best_score) { - best_bitslip = bitslip; - best_score = score; - } - /* exit */ - if (bitslip == SDRAM_PHY_BITSLIPS-1) - break; - /* increment bitslip */ - read_bitslip_inc(module); - } - - /* select best read window */ - printf("best: m%d, b%02d ", module, best_bitslip); - read_bitslip_rst(module); - for (bitslip=0; bitslip - -void sdrsw(void); -void sdrhw(void); -void sdrrow(unsigned int row); -void sdrrdbuf(int dq); -void sdrrd(unsigned int addr, int dq); -void sdrrderr(int count); -void sdrwr(unsigned int addr); - -void sdrwlon(void); -void sdrwloff(void); -int write_level(void); - -int sdrlevel(void); - -int memtest_silent(void); -int memtest(void); -int sdrinit(void); - -#if defined(DDRPHY_CMD_DELAY) || defined(USDDRPHY_DEBUG) -void ddrphy_cdly(unsigned int delay); -#endif - -#endif /* __SDRAM_H */ diff --git a/litex/soc/software/common.mak b/litex/soc/software/common.mak index fb97fc48..80891531 100644 --- a/litex/soc/software/common.mak +++ b/litex/soc/software/common.mak @@ -50,6 +50,7 @@ INCLUDES = -I$(SOC_DIRECTORY)/software/include/base \ -I$(SOC_DIRECTORY)/common \ -I$(BUILDINC_DIRECTORY) \ -I$(CPU_DIRECTORY) \ + -I$(SOC_DIRECTORY)/software/liblitedram \ -I$(SOC_DIRECTORY)/software/libliteeth COMMONFLAGS = $(DEPFLAGS) -Os $(CPUFLAGS) -g3 -fomit-frame-pointer -Wall -fno-builtin -nostdinc $(INCLUDES) CFLAGS = $(COMMONFLAGS) -fexceptions -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes diff --git a/litex/soc/software/liblitedram/Makefile b/litex/soc/software/liblitedram/Makefile new file mode 100644 index 00000000..07026325 --- /dev/null +++ b/litex/soc/software/liblitedram/Makefile @@ -0,0 +1,23 @@ +include ../include/generated/variables.mak +include $(SOC_DIRECTORY)/software/common.mak + +OBJECTS=sdram.o + +all: liblitedram.a + +liblitedram.a: $(OBJECTS) + $(AR) crs liblitedram.a $(OBJECTS) + +# pull in dependency info for *existing* .o files +-include $(OBJECTS:.o=.d) + +%.o: $(LIBLITEDRAM_DIRECTORY)/%.c + $(compile) + +%.o: %.S + $(assemble) + +.PHONY: all clean + +clean: + $(RM) $(OBJECTS) liblitedram.a .*~ *~ diff --git a/litex/soc/software/liblitedram/lfsr.h b/litex/soc/software/liblitedram/lfsr.h new file mode 100644 index 00000000..50dfccfc --- /dev/null +++ b/litex/soc/software/liblitedram/lfsr.h @@ -0,0 +1,109 @@ +#include + +/* + * Copyright (C) 2020, Anton Blanchard , IBM + * + * Redistribution and use in source and binary forms, with or without modification, + * are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Galois LFSR + * + * Polynomials verified with https://bitbucket.org/gallen/mlpolygen/ + */ +static inline unsigned long lfsr(unsigned long bits, unsigned long prev) +{ + static const unsigned long lfsr_taps[] = { + 0x0, + 0x0, + 0x3, + 0x6, + 0xc, + 0x14, + 0x30, + 0x60, + 0xb8, + 0x110, + 0x240, + 0x500, + 0x829, + 0x100d, + 0x2015, + 0x6000, + 0xd008, + 0x12000, + 0x20400, + 0x40023, + 0x90000, + 0x140000, + 0x300000, + 0x420000, + 0xe10000, + 0x1200000, + 0x2000023, + 0x4000013, + 0x9000000, + 0x14000000, + 0x20000029, + 0x48000000, + 0x80200003, +#if __WORDSIZE == 64 + 0x100080000, + 0x204000003, + 0x500000000, + 0x801000000, + 0x100000001f, + 0x2000000031, + 0x4400000000, + 0xa000140000, + 0x12000000000, + 0x300000c0000, + 0x63000000000, + 0xc0000030000, + 0x1b0000000000, + 0x300003000000, + 0x420000000000, + 0xc00000180000, + 0x1008000000000, + 0x3000000c00000, + 0x6000c00000000, + 0x9000000000000, + 0x18003000000000, + 0x30000000030000, + 0x40000040000000, + 0xc0000600000000, + 0x102000000000000, + 0x200004000000000, + 0x600003000000000, + 0xc00000000000000, + 0x1800300000000000, + 0x3000000000000030, + 0x6000000000000000, + 0x800000000000000d +#endif + }; + unsigned long lsb = prev & 1; + + prev >>= 1; + prev ^= (-lsb) & lfsr_taps[bits]; + + return prev; +} diff --git a/litex/soc/software/liblitedram/sdram.c b/litex/soc/software/liblitedram/sdram.c new file mode 100644 index 00000000..81b169cf --- /dev/null +++ b/litex/soc/software/liblitedram/sdram.c @@ -0,0 +1,1066 @@ +// This file is Copyright (c) 2013-2014 Sebastien Bourdeauducq +// This file is Copyright (c) 2013-2020 Florent Kermarrec +// This file is Copyright (c) 2018 Chris Ballance +// This file is Copyright (c) 2018 Dolu1990 +// This file is Copyright (c) 2019 Gabriel L. Somlo +// This file is Copyright (c) 2018 Jean-François Nguyen +// This file is Copyright (c) 2018 Sergiusz Bazanski +// This file is Copyright (c) 2018 Tim 'mithro' Ansell +// License: BSD + +#include + +#include +#include + +#ifdef CSR_SDRAM_BASE +#include +#endif +#include +#include +#include + +#include "sdram.h" +#include "lfsr.h" + +// FIXME(hack): If we don't have main ram, just target the sram instead. +#ifndef MAIN_RAM_BASE +#define MAIN_RAM_BASE SRAM_BASE +#endif + +__attribute__((unused)) static void cdelay(int i) +{ + while(i > 0) { + __asm__ volatile(CONFIG_CPU_NOP); + i--; + } +} + +#ifdef CSR_SDRAM_BASE + +#define DFII_ADDR_SHIFT CONFIG_CSR_ALIGNMENT/8 + +#define CSR_DATA_BYTES CONFIG_CSR_DATA_WIDTH/8 + +#define DFII_PIX_DATA_BYTES DFII_PIX_DATA_SIZE*CSR_DATA_BYTES + +void sdrsw(void) +{ + sdram_dfii_control_write(DFII_CONTROL_CKE|DFII_CONTROL_ODT|DFII_CONTROL_RESET_N); + printf("SDRAM now under software control\n"); +} + +void sdrhw(void) +{ + sdram_dfii_control_write(DFII_CONTROL_SEL); + printf("SDRAM now under hardware control\n"); +} + +void sdrrow(unsigned int row) +{ + if(row == 0) { + sdram_dfii_pi0_address_write(0x0000); + sdram_dfii_pi0_baddress_write(0); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); + cdelay(15); + } else { + sdram_dfii_pi0_address_write(row); + sdram_dfii_pi0_baddress_write(0); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CS); + cdelay(15); + } +} + +void sdrrdbuf(int dq) +{ + int i, p; + int first_byte, step; + unsigned char buf[DFII_PIX_DATA_BYTES]; + + if(dq < 0) { + first_byte = 0; + step = 1; + } else { + first_byte = DFII_PIX_DATA_BYTES/2 - 1 - dq; + step = DFII_PIX_DATA_BYTES/2; + } + + for(p=0;p 0) { + ddrphy_cdly_inc_write(1); + cdelay(1000); + delay--; + } +#if CSR_DDRPHY_EN_VTC_ADDR + ddrphy_en_vtc_write(1); +#endif +} +#endif + +#ifdef SDRAM_PHY_WRITE_LEVELING_CAPABLE +void sdrwlon(void) +{ + sdram_dfii_pi0_address_write(DDRX_MR1 | (1 << 7)); + sdram_dfii_pi0_baddress_write(1); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); + +#ifdef SDRAM_PHY_DDR4_RDIMM + sdram_dfii_pi0_address_write((DDRX_MR1 | (1 << 7)) ^ 0x2BF8) ; + sdram_dfii_pi0_baddress_write(1 ^ 0xF); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); +#endif + + ddrphy_wlevel_en_write(1); +} + +void sdrwloff(void) +{ + sdram_dfii_pi0_address_write(DDRX_MR1); + sdram_dfii_pi0_baddress_write(1); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); + +#ifdef SDRAM_PHY_DDR4_RDIMM + sdram_dfii_pi0_address_write(DDRX_MR1 ^ 0x2BF8); + sdram_dfii_pi0_baddress_write(1 ^ 0xF); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_CAS|DFII_COMMAND_WE|DFII_COMMAND_CS); +#endif + + ddrphy_wlevel_en_write(0); +} + +static void write_delay_rst(int module) { +#ifdef SDRAM_PHY_WRITE_LEVELING_REINIT + int i; +#endif + + /* sel module */ + ddrphy_dly_sel_write(1 << module); + + /* rst delay */ + ddrphy_wdly_dq_rst_write(1); + ddrphy_wdly_dqs_rst_write(1); +#ifdef SDRAM_PHY_WRITE_LEVELING_REINIT + for(i=0; i 32 + show_iter = (j%16 == 0) && show; +#endif + for (k=0; k zero_count) + taps_scan[j] = 1; + else + taps_scan[j] = 0; + if (show_iter) + printf("%d", taps_scan[j]); + write_delay_inc(i); + cdelay(10); + } + if (show) + printf("|"); + + /* find longer 1 window and set delay at the 0/1 transition */ + one_window_active = 0; + one_window_start = 0; + one_window_count = 0; + one_window_best_start = 0; + one_window_best_count = -1; + delays[i] = -1; + for(j=0;j one_window_best_count) { + one_window_best_start = one_window_start; + one_window_best_count = one_window_count; + } + } + } else { + if (taps_scan[j]) { + one_window_active = 1; + one_window_start = j; + } + } + } + /* succeed only if the start of a 1s window has been found */ + if (one_window_best_count > 0 && one_window_best_start > 0) { + delays[i] = one_window_best_start; + + /* configure write delay */ + write_delay_rst(i); + for(j=0; j=0;i--) { + if(delays[i] < 0) + ok = 0; + } + + return ok; +} + +static void write_level_cdly_range(unsigned int *best_error, int *best_cdly, + int cdly_start, int cdly_stop, int cdly_step) +{ + int cdly; + int cdly_actual = 0; + int delays[SDRAM_PHY_MODULES]; + + /* scan through the range */ + ddrphy_cdly_rst_write(1); + for (cdly = cdly_start; cdly < cdly_stop; cdly += cdly_step) { + /* increment cdly to current value */ + while (cdly_actual < cdly) { + ddrphy_cdly_inc_write(1); + cdelay(10); + cdly_actual++; + } + + /* write level using this delay */ + if (write_level_scan(delays, 8, 0)) { + /* use the mean of delays for error calulation */ + int delay_mean = 0; + for (int i=0; i < SDRAM_PHY_MODULES; ++i) { + delay_mean += delays[i]; + } + delay_mean /= SDRAM_PHY_MODULES; + + /* we want it to be at the start */ + int ideal_delay = 4*SDRAM_PHY_DELAYS/32; + int error = ideal_delay - delay_mean; + if (error < 0) + error *= -1; + + if (error < *best_error) { + *best_cdly = cdly; + *best_error = error; + } + printf("1"); + } else { + printf("0"); + } + } +} + +int write_level(void) +{ + int delays[SDRAM_PHY_MODULES]; + unsigned int best_error = ~0u; + int best_cdly = -1; + int cdly_range_start; + int cdly_range_end; + int cdly_range_step; + + printf("Command/Clk scan:\n"); + + /* Center write leveling by varying cdly. Searching through all possible + * values is slow, but we can use a simple optimization method of iterativly + * scanning smaller ranges with decreasing step */ + cdly_range_start = 0; + cdly_range_end = SDRAM_PHY_DELAYS; + if (SDRAM_PHY_DELAYS > 32) + cdly_range_step = SDRAM_PHY_DELAYS/8; + else + cdly_range_step = 1; + while (cdly_range_step > 0) { + printf("|"); + write_level_cdly_range(&best_error, &best_cdly, + cdly_range_start, cdly_range_end, cdly_range_step); + + /* small optimization - stop if we have zero error */ + if (best_error == 0) + break; + + /* use best result as the middle of next range */ + cdly_range_start = best_cdly - cdly_range_step; + cdly_range_end = best_cdly + cdly_range_step + 1; + if (cdly_range_start < 0) + cdly_range_start = 0; + if (cdly_range_end > 512) + cdly_range_end = 512; + + cdly_range_step /= 4; + } + printf("| best: %d\n", best_cdly); + + /* if we found any working delay then set it */ + if (best_cdly >= 0) { + ddrphy_cdly_rst_write(1); + for (int i = 0; i < best_cdly; ++i) { + ddrphy_cdly_inc_write(1); + cdelay(10); + } + } + + printf("Data scan:\n"); + + /* re-run write leveling the final time */ + if (!write_level_scan(delays, 128, 1)) + return 0; + + return best_cdly >= 0; +} + + +#endif /* SDRAM_PHY_WRITE_LEVELING_CAPABLE */ + +static void read_delay_rst(int module) { + /* sel module */ + ddrphy_dly_sel_write(1 << module); + + /* rst delay */ + ddrphy_rdly_dq_rst_write(1); + + /* unsel module */ + ddrphy_dly_sel_write(0); + +#ifdef SDRAM_PHY_ECP5DDRPHY + /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ + ddrphy_dly_sel_write(0xFF); + ddrphy_dly_sel_write(0); +#endif +} + +static void read_delay_inc(int module) { + /* sel module */ + ddrphy_dly_sel_write(1 << module); + + /* inc delay */ + ddrphy_rdly_dq_inc_write(1); + + /* unsel module */ + ddrphy_dly_sel_write(0); + +#ifdef SDRAM_PHY_ECP5DDRPHY + /* Sync all DQSBUFM's, By toggling all dly_sel (DQSBUFM.PAUSE) lines. */ + ddrphy_dly_sel_write(0xFF); + ddrphy_dly_sel_write(0); +#endif +} + +static void read_bitslip_rst(char m) +{ + /* sel module */ + ddrphy_dly_sel_write(1 << m); + + /* inc delay */ + ddrphy_rdly_dq_bitslip_rst_write(1); + + /* unsel module */ + ddrphy_dly_sel_write(0); +} + + +static void read_bitslip_inc(char m) +{ + /* sel module */ + ddrphy_dly_sel_write(1 << m); + + /* inc delay */ + ddrphy_rdly_dq_bitslip_write(1); + + /* unsel module */ + ddrphy_dly_sel_write(0); +} + +static int read_level_scan(int module, int bitslip) +{ + unsigned int prv; + unsigned char prs[SDRAM_PHY_PHASES][DFII_PIX_DATA_BYTES]; + unsigned char tst[DFII_PIX_DATA_BYTES]; + int p, i; + int score; + + /* Generate pseudo-random sequence */ + prv = 42; + for(p=0;p 32 + show = (i%16 == 0); +#endif +#ifdef SDRAM_PHY_ECP5DDRPHY + ddrphy_burstdet_clr_write(1); +#endif + command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA); + cdelay(15); + for(p=0;p> module) & 0x1) != 1) + working = 0; +#endif + if (show) + printf("%d", working); + score += working; + read_delay_inc(module); + } + printf("| "); + + /* Precharge */ + sdram_dfii_pi0_address_write(0); + sdram_dfii_pi0_baddress_write(0); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); + cdelay(15); + + return score; +} + +static void read_level(int module) +{ + unsigned int prv; + unsigned char prs[SDRAM_PHY_PHASES][DFII_PIX_DATA_BYTES]; + unsigned char tst[DFII_PIX_DATA_BYTES]; + int p, i; + int working; + int delay, delay_min, delay_max; + + printf("delays: "); + + /* Generate pseudo-random sequence */ + prv = 42; + for(p=0;p> module) & 0x1) != 1) + working = 0; +#endif + if(working) + break; + delay++; + if(delay >= SDRAM_PHY_DELAYS) + break; + read_delay_inc(module); + } + delay_min = delay; + + /* Get a bit further into the working zone */ +#if SDRAM_PHY_DELAYS > 32 + for(i=0;i<16;i++) { + delay += 1; + read_delay_inc(module); + } +#else + delay++; + read_delay_inc(module); +#endif + + /* Find largest working delay */ + while(1) { +#ifdef SDRAM_PHY_ECP5DDRPHY + ddrphy_burstdet_clr_write(1); +#endif + command_prd(DFII_COMMAND_CAS|DFII_COMMAND_CS|DFII_COMMAND_RDDATA); + cdelay(15); + working = 1; + for(p=0;p> module) & 0x1) != 1) + working = 0; +#endif + if(!working) + break; + delay++; + if(delay >= SDRAM_PHY_DELAYS) + break; + read_delay_inc(module); + } + delay_max = delay; + + if (delay_min >= SDRAM_PHY_DELAYS) + printf("-"); + else + printf("%02d+-%02d", (delay_min+delay_max)/2, (delay_max-delay_min)/2); + + /* Set delay to the middle */ + read_delay_rst(module); + for(i=0;i<(delay_min+delay_max)/2;i++) + read_delay_inc(module); + + /* Precharge */ + sdram_dfii_pi0_address_write(0); + sdram_dfii_pi0_baddress_write(0); + command_p0(DFII_COMMAND_RAS|DFII_COMMAND_WE|DFII_COMMAND_CS); + cdelay(15); +} +#endif /* CSR_DDRPHY_BASE */ + +#endif /* CSR_SDRAM_BASE */ + +static unsigned int seed_to_data_32(unsigned int seed, int random) +{ + if (random) + return lfsr(32, seed); + else + return seed + 1; +} + +static unsigned short seed_to_data_16(unsigned short seed, int random) +{ + if (random) + return lfsr(16, seed); + else + return seed + 1; +} + +#define ONEZERO 0xAAAAAAAA +#define ZEROONE 0x55555555 + +#ifndef MEMTEST_BUS_SIZE +#define MEMTEST_BUS_SIZE (512) +#endif + +//#define MEMTEST_BUS_DEBUG + +static int memtest_bus(void) +{ + volatile unsigned int *array = (unsigned int *)MAIN_RAM_BASE; + int i, errors; + unsigned int rdata; + + errors = 0; + + for(i=0;i best_score) { + best_bitslip = bitslip; + best_score = score; + } + /* exit */ + if (bitslip == SDRAM_PHY_BITSLIPS-1) + break; + /* increment bitslip */ + read_bitslip_inc(module); + } + + /* select best read window */ + printf("best: m%d, b%02d ", module, best_bitslip); + read_bitslip_rst(module); + for (bitslip=0; bitslip + +void sdrsw(void); +void sdrhw(void); +void sdrrow(unsigned int row); +void sdrrdbuf(int dq); +void sdrrd(unsigned int addr, int dq); +void sdrrderr(int count); +void sdrwr(unsigned int addr); + +void sdrwlon(void); +void sdrwloff(void); +int write_level(void); + +int sdrlevel(void); + +int memtest_silent(void); +int memtest(void); +int sdrinit(void); + +#if defined(DDRPHY_CMD_DELAY) || defined(USDDRPHY_DEBUG) +void ddrphy_cdly(unsigned int delay); +#endif + +#endif /* __SDRAM_H */