kernel/mem.cc

   1 /*
   2  *  yosys -- Yosys Open SYnthesis Suite
   3  *
   4  *  Copyright (C) 2020  Marcelina Kościelnicka <mwk@0x04.net>
   5  *
   6  *  Permission to use, copy, modify, and/or distribute this software for any
   7  *  purpose with or without fee is hereby granted, provided that the above
   8  *  copyright notice and this permission notice appear in all copies.
   9  *
  10  *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  *
  18  */
  19
  20 #include "kernel/mem.h"
  21 #include "kernel/ff.h"
  22
  23 USING_YOSYS_NAMESPACE
  24
  25 void Mem::remove() {
  26         if (cell) {
  27                 module->remove(cell);
  28                 cell = nullptr;
  29         }
  30         if (mem) {
  31                 module->memories.erase(mem->name);
  32                 delete mem;
  33                 mem = nullptr;
  34         }
  35         for (auto &port : rd_ports) {
  36                 if (port.cell) {
  37                         module->remove(port.cell);
  38                         port.cell = nullptr;
  39                 }
  40         }
  41         for (auto &port : wr_ports) {
  42                 if (port.cell) {
  43                         module->remove(port.cell);
  44                         port.cell = nullptr;
  45                 }
  46         }
  47         for (auto &init : inits) {
  48                 if (init.cell) {
  49                         module->remove(init.cell);
  50                         init.cell = nullptr;
  51                 }
  52         }
  53 }
  54
  55 void Mem::emit() {
  56         check();
  57         std::vector<int> rd_left;
  58         for (int i = 0; i < GetSize(rd_ports); i++) {
  59                 auto &port = rd_ports[i];
  60                 if (port.removed) {
  61                         if (port.cell) {
  62                                 module->remove(port.cell);
  63                         }
  64                 } else {
  65                         rd_left.push_back(i);
  66                 }
  67         }
  68         std::vector<int> wr_left;
  69         for (int i = 0; i < GetSize(wr_ports); i++) {
  70                 auto &port = wr_ports[i];
  71                 if (port.removed) {
  72                         if (port.cell) {
  73                                 module->remove(port.cell);
  74                         }
  75                 } else {
  76                         wr_left.push_back(i);
  77                 }
  78         }
  79         for (int i = 0; i < GetSize(rd_left); i++)
  80                 if (i != rd_left[i])
  81                         std::swap(rd_ports[i], rd_ports[rd_left[i]]);
  82         rd_ports.resize(GetSize(rd_left));
  83         for (int i = 0; i < GetSize(wr_left); i++)
  84                 if (i != wr_left[i])
  85                         std::swap(wr_ports[i], wr_ports[wr_left[i]]);
  86         wr_ports.resize(GetSize(wr_left));
  87
  88         // for future: handle transparency mask here
  89
  90         for (auto &port : wr_ports) {
  91                 for (int i = 0; i < GetSize(wr_left); i++)
  92                         port.priority_mask[i] = port.priority_mask[wr_left[i]];
  93                 port.priority_mask.resize(GetSize(wr_left));
  94         }
  95
  96         if (packed) {
  97                 if (mem) {
  98                         module->memories.erase(mem->name);
  99                         delete mem;
 100                         mem = nullptr;
 101                 }
 102                 if (!cell) {
 103                         if (memid.empty())
 104                                 memid = NEW_ID;
 105                         cell = module->addCell(memid, ID($mem));
 106                 }
 107                 cell->attributes = attributes;
 108                 cell->parameters[ID::MEMID] = Const(memid.str());
 109                 cell->parameters[ID::WIDTH] = Const(width);
 110                 cell->parameters[ID::OFFSET] = Const(start_offset);
 111                 cell->parameters[ID::SIZE] = Const(size);
 112                 Const rd_wide_continuation, rd_clk_enable, rd_clk_polarity, rd_transparent;
 113                 Const wr_wide_continuation, wr_clk_enable, wr_clk_polarity;
 114                 SigSpec rd_clk, rd_en, rd_addr, rd_data;
 115                 SigSpec wr_clk, wr_en, wr_addr, wr_data;
 116                 int abits = 0;
 117                 for (auto &port : rd_ports)
 118                         abits = std::max(abits, GetSize(port.addr));
 119                 for (auto &port : wr_ports)
 120                         abits = std::max(abits, GetSize(port.addr));
 121                 cell->parameters[ID::ABITS] = Const(abits);
 122                 for (auto &port : rd_ports) {
 123                         // TODO: remove
 124                         log_assert(port.arst == State::S0);
 125                         log_assert(port.srst == State::S0);
 126                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 127                         if (port.cell) {
 128                                 module->remove(port.cell);
 129                                 port.cell = nullptr;
 130                         }
 131                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 132                         {
 133                                 rd_wide_continuation.bits.push_back(State(sub != 0));
 134                                 rd_clk_enable.bits.push_back(State(port.clk_enable));
 135                                 rd_clk_polarity.bits.push_back(State(port.clk_polarity));
 136                                 rd_transparent.bits.push_back(State(port.transparent));
 137                                 rd_clk.append(port.clk);
 138                                 rd_en.append(port.en);
 139                                 SigSpec addr = port.sub_addr(sub);
 140                                 addr.extend_u0(abits, false);
 141                                 rd_addr.append(addr);
 142                                 log_assert(GetSize(addr) == abits);
 143                         }
 144                         rd_data.append(port.data);
 145                 }
 146                 if (rd_ports.empty()) {
 147                         rd_wide_continuation = State::S0;
 148                         rd_clk_enable = State::S0;
 149                         rd_clk_polarity = State::S0;
 150                         rd_transparent = State::S0;
 151                 }
 152                 cell->parameters[ID::RD_PORTS] = Const(GetSize(rd_clk));
 153                 cell->parameters[ID::RD_CLK_ENABLE] = rd_clk_enable;
 154                 cell->parameters[ID::RD_CLK_POLARITY] = rd_clk_polarity;
 155                 cell->parameters[ID::RD_TRANSPARENT] = rd_transparent;
 156                 cell->setPort(ID::RD_CLK, rd_clk);
 157                 cell->setPort(ID::RD_EN, rd_en);
 158                 cell->setPort(ID::RD_ADDR, rd_addr);
 159                 cell->setPort(ID::RD_DATA, rd_data);
 160                 for (auto &port : wr_ports) {
 161                         if (port.cell) {
 162                                 module->remove(port.cell);
 163                                 port.cell = nullptr;
 164                         }
 165                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 166                         {
 167                                 wr_wide_continuation.bits.push_back(State(sub != 0));
 168                                 wr_clk_enable.bits.push_back(State(port.clk_enable));
 169                                 wr_clk_polarity.bits.push_back(State(port.clk_polarity));
 170                                 wr_clk.append(port.clk);
 171                                 SigSpec addr = port.sub_addr(sub);
 172                                 addr.extend_u0(abits, false);
 173                                 wr_addr.append(addr);
 174                                 log_assert(GetSize(addr) == abits);
 175                         }
 176                         wr_en.append(port.en);
 177                         wr_data.append(port.data);
 178                 }
 179                 if (wr_ports.empty()) {
 180                         wr_wide_continuation = State::S0;
 181                         wr_clk_enable = State::S0;
 182                         wr_clk_polarity = State::S0;
 183                 }
 184                 cell->parameters[ID::WR_PORTS] = Const(GetSize(wr_clk));
 185                 cell->parameters[ID::WR_CLK_ENABLE] = wr_clk_enable;
 186                 cell->parameters[ID::WR_CLK_POLARITY] = wr_clk_polarity;
 187                 cell->setPort(ID::WR_CLK, wr_clk);
 188                 cell->setPort(ID::WR_EN, wr_en);
 189                 cell->setPort(ID::WR_ADDR, wr_addr);
 190                 cell->setPort(ID::WR_DATA, wr_data);
 191                 for (auto &init : inits) {
 192                         if (init.cell) {
 193                                 module->remove(init.cell);
 194                                 init.cell = nullptr;
 195                         }
 196                 }
 197                 cell->parameters[ID::INIT] = get_init_data();
 198         } else {
 199                 if (cell) {
 200                         module->remove(cell);
 201                         cell = nullptr;
 202                 }
 203                 if (!mem) {
 204                         if (memid.empty())
 205                                 memid = NEW_ID;
 206                         mem = new RTLIL::Memory;
 207                         mem->name = memid;
 208                         module->memories[memid] = mem;
 209                 }
 210                 mem->width = width;
 211                 mem->start_offset = start_offset;
 212                 mem->size = size;
 213                 for (auto &port : rd_ports) {
 214                         // TODO: remove
 215                         log_assert(port.arst == State::S0);
 216                         log_assert(port.srst == State::S0);
 217                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 218                         if (!port.cell)
 219                                 port.cell = module->addCell(NEW_ID, ID($memrd));
 220                         port.cell->parameters[ID::MEMID] = memid.str();
 221                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 222                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 223                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 224                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 225                         port.cell->parameters[ID::TRANSPARENT] = port.transparent;
 226                         port.cell->setPort(ID::CLK, port.clk);
 227                         port.cell->setPort(ID::EN, port.en);
 228                         port.cell->setPort(ID::ADDR, port.addr);
 229                         port.cell->setPort(ID::DATA, port.data);
 230                 }
 231                 int idx = 0;
 232                 for (auto &port : wr_ports) {
 233                         if (!port.cell)
 234                                 port.cell = module->addCell(NEW_ID, ID($memwr));
 235                         port.cell->parameters[ID::MEMID] = memid.str();
 236                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 237                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 238                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 239                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 240                         port.cell->parameters[ID::PRIORITY] = idx++;
 241                         port.cell->setPort(ID::CLK, port.clk);
 242                         port.cell->setPort(ID::EN, port.en);
 243                         port.cell->setPort(ID::ADDR, port.addr);
 244                         port.cell->setPort(ID::DATA, port.data);
 245                 }
 246                 idx = 0;
 247                 for (auto &init : inits) {
 248                         if (!init.cell)
 249                                 init.cell = module->addCell(NEW_ID, ID($meminit));
 250                         init.cell->parameters[ID::MEMID] = memid.str();
 251                         init.cell->parameters[ID::ABITS] = GetSize(init.addr);
 252                         init.cell->parameters[ID::WIDTH] = width;
 253                         init.cell->parameters[ID::WORDS] = GetSize(init.data) / width;
 254                         init.cell->parameters[ID::PRIORITY] = idx++;
 255                         init.cell->setPort(ID::ADDR, init.addr);
 256                         init.cell->setPort(ID::DATA, init.data);
 257                 }
 258         }
 259 }
 260
 261 void Mem::clear_inits() {
 262         for (auto &init : inits)
 263                 if (init.cell)
 264                         module->remove(init.cell);
 265         inits.clear();
 266 }
 267
 268 Const Mem::get_init_data() const {
 269         Const init_data(State::Sx, width * size);
 270         for (auto &init : inits) {
 271                 int offset = (init.addr.as_int() - start_offset) * width;
 272                 for (int i = 0; i < GetSize(init.data); i++)
 273                         if (0 <= i+offset && i+offset < GetSize(init_data))
 274                                 init_data.bits[i+offset] = init.data.bits[i];
 275         }
 276         return init_data;
 277 }
 278
 279 void Mem::check() {
 280         int max_wide_log2 = 0;
 281         for (auto &port : rd_ports) {
 282                 if (port.removed)
 283                         continue;
 284                 log_assert(GetSize(port.clk) == 1);
 285                 log_assert(GetSize(port.en) == 1);
 286                 log_assert(GetSize(port.arst) == 1);
 287                 log_assert(GetSize(port.srst) == 1);
 288                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 289                 log_assert(GetSize(port.init_value) == (width << port.wide_log2));
 290                 log_assert(GetSize(port.arst_value) == (width << port.wide_log2));
 291                 log_assert(GetSize(port.srst_value) == (width << port.wide_log2));
 292                 if (!port.clk_enable) {
 293                         log_assert(!port.transparent);
 294                         log_assert(port.en == State::S1);
 295                         log_assert(port.arst == State::S0);
 296                         log_assert(port.srst == State::S0);
 297                 }
 298                 for (int j = 0; j < port.wide_log2; j++) {
 299                         log_assert(port.addr[j] == State::S0);
 300                 }
 301                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 302         }
 303         for (int i = 0; i < GetSize(wr_ports); i++) {
 304                 auto &port = wr_ports[i];
 305                 if (port.removed)
 306                         continue;
 307                 log_assert(GetSize(port.clk) == 1);
 308                 log_assert(GetSize(port.en) == (width << port.wide_log2));
 309                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 310                 for (int j = 0; j < port.wide_log2; j++) {
 311                         log_assert(port.addr[j] == State::S0);
 312                 }
 313                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 314                 log_assert(GetSize(port.priority_mask) == GetSize(wr_ports));
 315                 for (int j = 0; j < GetSize(wr_ports); j++) {
 316                         auto &wport = wr_ports[j];
 317                         if (port.priority_mask[j] && !wport.removed) {
 318                                 log_assert(j < i);
 319                                 log_assert(port.clk_enable == wport.clk_enable);
 320                                 if (port.clk_enable) {
 321                                         log_assert(port.clk == wport.clk);
 322                                         log_assert(port.clk_polarity == wport.clk_polarity);
 323                                 }
 324                         }
 325                 }
 326         }
 327         int mask = (1 << max_wide_log2) - 1;
 328         log_assert(!(start_offset & mask));
 329         log_assert(!(size & mask));
 330 }
 331
 332 namespace {
 333
 334         struct MemIndex {
 335                 dict<IdString, pool<Cell *>> rd_ports;
 336                 dict<IdString, pool<Cell *>> wr_ports;
 337                 dict<IdString, pool<Cell *>> inits;
 338                 MemIndex (Module *module) {
 339                         for (auto cell: module->cells()) {
 340                                 if (cell->type == ID($memwr))
 341                                         wr_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 342                                 else if (cell->type == ID($memrd))
 343                                         rd_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 344                                 else if (cell->type == ID($meminit))
 345                                         inits[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 346                         }
 347                 }
 348         };
 349
 350         Mem mem_from_memory(Module *module, RTLIL::Memory *mem, const MemIndex &index) {
 351                 Mem res(module, mem->name, mem->width, mem->start_offset, mem->size);
 352                 res.packed = false;
 353                 res.mem = mem;
 354                 res.attributes = mem->attributes;
 355                 if (index.rd_ports.count(mem->name)) {
 356                         for (auto cell : index.rd_ports.at(mem->name)) {
 357                                 MemRd mrd;
 358                                 mrd.cell = cell;
 359                                 mrd.attributes = cell->attributes;
 360                                 mrd.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 361                                 mrd.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 362                                 mrd.transparent = cell->parameters.at(ID::TRANSPARENT).as_bool();
 363                                 mrd.clk = cell->getPort(ID::CLK);
 364                                 mrd.en = cell->getPort(ID::EN);
 365                                 mrd.addr = cell->getPort(ID::ADDR);
 366                                 mrd.data = cell->getPort(ID::DATA);
 367                                 mrd.wide_log2 = ceil_log2(GetSize(mrd.data) / mem->width);
 368                                 mrd.ce_over_srst = false;
 369                                 mrd.arst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 370                                 mrd.srst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 371                                 mrd.init_value = Const(State::Sx, mem->width << mrd.wide_log2);
 372                                 mrd.srst = State::S0;
 373                                 mrd.arst = State::S0;
 374                                 if (!mrd.clk_enable) {
 375                                         // Fix some patterns that we'll allow for backwards compatibility,
 376                                         // but don't want to see moving forwards: async transparent
 377                                         // ports (inherently meaningless) and async ports without
 378                                         // const 1 tied to EN bit (which may mean a latch in the future).
 379                                         mrd.transparent = false;
 380                                         if (mrd.en == State::Sx)
 381                                                 mrd.en = State::S1;
 382                                 }
 383                                 res.rd_ports.push_back(mrd);
 384                         }
 385                 }
 386                 if (index.wr_ports.count(mem->name)) {
 387                         std::vector<std::pair<int, MemWr>> ports;
 388                         for (auto cell : index.wr_ports.at(mem->name)) {
 389                                 MemWr mwr;
 390                                 mwr.cell = cell;
 391                                 mwr.attributes = cell->attributes;
 392                                 mwr.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 393                                 mwr.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 394                                 mwr.clk = cell->getPort(ID::CLK);
 395                                 mwr.en = cell->getPort(ID::EN);
 396                                 mwr.addr = cell->getPort(ID::ADDR);
 397                                 mwr.data = cell->getPort(ID::DATA);
 398                                 mwr.wide_log2 = ceil_log2(GetSize(mwr.data) / mem->width);
 399                                 ports.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), mwr));
 400                         }
 401                         std::sort(ports.begin(), ports.end(), [](const std::pair<int, MemWr> &a, const std::pair<int, MemWr> &b) { return a.first < b.first; });
 402                         for (auto &it : ports)
 403                                 res.wr_ports.push_back(it.second);
 404                 }
 405                 if (index.inits.count(mem->name)) {
 406                         std::vector<std::pair<int, MemInit>> inits;
 407                         for (auto cell : index.inits.at(mem->name)) {
 408                                 MemInit init;
 409                                 init.cell = cell;
 410                                 init.attributes = cell->attributes;
 411                                 auto addr = cell->getPort(ID::ADDR);
 412                                 auto data = cell->getPort(ID::DATA);
 413                                 if (!addr.is_fully_const())
 414                                         log_error("Non-constant address %s in memory initialization %s.\n", log_signal(addr), log_id(cell));
 415                                 if (!data.is_fully_const())
 416                                         log_error("Non-constant data %s in memory initialization %s.\n", log_signal(data), log_id(cell));
 417                                 init.addr = addr.as_const();
 418                                 init.data = data.as_const();
 419                                 inits.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), init));
 420                         }
 421                         std::sort(inits.begin(), inits.end(), [](const std::pair<int, MemInit> &a, const std::pair<int, MemInit> &b) { return a.first < b.first; });
 422                         for (auto &it : inits)
 423                                 res.inits.push_back(it.second);
 424                 }
 425                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 426                         auto &port = res.wr_ports[i];
 427                         port.priority_mask.resize(GetSize(res.wr_ports));
 428                         for (int j = 0; j < i; j++) {
 429                                 auto &oport = res.wr_ports[j];
 430                                 if (port.clk_enable != oport.clk_enable)
 431                                         continue;
 432                                 if (port.clk_enable && port.clk != oport.clk)
 433                                         continue;
 434                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 435                                         continue;
 436                                 port.priority_mask[j] = true;
 437                         }
 438                 }
 439                 res.check();
 440                 return res;
 441         }
 442
 443         Mem mem_from_cell(Cell *cell) {
 444                 Mem res(cell->module, cell->parameters.at(ID::MEMID).decode_string(),
 445                         cell->parameters.at(ID::WIDTH).as_int(),
 446                         cell->parameters.at(ID::OFFSET).as_int(),
 447                         cell->parameters.at(ID::SIZE).as_int()
 448                 );
 449                 int abits = cell->parameters.at(ID::ABITS).as_int();
 450                 res.packed = true;
 451                 res.cell = cell;
 452                 res.attributes = cell->attributes;
 453                 Const &init = cell->parameters.at(ID::INIT);
 454                 if (!init.is_fully_undef()) {
 455                         int pos = 0;
 456                         while (pos < res.size) {
 457                                 Const word = init.extract(pos * res.width, res.width, State::Sx);
 458                                 if (word.is_fully_undef()) {
 459                                         pos++;
 460                                 } else {
 461                                         int epos;
 462                                         for (epos = pos; epos < res.size; epos++) {
 463                                                 Const eword = init.extract(epos * res.width, res.width, State::Sx);
 464                                                 if (eword.is_fully_undef())
 465                                                         break;
 466                                         }
 467                                         MemInit minit;
 468                                         minit.addr = res.start_offset + pos;
 469                                         minit.data = init.extract(pos * res.width, (epos - pos) * res.width, State::Sx);
 470                                         res.inits.push_back(minit);
 471                                         pos = epos;
 472                                 }
 473                         }
 474                 }
 475                 for (int i = 0; i < cell->parameters.at(ID::RD_PORTS).as_int(); i++) {
 476                         MemRd mrd;
 477                         mrd.wide_log2 = 0;
 478                         mrd.clk_enable = cell->parameters.at(ID::RD_CLK_ENABLE).extract(i, 1).as_bool();
 479                         mrd.clk_polarity = cell->parameters.at(ID::RD_CLK_POLARITY).extract(i, 1).as_bool();
 480                         mrd.transparent = cell->parameters.at(ID::RD_TRANSPARENT).extract(i, 1).as_bool();
 481                         mrd.clk = cell->getPort(ID::RD_CLK).extract(i, 1);
 482                         mrd.en = cell->getPort(ID::RD_EN).extract(i, 1);
 483                         mrd.addr = cell->getPort(ID::RD_ADDR).extract(i * abits, abits);
 484                         mrd.data = cell->getPort(ID::RD_DATA).extract(i * res.width, res.width);
 485                         mrd.ce_over_srst = false;
 486                         mrd.arst_value = Const(State::Sx, res.width << mrd.wide_log2);
 487                         mrd.srst_value = Const(State::Sx, res.width << mrd.wide_log2);
 488                         mrd.init_value = Const(State::Sx, res.width << mrd.wide_log2);
 489                         mrd.srst = State::S0;
 490                         mrd.arst = State::S0;
 491                         res.rd_ports.push_back(mrd);
 492                 }
 493                 for (int i = 0; i < cell->parameters.at(ID::WR_PORTS).as_int(); i++) {
 494                         MemWr mwr;
 495                         mwr.wide_log2 = 0;
 496                         mwr.clk_enable = cell->parameters.at(ID::WR_CLK_ENABLE).extract(i, 1).as_bool();
 497                         mwr.clk_polarity = cell->parameters.at(ID::WR_CLK_POLARITY).extract(i, 1).as_bool();
 498                         mwr.clk = cell->getPort(ID::WR_CLK).extract(i, 1);
 499                         mwr.en = cell->getPort(ID::WR_EN).extract(i * res.width, res.width);
 500                         mwr.addr = cell->getPort(ID::WR_ADDR).extract(i * abits, abits);
 501                         mwr.data = cell->getPort(ID::WR_DATA).extract(i * res.width, res.width);
 502                         res.wr_ports.push_back(mwr);
 503                 }
 504                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 505                         auto &port = res.wr_ports[i];
 506                         port.priority_mask.resize(GetSize(res.wr_ports));
 507                         for (int j = 0; j < i; j++) {
 508                                 auto &oport = res.wr_ports[j];
 509                                 if (port.clk_enable != oport.clk_enable)
 510                                         continue;
 511                                 if (port.clk_enable && port.clk != oport.clk)
 512                                         continue;
 513                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 514                                         continue;
 515                                 port.priority_mask[j] = true;
 516                         }
 517                 }
 518                 res.check();
 519                 return res;
 520         }
 521
 522 }
 523
 524 std::vector<Mem> Mem::get_all_memories(Module *module) {
 525         std::vector<Mem> res;
 526         MemIndex index(module);
 527         for (auto it: module->memories) {
 528                 res.push_back(mem_from_memory(module, it.second, index));
 529         }
 530         for (auto cell: module->cells()) {
 531                 if (cell->type == ID($mem))
 532                         res.push_back(mem_from_cell(cell));
 533         }
 534         return res;
 535 }
 536
 537 std::vector<Mem> Mem::get_selected_memories(Module *module) {
 538         std::vector<Mem> res;
 539         MemIndex index(module);
 540         for (auto it: module->memories) {
 541                 if (module->design->selected(module, it.second))
 542                         res.push_back(mem_from_memory(module, it.second, index));
 543         }
 544         for (auto cell: module->selected_cells()) {
 545                 if (cell->type == ID($mem))
 546                         res.push_back(mem_from_cell(cell));
 547         }
 548         return res;
 549 }
 550
 551 Cell *Mem::extract_rdff(int idx, FfInitVals *initvals) {
 552         MemRd &port = rd_ports[idx];
 553
 554         if (!port.clk_enable)
 555                 return nullptr;
 556
 557         Cell *c;
 558
 559         // There are two ways to handle rdff extraction when transparency is involved:
 560         //
 561         // - if all of the following conditions are true, put the FF on address input:
 562         //
 563         //   - the port has no clock enable, no reset, and no initial value
 564         //   - the port is transparent wrt all write ports (implying they also share
 565         //     the clock domain)
 566         //
 567         // - otherwise, put the FF on the data output, and make bypass paths for
 568         //   all write ports wrt which this port is transparent
 569         bool trans_use_addr = port.transparent;
 570
 571         // If there are no write ports at all, we could possibly use either way; do data
 572         // FF in this case.
 573         if (GetSize(wr_ports) == 0)
 574                 trans_use_addr = false;
 575
 576         if (port.en != State::S1 || port.srst != State::S0 || port.arst != State::S0 || !port.init_value.is_fully_undef())
 577                 trans_use_addr = false;
 578
 579         if (trans_use_addr)
 580         {
 581                 // Do not put a register in front of constant address bits — this is both
 582                 // unnecessary and will break wide ports.
 583                 int width = 0;
 584                 for (int i = 0; i < GetSize(port.addr); i++)
 585                         if (port.addr[i].wire)
 586                                 width++;
 587
 588                 if (width)
 589                 {
 590                         SigSpec sig_q = module->addWire(stringf("$%s$rdreg[%d]$q", memid.c_str(), idx), width);
 591                         SigSpec sig_d;
 592
 593                         int pos = 0;
 594                         for (int i = 0; i < GetSize(port.addr); i++)
 595                                 if (port.addr[i].wire) {
 596                                         sig_d.append(port.addr[i]);
 597                                         port.addr[i] = sig_q[pos++];
 598                                 }
 599
 600                         c = module->addDff(stringf("$%s$rdreg[%d]", memid.c_str(), idx), port.clk, sig_d, sig_q, port.clk_polarity);
 601                 } else {
 602                         c = nullptr;
 603                 }
 604         }
 605         else
 606         {
 607                 log_assert(port.arst == State::S0 || port.srst == State::S0);
 608
 609                 SigSpec async_d = module->addWire(stringf("$%s$rdreg[%d]$d", memid.c_str(), idx), GetSize(port.data));
 610                 SigSpec sig_d = async_d;
 611
 612                 for (int i = 0; i < GetSize(wr_ports); i++) {
 613                         auto &wport = wr_ports[i];
 614                         if (port.transparent) {
 615                                 log_assert(wport.clk_enable);
 616                                 log_assert(wport.clk == port.clk);
 617                                 log_assert(wport.clk_enable == port.clk_enable);
 618                                 int min_wide_log2 = std::min(port.wide_log2, wport.wide_log2);
 619                                 int max_wide_log2 = std::max(port.wide_log2, wport.wide_log2);
 620                                 bool wide_write = wport.wide_log2 > port.wide_log2;
 621                                 for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 622                                         SigSpec raddr = port.addr;
 623                                         SigSpec waddr = wport.addr;
 624                                         if (wide_write)
 625                                                 waddr = wport.sub_addr(sub);
 626                                         else
 627                                                 raddr = port.sub_addr(sub);
 628                                         SigSpec addr_eq;
 629                                         if (raddr != waddr)
 630                                                 addr_eq = module->Eq(stringf("$%s$rdtransen[%d][%d][%d]$d", memid.c_str(), idx, i, sub), raddr, waddr);
 631                                         int pos = 0;
 632                                         int ewidth = width << min_wide_log2;
 633                                         int wsub = wide_write ? sub : 0;
 634                                         int rsub = wide_write ? 0 : sub;
 635                                         while (pos < ewidth) {
 636                                                 int epos = pos;
 637                                                 while (epos < ewidth && wport.en[epos + wsub * width] == wport.en[pos + wsub * width])
 638                                                         epos++;
 639                                                 SigSpec cur = sig_d.extract(pos + rsub * width, epos-pos);
 640                                                 SigSpec other = wport.data.extract(pos + wsub * width, epos-pos);
 641                                                 SigSpec cond;
 642                                                 if (raddr != waddr)
 643                                                         cond = module->And(stringf("$%s$rdtransgate[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), wport.en[pos + wsub * width], addr_eq);
 644                                                 else
 645                                                         cond = wport.en[pos + wsub * width];
 646                                                 SigSpec merged = module->Mux(stringf("$%s$rdtransmux[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), cur, other, cond);
 647                                                 sig_d.replace(pos + rsub * width, merged);
 648                                                 pos = epos;
 649                                         }
 650                                 }
 651                         }
 652                 }
 653
 654                 IdString name = stringf("$%s$rdreg[%d]", memid.c_str(), idx);
 655                 FfData ff(initvals);
 656                 ff.width = GetSize(port.data);
 657                 ff.has_clk = true;
 658                 ff.sig_clk = port.clk;
 659                 ff.pol_clk = port.clk_polarity;
 660                 if (port.en != State::S1) {
 661                         ff.has_en = true;
 662                         ff.pol_en = true;
 663                         ff.sig_en = port.en;
 664                 }
 665                 if (port.arst != State::S0) {
 666                         ff.has_arst = true;
 667                         ff.pol_arst = true;
 668                         ff.sig_arst = port.arst;
 669                         ff.val_arst = port.arst_value;
 670                 }
 671                 if (port.srst != State::S0) {
 672                         ff.has_srst = true;
 673                         ff.pol_srst = true;
 674                         ff.sig_srst = port.srst;
 675                         ff.val_srst = port.srst_value;
 676                         ff.ce_over_srst = ff.has_en && port.ce_over_srst;
 677                 }
 678                 ff.sig_d = sig_d;
 679                 ff.sig_q = port.data;
 680                 ff.val_init = port.init_value;
 681                 port.data = async_d;
 682                 c = ff.emit(module, name);
 683         }
 684
 685         log("Extracted %s FF from read port %d of %s.%s: %s\n", trans_use_addr ? "addr" : "data",
 686                         idx, log_id(module), log_id(memid), log_id(c));
 687
 688         port.en = State::S1;
 689         port.clk = State::S0;
 690         port.arst = State::S0;
 691         port.srst = State::S0;
 692         port.clk_enable = false;
 693         port.clk_polarity = true;
 694         port.transparent = false;
 695         port.ce_over_srst = false;
 696         port.arst_value = Const(State::Sx, GetSize(port.data));
 697         port.srst_value = Const(State::Sx, GetSize(port.data));
 698         port.init_value = Const(State::Sx, GetSize(port.data));
 699
 700         return c;
 701 }
 702
 703 void Mem::narrow() {
 704         // NOTE: several passes depend on this function not modifying
 705         // the design at all until (and unless) emit() is called.
 706         // Be careful to preserve this.
 707         std::vector<MemRd> new_rd_ports;
 708         std::vector<MemWr> new_wr_ports;
 709         std::vector<std::pair<int, int>> new_rd_map;
 710         std::vector<std::pair<int, int>> new_wr_map;
 711         for (int i = 0; i < GetSize(rd_ports); i++) {
 712                 auto &port = rd_ports[i];
 713                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 714                         new_rd_map.push_back(std::make_pair(i, sub));
 715                 }
 716         }
 717         for (int i = 0; i < GetSize(wr_ports); i++) {
 718                 auto &port = wr_ports[i];
 719                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 720                         new_wr_map.push_back(std::make_pair(i, sub));
 721                 }
 722         }
 723         for (auto &it : new_rd_map) {
 724                 MemRd &orig = rd_ports[it.first];
 725                 MemRd port = orig;
 726                 if (it.second != 0)
 727                         port.cell = nullptr;
 728                 if (port.wide_log2) {
 729                         port.data = port.data.extract(it.second * width, width);
 730                         port.init_value = port.init_value.extract(it.second * width, width);
 731                         port.arst_value = port.arst_value.extract(it.second * width, width);
 732                         port.srst_value = port.srst_value.extract(it.second * width, width);
 733                         port.addr = port.sub_addr(it.second);
 734                         port.wide_log2 = 0;
 735                 }
 736                 new_rd_ports.push_back(port);
 737         }
 738         for (auto &it : new_wr_map) {
 739                 MemWr &orig = wr_ports[it.first];
 740                 MemWr port = orig;
 741                 if (it.second != 0)
 742                         port.cell = nullptr;
 743                 if (port.wide_log2) {
 744                         port.data = port.data.extract(it.second * width, width);
 745                         port.en = port.en.extract(it.second * width, width);
 746                         port.addr = port.sub_addr(it.second);
 747                         port.wide_log2 = 0;
 748                 }
 749                 port.priority_mask.clear();
 750                 for (auto &it2 : new_wr_map)
 751                         port.priority_mask.push_back(orig.priority_mask[it2.first]);
 752                 new_wr_ports.push_back(port);
 753         }
 754         std::swap(rd_ports, new_rd_ports);
 755         std::swap(wr_ports, new_wr_ports);
 756 }
 757
 758 void Mem::emulate_priority(int idx1, int idx2)
 759 {
 760         auto &port1 = wr_ports[idx1];
 761         auto &port2 = wr_ports[idx2];
 762         if (!port2.priority_mask[idx1])
 763                 return;
 764         int min_wide_log2 = std::min(port1.wide_log2, port2.wide_log2);
 765         int max_wide_log2 = std::max(port1.wide_log2, port2.wide_log2);
 766         bool wide1 = port1.wide_log2 > port2.wide_log2;
 767         for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 768                 SigSpec addr1 = port1.addr;
 769                 SigSpec addr2 = port2.addr;
 770                 if (wide1)
 771                         addr1 = port1.sub_addr(sub);
 772                 else
 773                         addr2 = port2.sub_addr(sub);
 774                 SigSpec addr_eq = module->Eq(NEW_ID, addr1, addr2);
 775                 int ewidth = width << min_wide_log2;
 776                 int sub1 = wide1 ? sub : 0;
 777                 int sub2 = wide1 ? 0 : sub;
 778                 dict<std::pair<SigBit, SigBit>, SigBit> cache;
 779                 for (int pos = 0; pos < ewidth; pos++) {
 780                         SigBit &en1 = port1.en[pos + sub1 * width];
 781                         SigBit &en2 = port2.en[pos + sub2 * width];
 782                         std::pair<SigBit, SigBit> key(en1, en2);
 783                         if (cache.count(key)) {
 784                                 en1 = cache[key];
 785                         } else {
 786                                 SigBit active2 = module->And(NEW_ID, addr_eq, en2);
 787                                 SigBit nactive2 = module->Not(NEW_ID, active2);
 788                                 en1 = cache[key] = module->And(NEW_ID, en1, nactive2);
 789                         }
 790                 }
 791         }
 792         port2.priority_mask[idx1] = false;
 793 }
 794
 795 void Mem::prepare_wr_merge(int idx1, int idx2) {
 796         log_assert(idx1 < idx2);
 797         auto &port1 = wr_ports[idx1];
 798         auto &port2 = wr_ports[idx2];
 799         // If port 2 has priority over a port before port 1, make port 1 have priority too.
 800         for (int i = 0; i < idx1; i++)
 801                 if (port2.priority_mask[i])
 802                         port1.priority_mask[i] = true;
 803         // If port 2 has priority over a port after port 1, emulate it.
 804         for (int i = idx1 + 1; i < idx2; i++)
 805                 if (port2.priority_mask[i])
 806                         emulate_priority(i, idx2);
 807         // If some port had priority over port 2, make it have priority over the merged port too.
 808         for (int i = idx2 + 1; i < GetSize(wr_ports); i++) {
 809                 auto &oport = wr_ports[i];
 810                 if (oport.priority_mask[idx2])
 811                         oport.priority_mask[idx1] = true;
 812         }
 813 }
 814
 815 void Mem::widen_prep(int wide_log2) {
 816         // Make sure start_offset and size are aligned to the port width,
 817         // adjust if necessary.
 818         int mask = ((1 << wide_log2) - 1);
 819         int delta = start_offset & mask;
 820         start_offset -= delta;
 821         size += delta;
 822         if (size & mask) {
 823                 size |= mask;
 824                 size++;
 825         }
 826 }
 827
 828 void Mem::widen_wr_port(int idx, int wide_log2) {
 829         widen_prep(wide_log2);
 830         auto &port = wr_ports[idx];
 831         log_assert(port.wide_log2 <= wide_log2);
 832         if (port.wide_log2 < wide_log2) {
 833                 SigSpec new_data, new_en;
 834                 SigSpec addr_lo = port.addr.extract(0, wide_log2);
 835                 for (int sub = 0; sub < (1 << wide_log2); sub += (1 << port.wide_log2))
 836                 {
 837                         Const cur_addr_lo(sub, wide_log2);
 838                         if (addr_lo == cur_addr_lo) {
 839                                 // Always writes to this subword.
 840                                 new_data.append(port.data);
 841                                 new_en.append(port.en);
 842                         } else if (addr_lo.is_fully_const()) {
 843                                 // Never writes to this subword.
 844                                 new_data.append(Const(State::Sx, GetSize(port.data)));
 845                                 new_en.append(Const(State::S0, GetSize(port.data)));
 846                         } else {
 847                                 // May or may not write to this subword.
 848                                 new_data.append(port.data);
 849                                 SigSpec addr_eq = module->Eq(NEW_ID, addr_lo, cur_addr_lo);
 850                                 SigSpec en = module->Mux(NEW_ID, Const(State::S0, GetSize(port.data)), port.en, addr_eq);
 851                                 new_en.append(en);
 852                         }
 853                 }
 854                 port.addr.replace(port.wide_log2, Const(State::S0, wide_log2 - port.wide_log2));
 855                 port.data = new_data;
 856                 port.en = new_en;
 857                 port.wide_log2 = wide_log2;
 858         }
 859 }