kernel/mem.cc

   1 /*
   2  *  yosys -- Yosys Open SYnthesis Suite
   3  *
   4  *  Copyright (C) 2020  Marcelina Kościelnicka <mwk@0x04.net>
   5  *
   6  *  Permission to use, copy, modify, and/or distribute this software for any
   7  *  purpose with or without fee is hereby granted, provided that the above
   8  *  copyright notice and this permission notice appear in all copies.
   9  *
  10  *  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  *  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  *  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  *  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  *  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  *  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  *  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  *
  18  */
  19
  20 #include "kernel/mem.h"
  21 #include "kernel/ff.h"
  22
  23 USING_YOSYS_NAMESPACE
  24
  25 void Mem::remove() {
  26         if (cell) {
  27                 module->remove(cell);
  28                 cell = nullptr;
  29         }
  30         if (mem) {
  31                 module->memories.erase(mem->name);
  32                 delete mem;
  33                 mem = nullptr;
  34         }
  35         for (auto &port : rd_ports) {
  36                 if (port.cell) {
  37                         module->remove(port.cell);
  38                         port.cell = nullptr;
  39                 }
  40         }
  41         for (auto &port : wr_ports) {
  42                 if (port.cell) {
  43                         module->remove(port.cell);
  44                         port.cell = nullptr;
  45                 }
  46         }
  47         for (auto &init : inits) {
  48                 if (init.cell) {
  49                         module->remove(init.cell);
  50                         init.cell = nullptr;
  51                 }
  52         }
  53 }
  54
  55 void Mem::emit() {
  56         check();
  57         std::vector<int> rd_left;
  58         for (int i = 0; i < GetSize(rd_ports); i++) {
  59                 auto &port = rd_ports[i];
  60                 if (port.removed) {
  61                         if (port.cell) {
  62                                 module->remove(port.cell);
  63                         }
  64                 } else {
  65                         rd_left.push_back(i);
  66                 }
  67         }
  68         std::vector<int> wr_left;
  69         for (int i = 0; i < GetSize(wr_ports); i++) {
  70                 auto &port = wr_ports[i];
  71                 if (port.removed) {
  72                         if (port.cell) {
  73                                 module->remove(port.cell);
  74                         }
  75                 } else {
  76                         wr_left.push_back(i);
  77                 }
  78         }
  79         std::vector<int> init_left;
  80         for (int i = 0; i < GetSize(inits); i++) {
  81                 auto &init = inits[i];
  82                 if (init.removed) {
  83                         if (init.cell) {
  84                                 module->remove(init.cell);
  85                         }
  86                 } else {
  87                         init_left.push_back(i);
  88                 }
  89         }
  90         for (int i = 0; i < GetSize(rd_left); i++)
  91                 if (i != rd_left[i])
  92                         std::swap(rd_ports[i], rd_ports[rd_left[i]]);
  93         rd_ports.resize(GetSize(rd_left));
  94         for (int i = 0; i < GetSize(wr_left); i++)
  95                 if (i != wr_left[i])
  96                         std::swap(wr_ports[i], wr_ports[wr_left[i]]);
  97         wr_ports.resize(GetSize(wr_left));
  98         for (int i = 0; i < GetSize(init_left); i++)
  99                 if (i != init_left[i])
 100                         std::swap(inits[i], inits[init_left[i]]);
 101         inits.resize(GetSize(init_left));
 102
 103         // for future: handle transparency mask here
 104
 105         for (auto &port : wr_ports) {
 106                 for (int i = 0; i < GetSize(wr_left); i++)
 107                         port.priority_mask[i] = port.priority_mask[wr_left[i]];
 108                 port.priority_mask.resize(GetSize(wr_left));
 109         }
 110
 111         if (packed) {
 112                 if (mem) {
 113                         module->memories.erase(mem->name);
 114                         delete mem;
 115                         mem = nullptr;
 116                 }
 117                 if (!cell) {
 118                         if (memid.empty())
 119                                 memid = NEW_ID;
 120                         cell = module->addCell(memid, ID($mem));
 121                 }
 122                 cell->attributes = attributes;
 123                 cell->parameters[ID::MEMID] = Const(memid.str());
 124                 cell->parameters[ID::WIDTH] = Const(width);
 125                 cell->parameters[ID::OFFSET] = Const(start_offset);
 126                 cell->parameters[ID::SIZE] = Const(size);
 127                 Const rd_wide_continuation, rd_clk_enable, rd_clk_polarity, rd_transparent;
 128                 Const wr_wide_continuation, wr_clk_enable, wr_clk_polarity;
 129                 SigSpec rd_clk, rd_en, rd_addr, rd_data;
 130                 SigSpec wr_clk, wr_en, wr_addr, wr_data;
 131                 int abits = 0;
 132                 for (auto &port : rd_ports)
 133                         abits = std::max(abits, GetSize(port.addr));
 134                 for (auto &port : wr_ports)
 135                         abits = std::max(abits, GetSize(port.addr));
 136                 cell->parameters[ID::ABITS] = Const(abits);
 137                 for (auto &port : rd_ports) {
 138                         // TODO: remove
 139                         log_assert(port.arst == State::S0);
 140                         log_assert(port.srst == State::S0);
 141                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 142                         if (port.cell) {
 143                                 module->remove(port.cell);
 144                                 port.cell = nullptr;
 145                         }
 146                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 147                         {
 148                                 rd_wide_continuation.bits.push_back(State(sub != 0));
 149                                 rd_clk_enable.bits.push_back(State(port.clk_enable));
 150                                 rd_clk_polarity.bits.push_back(State(port.clk_polarity));
 151                                 rd_transparent.bits.push_back(State(port.transparent));
 152                                 rd_clk.append(port.clk);
 153                                 rd_en.append(port.en);
 154                                 SigSpec addr = port.sub_addr(sub);
 155                                 addr.extend_u0(abits, false);
 156                                 rd_addr.append(addr);
 157                                 log_assert(GetSize(addr) == abits);
 158                         }
 159                         rd_data.append(port.data);
 160                 }
 161                 if (rd_ports.empty()) {
 162                         rd_wide_continuation = State::S0;
 163                         rd_clk_enable = State::S0;
 164                         rd_clk_polarity = State::S0;
 165                         rd_transparent = State::S0;
 166                 }
 167                 cell->parameters[ID::RD_PORTS] = Const(GetSize(rd_clk));
 168                 cell->parameters[ID::RD_CLK_ENABLE] = rd_clk_enable;
 169                 cell->parameters[ID::RD_CLK_POLARITY] = rd_clk_polarity;
 170                 cell->parameters[ID::RD_TRANSPARENT] = rd_transparent;
 171                 cell->setPort(ID::RD_CLK, rd_clk);
 172                 cell->setPort(ID::RD_EN, rd_en);
 173                 cell->setPort(ID::RD_ADDR, rd_addr);
 174                 cell->setPort(ID::RD_DATA, rd_data);
 175                 for (auto &port : wr_ports) {
 176                         if (port.cell) {
 177                                 module->remove(port.cell);
 178                                 port.cell = nullptr;
 179                         }
 180                         for (int sub = 0; sub < (1 << port.wide_log2); sub++)
 181                         {
 182                                 wr_wide_continuation.bits.push_back(State(sub != 0));
 183                                 wr_clk_enable.bits.push_back(State(port.clk_enable));
 184                                 wr_clk_polarity.bits.push_back(State(port.clk_polarity));
 185                                 wr_clk.append(port.clk);
 186                                 SigSpec addr = port.sub_addr(sub);
 187                                 addr.extend_u0(abits, false);
 188                                 wr_addr.append(addr);
 189                                 log_assert(GetSize(addr) == abits);
 190                         }
 191                         wr_en.append(port.en);
 192                         wr_data.append(port.data);
 193                 }
 194                 if (wr_ports.empty()) {
 195                         wr_wide_continuation = State::S0;
 196                         wr_clk_enable = State::S0;
 197                         wr_clk_polarity = State::S0;
 198                 }
 199                 cell->parameters[ID::WR_PORTS] = Const(GetSize(wr_clk));
 200                 cell->parameters[ID::WR_CLK_ENABLE] = wr_clk_enable;
 201                 cell->parameters[ID::WR_CLK_POLARITY] = wr_clk_polarity;
 202                 cell->setPort(ID::WR_CLK, wr_clk);
 203                 cell->setPort(ID::WR_EN, wr_en);
 204                 cell->setPort(ID::WR_ADDR, wr_addr);
 205                 cell->setPort(ID::WR_DATA, wr_data);
 206                 for (auto &init : inits) {
 207                         if (init.cell) {
 208                                 module->remove(init.cell);
 209                                 init.cell = nullptr;
 210                         }
 211                 }
 212                 cell->parameters[ID::INIT] = get_init_data();
 213         } else {
 214                 if (cell) {
 215                         module->remove(cell);
 216                         cell = nullptr;
 217                 }
 218                 if (!mem) {
 219                         if (memid.empty())
 220                                 memid = NEW_ID;
 221                         mem = new RTLIL::Memory;
 222                         mem->name = memid;
 223                         module->memories[memid] = mem;
 224                 }
 225                 mem->width = width;
 226                 mem->start_offset = start_offset;
 227                 mem->size = size;
 228                 mem->attributes = attributes;
 229                 for (auto &port : rd_ports) {
 230                         // TODO: remove
 231                         log_assert(port.arst == State::S0);
 232                         log_assert(port.srst == State::S0);
 233                         log_assert(port.init_value == Const(State::Sx, width << port.wide_log2));
 234                         if (!port.cell)
 235                                 port.cell = module->addCell(NEW_ID, ID($memrd));
 236                         port.cell->attributes = port.attributes;
 237                         port.cell->parameters[ID::MEMID] = memid.str();
 238                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 239                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 240                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 241                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 242                         port.cell->parameters[ID::TRANSPARENT] = port.transparent;
 243                         port.cell->setPort(ID::CLK, port.clk);
 244                         port.cell->setPort(ID::EN, port.en);
 245                         port.cell->setPort(ID::ADDR, port.addr);
 246                         port.cell->setPort(ID::DATA, port.data);
 247                 }
 248                 int idx = 0;
 249                 for (auto &port : wr_ports) {
 250                         if (!port.cell)
 251                                 port.cell = module->addCell(NEW_ID, ID($memwr));
 252                         port.cell->attributes = port.attributes;
 253                         port.cell->parameters[ID::MEMID] = memid.str();
 254                         port.cell->parameters[ID::ABITS] = GetSize(port.addr);
 255                         port.cell->parameters[ID::WIDTH] = width << port.wide_log2;
 256                         port.cell->parameters[ID::CLK_ENABLE] = port.clk_enable;
 257                         port.cell->parameters[ID::CLK_POLARITY] = port.clk_polarity;
 258                         port.cell->parameters[ID::PRIORITY] = idx++;
 259                         port.cell->setPort(ID::CLK, port.clk);
 260                         port.cell->setPort(ID::EN, port.en);
 261                         port.cell->setPort(ID::ADDR, port.addr);
 262                         port.cell->setPort(ID::DATA, port.data);
 263                 }
 264                 idx = 0;
 265                 for (auto &init : inits) {
 266                         if (!init.cell)
 267                                 init.cell = module->addCell(NEW_ID, ID($meminit));
 268                         init.cell->attributes = init.attributes;
 269                         init.cell->parameters[ID::MEMID] = memid.str();
 270                         init.cell->parameters[ID::ABITS] = GetSize(init.addr);
 271                         init.cell->parameters[ID::WIDTH] = width;
 272                         init.cell->parameters[ID::WORDS] = GetSize(init.data) / width;
 273                         init.cell->parameters[ID::PRIORITY] = idx++;
 274                         init.cell->setPort(ID::ADDR, init.addr);
 275                         init.cell->setPort(ID::DATA, init.data);
 276                 }
 277         }
 278 }
 279
 280 void Mem::clear_inits() {
 281         for (auto &init : inits)
 282                 init.removed = true;
 283 }
 284
 285 Const Mem::get_init_data() const {
 286         Const init_data(State::Sx, width * size);
 287         for (auto &init : inits) {
 288                 if (init.removed)
 289                         continue;
 290                 int offset = (init.addr.as_int() - start_offset) * width;
 291                 for (int i = 0; i < GetSize(init.data); i++)
 292                         if (0 <= i+offset && i+offset < GetSize(init_data))
 293                                 init_data.bits[i+offset] = init.data.bits[i];
 294         }
 295         return init_data;
 296 }
 297
 298 void Mem::check() {
 299         int max_wide_log2 = 0;
 300         for (auto &port : rd_ports) {
 301                 if (port.removed)
 302                         continue;
 303                 log_assert(GetSize(port.clk) == 1);
 304                 log_assert(GetSize(port.en) == 1);
 305                 log_assert(GetSize(port.arst) == 1);
 306                 log_assert(GetSize(port.srst) == 1);
 307                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 308                 log_assert(GetSize(port.init_value) == (width << port.wide_log2));
 309                 log_assert(GetSize(port.arst_value) == (width << port.wide_log2));
 310                 log_assert(GetSize(port.srst_value) == (width << port.wide_log2));
 311                 if (!port.clk_enable) {
 312                         log_assert(!port.transparent);
 313                         log_assert(port.en == State::S1);
 314                         log_assert(port.arst == State::S0);
 315                         log_assert(port.srst == State::S0);
 316                 }
 317                 for (int j = 0; j < port.wide_log2; j++) {
 318                         log_assert(port.addr[j] == State::S0);
 319                 }
 320                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 321         }
 322         for (int i = 0; i < GetSize(wr_ports); i++) {
 323                 auto &port = wr_ports[i];
 324                 if (port.removed)
 325                         continue;
 326                 log_assert(GetSize(port.clk) == 1);
 327                 log_assert(GetSize(port.en) == (width << port.wide_log2));
 328                 log_assert(GetSize(port.data) == (width << port.wide_log2));
 329                 for (int j = 0; j < port.wide_log2; j++) {
 330                         log_assert(port.addr[j] == State::S0);
 331                 }
 332                 max_wide_log2 = std::max(max_wide_log2, port.wide_log2);
 333                 log_assert(GetSize(port.priority_mask) == GetSize(wr_ports));
 334                 for (int j = 0; j < GetSize(wr_ports); j++) {
 335                         auto &wport = wr_ports[j];
 336                         if (port.priority_mask[j] && !wport.removed) {
 337                                 log_assert(j < i);
 338                                 log_assert(port.clk_enable == wport.clk_enable);
 339                                 if (port.clk_enable) {
 340                                         log_assert(port.clk == wport.clk);
 341                                         log_assert(port.clk_polarity == wport.clk_polarity);
 342                                 }
 343                         }
 344                 }
 345         }
 346         int mask = (1 << max_wide_log2) - 1;
 347         log_assert(!(start_offset & mask));
 348         log_assert(!(size & mask));
 349 }
 350
 351 namespace {
 352
 353         struct MemIndex {
 354                 dict<IdString, pool<Cell *>> rd_ports;
 355                 dict<IdString, pool<Cell *>> wr_ports;
 356                 dict<IdString, pool<Cell *>> inits;
 357                 MemIndex (Module *module) {
 358                         for (auto cell: module->cells()) {
 359                                 if (cell->type == ID($memwr))
 360                                         wr_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 361                                 else if (cell->type == ID($memrd))
 362                                         rd_ports[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 363                                 else if (cell->type == ID($meminit))
 364                                         inits[cell->parameters.at(ID::MEMID).decode_string()].insert(cell);
 365                         }
 366                 }
 367         };
 368
 369         Mem mem_from_memory(Module *module, RTLIL::Memory *mem, const MemIndex &index) {
 370                 Mem res(module, mem->name, mem->width, mem->start_offset, mem->size);
 371                 res.packed = false;
 372                 res.mem = mem;
 373                 res.attributes = mem->attributes;
 374                 if (index.rd_ports.count(mem->name)) {
 375                         for (auto cell : index.rd_ports.at(mem->name)) {
 376                                 MemRd mrd;
 377                                 mrd.cell = cell;
 378                                 mrd.attributes = cell->attributes;
 379                                 mrd.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 380                                 mrd.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 381                                 mrd.transparent = cell->parameters.at(ID::TRANSPARENT).as_bool();
 382                                 mrd.clk = cell->getPort(ID::CLK);
 383                                 mrd.en = cell->getPort(ID::EN);
 384                                 mrd.addr = cell->getPort(ID::ADDR);
 385                                 mrd.data = cell->getPort(ID::DATA);
 386                                 mrd.wide_log2 = ceil_log2(GetSize(mrd.data) / mem->width);
 387                                 mrd.ce_over_srst = false;
 388                                 mrd.arst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 389                                 mrd.srst_value = Const(State::Sx, mem->width << mrd.wide_log2);
 390                                 mrd.init_value = Const(State::Sx, mem->width << mrd.wide_log2);
 391                                 mrd.srst = State::S0;
 392                                 mrd.arst = State::S0;
 393                                 if (!mrd.clk_enable) {
 394                                         // Fix some patterns that we'll allow for backwards compatibility,
 395                                         // but don't want to see moving forwards: async transparent
 396                                         // ports (inherently meaningless) and async ports without
 397                                         // const 1 tied to EN bit (which may mean a latch in the future).
 398                                         mrd.transparent = false;
 399                                         if (mrd.en == State::Sx)
 400                                                 mrd.en = State::S1;
 401                                 }
 402                                 res.rd_ports.push_back(mrd);
 403                         }
 404                 }
 405                 if (index.wr_ports.count(mem->name)) {
 406                         std::vector<std::pair<int, MemWr>> ports;
 407                         for (auto cell : index.wr_ports.at(mem->name)) {
 408                                 MemWr mwr;
 409                                 mwr.cell = cell;
 410                                 mwr.attributes = cell->attributes;
 411                                 mwr.clk_enable = cell->parameters.at(ID::CLK_ENABLE).as_bool();
 412                                 mwr.clk_polarity = cell->parameters.at(ID::CLK_POLARITY).as_bool();
 413                                 mwr.clk = cell->getPort(ID::CLK);
 414                                 mwr.en = cell->getPort(ID::EN);
 415                                 mwr.addr = cell->getPort(ID::ADDR);
 416                                 mwr.data = cell->getPort(ID::DATA);
 417                                 mwr.wide_log2 = ceil_log2(GetSize(mwr.data) / mem->width);
 418                                 ports.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), mwr));
 419                         }
 420                         std::sort(ports.begin(), ports.end(), [](const std::pair<int, MemWr> &a, const std::pair<int, MemWr> &b) { return a.first < b.first; });
 421                         for (auto &it : ports)
 422                                 res.wr_ports.push_back(it.second);
 423                 }
 424                 if (index.inits.count(mem->name)) {
 425                         std::vector<std::pair<int, MemInit>> inits;
 426                         for (auto cell : index.inits.at(mem->name)) {
 427                                 MemInit init;
 428                                 init.cell = cell;
 429                                 init.attributes = cell->attributes;
 430                                 auto addr = cell->getPort(ID::ADDR);
 431                                 auto data = cell->getPort(ID::DATA);
 432                                 if (!addr.is_fully_const())
 433                                         log_error("Non-constant address %s in memory initialization %s.\n", log_signal(addr), log_id(cell));
 434                                 if (!data.is_fully_const())
 435                                         log_error("Non-constant data %s in memory initialization %s.\n", log_signal(data), log_id(cell));
 436                                 init.addr = addr.as_const();
 437                                 init.data = data.as_const();
 438                                 inits.push_back(std::make_pair(cell->parameters.at(ID::PRIORITY).as_int(), init));
 439                         }
 440                         std::sort(inits.begin(), inits.end(), [](const std::pair<int, MemInit> &a, const std::pair<int, MemInit> &b) { return a.first < b.first; });
 441                         for (auto &it : inits)
 442                                 res.inits.push_back(it.second);
 443                 }
 444                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 445                         auto &port = res.wr_ports[i];
 446                         port.priority_mask.resize(GetSize(res.wr_ports));
 447                         for (int j = 0; j < i; j++) {
 448                                 auto &oport = res.wr_ports[j];
 449                                 if (port.clk_enable != oport.clk_enable)
 450                                         continue;
 451                                 if (port.clk_enable && port.clk != oport.clk)
 452                                         continue;
 453                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 454                                         continue;
 455                                 port.priority_mask[j] = true;
 456                         }
 457                 }
 458                 res.check();
 459                 return res;
 460         }
 461
 462         Mem mem_from_cell(Cell *cell) {
 463                 Mem res(cell->module, cell->parameters.at(ID::MEMID).decode_string(),
 464                         cell->parameters.at(ID::WIDTH).as_int(),
 465                         cell->parameters.at(ID::OFFSET).as_int(),
 466                         cell->parameters.at(ID::SIZE).as_int()
 467                 );
 468                 int abits = cell->parameters.at(ID::ABITS).as_int();
 469                 res.packed = true;
 470                 res.cell = cell;
 471                 res.attributes = cell->attributes;
 472                 Const &init = cell->parameters.at(ID::INIT);
 473                 if (!init.is_fully_undef()) {
 474                         int pos = 0;
 475                         while (pos < res.size) {
 476                                 Const word = init.extract(pos * res.width, res.width, State::Sx);
 477                                 if (word.is_fully_undef()) {
 478                                         pos++;
 479                                 } else {
 480                                         int epos;
 481                                         for (epos = pos; epos < res.size; epos++) {
 482                                                 Const eword = init.extract(epos * res.width, res.width, State::Sx);
 483                                                 if (eword.is_fully_undef())
 484                                                         break;
 485                                         }
 486                                         MemInit minit;
 487                                         minit.addr = res.start_offset + pos;
 488                                         minit.data = init.extract(pos * res.width, (epos - pos) * res.width, State::Sx);
 489                                         res.inits.push_back(minit);
 490                                         pos = epos;
 491                                 }
 492                         }
 493                 }
 494                 for (int i = 0; i < cell->parameters.at(ID::RD_PORTS).as_int(); i++) {
 495                         MemRd mrd;
 496                         mrd.wide_log2 = 0;
 497                         mrd.clk_enable = cell->parameters.at(ID::RD_CLK_ENABLE).extract(i, 1).as_bool();
 498                         mrd.clk_polarity = cell->parameters.at(ID::RD_CLK_POLARITY).extract(i, 1).as_bool();
 499                         mrd.transparent = cell->parameters.at(ID::RD_TRANSPARENT).extract(i, 1).as_bool();
 500                         mrd.clk = cell->getPort(ID::RD_CLK).extract(i, 1);
 501                         mrd.en = cell->getPort(ID::RD_EN).extract(i, 1);
 502                         mrd.addr = cell->getPort(ID::RD_ADDR).extract(i * abits, abits);
 503                         mrd.data = cell->getPort(ID::RD_DATA).extract(i * res.width, res.width);
 504                         mrd.ce_over_srst = false;
 505                         mrd.arst_value = Const(State::Sx, res.width << mrd.wide_log2);
 506                         mrd.srst_value = Const(State::Sx, res.width << mrd.wide_log2);
 507                         mrd.init_value = Const(State::Sx, res.width << mrd.wide_log2);
 508                         mrd.srst = State::S0;
 509                         mrd.arst = State::S0;
 510                         res.rd_ports.push_back(mrd);
 511                 }
 512                 for (int i = 0; i < cell->parameters.at(ID::WR_PORTS).as_int(); i++) {
 513                         MemWr mwr;
 514                         mwr.wide_log2 = 0;
 515                         mwr.clk_enable = cell->parameters.at(ID::WR_CLK_ENABLE).extract(i, 1).as_bool();
 516                         mwr.clk_polarity = cell->parameters.at(ID::WR_CLK_POLARITY).extract(i, 1).as_bool();
 517                         mwr.clk = cell->getPort(ID::WR_CLK).extract(i, 1);
 518                         mwr.en = cell->getPort(ID::WR_EN).extract(i * res.width, res.width);
 519                         mwr.addr = cell->getPort(ID::WR_ADDR).extract(i * abits, abits);
 520                         mwr.data = cell->getPort(ID::WR_DATA).extract(i * res.width, res.width);
 521                         res.wr_ports.push_back(mwr);
 522                 }
 523                 for (int i = 0; i < GetSize(res.wr_ports); i++) {
 524                         auto &port = res.wr_ports[i];
 525                         port.priority_mask.resize(GetSize(res.wr_ports));
 526                         for (int j = 0; j < i; j++) {
 527                                 auto &oport = res.wr_ports[j];
 528                                 if (port.clk_enable != oport.clk_enable)
 529                                         continue;
 530                                 if (port.clk_enable && port.clk != oport.clk)
 531                                         continue;
 532                                 if (port.clk_enable && port.clk_polarity != oport.clk_polarity)
 533                                         continue;
 534                                 port.priority_mask[j] = true;
 535                         }
 536                 }
 537                 res.check();
 538                 return res;
 539         }
 540
 541 }
 542
 543 std::vector<Mem> Mem::get_all_memories(Module *module) {
 544         std::vector<Mem> res;
 545         MemIndex index(module);
 546         for (auto it: module->memories) {
 547                 res.push_back(mem_from_memory(module, it.second, index));
 548         }
 549         for (auto cell: module->cells()) {
 550                 if (cell->type == ID($mem))
 551                         res.push_back(mem_from_cell(cell));
 552         }
 553         return res;
 554 }
 555
 556 std::vector<Mem> Mem::get_selected_memories(Module *module) {
 557         std::vector<Mem> res;
 558         MemIndex index(module);
 559         for (auto it: module->memories) {
 560                 if (module->design->selected(module, it.second))
 561                         res.push_back(mem_from_memory(module, it.second, index));
 562         }
 563         for (auto cell: module->selected_cells()) {
 564                 if (cell->type == ID($mem))
 565                         res.push_back(mem_from_cell(cell));
 566         }
 567         return res;
 568 }
 569
 570 Cell *Mem::extract_rdff(int idx, FfInitVals *initvals) {
 571         MemRd &port = rd_ports[idx];
 572
 573         if (!port.clk_enable)
 574                 return nullptr;
 575
 576         Cell *c;
 577
 578         // There are two ways to handle rdff extraction when transparency is involved:
 579         //
 580         // - if all of the following conditions are true, put the FF on address input:
 581         //
 582         //   - the port has no clock enable, no reset, and no initial value
 583         //   - the port is transparent wrt all write ports (implying they also share
 584         //     the clock domain)
 585         //
 586         // - otherwise, put the FF on the data output, and make bypass paths for
 587         //   all write ports wrt which this port is transparent
 588         bool trans_use_addr = port.transparent;
 589
 590         // If there are no write ports at all, we could possibly use either way; do data
 591         // FF in this case.
 592         if (GetSize(wr_ports) == 0)
 593                 trans_use_addr = false;
 594
 595         if (port.en != State::S1 || port.srst != State::S0 || port.arst != State::S0 || !port.init_value.is_fully_undef())
 596                 trans_use_addr = false;
 597
 598         if (trans_use_addr)
 599         {
 600                 // Do not put a register in front of constant address bits — this is both
 601                 // unnecessary and will break wide ports.
 602                 int width = 0;
 603                 for (int i = 0; i < GetSize(port.addr); i++)
 604                         if (port.addr[i].wire)
 605                                 width++;
 606
 607                 if (width)
 608                 {
 609                         SigSpec sig_q = module->addWire(stringf("$%s$rdreg[%d]$q", memid.c_str(), idx), width);
 610                         SigSpec sig_d;
 611
 612                         int pos = 0;
 613                         for (int i = 0; i < GetSize(port.addr); i++)
 614                                 if (port.addr[i].wire) {
 615                                         sig_d.append(port.addr[i]);
 616                                         port.addr[i] = sig_q[pos++];
 617                                 }
 618
 619                         c = module->addDff(stringf("$%s$rdreg[%d]", memid.c_str(), idx), port.clk, sig_d, sig_q, port.clk_polarity);
 620                 } else {
 621                         c = nullptr;
 622                 }
 623         }
 624         else
 625         {
 626                 log_assert(port.arst == State::S0 || port.srst == State::S0);
 627
 628                 SigSpec async_d = module->addWire(stringf("$%s$rdreg[%d]$d", memid.c_str(), idx), GetSize(port.data));
 629                 SigSpec sig_d = async_d;
 630
 631                 for (int i = 0; i < GetSize(wr_ports); i++) {
 632                         auto &wport = wr_ports[i];
 633                         if (port.transparent) {
 634                                 log_assert(wport.clk_enable);
 635                                 log_assert(wport.clk == port.clk);
 636                                 log_assert(wport.clk_enable == port.clk_enable);
 637                                 int min_wide_log2 = std::min(port.wide_log2, wport.wide_log2);
 638                                 int max_wide_log2 = std::max(port.wide_log2, wport.wide_log2);
 639                                 bool wide_write = wport.wide_log2 > port.wide_log2;
 640                                 for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 641                                         SigSpec raddr = port.addr;
 642                                         SigSpec waddr = wport.addr;
 643                                         if (wide_write)
 644                                                 waddr = wport.sub_addr(sub);
 645                                         else
 646                                                 raddr = port.sub_addr(sub);
 647                                         SigSpec addr_eq;
 648                                         if (raddr != waddr)
 649                                                 addr_eq = module->Eq(stringf("$%s$rdtransen[%d][%d][%d]$d", memid.c_str(), idx, i, sub), raddr, waddr);
 650                                         int pos = 0;
 651                                         int ewidth = width << min_wide_log2;
 652                                         int wsub = wide_write ? sub : 0;
 653                                         int rsub = wide_write ? 0 : sub;
 654                                         while (pos < ewidth) {
 655                                                 int epos = pos;
 656                                                 while (epos < ewidth && wport.en[epos + wsub * width] == wport.en[pos + wsub * width])
 657                                                         epos++;
 658                                                 SigSpec cur = sig_d.extract(pos + rsub * width, epos-pos);
 659                                                 SigSpec other = wport.data.extract(pos + wsub * width, epos-pos);
 660                                                 SigSpec cond;
 661                                                 if (raddr != waddr)
 662                                                         cond = module->And(stringf("$%s$rdtransgate[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), wport.en[pos + wsub * width], addr_eq);
 663                                                 else
 664                                                         cond = wport.en[pos + wsub * width];
 665                                                 SigSpec merged = module->Mux(stringf("$%s$rdtransmux[%d][%d][%d][%d]$d", memid.c_str(), idx, i, sub, pos), cur, other, cond);
 666                                                 sig_d.replace(pos + rsub * width, merged);
 667                                                 pos = epos;
 668                                         }
 669                                 }
 670                         }
 671                 }
 672
 673                 IdString name = stringf("$%s$rdreg[%d]", memid.c_str(), idx);
 674                 FfData ff(initvals);
 675                 ff.width = GetSize(port.data);
 676                 ff.has_clk = true;
 677                 ff.sig_clk = port.clk;
 678                 ff.pol_clk = port.clk_polarity;
 679                 if (port.en != State::S1) {
 680                         ff.has_en = true;
 681                         ff.pol_en = true;
 682                         ff.sig_en = port.en;
 683                 }
 684                 if (port.arst != State::S0) {
 685                         ff.has_arst = true;
 686                         ff.pol_arst = true;
 687                         ff.sig_arst = port.arst;
 688                         ff.val_arst = port.arst_value;
 689                 }
 690                 if (port.srst != State::S0) {
 691                         ff.has_srst = true;
 692                         ff.pol_srst = true;
 693                         ff.sig_srst = port.srst;
 694                         ff.val_srst = port.srst_value;
 695                         ff.ce_over_srst = ff.has_en && port.ce_over_srst;
 696                 }
 697                 ff.sig_d = sig_d;
 698                 ff.sig_q = port.data;
 699                 ff.val_init = port.init_value;
 700                 port.data = async_d;
 701                 c = ff.emit(module, name);
 702         }
 703
 704         log("Extracted %s FF from read port %d of %s.%s: %s\n", trans_use_addr ? "addr" : "data",
 705                         idx, log_id(module), log_id(memid), log_id(c));
 706
 707         port.en = State::S1;
 708         port.clk = State::S0;
 709         port.arst = State::S0;
 710         port.srst = State::S0;
 711         port.clk_enable = false;
 712         port.clk_polarity = true;
 713         port.transparent = false;
 714         port.ce_over_srst = false;
 715         port.arst_value = Const(State::Sx, GetSize(port.data));
 716         port.srst_value = Const(State::Sx, GetSize(port.data));
 717         port.init_value = Const(State::Sx, GetSize(port.data));
 718
 719         return c;
 720 }
 721
 722 void Mem::narrow() {
 723         // NOTE: several passes depend on this function not modifying
 724         // the design at all until (and unless) emit() is called.
 725         // Be careful to preserve this.
 726         std::vector<MemRd> new_rd_ports;
 727         std::vector<MemWr> new_wr_ports;
 728         std::vector<std::pair<int, int>> new_rd_map;
 729         std::vector<std::pair<int, int>> new_wr_map;
 730         for (int i = 0; i < GetSize(rd_ports); i++) {
 731                 auto &port = rd_ports[i];
 732                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 733                         new_rd_map.push_back(std::make_pair(i, sub));
 734                 }
 735         }
 736         for (int i = 0; i < GetSize(wr_ports); i++) {
 737                 auto &port = wr_ports[i];
 738                 for (int sub = 0; sub < (1 << port.wide_log2); sub++) {
 739                         new_wr_map.push_back(std::make_pair(i, sub));
 740                 }
 741         }
 742         for (auto &it : new_rd_map) {
 743                 MemRd &orig = rd_ports[it.first];
 744                 MemRd port = orig;
 745                 if (it.second != 0)
 746                         port.cell = nullptr;
 747                 if (port.wide_log2) {
 748                         port.data = port.data.extract(it.second * width, width);
 749                         port.init_value = port.init_value.extract(it.second * width, width);
 750                         port.arst_value = port.arst_value.extract(it.second * width, width);
 751                         port.srst_value = port.srst_value.extract(it.second * width, width);
 752                         port.addr = port.sub_addr(it.second);
 753                         port.wide_log2 = 0;
 754                 }
 755                 new_rd_ports.push_back(port);
 756         }
 757         for (auto &it : new_wr_map) {
 758                 MemWr &orig = wr_ports[it.first];
 759                 MemWr port = orig;
 760                 if (it.second != 0)
 761                         port.cell = nullptr;
 762                 if (port.wide_log2) {
 763                         port.data = port.data.extract(it.second * width, width);
 764                         port.en = port.en.extract(it.second * width, width);
 765                         port.addr = port.sub_addr(it.second);
 766                         port.wide_log2 = 0;
 767                 }
 768                 port.priority_mask.clear();
 769                 for (auto &it2 : new_wr_map)
 770                         port.priority_mask.push_back(orig.priority_mask[it2.first]);
 771                 new_wr_ports.push_back(port);
 772         }
 773         std::swap(rd_ports, new_rd_ports);
 774         std::swap(wr_ports, new_wr_ports);
 775 }
 776
 777 void Mem::emulate_priority(int idx1, int idx2)
 778 {
 779         auto &port1 = wr_ports[idx1];
 780         auto &port2 = wr_ports[idx2];
 781         if (!port2.priority_mask[idx1])
 782                 return;
 783         int min_wide_log2 = std::min(port1.wide_log2, port2.wide_log2);
 784         int max_wide_log2 = std::max(port1.wide_log2, port2.wide_log2);
 785         bool wide1 = port1.wide_log2 > port2.wide_log2;
 786         for (int sub = 0; sub < (1 << max_wide_log2); sub += (1 << min_wide_log2)) {
 787                 SigSpec addr1 = port1.addr;
 788                 SigSpec addr2 = port2.addr;
 789                 if (wide1)
 790                         addr1 = port1.sub_addr(sub);
 791                 else
 792                         addr2 = port2.sub_addr(sub);
 793                 SigSpec addr_eq = module->Eq(NEW_ID, addr1, addr2);
 794                 int ewidth = width << min_wide_log2;
 795                 int sub1 = wide1 ? sub : 0;
 796                 int sub2 = wide1 ? 0 : sub;
 797                 dict<std::pair<SigBit, SigBit>, SigBit> cache;
 798                 for (int pos = 0; pos < ewidth; pos++) {
 799                         SigBit &en1 = port1.en[pos + sub1 * width];
 800                         SigBit &en2 = port2.en[pos + sub2 * width];
 801                         std::pair<SigBit, SigBit> key(en1, en2);
 802                         if (cache.count(key)) {
 803                                 en1 = cache[key];
 804                         } else {
 805                                 SigBit active2 = module->And(NEW_ID, addr_eq, en2);
 806                                 SigBit nactive2 = module->Not(NEW_ID, active2);
 807                                 en1 = cache[key] = module->And(NEW_ID, en1, nactive2);
 808                         }
 809                 }
 810         }
 811         port2.priority_mask[idx1] = false;
 812 }
 813
 814 void Mem::prepare_wr_merge(int idx1, int idx2) {
 815         log_assert(idx1 < idx2);
 816         auto &port1 = wr_ports[idx1];
 817         auto &port2 = wr_ports[idx2];
 818         // If port 2 has priority over a port before port 1, make port 1 have priority too.
 819         for (int i = 0; i < idx1; i++)
 820                 if (port2.priority_mask[i])
 821                         port1.priority_mask[i] = true;
 822         // If port 2 has priority over a port after port 1, emulate it.
 823         for (int i = idx1 + 1; i < idx2; i++)
 824                 if (port2.priority_mask[i])
 825                         emulate_priority(i, idx2);
 826         // If some port had priority over port 2, make it have priority over the merged port too.
 827         for (int i = idx2 + 1; i < GetSize(wr_ports); i++) {
 828                 auto &oport = wr_ports[i];
 829                 if (oport.priority_mask[idx2])
 830                         oport.priority_mask[idx1] = true;
 831         }
 832 }
 833
 834 void Mem::widen_prep(int wide_log2) {
 835         // Make sure start_offset and size are aligned to the port width,
 836         // adjust if necessary.
 837         int mask = ((1 << wide_log2) - 1);
 838         int delta = start_offset & mask;
 839         start_offset -= delta;
 840         size += delta;
 841         if (size & mask) {
 842                 size |= mask;
 843                 size++;
 844         }
 845 }
 846
 847 void Mem::widen_wr_port(int idx, int wide_log2) {
 848         widen_prep(wide_log2);
 849         auto &port = wr_ports[idx];
 850         log_assert(port.wide_log2 <= wide_log2);
 851         if (port.wide_log2 < wide_log2) {
 852                 SigSpec new_data, new_en;
 853                 SigSpec addr_lo = port.addr.extract(0, wide_log2);
 854                 for (int sub = 0; sub < (1 << wide_log2); sub += (1 << port.wide_log2))
 855                 {
 856                         Const cur_addr_lo(sub, wide_log2);
 857                         if (addr_lo == cur_addr_lo) {
 858                                 // Always writes to this subword.
 859                                 new_data.append(port.data);
 860                                 new_en.append(port.en);
 861                         } else if (addr_lo.is_fully_const()) {
 862                                 // Never writes to this subword.
 863                                 new_data.append(Const(State::Sx, GetSize(port.data)));
 864                                 new_en.append(Const(State::S0, GetSize(port.data)));
 865                         } else {
 866                                 // May or may not write to this subword.
 867                                 new_data.append(port.data);
 868                                 SigSpec addr_eq = module->Eq(NEW_ID, addr_lo, cur_addr_lo);
 869                                 SigSpec en = module->Mux(NEW_ID, Const(State::S0, GetSize(port.data)), port.en, addr_eq);
 870                                 new_en.append(en);
 871                         }
 872                 }
 873                 port.addr.replace(port.wide_log2, Const(State::S0, wide_log2 - port.wide_log2));
 874                 port.data = new_data;
 875                 port.en = new_en;
 876                 port.wide_log2 = wide_log2;
 877         }
 878 }