src/mem/protocol/MOESI_CMP_token-L1cache.sm

   1 /*
   2  * Copyright (c) 1999-2013 Mark D. Hill and David A. Wood
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are
   7  * met: redistributions of source code must retain the above copyright
   8  * notice, this list of conditions and the following disclaimer;
   9  * redistributions in binary form must reproduce the above copyright
  10  * notice, this list of conditions and the following disclaimer in the
  11  * documentation and/or other materials provided with the distribution;
  12  * neither the name of the copyright holders nor the names of its
  13  * contributors may be used to endorse or promote products derived from
  14  * this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  19  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  20  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  21  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  22  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 /*
  30  * $Id: MOESI_CMP_token-L1cache.sm 1.22 05/01/19 15:55:39-06:00 beckmann@s0-28.cs.wisc.edu $
  31  *
  32  */
  33
  34 machine(L1Cache, "Token protocol")
  35  : Sequencer * sequencer;
  36    CacheMemory * L1Icache;
  37    CacheMemory * L1Dcache;
  38    int l2_select_num_bits;
  39    int N_tokens;
  40
  41    Cycles l1_request_latency := 2;
  42    Cycles l1_response_latency := 2;
  43    int retry_threshold := 1;
  44    Cycles fixed_timeout_latency := 100;
  45    Cycles reissue_wakeup_latency := 10;
  46    Cycles use_timeout_latency := 50;
  47
  48    bool dynamic_timeout_enabled := "True";
  49    bool no_mig_atomic := "True";
  50    bool send_evictions;
  51
  52    // Message Queues
  53    // From this node's L1 cache TO the network
  54
  55    // a local L1 -> this L2 bank
  56    MessageBuffer * responseFromL1Cache, network="To", virtual_network="4",
  57         ordered="false", vnet_type="response";
  58    MessageBuffer * persistentFromL1Cache, network="To", virtual_network="3",
  59         ordered="true", vnet_type="persistent";
  60    // a local L1 -> this L2 bank, currently ordered with directory forwarded requests
  61    MessageBuffer * requestFromL1Cache, network="To", virtual_network="1",
  62         ordered="false", vnet_type="request";
  63
  64
  65    // To this node's L1 cache FROM the network
  66
  67    // a L2 bank -> this L1
  68    MessageBuffer * responseToL1Cache, network="From", virtual_network="4",
  69         ordered="false", vnet_type="response";
  70    MessageBuffer * persistentToL1Cache, network="From", virtual_network="3",
  71         ordered="true", vnet_type="persistent";
  72    // a L2 bank -> this L1
  73    MessageBuffer * requestToL1Cache, network="From", virtual_network="1",
  74         ordered="false", vnet_type="request";
  75
  76 {
  77   // STATES
  78   state_declaration(State, desc="Cache states", default="L1Cache_State_I") {
  79     // Base states
  80     NP, AccessPermission:Invalid, "NP", desc="Not Present";
  81     I, AccessPermission:Invalid, "I", desc="Idle";
  82     S, AccessPermission:Read_Only, "S", desc="Shared";
  83     O, AccessPermission:Read_Only, "O", desc="Owned";
  84     M, AccessPermission:Read_Only, "M", desc="Modified (dirty)";
  85     MM, AccessPermission:Read_Write, "MM", desc="Modified (dirty and locally modified)";
  86     M_W, AccessPermission:Read_Only, "M^W", desc="Modified (dirty), waiting";
  87     MM_W, AccessPermission:Read_Write, "MM^W", desc="Modified (dirty and locally modified), waiting";
  88
  89     // Transient States
  90     IM, AccessPermission:Busy, "IM", desc="Issued GetX";
  91     SM, AccessPermission:Read_Only, "SM", desc="Issued GetX, we still have an old copy of the line";
  92     OM, AccessPermission:Read_Only, "OM", desc="Issued GetX, received data";
  93     IS, AccessPermission:Busy, "IS", desc="Issued GetS";
  94
  95     // Locked states
  96     I_L, AccessPermission:Busy, "I^L", desc="Invalid, Locked";
  97     S_L, AccessPermission:Busy, "S^L", desc="Shared, Locked";
  98     IM_L, AccessPermission:Busy, "IM^L", desc="Invalid, Locked, trying to go to Modified";
  99     SM_L, AccessPermission:Busy, "SM^L", desc="Shared, Locked, trying to go to Modified";
 100     IS_L, AccessPermission:Busy, "IS^L", desc="Invalid, Locked, trying to go to Shared";
 101   }
 102
 103   // EVENTS
 104   enumeration(Event, desc="Cache events") {
 105     Load,            desc="Load request from the processor";
 106     Ifetch,          desc="I-fetch request from the processor";
 107     Store,           desc="Store request from the processor";
 108     Atomic,          desc="Atomic request from the processor";
 109     L1_Replacement,  desc="L1 Replacement";
 110
 111     // Responses
 112     Data_Shared,             desc="Received a data message, we are now a sharer";
 113     Data_Owner,              desc="Received a data message, we are now the owner";
 114     Data_All_Tokens,   desc="Received a data message, we are now the owner, we now have all the tokens";
 115     Ack,                     desc="Received an ack message";
 116     Ack_All_Tokens,          desc="Received an ack message, we now have all the tokens";
 117
 118     // Requests
 119     Transient_GETX,  desc="A GetX from another processor";
 120     Transient_Local_GETX,  desc="A GetX from another processor";
 121     Transient_GETS,  desc="A GetS from another processor";
 122     Transient_Local_GETS,  desc="A GetS from another processor";
 123     Transient_GETS_Last_Token,  desc="A GetS from another processor";
 124     Transient_Local_GETS_Last_Token,  desc="A GetS from another processor";
 125
 126     // Lock/Unlock for distributed
 127     Persistent_GETX,     desc="Another processor has priority to read/write";
 128     Persistent_GETS,     desc="Another processor has priority to read";
 129     Persistent_GETS_Last_Token, desc="Another processor has priority to read, no more tokens";
 130     Own_Lock_or_Unlock,  desc="This processor now has priority";
 131
 132     // Triggers
 133     Request_Timeout,         desc="Timeout";
 134     Use_TimeoutStarverX,             desc="Timeout";
 135     Use_TimeoutStarverS,             desc="Timeout";
 136     Use_TimeoutNoStarvers,             desc="Timeout";
 137     Use_TimeoutNoStarvers_NoMig,     desc="Timeout Don't Migrate";
 138   }
 139
 140   // TYPES
 141
 142   // CacheEntry
 143   structure(Entry, desc="...", interface="AbstractCacheEntry") {
 144     State CacheState,        desc="cache state";
 145     bool Dirty,              desc="Is the data dirty (different than memory)?";
 146     int Tokens,              desc="The number of tokens we're holding for the line";
 147     DataBlock DataBlk,       desc="data for the block";
 148   }
 149
 150
 151   // TBE fields
 152   structure(TBE, desc="...") {
 153     Address Addr,                      desc="Physical address for this TBE";
 154     State TBEState,                       desc="Transient state";
 155     int IssueCount,      default="0",     desc="The number of times we've issued a request for this line.";
 156     Address PC,                           desc="Program counter of request";
 157
 158     bool WentPersistent, default="false",  desc="Request went persistent";
 159     bool ExternalResponse, default="false", desc="Response came from an external controller";
 160     bool IsAtomic, default="false",       desc="Request was an atomic request";
 161
 162     AccessType TypeOfAccess,                desc="Type of request (used for profiling)";
 163     Cycles IssueTime,                       desc="Time the request was issued";
 164     RubyAccessMode AccessMode,    desc="user/supervisor access type";
 165     PrefetchBit Prefetch,         desc="Is this a prefetch request";
 166   }
 167
 168   structure(TBETable, external="yes") {
 169     TBE lookup(Address);
 170     void allocate(Address);
 171     void deallocate(Address);
 172     bool isPresent(Address);
 173   }
 174
 175   structure(PersistentTable, external="yes") {
 176     void persistentRequestLock(Address, MachineID, AccessType);
 177     void persistentRequestUnlock(Address, MachineID);
 178     bool okToIssueStarving(Address, MachineID);
 179     MachineID findSmallest(Address);
 180     AccessType typeOfSmallest(Address);
 181     void markEntries(Address);
 182     bool isLocked(Address);
 183     int countStarvingForAddress(Address);
 184     int countReadStarvingForAddress(Address);
 185   }
 186
 187   void set_cache_entry(AbstractCacheEntry b);
 188   void unset_cache_entry();
 189   void set_tbe(TBE b);
 190   void unset_tbe();
 191   void wakeUpAllBuffers();
 192   void wakeUpBuffers(Address a);
 193   Cycles curCycle();
 194
 195   TBETable L1_TBEs, template="<L1Cache_TBE>", constructor="m_number_of_TBEs";
 196
 197   MessageBuffer mandatoryQueue, ordered="false", abstract_chip_ptr="true";
 198
 199   bool starving, default="false";
 200   int l2_select_low_bit, default="RubySystem::getBlockSizeBits()";
 201
 202   PersistentTable persistentTable;
 203   TimerTable useTimerTable;
 204   TimerTable reissueTimerTable;
 205
 206   int outstandingRequests, default="0";
 207   int outstandingPersistentRequests, default="0";
 208
 209   // Constant that provides hysteresis for calculated the estimated average
 210   int averageLatencyHysteresis, default="(8)";
 211   Cycles averageLatencyCounter,
 212         default="(Cycles(500) << (*m_averageLatencyHysteresis_ptr))";
 213
 214   Cycles averageLatencyEstimate() {
 215     DPRINTF(RubySlicc, "%d\n",
 216             (averageLatencyCounter >> averageLatencyHysteresis));
 217     return averageLatencyCounter >> averageLatencyHysteresis;
 218   }
 219
 220   void updateAverageLatencyEstimate(Cycles latency) {
 221     DPRINTF(RubySlicc, "%d\n", latency);
 222
 223     // By subtracting the current average and then adding the most
 224     // recent sample, we calculate an estimate of the recent average.
 225     // If we simply used a running sum and divided by the total number
 226     // of entries, the estimate of the average would adapt very slowly
 227     // after the execution has run for a long time.
 228     // averageLatencyCounter := averageLatencyCounter - averageLatencyEstimate() + latency;
 229
 230     averageLatencyCounter := averageLatencyCounter - averageLatencyEstimate() + latency;
 231   }
 232
 233   Entry getCacheEntry(Address addr), return_by_pointer="yes" {
 234     Entry L1Dcache_entry := static_cast(Entry, "pointer", L1Dcache.lookup(addr));
 235     if(is_valid(L1Dcache_entry)) {
 236       return L1Dcache_entry;
 237     }
 238
 239     Entry L1Icache_entry := static_cast(Entry, "pointer", L1Icache.lookup(addr));
 240     return L1Icache_entry;
 241   }
 242
 243   DataBlock getDataBlock(Address addr), return_by_ref="yes" {
 244     return getCacheEntry(addr).DataBlk;
 245   }
 246
 247   Entry getL1DCacheEntry(Address addr), return_by_pointer="yes" {
 248     Entry L1Dcache_entry := static_cast(Entry, "pointer", L1Dcache.lookup(addr));
 249     return L1Dcache_entry;
 250   }
 251
 252   Entry getL1ICacheEntry(Address addr), return_by_pointer="yes" {
 253     Entry L1Icache_entry := static_cast(Entry, "pointer", L1Icache.lookup(addr));
 254     return L1Icache_entry;
 255   }
 256
 257   int getTokens(Entry cache_entry) {
 258     if (is_valid(cache_entry)) {
 259       return cache_entry.Tokens;
 260     }
 261     return 0;
 262   }
 263
 264   State getState(TBE tbe, Entry cache_entry, Address addr) {
 265
 266     if (is_valid(tbe)) {
 267       return tbe.TBEState;
 268     } else if (is_valid(cache_entry)) {
 269       return cache_entry.CacheState;
 270     } else {
 271       if (persistentTable.isLocked(addr) && (persistentTable.findSmallest(addr) != machineID)) {
 272       // Not in cache, in persistent table, but this processor isn't highest priority
 273       return State:I_L;
 274       } else {
 275         return State:NP;
 276       }
 277     }
 278   }
 279
 280   void setState(TBE tbe, Entry cache_entry, Address addr, State state) {
 281     assert((L1Dcache.isTagPresent(addr) && L1Icache.isTagPresent(addr)) == false);
 282
 283     if (is_valid(tbe)) {
 284       assert(state != State:I);
 285       assert(state != State:S);
 286       assert(state != State:O);
 287       assert(state != State:MM);
 288       assert(state != State:M);
 289       tbe.TBEState := state;
 290     }
 291
 292     if (is_valid(cache_entry)) {
 293       // Make sure the token count is in range
 294       assert(cache_entry.Tokens >= 0);
 295       assert(cache_entry.Tokens <= max_tokens());
 296       assert(cache_entry.Tokens != (max_tokens() / 2));
 297
 298       if ((state == State:I_L) ||
 299           (state == State:IM_L) ||
 300           (state == State:IS_L)) {
 301         // Make sure we have no tokens in the "Invalid, locked" states
 302           assert(cache_entry.Tokens == 0);
 303
 304         // Make sure the line is locked
 305         // assert(persistentTable.isLocked(addr));
 306
 307         // But we shouldn't have highest priority for it
 308         // assert(persistentTable.findSmallest(addr) != id);
 309
 310       } else if ((state == State:S_L) ||
 311                  (state == State:SM_L)) {
 312         assert(cache_entry.Tokens >= 1);
 313         assert(cache_entry.Tokens < (max_tokens() / 2));
 314
 315         // Make sure the line is locked...
 316         // assert(persistentTable.isLocked(addr));
 317
 318         // ...But we shouldn't have highest priority for it...
 319         // assert(persistentTable.findSmallest(addr) != id);
 320
 321         // ...And it must be a GETS request
 322         // assert(persistentTable.typeOfSmallest(addr) == AccessType:Read);
 323
 324       } else {
 325
 326         // If there is an entry in the persistent table of this block,
 327         // this processor needs to have an entry in the table for this
 328         // block, and that entry better be the smallest (highest
 329         // priority).  Otherwise, the state should have been one of
 330         // locked states
 331
 332         //if (persistentTable.isLocked(addr)) {
 333         //  assert(persistentTable.findSmallest(addr) == id);
 334         //}
 335       }
 336
 337       // in M and E you have all the tokens
 338       if (state == State:MM || state == State:M || state == State:MM_W || state == State:M_W) {
 339         assert(cache_entry.Tokens == max_tokens());
 340       }
 341
 342       // in NP you have no tokens
 343       if (state == State:NP) {
 344         assert(cache_entry.Tokens == 0);
 345       }
 346
 347       // You have at least one token in S-like states
 348       if (state == State:S || state == State:SM) {
 349         assert(cache_entry.Tokens > 0);
 350       }
 351
 352       // You have at least half the token in O-like states
 353       if (state == State:O && state == State:OM) {
 354         assert(cache_entry.Tokens > (max_tokens() / 2));
 355       }
 356
 357       cache_entry.CacheState := state;
 358     }
 359   }
 360
 361   AccessPermission getAccessPermission(Address addr) {
 362     TBE tbe := L1_TBEs[addr];
 363     if(is_valid(tbe)) {
 364       return L1Cache_State_to_permission(tbe.TBEState);
 365     }
 366
 367     Entry cache_entry := getCacheEntry(addr);
 368     if(is_valid(cache_entry)) {
 369       return L1Cache_State_to_permission(cache_entry.CacheState);
 370     }
 371
 372     return AccessPermission:NotPresent;
 373   }
 374
 375   void setAccessPermission(Entry cache_entry, Address addr, State state) {
 376     if (is_valid(cache_entry)) {
 377       cache_entry.changePermission(L1Cache_State_to_permission(state));
 378     }
 379   }
 380
 381   Event mandatory_request_type_to_event(RubyRequestType type) {
 382     if (type == RubyRequestType:LD) {
 383       return Event:Load;
 384     } else if (type == RubyRequestType:IFETCH) {
 385       return Event:Ifetch;
 386     } else if (type == RubyRequestType:ST) {
 387       return Event:Store;
 388     } else if (type == RubyRequestType:ATOMIC) {
 389       if (no_mig_atomic) {
 390         return Event:Atomic;
 391       } else {
 392         return Event:Store;
 393       }
 394     } else {
 395       error("Invalid RubyRequestType");
 396     }
 397   }
 398
 399   AccessType cache_request_type_to_access_type(RubyRequestType type) {
 400     if ((type == RubyRequestType:LD) || (type == RubyRequestType:IFETCH)) {
 401       return AccessType:Read;
 402     } else if ((type == RubyRequestType:ST) || (type == RubyRequestType:ATOMIC)) {
 403       return AccessType:Write;
 404     } else {
 405       error("Invalid RubyRequestType");
 406     }
 407   }
 408
 409   // NOTE: direct local hits should not call this function
 410   bool isExternalHit(Address addr, MachineID sender) {
 411     if (machineIDToMachineType(sender) == MachineType:L1Cache) {
 412       return true;
 413     } else if (machineIDToMachineType(sender) == MachineType:L2Cache) {
 414
 415       if (sender == mapAddressToRange(addr, MachineType:L2Cache,
 416                       l2_select_low_bit, l2_select_num_bits, intToID(0))) {
 417         return false;
 418       } else {
 419         return  true;
 420       }
 421     }
 422
 423     return true;
 424   }
 425
 426   bool okToIssueStarving(Address addr, MachineID machineID) {
 427     return persistentTable.okToIssueStarving(addr, machineID);
 428   }
 429
 430   void markPersistentEntries(Address addr) {
 431     persistentTable.markEntries(addr);
 432   }
 433
 434   void setExternalResponse(TBE tbe) {
 435     assert(is_valid(tbe));
 436     tbe.ExternalResponse := true;
 437   }
 438
 439   bool IsAtomic(TBE tbe) {
 440     assert(is_valid(tbe));
 441     return tbe.IsAtomic;
 442   }
 443
 444   // ** OUT_PORTS **
 445   out_port(persistentNetwork_out, PersistentMsg, persistentFromL1Cache);
 446   out_port(requestNetwork_out, RequestMsg, requestFromL1Cache);
 447   out_port(responseNetwork_out, ResponseMsg, responseFromL1Cache);
 448   out_port(requestRecycle_out, RequestMsg, requestToL1Cache);
 449
 450   // ** IN_PORTS **
 451
 452   // Use Timer
 453   in_port(useTimerTable_in, Address, useTimerTable, rank=5) {
 454     if (useTimerTable_in.isReady()) {
 455       TBE tbe := L1_TBEs[useTimerTable.readyAddress()];
 456
 457       if (persistentTable.isLocked(useTimerTable.readyAddress()) &&
 458           (persistentTable.findSmallest(useTimerTable.readyAddress()) != machineID)) {
 459         if (persistentTable.typeOfSmallest(useTimerTable.readyAddress()) == AccessType:Write) {
 460           trigger(Event:Use_TimeoutStarverX, useTimerTable.readyAddress(),
 461                   getCacheEntry(useTimerTable.readyAddress()), tbe);
 462         } else {
 463           trigger(Event:Use_TimeoutStarverS, useTimerTable.readyAddress(),
 464                   getCacheEntry(useTimerTable.readyAddress()), tbe);
 465         }
 466       } else {
 467         if (no_mig_atomic && IsAtomic(tbe)) {
 468           trigger(Event:Use_TimeoutNoStarvers_NoMig, useTimerTable.readyAddress(),
 469                   getCacheEntry(useTimerTable.readyAddress()), tbe);
 470         } else {
 471           trigger(Event:Use_TimeoutNoStarvers, useTimerTable.readyAddress(),
 472                   getCacheEntry(useTimerTable.readyAddress()), tbe);
 473         }
 474       }
 475     }
 476   }
 477
 478   // Reissue Timer
 479   in_port(reissueTimerTable_in, Address, reissueTimerTable, rank=4) {
 480     if (reissueTimerTable_in.isReady()) {
 481       trigger(Event:Request_Timeout, reissueTimerTable.readyAddress(),
 482               getCacheEntry(reissueTimerTable.readyAddress()),
 483               L1_TBEs[reissueTimerTable.readyAddress()]);
 484     }
 485   }
 486
 487   // Persistent Network
 488   in_port(persistentNetwork_in, PersistentMsg, persistentToL1Cache, rank=3) {
 489     if (persistentNetwork_in.isReady()) {
 490       peek(persistentNetwork_in, PersistentMsg, block_on="Addr") {
 491         assert(in_msg.Destination.isElement(machineID));
 492
 493         // Apply the lockdown or unlockdown message to the table
 494         if (in_msg.Type == PersistentRequestType:GETX_PERSISTENT) {
 495           persistentTable.persistentRequestLock(in_msg.Addr, in_msg.Requestor, AccessType:Write);
 496         } else if (in_msg.Type == PersistentRequestType:GETS_PERSISTENT) {
 497           persistentTable.persistentRequestLock(in_msg.Addr, in_msg.Requestor, AccessType:Read);
 498         } else if (in_msg.Type == PersistentRequestType:DEACTIVATE_PERSISTENT) {
 499           persistentTable.persistentRequestUnlock(in_msg.Addr, in_msg.Requestor);
 500         } else {
 501           error("Unexpected message");
 502         }
 503
 504         // React to the message based on the current state of the table
 505         Entry cache_entry := getCacheEntry(in_msg.Addr);
 506         TBE tbe := L1_TBEs[in_msg.Addr];
 507
 508         if (persistentTable.isLocked(in_msg.Addr)) {
 509           if (persistentTable.findSmallest(in_msg.Addr) == machineID) {
 510             // Our Own Lock - this processor is highest priority
 511             trigger(Event:Own_Lock_or_Unlock, in_msg.Addr,
 512                     cache_entry, tbe);
 513           } else {
 514             if (persistentTable.typeOfSmallest(in_msg.Addr) == AccessType:Read) {
 515               if (getTokens(cache_entry) == 1 ||
 516                   getTokens(cache_entry) == (max_tokens() / 2) + 1) {
 517                 trigger(Event:Persistent_GETS_Last_Token, in_msg.Addr,
 518                         cache_entry, tbe);
 519               } else {
 520                 trigger(Event:Persistent_GETS, in_msg.Addr,
 521                         cache_entry, tbe);
 522               }
 523             } else {
 524               trigger(Event:Persistent_GETX, in_msg.Addr,
 525                       cache_entry, tbe);
 526             }
 527           }
 528         } else {
 529           // Unlock case - no entries in the table
 530           trigger(Event:Own_Lock_or_Unlock, in_msg.Addr,
 531                   cache_entry, tbe);
 532         }
 533       }
 534     }
 535   }
 536
 537   // Response Network
 538   in_port(responseNetwork_in, ResponseMsg, responseToL1Cache, rank=2) {
 539     if (responseNetwork_in.isReady()) {
 540       peek(responseNetwork_in, ResponseMsg, block_on="Addr") {
 541         assert(in_msg.Destination.isElement(machineID));
 542
 543         Entry cache_entry := getCacheEntry(in_msg.Addr);
 544         TBE tbe := L1_TBEs[in_msg.Addr];
 545
 546         // Mark TBE flag if response received off-chip.  Use this to update average latency estimate
 547         if ( machineIDToMachineType(in_msg.Sender) == MachineType:L2Cache ) {
 548
 549           if (in_msg.Sender == mapAddressToRange(in_msg.Addr,
 550                                  MachineType:L2Cache, l2_select_low_bit,
 551                                  l2_select_num_bits, intToID(0))) {
 552
 553             // came from an off-chip L2 cache
 554             if (is_valid(tbe)) {
 555                // L1_TBEs[in_msg.Addr].ExternalResponse := true;
 556                // profile_offchipL2_response(in_msg.Addr);
 557             }
 558           }
 559           else {
 560                // profile_onchipL2_response(in_msg.Addr );
 561           }
 562         } else if ( machineIDToMachineType(in_msg.Sender) == MachineType:Directory ) {
 563           if (is_valid(tbe)) {
 564             setExternalResponse(tbe);
 565             // profile_memory_response( in_msg.Addr);
 566           }
 567         } else if ( machineIDToMachineType(in_msg.Sender) == MachineType:L1Cache) {
 568           //if (isLocalProcessor(machineID, in_msg.Sender) == false) {
 569             //if (is_valid(tbe)) {
 570                // tbe.ExternalResponse := true;
 571                // profile_offchipL1_response(in_msg.Addr );
 572             //}
 573           //}
 574           //else {
 575                // profile_onchipL1_response(in_msg.Addr );
 576           //}
 577         } else {
 578           error("unexpected SenderMachine");
 579         }
 580
 581
 582         if (getTokens(cache_entry) + in_msg.Tokens != max_tokens()) {
 583           if (in_msg.Type == CoherenceResponseType:ACK) {
 584             assert(in_msg.Tokens < (max_tokens() / 2));
 585             trigger(Event:Ack, in_msg.Addr, cache_entry, tbe);
 586           } else if (in_msg.Type == CoherenceResponseType:DATA_OWNER) {
 587             trigger(Event:Data_Owner, in_msg.Addr, cache_entry, tbe);
 588           } else if (in_msg.Type == CoherenceResponseType:DATA_SHARED) {
 589             assert(in_msg.Tokens < (max_tokens() / 2));
 590             trigger(Event:Data_Shared, in_msg.Addr, cache_entry, tbe);
 591           } else {
 592             error("Unexpected message");
 593           }
 594         } else {
 595           if (in_msg.Type == CoherenceResponseType:ACK) {
 596             assert(in_msg.Tokens < (max_tokens() / 2));
 597             trigger(Event:Ack_All_Tokens, in_msg.Addr, cache_entry, tbe);
 598           } else if (in_msg.Type == CoherenceResponseType:DATA_OWNER || in_msg.Type == CoherenceResponseType:DATA_SHARED) {
 599             trigger(Event:Data_All_Tokens, in_msg.Addr, cache_entry, tbe);
 600           } else {
 601             error("Unexpected message");
 602           }
 603         }
 604       }
 605     }
 606   }
 607
 608   // Request Network
 609   in_port(requestNetwork_in, RequestMsg, requestToL1Cache) {
 610     if (requestNetwork_in.isReady()) {
 611       peek(requestNetwork_in, RequestMsg, block_on="Addr") {
 612         assert(in_msg.Destination.isElement(machineID));
 613
 614         Entry cache_entry := getCacheEntry(in_msg.Addr);
 615         TBE tbe := L1_TBEs[in_msg.Addr];
 616
 617         if (in_msg.Type == CoherenceRequestType:GETX) {
 618           if (in_msg.isLocal) {
 619             trigger(Event:Transient_Local_GETX, in_msg.Addr,
 620                     cache_entry, tbe);
 621           }
 622           else {
 623             trigger(Event:Transient_GETX, in_msg.Addr,
 624                     cache_entry, tbe);
 625           }
 626         } else if (in_msg.Type == CoherenceRequestType:GETS) {
 627           if (getTokens(cache_entry) == 1 ||
 628               getTokens(cache_entry) == (max_tokens() / 2) + 1) {
 629             if (in_msg.isLocal) {
 630               trigger(Event:Transient_Local_GETS_Last_Token, in_msg.Addr,
 631                       cache_entry, tbe);
 632             }
 633             else {
 634               trigger(Event:Transient_GETS_Last_Token, in_msg.Addr,
 635                       cache_entry, tbe);
 636             }
 637           }
 638           else {
 639             if (in_msg.isLocal) {
 640               trigger(Event:Transient_Local_GETS, in_msg.Addr,
 641                       cache_entry, tbe);
 642             }
 643             else {
 644               trigger(Event:Transient_GETS, in_msg.Addr,
 645                       cache_entry, tbe);
 646             }
 647           }
 648         } else {
 649           error("Unexpected message");
 650         }
 651       }
 652     }
 653   }
 654
 655   // Mandatory Queue
 656   in_port(mandatoryQueue_in, RubyRequest, mandatoryQueue, desc="...", rank=0) {
 657     if (mandatoryQueue_in.isReady()) {
 658       peek(mandatoryQueue_in, RubyRequest, block_on="LineAddress") {
 659         // Check for data access to blocks in I-cache and ifetchs to blocks in D-cache
 660
 661         TBE tbe := L1_TBEs[in_msg.LineAddress];
 662
 663         if (in_msg.Type == RubyRequestType:IFETCH) {
 664           // ** INSTRUCTION ACCESS ***
 665
 666           Entry L1Icache_entry := getL1ICacheEntry(in_msg.LineAddress);
 667           if (is_valid(L1Icache_entry)) {
 668             // The tag matches for the L1, so the L1 fetches the line.
 669             // We know it can't be in the L2 due to exclusion.
 670             trigger(mandatory_request_type_to_event(in_msg.Type),
 671                     in_msg.LineAddress, L1Icache_entry, tbe);
 672           } else {
 673
 674             // Check to see if it is in the OTHER L1
 675             Entry L1Dcache_entry := getL1DCacheEntry(in_msg.LineAddress);
 676             if (is_valid(L1Dcache_entry)) {
 677               // The block is in the wrong L1, try to write it to the L2
 678                 trigger(Event:L1_Replacement, in_msg.LineAddress,
 679                         L1Dcache_entry, tbe);
 680             }
 681
 682             if (L1Icache.cacheAvail(in_msg.LineAddress)) {
 683               // L1 does't have the line, but we have space for it in the L1
 684               trigger(mandatory_request_type_to_event(in_msg.Type),
 685                       in_msg.LineAddress, L1Icache_entry, tbe);
 686             } else {
 687               // No room in the L1, so we need to make room
 688               trigger(Event:L1_Replacement,
 689                       L1Icache.cacheProbe(in_msg.LineAddress),
 690                       getL1ICacheEntry(L1Icache.cacheProbe(in_msg.LineAddress)),
 691                       L1_TBEs[L1Icache.cacheProbe(in_msg.LineAddress)]);
 692             }
 693           }
 694         } else {
 695           // *** DATA ACCESS ***
 696
 697           Entry L1Dcache_entry := getL1DCacheEntry(in_msg.LineAddress);
 698           if (is_valid(L1Dcache_entry)) {
 699             // The tag matches for the L1, so the L1 fetches the line.
 700             // We know it can't be in the L2 due to exclusion.
 701             trigger(mandatory_request_type_to_event(in_msg.Type),
 702                     in_msg.LineAddress, L1Dcache_entry, tbe);
 703           } else {
 704
 705             // Check to see if it is in the OTHER L1
 706             Entry L1Icache_entry := getL1ICacheEntry(in_msg.LineAddress);
 707             if (is_valid(L1Icache_entry)) {
 708               // The block is in the wrong L1, try to write it to the L2
 709               trigger(Event:L1_Replacement, in_msg.LineAddress,
 710                       L1Icache_entry, tbe);
 711             }
 712
 713             if (L1Dcache.cacheAvail(in_msg.LineAddress)) {
 714               // L1 does't have the line, but we have space for it in the L1
 715               trigger(mandatory_request_type_to_event(in_msg.Type),
 716                       in_msg.LineAddress, L1Dcache_entry, tbe);
 717             } else {
 718               // No room in the L1, so we need to make room
 719               trigger(Event:L1_Replacement,
 720                       L1Dcache.cacheProbe(in_msg.LineAddress),
 721                       getL1DCacheEntry(L1Dcache.cacheProbe(in_msg.LineAddress)),
 722                       L1_TBEs[L1Dcache.cacheProbe(in_msg.LineAddress)]);
 723             }
 724           }
 725         }
 726       }
 727     }
 728   }
 729
 730   // ACTIONS
 731
 732   action(a_issueReadRequest, "a", desc="Issue GETS") {
 733       assert(is_valid(tbe));
 734       if (tbe.IssueCount == 0) {
 735         // Update outstanding requests
 736         //profile_outstanding_request(outstandingRequests);
 737         outstandingRequests := outstandingRequests + 1;
 738       }
 739
 740       if (tbe.IssueCount >= retry_threshold) {
 741         // Issue a persistent request if possible
 742         if (okToIssueStarving(address, machineID) && (starving == false)) {
 743           enqueue(persistentNetwork_out, PersistentMsg, l1_request_latency) {
 744             out_msg.Addr := address;
 745             out_msg.Type := PersistentRequestType:GETS_PERSISTENT;
 746             out_msg.Requestor := machineID;
 747             out_msg.Destination.broadcast(MachineType:L1Cache);
 748
 749             //
 750             // Currently the configuration system limits the system to only one
 751             // chip.  Therefore, if we assume one shared L2 cache, then only one
 752             // pertinent L2 cache exist.
 753             //
 754             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 755
 756             out_msg.Destination.add(mapAddressToRange(address,
 757                                       MachineType:L2Cache, l2_select_low_bit,
 758                                       l2_select_num_bits, intToID(0)));
 759
 760             out_msg.Destination.add(map_Address_to_Directory(address));
 761             out_msg.MessageSize := MessageSizeType:Persistent_Control;
 762             out_msg.Prefetch := tbe.Prefetch;
 763             out_msg.AccessMode := tbe.AccessMode;
 764           }
 765           markPersistentEntries(address);
 766           starving := true;
 767
 768           if (tbe.IssueCount == 0) {
 769             //profile_persistent_prediction(address, tbe.TypeOfAccess);
 770           }
 771
 772           // Update outstanding requests
 773           //profile_outstanding_persistent_request(outstandingPersistentRequests);
 774           outstandingPersistentRequests := outstandingPersistentRequests + 1;
 775
 776           // Increment IssueCount
 777           tbe.IssueCount := tbe.IssueCount + 1;
 778
 779           tbe.WentPersistent := true;
 780
 781           // Do not schedule a wakeup, a persistent requests will always complete
 782         }
 783         else {
 784
 785           // We'd like to issue a persistent request, but are not allowed
 786           // to issue a P.R. right now.  This, we do not increment the
 787           // IssueCount.
 788
 789           // Set a wakeup timer
 790           reissueTimerTable.set(address, reissue_wakeup_latency);
 791
 792         }
 793       } else {
 794         // Make a normal request
 795         enqueue(requestNetwork_out, RequestMsg, l1_request_latency) {
 796           out_msg.Addr := address;
 797           out_msg.Type := CoherenceRequestType:GETS;
 798           out_msg.Requestor := machineID;
 799           out_msg.Destination.add(mapAddressToRange(address,
 800                                     MachineType:L2Cache, l2_select_low_bit,
 801                                     l2_select_num_bits, intToID(0)));
 802
 803           out_msg.RetryNum := tbe.IssueCount;
 804           if (tbe.IssueCount == 0) {
 805             out_msg.MessageSize := MessageSizeType:Request_Control;
 806           } else {
 807             out_msg.MessageSize := MessageSizeType:Reissue_Control;
 808           }
 809           out_msg.Prefetch := tbe.Prefetch;
 810           out_msg.AccessMode := tbe.AccessMode;
 811         }
 812
 813         // send to other local L1s, with local bit set
 814         enqueue(requestNetwork_out, RequestMsg, l1_request_latency) {
 815           out_msg.Addr := address;
 816           out_msg.Type := CoherenceRequestType:GETS;
 817           out_msg.Requestor := machineID;
 818           //
 819           // Since only one chip, assuming all L1 caches are local
 820           //
 821           //out_msg.Destination := getOtherLocalL1IDs(machineID);
 822           out_msg.Destination.broadcast(MachineType:L1Cache);
 823           out_msg.Destination.remove(machineID);
 824
 825           out_msg.RetryNum := tbe.IssueCount;
 826           out_msg.isLocal := true;
 827           if (tbe.IssueCount == 0) {
 828             out_msg.MessageSize := MessageSizeType:Broadcast_Control;
 829           } else {
 830             out_msg.MessageSize := MessageSizeType:Broadcast_Control;
 831           }
 832           out_msg.Prefetch := tbe.Prefetch;
 833           out_msg.AccessMode := tbe.AccessMode;
 834         }
 835
 836         // Increment IssueCount
 837         tbe.IssueCount := tbe.IssueCount + 1;
 838
 839         // Set a wakeup timer
 840
 841         if (dynamic_timeout_enabled) {
 842           reissueTimerTable.set(address, (5 * averageLatencyEstimate()) / 4);
 843         } else {
 844           reissueTimerTable.set(address, fixed_timeout_latency);
 845         }
 846
 847       }
 848   }
 849
 850   action(b_issueWriteRequest, "b", desc="Issue GETX") {
 851
 852       assert(is_valid(tbe));
 853       if (tbe.IssueCount == 0) {
 854         // Update outstanding requests
 855         //profile_outstanding_request(outstandingRequests);
 856         outstandingRequests := outstandingRequests + 1;
 857       }
 858
 859       if (tbe.IssueCount >= retry_threshold) {
 860         // Issue a persistent request if possible
 861         if ( okToIssueStarving(address, machineID) && (starving == false)) {
 862           enqueue(persistentNetwork_out, PersistentMsg, l1_request_latency) {
 863             out_msg.Addr := address;
 864             out_msg.Type := PersistentRequestType:GETX_PERSISTENT;
 865             out_msg.Requestor := machineID;
 866             out_msg.Destination.broadcast(MachineType:L1Cache);
 867
 868             //
 869             // Currently the configuration system limits the system to only one
 870             // chip.  Therefore, if we assume one shared L2 cache, then only one
 871             // pertinent L2 cache exist.
 872             //
 873             //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
 874
 875             out_msg.Destination.add(mapAddressToRange(address,
 876                                       MachineType:L2Cache, l2_select_low_bit,
 877                                       l2_select_num_bits, intToID(0)));
 878
 879             out_msg.Destination.add(map_Address_to_Directory(address));
 880             out_msg.MessageSize := MessageSizeType:Persistent_Control;
 881             out_msg.Prefetch := tbe.Prefetch;
 882             out_msg.AccessMode := tbe.AccessMode;
 883           }
 884           markPersistentEntries(address);
 885           starving := true;
 886
 887           // Update outstanding requests
 888           //profile_outstanding_persistent_request(outstandingPersistentRequests);
 889           outstandingPersistentRequests := outstandingPersistentRequests + 1;
 890
 891           if (tbe.IssueCount == 0) {
 892             //profile_persistent_prediction(address, tbe.TypeOfAccess);
 893           }
 894
 895           // Increment IssueCount
 896           tbe.IssueCount := tbe.IssueCount + 1;
 897
 898           tbe.WentPersistent := true;
 899
 900           // Do not schedule a wakeup, a persistent requests will always complete
 901         }
 902         else {
 903
 904           // We'd like to issue a persistent request, but are not allowed
 905           // to issue a P.R. right now.  This, we do not increment the
 906           // IssueCount.
 907
 908           // Set a wakeup timer
 909           reissueTimerTable.set(address, reissue_wakeup_latency);
 910         }
 911
 912       } else  {
 913         // Make a normal request
 914         enqueue(requestNetwork_out, RequestMsg, l1_request_latency) {
 915           out_msg.Addr := address;
 916           out_msg.Type := CoherenceRequestType:GETX;
 917           out_msg.Requestor := machineID;
 918
 919           out_msg.Destination.add(mapAddressToRange(address,
 920                                     MachineType:L2Cache, l2_select_low_bit,
 921                                     l2_select_num_bits, intToID(0)));
 922
 923           out_msg.RetryNum := tbe.IssueCount;
 924
 925           if (tbe.IssueCount == 0) {
 926             out_msg.MessageSize := MessageSizeType:Request_Control;
 927           } else {
 928             out_msg.MessageSize := MessageSizeType:Reissue_Control;
 929           }
 930           out_msg.Prefetch := tbe.Prefetch;
 931           out_msg.AccessMode := tbe.AccessMode;
 932         }
 933
 934         // send to other local L1s too
 935         enqueue(requestNetwork_out, RequestMsg, l1_request_latency) {
 936           out_msg.Addr := address;
 937           out_msg.Type := CoherenceRequestType:GETX;
 938           out_msg.Requestor := machineID;
 939           out_msg.isLocal := true;
 940
 941           //
 942           // Since only one chip, assuming all L1 caches are local
 943           //
 944           //out_msg.Destination := getOtherLocalL1IDs(machineID);
 945           out_msg.Destination.broadcast(MachineType:L1Cache);
 946           out_msg.Destination.remove(machineID);
 947
 948           out_msg.RetryNum := tbe.IssueCount;
 949           if (tbe.IssueCount == 0) {
 950             out_msg.MessageSize := MessageSizeType:Broadcast_Control;
 951           } else {
 952             out_msg.MessageSize := MessageSizeType:Broadcast_Control;
 953           }
 954           out_msg.Prefetch := tbe.Prefetch;
 955           out_msg.AccessMode := tbe.AccessMode;
 956         }
 957
 958         // Increment IssueCount
 959         tbe.IssueCount := tbe.IssueCount + 1;
 960
 961         DPRINTF(RubySlicc, "incremented issue count to %d\n",
 962                 tbe.IssueCount);
 963
 964         // Set a wakeup timer
 965         if (dynamic_timeout_enabled) {
 966           reissueTimerTable.set(address, (5 * averageLatencyEstimate()) / 4);
 967         } else {
 968           reissueTimerTable.set(address, fixed_timeout_latency);
 969         }
 970       }
 971   }
 972
 973   action(bb_bounceResponse, "\b", desc="Bounce tokens and data to memory") {
 974     peek(responseNetwork_in, ResponseMsg) {
 975       // FIXME, should use a 3rd vnet
 976       enqueue(responseNetwork_out, ResponseMsg, 1) {
 977         out_msg.Addr := address;
 978         out_msg.Type := in_msg.Type;
 979         out_msg.Sender := machineID;
 980         out_msg.Destination.add(map_Address_to_Directory(address));
 981         out_msg.Tokens := in_msg.Tokens;
 982         out_msg.MessageSize := in_msg.MessageSize;
 983         out_msg.DataBlk := in_msg.DataBlk;
 984         out_msg.Dirty := in_msg.Dirty;
 985       }
 986     }
 987   }
 988
 989   action(c_ownedReplacement, "c", desc="Issue writeback") {
 990     assert(is_valid(cache_entry));
 991     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
 992       out_msg.Addr := address;
 993       out_msg.Sender := machineID;
 994
 995       out_msg.Destination.add(mapAddressToRange(address,
 996                                 MachineType:L2Cache, l2_select_low_bit,
 997                                 l2_select_num_bits, intToID(0)));
 998
 999       out_msg.Tokens := cache_entry.Tokens;
1000       out_msg.DataBlk := cache_entry.DataBlk;
1001       out_msg.Dirty := cache_entry.Dirty;
1002       out_msg.Type := CoherenceResponseType:WB_OWNED;
1003
1004       // always send the data?
1005       out_msg.MessageSize := MessageSizeType:Writeback_Data;
1006     }
1007     cache_entry.Tokens := 0;
1008   }
1009
1010   action(cc_sharedReplacement, "\c", desc="Issue shared writeback") {
1011
1012     // don't send writeback if replacing block with no tokens
1013     assert(is_valid(cache_entry));
1014     assert (cache_entry.Tokens > 0);
1015     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1016         out_msg.Addr := address;
1017         out_msg.Sender := machineID;
1018
1019         out_msg.Destination.add(mapAddressToRange(address,
1020                                   MachineType:L2Cache, l2_select_low_bit,
1021                                   l2_select_num_bits, intToID(0)));
1022
1023         out_msg.Tokens := cache_entry.Tokens;
1024         out_msg.DataBlk := cache_entry.DataBlk;
1025         // assert(cache_entry.Dirty == false);
1026         out_msg.Dirty := false;
1027
1028         out_msg.MessageSize := MessageSizeType:Writeback_Data;
1029         out_msg.Type := CoherenceResponseType:WB_SHARED_DATA;
1030     }
1031     cache_entry.Tokens := 0;
1032   }
1033
1034   action(tr_tokenReplacement, "tr", desc="Issue token writeback") {
1035     assert(is_valid(cache_entry));
1036     if (cache_entry.Tokens > 0) {
1037       enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1038         out_msg.Addr := address;
1039         out_msg.Sender := machineID;
1040
1041         out_msg.Destination.add(mapAddressToRange(address,
1042                                   MachineType:L2Cache, l2_select_low_bit,
1043                                   l2_select_num_bits, intToID(0)));
1044
1045         out_msg.Tokens := cache_entry.Tokens;
1046         out_msg.DataBlk := cache_entry.DataBlk;
1047         // assert(cache_entry.Dirty == false);
1048         out_msg.Dirty := false;
1049
1050         // always send the data?
1051         out_msg.MessageSize := MessageSizeType:Writeback_Control;
1052         out_msg.Type := CoherenceResponseType:WB_TOKENS;
1053       }
1054     }
1055     cache_entry.Tokens := 0;
1056   }
1057
1058
1059   action(d_sendDataWithToken, "d", desc="Send data and a token from cache to requestor") {
1060     assert(is_valid(cache_entry));
1061     peek(requestNetwork_in, RequestMsg) {
1062       enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1063         out_msg.Addr := address;
1064         out_msg.Type := CoherenceResponseType:DATA_SHARED;
1065         out_msg.Sender := machineID;
1066         out_msg.Destination.add(in_msg.Requestor);
1067         out_msg.Tokens := 1;
1068         out_msg.DataBlk := cache_entry.DataBlk;
1069         // out_msg.Dirty := cache_entry.Dirty;
1070         out_msg.Dirty := false;
1071         if (in_msg.isLocal) {
1072           out_msg.MessageSize := MessageSizeType:ResponseLocal_Data;
1073         } else {
1074           out_msg.MessageSize := MessageSizeType:Response_Data;
1075         }
1076       }
1077     }
1078     cache_entry.Tokens := cache_entry.Tokens - 1;
1079     assert(cache_entry.Tokens >= 1);
1080   }
1081
1082   action(d_sendDataWithNTokenIfAvail, "\dd", desc="Send data and a token from cache to requestor") {
1083     assert(is_valid(cache_entry));
1084     peek(requestNetwork_in, RequestMsg) {
1085       if (cache_entry.Tokens > (N_tokens + (max_tokens() / 2))) {
1086         enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1087           out_msg.Addr := address;
1088           out_msg.Type := CoherenceResponseType:DATA_SHARED;
1089           out_msg.Sender := machineID;
1090           out_msg.Destination.add(in_msg.Requestor);
1091           out_msg.Tokens := N_tokens;
1092           out_msg.DataBlk := cache_entry.DataBlk;
1093           // out_msg.Dirty := cache_entry.Dirty;
1094           out_msg.Dirty := false;
1095           if (in_msg.isLocal) {
1096             out_msg.MessageSize := MessageSizeType:ResponseLocal_Data;
1097           } else {
1098             out_msg.MessageSize := MessageSizeType:Response_Data;
1099           }
1100         }
1101         cache_entry.Tokens := cache_entry.Tokens - N_tokens;
1102       }
1103       else if (cache_entry.Tokens > 1) {
1104         enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1105           out_msg.Addr := address;
1106           out_msg.Type := CoherenceResponseType:DATA_SHARED;
1107           out_msg.Sender := machineID;
1108           out_msg.Destination.add(in_msg.Requestor);
1109           out_msg.Tokens := 1;
1110           out_msg.DataBlk := cache_entry.DataBlk;
1111           // out_msg.Dirty := cache_entry.Dirty;
1112           out_msg.Dirty := false;
1113           if (in_msg.isLocal) {
1114             out_msg.MessageSize := MessageSizeType:ResponseLocal_Data;
1115           } else {
1116             out_msg.MessageSize := MessageSizeType:Response_Data;
1117           }
1118         }
1119         cache_entry.Tokens := cache_entry.Tokens - 1;
1120       }
1121     }
1122 //    assert(cache_entry.Tokens >= 1);
1123   }
1124
1125   action(dd_sendDataWithAllTokens, "\d", desc="Send data and all tokens from cache to requestor") {
1126     peek(requestNetwork_in, RequestMsg) {
1127     assert(is_valid(cache_entry));
1128       enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1129         out_msg.Addr := address;
1130         out_msg.Type := CoherenceResponseType:DATA_OWNER;
1131         out_msg.Sender := machineID;
1132         out_msg.Destination.add(in_msg.Requestor);
1133         assert(cache_entry.Tokens > (max_tokens() / 2));
1134         out_msg.Tokens := cache_entry.Tokens;
1135         out_msg.DataBlk := cache_entry.DataBlk;
1136         out_msg.Dirty := cache_entry.Dirty;
1137         if (in_msg.isLocal) {
1138           out_msg.MessageSize := MessageSizeType:ResponseLocal_Data;
1139         } else {
1140           out_msg.MessageSize := MessageSizeType:Response_Data;
1141         }
1142       }
1143     }
1144     cache_entry.Tokens := 0;
1145   }
1146
1147   action(e_sendAckWithCollectedTokens, "e", desc="Send ack with the tokens we've collected thus far.") {
1148     // assert(persistentTable.findSmallest(address) != id); // Make sure we never bounce tokens to ourself
1149     assert(is_valid(cache_entry));
1150     if (cache_entry.Tokens > 0) {
1151       enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1152         out_msg.Addr := address;
1153         if (cache_entry.Tokens > (max_tokens() / 2)) {
1154           out_msg.Type := CoherenceResponseType:DATA_OWNER;
1155         } else {
1156           out_msg.Type := CoherenceResponseType:ACK;
1157         }
1158         out_msg.Sender := machineID;
1159         out_msg.Destination.add(persistentTable.findSmallest(address));
1160         assert(cache_entry.Tokens >= 1);
1161         out_msg.Tokens := cache_entry.Tokens;
1162         out_msg.DataBlk := cache_entry.DataBlk;
1163         out_msg.MessageSize := MessageSizeType:Response_Control;
1164       }
1165     }
1166     cache_entry.Tokens := 0;
1167   }
1168
1169   action(ee_sendDataWithAllTokens, "\e", desc="Send data and all tokens from cache to starver") {
1170     //assert(persistentTable.findSmallest(address) != id); // Make sure we never bounce tokens to ourself
1171     assert(is_valid(cache_entry));
1172     assert(cache_entry.Tokens > 0);
1173     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1174       out_msg.Addr := address;
1175       out_msg.Type := CoherenceResponseType:DATA_OWNER;
1176       out_msg.Sender := machineID;
1177       out_msg.Destination.add(persistentTable.findSmallest(address));
1178       assert(cache_entry.Tokens > (max_tokens() / 2));
1179       out_msg.Tokens := cache_entry.Tokens;
1180       out_msg.DataBlk := cache_entry.DataBlk;
1181       out_msg.Dirty := cache_entry.Dirty;
1182       out_msg.MessageSize := MessageSizeType:Response_Data;
1183     }
1184     cache_entry.Tokens := 0;
1185   }
1186
1187   action(f_sendAckWithAllButNorOneTokens, "f", desc="Send ack with all our tokens but one to starver.") {
1188     //assert(persistentTable.findSmallest(address) != id); // Make sure we never bounce tokens to ourself
1189     assert(is_valid(cache_entry));
1190     assert(cache_entry.Tokens > 0);
1191     if (cache_entry.Tokens > 1) {
1192       enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1193         out_msg.Addr := address;
1194         if (cache_entry.Tokens > (max_tokens() / 2)) {
1195           out_msg.Type := CoherenceResponseType:DATA_OWNER;
1196         } else {
1197           out_msg.Type := CoherenceResponseType:ACK;
1198         }
1199         out_msg.Sender := machineID;
1200         out_msg.Destination.add(persistentTable.findSmallest(address));
1201         assert(cache_entry.Tokens >= 1);
1202         if (cache_entry.Tokens > N_tokens) {
1203           out_msg.Tokens := cache_entry.Tokens - N_tokens;
1204         } else {
1205           out_msg.Tokens := cache_entry.Tokens - 1;
1206         }
1207         out_msg.DataBlk := cache_entry.DataBlk;
1208         out_msg.MessageSize := MessageSizeType:Response_Control;
1209       }
1210     }
1211     if (cache_entry.Tokens > N_tokens) {
1212       cache_entry.Tokens := N_tokens;
1213     } else {
1214       cache_entry.Tokens := 1;
1215     }
1216   }
1217
1218   action(ff_sendDataWithAllButNorOneTokens, "\f", desc="Send data and out tokens but one to starver") {
1219     //assert(persistentTable.findSmallest(address) != id); // Make sure we never bounce tokens to ourself
1220     assert(is_valid(cache_entry));
1221     assert(cache_entry.Tokens > ((max_tokens() / 2) + 1));
1222     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1223         out_msg.Addr := address;
1224         out_msg.Type := CoherenceResponseType:DATA_OWNER;
1225         out_msg.Sender := machineID;
1226         out_msg.Destination.add(persistentTable.findSmallest(address));
1227         if (cache_entry.Tokens > (N_tokens + (max_tokens() / 2))) {
1228           out_msg.Tokens := cache_entry.Tokens - N_tokens;
1229         } else {
1230           out_msg.Tokens := cache_entry.Tokens - 1;
1231         }
1232         assert(out_msg.Tokens > (max_tokens() / 2));
1233         out_msg.DataBlk := cache_entry.DataBlk;
1234         out_msg.Dirty := cache_entry.Dirty;
1235         out_msg.MessageSize := MessageSizeType:Response_Data;
1236     }
1237     if (cache_entry.Tokens > (N_tokens + (max_tokens() / 2))) {
1238       cache_entry.Tokens := N_tokens;
1239     } else {
1240       cache_entry.Tokens := 1;
1241     }
1242   }
1243
1244   action(fo_sendDataWithOwnerToken, "fo", desc="Send data and owner tokens") {
1245     assert(is_valid(cache_entry));
1246     assert(cache_entry.Tokens == ((max_tokens() / 2) + 1));
1247     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1248         out_msg.Addr := address;
1249         out_msg.Type := CoherenceResponseType:DATA_OWNER;
1250         out_msg.Sender := machineID;
1251         out_msg.Destination.add(persistentTable.findSmallest(address));
1252         out_msg.Tokens := cache_entry.Tokens;
1253         assert(out_msg.Tokens > (max_tokens() / 2));
1254         out_msg.DataBlk := cache_entry.DataBlk;
1255         out_msg.Dirty := cache_entry.Dirty;
1256         out_msg.MessageSize := MessageSizeType:Response_Data;
1257     }
1258     cache_entry.Tokens := 0;
1259   }
1260
1261   action(g_bounceResponseToStarver, "g", desc="Redirect response to starving processor") {
1262     // assert(persistentTable.isLocked(address));
1263
1264     peek(responseNetwork_in, ResponseMsg) {
1265       // assert(persistentTable.findSmallest(address) != id); // Make sure we never bounce tokens to ourself
1266       // FIXME, should use a 3rd vnet in some cases
1267       enqueue(responseNetwork_out, ResponseMsg, 1) {
1268         out_msg.Addr := address;
1269         out_msg.Type := in_msg.Type;
1270         out_msg.Sender := machineID;
1271         out_msg.Destination.add(persistentTable.findSmallest(address));
1272         out_msg.Tokens := in_msg.Tokens;
1273         out_msg.DataBlk := in_msg.DataBlk;
1274         out_msg.Dirty := in_msg.Dirty;
1275         out_msg.MessageSize := in_msg.MessageSize;
1276       }
1277     }
1278   }
1279
1280
1281   action(h_load_hit, "h", desc="Notify sequencer the load completed.") {
1282     assert(is_valid(cache_entry));
1283     DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
1284             address, cache_entry.DataBlk);
1285
1286     sequencer.readCallback(address, cache_entry.DataBlk, false,
1287                            MachineType:L1Cache);
1288   }
1289
1290   action(x_external_load_hit, "x", desc="Notify sequencer the load completed.") {
1291     assert(is_valid(cache_entry));
1292     DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
1293             address, cache_entry.DataBlk);
1294     peek(responseNetwork_in, ResponseMsg) {
1295       sequencer.readCallback(address, cache_entry.DataBlk,
1296                              isExternalHit(address, in_msg.Sender),
1297                              machineIDToMachineType(in_msg.Sender));
1298     }
1299   }
1300
1301   action(hh_store_hit, "\h", desc="Notify sequencer that store completed.") {
1302     assert(is_valid(cache_entry));
1303     DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
1304             address, cache_entry.DataBlk);
1305
1306     sequencer.writeCallback(address, cache_entry.DataBlk, false,
1307                             MachineType:L1Cache);
1308     cache_entry.Dirty := true;
1309     DPRINTF(RubySlicc, "%s\n", cache_entry.DataBlk);
1310   }
1311
1312   action(xx_external_store_hit, "\x", desc="Notify sequencer that store completed.") {
1313     assert(is_valid(cache_entry));
1314     DPRINTF(RubySlicc, "Address: %s, Data Block: %s\n",
1315             address, cache_entry.DataBlk);
1316     peek(responseNetwork_in, ResponseMsg) {
1317       sequencer.writeCallback(address, cache_entry.DataBlk,
1318                               isExternalHit(address, in_msg.Sender),
1319                               machineIDToMachineType(in_msg.Sender));
1320     }
1321     cache_entry.Dirty := true;
1322     DPRINTF(RubySlicc, "%s\n", cache_entry.DataBlk);
1323   }
1324
1325   action(i_allocateTBE, "i", desc="Allocate TBE") {
1326     check_allocate(L1_TBEs);
1327     L1_TBEs.allocate(address);
1328     set_tbe(L1_TBEs[address]);
1329     tbe.IssueCount := 0;
1330     peek(mandatoryQueue_in, RubyRequest) {
1331       tbe.PC := in_msg.ProgramCounter;
1332       tbe.TypeOfAccess := cache_request_type_to_access_type(in_msg.Type);
1333       if (in_msg.Type == RubyRequestType:ATOMIC) {
1334         tbe.IsAtomic := true;
1335       }
1336       tbe.Prefetch := in_msg.Prefetch;
1337       tbe.AccessMode := in_msg.AccessMode;
1338     }
1339     tbe.IssueTime := curCycle();
1340   }
1341
1342   action(ta_traceStalledAddress, "ta", desc="Trace Stalled Address") {
1343     peek(mandatoryQueue_in, RubyRequest) {
1344       APPEND_TRANSITION_COMMENT(in_msg.LineAddress);
1345     }
1346   }
1347
1348   action(j_unsetReissueTimer, "j", desc="Unset reissue timer.") {
1349     if (reissueTimerTable.isSet(address)) {
1350       reissueTimerTable.unset(address);
1351     }
1352   }
1353
1354   action(jj_unsetUseTimer, "\j", desc="Unset use timer.") {
1355     useTimerTable.unset(address);
1356   }
1357
1358   action(k_popMandatoryQueue, "k", desc="Pop mandatory queue.") {
1359     mandatoryQueue_in.dequeue();
1360   }
1361
1362   action(l_popPersistentQueue, "l", desc="Pop persistent queue.") {
1363     persistentNetwork_in.dequeue();
1364   }
1365
1366   action(m_popRequestQueue, "m", desc="Pop request queue.") {
1367     requestNetwork_in.dequeue();
1368   }
1369
1370   action(n_popResponseQueue, "n", desc="Pop response queue") {
1371     responseNetwork_in.dequeue();
1372   }
1373
1374   action(o_scheduleUseTimeout, "o", desc="Schedule a use timeout.") {
1375     useTimerTable.set(address, use_timeout_latency);
1376   }
1377
1378   action(p_informL2AboutTokenLoss, "p", desc="Inform L2 about loss of all tokens") {
1379     enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1380        out_msg.Addr := address;
1381        out_msg.Type := CoherenceResponseType:INV;
1382        out_msg.Tokens := 0;
1383        out_msg.Sender := machineID;
1384
1385        out_msg.Destination.add(mapAddressToRange(address,
1386                                  MachineType:L2Cache, l2_select_low_bit,
1387                                  l2_select_num_bits, intToID(0)));
1388        out_msg.MessageSize := MessageSizeType:Response_Control;
1389     }
1390   }
1391
1392   action(q_updateTokensFromResponse, "q", desc="Update the token count based on the incoming response message") {
1393     peek(responseNetwork_in, ResponseMsg) {
1394       assert(is_valid(cache_entry));
1395       assert(in_msg.Tokens != 0);
1396       DPRINTF(RubySlicc, "L1 received tokens for address: %s, tokens: %d\n",
1397               in_msg.Addr, in_msg.Tokens);
1398       cache_entry.Tokens := cache_entry.Tokens + in_msg.Tokens;
1399       DPRINTF(RubySlicc, "%d\n", cache_entry.Tokens);
1400
1401       if (cache_entry.Dirty == false && in_msg.Dirty) {
1402         cache_entry.Dirty := true;
1403       }
1404     }
1405   }
1406
1407   action(s_deallocateTBE, "s", desc="Deallocate TBE") {
1408
1409     assert(is_valid(tbe));
1410     if (tbe.WentPersistent) {
1411       // assert(starving);
1412       outstandingRequests := outstandingRequests - 1;
1413       enqueue(persistentNetwork_out, PersistentMsg, l1_request_latency) {
1414         out_msg.Addr := address;
1415         out_msg.Type := PersistentRequestType:DEACTIVATE_PERSISTENT;
1416         out_msg.Requestor := machineID;
1417         out_msg.Destination.broadcast(MachineType:L1Cache);
1418
1419         //
1420         // Currently the configuration system limits the system to only one
1421         // chip.  Therefore, if we assume one shared L2 cache, then only one
1422         // pertinent L2 cache exist.
1423         //
1424         //out_msg.Destination.addNetDest(getAllPertinentL2Banks(address));
1425
1426         out_msg.Destination.add(mapAddressToRange(address,
1427                                   MachineType:L2Cache, l2_select_low_bit,
1428                                   l2_select_num_bits, intToID(0)));
1429
1430         out_msg.Destination.add(map_Address_to_Directory(address));
1431         out_msg.MessageSize := MessageSizeType:Persistent_Control;
1432       }
1433       starving := false;
1434     }
1435
1436     // Update average latency
1437     if (tbe.IssueCount <= 1) {
1438       if (tbe.ExternalResponse) {
1439         updateAverageLatencyEstimate(curCycle() - tbe.IssueTime);
1440       }
1441     }
1442
1443     // Profile
1444     //if (tbe.WentPersistent) {
1445     //  profile_token_retry(address, tbe.TypeOfAccess, 2);
1446     //}
1447     //else {
1448     //  profile_token_retry(address, tbe.TypeOfAccess, 1);
1449     //}
1450
1451     //profile_token_retry(address, tbe.TypeOfAccess, tbe.IssueCount);
1452     L1_TBEs.deallocate(address);
1453     unset_tbe();
1454   }
1455
1456   action(t_sendAckWithCollectedTokens, "t", desc="Send ack with the tokens we've collected thus far.") {
1457     assert(is_valid(cache_entry));
1458     if (cache_entry.Tokens > 0) {
1459       peek(requestNetwork_in, RequestMsg) {
1460         enqueue(responseNetwork_out, ResponseMsg, l1_response_latency) {
1461           out_msg.Addr := address;
1462           if (cache_entry.Tokens > (max_tokens() / 2)) {
1463             out_msg.Type := CoherenceResponseType:DATA_OWNER;
1464           } else {
1465             out_msg.Type := CoherenceResponseType:ACK;
1466           }
1467           out_msg.Sender := machineID;
1468           out_msg.Destination.add(in_msg.Requestor);
1469           assert(cache_entry.Tokens >= 1);
1470           out_msg.Tokens := cache_entry.Tokens;
1471           out_msg.DataBlk := cache_entry.DataBlk;
1472           out_msg.MessageSize := MessageSizeType:Response_Control;
1473         }
1474       }
1475     }
1476     cache_entry.Tokens := 0;
1477   }
1478
1479   action(u_writeDataToCache, "u", desc="Write data to cache") {
1480     peek(responseNetwork_in, ResponseMsg) {
1481       assert(is_valid(cache_entry));
1482       cache_entry.DataBlk := in_msg.DataBlk;
1483       if (cache_entry.Dirty == false && in_msg.Dirty) {
1484         cache_entry.Dirty := in_msg.Dirty;
1485       }
1486
1487     }
1488   }
1489
1490   action(gg_deallocateL1CacheBlock, "\g", desc="Deallocate cache block.  Sets the cache to invalid, allowing a replacement in parallel with a fetch.") {
1491     assert(getTokens(cache_entry) == 0);
1492     if (L1Dcache.isTagPresent(address)) {
1493       L1Dcache.deallocate(address);
1494     } else {
1495       L1Icache.deallocate(address);
1496     }
1497     unset_cache_entry();
1498   }
1499
1500   action(ii_allocateL1DCacheBlock, "\i", desc="Set L1 D-cache tag equal to tag of block B.") {
1501     if (is_valid(cache_entry)) {
1502     } else {
1503       set_cache_entry(L1Dcache.allocate(address, new Entry));
1504     }
1505   }
1506
1507   action(pp_allocateL1ICacheBlock, "\p", desc="Set L1 I-cache tag equal to tag of block B.") {
1508     if (is_valid(cache_entry)) {
1509     } else {
1510       set_cache_entry(L1Icache.allocate(address, new Entry));
1511     }
1512   }
1513
1514   action(forward_eviction_to_cpu, "\cc", desc="sends eviction information to the processor") {
1515     if (send_evictions) {
1516       DPRINTF(RubySlicc, "Sending invalidation for %s to the CPU\n", address);
1517       sequencer.evictionCallback(address);
1518     }
1519   }
1520
1521   action(uu_profileInstMiss, "\uim", desc="Profile the demand miss") {
1522       ++L1Icache.demand_misses;
1523   }
1524
1525   action(uu_profileInstHit, "\uih", desc="Profile the demand hit") {
1526       ++L1Icache.demand_hits;
1527   }
1528
1529   action(uu_profileDataMiss, "\udm", desc="Profile the demand miss") {
1530       ++L1Dcache.demand_misses;
1531   }
1532
1533   action(uu_profileDataHit, "\udh", desc="Profile the demand hit") {
1534       ++L1Dcache.demand_hits;
1535   }
1536
1537   action(w_assertIncomingDataAndCacheDataMatch, "w", desc="Assert that the incoming data and the data in the cache match") {
1538     peek(responseNetwork_in, ResponseMsg) {
1539       assert(is_valid(cache_entry));
1540       assert(cache_entry.DataBlk == in_msg.DataBlk);
1541     }
1542   }
1543
1544   action(zz_stallAndWaitMandatoryQueue, "\z", desc="Send the head of the mandatory queue to the back of the queue.") {
1545     peek(mandatoryQueue_in, RubyRequest) {
1546       APPEND_TRANSITION_COMMENT(in_msg.LineAddress);
1547     }
1548     stall_and_wait(mandatoryQueue_in, address);
1549   }
1550
1551   action(kd_wakeUpDependents, "kd", desc="wake-up dependents") {
1552     wakeUpBuffers(address);
1553   }
1554
1555   action(ka_wakeUpAllDependents, "ka", desc="wake-up all dependents") {
1556     wakeUpAllBuffers();
1557   }
1558
1559   //*****************************************************
1560   // TRANSITIONS
1561   //*****************************************************
1562
1563   // Transitions for Load/Store/L2_Replacement from transient states
1564   transition({IM, SM, OM, IS, IM_L, IS_L, I_L, S_L, SM_L, M_W, MM_W}, L1_Replacement) {
1565     ta_traceStalledAddress;
1566     zz_stallAndWaitMandatoryQueue;
1567   }
1568
1569   transition({IM, SM, OM, IS, IM_L, IS_L, SM_L}, {Store, Atomic}) {
1570     zz_stallAndWaitMandatoryQueue;
1571   }
1572
1573   transition({IM, IS, IM_L, IS_L}, {Load, Ifetch}) {
1574     zz_stallAndWaitMandatoryQueue;
1575   }
1576
1577   // Lockdowns
1578   transition({NP, I, S, O, M, MM, M_W, MM_W, IM, SM, OM, IS}, Own_Lock_or_Unlock) {
1579     l_popPersistentQueue;
1580   }
1581
1582   // Transitions from NP
1583   transition(NP, Load, IS) {
1584     ii_allocateL1DCacheBlock;
1585     i_allocateTBE;
1586     a_issueReadRequest;
1587     uu_profileDataMiss;
1588     k_popMandatoryQueue;
1589   }
1590
1591   transition(NP, Ifetch, IS) {
1592     pp_allocateL1ICacheBlock;
1593     i_allocateTBE;
1594     a_issueReadRequest;
1595     uu_profileInstMiss;
1596     k_popMandatoryQueue;
1597   }
1598
1599   transition(NP, {Store, Atomic}, IM) {
1600     ii_allocateL1DCacheBlock;
1601     i_allocateTBE;
1602     b_issueWriteRequest;
1603     uu_profileDataMiss;
1604     k_popMandatoryQueue;
1605   }
1606
1607   transition(NP, {Ack, Data_Shared, Data_Owner, Data_All_Tokens}) {
1608     bb_bounceResponse;
1609     n_popResponseQueue;
1610   }
1611
1612   transition(NP, {Transient_GETX, Transient_Local_GETX, Transient_GETS, Transient_Local_GETS}) {
1613     m_popRequestQueue;
1614   }
1615
1616   transition(NP, {Persistent_GETX, Persistent_GETS, Persistent_GETS_Last_Token}, I_L) {
1617     l_popPersistentQueue;
1618   }
1619
1620   // Transitions from Idle
1621   transition(I, Load, IS) {
1622     i_allocateTBE;
1623     a_issueReadRequest;
1624     uu_profileDataMiss;
1625     k_popMandatoryQueue;
1626   }
1627
1628   transition(I, Ifetch, IS) {
1629     i_allocateTBE;
1630     a_issueReadRequest;
1631     uu_profileInstMiss;
1632     k_popMandatoryQueue;
1633   }
1634
1635   transition(I, {Store, Atomic}, IM) {
1636     i_allocateTBE;
1637     b_issueWriteRequest;
1638     uu_profileDataMiss;
1639     k_popMandatoryQueue;
1640   }
1641
1642   transition(I, L1_Replacement) {
1643     ta_traceStalledAddress;
1644     tr_tokenReplacement;
1645     gg_deallocateL1CacheBlock;
1646     ka_wakeUpAllDependents;
1647   }
1648
1649   transition(I, {Transient_GETX, Transient_Local_GETX}) {
1650     t_sendAckWithCollectedTokens;
1651     m_popRequestQueue;
1652   }
1653
1654   transition(I, {Transient_GETS, Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token, Transient_Local_GETS}) {
1655     m_popRequestQueue;
1656   }
1657
1658   transition(I, {Persistent_GETX, Persistent_GETS, Persistent_GETS_Last_Token}, I_L) {
1659     e_sendAckWithCollectedTokens;
1660     l_popPersistentQueue;
1661   }
1662
1663   transition(I_L, {Persistent_GETX, Persistent_GETS, Persistent_GETS_Last_Token}) {
1664     l_popPersistentQueue;
1665   }
1666
1667   transition(I, Ack) {
1668     q_updateTokensFromResponse;
1669     n_popResponseQueue;
1670   }
1671
1672   transition(I, Data_Shared, S) {
1673     u_writeDataToCache;
1674     q_updateTokensFromResponse;
1675     n_popResponseQueue;
1676   }
1677
1678   transition(I, Data_Owner, O) {
1679     u_writeDataToCache;
1680     q_updateTokensFromResponse;
1681     n_popResponseQueue;
1682   }
1683
1684   transition(I, Data_All_Tokens, M) {
1685     u_writeDataToCache;
1686     q_updateTokensFromResponse;
1687     n_popResponseQueue;
1688   }
1689
1690   // Transitions from Shared
1691   transition({S, SM, S_L, SM_L}, Load) {
1692     h_load_hit;
1693     uu_profileDataHit;
1694     k_popMandatoryQueue;
1695   }
1696
1697   transition({S, SM, S_L, SM_L}, Ifetch) {
1698     h_load_hit;
1699     uu_profileInstHit;
1700     k_popMandatoryQueue;
1701   }
1702
1703   transition(S, {Store, Atomic}, SM) {
1704     i_allocateTBE;
1705     b_issueWriteRequest;
1706     uu_profileDataMiss;
1707     k_popMandatoryQueue;
1708   }
1709
1710   transition(S, L1_Replacement, I) {
1711     ta_traceStalledAddress;
1712     cc_sharedReplacement; // Only needed in some cases
1713     forward_eviction_to_cpu;
1714     gg_deallocateL1CacheBlock;
1715     ka_wakeUpAllDependents;
1716   }
1717
1718   transition(S, {Transient_GETX, Transient_Local_GETX}, I) {
1719     t_sendAckWithCollectedTokens;
1720     p_informL2AboutTokenLoss;
1721     forward_eviction_to_cpu
1722     m_popRequestQueue;
1723   }
1724
1725   // only owner responds to non-local requests
1726   transition(S, Transient_GETS) {
1727     m_popRequestQueue;
1728   }
1729
1730   transition(S, Transient_Local_GETS) {
1731     d_sendDataWithToken;
1732     m_popRequestQueue;
1733   }
1734
1735   transition(S, {Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token}) {
1736     m_popRequestQueue;
1737   }
1738
1739   transition({S, S_L}, Persistent_GETX, I_L) {
1740     e_sendAckWithCollectedTokens;
1741     p_informL2AboutTokenLoss;
1742     forward_eviction_to_cpu
1743     l_popPersistentQueue;
1744   }
1745
1746   transition(S, {Persistent_GETS, Persistent_GETS_Last_Token}, S_L) {
1747     f_sendAckWithAllButNorOneTokens;
1748     l_popPersistentQueue;
1749   }
1750
1751   transition(S_L, {Persistent_GETS, Persistent_GETS_Last_Token}) {
1752     l_popPersistentQueue;
1753   }
1754
1755   transition(S, Ack) {
1756     q_updateTokensFromResponse;
1757     n_popResponseQueue;
1758   }
1759
1760   transition(S, Data_Shared) {
1761     w_assertIncomingDataAndCacheDataMatch;
1762     q_updateTokensFromResponse;
1763     n_popResponseQueue;
1764   }
1765
1766   transition(S, Data_Owner, O) {
1767     w_assertIncomingDataAndCacheDataMatch;
1768     q_updateTokensFromResponse;
1769     n_popResponseQueue;
1770   }
1771
1772   transition(S, Data_All_Tokens, M) {
1773     w_assertIncomingDataAndCacheDataMatch;
1774     q_updateTokensFromResponse;
1775     n_popResponseQueue;
1776   }
1777
1778   // Transitions from Owned
1779   transition({O, OM}, Ifetch) {
1780     h_load_hit;
1781     uu_profileInstHit;
1782     k_popMandatoryQueue;
1783   }
1784
1785   transition({O, OM}, Load) {
1786     h_load_hit;
1787     uu_profileDataHit;
1788     k_popMandatoryQueue;
1789   }
1790
1791   transition(O, {Store, Atomic}, OM) {
1792     i_allocateTBE;
1793     b_issueWriteRequest;
1794     uu_profileDataMiss;
1795     k_popMandatoryQueue;
1796   }
1797
1798   transition(O, L1_Replacement, I) {
1799     ta_traceStalledAddress;
1800     c_ownedReplacement;
1801     forward_eviction_to_cpu
1802     gg_deallocateL1CacheBlock;
1803     ka_wakeUpAllDependents;
1804   }
1805
1806   transition(O, {Transient_GETX, Transient_Local_GETX}, I) {
1807     dd_sendDataWithAllTokens;
1808     p_informL2AboutTokenLoss;
1809     forward_eviction_to_cpu
1810     m_popRequestQueue;
1811   }
1812
1813   transition(O, Persistent_GETX, I_L) {
1814     ee_sendDataWithAllTokens;
1815     p_informL2AboutTokenLoss;
1816     forward_eviction_to_cpu
1817     l_popPersistentQueue;
1818   }
1819
1820   transition(O, Persistent_GETS, S_L) {
1821     ff_sendDataWithAllButNorOneTokens;
1822     l_popPersistentQueue;
1823   }
1824
1825   transition(O, Persistent_GETS_Last_Token, I_L) {
1826     fo_sendDataWithOwnerToken;
1827     forward_eviction_to_cpu
1828     l_popPersistentQueue;
1829   }
1830
1831   transition(O, Transient_GETS) {
1832     d_sendDataWithToken;
1833     m_popRequestQueue;
1834   }
1835
1836   transition(O, Transient_Local_GETS) {
1837     d_sendDataWithToken;
1838     m_popRequestQueue;
1839   }
1840
1841   // ran out of tokens, wait for it to go persistent
1842   transition(O, {Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token}) {
1843     m_popRequestQueue;
1844   }
1845
1846   transition(O, Ack) {
1847     q_updateTokensFromResponse;
1848     n_popResponseQueue;
1849   }
1850
1851   transition(O, Ack_All_Tokens, M) {
1852     q_updateTokensFromResponse;
1853     n_popResponseQueue;
1854   }
1855
1856   transition(O, Data_Shared) {
1857     w_assertIncomingDataAndCacheDataMatch;
1858     q_updateTokensFromResponse;
1859     n_popResponseQueue;
1860   }
1861
1862   transition(O, Data_All_Tokens, M) {
1863     w_assertIncomingDataAndCacheDataMatch;
1864     q_updateTokensFromResponse;
1865     n_popResponseQueue;
1866   }
1867
1868   // Transitions from Modified
1869   transition({MM, MM_W}, Ifetch) {
1870     h_load_hit;
1871     uu_profileInstHit;
1872     k_popMandatoryQueue;
1873   }
1874
1875   transition({MM, MM_W}, Load) {
1876     h_load_hit;
1877     uu_profileDataHit;
1878     k_popMandatoryQueue;
1879   }
1880
1881   transition({MM_W}, {Store, Atomic}) {
1882     hh_store_hit;
1883     uu_profileDataHit;
1884     k_popMandatoryQueue;
1885   }
1886
1887   transition(MM, Store) {
1888     hh_store_hit;
1889     uu_profileDataHit;
1890     k_popMandatoryQueue;
1891   }
1892
1893   transition(MM, Atomic, M) {
1894     hh_store_hit;
1895     uu_profileDataHit;
1896     k_popMandatoryQueue;
1897   }
1898
1899   transition(MM, L1_Replacement, I) {
1900     ta_traceStalledAddress;
1901     c_ownedReplacement;
1902     forward_eviction_to_cpu
1903     gg_deallocateL1CacheBlock;
1904     ka_wakeUpAllDependents;
1905   }
1906
1907   transition(MM, {Transient_GETX, Transient_Local_GETX, Transient_GETS, Transient_Local_GETS}, I) {
1908     dd_sendDataWithAllTokens;
1909     p_informL2AboutTokenLoss;
1910     forward_eviction_to_cpu
1911     m_popRequestQueue;
1912   }
1913
1914   transition({MM_W}, {Transient_GETX, Transient_Local_GETX, Transient_GETS, Transient_Local_GETS}) { // Ignore the request
1915     m_popRequestQueue;
1916   }
1917
1918   // Implement the migratory sharing optimization, even for persistent requests
1919   transition(MM, {Persistent_GETX, Persistent_GETS}, I_L) {
1920     ee_sendDataWithAllTokens;
1921     p_informL2AboutTokenLoss;
1922     forward_eviction_to_cpu
1923     l_popPersistentQueue;
1924   }
1925
1926   // ignore persistent requests in lockout period
1927   transition(MM_W, {Persistent_GETX, Persistent_GETS}) {
1928     l_popPersistentQueue;
1929   }
1930
1931   transition(MM_W, Use_TimeoutNoStarvers, MM) {
1932     s_deallocateTBE;
1933     jj_unsetUseTimer;
1934     kd_wakeUpDependents;
1935   }
1936
1937   transition(MM_W, Use_TimeoutNoStarvers_NoMig, M) {
1938     s_deallocateTBE;
1939     jj_unsetUseTimer;
1940     kd_wakeUpDependents;
1941   }
1942
1943   // Transitions from Dirty Exclusive
1944   transition({M, M_W}, Ifetch) {
1945     h_load_hit;
1946     uu_profileInstHit;
1947     k_popMandatoryQueue;
1948   }
1949
1950   transition({M, M_W}, Load) {
1951     h_load_hit;
1952     uu_profileDataHit;
1953     k_popMandatoryQueue;
1954   }
1955
1956   transition(M, Store, MM) {
1957     hh_store_hit;
1958     uu_profileDataHit;
1959     k_popMandatoryQueue;
1960   }
1961
1962   transition(M, Atomic) {
1963     hh_store_hit;
1964     uu_profileDataHit;
1965     k_popMandatoryQueue;
1966   }
1967
1968   transition(M_W, Store, MM_W) {
1969     hh_store_hit;
1970     uu_profileDataHit;
1971     k_popMandatoryQueue;
1972   }
1973
1974   transition(M_W, Atomic) {
1975     hh_store_hit;
1976     uu_profileDataHit;
1977     k_popMandatoryQueue;
1978   }
1979
1980   transition(M, L1_Replacement, I) {
1981     ta_traceStalledAddress;
1982     c_ownedReplacement;
1983     forward_eviction_to_cpu
1984     gg_deallocateL1CacheBlock;
1985     ka_wakeUpAllDependents;
1986   }
1987
1988   transition(M, {Transient_GETX, Transient_Local_GETX}, I) {
1989     dd_sendDataWithAllTokens;
1990     p_informL2AboutTokenLoss;
1991     forward_eviction_to_cpu
1992     m_popRequestQueue;
1993   }
1994
1995   transition(M, Transient_Local_GETS, O) {
1996     d_sendDataWithToken;
1997     m_popRequestQueue;
1998   }
1999
2000   transition(M, Transient_GETS, O) {
2001     d_sendDataWithNTokenIfAvail;
2002     m_popRequestQueue;
2003   }
2004
2005   transition(M_W, {Transient_GETX, Transient_Local_GETX, Transient_GETS, Transient_Local_GETS}) { // Ignore the request
2006     m_popRequestQueue;
2007   }
2008
2009   transition(M, Persistent_GETX, I_L) {
2010     ee_sendDataWithAllTokens;
2011     p_informL2AboutTokenLoss;
2012     forward_eviction_to_cpu
2013     l_popPersistentQueue;
2014   }
2015
2016   transition(M, Persistent_GETS, S_L) {
2017     ff_sendDataWithAllButNorOneTokens;
2018     l_popPersistentQueue;
2019   }
2020
2021   // ignore persistent requests in lockout period
2022   transition(M_W, {Persistent_GETX, Persistent_GETS}) {
2023     l_popPersistentQueue;
2024   }
2025
2026   transition(M_W, Use_TimeoutStarverS, S_L) {
2027     s_deallocateTBE;
2028     ff_sendDataWithAllButNorOneTokens;
2029     jj_unsetUseTimer;
2030   }
2031
2032   // someone unlocked during timeout
2033   transition(M_W, {Use_TimeoutNoStarvers, Use_TimeoutNoStarvers_NoMig}, M) {
2034     s_deallocateTBE;
2035     jj_unsetUseTimer;
2036     kd_wakeUpDependents;
2037   }
2038
2039   transition(M_W, Use_TimeoutStarverX, I_L) {
2040     s_deallocateTBE;
2041     ee_sendDataWithAllTokens;
2042     forward_eviction_to_cpu;
2043     p_informL2AboutTokenLoss;
2044     jj_unsetUseTimer;
2045   }
2046
2047   // migratory
2048   transition(MM_W, {Use_TimeoutStarverX, Use_TimeoutStarverS}, I_L) {
2049     s_deallocateTBE;
2050     ee_sendDataWithAllTokens;
2051     forward_eviction_to_cpu;
2052     p_informL2AboutTokenLoss;
2053     jj_unsetUseTimer;
2054
2055   }
2056
2057   // Transient_GETX and Transient_GETS in transient states
2058   transition(OM, {Transient_GETX, Transient_Local_GETX, Transient_GETS, Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token, Transient_Local_GETS}) {
2059     m_popRequestQueue;  // Even if we have the data, we can pretend we don't have it yet.
2060   }
2061
2062   transition(IS, {Transient_GETX, Transient_Local_GETX}) {
2063     t_sendAckWithCollectedTokens;
2064     m_popRequestQueue;
2065   }
2066
2067   transition(IS, {Transient_GETS, Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token, Transient_Local_GETS}) {
2068     m_popRequestQueue;
2069   }
2070
2071   transition(IS, {Persistent_GETX, Persistent_GETS, Persistent_GETS_Last_Token}, IS_L) {
2072     e_sendAckWithCollectedTokens;
2073     l_popPersistentQueue;
2074   }
2075
2076   transition(IS_L, {Persistent_GETX, Persistent_GETS}) {
2077     l_popPersistentQueue;
2078   }
2079
2080   transition(IM, {Persistent_GETX, Persistent_GETS, Persistent_GETS_Last_Token}, IM_L) {
2081     e_sendAckWithCollectedTokens;
2082     l_popPersistentQueue;
2083   }
2084
2085   transition(IM_L, {Persistent_GETX, Persistent_GETS}) {
2086     l_popPersistentQueue;
2087   }
2088
2089   transition({SM, SM_L}, Persistent_GETX, IM_L) {
2090     e_sendAckWithCollectedTokens;
2091     forward_eviction_to_cpu
2092     l_popPersistentQueue;
2093   }
2094
2095   transition(SM, {Persistent_GETS, Persistent_GETS_Last_Token}, SM_L) {
2096     f_sendAckWithAllButNorOneTokens;
2097     l_popPersistentQueue;
2098   }
2099
2100   transition(SM_L, {Persistent_GETS, Persistent_GETS_Last_Token}) {
2101     l_popPersistentQueue;
2102   }
2103
2104   transition(OM, Persistent_GETX, IM_L) {
2105     ee_sendDataWithAllTokens;
2106     forward_eviction_to_cpu
2107     l_popPersistentQueue;
2108   }
2109
2110   transition(OM, Persistent_GETS, SM_L) {
2111     ff_sendDataWithAllButNorOneTokens;
2112     l_popPersistentQueue;
2113   }
2114
2115   transition(OM, Persistent_GETS_Last_Token, IM_L) {
2116     fo_sendDataWithOwnerToken;
2117     l_popPersistentQueue;
2118   }
2119
2120   // Transitions from IM/SM
2121
2122   transition({IM, SM}, Ack) {
2123     q_updateTokensFromResponse;
2124     n_popResponseQueue;
2125   }
2126
2127   transition(IM, Data_Shared, SM) {
2128     u_writeDataToCache;
2129     q_updateTokensFromResponse;
2130     n_popResponseQueue;
2131   }
2132
2133   transition(IM, Data_Owner, OM) {
2134     u_writeDataToCache;
2135     q_updateTokensFromResponse;
2136     n_popResponseQueue;
2137   }
2138
2139   transition(IM, Data_All_Tokens, MM_W) {
2140     u_writeDataToCache;
2141     q_updateTokensFromResponse;
2142     xx_external_store_hit;
2143     o_scheduleUseTimeout;
2144     j_unsetReissueTimer;
2145     n_popResponseQueue;
2146     kd_wakeUpDependents;
2147   }
2148
2149   transition(SM, Data_Shared) {
2150     w_assertIncomingDataAndCacheDataMatch;
2151     q_updateTokensFromResponse;
2152     n_popResponseQueue;
2153   }
2154
2155   transition(SM, Data_Owner, OM) {
2156     w_assertIncomingDataAndCacheDataMatch;
2157     q_updateTokensFromResponse;
2158     n_popResponseQueue;
2159   }
2160
2161   transition(SM, Data_All_Tokens, MM_W) {
2162     w_assertIncomingDataAndCacheDataMatch;
2163     q_updateTokensFromResponse;
2164     xx_external_store_hit;
2165     o_scheduleUseTimeout;
2166     j_unsetReissueTimer;
2167     n_popResponseQueue;
2168     kd_wakeUpDependents;
2169   }
2170
2171   transition({IM, SM}, {Transient_GETX, Transient_Local_GETX}, IM) { // We don't have the data yet, but we might have collected some tokens.  We give them up here to avoid livelock
2172     t_sendAckWithCollectedTokens;
2173     forward_eviction_to_cpu;
2174     m_popRequestQueue;
2175   }
2176
2177   transition({IM, SM}, {Transient_GETS, Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token, Transient_Local_GETS}) {
2178     m_popRequestQueue;
2179   }
2180
2181   transition({IM, SM}, Request_Timeout) {
2182     j_unsetReissueTimer;
2183     b_issueWriteRequest;
2184   }
2185
2186   // Transitions from OM
2187
2188   transition(OM, Ack) {
2189     q_updateTokensFromResponse;
2190     n_popResponseQueue;
2191   }
2192
2193   transition(OM, Ack_All_Tokens, MM_W) {
2194     q_updateTokensFromResponse;
2195     xx_external_store_hit;
2196     o_scheduleUseTimeout;
2197     j_unsetReissueTimer;
2198     n_popResponseQueue;
2199     kd_wakeUpDependents;
2200   }
2201
2202   transition(OM, Data_Shared) {
2203     w_assertIncomingDataAndCacheDataMatch;
2204     q_updateTokensFromResponse;
2205     n_popResponseQueue;
2206   }
2207
2208   transition(OM, Data_All_Tokens, MM_W) {
2209     w_assertIncomingDataAndCacheDataMatch;
2210     q_updateTokensFromResponse;
2211     xx_external_store_hit;
2212     o_scheduleUseTimeout;
2213     j_unsetReissueTimer;
2214     n_popResponseQueue;
2215     kd_wakeUpDependents;
2216   }
2217
2218   transition(OM, Request_Timeout) {
2219     j_unsetReissueTimer;
2220     b_issueWriteRequest;
2221   }
2222
2223   // Transitions from IS
2224
2225   transition(IS, Ack) {
2226     q_updateTokensFromResponse;
2227     n_popResponseQueue;
2228   }
2229
2230   transition(IS, Data_Shared, S) {
2231     u_writeDataToCache;
2232     q_updateTokensFromResponse;
2233     x_external_load_hit;
2234     s_deallocateTBE;
2235     j_unsetReissueTimer;
2236     n_popResponseQueue;
2237     kd_wakeUpDependents;
2238   }
2239
2240   transition(IS, Data_Owner, O) {
2241     u_writeDataToCache;
2242     q_updateTokensFromResponse;
2243     x_external_load_hit;
2244     s_deallocateTBE;
2245     j_unsetReissueTimer;
2246     n_popResponseQueue;
2247     kd_wakeUpDependents;
2248   }
2249
2250   transition(IS, Data_All_Tokens, M_W) {
2251     u_writeDataToCache;
2252     q_updateTokensFromResponse;
2253     x_external_load_hit;
2254     o_scheduleUseTimeout;
2255     j_unsetReissueTimer;
2256     n_popResponseQueue;
2257     kd_wakeUpDependents;
2258   }
2259
2260   transition(IS, Request_Timeout) {
2261     j_unsetReissueTimer;
2262     a_issueReadRequest;
2263   }
2264
2265   // Transitions from I_L
2266
2267   transition(I_L, Load, IS_L) {
2268     ii_allocateL1DCacheBlock;
2269     i_allocateTBE;
2270     a_issueReadRequest;
2271     uu_profileDataMiss;
2272     k_popMandatoryQueue;
2273   }
2274
2275   transition(I_L, Ifetch, IS_L) {
2276     pp_allocateL1ICacheBlock;
2277     i_allocateTBE;
2278     a_issueReadRequest;
2279     uu_profileInstMiss;
2280     k_popMandatoryQueue;
2281   }
2282
2283   transition(I_L, {Store, Atomic}, IM_L) {
2284     ii_allocateL1DCacheBlock;
2285     i_allocateTBE;
2286     b_issueWriteRequest;
2287     uu_profileDataMiss;
2288     k_popMandatoryQueue;
2289   }
2290
2291
2292   // Transitions from S_L
2293
2294   transition(S_L, {Store, Atomic}, SM_L) {
2295     i_allocateTBE;
2296     b_issueWriteRequest;
2297     uu_profileDataMiss;
2298     k_popMandatoryQueue;
2299   }
2300
2301   // Other transitions from *_L states
2302
2303   transition({I_L, IM_L, IS_L, S_L, SM_L}, {Transient_GETS, Transient_GETS_Last_Token, Transient_Local_GETS_Last_Token, Transient_Local_GETS, Transient_GETX, Transient_Local_GETX}) {
2304     m_popRequestQueue;
2305   }
2306
2307   transition({I_L, IM_L, IS_L, S_L, SM_L}, Ack) {
2308     g_bounceResponseToStarver;
2309     n_popResponseQueue;
2310   }
2311
2312   transition({I_L, IM_L, S_L, SM_L}, {Data_Shared, Data_Owner}) {
2313     g_bounceResponseToStarver;
2314     n_popResponseQueue;
2315   }
2316
2317   transition({I_L, S_L}, Data_All_Tokens) {
2318     g_bounceResponseToStarver;
2319     n_popResponseQueue;
2320   }
2321
2322   transition(IS_L, Request_Timeout) {
2323     j_unsetReissueTimer;
2324     a_issueReadRequest;
2325   }
2326
2327   transition({IM_L, SM_L}, Request_Timeout) {
2328     j_unsetReissueTimer;
2329     b_issueWriteRequest;
2330   }
2331
2332   // Opportunisticly Complete the memory operation in the following
2333   // cases.  Note: these transitions could just use
2334   // g_bounceResponseToStarver, but if we have the data and tokens, we
2335   // might as well complete the memory request while we have the
2336   // chance (and then immediately forward on the data)
2337
2338   transition(IM_L, Data_All_Tokens, MM_W) {
2339     u_writeDataToCache;
2340     q_updateTokensFromResponse;
2341     xx_external_store_hit;
2342     j_unsetReissueTimer;
2343     o_scheduleUseTimeout;
2344     n_popResponseQueue;
2345     kd_wakeUpDependents;
2346   }
2347
2348   transition(SM_L, Data_All_Tokens, S_L) {
2349     u_writeDataToCache;
2350     q_updateTokensFromResponse;
2351     xx_external_store_hit;
2352     ff_sendDataWithAllButNorOneTokens;
2353     s_deallocateTBE;
2354     j_unsetReissueTimer;
2355     n_popResponseQueue;
2356   }
2357
2358   transition(IS_L, Data_Shared, I_L) {
2359     u_writeDataToCache;
2360     q_updateTokensFromResponse;
2361     x_external_load_hit;
2362     s_deallocateTBE;
2363     e_sendAckWithCollectedTokens;
2364     p_informL2AboutTokenLoss;
2365     j_unsetReissueTimer;
2366     n_popResponseQueue;
2367   }
2368
2369   transition(IS_L, Data_Owner, I_L) {
2370     u_writeDataToCache;
2371     q_updateTokensFromResponse;
2372     x_external_load_hit;
2373     ee_sendDataWithAllTokens;
2374     s_deallocateTBE;
2375     p_informL2AboutTokenLoss;
2376     j_unsetReissueTimer;
2377     n_popResponseQueue;
2378   }
2379
2380   transition(IS_L, Data_All_Tokens, M_W) {
2381     u_writeDataToCache;
2382     q_updateTokensFromResponse;
2383     x_external_load_hit;
2384     j_unsetReissueTimer;
2385     o_scheduleUseTimeout;
2386     n_popResponseQueue;
2387     kd_wakeUpDependents;
2388   }
2389
2390   // Own_Lock_or_Unlock
2391
2392   transition(I_L, Own_Lock_or_Unlock, I) {
2393     l_popPersistentQueue;
2394     kd_wakeUpDependents;
2395   }
2396
2397   transition(S_L, Own_Lock_or_Unlock, S) {
2398     l_popPersistentQueue;
2399     kd_wakeUpDependents;
2400   }
2401
2402   transition(IM_L, Own_Lock_or_Unlock, IM) {
2403     l_popPersistentQueue;
2404     kd_wakeUpDependents;
2405   }
2406
2407   transition(IS_L, Own_Lock_or_Unlock, IS) {
2408     l_popPersistentQueue;
2409     kd_wakeUpDependents;
2410   }
2411
2412   transition(SM_L, Own_Lock_or_Unlock, SM) {
2413     l_popPersistentQueue;
2414     kd_wakeUpDependents;
2415   }
2416 }