X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fmem%2Fruby%2Fsystem%2FSystem.cc;h=98cf50e9c5d837bb4a099f96c6e93c4b1eb7faa1;hb=96091f358b97a10e261422aebefd6b5c187c2a60;hp=01e50ef0b5bc3246dc9b479c7de5ece17de832e1;hpb=6aed4d4f931f0eca15d1211e7e28a06d66f16d32;p=gem5.git diff --git a/src/mem/ruby/system/System.cc b/src/mem/ruby/system/System.cc index 01e50ef0b..98cf50e9c 100644 --- a/src/mem/ruby/system/System.cc +++ b/src/mem/ruby/system/System.cc @@ -30,6 +30,7 @@ #include #include +#include #include "base/intmath.hh" #include "base/statistics.hh" @@ -37,8 +38,8 @@ #include "debug/RubySystem.hh" #include "mem/ruby/common/Address.hh" #include "mem/ruby/network/Network.hh" -#include "mem/ruby/profiler/Profiler.hh" #include "mem/ruby/system/System.hh" +#include "mem/simple_mem.hh" #include "sim/eventq.hh" #include "sim/simulate.hh" @@ -46,17 +47,19 @@ using namespace std; int RubySystem::m_random_seed; bool RubySystem::m_randomization; -int RubySystem::m_block_size_bytes; -int RubySystem::m_block_size_bits; -uint64 RubySystem::m_memory_size_bytes; -int RubySystem::m_memory_size_bits; +uint32_t RubySystem::m_block_size_bytes; +uint32_t RubySystem::m_block_size_bits; +uint32_t RubySystem::m_memory_size_bits; +bool RubySystem::m_warmup_enabled = false; +// To look forward to allowing multiple RubySystem instances, track the number +// of RubySystems that need to be warmed up on checkpoint restore. +unsigned RubySystem::m_systems_to_warmup = 0; +bool RubySystem::m_cooldown_enabled = false; RubySystem::RubySystem(const Params *p) - : ClockedObject(p) + : ClockedObject(p), m_access_backing_store(p->access_backing_store), + m_cache_recorder(NULL) { - if (g_system_ptr != NULL) - fatal("Only one RubySystem object currently allowed.\n"); - m_random_seed = p->random_seed; srandom(m_random_seed); m_randomization = p->randomization; @@ -64,132 +67,51 @@ RubySystem::RubySystem(const Params *p) m_block_size_bytes = p->block_size_bytes; assert(isPowerOf2(m_block_size_bytes)); m_block_size_bits = floorLog2(m_block_size_bytes); + m_memory_size_bits = p->memory_size_bits; - m_memory_size_bytes = p->mem_size; - if (m_memory_size_bytes == 0) { - m_memory_size_bits = 0; - } else { - m_memory_size_bits = ceilLog2(m_memory_size_bytes); - } - - g_system_ptr = this; - if (p->no_mem_vec) { - m_mem_vec_ptr = NULL; - } else { - m_mem_vec_ptr = new MemoryVector; - m_mem_vec_ptr->resize(m_memory_size_bytes); - } - - // Print ruby configuration and stats at exit and when asked for - Stats::registerDumpCallback(new RubyDumpStatsCallback(p->stats_filename, - this)); + // Resize to the size of different machine types + m_abstract_controls.resize(MachineType_NUM); - m_warmup_enabled = false; - m_cooldown_enabled = false; -} - -void -RubySystem::init() -{ - m_profiler_ptr->clearStats(); - m_network_ptr->clearStats(); + // Collate the statistics before they are printed. + Stats::registerDumpCallback(new RubyStatsCallback(this)); + // Create the profiler + m_profiler = new Profiler(p, this); + m_phys_mem = p->phys_mem; } void RubySystem::registerNetwork(Network* network_ptr) { - m_network_ptr = network_ptr; -} - -void -RubySystem::registerProfiler(Profiler* profiler_ptr) -{ - m_profiler_ptr = profiler_ptr; + m_network = network_ptr; } void RubySystem::registerAbstractController(AbstractController* cntrl) { - m_abs_cntrl_vec.push_back(cntrl); -} + m_abs_cntrl_vec.push_back(cntrl); -void -RubySystem::registerSparseMemory(SparseMemory* s) -{ - m_sparse_memory_vector.push_back(s); -} - -void -RubySystem::registerMemController(MemoryControl *mc) { - m_memory_controller_vec.push_back(mc); + MachineID id = cntrl->getMachineID(); + m_abstract_controls[id.getType()][id.getNum()] = cntrl; } RubySystem::~RubySystem() { - delete m_network_ptr; - delete m_profiler_ptr; - if (m_mem_vec_ptr) - delete m_mem_vec_ptr; + delete m_network; + delete m_profiler; } void -RubySystem::printStats(ostream& out) -{ - const time_t T = time(NULL); - tm *localTime = localtime(&T); - char buf[100]; - strftime(buf, 100, "%b/%d/%Y %H:%M:%S", localTime); - - out << "Real time: " << buf << endl; - - m_profiler_ptr->printStats(out); - m_network_ptr->printStats(out); -} - -void -RubySystem::writeCompressedTrace(uint8_t *raw_data, string filename, - uint64 uncompressed_trace_size) +RubySystem::makeCacheRecorder(uint8_t *uncompressed_trace, + uint64 cache_trace_size, + uint64 block_size_bytes) { - // Create the checkpoint file for the memory - string thefile = Checkpoint::dir() + "/" + filename.c_str(); - - int fd = creat(thefile.c_str(), 0664); - if (fd < 0) { - perror("creat"); - fatal("Can't open memory trace file '%s'\n", filename); - } - - gzFile compressedMemory = gzdopen(fd, "wb"); - if (compressedMemory == NULL) - fatal("Insufficient memory to allocate compression state for %s\n", - filename); - - if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) != - uncompressed_trace_size) { - fatal("Write failed on memory trace file '%s'\n", filename); - } - - if (gzclose(compressedMemory)) { - fatal("Close failed on memory trace file '%s'\n", filename); - } - delete raw_data; -} - -void -RubySystem::serialize(std::ostream &os) -{ - m_cooldown_enabled = true; - vector sequencer_map; Sequencer* sequencer_ptr = NULL; - int cntrl_id = -1; - for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); if (sequencer_ptr == NULL) { sequencer_ptr = sequencer_map[cntrl]; - cntrl_id = cntrl; } } @@ -201,21 +123,46 @@ RubySystem::serialize(std::ostream &os) } } - DPRINTF(RubyCacheTrace, "Recording Cache Trace\n"); + // Remove the old CacheRecorder if it's still hanging about. + if (m_cache_recorder != NULL) { + delete m_cache_recorder; + } + // Create the CacheRecorder and record the cache trace - m_cache_recorder = new CacheRecorder(NULL, 0, sequencer_map); + m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size, + sequencer_map, block_size_bytes); +} + +void +RubySystem::memWriteback() +{ + m_cooldown_enabled = true; + // Make the trace so we know what to write back. + DPRINTF(RubyCacheTrace, "Recording Cache Trace\n"); + makeCacheRecorder(NULL, 0, getBlockSizeBytes()); for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { m_abs_cntrl_vec[cntrl]->recordCacheTrace(cntrl, m_cache_recorder); } - DPRINTF(RubyCacheTrace, "Cache Trace Complete\n"); + // save the current tick value Tick curtick_original = curTick(); - // save the event queue head - Event* eventq_head = eventq->replaceHead(NULL); - DPRINTF(RubyCacheTrace, "Recording current tick %ld and event queue\n", - curtick_original); + DPRINTF(RubyCacheTrace, "Recording current tick %ld\n", curtick_original); + + // Deschedule all prior events on the event queue, but record the tick they + // were scheduled at so they can be restored correctly later. + list > original_events; + while (!eventq->empty()) { + Event *curr_head = eventq->getHead(); + if (curr_head->isAutoDelete()) { + DPRINTF(RubyCacheTrace, "Event %s auto-deletes when descheduled," + " not recording\n", curr_head->name()); + } else { + original_events.push_back(make_pair(curr_head, curr_head->when())); + } + eventq->deschedule(curr_head); + } // Schedule an event to start cache cooldown DPRINTF(RubyCacheTrace, "Starting cache flush\n"); @@ -223,32 +170,88 @@ RubySystem::serialize(std::ostream &os) simulate(); DPRINTF(RubyCacheTrace, "Cache flush complete\n"); - // Restore eventq head - eventq_head = eventq->replaceHead(eventq_head); + // Deschedule any events left on the event queue. + while (!eventq->empty()) { + eventq->deschedule(eventq->getHead()); + } + // Restore curTick setCurTick(curtick_original); - uint8_t *raw_data = NULL; + // Restore all events that were originally on the event queue. This is + // done after setting curTick back to its original value so that events do + // not seem to be scheduled in the past. + while (!original_events.empty()) { + pair event = original_events.back(); + eventq->schedule(event.first, event.second); + original_events.pop_back(); + } - if (m_mem_vec_ptr != NULL) { - uint64 memory_trace_size = m_mem_vec_ptr->collatePages(raw_data); + // No longer flushing back to memory. + m_cooldown_enabled = false; - string memory_trace_file = name() + ".memory.gz"; - writeCompressedTrace(raw_data, memory_trace_file, - memory_trace_size); + // There are several issues with continuing simulation after calling + // memWriteback() at the moment, that stem from taking events off the + // queue, simulating again, and then putting them back on, whilst + // pretending that no time has passed. One is that some events will have + // been deleted, so can't be put back. Another is that any object + // recording the tick something happens may end up storing a tick in the + // future. A simple warning here alerts the user that things may not work + // as expected. + warn_once("Ruby memory writeback is experimental. Continuing simulation " + "afterwards may not always work as intended."); + + // Keep the cache recorder around so that we can dump the trace if a + // checkpoint is immediately taken. +} - SERIALIZE_SCALAR(memory_trace_file); - SERIALIZE_SCALAR(memory_trace_size); +void +RubySystem::writeCompressedTrace(uint8_t *raw_data, string filename, + uint64 uncompressed_trace_size) +{ + // Create the checkpoint file for the memory + string thefile = CheckpointIn::dir() + "/" + filename.c_str(); - } else { - for (int i = 0; i < m_sparse_memory_vector.size(); ++i) { - m_sparse_memory_vector[i]->recordBlocks(cntrl_id, - m_cache_recorder); - } + int fd = creat(thefile.c_str(), 0664); + if (fd < 0) { + perror("creat"); + fatal("Can't open memory trace file '%s'\n", filename); } - // Aggergate the trace entries together into a single array - raw_data = new uint8_t[4096]; + gzFile compressedMemory = gzdopen(fd, "wb"); + if (compressedMemory == NULL) + fatal("Insufficient memory to allocate compression state for %s\n", + filename); + + if (gzwrite(compressedMemory, raw_data, uncompressed_trace_size) != + uncompressed_trace_size) { + fatal("Write failed on memory trace file '%s'\n", filename); + } + + if (gzclose(compressedMemory)) { + fatal("Close failed on memory trace file '%s'\n", filename); + } + delete[] raw_data; +} + +void +RubySystem::serializeOld(CheckpointOut &cp) +{ + // Store the cache-block size, so we are able to restore on systems with a + // different cache-block size. CacheRecorder depends on the correct + // cache-block size upon unserializing. + uint64 block_size_bytes = getBlockSizeBytes(); + SERIALIZE_SCALAR(block_size_bytes); + + // Check that there's a valid trace to use. If not, then memory won't be + // up-to-date and the simulation will probably fail when restoring from the + // checkpoint. + if (m_cache_recorder == NULL) { + fatal("Call memWriteback() before serialize() to create ruby trace"); + } + + // Aggregate the trace entries together into a single array + uint8_t *raw_data = new uint8_t[4096]; uint64 cache_trace_size = m_cache_recorder->aggregateRecords(&raw_data, 4096); string cache_trace_file = name() + ".cache.gz"; @@ -257,7 +260,9 @@ RubySystem::serialize(std::ostream &os) SERIALIZE_SCALAR(cache_trace_file); SERIALIZE_SCALAR(cache_trace_size); - m_cooldown_enabled = false; + // Now finished with the cache recorder. + delete m_cache_recorder; + m_cache_recorder = NULL; } void @@ -292,66 +297,54 @@ RubySystem::readCompressedTrace(string filename, uint8_t *&raw_data, } void -RubySystem::unserialize(Checkpoint *cp, const string §ion) +RubySystem::unserialize(CheckpointIn &cp) { - // - // The main purpose for clearing stats in the unserialize process is so - // that the profiler can correctly set its start time to the unserialized - // value of curTick() - // - resetStats(); uint8_t *uncompressed_trace = NULL; - if (m_mem_vec_ptr != NULL) { - string memory_trace_file; - uint64 memory_trace_size = 0; - - UNSERIALIZE_SCALAR(memory_trace_file); - UNSERIALIZE_SCALAR(memory_trace_size); - memory_trace_file = cp->cptDir + "/" + memory_trace_file; - - readCompressedTrace(memory_trace_file, uncompressed_trace, - memory_trace_size); - m_mem_vec_ptr->populatePages(uncompressed_trace); - - delete uncompressed_trace; - uncompressed_trace = NULL; - } + // This value should be set to the checkpoint-system's block-size. + // Optional, as checkpoints without it can be run if the + // checkpoint-system's block-size == current block-size. + uint64 block_size_bytes = getBlockSizeBytes(); + UNSERIALIZE_OPT_SCALAR(block_size_bytes); string cache_trace_file; uint64 cache_trace_size = 0; UNSERIALIZE_SCALAR(cache_trace_file); UNSERIALIZE_SCALAR(cache_trace_size); - cache_trace_file = cp->cptDir + "/" + cache_trace_file; + cache_trace_file = cp.cptDir + "/" + cache_trace_file; readCompressedTrace(cache_trace_file, uncompressed_trace, cache_trace_size); m_warmup_enabled = true; + m_systems_to_warmup++; - vector sequencer_map; - Sequencer* t = NULL; - for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { - sequencer_map.push_back(m_abs_cntrl_vec[cntrl]->getSequencer()); - if (t == NULL) t = sequencer_map[cntrl]; - } - - assert(t != NULL); - - for (int cntrl = 0; cntrl < m_abs_cntrl_vec.size(); cntrl++) { - if (sequencer_map[cntrl] == NULL) { - sequencer_map[cntrl] = t; - } - } - - m_cache_recorder = new CacheRecorder(uncompressed_trace, cache_trace_size, - sequencer_map); + // Create the cache recorder that will hang around until startup. + makeCacheRecorder(uncompressed_trace, cache_trace_size, block_size_bytes); } void RubySystem::startup() { + + // Ruby restores state from a checkpoint by resetting the clock to 0 and + // playing the requests that can possibly re-generate the cache state. + // The clock value is set to the actual checkpointed value once all the + // requests have been executed. + // + // This way of restoring state is pretty finicky. For example, if a + // Ruby component reads time before the state has been restored, it would + // cache this value and hence its clock would not be reset to 0, when + // Ruby resets the global clock. This can potentially result in a + // deadlock. + // + // The solution is that no Ruby component should read time before the + // simulation starts. And then one also needs to hope that the time + // Ruby finishes restoring the state is less than the time when the + // state was checkpointed. + if (m_warmup_enabled) { + DPRINTF(RubyCacheTrace, "Starting ruby cache warmup\n"); // save the current tick value Tick curtick_original = curTick(); // save the event queue head @@ -366,12 +359,9 @@ RubySystem::startup() delete m_cache_recorder; m_cache_recorder = NULL; - m_warmup_enabled = false; - - // reset DRAM so that it's not waiting for events on the old event - // queue - for (int i = 0; i < m_memory_controller_vec.size(); ++i) { - m_memory_controller_vec[i]->reset(); + m_systems_to_warmup--; + if (m_systems_to_warmup == 0) { + m_warmup_enabled = false; } // Restore eventq head @@ -380,23 +370,24 @@ RubySystem::startup() setCurTick(curtick_original); resetClock(); } + + resetStats(); } void RubySystem::RubyEvent::process() { - if (ruby_system->m_warmup_enabled) { - ruby_system->m_cache_recorder->enqueueNextFetchRequest(); - } else if (ruby_system->m_cooldown_enabled) { - ruby_system->m_cache_recorder->enqueueNextFlushRequest(); + if (RubySystem::getWarmupEnabled()) { + m_ruby_system->m_cache_recorder->enqueueNextFetchRequest(); + } else if (RubySystem::getCooldownEnabled()) { + m_ruby_system->m_cache_recorder->enqueueNextFlushRequest(); } } void RubySystem::resetStats() { - m_profiler_ptr->clearStats(); - m_network_ptr->clearStats(); + m_start_cycle = curCycle(); } bool @@ -439,10 +430,6 @@ RubySystem::functionalRead(PacketPtr pkt) } assert(num_rw <= 1); - uint8_t *data = pkt->getPtr(true); - unsigned int size_in_bytes = pkt->getSize(); - unsigned startByte = address.getAddress() - line_address.getAddress(); - // This if case is meant to capture what happens in a Broadcast/Snoop // protocol where the block does not exist in the cache hierarchy. You // only want to read from the Backing_Store memory if there is no copy in @@ -451,20 +438,12 @@ RubySystem::functionalRead(PacketPtr pkt) // The reason is because the Backing_Store memory could easily be stale, if // there are copies floating around the cache hierarchy, so you want to read // it only if it's not in the cache hierarchy at all. - if (num_invalid == (num_controllers - 1) && - num_backing_store == 1) { + if (num_invalid == (num_controllers - 1) && num_backing_store == 1) { DPRINTF(RubySystem, "only copy in Backing_Store memory, read from it\n"); for (unsigned int i = 0; i < num_controllers; ++i) { access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address); if (access_perm == AccessPermission_Backing_Store) { - DataBlock& block = m_abs_cntrl_vec[i]-> - getDataBlock(line_address); - - DPRINTF(RubySystem, "reading from %s block %s\n", - m_abs_cntrl_vec[i]->name(), block); - for (unsigned i = 0; i < size_in_bytes; ++i) { - data[i] = block.getByte(i + startByte); - } + m_abs_cntrl_vec[i]->functionalRead(line_address, pkt); return true; } } @@ -482,14 +461,7 @@ RubySystem::functionalRead(PacketPtr pkt) access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_address); if (access_perm == AccessPermission_Read_Only || access_perm == AccessPermission_Read_Write) { - DataBlock& block = m_abs_cntrl_vec[i]-> - getDataBlock(line_address); - - DPRINTF(RubySystem, "reading from %s block %s\n", - m_abs_cntrl_vec[i]->name(), block); - for (unsigned i = 0; i < size_in_bytes; ++i) { - data[i] = block.getByte(i + startByte); - } + m_abs_cntrl_vec[i]->functionalRead(line_address, pkt); return true; } } @@ -512,33 +484,21 @@ RubySystem::functionalWrite(PacketPtr pkt) DPRINTF(RubySystem, "Functional Write request for %s\n",addr); - uint8_t *data = pkt->getPtr(true); - unsigned int size_in_bytes = pkt->getSize(); - unsigned startByte = addr.getAddress() - line_addr.getAddress(); + uint32_t M5_VAR_USED num_functional_writes = 0; for (unsigned int i = 0; i < num_controllers;++i) { - m_abs_cntrl_vec[i]->functionalWriteBuffers(pkt); + num_functional_writes += + m_abs_cntrl_vec[i]->functionalWriteBuffers(pkt); access_perm = m_abs_cntrl_vec[i]->getAccessPermission(line_addr); if (access_perm != AccessPermission_Invalid && access_perm != AccessPermission_NotPresent) { - - DataBlock& block = m_abs_cntrl_vec[i]->getDataBlock(line_addr); - DPRINTF(RubySystem, "%s\n",block); - for (unsigned i = 0; i < size_in_bytes; ++i) { - block.setByte(i + startByte, data[i]); - } - DPRINTF(RubySystem, "%s\n",block); + num_functional_writes += + m_abs_cntrl_vec[i]->functionalWrite(line_addr, pkt); } } - uint32_t M5_VAR_USED num_functional_writes = 0; - for (unsigned int i = 0; i < m_memory_controller_vec.size() ;++i) { - num_functional_writes += - m_memory_controller_vec[i]->functionalWriteBuffers(pkt); - } - - num_functional_writes += m_network_ptr->functionalWrite(pkt); + num_functional_writes += m_network->functionalWrite(pkt); DPRINTF(RubySystem, "Messages written = %u\n", num_functional_writes); return true; @@ -601,13 +561,3 @@ RubySystemParams::create() { return new RubySystem(this); } - -/** - * virtual process function that is invoked when the callback - * queue is executed. - */ -void -RubyDumpStatsCallback::process() -{ - ruby_system->printStats(*os); -}