Get rid of the Statistics and Statreset ParamContexts, and
[gem5.git] / src / base / stats / mysql.cc
1 /*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Nathan Binkert
29 */
30
31 #include <cassert>
32 #include <map>
33 #include <sstream>
34 #include <string>
35 #include <vector>
36
37 #include "base/misc.hh"
38 #include "base/mysql.hh"
39 #include "base/statistics.hh"
40 #include "base/stats/flags.hh"
41 #include "base/stats/mysql.hh"
42 #include "base/stats/mysql_run.hh"
43 #include "base/stats/statdb.hh"
44 #include "base/stats/types.hh"
45 #include "base/str.hh"
46 #include "sim/host.hh"
47
48 using namespace std;
49
50 namespace Stats {
51
52 void
53 MySqlRun::connect(const string &host, const string &user, const string &passwd,
54 const string &db, const string &name, const string &sample,
55 const string &project)
56 {
57 if (connected())
58 panic("can only get one database connection at this time!");
59
60 mysql.connect(host, user, passwd, db);
61 if (mysql.error)
62 panic("could not connect to database server\n%s\n", mysql.error);
63
64 if (mysql.autocommit(false))
65 panic("could not set autocommit\n%s\n", mysql.error);
66
67 remove(name);
68 //cleanup();
69 setup(name, sample, user, project);
70 }
71
72 void
73 MySqlRun::setup(const string &name, const string &sample, const string &user,
74 const string &project)
75 {
76 assert(mysql.connected());
77
78 stringstream insert;
79 ccprintf(insert,
80 "INSERT INTO "
81 "runs(rn_name,rn_sample,rn_user,rn_project,rn_date,rn_expire)"
82 "values(\"%s\", \"%s\", \"%s\", \"%s\", NOW(),"
83 "DATE_ADD(CURDATE(), INTERVAL 31 DAY))",
84 name, sample, user, project);
85
86 mysql.query(insert);
87 if (mysql.error)
88 panic("could not get a run\n%s\n", mysql.error);
89
90 run_id = mysql.insert_id();
91 if (mysql.commit())
92 panic("could not commit transaction\n%s\n", mysql.error);
93 }
94
95 void
96 MySqlRun::remove(const string &name)
97 {
98 assert(mysql.connected());
99 stringstream sql;
100 ccprintf(sql, "DELETE FROM runs WHERE rn_name=\"%s\"", name);
101 mysql.query(sql);
102 if (mysql.error)
103 panic("could not delete run\n%s\n", mysql.error);
104 if (mysql.commit())
105 panic("could not commit transaction\n%s\n", mysql.error);
106 }
107
108 void
109 MySqlRun::cleanup()
110 {
111 assert(mysql.connected());
112
113 mysql.query("DELETE data "
114 "FROM data "
115 "LEFT JOIN runs ON dt_run=rn_id "
116 "WHERE rn_id IS NULL");
117
118 if (mysql.commit())
119 panic("could not commit transaction\n%s\n", mysql.error);
120
121 mysql.query("DELETE formula_ref "
122 "FROM formula_ref "
123 "LEFT JOIN runs ON fr_run=rn_id "
124 "WHERE rn_id IS NULL");
125
126 if (mysql.commit())
127 panic("could not commit transaction\n%s\n", mysql.error);
128
129 mysql.query("DELETE formulas "
130 "FROM formulas "
131 "LEFT JOIN formula_ref ON fm_stat=fr_stat "
132 "WHERE fr_stat IS NULL");
133
134 if (mysql.commit())
135 panic("could not commit transaction\n%s\n", mysql.error);
136
137 mysql.query("DELETE stats "
138 "FROM stats "
139 "LEFT JOIN data ON st_id=dt_stat "
140 "WHERE dt_stat IS NULL");
141
142 if (mysql.commit())
143 panic("could not commit transaction\n%s\n", mysql.error);
144
145 mysql.query("DELETE subdata "
146 "FROM subdata "
147 "LEFT JOIN data ON sd_stat=dt_stat "
148 "WHERE dt_stat IS NULL");
149
150 if (mysql.commit())
151 panic("could not commit transaction\n%s\n", mysql.error);
152
153 mysql.query("DELETE events"
154 "FROM events"
155 "LEFT JOIN runs ON ev_run=rn_id"
156 "WHERE rn_id IS NULL");
157
158 if (mysql.commit())
159 panic("could not commit transaction\n%s\n", mysql.error);
160
161 mysql.query("DELETE event_names"
162 "FROM event_names"
163 "LEFT JOIN events ON en_id=ev_event"
164 "WHERE ev_event IS NULL");
165
166 if (mysql.commit())
167 panic("could not commit transaction\n%s\n", mysql.error);
168 }
169
170 void
171 SetupStat::init()
172 {
173 name = "";
174 descr = "";
175 type = "";
176 print = false;
177 prereq = 0;
178 prec = -1;
179 nozero = false;
180 nonan = false;
181 total = false;
182 pdf = false;
183 cdf = false;
184 min = 0;
185 max = 0;
186 bktsize = 0;
187 size = 0;
188 }
189
190 unsigned
191 SetupStat::setup()
192 {
193 MySQL::Connection &mysql = run->conn();
194
195 stringstream insert;
196 ccprintf(insert,
197 "INSERT INTO "
198 "stats(st_name, st_descr, st_type, st_print, st_prereq, "
199 "st_prec, st_nozero, st_nonan, st_total, st_pdf, st_cdf, "
200 "st_min, st_max, st_bktsize, st_size)"
201 "values(\"%s\",\"%s\",\"%s\","
202 " %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)",
203 name, descr, type, print, prereq, (int)prec, nozero, nonan,
204 total, pdf, cdf,
205 min, max, bktsize, size);
206
207 mysql.query(insert);
208 if (!mysql.error) {
209 int id = mysql.insert_id();
210 if (mysql.commit())
211 panic("could not commit transaction\n%s\n", mysql.error);
212 return id;
213 }
214
215 stringstream select;
216 ccprintf(select, "SELECT * FROM stats WHERE st_name=\"%s\"", name);
217
218 mysql.query(select);
219 MySQL::Result result = mysql.store_result();
220 if (!result)
221 panic("could not find stat\n%s\n", mysql.error);
222
223 assert(result.num_fields() == 16);
224 MySQL::Row row = result.fetch_row();
225 if (!row)
226 panic("could not get stat row\n%s\n", mysql.error);
227
228 bool tb;
229 int8_t ti8;
230 uint16_t tu16;
231 int64_t ti64;
232 uint64_t tu64;
233
234 if (name != (char *)row[1])
235 panic("failed stat check on %s:name. %s != %s\n",
236 name, name, row[1]);
237
238 if (descr != (char *)row[2])
239 panic("failed stat check on %s:descr. %s != %s\n",
240 name, descr, row[2]);
241
242 if (type != (char *)row[3])
243 panic("failed stat check on %s:type. %s != %s\n",
244 name, type, row[3]);
245
246 if (!to_number(row[4], tb) || print != tb)
247 panic("failed stat check on %s:print. %d != %d\n",
248 name, print, tb);
249
250 if (!to_number(row[6], ti8) || prec != ti8)
251 panic("failed stat check on %s:prec. %d != %d\n",
252 name, prec, ti8);
253
254 if (!to_number(row[7], tb) || nozero != tb)
255 panic("failed stat check on %s:nozero. %d != %d\n",
256 name, nozero, tb);
257
258 if (!to_number(row[8], tb) || nonan != tb)
259 panic("failed stat check on %s:nonan. %d != %d\n",
260 name, nonan, tb);
261
262 if (!to_number(row[9], tb) || total != tb)
263 panic("failed stat check on %s:total. %d != %d\n",
264 name, total, tb);
265
266 if (!to_number(row[10], tb) || pdf != tb)
267 panic("failed stat check on %s:pdf. %d != %d\n",
268 name, pdf, tb);
269
270 if (!to_number(row[11], tb) || cdf != tb)
271 panic("failed stat check on %s:cdf. %d != %d\n",
272 name, cdf, tb);
273
274 if (!to_number(row[12], ti64) || min != ti64)
275 panic("failed stat check on %s:min. %d != %d\n",
276 name, min, ti64);
277
278 if (!to_number(row[13], ti64) || max != ti64)
279 panic("failed stat check on %s:max. %d != %d\n",
280 name, max, ti64);
281
282 if (!to_number(row[14], tu64) || bktsize != tu64)
283 panic("failed stat check on %s:bktsize. %d != %d\n",
284 name, bktsize, tu64);
285
286 if (!to_number(row[15], tu16) || size != tu16)
287 panic("failed stat check on %s:size. %d != %d\n",
288 name, size, tu16);
289
290 to_number(row[5], prereq);
291 uint16_t statid;
292 to_number(row[0], statid);
293 return statid;
294 }
295
296 InsertData::InsertData()
297 {
298 query = new char[maxsize + 1];
299 size = 0;
300 flush();
301 }
302
303 InsertData::~InsertData()
304 {
305 delete [] query;
306 }
307
308 void
309 InsertData::flush()
310 {
311 if (size) {
312 MySQL::Connection &mysql = run->conn();
313 assert(mysql.connected());
314 mysql.query(query);
315 if (mysql.error)
316 panic("could not insert data\n%s\n", mysql.error);
317 if (mysql.commit())
318 panic("could not commit transaction\n%s\n", mysql.error);
319 }
320
321 query[0] = '\0';
322 size = 0;
323 first = true;
324 strcpy(query, "INSERT INTO "
325 "data(dt_stat,dt_x,dt_y,dt_run,dt_tick,dt_data) "
326 "values");
327 size = strlen(query);
328 }
329
330 void
331 InsertData::insert()
332 {
333 if (size + 1024 > maxsize)
334 flush();
335
336 if (!first) {
337 query[size++] = ',';
338 query[size] = '\0';
339 }
340
341 first = false;
342
343 size += sprintf(query + size, "(%u,%d,%d,%u,%llu,\"%f\")",
344 stat, x, y, run->run(), (unsigned long long)tick,
345 data);
346 }
347
348 struct InsertSubData
349 {
350 uint16_t stat;
351 int16_t x;
352 int16_t y;
353 string name;
354 string descr;
355
356 void setup();
357 };
358
359 void
360 InsertSubData::setup()
361 {
362 MySQL::Connection &mysql = run->conn();
363 assert(mysql.connected());
364 stringstream insert;
365 ccprintf(insert,
366 "INSERT INTO subdata(sd_stat,sd_x,sd_y,sd_name,sd_descr) "
367 "values(%d,%d,%d,\"%s\",\"%s\")",
368 stat, x, y, name, descr);
369
370 mysql.query(insert);
371 // if (mysql.error)
372 // panic("could not insert subdata\n%s\n", mysql.error);
373
374 if (mysql.commit())
375 panic("could not commit transaction\n%s\n", mysql.error);
376 }
377
378 void
379 InsertFormula(uint16_t stat, const string &formula)
380 {
381 MySQL::Connection &mysql = run->conn();
382 assert(mysql.connected());
383 stringstream insert_formula;
384 ccprintf(insert_formula,
385 "INSERT INTO formulas(fm_stat,fm_formula) values(%d, \"%s\")",
386 stat, formula);
387
388 mysql.query(insert_formula);
389 // if (mysql.error)
390 // panic("could not insert formula\n%s\n", mysql.error);
391
392 stringstream insert_ref;
393 ccprintf(insert_ref,
394 "INSERT INTO formula_ref(fr_stat,fr_run) values(%d, %d)",
395 stat, run->run());
396
397 mysql.query(insert_ref);
398 // if (mysql.error)
399 // panic("could not insert formula reference\n%s\n", mysql.error);
400
401 if (mysql.commit())
402 panic("could not commit transaction\n%s\n", mysql.error);
403 }
404
405 void
406 UpdatePrereq(uint16_t stat, uint16_t prereq)
407 {
408 MySQL::Connection &mysql = run->conn();
409 assert(mysql.connected());
410 stringstream update;
411 ccprintf(update, "UPDATE stats SET st_prereq=%d WHERE st_id=%d",
412 prereq, stat);
413 mysql.query(update);
414 if (mysql.error)
415 panic("could not update prereq\n%s\n", mysql.error);
416
417 if (mysql.commit())
418 panic("could not commit transaction\n%s\n", mysql.error);
419 }
420
421 MySql::MySql()
422 : run(new MySqlRun)
423 {}
424
425 MySql::~MySql()
426 {
427 delete run;
428 }
429
430 void
431 MySql::connect(const string &host, const string &user, const string &passwd,
432 const string &db, const string &name, const string &sample,
433 const string &project)
434 {
435 run->connect(host, user, passwd, db, name, sample, project);
436 }
437
438 bool
439 MySql::connected() const
440 {
441 run->connected();
442 }
443
444 void
445 MySql::configure()
446 {
447 /*
448 * set up all stats!
449 */
450 using namespace Database;
451
452 MySQL::Connection &mysql = run->conn();
453
454 stat_list_t::const_iterator i, end = stats().end();
455 for (i = stats().begin(); i != end; ++i) {
456 (*i)->visit(*this);
457 }
458
459 for (i = stats().begin(); i != end; ++i) {
460 StatData *data = *i;
461 if (data->prereq) {
462 uint16_t stat_id = find(data->id);
463 uint16_t prereq_id = find(data->prereq->id);
464 assert(stat_id && prereq_id);
465
466 UpdatePrereq(stat_id, prereq_id);
467 }
468 }
469
470 if (mysql.commit())
471 panic("could not commit transaction\n%s\n", mysql.error);
472
473 configured = true;
474 }
475
476
477 bool
478 MySql::configure(const StatData &data, string type)
479 {
480 stat.init();
481 stat.name = data.name;
482 stat.descr = data.desc;
483 stat.type = type;
484 stat.print = data.flags & print;
485 stat.prec = data.precision;
486 stat.nozero = data.flags & nozero;
487 stat.nonan = data.flags & nonan;
488 stat.total = data.flags & total;
489 stat.pdf = data.flags & pdf;
490 stat.cdf = data.flags & cdf;
491
492 return stat.print;
493 }
494
495 void
496 MySql::configure(const ScalarData &data)
497 {
498 if (!configure(data, "SCALAR"))
499 return;
500
501 insert(data.id, stat.setup());
502 }
503
504 void
505 MySql::configure(const VectorData &data)
506 {
507 if (!configure(data, "VECTOR"))
508 return;
509
510 uint16_t statid = stat.setup();
511
512 if (!data.subnames.empty()) {
513 InsertSubData subdata;
514 subdata.stat = statid;
515 subdata.y = 0;
516 for (int i = 0; i < data.subnames.size(); ++i) {
517 subdata.x = i;
518 subdata.name = data.subnames[i];
519 subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];
520
521 if (!subdata.name.empty() || !subdata.descr.empty())
522 subdata.setup();
523 }
524 }
525
526 insert(data.id, statid);
527 }
528
529 void
530 MySql::configure(const DistData &data)
531 {
532 if (!configure(data, "DIST"))
533 return;
534
535 if (!data.data.fancy) {
536 stat.size = data.data.size;
537 stat.min = data.data.min;
538 stat.max = data.data.max;
539 stat.bktsize = data.data.bucket_size;
540 }
541 insert(data.id, stat.setup());
542 }
543
544 void
545 MySql::configure(const VectorDistData &data)
546 {
547 if (!configure(data, "VECTORDIST"))
548 return;
549
550 if (!data.data[0].fancy) {
551 stat.size = data.data[0].size;
552 stat.min = data.data[0].min;
553 stat.max = data.data[0].max;
554 stat.bktsize = data.data[0].bucket_size;
555 }
556
557 uint16_t statid = stat.setup();
558
559 if (!data.subnames.empty()) {
560 InsertSubData subdata;
561 subdata.stat = statid;
562 subdata.y = 0;
563 for (int i = 0; i < data.subnames.size(); ++i) {
564 subdata.x = i;
565 subdata.name = data.subnames[i];
566 subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];
567 if (!subdata.name.empty() || !subdata.descr.empty())
568 subdata.setup();
569 }
570 }
571
572 insert(data.id, statid);
573 }
574
575 void
576 MySql::configure(const Vector2dData &data)
577 {
578 if (!configure(data, "VECTOR2D"))
579 return;
580
581 uint16_t statid = stat.setup();
582
583 if (!data.subnames.empty()) {
584 InsertSubData subdata;
585 subdata.stat = statid;
586 subdata.y = -1;
587 for (int i = 0; i < data.subnames.size(); ++i) {
588 subdata.x = i;
589 subdata.name = data.subnames[i];
590 subdata.descr = data.subdescs.empty() ? "" : data.subdescs[i];
591 if (!subdata.name.empty() || !subdata.descr.empty())
592 subdata.setup();
593 }
594 }
595
596 if (!data.y_subnames.empty()) {
597 InsertSubData subdata;
598 subdata.stat = statid;
599 subdata.x = -1;
600 subdata.descr = "";
601 for (int i = 0; i < data.y_subnames.size(); ++i) {
602 subdata.y = i;
603 subdata.name = data.y_subnames[i];
604 if (!subdata.name.empty())
605 subdata.setup();
606 }
607 }
608
609 insert(data.id, statid);
610 }
611
612 void
613 MySql::configure(const FormulaData &data)
614 {
615 configure(data, "FORMULA");
616 insert(data.id, stat.setup());
617 InsertFormula(find(data.id), data.str());
618 }
619
620 bool
621 MySql::valid() const
622 {
623 return run->connected();
624 }
625
626 void
627 MySql::output()
628 {
629 using namespace Database;
630 assert(valid());
631
632 if (!configured)
633 configure();
634
635 // store sample #
636 newdata.tick = curTick;
637
638 MySQL::Connection &mysql = run->conn();
639
640 Database::stat_list_t::const_iterator i, end = Database::stats().end();
641 for (i = Database::stats().begin(); i != end; ++i) {
642 StatData *stat = *i;
643 stat->visit(*this);
644 if (mysql.commit())
645 panic("could not commit transaction\n%s\n", mysql.error);
646 }
647
648 newdata.flush();
649 }
650
651 void
652 MySql::output(const ScalarData &data)
653 {
654 if (!(data.flags & print))
655 return;
656
657 newdata.stat = find(data.id);
658 newdata.x = 0;
659 newdata.y = 0;
660 newdata.data = data.value();
661
662 newdata.insert();
663 }
664
665 void
666 MySql::output(const VectorData &data)
667 {
668 if (!(data.flags & print))
669 return;
670
671 newdata.stat = find(data.id);
672 newdata.y = 0;
673
674 const VCounter &cvec = data.value();
675 int size = data.size();
676 for (int x = 0; x < size; x++) {
677 newdata.x = x;
678 newdata.data = cvec[x];
679 newdata.insert();
680 }
681 }
682
683 void
684 MySql::output(const DistDataData &data)
685 {
686 const int db_sum = -1;
687 const int db_squares = -2;
688 const int db_samples = -3;
689 const int db_min_val = -4;
690 const int db_max_val = -5;
691 const int db_underflow = -6;
692 const int db_overflow = -7;
693
694 newdata.x = db_sum;
695 newdata.data = data.sum;
696 newdata.insert();
697
698 newdata.x = db_squares;
699 newdata.data = data.squares;
700 newdata.insert();
701
702 newdata.x = db_samples;
703 newdata.data = data.samples;
704 newdata.insert();
705
706 if (data.samples && !data.fancy) {
707 newdata.x = db_min_val;
708 newdata.data = data.min_val;
709 newdata.insert();
710
711 newdata.x = db_max_val;
712 newdata.data = data.max_val;
713 newdata.insert();
714
715 newdata.x = db_underflow;
716 newdata.data = data.underflow;
717 newdata.insert();
718
719 newdata.x = db_overflow;
720 newdata.data = data.overflow;
721 newdata.insert();
722
723 int size = data.cvec.size();
724 for (int x = 0; x < size; x++) {
725 newdata.x = x;
726 newdata.data = data.cvec[x];
727 newdata.insert();
728 }
729 }
730 }
731
732
733 void
734 MySql::output(const DistData &data)
735 {
736 if (!(data.flags & print))
737 return;
738
739 newdata.stat = find(data.id);
740 newdata.y = 0;
741 output(data.data);
742 }
743
744 void
745 MySql::output(const VectorDistData &data)
746 {
747 if (!(data.flags & print))
748 return;
749
750 newdata.stat = find(data.id);
751
752 int size = data.data.size();
753 for (int y = 0; y < size; ++y) {
754 newdata.y = y;
755 output(data.data[y]);
756 }
757 }
758
759 void
760 MySql::output(const Vector2dData &data)
761 {
762 if (!(data.flags & print))
763 return;
764
765 newdata.stat = find(data.id);
766
767 int index = 0;
768 for (int x = 0; x < data.x; x++) {
769 newdata.x = x;
770 for (int y = 0; y < data.y; y++) {
771 newdata.y = y;
772 newdata.data = data.cvec[index++];
773 newdata.insert();
774 }
775 }
776 }
777
778 void
779 MySql::output(const FormulaData &data)
780 {
781 }
782
783 /*
784 * Implement the visitor
785 */
786 void
787 MySql::visit(const ScalarData &data)
788 {
789 if (!configured)
790 configure(data);
791 else
792 output(data);
793 }
794
795 void
796 MySql::visit(const VectorData &data)
797 {
798 if (!configured)
799 configure(data);
800 else
801 output(data);
802 }
803
804 void
805 MySql::visit(const DistData &data)
806 {
807 return;
808 if (!configured)
809 configure(data);
810 else
811 output(data);
812 }
813
814 void
815 MySql::visit(const VectorDistData &data)
816 {
817 return;
818 if (!configured)
819 configure(data);
820 else
821 output(data);
822 }
823
824 void
825 MySql::visit(const Vector2dData &data)
826 {
827 return;
828 if (!configured)
829 configure(data);
830 else
831 output(data);
832 }
833
834 void
835 MySql::visit(const FormulaData &data)
836 {
837 if (!configured)
838 configure(data);
839 else
840 output(data);
841 }
842
843 bool
844 initMySQL(string host, string user, string password, string database,
845 string name, string sample, string project)
846 {
847 extern list<Output *> OutputList;
848 static MySql mysql;
849
850 if (mysql.connected())
851 return false;
852
853 if (user.empty())
854 user = username();
855
856 mysql.connect(host, user, password, database, name, sample, project);
857 OutputList.push_back(&mysql);
858
859 return true;
860 }