includes: sort all includes
[gem5.git] / src / base / stats / mysql.cc
1 /*
2 * Copyright (c) 2004-2005 The Regents of The University of Michigan
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are
7 * met: redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer;
9 * redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution;
12 * neither the name of the copyright holders nor the names of its
13 * contributors may be used to endorse or promote products derived from
14 * this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * Authors: Nathan Binkert
29 */
30
31 #include <cassert>
32 #include <cstdio>
33 #include <map>
34 #include <sstream>
35 #include <string>
36 #include <vector>
37
38 #include "base/stats/info.hh"
39 #include "base/stats/mysql.hh"
40 #include "base/stats/mysql_run.hh"
41 #include "base/stats/types.hh"
42 #include "base/misc.hh"
43 #include "base/mysql.hh"
44 #include "base/statistics.hh"
45 #include "base/str.hh"
46 #include "base/types.hh"
47 #include "base/userinfo.hh"
48 #include "sim/core.hh"
49
50 using namespace std;
51
52 namespace Stats {
53
54 void
55 MySqlRun::connect(const string &host, const string &user, const string &passwd,
56 const string &db, const string &name, const string &sample,
57 const string &project)
58 {
59 if (connected())
60 panic("can only get one database connection at this time!");
61
62 mysql.connect(host, user, passwd, db);
63 if (mysql.error)
64 panic("could not connect to database server\n%s\n", mysql.error);
65
66 if (mysql.autocommit(false))
67 panic("could not set autocommit\n%s\n", mysql.error);
68
69 remove(name);
70 //cleanup();
71 setup(name, sample, user, project);
72 }
73
74 void
75 MySqlRun::setup(const string &name, const string &sample, const string &user,
76 const string &project)
77 {
78 assert(mysql.connected());
79
80 stringstream insert;
81 ccprintf(insert,
82 "INSERT INTO "
83 "runs(rn_name,rn_sample,rn_user,rn_project,rn_date,rn_expire)"
84 "values(\"%s\", \"%s\", \"%s\", \"%s\", NOW(),"
85 "DATE_ADD(CURDATE(), INTERVAL 31 DAY))",
86 name, sample, user, project);
87
88 mysql.query(insert);
89 if (mysql.error)
90 panic("could not get a run\n%s\n", mysql.error);
91
92 run_id = mysql.insert_id();
93 if (mysql.commit())
94 panic("could not commit transaction\n%s\n", mysql.error);
95 }
96
97 void
98 MySqlRun::remove(const string &name)
99 {
100 assert(mysql.connected());
101 stringstream sql;
102 ccprintf(sql, "DELETE FROM runs WHERE rn_name=\"%s\"", name);
103 mysql.query(sql);
104 if (mysql.error)
105 panic("could not delete run\n%s\n", mysql.error);
106 if (mysql.commit())
107 panic("could not commit transaction\n%s\n", mysql.error);
108 }
109
110 void
111 MySqlRun::cleanup()
112 {
113 assert(mysql.connected());
114
115 mysql.query("DELETE data "
116 "FROM data "
117 "LEFT JOIN runs ON dt_run=rn_id "
118 "WHERE rn_id IS NULL");
119
120 if (mysql.commit())
121 panic("could not commit transaction\n%s\n", mysql.error);
122
123 mysql.query("DELETE formula_ref "
124 "FROM formula_ref "
125 "LEFT JOIN runs ON fr_run=rn_id "
126 "WHERE rn_id IS NULL");
127
128 if (mysql.commit())
129 panic("could not commit transaction\n%s\n", mysql.error);
130
131 mysql.query("DELETE formulas "
132 "FROM formulas "
133 "LEFT JOIN formula_ref ON fm_stat=fr_stat "
134 "WHERE fr_stat IS NULL");
135
136 if (mysql.commit())
137 panic("could not commit transaction\n%s\n", mysql.error);
138
139 mysql.query("DELETE stats "
140 "FROM stats "
141 "LEFT JOIN data ON st_id=dt_stat "
142 "WHERE dt_stat IS NULL");
143
144 if (mysql.commit())
145 panic("could not commit transaction\n%s\n", mysql.error);
146
147 mysql.query("DELETE subdata "
148 "FROM subdata "
149 "LEFT JOIN data ON sd_stat=dt_stat "
150 "WHERE dt_stat IS NULL");
151
152 if (mysql.commit())
153 panic("could not commit transaction\n%s\n", mysql.error);
154 }
155
156 void
157 SetupStat::init()
158 {
159 name = "";
160 descr = "";
161 type = "";
162 print = false;
163 prereq = 0;
164 prec = -1;
165 nozero = false;
166 nonan = false;
167 total = false;
168 pdf = false;
169 cdf = false;
170 min = 0;
171 max = 0;
172 bktsize = 0;
173 size = 0;
174 }
175
176 unsigned
177 SetupStat::setup(MySqlRun *run)
178 {
179 MySQL::Connection &mysql = run->conn();
180
181 stringstream insert;
182 ccprintf(insert,
183 "INSERT INTO "
184 "stats(st_name, st_descr, st_type, st_print, st_prereq, "
185 "st_prec, st_nozero, st_nonan, st_total, st_pdf, st_cdf, "
186 "st_min, st_max, st_bktsize, st_size)"
187 "values(\"%s\",\"%s\",\"%s\","
188 " %d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)",
189 name, descr, type, print, prereq, (int)prec, nozero, nonan,
190 total, pdf, cdf,
191 min, max, bktsize, size);
192
193 mysql.query(insert);
194 if (!mysql.error) {
195 int id = mysql.insert_id();
196 if (mysql.commit())
197 panic("could not commit transaction\n%s\n", mysql.error);
198 return id;
199 }
200
201 stringstream select;
202 ccprintf(select, "SELECT * FROM stats WHERE st_name=\"%s\"", name);
203
204 mysql.query(select);
205 MySQL::Result result = mysql.store_result();
206 if (!result)
207 panic("could not find stat\n%s\n", mysql.error);
208
209 assert(result.num_fields() == 16);
210 MySQL::Row row = result.fetch_row();
211 if (!row)
212 panic("could not get stat row\n%s\n", mysql.error);
213
214 bool tb;
215 int8_t ti8;
216 uint16_t tu16;
217 int64_t ti64;
218 uint64_t tu64;
219
220 if (name != (char *)row[1])
221 panic("failed stat check on %s:name. %s != %s\n",
222 name, name, row[1]);
223
224 if (descr != (char *)row[2])
225 panic("failed stat check on %s:descr. %s != %s\n",
226 name, descr, row[2]);
227
228 if (type != (char *)row[3])
229 panic("failed stat check on %s:type. %s != %s\n",
230 name, type, row[3]);
231
232 if (!to_number(row[4], tb) || print != tb)
233 panic("failed stat check on %s:print. %d != %d\n",
234 name, print, tb);
235
236 if (!to_number(row[6], ti8) || prec != ti8)
237 panic("failed stat check on %s:prec. %d != %d\n",
238 name, prec, ti8);
239
240 if (!to_number(row[7], tb) || nozero != tb)
241 panic("failed stat check on %s:nozero. %d != %d\n",
242 name, nozero, tb);
243
244 if (!to_number(row[8], tb) || nonan != tb)
245 panic("failed stat check on %s:nonan. %d != %d\n",
246 name, nonan, tb);
247
248 if (!to_number(row[9], tb) || total != tb)
249 panic("failed stat check on %s:total. %d != %d\n",
250 name, total, tb);
251
252 if (!to_number(row[10], tb) || pdf != tb)
253 panic("failed stat check on %s:pdf. %d != %d\n",
254 name, pdf, tb);
255
256 if (!to_number(row[11], tb) || cdf != tb)
257 panic("failed stat check on %s:cdf. %d != %d\n",
258 name, cdf, tb);
259
260 if (!to_number(row[12], ti64) || min != ti64)
261 panic("failed stat check on %s:min. %d != %d\n",
262 name, min, ti64);
263
264 if (!to_number(row[13], ti64) || max != ti64)
265 panic("failed stat check on %s:max. %d != %d\n",
266 name, max, ti64);
267
268 if (!to_number(row[14], tu64) || bktsize != tu64)
269 panic("failed stat check on %s:bktsize. %d != %d\n",
270 name, bktsize, tu64);
271
272 if (!to_number(row[15], tu16) || size != tu16)
273 panic("failed stat check on %s:size. %d != %d\n",
274 name, size, tu16);
275
276 to_number(row[5], prereq);
277 uint16_t statid;
278 to_number(row[0], statid);
279 return statid;
280 }
281
282 InsertData::InsertData(MySqlRun *_run)
283 : run(_run)
284 {
285 query = new char[maxsize + 1];
286 size = 0;
287 flush();
288 }
289
290 InsertData::~InsertData()
291 {
292 delete [] query;
293 }
294
295 void
296 InsertData::flush()
297 {
298 if (size) {
299 MySQL::Connection &mysql = run->conn();
300 assert(mysql.connected());
301 mysql.query(query);
302 if (mysql.error)
303 panic("could not insert data\n%s\n", mysql.error);
304 if (mysql.commit())
305 panic("could not commit transaction\n%s\n", mysql.error);
306 }
307
308 query[0] = '\0';
309 size = 0;
310 first = true;
311 strcpy(query, "INSERT INTO "
312 "data(dt_stat,dt_x,dt_y,dt_run,dt_tick,dt_data) "
313 "values");
314 size = strlen(query);
315 }
316
317 void
318 InsertData::insert()
319 {
320 if (size + 1024 > maxsize)
321 flush();
322
323 if (!first) {
324 query[size++] = ',';
325 query[size] = '\0';
326 }
327
328 first = false;
329
330 size += sprintf(query + size, "(%u,%d,%d,%u,%llu,\"%f\")",
331 stat, x, y, run->run(), (unsigned long long)tick,
332 data);
333 }
334
335 struct InsertSubData
336 {
337 uint16_t stat;
338 int16_t x;
339 int16_t y;
340 string name;
341 string descr;
342
343 void setup(MySqlRun *run);
344 };
345
346 void
347 InsertSubData::setup(MySqlRun *run)
348 {
349 MySQL::Connection &mysql = run->conn();
350 assert(mysql.connected());
351 stringstream insert;
352 ccprintf(insert,
353 "INSERT INTO subdata(sd_stat,sd_x,sd_y,sd_name,sd_descr) "
354 "values(%d,%d,%d,\"%s\",\"%s\")",
355 stat, x, y, name, descr);
356
357 mysql.query(insert);
358 // if (mysql.error)
359 // panic("could not insert subdata\n%s\n", mysql.error);
360
361 if (mysql.commit())
362 panic("could not commit transaction\n%s\n", mysql.error);
363 }
364
365 MySql::MySql()
366 : run(new MySqlRun), newdata(run)
367 {}
368
369 MySql::~MySql()
370 {
371 delete run;
372 }
373
374 void
375 MySql::connect(const string &host, const string &user, const string &passwd,
376 const string &db, const string &name, const string &sample,
377 const string &project)
378 {
379 run->connect(host, user, passwd, db, name, sample, project);
380 }
381
382 bool
383 MySql::connected() const
384 {
385 return run->connected();
386 }
387
388 void
389 MySql::configure()
390 {
391 /*
392 * set up all stats!
393 */
394 MySQL::Connection &mysql = run->conn();
395
396 list<Info *>::const_iterator i, end = statsList().end();
397 for (i = statsList().begin(); i != end; ++i) {
398 (*i)->visit(*this);
399 }
400
401 for (i = statsList().begin(); i != end; ++i) {
402 Info *info = *i;
403 if (info->prereq) {
404 // update the prerequisite
405 uint16_t stat_id = find(info->id);
406 uint16_t prereq_id = find(info->prereq->id);
407 assert(stat_id && prereq_id);
408
409 stringstream update;
410 ccprintf(update, "UPDATE stats SET st_prereq=%d WHERE st_id=%d",
411 prereq_id, stat_id);
412 mysql.query(update);
413 if (mysql.error)
414 panic("could not update prereq\n%s\n", mysql.error);
415
416 if (mysql.commit())
417 panic("could not commit transaction\n%s\n", mysql.error);
418 }
419 }
420
421 if (mysql.commit())
422 panic("could not commit transaction\n%s\n", mysql.error);
423
424 configured = true;
425 }
426
427 bool
428 MySql::configure(const Info &info, string type)
429 {
430 stat.init();
431 stat.name = info.name;
432 stat.descr = info.desc;
433 stat.type = type;
434 stat.print = info.flags & display;
435 stat.prec = info.precision;
436 stat.nozero = info.flags & nozero;
437 stat.nonan = info.flags & nonan;
438 stat.total = info.flags & total;
439 stat.pdf = info.flags & pdf;
440 stat.cdf = info.flags & cdf;
441
442 return stat.print;
443 }
444
445 void
446 MySql::configure(const ScalarInfo &info)
447 {
448 if (!configure(info, "SCALAR"))
449 return;
450
451 insert(info.id, stat.setup(run));
452 }
453
454 void
455 MySql::configure(const VectorInfo &info)
456 {
457 if (!configure(info, "VECTOR"))
458 return;
459
460 uint16_t statid = stat.setup(run);
461
462 if (!info.subnames.empty()) {
463 InsertSubData subdata;
464 subdata.stat = statid;
465 subdata.y = 0;
466 for (off_type i = 0; i < info.subnames.size(); ++i) {
467 subdata.x = i;
468 subdata.name = info.subnames[i];
469 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
470
471 if (!subdata.name.empty() || !subdata.descr.empty())
472 subdata.setup(run);
473 }
474 }
475
476 insert(info.id, statid);
477 }
478
479 void
480 MySql::configure(const DistInfo &info)
481 {
482 if (!configure(info, "DIST"))
483 return;
484
485 const DistStor::Params *params =
486 dynamic_cast<const DistStor::Params *>(info.storageParams);
487 if (params) {
488 assert(params->type == Dist);
489 stat.size = params->buckets;
490 stat.min = params->min;
491 stat.max = params->max;
492 stat.bktsize = params->bucket_size;
493 }
494 insert(info.id, stat.setup(run));
495 }
496
497 void
498 MySql::configure(const VectorDistInfo &info)
499 {
500 if (!configure(info, "VECTORDIST"))
501 return;
502
503 const DistStor::Params *params =
504 dynamic_cast<const DistStor::Params *>(info.storageParams);
505 if (params) {
506 assert(params->type == Dist);
507 stat.size = params->buckets;
508 stat.min = params->min;
509 stat.max = params->max;
510 stat.bktsize = params->bucket_size;
511 }
512
513 uint16_t statid = stat.setup(run);
514
515 if (!info.subnames.empty()) {
516 InsertSubData subdata;
517 subdata.stat = statid;
518 subdata.y = 0;
519 for (off_type i = 0; i < info.subnames.size(); ++i) {
520 subdata.x = i;
521 subdata.name = info.subnames[i];
522 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
523 if (!subdata.name.empty() || !subdata.descr.empty())
524 subdata.setup(run);
525 }
526 }
527
528 insert(info.id, statid);
529 }
530
531 void
532 MySql::configure(const Vector2dInfo &info)
533 {
534 if (!configure(info, "VECTOR2D"))
535 return;
536
537 uint16_t statid = stat.setup(run);
538
539 if (!info.subnames.empty()) {
540 InsertSubData subdata;
541 subdata.stat = statid;
542 subdata.y = -1;
543 for (off_type i = 0; i < info.subnames.size(); ++i) {
544 subdata.x = i;
545 subdata.name = info.subnames[i];
546 subdata.descr = info.subdescs.empty() ? "" : info.subdescs[i];
547 if (!subdata.name.empty() || !subdata.descr.empty())
548 subdata.setup(run);
549 }
550 }
551
552 if (!info.y_subnames.empty()) {
553 InsertSubData subdata;
554 subdata.stat = statid;
555 subdata.x = -1;
556 subdata.descr = "";
557 for (off_type i = 0; i < info.y_subnames.size(); ++i) {
558 subdata.y = i;
559 subdata.name = info.y_subnames[i];
560 if (!subdata.name.empty())
561 subdata.setup(run);
562 }
563 }
564
565 insert(info.id, statid);
566 }
567
568 void
569 MySql::configure(const FormulaInfo &info)
570 {
571 MySQL::Connection &mysql = run->conn();
572 assert(mysql.connected());
573
574 configure(info, "FORMULA");
575 insert(info.id, stat.setup(run));
576
577 uint16_t stat = find(info.id);
578 string formula = info.str();
579
580 stringstream insert_formula;
581 ccprintf(insert_formula,
582 "INSERT INTO formulas(fm_stat,fm_formula) values(%d, \"%s\")",
583 stat, formula);
584
585 mysql.query(insert_formula);
586 // if (mysql.error)
587 // panic("could not insert formula\n%s\n", mysql.error);
588
589 stringstream insert_ref;
590 ccprintf(insert_ref,
591 "INSERT INTO formula_ref(fr_stat,fr_run) values(%d, %d)",
592 stat, run->run());
593
594 mysql.query(insert_ref);
595 // if (mysql.error)
596 // panic("could not insert formula reference\n%s\n", mysql.error);
597
598 if (mysql.commit())
599 panic("could not commit transaction\n%s\n", mysql.error);
600 }
601
602 bool
603 MySql::valid() const
604 {
605 return run->connected();
606 }
607
608 void
609 MySql::output()
610 {
611 assert(valid());
612
613 if (!configured)
614 configure();
615
616 // store sample #
617 newdata.tick = curTick();
618
619 MySQL::Connection &mysql = run->conn();
620
621 list<Info *>::const_iterator i, end = statsList().end();
622 for (i = statsList().begin(); i != end; ++i) {
623 Info *stat = *i;
624 stat->visit(*this);
625 if (mysql.commit())
626 panic("could not commit transaction\n%s\n", mysql.error);
627 }
628
629 newdata.flush();
630 }
631
632 void
633 MySql::output(const ScalarInfo &info)
634 {
635 if (!(info.flags & display))
636 return;
637
638 newdata.stat = find(info.id);
639 newdata.x = 0;
640 newdata.y = 0;
641 newdata.data = info.value();
642
643 newdata.insert();
644 }
645
646 void
647 MySql::output(const VectorInfo &info)
648 {
649 if (!(info.flags & display))
650 return;
651
652 newdata.stat = find(info.id);
653 newdata.y = 0;
654
655 const VCounter &cvec = info.value();
656 size_type size = info.size();
657 for (off_type x = 0; x < size; x++) {
658 newdata.x = x;
659 newdata.data = cvec[x];
660 newdata.insert();
661 }
662 }
663
664 void
665 MySql::output(const DistData &data, const DistParams *params)
666 {
667 const int db_sum = -1;
668 const int db_squares = -2;
669 const int db_samples = -3;
670 const int db_min_val = -4;
671 const int db_max_val = -5;
672 const int db_underflow = -6;
673 const int db_overflow = -7;
674
675 newdata.x = db_sum;
676 newdata.data = data.sum;
677 newdata.insert();
678
679 newdata.x = db_squares;
680 newdata.data = data.squares;
681 newdata.insert();
682
683 newdata.x = db_samples;
684 newdata.data = data.samples;
685 newdata.insert();
686
687 if (data.samples && params->type == Dist) {
688 newdata.x = db_min_val;
689 newdata.data = data.min_val;
690 newdata.insert();
691
692 newdata.x = db_max_val;
693 newdata.data = data.max_val;
694 newdata.insert();
695
696 newdata.x = db_underflow;
697 newdata.data = data.underflow;
698 newdata.insert();
699
700 newdata.x = db_overflow;
701 newdata.data = data.overflow;
702 newdata.insert();
703
704 size_type size = data.cvec.size();
705 for (off_type x = 0; x < size; x++) {
706 newdata.x = x;
707 newdata.data = data.cvec[x];
708 newdata.insert();
709 }
710 }
711 }
712
713 void
714 MySql::output(const DistInfo &info)
715 {
716 if (!(info.flags & display))
717 return;
718
719 newdata.stat = find(info.id);
720 newdata.y = 0;
721 output(info.data, safe_cast<const DistParams *>(info.storageParams));
722 }
723
724 void
725 MySql::output(const VectorDistInfo &info)
726 {
727 if (!(info.flags & display))
728 return;
729
730 newdata.stat = find(info.id);
731
732 size_type size = info.data.size();
733 for (off_type y = 0; y < size; ++y) {
734 newdata.y = y;
735 output(info.data[y],
736 safe_cast<const DistParams *>(info.storageParams));
737 }
738 }
739
740 void
741 MySql::output(const Vector2dInfo &info)
742 {
743 if (!(info.flags & display))
744 return;
745
746 newdata.stat = find(info.id);
747
748 off_type index = 0;
749 for (off_type x = 0; x < info.x; x++) {
750 newdata.x = x;
751 for (off_type y = 0; y < info.y; y++) {
752 newdata.y = y;
753 newdata.data = info.cvec[index++];
754 newdata.insert();
755 }
756 }
757 }
758
759 void
760 MySql::output(const FormulaInfo &info)
761 {
762 }
763
764 /*
765 * Implement the visitor
766 */
767 void
768 MySql::visit(const ScalarInfo &info)
769 {
770 if (!configured)
771 configure(info);
772 else
773 output(info);
774 }
775
776 void
777 MySql::visit(const VectorInfo &info)
778 {
779 if (!configured)
780 configure(info);
781 else
782 output(info);
783 }
784
785 void
786 MySql::visit(const DistInfo &info)
787 {
788 return;
789 if (!configured)
790 configure(info);
791 else
792 output(info);
793 }
794
795 void
796 MySql::visit(const VectorDistInfo &info)
797 {
798 return;
799 if (!configured)
800 configure(info);
801 else
802 output(info);
803 }
804
805 void
806 MySql::visit(const Vector2dInfo &info)
807 {
808 return;
809 if (!configured)
810 configure(info);
811 else
812 output(info);
813 }
814
815 void
816 MySql::visit(const FormulaInfo &info)
817 {
818 if (!configured)
819 configure(info);
820 else
821 output(info);
822 }
823
824 bool
825 initMySQL(string host, string user, string password, string database,
826 string project, string name, string sample)
827 {
828 extern list<Output *> OutputList;
829 static MySql mysql;
830
831 if (mysql.connected())
832 return false;
833
834 mysql.connect(host, user, password, database, name, sample, project);
835 OutputList.push_back(&mysql);
836
837 return true;
838 }
839
840 } // namespace Stats