1 /** 2 Copyright: Copyright (c) 2018, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 This module contains the schema to initialize the database. 11 12 To ensure that the upgrade path for a database always work a database is 13 created at the "lowest supported" and upgraded to the latest. 14 15 # How to add schema change 16 17 1. add an upgrade function, upgradeVX. 18 19 The function makeUpgradeTable will then automatically find it and use it. X 20 **must** be the version upgrading FROM. 21 22 # Style 23 A database schema upgrade path shall have a comment stating what date it was added. 24 Each change to the database schema must have an equal upgrade added. 25 26 # Sqlite3 27 From the sqlite3 manual $(LINK https://www.sqlite.org/datatype3.html): 28 Each value stored in an SQLite database (or manipulated by the database 29 engine) has one of the following storage classes: 30 31 NULL. The value is a NULL value. 32 33 INTEGER. The value is a signed integer, stored in 1, 2, 3, 4, 6, or 8 bytes 34 depending on the magnitude of the value. 35 36 REAL. The value is a floating point value, stored as an 8-byte IEEE floating 37 point number. 38 39 TEXT. The value is a text string, stored using the database encoding (UTF-8, 40 UTF-16BE or UTF-16LE). 41 42 BLOB. The value is a blob of data, stored exactly as it was input. 43 44 A storage class is more general than a datatype. The INTEGER storage class, for 45 example, includes 6 different integer datatypes of different lengths. This 46 makes a difference on disk. But as soon as INTEGER values are read off of disk 47 and into memory for processing, they are converted to the most general datatype 48 (8-byte signed integer). And so for the most part, "storage class" is 49 indistinguishable from "datatype" and the two terms can be used 50 interchangeably. 51 52 # ON DELETE CASCADE 53 54 when a ON DELETE CASCADE is added an index should be created too of the childs 55 foreign key. 56 57 From the sqlite documentation: 58 59 Indices are not required for child key columns but they are almost always 60 beneficial. […] 61 62 Each time an application deletes a row from the ... parent table, it performs 63 [a query] to search for referencing rows in the ... child table. 64 65 If this query returns any rows at all, then SQLite concludes that deleting the 66 row from the parent table would violate the foreign key constraint and returns 67 an error. Similar queries may be run if the content of the parent key is 68 modified or a new row is inserted into the parent table. If these queries 69 cannot use an index, they are forced to do a linear scan of the entire child 70 table. In a non-trivial database, this may be prohibitively expensive. 71 72 So, in most real systems, an index should be created on the child key columns 73 of each foreign key constraint. The child key index does not have to be (and 74 usually will not be) a UNIQUE index. 75 76 */ 77 module dextool.plugin.mutate.backend.database.schema; 78 79 import logger = std.experimental.logger; 80 import std.array : array, empty; 81 import std.datetime : SysTime, dur, Clock; 82 import std.exception : collectException; 83 import std.format : format; 84 85 import dextool.plugin.mutate.backend.type : Language; 86 87 import d2sqlite3 : SqlDatabase = Database; 88 import miniorm : Miniorm, TableName, buildSchema, ColumnParam, TableForeignKey, TableConstraint, 89 TablePrimaryKey, KeyRef, KeyParam, ColumnName, delete_, insert, select, spinSql, silentLog; 90 91 immutable allTestCaseTable = "all_test_case"; 92 immutable configVersionTable = "config_version"; 93 immutable depFileTable = "dependency_file"; 94 immutable depRootTable = "rel_dependency_root"; 95 immutable dextoolVersionTable = "dextool_version"; 96 immutable filesTable = "files"; 97 immutable killedTestCaseTable = "killed_test_case"; 98 immutable markedMutantTable = "marked_mutant"; 99 immutable mutantMemOverloadWorklistTable = "mutant_memoverload_worklist"; 100 immutable mutantTimeoutCtxTable = "mutant_timeout_ctx"; 101 immutable mutantTimeoutWorklistTable = "mutant_timeout_worklist"; 102 immutable mutantWorklistTable = "mutant_worklist"; 103 immutable mutationFileScoreHistoryTable = "mutation_file_score_history"; 104 immutable mutationPointTable = "mutation_point"; 105 immutable mutationScoreHistoryTable = "mutation_score_history"; 106 immutable mutationStatusTable = "mutation_status"; 107 immutable mutationTable = "mutation"; 108 immutable nomutDataTable = "nomut_data"; 109 immutable nomutTable = "nomut"; 110 immutable rawSrcMetadataTable = "raw_src_metadata"; 111 immutable runtimeHistoryTable = "test_cmd_runtime_history"; 112 immutable schemaFragmentV2Table = "schema_fragment_v2"; 113 immutable schemaMutantQTable = "schema_mutant_q"; 114 immutable schemaMutantV2Table = "schemata_mutant_v2"; 115 immutable schemaSizeQTable = "schema_size_q"; 116 immutable schemaVersionTable = "schema_version"; 117 immutable srcCovInfoTable = "src_cov_info"; 118 immutable srcCovTable = "src_cov_instr"; 119 immutable srcCovTimeStampTable = "src_cov_timestamp"; 120 immutable srcMetadataTable = "src_metadata"; 121 immutable testCmdMutatedTable = "test_cmd_mutated"; 122 immutable testCmdOriginalTable = "test_cmd_original"; 123 immutable testCmdRelMutantTable = "test_cmd_rel_mutant"; 124 immutable testCmdTable = "test_cmd"; 125 immutable testFilesTable = "test_files"; 126 127 private immutable schemataTable = "schemata"; 128 private immutable schemataMutantTable = "schemata_mutant"; 129 private immutable schemataUsedTable = "schemata_used"; 130 private immutable invalidSchemataTable = "invalid_schemata"; 131 private immutable schemataWorkListTable = "schemata_worklist"; 132 private immutable testCaseTableV1 = "test_case"; 133 private immutable schemataFragmentTable = "schemata_fragment"; 134 135 /** Initialize or open an existing database. 136 * 137 * Params: 138 * p = path where to initialize a new database or open an existing 139 * 140 * Returns: an open sqlite3 database object. 141 */ 142 Miniorm initializeDB(const string p) @trusted 143 in { 144 assert(p.length != 0); 145 } 146 do { 147 import std.file : exists; 148 import my.file : followSymlink; 149 import my.optional; 150 import my.path : Path; 151 import d2sqlite3 : SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE; 152 153 static void setPragmas(ref SqlDatabase db) { 154 // dfmt off 155 auto pragmas = [ 156 // required for foreign keys with cascade to work 157 "PRAGMA foreign_keys=ON;", 158 ]; 159 // dfmt on 160 161 foreach (p; pragmas) { 162 db.run(p); 163 } 164 } 165 166 const isOldDb = exists(followSymlink(Path(p)).orElse(Path(p)).toString); 167 SqlDatabase sqliteDb; 168 scope (success) 169 setPragmas(sqliteDb); 170 171 logger.trace("Opening database ", p); 172 try { 173 sqliteDb = SqlDatabase(p, SQLITE_OPEN_READWRITE); 174 } catch (Exception e) { 175 logger.trace(e.msg); 176 logger.trace("Initializing a new sqlite3 database"); 177 sqliteDb = SqlDatabase(p, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE); 178 } 179 180 auto db = Miniorm(sqliteDb); 181 182 auto tbl = makeUpgradeTable; 183 const longTimeout = 10.dur!"minutes"; 184 try { 185 if (isOldDb && spinSql!(() => getSchemaVersion(db), 186 silentLog)(10.dur!"seconds") >= tbl.latestSchemaVersion) 187 return db; 188 } catch (Exception e) { 189 logger.info("The database is probably locked. Will keep trying to open for ", longTimeout); 190 } 191 if (isOldDb && spinSql!(() => getSchemaVersion(db))(longTimeout) >= tbl.latestSchemaVersion) 192 return db; 193 194 // TODO: remove all key off in upgrade schemas. 195 const giveUpAfter = Clock.currTime + longTimeout; 196 bool failed = true; 197 while (failed && Clock.currTime < giveUpAfter) { 198 try { 199 auto trans = db.transaction; 200 db.run("PRAGMA foreign_keys=OFF;"); 201 upgrade(db, tbl); 202 trans.commit; 203 failed = false; 204 } catch (Exception e) { 205 logger.trace(e.msg); 206 } 207 } 208 209 if (failed) { 210 logger.error("Unable to upgrade the database to the latest schema"); 211 throw new Exception(null); 212 } 213 214 return db; 215 } 216 217 package: 218 219 // metadata about mutants that occur on a line extracted from the source code. 220 // It is intended to further refined. 221 // nomut = if the line should ignore mutants. 222 // tag = a user defined tag for a NOMUT. 223 // comment = a user defined comment. 224 @TableName(rawSrcMetadataTable) 225 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 226 @TableConstraint("unique_line_in_file UNIQUE (file_id, line)") 227 struct RawSrcMetadata { 228 long id; 229 230 @ColumnName("file_id") 231 long fileId; 232 233 @ColumnParam("") 234 uint line; 235 236 @ColumnParam("") 237 long nomut; 238 239 @ColumnParam("") 240 string tag; 241 242 @ColumnParam("") 243 string comment; 244 } 245 246 @TableName(srcMetadataTable) 247 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE")) 248 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 249 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 250 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 251 struct SrcMetadataTable { 252 @ColumnName("mut_id") 253 long mutationId; 254 255 @ColumnName("st_id") 256 long mutationStatusId; 257 258 @ColumnName("mp_id") 259 long mutationPointId; 260 261 @ColumnName("file_id") 262 long fileId; 263 264 @ColumnName("nomut") 265 long nomutCount; 266 } 267 268 // Reconstruct the nomut table in Miniorm. 269 @TableName(nomutTable) 270 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 271 struct NomutTbl { 272 @ColumnName("mp_id") 273 long mutationPointId; 274 275 long line; 276 277 /// != 0 when a nomut is tagged on the line. 278 long status; 279 } 280 281 @TableName(nomutDataTable) 282 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 283 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 284 struct NomutDataTbl { 285 @ColumnName("st_id") 286 long mutantStatusId; 287 288 @ColumnName("mp_id") 289 long mutationPointId; 290 291 long line; 292 293 @ColumnParam("") 294 string tag; 295 296 @ColumnParam("") 297 string comment; 298 } 299 300 @TableName(schemaVersionTable) 301 struct VersionTbl { 302 @ColumnName("version") 303 long version_; 304 } 305 306 @TableName(dextoolVersionTable) 307 struct DextoolVersionTable { 308 long checksum; 309 } 310 311 @TableName(configVersionTable) 312 struct ConfigVersionTable { 313 long checksum; 314 } 315 316 @TableName(filesTable) 317 @TableConstraint("unique_ UNIQUE (path)") 318 struct FilesTbl { 319 long id; 320 321 string path; 322 323 // checksum of the file content, 64bit. 324 long checksum; 325 Language lang; 326 327 @ColumnName("timestamp") 328 SysTime timeStamp; 329 330 /// True if the file is a root. 331 bool root; 332 } 333 334 @TableName(testFilesTable) 335 @TableConstraint("unique_ UNIQUE (path)") 336 struct TestFilesTable { 337 long id; 338 339 string path; 340 341 /// checksum is 64bit. 342 long checksum; 343 344 /// Last time a change to the test file where detected. 345 @ColumnName("timestamp") 346 SysTime timeStamp; 347 } 348 349 /// there shall never exist two mutations points for the same file+offset. 350 @TableName(mutationPointTable) 351 @TableConstraint("file_offset UNIQUE (file_id, offset_begin, offset_end)") 352 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 353 struct MutationPointTbl { 354 long id; 355 long file_id; 356 uint offset_begin; 357 uint offset_end; 358 359 /// line start from zero 360 @ColumnParam("") 361 uint line; 362 @ColumnParam("") 363 uint column; 364 365 @ColumnParam("") 366 uint line_end; 367 368 @ColumnParam("") 369 uint column_end; 370 } 371 372 @TableName(mutationTable) 373 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 374 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 375 @TableConstraint("unique_ UNIQUE (mp_id, st_id, kind)") 376 struct MutationTbl { 377 long id; 378 379 long mp_id; 380 381 long st_id; 382 383 long kind; 384 } 385 386 /** 387 * This could use an intermediate adapter table to normalise the test_case data 388 * but I chose not to do that because it makes it harder to add test cases and 389 * do a cleanup. 390 */ 391 @TableName(killedTestCaseTable) 392 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 393 @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE")) 394 @TableConstraint("unique_ UNIQUE (st_id, tc_id)") 395 struct TestCaseKilledTbl { 396 long id; 397 398 @ColumnName("st_id") 399 long mutationStatusId; 400 @ColumnName("tc_id") 401 long testCaseId; 402 403 // location is a filesystem location or other suitable helper for a user to 404 // locate the test. 405 @ColumnParam("") 406 string location; 407 } 408 409 /** 410 * Track all test cases that has been found by the test suite output analyzer. 411 * Useful to find test cases that has never killed any mutant. 412 * name should match test_case_killed_v2_tbl 413 * TODO: name should be the primary key. on a conflict a counter should be updated. 414 */ 415 @TableName(allTestCaseTable) 416 @TableConstraint("unique_ UNIQUE (name)") 417 struct AllTestCaseTbl { 418 long id; 419 string name; 420 421 @ColumnName("is_new") 422 bool isNew; 423 } 424 425 /** 426 * The status of a mutant. if it is killed or otherwise. multiple mutation 427 * operators can result in the same change of the source code. By coupling the 428 * mutant status to the checksum of the source code change it means that two 429 * mutations that have the same checksum will "cooperate". 430 * 431 * compile_time_ms = time it took to compile the program for the mutant 432 * test_time_ms = time it took to run the test suite 433 * updated_ts = is when the status where last updated. Seconds at UTC+0. 434 * added_ts = when the mutant where added to the system. UTC+0. 435 */ 436 @TableName(mutationStatusTable) 437 struct MutationStatusTbl { 438 // 64 bit checksum. 439 long id; 440 441 /// Mutation.Status 442 long status; 443 444 @ColumnName("exit_code") 445 int exitCode; 446 447 @ColumnName("compile_time_ms") 448 long compileTimeMs; 449 450 @ColumnName("test_time_ms") 451 long testTimeMs; 452 453 @ColumnParam("") 454 @ColumnName("update_ts") 455 SysTime updated; 456 457 @ColumnParam("") 458 @ColumnName("added_ts") 459 SysTime added; 460 461 /// Priority of the mutant used when testing. 462 long prio; 463 } 464 465 /// Mutants that should be tested. 466 @TableName(mutantWorklistTable) 467 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 468 struct MutantWorklistTbl { 469 long id; 470 long prio; 471 } 472 473 /** Memory overload mutants that are re-tested one extra time. 474 */ 475 @TableName(mutantMemOverloadWorklistTable) 476 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 477 struct MutantMemOverloadtWorklistTbl { 478 long id; 479 } 480 481 /** Timeout mutants that are re-tested until none of them change status from 482 * timeout. 483 */ 484 @TableName(mutantTimeoutWorklistTable) 485 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 486 struct MutantTimeoutWorklistTbl { 487 long id; 488 /// iterations that the mutant have been tested. 489 long iter; 490 } 491 492 /** The defaults for the schema is the state that the state machine start in. 493 * 494 * This mean that if there are nothing in the database then `.init` is the correct starting point. 495 */ 496 @TableName(mutantTimeoutCtxTable) 497 struct MutantTimeoutCtxTbl { 498 /// What iteration the timeout testing is at. 499 long iter; 500 501 /// Last count of the mutants in the worklist that where in the timeout state. 502 long worklistCount; 503 504 enum State { 505 init_, 506 running, 507 done 508 } 509 510 /// State of the timeout algorithm. 511 State state; 512 } 513 514 /** The checksum is used as primary key to reduce the need to "peek" into the 515 * database when updating, adding, linking tables. 516 * 517 * Both `st_id` and `mut_id` are values that sqlite can reuse between analyzes 518 * if they have been previously removed thus the only assured connection 519 * between a marked mutant and future code changes is the checksum. 520 */ 521 @TableName(markedMutantTable) 522 @TablePrimaryKey("checksum") 523 struct MarkedMutantTbl { 524 // TODO: can be removed in the future considering mutationStatusId is the 525 // checksum from v55+. 526 /// Checksum of the mutant status the marking is related to. 527 /// it is the mutationStatusId id. 528 long checksum; 529 530 /// updated each analyze. 531 @ColumnName("st_id") 532 long mutationStatusId; 533 534 uint line; 535 uint column; 536 string path; 537 538 /// The status it should always be changed to. 539 long toStatus; 540 541 /// Time when the mutant where marked. 542 SysTime time; 543 544 string rationale; 545 546 string mutText; 547 } 548 549 @TableName(schemaFragmentV2Table) 550 @TableForeignKey("file_id", KeyRef(filesTable ~ "(id)"), KeyParam("ON DELETE CASCADE")) 551 @TableConstraint("unique_ UNIQUE (file_id, text, offset_begin, offset_end)") 552 struct SchemaFragmentV2Table { 553 long id; 554 555 @ColumnName("file_id") 556 long fileId; 557 558 @ColumnParam("") 559 const(ubyte)[] text; 560 561 @ColumnName("offset_begin") 562 uint offsetBegin; 563 @ColumnName("offset_end") 564 uint offsetEnd; 565 } 566 567 @TableName(schemaMutantV2Table) 568 @TableForeignKey("fragment_id", KeyRef(schemaFragmentV2Table ~ "(id)"), 569 KeyParam("ON DELETE CASCADE")) 570 @TableForeignKey("st_id", KeyRef(mutationStatusTable ~ "(id)"), KeyParam("ON DELETE CASCADE")) 571 @TableConstraint("unique_ UNIQUE (fragment_id, st_id)") 572 struct SchemaMutantV2Table { 573 @ColumnName("fragment_id") 574 long fragmentId; 575 576 @ColumnName("st_id") 577 long statusId; 578 } 579 580 @TableName(schemaMutantQTable) 581 @TableConstraint("unique_ UNIQUE (kind, path)") 582 struct SchemaMutantKindQTable { 583 long id; 584 585 /// mutant subtype 586 long kind; 587 588 // max 100 589 long probability; 590 591 // 64 bit checksum of the path 592 long path; 593 } 594 595 @TableName(schemaSizeQTable) 596 struct SchemaSizeQTable { 597 long id; 598 long size; 599 } 600 601 /** The runtime of the test commands. 602 * 603 * By storing the data it reduces the need to run the test suite multiple times 604 * to get the minimum. 605 */ 606 @TableName(runtimeHistoryTable) 607 struct RuntimeHistoryTable { 608 long id; 609 610 /// when the measurement was taken. 611 @ColumnName("time") 612 SysTime timeStamp; 613 614 @ColumnName("time_ms") 615 long timeMs; 616 } 617 618 @TableName(mutationScoreHistoryTable) 619 @TableConstraint("time UNIQUE (time)") 620 struct MutationScoreHistoryTable { 621 long id; 622 623 /// when the measurement was taken. 624 @ColumnName("time") 625 SysTime timeStamp; 626 627 double score; 628 } 629 630 @TableName(mutationFileScoreHistoryTable) 631 @TableConstraint("date UNIQUE (time_stamp, file_path)") 632 struct MutationFileScoreHistoryTable { 633 long id; 634 635 /// when the measurement was taken. 636 @ColumnName("time_stamp") 637 SysTime timeStamp; 638 639 double score; 640 641 @ColumnName("file_path") 642 string filePath; 643 } 644 645 /** All functions that has been discovered in the source code. 646 * The `offset_begin` is where instrumentation points can be injected. 647 */ 648 @TableName(srcCovTable) 649 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 650 @TableConstraint("file_offset UNIQUE (file_id, begin, end)") 651 struct CoverageCodeRegionTable { 652 long id; 653 654 @ColumnName("file_id") 655 long fileId; 656 657 /// the region in the files, in bytes. 658 uint begin; 659 uint end; 660 } 661 662 /** Each coverage region that has a valid status has an entry in this table. 663 * It do mean that there can be region that do not exist in this table. That 664 * mean that something went wrong when gathering the data. 665 */ 666 @TableName(srcCovInfoTable) 667 @TableForeignKey("id", KeyRef("src_cov_instr(id)"), KeyParam("ON DELETE CASCADE")) 668 struct CoverageInfoTable { 669 long id; 670 671 /// True if the region has been visited. 672 bool status; 673 } 674 675 /// When the coverage information was gathered. 676 @TableName(srcCovTimeStampTable) 677 struct CoverageTimeTtampTable { 678 long id; 679 680 @ColumnName("timestamp") 681 SysTime timeStamp; 682 } 683 684 /** Files that roots are dependent on. They do not need to contain mutants. 685 */ 686 @TableName(depFileTable) 687 @TableConstraint("unique_ UNIQUE (file)") 688 struct DependencyFileTable { 689 long id; 690 691 string file; 692 693 /// checksum is 64bit. 694 long checksum; 695 } 696 697 @TableName(depRootTable) 698 @TableForeignKey("dep_id", KeyRef("dependency_file(id)"), KeyParam("ON DELETE CASCADE")) 699 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 700 @TableConstraint("unique_ UNIQUE (dep_id, file_id)") 701 struct DependencyRootTable { 702 @ColumnName("dep_id") 703 long depFileId; 704 705 @ColumnName("file_id") 706 long fileId; 707 } 708 709 @TableName(testCmdOriginalTable) 710 @TablePrimaryKey("checksum") 711 @TableConstraint("unique_ UNIQUE (cmd_id)") 712 @TableForeignKey("cmd_id", KeyRef("test_cmd(id)"), KeyParam("ON DELETE CASCADE")) 713 struct TestCmdOriginalTable { 714 long checksum; 715 716 @ColumnName("cmd_id") 717 long testCmdId; 718 } 719 720 @TableName(testCmdMutatedTable) 721 @TablePrimaryKey("checksum") 722 struct TestCmdMutatedTable { 723 long checksum; 724 725 /// Mutation.Status 726 long status; 727 728 /// when the measurement was taken. 729 @ColumnName("timestamp") 730 SysTime timeStamp; 731 } 732 733 @TableName(testCmdTable) 734 @TableConstraint("unique_ UNIQUE (cmd)") 735 struct TestCmdTable { 736 long id; 737 string cmd; 738 } 739 740 @TableName(testCmdRelMutantTable) 741 @TableForeignKey("cmd_id", KeyRef("test_cmd(id)"), KeyParam("ON DELETE CASCADE")) 742 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 743 @TableConstraint("unique_ UNIQUE (cmd_id, st_id)") 744 struct TestCmdRelMutantTable { 745 long id; 746 747 @ColumnName("cmd_id") 748 long testCmdId; 749 750 @ColumnName("st_id") 751 long statusId; 752 } 753 754 void updateSchemaVersion(ref Miniorm db, long ver) nothrow { 755 try { 756 db.run(delete_!VersionTbl); 757 db.run(insert!VersionTbl.insert, VersionTbl(ver)); 758 } catch (Exception e) { 759 logger.error(e.msg).collectException; 760 } 761 } 762 763 long getSchemaVersion(ref Miniorm db) { 764 auto v = db.run(select!VersionTbl); 765 return v.empty ? 0 : v.front.version_; 766 } 767 768 void upgrade(ref Miniorm db, UpgradeTable tbl) { 769 import d2sqlite3; 770 771 immutable maxIndex = 30; 772 773 alias upgradeFunc = void function(ref Miniorm db); 774 775 bool hasUpdated; 776 777 bool running = true; 778 while (running) { 779 const version_ = () { 780 // first time the version table do not exist thus fail. 781 try { 782 return getSchemaVersion(db); 783 } catch (Exception e) { 784 } 785 return 0; 786 }(); 787 788 if (version_ >= tbl.latestSchemaVersion) { 789 running = false; 790 break; 791 } 792 793 logger.infof("Upgrading database from %s", version_).collectException; 794 795 if (!hasUpdated) 796 try { 797 // only do this once and always before any changes to the database. 798 foreach (i; 0 .. maxIndex) { 799 db.run(format!"DROP INDEX IF EXISTS i%s"(i)); 800 } 801 } catch (Exception e) { 802 logger.warning(e.msg).collectException; 803 logger.warning("Unable to drop database indexes").collectException; 804 } 805 806 if (auto f = version_ in tbl) { 807 try { 808 hasUpdated = true; 809 810 (*f)(db); 811 if (version_ != 0) 812 updateSchemaVersion(db, version_ + 1); 813 } catch (Exception e) { 814 logger.trace(e).collectException; 815 logger.error(e.msg).collectException; 816 logger.warningf("Unable to upgrade a database of version %s", 817 version_).collectException; 818 logger.warning("This might impact the functionality. It is unwise to continue") 819 .collectException; 820 throw e; 821 } 822 } else { 823 logger.info("Upgrade successful").collectException; 824 running = false; 825 } 826 } 827 828 // add indexes assuming the lastest database schema 829 if (hasUpdated) 830 try { 831 int i; 832 db.run(format!"CREATE INDEX i%s ON %s(path)"(i++, filesTable)); 833 db.run(format!"CREATE INDEX i%s ON %s(path)"(i++, testFilesTable)); 834 835 // improve getTestCaseMutantKills by 10x 836 db.run(format!"CREATE INDEX i%s ON %s(tc_id,st_id)"(i++, killedTestCaseTable)); 837 db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, mutationTable)); 838 839 // all on delete cascade 840 db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, rawSrcMetadataTable)); 841 db.run(format!"CREATE INDEX i%s ON %s(mut_id)"(i++, srcMetadataTable)); 842 db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, srcMetadataTable)); 843 db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, srcMetadataTable)); 844 db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, srcMetadataTable)); 845 db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, nomutTable)); 846 db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, nomutDataTable)); 847 db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, nomutDataTable)); 848 db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, mutationPointTable)); 849 db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, mutationTable)); 850 db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, mutationTable)); 851 db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, killedTestCaseTable)); 852 db.run(format!"CREATE INDEX i%s ON %s(tc_id)"(i++, killedTestCaseTable)); 853 db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, srcCovTable)); 854 db.run(format!"CREATE INDEX i%s ON %s(dep_id)"(i++, depRootTable)); 855 db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, depRootTable)); 856 857 assert(i <= maxIndex); 858 } catch (Exception e) { 859 logger.warning(e.msg).collectException; 860 logger.warning("Unable to create database indexes").collectException; 861 } 862 } 863 864 /** If the database start it version 0, not initialized, then initialize to the 865 * latest schema version. 866 */ 867 void upgradeV0(ref Miniorm db) { 868 auto tbl = makeUpgradeTable; 869 870 db.run(buildSchema!(VersionTbl, RawSrcMetadata, FilesTbl, 871 MutationPointTbl, MutationTbl, TestCaseKilledTbl, AllTestCaseTbl, 872 MutationStatusTbl, MutantTimeoutCtxTbl, MutantTimeoutWorklistTbl, 873 MarkedMutantTbl, SrcMetadataTable, NomutTbl, NomutDataTbl, 874 NomutDataTbl, SchemaMutantKindQTable, SchemaSizeQTable, 875 MutantWorklistTbl, 876 RuntimeHistoryTable, 877 MutationScoreHistoryTable, 878 MutationFileScoreHistoryTable, TestFilesTable, CoverageCodeRegionTable, 879 CoverageInfoTable, CoverageTimeTtampTable, DependencyFileTable, 880 DependencyRootTable, DextoolVersionTable, ConfigVersionTable, 881 TestCmdOriginalTable, 882 TestCmdMutatedTable, 883 MutantMemOverloadtWorklistTbl, TestCmdRelMutantTable, 884 TestCmdTable, SchemaMutantV2Table, SchemaFragmentV2Table)); 885 886 updateSchemaVersion(db, tbl.latestSchemaVersion); 887 } 888 889 /// 2018-04-08 890 void upgradeV1(ref Miniorm db) { 891 @TableName(allTestCaseTable) 892 struct AllTestCaseTbl { 893 long id; 894 895 @ColumnParam("") 896 string name; 897 } 898 899 @TableName(testCaseTableV1) 900 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE")) 901 static struct TestCaseKilledTblV1 { 902 ulong id; 903 904 @ColumnName("mut_id") 905 ulong mutantId; 906 907 /// test_case is whatever identifier the user choose. 908 @ColumnName("test_case") 909 string testCase; 910 } 911 912 db.run(buildSchema!(TestCaseKilledTblV1, AllTestCaseTbl)); 913 } 914 915 /// 2018-04-22 916 void upgradeV2(ref Miniorm db) { 917 @TableName(filesTable) 918 static struct FilesTbl { 919 ulong id; 920 921 @ColumnParam("") 922 string path; 923 924 ulong checksum0; 925 ulong checksum1; 926 Language lang; 927 } 928 929 immutable newTbl = "new_" ~ filesTable; 930 931 db.run(buildSchema!FilesTbl("new_")); 932 db.run(format("INSERT INTO %s (id,path,checksum0,checksum1) SELECT * FROM %s", 933 newTbl, filesTable)); 934 db.replaceTbl(newTbl, filesTable); 935 } 936 937 /// 2018-09-01 938 void upgradeV3(ref Miniorm db) { 939 @TableName(killedTestCaseTable) 940 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE")) 941 struct TestCaseKilledTblV2 { 942 ulong id; 943 944 @ColumnName("mut_id") 945 ulong mutantId; 946 947 @ColumnParam("") 948 string name; 949 950 // location is a filesystem location or other suitable helper for a user to 951 // locate the test. 952 @ColumnParam("") 953 string location; 954 } 955 956 db.run(buildSchema!TestCaseKilledTblV2); 957 db.run(format("INSERT INTO %s (id,mut_id,name) SELECT * FROM %s", 958 killedTestCaseTable, testCaseTableV1)); 959 db.run(format("DROP TABLE %s", testCaseTableV1)); 960 961 db.run(buildSchema!AllTestCaseTbl); 962 } 963 964 /// 2018-09-24 965 void upgradeV4(ref Miniorm db) { 966 @TableName(killedTestCaseTable) 967 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE")) 968 @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE")) 969 static struct TestCaseKilledTblV3 { 970 ulong id; 971 972 @ColumnName("mut_id") 973 ulong mutantId; 974 @ColumnName("tc_id") 975 ulong testCaseId; 976 977 // location is a filesystem location or other suitable helper for a user to 978 // locate the test. 979 @ColumnParam("") 980 string location; 981 } 982 983 immutable newTbl = "new_" ~ killedTestCaseTable; 984 985 db.run(buildSchema!TestCaseKilledTblV3("new_")); 986 987 // add all missing test cases to all_test_case 988 db.run(format("INSERT INTO %s (name) SELECT DISTINCT t1.name FROM %s t1 LEFT JOIN %s t2 ON t2.name = t1.name WHERE t2.name IS NULL", 989 allTestCaseTable, killedTestCaseTable, allTestCaseTable)); 990 // https://stackoverflow.com/questions/2686254/how-to-select-all-records-from-one-table-that-do-not-exist-in-another-table 991 //Q: What is happening here? 992 // 993 //A: Conceptually, we select all rows from table1 and for each row we 994 //attempt to find a row in table2 with the same value for the name column. 995 //If there is no such row, we just leave the table2 portion of our result 996 //empty for that row. Then we constrain our selection by picking only those 997 //rows in the result where the matching row does not exist. Finally, We 998 //ignore all fields from our result except for the name column (the one we 999 //are sure that exists, from table1). 1000 // 1001 //While it may not be the most performant method possible in all cases, it 1002 //should work in basically every database engine ever that attempts to 1003 //implement ANSI 92 SQL 1004 1005 // This do NOT WORK. The result is that that this upgrade is broken because 1006 // it drops all maps between killed_test_case and mutation. 1007 //db.run(format("INSERT INTO %s (id,mut_id,tc_id,location) SELECT t1.id,t1.mut_id,t2.id,t1.location FROM %s t1 INNER JOIN %s t2 ON t1.name = t2.name", 1008 // newTbl, killedTestCaseTable, allTestCaseTable)); 1009 1010 db.replaceTbl(newTbl, killedTestCaseTable); 1011 } 1012 1013 /** 2018-09-30 1014 * 1015 * This upgrade will drop all existing mutations and thus all results. 1016 * It is too complex trying to upgrade and keep the results. 1017 * 1018 * When removing this function also remove the status field in mutation_v2_tbl. 1019 */ 1020 void upgradeV5(ref Miniorm db) { 1021 @TableName(filesTable) 1022 @TableConstraint("unique_ UNIQUE (path)") 1023 struct FilesTbl { 1024 long id; 1025 1026 @ColumnParam("") 1027 string path; 1028 1029 /// checksum is 128bit. 1030 long checksum0; 1031 long checksum1; 1032 Language lang; 1033 } 1034 1035 @TableName(mutationTable) 1036 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 1037 @TableForeignKey("st_id", KeyRef("mutation_status(id)")) 1038 @TableConstraint("unique_ UNIQUE (mp_id, kind)") 1039 static struct MutationTbl { 1040 ulong id; 1041 1042 ulong mp_id; 1043 1044 @ColumnParam("") 1045 ulong st_id; 1046 1047 ulong kind; 1048 1049 @ColumnParam("") 1050 ulong status; 1051 1052 /// time in ms spent on verifying the mutant 1053 @ColumnParam("") 1054 ulong time; 1055 } 1056 1057 @TableName(mutationStatusTable) 1058 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1059 static struct MutationStatusTbl { 1060 ulong id; 1061 ulong status; 1062 ulong checksum0; 1063 ulong checksum1; 1064 } 1065 1066 immutable new_mut_tbl = "new_" ~ mutationTable; 1067 db.run(buildSchema!MutationStatusTbl); 1068 1069 db.run(format("DROP TABLE %s", mutationTable)); 1070 db.run(buildSchema!MutationTbl); 1071 1072 immutable newFilesTbl = "new_" ~ filesTable; 1073 db.run(buildSchema!FilesTbl("new_")); 1074 db.run(format("INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang) SELECT * FROM %s", 1075 newFilesTbl, filesTable)); 1076 db.replaceTbl(newFilesTbl, filesTable); 1077 } 1078 1079 /// 2018-10-11 1080 void upgradeV6(ref Miniorm db) { 1081 @TableName(mutationStatusTable) 1082 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1083 static struct MutationStatusTbl { 1084 ulong id; 1085 ulong status; 1086 ulong time; 1087 SysTime timestamp; 1088 ulong checksum0; 1089 ulong checksum1; 1090 } 1091 1092 @TableName(mutationTable) 1093 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 1094 @TableForeignKey("st_id", KeyRef("mutation_status(id)")) 1095 @TableConstraint("unique_ UNIQUE (mp_id, kind)") 1096 struct MutationTbl { 1097 long id; 1098 1099 long mp_id; 1100 1101 @ColumnParam("") 1102 long st_id; 1103 1104 long kind; 1105 } 1106 1107 immutable new_mut_tbl = "new_" ~ mutationTable; 1108 1109 db.run(buildSchema!MutationTbl("new_")); 1110 1111 db.run(format("INSERT INTO %s (id,mp_id,st_id,kind) SELECT id,mp_id,st_id,kind FROM %s", 1112 new_mut_tbl, mutationTable)); 1113 db.replaceTbl(new_mut_tbl, mutationTable); 1114 1115 immutable new_muts_tbl = "new_" ~ mutationStatusTable; 1116 db.run(buildSchema!MutationStatusTbl("new_")); 1117 db.run(format("INSERT INTO %s (id,status,checksum0,checksum1) SELECT id,status,checksum0,checksum1 FROM %s", 1118 new_muts_tbl, mutationStatusTable)); 1119 db.replaceTbl(new_muts_tbl, mutationStatusTable); 1120 } 1121 1122 /// 2018-10-15 1123 void upgradeV7(ref Miniorm db) { 1124 @TableName(killedTestCaseTable) 1125 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1126 @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE")) 1127 struct TestCaseKilledTbl { 1128 long id; 1129 1130 @ColumnName("st_id") 1131 long mutationStatusId; 1132 @ColumnName("tc_id") 1133 long testCaseId; 1134 1135 // location is a filesystem location or other suitable helper for a user to 1136 // locate the test. 1137 @ColumnParam("") 1138 string location; 1139 } 1140 1141 immutable newTbl = "new_" ~ killedTestCaseTable; 1142 1143 db.run(buildSchema!TestCaseKilledTbl("new_")); 1144 1145 db.run(format("INSERT INTO %s (id,st_id,tc_id,location) 1146 SELECT t0.id,t1.st_id,t0.tc_id,t0.location 1147 FROM %s t0, %s t1 1148 WHERE 1149 t0.mut_id = t1.id", newTbl, killedTestCaseTable, mutationTable)); 1150 1151 db.replaceTbl(newTbl, killedTestCaseTable); 1152 } 1153 1154 /// 2018-10-20 1155 void upgradeV8(ref Miniorm db) { 1156 immutable newTbl = "new_" ~ mutationPointTable; 1157 db.run(buildSchema!MutationPointTbl("new_")); 1158 db.run(format("INSERT INTO %s (id,file_id,offset_begin,offset_end,line,column) 1159 SELECT t0.id,t0.file_id,t0.offset_begin,t0.offset_end,t0.line,t0.column 1160 FROM %s t0", newTbl, mutationPointTable)); 1161 1162 db.replaceTbl(newTbl, mutationPointTable); 1163 } 1164 1165 /// 2018-11-10 1166 void upgradeV9(ref Miniorm db) { 1167 @TableName(mutationStatusTable) 1168 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1169 struct MutationStatusTbl { 1170 long id; 1171 long status; 1172 1173 @ColumnParam("") 1174 long time; 1175 1176 @ColumnName("test_cnt") 1177 long testCnt; 1178 1179 @ColumnParam("") 1180 @ColumnName("update_ts") 1181 SysTime updated; 1182 1183 @ColumnParam("") 1184 @ColumnName("added_ts") 1185 SysTime added; 1186 1187 long checksum0; 1188 long checksum1; 1189 } 1190 1191 immutable newTbl = "new_" ~ mutationStatusTable; 1192 db.run(buildSchema!MutationStatusTbl("new_")); 1193 db.run(format("INSERT INTO %s (id,status,time,test_cnt,update_ts,checksum0,checksum1) 1194 SELECT t0.id,t0.status,t0.time,0,t0.timestamp,t0.checksum0,t0.checksum1 1195 FROM %s t0", newTbl, mutationStatusTable)); 1196 1197 replaceTbl(db, newTbl, mutationStatusTable); 1198 } 1199 1200 /// 2018-11-25 1201 void upgradeV10(ref Miniorm db) { 1202 @TableName(rawSrcMetadataTable) 1203 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 1204 @TableConstraint("unique_line_in_file UNIQUE (file_id, line)") 1205 struct RawSrcMetadata { 1206 ulong id; 1207 1208 @ColumnName("file_id") 1209 ulong fileId; 1210 1211 @ColumnParam("") 1212 uint line; 1213 1214 @ColumnParam("") 1215 ulong nomut; 1216 } 1217 1218 db.run(buildSchema!RawSrcMetadata); 1219 void makeSrcMetadataView(ref Miniorm db) { 1220 // check if a NOMUT is on or between the start and end of a mutant. 1221 immutable src_metadata_v1_tbl = "CREATE VIEW %s 1222 AS 1223 SELECT 1224 t0.id AS mut_id, 1225 t1.id AS st_id, 1226 t2.id AS mp_id, 1227 t3.id AS file_id, 1228 (SELECT count(*) FROM %s in_t0, %s in_t1 1229 WHERE 1230 in_t0.file_id = in_t1.file_id AND 1231 t0.mp_id = in_t0.id AND 1232 (in_t1.line BETWEEN in_t0.line AND in_t0.line_end)) AS nomut 1233 FROM %s t0, %s t1, %s t2, %s t3 1234 WHERE 1235 t0.mp_id = t2.id AND 1236 t0.st_id = t1.id AND 1237 t2.file_id = t3.id 1238 "; 1239 1240 db.run(format(src_metadata_v1_tbl, srcMetadataTable, mutationPointTable, rawSrcMetadataTable, 1241 mutationTable, mutationStatusTable, mutationPointTable, filesTable)); 1242 } 1243 1244 makeSrcMetadataView(db); 1245 } 1246 1247 /// 2019-04-06 1248 void upgradeV11(ref Miniorm db) { 1249 immutable newTbl = "new_" ~ rawSrcMetadataTable; 1250 db.run(buildSchema!RawSrcMetadata("new_")); 1251 db.run(format!"INSERT INTO %s (id,file_id,line,nomut) SELECT t.id,t.file_id,t.line,t.nomut FROM %s t"(newTbl, 1252 rawSrcMetadataTable)); 1253 replaceTbl(db, newTbl, rawSrcMetadataTable); 1254 1255 db.run(format("DROP VIEW %s", srcMetadataTable)).collectException; 1256 1257 // Associate metadata from lines with the mutation status. 1258 void makeSrcMetadataView(ref Miniorm db) { 1259 // check if a NOMUT is on or between the start and end of a mutant. 1260 immutable src_metadata_tbl = "CREATE VIEW %s 1261 AS 1262 SELECT DISTINCT 1263 t0.id AS mut_id, 1264 t1.id AS st_id, 1265 t2.id AS mp_id, 1266 t3.id AS file_id, 1267 (SELECT count(*) FROM %s WHERE nomut.mp_id = t2.id) as nomut 1268 FROM %s t0, %s t1, %s t2, %s t3 1269 WHERE 1270 t0.mp_id = t2.id AND 1271 t0.st_id = t1.id AND 1272 t2.file_id = t3.id"; 1273 db.run(format(src_metadata_tbl, srcMetadataTable, nomutTable, 1274 mutationTable, mutationStatusTable, mutationPointTable, filesTable)); 1275 1276 immutable nomut_tbl = "CREATE VIEW %s 1277 AS 1278 SELECT 1279 t0.id mp_id, 1280 t1.line line, 1281 count(*) status 1282 FROM %s t0, %s t1 1283 WHERE 1284 t0.file_id = t1.file_id AND 1285 (t1.line BETWEEN t0.line AND t0.line_end) 1286 GROUP BY 1287 t0.id"; 1288 db.run(format(nomut_tbl, nomutTable, mutationPointTable, rawSrcMetadataTable)); 1289 1290 immutable nomut_data_tbl = "CREATE VIEW %s 1291 AS 1292 SELECT 1293 t0.id as mut_id, 1294 t0.mp_id as mp_id, 1295 t1.line as line, 1296 t1.tag as tag, 1297 t1.comment as comment 1298 FROM %s t0, %s t1, %s t2 1299 WHERE 1300 t0.mp_id = t2.mp_id AND 1301 t1.line = t2.line"; 1302 db.run(format(nomut_data_tbl, nomutDataTable, mutationTable, 1303 rawSrcMetadataTable, nomutTable)); 1304 } 1305 1306 makeSrcMetadataView(db); 1307 } 1308 1309 /// 2019-08-28 1310 void upgradeV12(ref Miniorm db) { 1311 /** Timeout mutants that are re-tested until none of them change status from 1312 * timeout. 1313 */ 1314 @TableName(mutantTimeoutWorklistTable) 1315 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1316 struct MutantTimeoutWorklistTbl { 1317 long id; 1318 } 1319 1320 db.run(buildSchema!(MutantTimeoutCtxTbl, MutantTimeoutWorklistTbl)); 1321 } 1322 1323 /// 2019-11-12 1324 void upgradeV13(ref Miniorm db) { 1325 @TableName(markedMutantTable) 1326 @TablePrimaryKey("st_id") 1327 struct MarkedMutantTbl { 1328 @ColumnName("st_id") 1329 long mutationStatusId; 1330 1331 @ColumnName("mut_id") 1332 long mutationId; 1333 1334 uint line; 1335 1336 uint column; 1337 1338 string path; 1339 1340 @ColumnName("to_status") 1341 ulong toStatus; 1342 1343 SysTime time; 1344 1345 string rationale; 1346 1347 @ColumnName("mut_text") 1348 string mutText; 1349 } 1350 1351 db.run(buildSchema!(MarkedMutantTbl)); 1352 } 1353 1354 /// 2020-01-12 1355 void upgradeV14(ref Miniorm db) { 1356 @TableName(nomutDataTable) 1357 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE")) 1358 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 1359 struct NomutDataTbl { 1360 @ColumnName("mut_id") 1361 long mutationId; 1362 1363 @ColumnName("mp_id") 1364 long mutationPointId; 1365 1366 long line; 1367 1368 @ColumnParam("") 1369 string tag; 1370 1371 @ColumnParam("") 1372 string comment; 1373 } 1374 1375 db.run(format!"DROP VIEW %s"(srcMetadataTable)); 1376 db.run(format!"DROP VIEW %s"(nomutTable)); 1377 db.run(format!"DROP VIEW %s"(nomutDataTable)); 1378 1379 db.run(buildSchema!(SrcMetadataTable, NomutTbl, NomutDataTbl)); 1380 logger.info("Re-execute analyze to update the NOMUT data"); 1381 } 1382 1383 /// 2020-01-21 1384 void upgradeV15(ref Miniorm db) { 1385 @TableName(markedMutantTable) 1386 @TablePrimaryKey("checksum0") 1387 struct MarkedMutantTbl { 1388 /// Checksum of the mutant status the marking is related to. 1389 long checksum0; 1390 long checksum1; 1391 1392 /// updated each analyze. 1393 @ColumnName("st_id") 1394 long mutationStatusId; 1395 1396 /// updated each analyze. 1397 @ColumnName("mut_id") 1398 long mutationId; 1399 1400 uint line; 1401 uint column; 1402 string path; 1403 1404 /// The status it should always be changed to. 1405 long toStatus; 1406 1407 /// Time when the mutant where marked. 1408 SysTime time; 1409 1410 string rationale; 1411 1412 string mutText; 1413 } 1414 1415 // fix bug in the marked mutant table 1416 db.run(format!"DROP TABLE %s"(markedMutantTable)); 1417 db.run(buildSchema!MarkedMutantTbl); 1418 logger.info("Dropping all marked mutants because of database changes"); 1419 } 1420 1421 /// 2020-02-12 1422 void upgradeV16(ref Miniorm db) { 1423 @TableName(schemataWorkListTable) 1424 @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 1425 static struct SchemataWorkListTable { 1426 long id; 1427 } 1428 1429 @TableName(schemataMutantTable) 1430 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1431 @TableForeignKey("schem_id", KeyRef("schemata_fragment(id)"), KeyParam("ON DELETE CASCADE")) 1432 static struct SchemataMutantTable { 1433 @ColumnName("st_id") 1434 long statusId; 1435 @ColumnName("schem_id") 1436 long schemaId; 1437 } 1438 1439 @TableName(schemataFragmentTable) 1440 @TableForeignKey("schem_id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 1441 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE")) 1442 struct SchemataFragmentTable { 1443 long id; 1444 1445 @ColumnName("schem_id") 1446 long schemataId; 1447 1448 @ColumnName("file_id") 1449 long fileId; 1450 1451 @ColumnName("order_") 1452 long order; 1453 1454 @ColumnParam("") 1455 const(ubyte)[] text; 1456 1457 @ColumnName("offset_begin") 1458 uint offsetBegin; 1459 @ColumnName("offset_end") 1460 uint offsetEnd; 1461 } 1462 1463 db.run(buildSchema!(SchemataFragmentTable, SchemataWorkListTable, SchemataMutantTable)); 1464 } 1465 1466 /// 2020-02-12 1467 void upgradeV17(ref Miniorm db) { 1468 @TableName(schemataTable) 1469 static struct SchemataTable { 1470 long id; 1471 } 1472 1473 db.run(buildSchema!(SchemataTable)); 1474 } 1475 1476 /// 2020-03-21 1477 void upgradeV18(ref Miniorm db) { 1478 // this force an old database to add indexes 1479 } 1480 1481 /// 2020-04-01 1482 void upgradeV19(ref Miniorm db) { 1483 db.run("DROP TABLE " ~ schemataWorkListTable); 1484 db.run("DROP TABLE " ~ schemataTable); 1485 db.run("DROP TABLE " ~ schemataMutantTable); 1486 1487 @TableName(invalidSchemataTable) 1488 @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 1489 struct InvalidSchemataTable { 1490 long id; 1491 } 1492 1493 @TableName(schemataTable) 1494 struct SchemataTable { 1495 long id; 1496 1497 // number of fragments the schemata consist of. 1498 // used to detect if a fragment has been removed because its related file 1499 // was changed. 1500 long fragments; 1501 1502 // runtime generated constant that make it possible to "prune" old 1503 // schematas automatically. it assumes that each new version of dextool may 1504 // contain updates to the schematas thus the old schemats should be 1505 // removed. 1506 @ColumnName("version") 1507 long version_; 1508 } 1509 1510 db.run(buildSchema!(SchemataTable, InvalidSchemataTable)); 1511 } 1512 1513 /// 2020-06-01 1514 void upgradeV20(ref Miniorm db) { 1515 @TableName(schemataMutantTable) 1516 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1517 @TableForeignKey("schem_id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 1518 @TableConstraint("unique_ UNIQUE (st_id, schem_id)") 1519 struct SchemataMutantTable { 1520 @ColumnName("st_id") 1521 long statusId; 1522 @ColumnName("schem_id") 1523 long schemaId; 1524 } 1525 1526 db.run("DROP TABLE " ~ schemataMutantTable); 1527 db.run(buildSchema!(SchemataMutantTable)); 1528 } 1529 1530 /// 2020-11-28 1531 void upgradeV21(ref Miniorm db) { 1532 @TableName(schemataUsedTable) 1533 @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 1534 struct SchemataUsedTable { 1535 long id; 1536 } 1537 1538 db.run("DROP TABLE " ~ invalidSchemataTable); 1539 db.run(buildSchema!(SchemataUsedTable)); 1540 } 1541 1542 /// 2020-11-28 1543 void upgradeV22(ref Miniorm db) { 1544 @TableName(mutantWorklistTable) 1545 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1546 struct MutantWorklistTbl { 1547 long id; 1548 } 1549 1550 db.run(buildSchema!(MutantWorklistTbl)); 1551 } 1552 1553 /// 2020-12-06 1554 void upgradeV23(ref Miniorm db) { 1555 db.run(buildSchema!(RuntimeHistoryTable)); 1556 } 1557 1558 /// 2020-12-06 1559 void upgradeV24(ref Miniorm db) { 1560 @TableName(mutationScoreHistoryTable) 1561 struct MutationScoreHistoryTable { 1562 long id; 1563 1564 /// when the measurement was taken. 1565 @ColumnName("time") 1566 SysTime timeStamp; 1567 1568 double score; 1569 } 1570 1571 db.run(buildSchema!(MutationScoreHistoryTable)); 1572 } 1573 1574 /// 2020-12-25 1575 void upgradeV25(ref Miniorm db) { 1576 import std.traits : EnumMembers; 1577 import dextool.plugin.mutate.backend.type : Mutation; 1578 1579 @TableName(mutationStatusTable) 1580 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1581 struct MutationStatusTbl { 1582 long id; 1583 long status; 1584 @ColumnName("exit_code") 1585 int exitCode; 1586 1587 @ColumnParam("") 1588 long time; 1589 1590 @ColumnName("test_cnt") 1591 long testCnt; 1592 1593 @ColumnParam("") 1594 @ColumnName("update_ts") 1595 SysTime updated; 1596 1597 @ColumnParam("") 1598 @ColumnName("added_ts") 1599 SysTime added; 1600 1601 long checksum0; 1602 long checksum1; 1603 } 1604 1605 immutable newTbl = "new_" ~ mutationStatusTable; 1606 db.run(buildSchema!MutationStatusTbl("new_")); 1607 1608 auto stmt = db.prepare(format( 1609 "INSERT INTO %s (id,status,exit_code,time,test_cnt,update_ts,added_ts,checksum0,checksum1) 1610 SELECT t.id,t.status,:ecode,t.time,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1 1611 FROM %s t WHERE t.status = :status", newTbl, mutationStatusTable)); 1612 1613 foreach (st; [EnumMembers!(Mutation.Status)]) { 1614 stmt.get.bind(":ecode", (st == Mutation.Status.killed) ? 1 : 0); 1615 stmt.get.bind(":status", cast(long) st); 1616 stmt.get.execute; 1617 stmt.get.reset; 1618 } 1619 1620 replaceTbl(db, newTbl, mutationStatusTable); 1621 } 1622 1623 /// 2020-12-25 1624 void upgradeV26(ref Miniorm db) { 1625 immutable newTbl = "new_" ~ killedTestCaseTable; 1626 db.run(buildSchema!TestCaseKilledTbl("new_")); 1627 1628 db.run(format("INSERT OR IGNORE INTO %s (id, st_id, tc_id, location) 1629 SELECT t.id, t.st_id, t.tc_id, t.location 1630 FROM %s t", newTbl, killedTestCaseTable)); 1631 replaceTbl(db, newTbl, killedTestCaseTable); 1632 } 1633 1634 /// 2020-12-25 1635 void upgradeV27(ref Miniorm db) { 1636 @TableName(allTestCaseTable) 1637 @TableConstraint("unique_ UNIQUE (name)") 1638 struct AllTestCaseTbl { 1639 long id; 1640 string name; 1641 } 1642 1643 immutable newTbl = "new_" ~ allTestCaseTable; 1644 db.run(buildSchema!AllTestCaseTbl("new_")); 1645 1646 db.run(format("INSERT OR IGNORE INTO %s (id, name) 1647 SELECT t.id, t.name 1648 FROM %s t", newTbl, allTestCaseTable)); 1649 replaceTbl(db, newTbl, allTestCaseTable); 1650 } 1651 1652 /// 2020-12-25 1653 void upgradeV28(ref Miniorm db) { 1654 import dextool.plugin.mutate.backend.type : Mutation; 1655 1656 @TableName(mutationStatusTable) 1657 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1658 struct MutationStatusTbl { 1659 long id; 1660 long status; 1661 @ColumnName("exit_code") 1662 int exitCode; 1663 1664 @ColumnName("compile_time_ms") 1665 long compileTimeMs; 1666 1667 @ColumnName("test_time_ms") 1668 long testTimeMs; 1669 1670 @ColumnName("test_cnt") 1671 long testCnt; 1672 1673 @ColumnParam("") 1674 @ColumnName("update_ts") 1675 SysTime updated; 1676 1677 @ColumnParam("") 1678 @ColumnName("added_ts") 1679 SysTime added; 1680 1681 long checksum0; 1682 long checksum1; 1683 } 1684 1685 immutable newTbl = "new_" ~ mutationStatusTable; 1686 db.run(buildSchema!MutationStatusTbl("new_")); 1687 1688 db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1) 1689 SELECT t.id,t.status,t.exit_code,0,t.time,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1 1690 FROM %s t WHERE t.time NOT NULL", newTbl, mutationStatusTable)); 1691 1692 db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1) 1693 SELECT t.id,t.status,t.exit_code,0,0,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1 1694 FROM %s t WHERE t.time IS NULL", newTbl, mutationStatusTable)); 1695 1696 replaceTbl(db, newTbl, mutationStatusTable); 1697 } 1698 1699 /// 2020-12-27 1700 void upgradeV29(ref Miniorm db) { 1701 @TableName(testFilesTable) 1702 @TableConstraint("unique_ UNIQUE (path)") 1703 struct TestFilesTable { 1704 long id; 1705 1706 string path; 1707 1708 /// checksum is 128bit. 1709 long checksum0; 1710 long checksum1; 1711 1712 /// Last time a change to the test file where detected. 1713 @ColumnName("timestamp") 1714 SysTime timeStamp; 1715 } 1716 1717 db.run(buildSchema!(TestFilesTable)); 1718 } 1719 1720 /// 2020-12-29 1721 void upgradeV30(ref Miniorm db) { 1722 import std.datetime : Clock; 1723 import miniorm : toSqliteDateTime; 1724 1725 @TableName(filesTable) 1726 @TableConstraint("unique_ UNIQUE (path)") 1727 struct FilesTbl { 1728 long id; 1729 1730 string path; 1731 1732 /// checksum is 128bit. 1733 long checksum0; 1734 long checksum1; 1735 Language lang; 1736 1737 @ColumnName("timestamp") 1738 SysTime timeStamp; 1739 } 1740 1741 immutable newFilesTbl = "new_" ~ filesTable; 1742 db.run(buildSchema!FilesTbl("new_")); 1743 auto stmt = db.prepare(format( 1744 "INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang,timestamp) 1745 SELECT id,path,checksum0,checksum1,lang,:time FROM %s", newFilesTbl, filesTable)); 1746 stmt.get.bind(":time", Clock.currTime.toSqliteDateTime); 1747 stmt.get.execute; 1748 db.replaceTbl(newFilesTbl, filesTable); 1749 } 1750 1751 /// 2020-12-29 1752 void upgradeV31(ref Miniorm db) { 1753 @TableName(srcCovInfoTable) 1754 @TableForeignKey("id", KeyRef("src_cov_info(id)"), KeyParam("ON DELETE CASCADE")) 1755 struct CoverageInfoTable { 1756 long id; 1757 1758 /// True if the region has been visited. 1759 bool status; 1760 } 1761 1762 db.run(buildSchema!(CoverageCodeRegionTable, CoverageInfoTable, CoverageTimeTtampTable)); 1763 } 1764 1765 /// 2021-01-02 1766 void upgradeV32(ref Miniorm db) { 1767 @TableName(filesTable) 1768 @TableConstraint("unique_ UNIQUE (path)") 1769 struct FilesTbl { 1770 long id; 1771 1772 string path; 1773 1774 // checksum of the file content, 128bit. 1775 long checksum0; 1776 long checksum1; 1777 Language lang; 1778 1779 @ColumnName("timestamp") 1780 SysTime timeStamp; 1781 1782 /// True if the file is a root. 1783 bool root; 1784 } 1785 1786 immutable newFilesTbl = "new_" ~ filesTable; 1787 db.run(buildSchema!FilesTbl("new_")); 1788 db.run(format("INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang,timestamp,root) 1789 SELECT id,path,checksum0,checksum1,lang,timestamp,0 FROM %s", 1790 newFilesTbl, filesTable)); 1791 db.replaceTbl(newFilesTbl, filesTable); 1792 } 1793 1794 /// 2021-01-15 1795 void upgradeV33(ref Miniorm db) { 1796 @TableName(depFileTable) 1797 @TableConstraint("unique_ UNIQUE (file)") 1798 struct DependencyFileTable { 1799 long id; 1800 1801 string file; 1802 1803 /// checksum is 128bit. 1804 long checksum0; 1805 long checksum1; 1806 } 1807 1808 db.run(buildSchema!(DependencyFileTable, DependencyRootTable)); 1809 1810 // add all existing files as being dependent on each other. 1811 // this forces, if any one of them changes, all to be re-analyzed. 1812 db.run(format("INSERT OR IGNORE INTO %1$s (file,checksum0,checksum1) 1813 SELECT path,0,0 FROM %2$s", depFileTable, filesTable)); 1814 db.run(format("INSERT OR IGNORE INTO %1$s (dep_id,file_id) 1815 SELECT t0.id,t1.id FROM %2$s t0, %3$s t1", depRootTable, 1816 depFileTable, filesTable)); 1817 } 1818 1819 /// 2021-03-14 1820 void upgradeV34(ref Miniorm db) { 1821 immutable newTbl = "new_" ~ mutantWorklistTable; 1822 db.run(buildSchema!MutantWorklistTbl("new_")); 1823 db.run(format("INSERT INTO %1$s (id,prio) SELECT id,0 FROM %2$s", newTbl, mutantWorklistTable)); 1824 db.replaceTbl(newTbl, mutantWorklistTable); 1825 } 1826 1827 /// 2021-03-28 1828 void upgradeV35(ref Miniorm db) { 1829 @TableName(mutationStatusTable) 1830 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1831 struct MutationStatusTbl { 1832 long id; 1833 long status; 1834 @ColumnName("exit_code") 1835 int exitCode; 1836 1837 @ColumnName("compile_time_ms") 1838 long compileTimeMs; 1839 1840 @ColumnName("test_time_ms") 1841 long testTimeMs; 1842 1843 @ColumnName("test_cnt") 1844 long testCnt; 1845 1846 @ColumnParam("") 1847 @ColumnName("update_ts") 1848 SysTime updated; 1849 1850 @ColumnParam("") 1851 @ColumnName("added_ts") 1852 SysTime added; 1853 1854 long checksum0; 1855 long checksum1; 1856 1857 /// Priority of the mutant used when testing. 1858 long prio; 1859 } 1860 1861 immutable newTbl = "new_" ~ mutationStatusTable; 1862 db.run(buildSchema!MutationStatusTbl("new_")); 1863 1864 db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1,prio) 1865 SELECT t.id,t.status,t.exit_code,t.compile_time_ms,t.test_time_ms,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1,0 1866 FROM %s t", newTbl, mutationStatusTable)); 1867 replaceTbl(db, newTbl, mutationStatusTable); 1868 } 1869 1870 /// 2021-03-29 1871 void upgradeV36(ref Miniorm db) { 1872 import dextool.plugin.mutate.backend.type : Mutation; 1873 1874 @TableName(mutationStatusTable) 1875 @TableConstraint("checksum UNIQUE (checksum0, checksum1)") 1876 struct MutationStatusTbl { 1877 long id; 1878 1879 /// Mutation.Status 1880 long status; 1881 1882 @ColumnName("exit_code") 1883 int exitCode; 1884 1885 @ColumnName("compile_time_ms") 1886 long compileTimeMs; 1887 1888 @ColumnName("test_time_ms") 1889 long testTimeMs; 1890 1891 @ColumnParam("") 1892 @ColumnName("update_ts") 1893 SysTime updated; 1894 1895 @ColumnParam("") 1896 @ColumnName("added_ts") 1897 SysTime added; 1898 1899 long checksum0; 1900 long checksum1; 1901 1902 /// Priority of the mutant used when testing. 1903 long prio; 1904 } 1905 1906 immutable newTbl = "new_" ~ mutationStatusTable; 1907 db.run(buildSchema!MutationStatusTbl("new_")); 1908 1909 db.run(format( 1910 "INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,update_ts,added_ts,checksum0,checksum1,prio) 1911 SELECT t.id,t.status,t.exit_code,t.compile_time_ms,t.test_time_ms,t.update_ts,t.added_ts,t.checksum0,t.checksum1,t.prio 1912 FROM %s t", newTbl, mutationStatusTable)); 1913 replaceTbl(db, newTbl, mutationStatusTable); 1914 } 1915 1916 // 2021-04-18 1917 void upgradeV37(ref Miniorm db) { 1918 db.run("DELETE FROM " ~ schemataTable); 1919 db.run(buildSchema!(DextoolVersionTable)); 1920 } 1921 1922 // 2021-04-19 1923 void upgradeV38(ref Miniorm db) { 1924 @TableName(schemataTable) 1925 struct SchemataTable { 1926 long id; 1927 1928 // number of fragments the schemata consist of. 1929 // used to detect if a fragment has been removed because its related file 1930 // was changed. 1931 long fragments; 1932 } 1933 1934 db.run("DROP TABLE " ~ schemataTable); 1935 db.run(buildSchema!(SchemataTable)); 1936 } 1937 1938 // 2021-04-23 1939 void upgradeV39(ref Miniorm db) { 1940 // this is just to force a re-measure of the test suite. 1941 db.run(format("DELETE FROM %s", runtimeHistoryTable)); 1942 } 1943 1944 // 2021-05-06 1945 void upgradeV40(ref Miniorm db) { 1946 // force an upgrade because after this release all scheman will be zipped. 1947 db.run("DELETE FROM " ~ schemataFragmentTable); 1948 db.run("DELETE FROM " ~ schemataMutantTable); 1949 db.run("DELETE FROM " ~ schemataTable); 1950 db.run("DELETE FROM " ~ schemataUsedTable); 1951 } 1952 1953 // 2021-05-09 1954 void upgradeV41(ref Miniorm db) { 1955 @TableName(testCmdOriginalTable) 1956 @TablePrimaryKey("checksum") 1957 @TableConstraint("unique_ UNIQUE (cmd)") 1958 struct TestCmdOriginalTable { 1959 long checksum; 1960 string cmd; 1961 } 1962 1963 db.run(buildSchema!(TestCmdOriginalTable)); 1964 } 1965 1966 // 2021-05-23 1967 void upgradeV42(ref Miniorm db) { 1968 db.run(buildSchema!(TestCmdMutatedTable)); 1969 } 1970 1971 // 2021-06-07 1972 void upgradeV43(ref Miniorm db) { 1973 @TableName(mutationTable) 1974 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE")) 1975 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE")) 1976 @TableConstraint("unique_ UNIQUE (mp_id, kind)") 1977 struct MutationTbl { 1978 long id; 1979 1980 long mp_id; 1981 1982 @ColumnParam("") 1983 long st_id; 1984 1985 long kind; 1986 } 1987 1988 immutable newTbl = "new_" ~ mutationTable; 1989 db.run(buildSchema!MutationTbl("new_")); 1990 1991 db.run(format("INSERT INTO %s (id,mp_id,st_id,kind) 1992 SELECT id,mp_id,st_id,kind FROM %s WHERE st_id NOT NULL", newTbl, mutationTable)); 1993 replaceTbl(db, newTbl, mutationTable); 1994 } 1995 1996 // 2021-08-30 1997 void upgradeV44(ref Miniorm db) { 1998 @TableName(schemaMutantQTable) 1999 @TablePrimaryKey("kind") 2000 struct SchemaMutantKindQ { 2001 /// mutant subtype 2002 long kind; 2003 2004 // max 100 2005 long probability; 2006 } 2007 2008 @TableName(schemataUsedTable) 2009 @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE")) 2010 struct SchemataUsedTable { 2011 enum SchemaStatus { 2012 /// no status exist 2013 none, 2014 /// the schema compiled and the test suite executed OK 2015 ok, 2016 /// either it failed to compile or the test suite failed 2017 broken, 2018 /// the schema only contain killed mutants 2019 allKilled, 2020 } 2021 2022 long id; 2023 2024 SchemaStatus status; 2025 } 2026 2027 db.run("DROP TABLE " ~ schemataUsedTable); 2028 db.run(buildSchema!(SchemataUsedTable, SchemaMutantKindQ)); 2029 } 2030 2031 // 2021-08-30 2032 void upgradeV45(ref Miniorm db) { 2033 db.run("DROP TABLE " ~ schemaMutantQTable); 2034 db.run(buildSchema!(SchemaMutantKindQTable)); 2035 } 2036 2037 // 2021-08-30 2038 void upgradeV46(ref Miniorm db) { 2039 db.run(buildSchema!SchemaSizeQTable); 2040 // drop probability because max state where changed thus all previous values are now 10x off 2041 db.run("DELETE FROM " ~ schemaMutantQTable); 2042 } 2043 2044 // 2021-11-01 2045 void upgradeV47(ref Miniorm db) { 2046 static immutable newTbl = "new_" ~ allTestCaseTable; 2047 db.run(buildSchema!AllTestCaseTbl("new_")); 2048 2049 db.run("INSERT INTO " ~ newTbl ~ " (id,name,is_new) SELECT id,name,0 FROM " ~ allTestCaseTable); 2050 replaceTbl(db, newTbl, allTestCaseTable); 2051 } 2052 2053 // 2021-11-02 2054 void upgradeV48(ref Miniorm db) { 2055 db.run(buildSchema!MutantMemOverloadtWorklistTbl); 2056 } 2057 2058 // 2021-11-02 2059 void upgradeV49(ref Miniorm db) { 2060 static immutable newTbl = "new_" ~ mutantTimeoutWorklistTable; 2061 db.run(buildSchema!MutantTimeoutWorklistTbl("new_")); 2062 2063 db.run("INSERT INTO " ~ newTbl ~ " (id,iter) SELECT id,0 FROM " ~ mutantTimeoutWorklistTable); 2064 2065 replaceTbl(db, newTbl, mutantTimeoutWorklistTable); 2066 } 2067 2068 // 2022-03-21 2069 void upgradeV50(ref Miniorm db) { 2070 db.run("DROP TABLE " ~ testCmdOriginalTable); 2071 db.run(buildSchema!(TestCmdRelMutantTable, TestCmdTable, TestCmdOriginalTable)); 2072 } 2073 2074 // 2022-06-14 2075 void upgradeV51(ref Miniorm db) { 2076 db.run("DROP TABLE " ~ srcCovInfoTable); 2077 db.run(buildSchema!CoverageInfoTable); 2078 } 2079 2080 // 2022-06-21 2081 void upgradeV52(ref Miniorm db) { 2082 @TableName(mutationFileScoreHistoryTable) 2083 struct MutationFileScoreHistoryTable { 2084 long id; 2085 2086 /// when the measurement was taken. 2087 @ColumnName("time_stamp") 2088 SysTime timeStamp; 2089 2090 double score; 2091 2092 @ColumnName("file_path") 2093 string filePath; 2094 } 2095 2096 db.run(buildSchema!MutationFileScoreHistoryTable); 2097 } 2098 2099 // 2022-08-10 2100 void upgradeV53(ref Miniorm db) { 2101 @TableName(mutationStatusTable) 2102 @TableConstraint("checksum UNIQUE (checksum)") 2103 struct MutationStatusTbl { 2104 long id; 2105 2106 /// Mutation.Status 2107 long status; 2108 2109 @ColumnName("exit_code") 2110 int exitCode; 2111 2112 @ColumnName("compile_time_ms") 2113 long compileTimeMs; 2114 2115 @ColumnName("test_time_ms") 2116 long testTimeMs; 2117 2118 @ColumnParam("") 2119 @ColumnName("update_ts") 2120 SysTime updated; 2121 2122 @ColumnParam("") 2123 @ColumnName("added_ts") 2124 SysTime added; 2125 2126 long checksum; 2127 2128 /// Priority of the mutant used when testing. 2129 long prio; 2130 } 2131 2132 @TableName(markedMutantTable) 2133 @TablePrimaryKey("checksum") 2134 struct MarkedMutantTbl { 2135 // TODO: can be removed in the future considering mutationStatusId is the 2136 // checksum from v55+. 2137 /// Checksum of the mutant status the marking is related to. 2138 /// it is the mutationStatusId id. 2139 long checksum; 2140 2141 /// updated each analyze. 2142 @ColumnName("st_id") 2143 long mutationStatusId; 2144 2145 /// updated each analyze. 2146 @ColumnName("mut_id") 2147 long mutationId; 2148 2149 uint line; 2150 uint column; 2151 string path; 2152 2153 /// The status it should always be changed to. 2154 long toStatus; 2155 2156 /// Time when the mutant where marked. 2157 SysTime time; 2158 2159 string rationale; 2160 2161 string mutText; 2162 } 2163 2164 const newFilesTbl = "new_" ~ filesTable; 2165 db.run(buildSchema!FilesTbl("new_")); 2166 db.run("INSERT INTO " ~ newFilesTbl 2167 ~ " (id, path, lang, timestamp, root, checksum) SELECT id,path,lang,timestamp,root,checksum0 FROM " 2168 ~ filesTable); 2169 replaceTbl(db, newFilesTbl, filesTable); 2170 2171 const newTestFilesTbl = "new_" ~ testFilesTable; 2172 db.run(buildSchema!TestFilesTable("new_")); 2173 db.run("INSERT INTO " ~ newTestFilesTbl 2174 ~ " (id,path,timestamp,checksum) SELECT id,path,timestamp,checksum0 FROM " 2175 ~ testFilesTable); 2176 replaceTbl(db, newTestFilesTbl, testFilesTable); 2177 2178 const newMutationStatusTbl = "new_" ~ mutationStatusTable; 2179 db.run(buildSchema!MutationStatusTbl("new_")); 2180 db.run("INSERT INTO " ~ newMutationStatusTbl ~ " (id,status,exit_code,checksum,compile_time_ms,test_time_ms,prio) SELECT id,status,exit_code,checksum0,compile_time_ms,test_time_ms,prio FROM " ~ mutationStatusTable); 2181 replaceTbl(db, newMutationStatusTbl, mutationStatusTable); 2182 2183 const newDepFileTbl = "new_" ~ depFileTable; 2184 db.run(buildSchema!DependencyFileTable("new_")); 2185 db.run("INSERT INTO " ~ newDepFileTbl 2186 ~ " (id, file, checksum) SELECT id,file,checksum0 FROM " ~ depFileTable); 2187 replaceTbl(db, newDepFileTbl, depFileTable); 2188 2189 const newMarkedMutantTbl = "new_" ~ markedMutantTable; 2190 db.run(buildSchema!MarkedMutantTbl("new_")); 2191 db.run("INSERT INTO " ~ newMarkedMutantTbl ~ " (checksum,st_id,mut_id,line,column,path,toStatus,time,rationale,mutText) SELECT checksum0,st_id,mut_id,line,column,path,toStatus,time,rationale,mutText FROM " ~ markedMutantTable); 2192 replaceTbl(db, newMarkedMutantTbl, markedMutantTable); 2193 } 2194 2195 // 2022-08-17 2196 void upgradeV54(ref Miniorm db) { 2197 static immutable newFileScoreTable = "new_" ~ mutationFileScoreHistoryTable; 2198 db.run(buildSchema!MutationFileScoreHistoryTable("new_")); 2199 db.run("INSERT INTO " ~ newFileScoreTable ~ " (id,time_stamp,score,file_path) SELECT id,strftime('%Y-%m-%d 00:00:00.0', time_stamp) AS time,score,file_path FROM " ~ mutationFileScoreHistoryTable ~ " GROUP BY date(time), file_path"); 2200 replaceTbl(db, newFileScoreTable, mutationFileScoreHistoryTable); 2201 2202 static immutable newMutationScoreTable = "new_" ~ mutationScoreHistoryTable; 2203 db.run(buildSchema!MutationScoreHistoryTable("new_")); 2204 db.run("INSERT INTO " ~ newMutationScoreTable 2205 ~ " (id,time,score) SELECT id,strftime('%Y-%m-%d 00:00:00.0', time),score FROM " 2206 ~ mutationScoreHistoryTable ~ " GROUP BY date(time)"); 2207 replaceTbl(db, newMutationScoreTable, mutationScoreHistoryTable); 2208 } 2209 2210 // 2022-08-23 2211 void upgradeV55(ref Miniorm db) { 2212 foreach (tbl; [ 2213 srcMetadataTable, mutationTable, killedTestCaseTable, 2214 mutantWorklistTable, mutantMemOverloadWorklistTable, 2215 mutantTimeoutWorklistTable, schemataMutantTable, testCmdRelMutantTable 2216 ]) { 2217 db.run("DELETE FROM " ~ tbl); 2218 } 2219 2220 db.run("DROP TABLE " ~ mutationStatusTable); 2221 db.run(buildSchema!MutationStatusTbl); 2222 } 2223 2224 // 2022-08-23 2225 void upgradeV56(ref Miniorm db) { 2226 foreach (tbl; [nomutDataTable, markedMutantTable]) { 2227 db.run("DROP TABLE " ~ tbl); 2228 } 2229 db.run(buildSchema!(NomutDataTbl, MarkedMutantTbl)); 2230 } 2231 2232 // 2022-08-30 2233 void upgradeV57(ref Miniorm db) { 2234 // fejk upgrade to force recalculation of indexes 2235 } 2236 2237 // 2022-09-02 2238 void upgradeV58(ref Miniorm db) { 2239 // clear out old junk values. those with zero mutants. 2240 db.run("DELETE FROM " ~ mutationFileScoreHistoryTable ~ " WHERE score=0"); 2241 } 2242 2243 // 2022-09-07 2244 void upgradeV59(ref Miniorm db) { 2245 db.run(buildSchema!(SchemaFragmentV2Table, SchemaMutantV2Table)); 2246 } 2247 2248 // 2022-10-09 2249 void upgradeV60(ref Miniorm db) { 2250 foreach (a; [ 2251 schemataFragmentTable, schemataUsedTable, schemataMutantTable, 2252 schemataTable 2253 ]) 2254 db.run("DROP TABLE " ~ a); 2255 } 2256 2257 void upgradeV61(ref Miniorm db) { 2258 immutable newTbl = "new_" ~ mutationTable; 2259 db.run(buildSchema!MutationTbl("new_")); 2260 2261 db.run("INSERT OR IGNORE INTO " ~ newTbl ~ " (id,mp_id,st_id,kind) 2262 SELECT id,mp_id,st_id,kind FROM " ~ mutationTable); 2263 replaceTbl(db, newTbl, mutationTable); 2264 } 2265 2266 void upgradeV62(ref Miniorm db) { 2267 db.run(buildSchema!ConfigVersionTable); 2268 } 2269 2270 void replaceTbl(ref Miniorm db, string src, string dst) { 2271 db.run("DROP TABLE " ~ dst); 2272 db.run("ALTER TABLE " ~ src ~ " RENAME TO " ~ dst); 2273 } 2274 2275 struct UpgradeTable { 2276 alias UpgradeFunc = void function(ref Miniorm db); 2277 UpgradeFunc[long] tbl; 2278 alias tbl this; 2279 2280 immutable long latestSchemaVersion; 2281 } 2282 2283 /** Inspects a module for functions starting with upgradeV to create a table of 2284 * functions that can be used to upgrade a database. 2285 */ 2286 UpgradeTable makeUpgradeTable() { 2287 import std.algorithm : sort, startsWith; 2288 import std.conv : to; 2289 import std.typecons : Tuple; 2290 2291 immutable prefix = "upgradeV"; 2292 2293 alias Module = dextool.plugin.mutate.backend.database.schema; 2294 2295 // the second parameter is the database version to upgrade FROM. 2296 alias UpgradeFx = Tuple!(UpgradeTable.UpgradeFunc, long); 2297 2298 UpgradeFx[] upgradeFx; 2299 long last_from; 2300 2301 static foreach (member; __traits(allMembers, Module)) { 2302 static if (member.startsWith(prefix)) 2303 upgradeFx ~= UpgradeFx(&__traits(getMember, Module, member), 2304 member[prefix.length .. $].to!long); 2305 } 2306 2307 typeof(UpgradeTable.tbl) tbl; 2308 foreach (fn; upgradeFx.sort!((a, b) => a[1] < b[1])) { 2309 last_from = fn[1]; 2310 tbl[last_from] = fn[0]; 2311 } 2312 2313 return UpgradeTable(tbl, last_from + 1); 2314 }