1 /**
2 Copyright: Copyright (c) 2018, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 This module contains the schema to initialize the database.
11 
12 To ensure that the upgrade path for a database always work a database is
13 created at the "lowest supported" and upgraded to the latest.
14 
15 # How to add schema change
16 
17 1. add an upgrade function, upgradeVX.
18 
19 The function makeUpgradeTable will then automatically find it and use it. X
20  **must** be the version upgrading FROM.
21 
22 # Style
23 A database schema upgrade path shall have a comment stating what date it was added.
24 Each change to the database schema must have an equal upgrade added.
25 
26 # Sqlite3
27 From the sqlite3 manual $(LINK https://www.sqlite.org/datatype3.html):
28 Each value stored in an SQLite database (or manipulated by the database
29 engine) has one of the following storage classes:
30 
31 NULL. The value is a NULL value.
32 
33 INTEGER. The value is a signed integer, stored in 1, 2, 3, 4, 6, or 8 bytes
34 depending on the magnitude of the value.
35 
36 REAL. The value is a floating point value, stored as an 8-byte IEEE floating
37 point number.
38 
39 TEXT. The value is a text string, stored using the database encoding (UTF-8,
40 UTF-16BE or UTF-16LE).
41 
42 BLOB. The value is a blob of data, stored exactly as it was input.
43 
44 A storage class is more general than a datatype. The INTEGER storage class, for
45 example, includes 6 different integer datatypes of different lengths.  This
46 makes a difference on disk. But as soon as INTEGER values are read off of disk
47 and into memory for processing, they are converted to the most general datatype
48 (8-byte signed integer). And so for the most part, "storage class" is
49 indistinguishable from "datatype" and the two terms can be used
50 interchangeably.
51 
52 # ON DELETE CASCADE
53 
54 when a ON DELETE CASCADE is added an index should be created too of the childs
55 foreign key.
56 
57 From the sqlite documentation:
58 
59 Indices are not required for child key columns but they are almost always
60 beneficial. […]
61 
62 Each time an application deletes a row from the ... parent table, it performs
63 [a query] to search for referencing rows in the ... child table.
64 
65 If this query returns any rows at all, then SQLite concludes that deleting the
66 row from the parent table would violate the foreign key constraint and returns
67 an error. Similar queries may be run if the content of the parent key is
68 modified or a new row is inserted into the parent table. If these queries
69 cannot use an index, they are forced to do a linear scan of the entire child
70 table. In a non-trivial database, this may be prohibitively expensive.
71 
72 So, in most real systems, an index should be created on the child key columns
73 of each foreign key constraint. The child key index does not have to be (and
74 usually will not be) a UNIQUE index.
75 
76 */
77 module dextool.plugin.mutate.backend.database.schema;
78 
79 import logger = std.experimental.logger;
80 import std.array : array, empty;
81 import std.datetime : SysTime, dur, Clock;
82 import std.exception : collectException;
83 import std.format : format;
84 
85 import dextool.plugin.mutate.backend.type : Language;
86 
87 import d2sqlite3 : SqlDatabase = Database;
88 import miniorm : Miniorm, TableName, buildSchema, ColumnParam, TableForeignKey, TableConstraint,
89     TablePrimaryKey, KeyRef, KeyParam, ColumnName, delete_, insert, select, spinSql, silentLog;
90 
91 immutable allTestCaseTable = "all_test_case";
92 immutable configVersionTable = "config_version";
93 immutable depFileTable = "dependency_file";
94 immutable depRootTable = "rel_dependency_root";
95 immutable dextoolVersionTable = "dextool_version";
96 immutable filesTable = "files";
97 immutable killedTestCaseTable = "killed_test_case";
98 immutable markedMutantTable = "marked_mutant";
99 immutable mutantMemOverloadWorklistTable = "mutant_memoverload_worklist";
100 immutable mutantTimeoutCtxTable = "mutant_timeout_ctx";
101 immutable mutantTimeoutWorklistTable = "mutant_timeout_worklist";
102 immutable mutantWorklistTable = "mutant_worklist";
103 immutable mutationFileScoreHistoryTable = "mutation_file_score_history";
104 immutable mutationPointTable = "mutation_point";
105 immutable mutationScoreHistoryTable = "mutation_score_history";
106 immutable mutationStatusTable = "mutation_status";
107 immutable mutationTable = "mutation";
108 immutable nomutDataTable = "nomut_data";
109 immutable nomutTable = "nomut";
110 immutable rawSrcMetadataTable = "raw_src_metadata";
111 immutable runtimeHistoryTable = "test_cmd_runtime_history";
112 immutable schemaFragmentV2Table = "schema_fragment_v2";
113 immutable schemaMutantQTable = "schema_mutant_q";
114 immutable schemaMutantV2Table = "schemata_mutant_v2";
115 immutable schemaSizeQTable = "schema_size_q";
116 immutable schemaVersionTable = "schema_version";
117 immutable srcCovInfoTable = "src_cov_info";
118 immutable srcCovTable = "src_cov_instr";
119 immutable srcCovTimeStampTable = "src_cov_timestamp";
120 immutable srcMetadataTable = "src_metadata";
121 immutable testCmdMutatedTable = "test_cmd_mutated";
122 immutable testCmdOriginalTable = "test_cmd_original";
123 immutable testCmdRelMutantTable = "test_cmd_rel_mutant";
124 immutable testCmdTable = "test_cmd";
125 immutable testFilesTable = "test_files";
126 
127 private immutable schemataTable = "schemata";
128 private immutable schemataMutantTable = "schemata_mutant";
129 private immutable schemataUsedTable = "schemata_used";
130 private immutable invalidSchemataTable = "invalid_schemata";
131 private immutable schemataWorkListTable = "schemata_worklist";
132 private immutable testCaseTableV1 = "test_case";
133 private immutable schemataFragmentTable = "schemata_fragment";
134 
135 /** Initialize or open an existing database.
136  *
137  * Params:
138  *  p = path where to initialize a new database or open an existing
139  *
140  * Returns: an open sqlite3 database object.
141  */
142 Miniorm initializeDB(const string p) @trusted
143 in {
144     assert(p.length != 0);
145 }
146 do {
147     import std.file : exists;
148     import my.file : followSymlink;
149     import my.optional;
150     import my.path : Path;
151     import d2sqlite3 : SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE;
152 
153     static void setPragmas(ref SqlDatabase db) {
154         // dfmt off
155         auto pragmas = [
156             // required for foreign keys with cascade to work
157             "PRAGMA foreign_keys=ON;",
158         ];
159         // dfmt on
160 
161         foreach (p; pragmas) {
162             db.run(p);
163         }
164     }
165 
166     const isOldDb = exists(followSymlink(Path(p)).orElse(Path(p)).toString);
167     SqlDatabase sqliteDb;
168     scope (success)
169         setPragmas(sqliteDb);
170 
171     logger.trace("Opening database ", p);
172     try {
173         sqliteDb = SqlDatabase(p, SQLITE_OPEN_READWRITE);
174     } catch (Exception e) {
175         logger.trace(e.msg);
176         logger.trace("Initializing a new sqlite3 database");
177         sqliteDb = SqlDatabase(p, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);
178     }
179 
180     auto db = Miniorm(sqliteDb);
181 
182     auto tbl = makeUpgradeTable;
183     const longTimeout = 10.dur!"minutes";
184     try {
185         if (isOldDb && spinSql!(() => getSchemaVersion(db),
186                 silentLog)(10.dur!"seconds") >= tbl.latestSchemaVersion)
187             return db;
188     } catch (Exception e) {
189         logger.info("The database is probably locked. Will keep trying to open for ", longTimeout);
190     }
191     if (isOldDb && spinSql!(() => getSchemaVersion(db))(longTimeout) >= tbl.latestSchemaVersion)
192         return db;
193 
194     // TODO: remove all key off in upgrade schemas.
195     const giveUpAfter = Clock.currTime + longTimeout;
196     bool failed = true;
197     while (failed && Clock.currTime < giveUpAfter) {
198         try {
199             auto trans = db.transaction;
200             db.run("PRAGMA foreign_keys=OFF;");
201             upgrade(db, tbl);
202             trans.commit;
203             failed = false;
204         } catch (Exception e) {
205             logger.trace(e.msg);
206         }
207     }
208 
209     if (failed) {
210         logger.error("Unable to upgrade the database to the latest schema");
211         throw new Exception(null);
212     }
213 
214     return db;
215 }
216 
217 package:
218 
219 // metadata about mutants that occur on a line extracted from the source code.
220 // It is intended to further refined.
221 // nomut = if the line should ignore mutants.
222 // tag = a user defined tag for a NOMUT.
223 // comment = a user defined comment.
224 @TableName(rawSrcMetadataTable)
225 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
226 @TableConstraint("unique_line_in_file UNIQUE (file_id, line)")
227 struct RawSrcMetadata {
228     long id;
229 
230     @ColumnName("file_id")
231     long fileId;
232 
233     @ColumnParam("")
234     uint line;
235 
236     @ColumnParam("")
237     long nomut;
238 
239     @ColumnParam("")
240     string tag;
241 
242     @ColumnParam("")
243     string comment;
244 }
245 
246 @TableName(srcMetadataTable)
247 @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE"))
248 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
249 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
250 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
251 struct SrcMetadataTable {
252     @ColumnName("mut_id")
253     long mutationId;
254 
255     @ColumnName("st_id")
256     long mutationStatusId;
257 
258     @ColumnName("mp_id")
259     long mutationPointId;
260 
261     @ColumnName("file_id")
262     long fileId;
263 
264     @ColumnName("nomut")
265     long nomutCount;
266 }
267 
268 // Reconstruct the nomut table in Miniorm.
269 @TableName(nomutTable)
270 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
271 struct NomutTbl {
272     @ColumnName("mp_id")
273     long mutationPointId;
274 
275     long line;
276 
277     /// != 0 when a nomut is tagged on the line.
278     long status;
279 }
280 
281 @TableName(nomutDataTable)
282 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
283 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
284 struct NomutDataTbl {
285     @ColumnName("st_id")
286     long mutantStatusId;
287 
288     @ColumnName("mp_id")
289     long mutationPointId;
290 
291     long line;
292 
293     @ColumnParam("")
294     string tag;
295 
296     @ColumnParam("")
297     string comment;
298 }
299 
300 @TableName(schemaVersionTable)
301 struct VersionTbl {
302     @ColumnName("version")
303     long version_;
304 }
305 
306 @TableName(dextoolVersionTable)
307 struct DextoolVersionTable {
308     long checksum;
309 }
310 
311 @TableName(configVersionTable)
312 struct ConfigVersionTable {
313     long checksum;
314 }
315 
316 @TableName(filesTable)
317 @TableConstraint("unique_ UNIQUE (path)")
318 struct FilesTbl {
319     long id;
320 
321     string path;
322 
323     // checksum of the file content, 64bit.
324     long checksum;
325     Language lang;
326 
327     @ColumnName("timestamp")
328     SysTime timeStamp;
329 
330     /// True if the file is a root.
331     bool root;
332 }
333 
334 @TableName(testFilesTable)
335 @TableConstraint("unique_ UNIQUE (path)")
336 struct TestFilesTable {
337     long id;
338 
339     string path;
340 
341     /// checksum is 64bit.
342     long checksum;
343 
344     /// Last time a change to the test file where detected.
345     @ColumnName("timestamp")
346     SysTime timeStamp;
347 }
348 
349 /// there shall never exist two mutations points for the same file+offset.
350 @TableName(mutationPointTable)
351 @TableConstraint("file_offset UNIQUE (file_id, offset_begin, offset_end)")
352 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
353 struct MutationPointTbl {
354     long id;
355     long file_id;
356     uint offset_begin;
357     uint offset_end;
358 
359     /// line start from zero
360     @ColumnParam("")
361     uint line;
362     @ColumnParam("")
363     uint column;
364 
365     @ColumnParam("")
366     uint line_end;
367 
368     @ColumnParam("")
369     uint column_end;
370 }
371 
372 @TableName(mutationTable)
373 @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
374 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
375 @TableConstraint("unique_ UNIQUE (mp_id, st_id, kind)")
376 struct MutationTbl {
377     long id;
378 
379     long mp_id;
380 
381     long st_id;
382 
383     long kind;
384 }
385 
386 /**
387  * This could use an intermediate adapter table to normalise the test_case data
388  * but I chose not to do that because it makes it harder to add test cases and
389  * do a cleanup.
390  */
391 @TableName(killedTestCaseTable)
392 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
393 @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE"))
394 @TableConstraint("unique_ UNIQUE (st_id, tc_id)")
395 struct TestCaseKilledTbl {
396     long id;
397 
398     @ColumnName("st_id")
399     long mutationStatusId;
400     @ColumnName("tc_id")
401     long testCaseId;
402 
403     // location is a filesystem location or other suitable helper for a user to
404     // locate the test.
405     @ColumnParam("")
406     string location;
407 }
408 
409 /**
410  * Track all test cases that has been found by the test suite output analyzer.
411  * Useful to find test cases that has never killed any mutant.
412  * name should match test_case_killed_v2_tbl
413  * TODO: name should be the primary key. on a conflict a counter should be updated.
414  */
415 @TableName(allTestCaseTable)
416 @TableConstraint("unique_ UNIQUE (name)")
417 struct AllTestCaseTbl {
418     long id;
419     string name;
420 
421     @ColumnName("is_new")
422     bool isNew;
423 }
424 
425 /**
426  * The status of a mutant. if it is killed or otherwise. multiple mutation
427  * operators can result in the same change of the source code. By coupling the
428  * mutant status to the checksum of the source code change it means that two
429  * mutations that have the same checksum will "cooperate".
430  *
431  * compile_time_ms = time it took to compile the program for the mutant
432  * test_time_ms = time it took to run the test suite
433  * updated_ts = is when the status where last updated. Seconds at UTC+0.
434  * added_ts = when the mutant where added to the system. UTC+0.
435  */
436 @TableName(mutationStatusTable)
437 struct MutationStatusTbl {
438     // 64 bit checksum.
439     long id;
440 
441     /// Mutation.Status
442     long status;
443 
444     @ColumnName("exit_code")
445     int exitCode;
446 
447     @ColumnName("compile_time_ms")
448     long compileTimeMs;
449 
450     @ColumnName("test_time_ms")
451     long testTimeMs;
452 
453     @ColumnParam("")
454     @ColumnName("update_ts")
455     SysTime updated;
456 
457     @ColumnParam("")
458     @ColumnName("added_ts")
459     SysTime added;
460 
461     /// Priority of the mutant used when testing.
462     long prio;
463 }
464 
465 /// Mutants that should be tested.
466 @TableName(mutantWorklistTable)
467 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
468 struct MutantWorklistTbl {
469     long id;
470     long prio;
471 }
472 
473 /** Memory overload mutants that are re-tested one extra time.
474  */
475 @TableName(mutantMemOverloadWorklistTable)
476 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
477 struct MutantMemOverloadtWorklistTbl {
478     long id;
479 }
480 
481 /** Timeout mutants that are re-tested until none of them change status from
482  * timeout.
483  */
484 @TableName(mutantTimeoutWorklistTable)
485 @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
486 struct MutantTimeoutWorklistTbl {
487     long id;
488     /// iterations that the mutant have been tested.
489     long iter;
490 }
491 
492 /** The defaults for the schema is the state that the state machine start in.
493  *
494  * This mean that if there are nothing in the database then `.init` is the correct starting point.
495  */
496 @TableName(mutantTimeoutCtxTable)
497 struct MutantTimeoutCtxTbl {
498     /// What iteration the timeout testing is at.
499     long iter;
500 
501     /// Last count of the mutants in the worklist that where in the timeout state.
502     long worklistCount;
503 
504     enum State {
505         init_,
506         running,
507         done
508     }
509 
510     /// State of the timeout algorithm.
511     State state;
512 }
513 
514 /** The checksum is used as primary key to reduce the need to "peek" into the
515  * database when updating, adding, linking tables.
516  *
517  * Both `st_id` and `mut_id` are values that sqlite can reuse between analyzes
518  * if they have been previously removed thus the only assured connection
519  * between a marked mutant and future code changes is the checksum.
520  */
521 @TableName(markedMutantTable)
522 @TablePrimaryKey("checksum")
523 struct MarkedMutantTbl {
524     // TODO: can be removed in the future considering mutationStatusId is the
525     // checksum from v55+.
526     /// Checksum of the mutant status the marking is related to.
527     /// it is the mutationStatusId id.
528     long checksum;
529 
530     /// updated each analyze.
531     @ColumnName("st_id")
532     long mutationStatusId;
533 
534     uint line;
535     uint column;
536     string path;
537 
538     /// The status it should always be changed to.
539     long toStatus;
540 
541     /// Time when the mutant where marked.
542     SysTime time;
543 
544     string rationale;
545 
546     string mutText;
547 }
548 
549 @TableName(schemaFragmentV2Table)
550 @TableForeignKey("file_id", KeyRef(filesTable ~ "(id)"), KeyParam("ON DELETE CASCADE"))
551 @TableConstraint("unique_ UNIQUE (file_id, text, offset_begin, offset_end)")
552 struct SchemaFragmentV2Table {
553     long id;
554 
555     @ColumnName("file_id")
556     long fileId;
557 
558     @ColumnParam("")
559     const(ubyte)[] text;
560 
561     @ColumnName("offset_begin")
562     uint offsetBegin;
563     @ColumnName("offset_end")
564     uint offsetEnd;
565 }
566 
567 @TableName(schemaMutantV2Table)
568 @TableForeignKey("fragment_id", KeyRef(schemaFragmentV2Table ~ "(id)"),
569         KeyParam("ON DELETE CASCADE"))
570 @TableForeignKey("st_id", KeyRef(mutationStatusTable ~ "(id)"), KeyParam("ON DELETE CASCADE"))
571 @TableConstraint("unique_ UNIQUE (fragment_id, st_id)")
572 struct SchemaMutantV2Table {
573     @ColumnName("fragment_id")
574     long fragmentId;
575 
576     @ColumnName("st_id")
577     long statusId;
578 }
579 
580 @TableName(schemaMutantQTable)
581 @TableConstraint("unique_ UNIQUE (kind, path)")
582 struct SchemaMutantKindQTable {
583     long id;
584 
585     /// mutant subtype
586     long kind;
587 
588     // max 100
589     long probability;
590 
591     // 64 bit checksum of the path
592     long path;
593 }
594 
595 @TableName(schemaSizeQTable)
596 struct SchemaSizeQTable {
597     long id;
598     long size;
599 }
600 
601 /** The runtime of the test commands.
602  *
603  * By storing the data it reduces the need to run the test suite multiple times
604  * to get the minimum.
605  */
606 @TableName(runtimeHistoryTable)
607 struct RuntimeHistoryTable {
608     long id;
609 
610     /// when the measurement was taken.
611     @ColumnName("time")
612     SysTime timeStamp;
613 
614     @ColumnName("time_ms")
615     long timeMs;
616 }
617 
618 @TableName(mutationScoreHistoryTable)
619 @TableConstraint("time UNIQUE (time)")
620 struct MutationScoreHistoryTable {
621     long id;
622 
623     /// when the measurement was taken.
624     @ColumnName("time")
625     SysTime timeStamp;
626 
627     double score;
628 }
629 
630 @TableName(mutationFileScoreHistoryTable)
631 @TableConstraint("date UNIQUE (time_stamp, file_path)")
632 struct MutationFileScoreHistoryTable {
633     long id;
634 
635     /// when the measurement was taken.
636     @ColumnName("time_stamp")
637     SysTime timeStamp;
638 
639     double score;
640 
641     @ColumnName("file_path")
642     string filePath;
643 }
644 
645 /** All functions that has been discovered in the source code.
646  * The `offset_begin` is where instrumentation points can be injected.
647  */
648 @TableName(srcCovTable)
649 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
650 @TableConstraint("file_offset UNIQUE (file_id, begin, end)")
651 struct CoverageCodeRegionTable {
652     long id;
653 
654     @ColumnName("file_id")
655     long fileId;
656 
657     /// the region in the files, in bytes.
658     uint begin;
659     uint end;
660 }
661 
662 /** Each coverage region that has a valid status has an entry in this table.
663  * It do mean that there can be region that do not exist in this table. That
664  * mean that something went wrong when gathering the data.
665  */
666 @TableName(srcCovInfoTable)
667 @TableForeignKey("id", KeyRef("src_cov_instr(id)"), KeyParam("ON DELETE CASCADE"))
668 struct CoverageInfoTable {
669     long id;
670 
671     /// True if the region has been visited.
672     bool status;
673 }
674 
675 /// When the coverage information was gathered.
676 @TableName(srcCovTimeStampTable)
677 struct CoverageTimeTtampTable {
678     long id;
679 
680     @ColumnName("timestamp")
681     SysTime timeStamp;
682 }
683 
684 /** Files that roots are dependent on. They do not need to contain mutants.
685  */
686 @TableName(depFileTable)
687 @TableConstraint("unique_ UNIQUE (file)")
688 struct DependencyFileTable {
689     long id;
690 
691     string file;
692 
693     /// checksum is 64bit.
694     long checksum;
695 }
696 
697 @TableName(depRootTable)
698 @TableForeignKey("dep_id", KeyRef("dependency_file(id)"), KeyParam("ON DELETE CASCADE"))
699 @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
700 @TableConstraint("unique_ UNIQUE (dep_id, file_id)")
701 struct DependencyRootTable {
702     @ColumnName("dep_id")
703     long depFileId;
704 
705     @ColumnName("file_id")
706     long fileId;
707 }
708 
709 @TableName(testCmdOriginalTable)
710 @TablePrimaryKey("checksum")
711 @TableConstraint("unique_ UNIQUE (cmd_id)")
712 @TableForeignKey("cmd_id", KeyRef("test_cmd(id)"), KeyParam("ON DELETE CASCADE"))
713 struct TestCmdOriginalTable {
714     long checksum;
715 
716     @ColumnName("cmd_id")
717     long testCmdId;
718 }
719 
720 @TableName(testCmdMutatedTable)
721 @TablePrimaryKey("checksum")
722 struct TestCmdMutatedTable {
723     long checksum;
724 
725     /// Mutation.Status
726     long status;
727 
728     /// when the measurement was taken.
729     @ColumnName("timestamp")
730     SysTime timeStamp;
731 }
732 
733 @TableName(testCmdTable)
734 @TableConstraint("unique_ UNIQUE (cmd)")
735 struct TestCmdTable {
736     long id;
737     string cmd;
738 }
739 
740 @TableName(testCmdRelMutantTable)
741 @TableForeignKey("cmd_id", KeyRef("test_cmd(id)"), KeyParam("ON DELETE CASCADE"))
742 @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
743 @TableConstraint("unique_ UNIQUE (cmd_id, st_id)")
744 struct TestCmdRelMutantTable {
745     long id;
746 
747     @ColumnName("cmd_id")
748     long testCmdId;
749 
750     @ColumnName("st_id")
751     long statusId;
752 }
753 
754 void updateSchemaVersion(ref Miniorm db, long ver) nothrow {
755     try {
756         db.run(delete_!VersionTbl);
757         db.run(insert!VersionTbl.insert, VersionTbl(ver));
758     } catch (Exception e) {
759         logger.error(e.msg).collectException;
760     }
761 }
762 
763 long getSchemaVersion(ref Miniorm db) {
764     auto v = db.run(select!VersionTbl);
765     return v.empty ? 0 : v.front.version_;
766 }
767 
768 void upgrade(ref Miniorm db, UpgradeTable tbl) {
769     import d2sqlite3;
770 
771     immutable maxIndex = 30;
772 
773     alias upgradeFunc = void function(ref Miniorm db);
774 
775     bool hasUpdated;
776 
777     bool running = true;
778     while (running) {
779         const version_ = () {
780             // first time the version table do not exist thus fail.
781             try {
782                 return getSchemaVersion(db);
783             } catch (Exception e) {
784             }
785             return 0;
786         }();
787 
788         if (version_ >= tbl.latestSchemaVersion) {
789             running = false;
790             break;
791         }
792 
793         logger.infof("Upgrading database from %s", version_).collectException;
794 
795         if (!hasUpdated)
796             try {
797                 // only do this once and always before any changes to the database.
798                 foreach (i; 0 .. maxIndex) {
799                     db.run(format!"DROP INDEX IF EXISTS i%s"(i));
800                 }
801             } catch (Exception e) {
802                 logger.warning(e.msg).collectException;
803                 logger.warning("Unable to drop database indexes").collectException;
804             }
805 
806         if (auto f = version_ in tbl) {
807             try {
808                 hasUpdated = true;
809 
810                 (*f)(db);
811                 if (version_ != 0)
812                     updateSchemaVersion(db, version_ + 1);
813             } catch (Exception e) {
814                 logger.trace(e).collectException;
815                 logger.error(e.msg).collectException;
816                 logger.warningf("Unable to upgrade a database of version %s",
817                         version_).collectException;
818                 logger.warning("This might impact the functionality. It is unwise to continue")
819                     .collectException;
820                 throw e;
821             }
822         } else {
823             logger.info("Upgrade successful").collectException;
824             running = false;
825         }
826     }
827 
828     // add indexes assuming the lastest database schema
829     if (hasUpdated)
830         try {
831             int i;
832             db.run(format!"CREATE INDEX i%s ON %s(path)"(i++, filesTable));
833             db.run(format!"CREATE INDEX i%s ON %s(path)"(i++, testFilesTable));
834 
835             // improve getTestCaseMutantKills by 10x
836             db.run(format!"CREATE INDEX i%s ON %s(tc_id,st_id)"(i++, killedTestCaseTable));
837             db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, mutationTable));
838 
839             // all on delete cascade
840             db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, rawSrcMetadataTable));
841             db.run(format!"CREATE INDEX i%s ON %s(mut_id)"(i++, srcMetadataTable));
842             db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, srcMetadataTable));
843             db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, srcMetadataTable));
844             db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, srcMetadataTable));
845             db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, nomutTable));
846             db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, nomutDataTable));
847             db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, nomutDataTable));
848             db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, mutationPointTable));
849             db.run(format!"CREATE INDEX i%s ON %s(mp_id)"(i++, mutationTable));
850             db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, mutationTable));
851             db.run(format!"CREATE INDEX i%s ON %s(st_id)"(i++, killedTestCaseTable));
852             db.run(format!"CREATE INDEX i%s ON %s(tc_id)"(i++, killedTestCaseTable));
853             db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, srcCovTable));
854             db.run(format!"CREATE INDEX i%s ON %s(dep_id)"(i++, depRootTable));
855             db.run(format!"CREATE INDEX i%s ON %s(file_id)"(i++, depRootTable));
856 
857             assert(i <= maxIndex);
858         } catch (Exception e) {
859             logger.warning(e.msg).collectException;
860             logger.warning("Unable to create database indexes").collectException;
861         }
862 }
863 
864 /** If the database start it version 0, not initialized, then initialize to the
865  * latest schema version.
866  */
867 void upgradeV0(ref Miniorm db) {
868     auto tbl = makeUpgradeTable;
869 
870     db.run(buildSchema!(VersionTbl, RawSrcMetadata, FilesTbl,
871             MutationPointTbl, MutationTbl, TestCaseKilledTbl, AllTestCaseTbl,
872             MutationStatusTbl, MutantTimeoutCtxTbl, MutantTimeoutWorklistTbl,
873             MarkedMutantTbl, SrcMetadataTable, NomutTbl, NomutDataTbl,
874             NomutDataTbl, SchemaMutantKindQTable, SchemaSizeQTable,
875             MutantWorklistTbl,
876             RuntimeHistoryTable,
877             MutationScoreHistoryTable,
878             MutationFileScoreHistoryTable, TestFilesTable, CoverageCodeRegionTable,
879             CoverageInfoTable, CoverageTimeTtampTable, DependencyFileTable,
880             DependencyRootTable, DextoolVersionTable, ConfigVersionTable,
881             TestCmdOriginalTable,
882             TestCmdMutatedTable,
883             MutantMemOverloadtWorklistTbl, TestCmdRelMutantTable,
884             TestCmdTable, SchemaMutantV2Table, SchemaFragmentV2Table));
885 
886     updateSchemaVersion(db, tbl.latestSchemaVersion);
887 }
888 
889 /// 2018-04-08
890 void upgradeV1(ref Miniorm db) {
891     @TableName(allTestCaseTable)
892     struct AllTestCaseTbl {
893         long id;
894 
895         @ColumnParam("")
896         string name;
897     }
898 
899     @TableName(testCaseTableV1)
900     @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE"))
901     static struct TestCaseKilledTblV1 {
902         ulong id;
903 
904         @ColumnName("mut_id")
905         ulong mutantId;
906 
907         /// test_case is whatever identifier the user choose.
908         @ColumnName("test_case")
909         string testCase;
910     }
911 
912     db.run(buildSchema!(TestCaseKilledTblV1, AllTestCaseTbl));
913 }
914 
915 /// 2018-04-22
916 void upgradeV2(ref Miniorm db) {
917     @TableName(filesTable)
918     static struct FilesTbl {
919         ulong id;
920 
921         @ColumnParam("")
922         string path;
923 
924         ulong checksum0;
925         ulong checksum1;
926         Language lang;
927     }
928 
929     immutable newTbl = "new_" ~ filesTable;
930 
931     db.run(buildSchema!FilesTbl("new_"));
932     db.run(format("INSERT INTO %s (id,path,checksum0,checksum1) SELECT * FROM %s",
933             newTbl, filesTable));
934     db.replaceTbl(newTbl, filesTable);
935 }
936 
937 /// 2018-09-01
938 void upgradeV3(ref Miniorm db) {
939     @TableName(killedTestCaseTable)
940     @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE"))
941     struct TestCaseKilledTblV2 {
942         ulong id;
943 
944         @ColumnName("mut_id")
945         ulong mutantId;
946 
947         @ColumnParam("")
948         string name;
949 
950         // location is a filesystem location or other suitable helper for a user to
951         // locate the test.
952         @ColumnParam("")
953         string location;
954     }
955 
956     db.run(buildSchema!TestCaseKilledTblV2);
957     db.run(format("INSERT INTO %s (id,mut_id,name) SELECT * FROM %s",
958             killedTestCaseTable, testCaseTableV1));
959     db.run(format("DROP TABLE %s", testCaseTableV1));
960 
961     db.run(buildSchema!AllTestCaseTbl);
962 }
963 
964 /// 2018-09-24
965 void upgradeV4(ref Miniorm db) {
966     @TableName(killedTestCaseTable)
967     @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE"))
968     @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE"))
969     static struct TestCaseKilledTblV3 {
970         ulong id;
971 
972         @ColumnName("mut_id")
973         ulong mutantId;
974         @ColumnName("tc_id")
975         ulong testCaseId;
976 
977         // location is a filesystem location or other suitable helper for a user to
978         // locate the test.
979         @ColumnParam("")
980         string location;
981     }
982 
983     immutable newTbl = "new_" ~ killedTestCaseTable;
984 
985     db.run(buildSchema!TestCaseKilledTblV3("new_"));
986 
987     // add all missing test cases to all_test_case
988     db.run(format("INSERT INTO %s (name) SELECT DISTINCT t1.name FROM %s t1 LEFT JOIN %s t2 ON t2.name = t1.name WHERE t2.name IS NULL",
989             allTestCaseTable, killedTestCaseTable, allTestCaseTable));
990     // https://stackoverflow.com/questions/2686254/how-to-select-all-records-from-one-table-that-do-not-exist-in-another-table
991     //Q: What is happening here?
992     //
993     //A: Conceptually, we select all rows from table1 and for each row we
994     //attempt to find a row in table2 with the same value for the name column.
995     //If there is no such row, we just leave the table2 portion of our result
996     //empty for that row. Then we constrain our selection by picking only those
997     //rows in the result where the matching row does not exist. Finally, We
998     //ignore all fields from our result except for the name column (the one we
999     //are sure that exists, from table1).
1000     //
1001     //While it may not be the most performant method possible in all cases, it
1002     //should work in basically every database engine ever that attempts to
1003     //implement ANSI 92 SQL
1004 
1005     // This do NOT WORK. The result is that that this upgrade is broken because
1006     // it drops all maps between killed_test_case and mutation.
1007     //db.run(format("INSERT INTO %s (id,mut_id,tc_id,location) SELECT t1.id,t1.mut_id,t2.id,t1.location FROM %s t1 INNER JOIN %s t2 ON t1.name = t2.name",
1008     //        newTbl, killedTestCaseTable, allTestCaseTable));
1009 
1010     db.replaceTbl(newTbl, killedTestCaseTable);
1011 }
1012 
1013 /** 2018-09-30
1014  *
1015  * This upgrade will drop all existing mutations and thus all results.
1016  * It is too complex trying to upgrade and keep the results.
1017  *
1018  * When removing this function also remove the status field in mutation_v2_tbl.
1019  */
1020 void upgradeV5(ref Miniorm db) {
1021     @TableName(filesTable)
1022     @TableConstraint("unique_ UNIQUE (path)")
1023     struct FilesTbl {
1024         long id;
1025 
1026         @ColumnParam("")
1027         string path;
1028 
1029         /// checksum is 128bit.
1030         long checksum0;
1031         long checksum1;
1032         Language lang;
1033     }
1034 
1035     @TableName(mutationTable)
1036     @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
1037     @TableForeignKey("st_id", KeyRef("mutation_status(id)"))
1038     @TableConstraint("unique_ UNIQUE (mp_id, kind)")
1039     static struct MutationTbl {
1040         ulong id;
1041 
1042         ulong mp_id;
1043 
1044         @ColumnParam("")
1045         ulong st_id;
1046 
1047         ulong kind;
1048 
1049         @ColumnParam("")
1050         ulong status;
1051 
1052         /// time in ms spent on verifying the mutant
1053         @ColumnParam("")
1054         ulong time;
1055     }
1056 
1057     @TableName(mutationStatusTable)
1058     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1059     static struct MutationStatusTbl {
1060         ulong id;
1061         ulong status;
1062         ulong checksum0;
1063         ulong checksum1;
1064     }
1065 
1066     immutable new_mut_tbl = "new_" ~ mutationTable;
1067     db.run(buildSchema!MutationStatusTbl);
1068 
1069     db.run(format("DROP TABLE %s", mutationTable));
1070     db.run(buildSchema!MutationTbl);
1071 
1072     immutable newFilesTbl = "new_" ~ filesTable;
1073     db.run(buildSchema!FilesTbl("new_"));
1074     db.run(format("INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang) SELECT * FROM %s",
1075             newFilesTbl, filesTable));
1076     db.replaceTbl(newFilesTbl, filesTable);
1077 }
1078 
1079 /// 2018-10-11
1080 void upgradeV6(ref Miniorm db) {
1081     @TableName(mutationStatusTable)
1082     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1083     static struct MutationStatusTbl {
1084         ulong id;
1085         ulong status;
1086         ulong time;
1087         SysTime timestamp;
1088         ulong checksum0;
1089         ulong checksum1;
1090     }
1091 
1092     @TableName(mutationTable)
1093     @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
1094     @TableForeignKey("st_id", KeyRef("mutation_status(id)"))
1095     @TableConstraint("unique_ UNIQUE (mp_id, kind)")
1096     struct MutationTbl {
1097         long id;
1098 
1099         long mp_id;
1100 
1101         @ColumnParam("")
1102         long st_id;
1103 
1104         long kind;
1105     }
1106 
1107     immutable new_mut_tbl = "new_" ~ mutationTable;
1108 
1109     db.run(buildSchema!MutationTbl("new_"));
1110 
1111     db.run(format("INSERT INTO %s (id,mp_id,st_id,kind) SELECT id,mp_id,st_id,kind FROM %s",
1112             new_mut_tbl, mutationTable));
1113     db.replaceTbl(new_mut_tbl, mutationTable);
1114 
1115     immutable new_muts_tbl = "new_" ~ mutationStatusTable;
1116     db.run(buildSchema!MutationStatusTbl("new_"));
1117     db.run(format("INSERT INTO %s (id,status,checksum0,checksum1) SELECT id,status,checksum0,checksum1 FROM %s",
1118             new_muts_tbl, mutationStatusTable));
1119     db.replaceTbl(new_muts_tbl, mutationStatusTable);
1120 }
1121 
1122 /// 2018-10-15
1123 void upgradeV7(ref Miniorm db) {
1124     @TableName(killedTestCaseTable)
1125     @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1126     @TableForeignKey("tc_id", KeyRef("all_test_case(id)"), KeyParam("ON DELETE CASCADE"))
1127     struct TestCaseKilledTbl {
1128         long id;
1129 
1130         @ColumnName("st_id")
1131         long mutationStatusId;
1132         @ColumnName("tc_id")
1133         long testCaseId;
1134 
1135         // location is a filesystem location or other suitable helper for a user to
1136         // locate the test.
1137         @ColumnParam("")
1138         string location;
1139     }
1140 
1141     immutable newTbl = "new_" ~ killedTestCaseTable;
1142 
1143     db.run(buildSchema!TestCaseKilledTbl("new_"));
1144 
1145     db.run(format("INSERT INTO %s (id,st_id,tc_id,location)
1146         SELECT t0.id,t1.st_id,t0.tc_id,t0.location
1147         FROM %s t0, %s t1
1148         WHERE
1149         t0.mut_id = t1.id", newTbl, killedTestCaseTable, mutationTable));
1150 
1151     db.replaceTbl(newTbl, killedTestCaseTable);
1152 }
1153 
1154 /// 2018-10-20
1155 void upgradeV8(ref Miniorm db) {
1156     immutable newTbl = "new_" ~ mutationPointTable;
1157     db.run(buildSchema!MutationPointTbl("new_"));
1158     db.run(format("INSERT INTO %s (id,file_id,offset_begin,offset_end,line,column)
1159         SELECT t0.id,t0.file_id,t0.offset_begin,t0.offset_end,t0.line,t0.column
1160         FROM %s t0", newTbl, mutationPointTable));
1161 
1162     db.replaceTbl(newTbl, mutationPointTable);
1163 }
1164 
1165 /// 2018-11-10
1166 void upgradeV9(ref Miniorm db) {
1167     @TableName(mutationStatusTable)
1168     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1169     struct MutationStatusTbl {
1170         long id;
1171         long status;
1172 
1173         @ColumnParam("")
1174         long time;
1175 
1176         @ColumnName("test_cnt")
1177         long testCnt;
1178 
1179         @ColumnParam("")
1180         @ColumnName("update_ts")
1181         SysTime updated;
1182 
1183         @ColumnParam("")
1184         @ColumnName("added_ts")
1185         SysTime added;
1186 
1187         long checksum0;
1188         long checksum1;
1189     }
1190 
1191     immutable newTbl = "new_" ~ mutationStatusTable;
1192     db.run(buildSchema!MutationStatusTbl("new_"));
1193     db.run(format("INSERT INTO %s (id,status,time,test_cnt,update_ts,checksum0,checksum1)
1194         SELECT t0.id,t0.status,t0.time,0,t0.timestamp,t0.checksum0,t0.checksum1
1195         FROM %s t0", newTbl, mutationStatusTable));
1196 
1197     replaceTbl(db, newTbl, mutationStatusTable);
1198 }
1199 
1200 /// 2018-11-25
1201 void upgradeV10(ref Miniorm db) {
1202     @TableName(rawSrcMetadataTable)
1203     @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
1204     @TableConstraint("unique_line_in_file UNIQUE (file_id, line)")
1205     struct RawSrcMetadata {
1206         ulong id;
1207 
1208         @ColumnName("file_id")
1209         ulong fileId;
1210 
1211         @ColumnParam("")
1212         uint line;
1213 
1214         @ColumnParam("")
1215         ulong nomut;
1216     }
1217 
1218     db.run(buildSchema!RawSrcMetadata);
1219     void makeSrcMetadataView(ref Miniorm db) {
1220         // check if a NOMUT is on or between the start and end of a mutant.
1221         immutable src_metadata_v1_tbl = "CREATE VIEW %s
1222             AS
1223             SELECT
1224             t0.id AS mut_id,
1225             t1.id AS st_id,
1226             t2.id AS mp_id,
1227             t3.id AS file_id,
1228             (SELECT count(*) FROM %s in_t0, %s in_t1
1229              WHERE
1230              in_t0.file_id = in_t1.file_id AND
1231              t0.mp_id = in_t0.id AND
1232              (in_t1.line BETWEEN in_t0.line AND in_t0.line_end)) AS nomut
1233                 FROM %s t0, %s t1, %s t2, %s t3
1234                 WHERE
1235                 t0.mp_id = t2.id AND
1236                 t0.st_id = t1.id AND
1237                 t2.file_id = t3.id
1238                 ";
1239 
1240         db.run(format(src_metadata_v1_tbl, srcMetadataTable, mutationPointTable, rawSrcMetadataTable,
1241                 mutationTable, mutationStatusTable, mutationPointTable, filesTable));
1242     }
1243 
1244     makeSrcMetadataView(db);
1245 }
1246 
1247 /// 2019-04-06
1248 void upgradeV11(ref Miniorm db) {
1249     immutable newTbl = "new_" ~ rawSrcMetadataTable;
1250     db.run(buildSchema!RawSrcMetadata("new_"));
1251     db.run(format!"INSERT INTO %s (id,file_id,line,nomut) SELECT t.id,t.file_id,t.line,t.nomut FROM %s t"(newTbl,
1252             rawSrcMetadataTable));
1253     replaceTbl(db, newTbl, rawSrcMetadataTable);
1254 
1255     db.run(format("DROP VIEW %s", srcMetadataTable)).collectException;
1256 
1257     // Associate metadata from lines with the mutation status.
1258     void makeSrcMetadataView(ref Miniorm db) {
1259         // check if a NOMUT is on or between the start and end of a mutant.
1260         immutable src_metadata_tbl = "CREATE VIEW %s
1261         AS
1262         SELECT DISTINCT
1263         t0.id AS mut_id,
1264         t1.id AS st_id,
1265         t2.id AS mp_id,
1266         t3.id AS file_id,
1267         (SELECT count(*) FROM %s WHERE nomut.mp_id = t2.id) as nomut
1268         FROM %s t0, %s t1, %s t2, %s t3
1269         WHERE
1270         t0.mp_id = t2.id AND
1271         t0.st_id = t1.id AND
1272         t2.file_id = t3.id";
1273         db.run(format(src_metadata_tbl, srcMetadataTable, nomutTable,
1274                 mutationTable, mutationStatusTable, mutationPointTable, filesTable));
1275 
1276         immutable nomut_tbl = "CREATE VIEW %s
1277         AS
1278         SELECT
1279         t0.id mp_id,
1280         t1.line line,
1281         count(*) status
1282         FROM %s t0, %s t1
1283         WHERE
1284         t0.file_id = t1.file_id AND
1285         (t1.line BETWEEN t0.line AND t0.line_end)
1286         GROUP BY
1287         t0.id";
1288         db.run(format(nomut_tbl, nomutTable, mutationPointTable, rawSrcMetadataTable));
1289 
1290         immutable nomut_data_tbl = "CREATE VIEW %s
1291         AS
1292         SELECT
1293         t0.id as mut_id,
1294         t0.mp_id as mp_id,
1295         t1.line as line,
1296         t1.tag as tag,
1297         t1.comment as comment
1298         FROM %s t0, %s t1, %s t2
1299         WHERE
1300         t0.mp_id = t2.mp_id AND
1301         t1.line = t2.line";
1302         db.run(format(nomut_data_tbl, nomutDataTable, mutationTable,
1303                 rawSrcMetadataTable, nomutTable));
1304     }
1305 
1306     makeSrcMetadataView(db);
1307 }
1308 
1309 /// 2019-08-28
1310 void upgradeV12(ref Miniorm db) {
1311     /** Timeout mutants that are re-tested until none of them change status from
1312      * timeout.
1313      */
1314     @TableName(mutantTimeoutWorklistTable)
1315     @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1316     struct MutantTimeoutWorklistTbl {
1317         long id;
1318     }
1319 
1320     db.run(buildSchema!(MutantTimeoutCtxTbl, MutantTimeoutWorklistTbl));
1321 }
1322 
1323 /// 2019-11-12
1324 void upgradeV13(ref Miniorm db) {
1325     @TableName(markedMutantTable)
1326     @TablePrimaryKey("st_id")
1327     struct MarkedMutantTbl {
1328         @ColumnName("st_id")
1329         long mutationStatusId;
1330 
1331         @ColumnName("mut_id")
1332         long mutationId;
1333 
1334         uint line;
1335 
1336         uint column;
1337 
1338         string path;
1339 
1340         @ColumnName("to_status")
1341         ulong toStatus;
1342 
1343         SysTime time;
1344 
1345         string rationale;
1346 
1347         @ColumnName("mut_text")
1348         string mutText;
1349     }
1350 
1351     db.run(buildSchema!(MarkedMutantTbl));
1352 }
1353 
1354 /// 2020-01-12
1355 void upgradeV14(ref Miniorm db) {
1356     @TableName(nomutDataTable)
1357     @TableForeignKey("mut_id", KeyRef("mutation(id)"), KeyParam("ON DELETE CASCADE"))
1358     @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
1359     struct NomutDataTbl {
1360         @ColumnName("mut_id")
1361         long mutationId;
1362 
1363         @ColumnName("mp_id")
1364         long mutationPointId;
1365 
1366         long line;
1367 
1368         @ColumnParam("")
1369         string tag;
1370 
1371         @ColumnParam("")
1372         string comment;
1373     }
1374 
1375     db.run(format!"DROP VIEW %s"(srcMetadataTable));
1376     db.run(format!"DROP VIEW %s"(nomutTable));
1377     db.run(format!"DROP VIEW %s"(nomutDataTable));
1378 
1379     db.run(buildSchema!(SrcMetadataTable, NomutTbl, NomutDataTbl));
1380     logger.info("Re-execute analyze to update the NOMUT data");
1381 }
1382 
1383 /// 2020-01-21
1384 void upgradeV15(ref Miniorm db) {
1385     @TableName(markedMutantTable)
1386     @TablePrimaryKey("checksum0")
1387     struct MarkedMutantTbl {
1388         /// Checksum of the mutant status the marking is related to.
1389         long checksum0;
1390         long checksum1;
1391 
1392         /// updated each analyze.
1393         @ColumnName("st_id")
1394         long mutationStatusId;
1395 
1396         /// updated each analyze.
1397         @ColumnName("mut_id")
1398         long mutationId;
1399 
1400         uint line;
1401         uint column;
1402         string path;
1403 
1404         /// The status it should always be changed to.
1405         long toStatus;
1406 
1407         /// Time when the mutant where marked.
1408         SysTime time;
1409 
1410         string rationale;
1411 
1412         string mutText;
1413     }
1414 
1415     // fix bug in the marked mutant table
1416     db.run(format!"DROP TABLE %s"(markedMutantTable));
1417     db.run(buildSchema!MarkedMutantTbl);
1418     logger.info("Dropping all marked mutants because of database changes");
1419 }
1420 
1421 /// 2020-02-12
1422 void upgradeV16(ref Miniorm db) {
1423     @TableName(schemataWorkListTable)
1424     @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
1425     static struct SchemataWorkListTable {
1426         long id;
1427     }
1428 
1429     @TableName(schemataMutantTable)
1430     @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1431     @TableForeignKey("schem_id", KeyRef("schemata_fragment(id)"), KeyParam("ON DELETE CASCADE"))
1432     static struct SchemataMutantTable {
1433         @ColumnName("st_id")
1434         long statusId;
1435         @ColumnName("schem_id")
1436         long schemaId;
1437     }
1438 
1439     @TableName(schemataFragmentTable)
1440     @TableForeignKey("schem_id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
1441     @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
1442     struct SchemataFragmentTable {
1443         long id;
1444 
1445         @ColumnName("schem_id")
1446         long schemataId;
1447 
1448         @ColumnName("file_id")
1449         long fileId;
1450 
1451         @ColumnName("order_")
1452         long order;
1453 
1454         @ColumnParam("")
1455         const(ubyte)[] text;
1456 
1457         @ColumnName("offset_begin")
1458         uint offsetBegin;
1459         @ColumnName("offset_end")
1460         uint offsetEnd;
1461     }
1462 
1463     db.run(buildSchema!(SchemataFragmentTable, SchemataWorkListTable, SchemataMutantTable));
1464 }
1465 
1466 /// 2020-02-12
1467 void upgradeV17(ref Miniorm db) {
1468     @TableName(schemataTable)
1469     static struct SchemataTable {
1470         long id;
1471     }
1472 
1473     db.run(buildSchema!(SchemataTable));
1474 }
1475 
1476 /// 2020-03-21
1477 void upgradeV18(ref Miniorm db) {
1478     // this force an old database to add indexes
1479 }
1480 
1481 /// 2020-04-01
1482 void upgradeV19(ref Miniorm db) {
1483     db.run("DROP TABLE " ~ schemataWorkListTable);
1484     db.run("DROP TABLE " ~ schemataTable);
1485     db.run("DROP TABLE " ~ schemataMutantTable);
1486 
1487     @TableName(invalidSchemataTable)
1488     @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
1489     struct InvalidSchemataTable {
1490         long id;
1491     }
1492 
1493     @TableName(schemataTable)
1494     struct SchemataTable {
1495         long id;
1496 
1497         // number of fragments the schemata consist of.
1498         // used to detect if a fragment has been removed because its related file
1499         // was changed.
1500         long fragments;
1501 
1502         // runtime generated constant that make it possible to "prune" old
1503         // schematas automatically. it assumes that each new version of dextool may
1504         // contain updates to the schematas thus the old schemats should be
1505         // removed.
1506         @ColumnName("version")
1507         long version_;
1508     }
1509 
1510     db.run(buildSchema!(SchemataTable, InvalidSchemataTable));
1511 }
1512 
1513 /// 2020-06-01
1514 void upgradeV20(ref Miniorm db) {
1515     @TableName(schemataMutantTable)
1516     @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1517     @TableForeignKey("schem_id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
1518     @TableConstraint("unique_ UNIQUE (st_id, schem_id)")
1519     struct SchemataMutantTable {
1520         @ColumnName("st_id")
1521         long statusId;
1522         @ColumnName("schem_id")
1523         long schemaId;
1524     }
1525 
1526     db.run("DROP TABLE " ~ schemataMutantTable);
1527     db.run(buildSchema!(SchemataMutantTable));
1528 }
1529 
1530 /// 2020-11-28
1531 void upgradeV21(ref Miniorm db) {
1532     @TableName(schemataUsedTable)
1533     @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
1534     struct SchemataUsedTable {
1535         long id;
1536     }
1537 
1538     db.run("DROP TABLE " ~ invalidSchemataTable);
1539     db.run(buildSchema!(SchemataUsedTable));
1540 }
1541 
1542 /// 2020-11-28
1543 void upgradeV22(ref Miniorm db) {
1544     @TableName(mutantWorklistTable)
1545     @TableForeignKey("id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1546     struct MutantWorklistTbl {
1547         long id;
1548     }
1549 
1550     db.run(buildSchema!(MutantWorklistTbl));
1551 }
1552 
1553 /// 2020-12-06
1554 void upgradeV23(ref Miniorm db) {
1555     db.run(buildSchema!(RuntimeHistoryTable));
1556 }
1557 
1558 /// 2020-12-06
1559 void upgradeV24(ref Miniorm db) {
1560     @TableName(mutationScoreHistoryTable)
1561     struct MutationScoreHistoryTable {
1562         long id;
1563 
1564         /// when the measurement was taken.
1565         @ColumnName("time")
1566         SysTime timeStamp;
1567 
1568         double score;
1569     }
1570 
1571     db.run(buildSchema!(MutationScoreHistoryTable));
1572 }
1573 
1574 /// 2020-12-25
1575 void upgradeV25(ref Miniorm db) {
1576     import std.traits : EnumMembers;
1577     import dextool.plugin.mutate.backend.type : Mutation;
1578 
1579     @TableName(mutationStatusTable)
1580     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1581     struct MutationStatusTbl {
1582         long id;
1583         long status;
1584         @ColumnName("exit_code")
1585         int exitCode;
1586 
1587         @ColumnParam("")
1588         long time;
1589 
1590         @ColumnName("test_cnt")
1591         long testCnt;
1592 
1593         @ColumnParam("")
1594         @ColumnName("update_ts")
1595         SysTime updated;
1596 
1597         @ColumnParam("")
1598         @ColumnName("added_ts")
1599         SysTime added;
1600 
1601         long checksum0;
1602         long checksum1;
1603     }
1604 
1605     immutable newTbl = "new_" ~ mutationStatusTable;
1606     db.run(buildSchema!MutationStatusTbl("new_"));
1607 
1608     auto stmt = db.prepare(format(
1609             "INSERT INTO %s (id,status,exit_code,time,test_cnt,update_ts,added_ts,checksum0,checksum1)
1610         SELECT t.id,t.status,:ecode,t.time,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1
1611         FROM %s t WHERE t.status = :status", newTbl, mutationStatusTable));
1612 
1613     foreach (st; [EnumMembers!(Mutation.Status)]) {
1614         stmt.get.bind(":ecode", (st == Mutation.Status.killed) ? 1 : 0);
1615         stmt.get.bind(":status", cast(long) st);
1616         stmt.get.execute;
1617         stmt.get.reset;
1618     }
1619 
1620     replaceTbl(db, newTbl, mutationStatusTable);
1621 }
1622 
1623 /// 2020-12-25
1624 void upgradeV26(ref Miniorm db) {
1625     immutable newTbl = "new_" ~ killedTestCaseTable;
1626     db.run(buildSchema!TestCaseKilledTbl("new_"));
1627 
1628     db.run(format("INSERT OR IGNORE INTO %s (id, st_id, tc_id, location)
1629         SELECT t.id, t.st_id, t.tc_id, t.location
1630         FROM %s t", newTbl, killedTestCaseTable));
1631     replaceTbl(db, newTbl, killedTestCaseTable);
1632 }
1633 
1634 /// 2020-12-25
1635 void upgradeV27(ref Miniorm db) {
1636     @TableName(allTestCaseTable)
1637     @TableConstraint("unique_ UNIQUE (name)")
1638     struct AllTestCaseTbl {
1639         long id;
1640         string name;
1641     }
1642 
1643     immutable newTbl = "new_" ~ allTestCaseTable;
1644     db.run(buildSchema!AllTestCaseTbl("new_"));
1645 
1646     db.run(format("INSERT OR IGNORE INTO %s (id, name)
1647         SELECT t.id, t.name
1648         FROM %s t", newTbl, allTestCaseTable));
1649     replaceTbl(db, newTbl, allTestCaseTable);
1650 }
1651 
1652 /// 2020-12-25
1653 void upgradeV28(ref Miniorm db) {
1654     import dextool.plugin.mutate.backend.type : Mutation;
1655 
1656     @TableName(mutationStatusTable)
1657     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1658     struct MutationStatusTbl {
1659         long id;
1660         long status;
1661         @ColumnName("exit_code")
1662         int exitCode;
1663 
1664         @ColumnName("compile_time_ms")
1665         long compileTimeMs;
1666 
1667         @ColumnName("test_time_ms")
1668         long testTimeMs;
1669 
1670         @ColumnName("test_cnt")
1671         long testCnt;
1672 
1673         @ColumnParam("")
1674         @ColumnName("update_ts")
1675         SysTime updated;
1676 
1677         @ColumnParam("")
1678         @ColumnName("added_ts")
1679         SysTime added;
1680 
1681         long checksum0;
1682         long checksum1;
1683     }
1684 
1685     immutable newTbl = "new_" ~ mutationStatusTable;
1686     db.run(buildSchema!MutationStatusTbl("new_"));
1687 
1688     db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1)
1689         SELECT t.id,t.status,t.exit_code,0,t.time,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1
1690         FROM %s t WHERE t.time NOT NULL", newTbl, mutationStatusTable));
1691 
1692     db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1)
1693         SELECT t.id,t.status,t.exit_code,0,0,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1
1694         FROM %s t WHERE t.time IS NULL", newTbl, mutationStatusTable));
1695 
1696     replaceTbl(db, newTbl, mutationStatusTable);
1697 }
1698 
1699 /// 2020-12-27
1700 void upgradeV29(ref Miniorm db) {
1701     @TableName(testFilesTable)
1702     @TableConstraint("unique_ UNIQUE (path)")
1703     struct TestFilesTable {
1704         long id;
1705 
1706         string path;
1707 
1708         /// checksum is 128bit.
1709         long checksum0;
1710         long checksum1;
1711 
1712         /// Last time a change to the test file where detected.
1713         @ColumnName("timestamp")
1714         SysTime timeStamp;
1715     }
1716 
1717     db.run(buildSchema!(TestFilesTable));
1718 }
1719 
1720 /// 2020-12-29
1721 void upgradeV30(ref Miniorm db) {
1722     import std.datetime : Clock;
1723     import miniorm : toSqliteDateTime;
1724 
1725     @TableName(filesTable)
1726     @TableConstraint("unique_ UNIQUE (path)")
1727     struct FilesTbl {
1728         long id;
1729 
1730         string path;
1731 
1732         /// checksum is 128bit.
1733         long checksum0;
1734         long checksum1;
1735         Language lang;
1736 
1737         @ColumnName("timestamp")
1738         SysTime timeStamp;
1739     }
1740 
1741     immutable newFilesTbl = "new_" ~ filesTable;
1742     db.run(buildSchema!FilesTbl("new_"));
1743     auto stmt = db.prepare(format(
1744             "INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang,timestamp)
1745                   SELECT id,path,checksum0,checksum1,lang,:time FROM %s", newFilesTbl, filesTable));
1746     stmt.get.bind(":time", Clock.currTime.toSqliteDateTime);
1747     stmt.get.execute;
1748     db.replaceTbl(newFilesTbl, filesTable);
1749 }
1750 
1751 /// 2020-12-29
1752 void upgradeV31(ref Miniorm db) {
1753     @TableName(srcCovInfoTable)
1754     @TableForeignKey("id", KeyRef("src_cov_info(id)"), KeyParam("ON DELETE CASCADE"))
1755     struct CoverageInfoTable {
1756         long id;
1757 
1758         /// True if the region has been visited.
1759         bool status;
1760     }
1761 
1762     db.run(buildSchema!(CoverageCodeRegionTable, CoverageInfoTable, CoverageTimeTtampTable));
1763 }
1764 
1765 /// 2021-01-02
1766 void upgradeV32(ref Miniorm db) {
1767     @TableName(filesTable)
1768     @TableConstraint("unique_ UNIQUE (path)")
1769     struct FilesTbl {
1770         long id;
1771 
1772         string path;
1773 
1774         // checksum of the file content, 128bit.
1775         long checksum0;
1776         long checksum1;
1777         Language lang;
1778 
1779         @ColumnName("timestamp")
1780         SysTime timeStamp;
1781 
1782         /// True if the file is a root.
1783         bool root;
1784     }
1785 
1786     immutable newFilesTbl = "new_" ~ filesTable;
1787     db.run(buildSchema!FilesTbl("new_"));
1788     db.run(format("INSERT OR IGNORE INTO %s (id,path,checksum0,checksum1,lang,timestamp,root)
1789                   SELECT id,path,checksum0,checksum1,lang,timestamp,0 FROM %s",
1790             newFilesTbl, filesTable));
1791     db.replaceTbl(newFilesTbl, filesTable);
1792 }
1793 
1794 /// 2021-01-15
1795 void upgradeV33(ref Miniorm db) {
1796     @TableName(depFileTable)
1797     @TableConstraint("unique_ UNIQUE (file)")
1798     struct DependencyFileTable {
1799         long id;
1800 
1801         string file;
1802 
1803         /// checksum is 128bit.
1804         long checksum0;
1805         long checksum1;
1806     }
1807 
1808     db.run(buildSchema!(DependencyFileTable, DependencyRootTable));
1809 
1810     // add all existing files as being dependent on each other.
1811     // this forces, if any one of them changes, all to be re-analyzed.
1812     db.run(format("INSERT OR IGNORE INTO %1$s (file,checksum0,checksum1)
1813                   SELECT path,0,0 FROM %2$s", depFileTable, filesTable));
1814     db.run(format("INSERT OR IGNORE INTO %1$s (dep_id,file_id)
1815                   SELECT t0.id,t1.id FROM %2$s t0, %3$s t1", depRootTable,
1816             depFileTable, filesTable));
1817 }
1818 
1819 /// 2021-03-14
1820 void upgradeV34(ref Miniorm db) {
1821     immutable newTbl = "new_" ~ mutantWorklistTable;
1822     db.run(buildSchema!MutantWorklistTbl("new_"));
1823     db.run(format("INSERT INTO %1$s (id,prio) SELECT id,0 FROM %2$s", newTbl, mutantWorklistTable));
1824     db.replaceTbl(newTbl, mutantWorklistTable);
1825 }
1826 
1827 /// 2021-03-28
1828 void upgradeV35(ref Miniorm db) {
1829     @TableName(mutationStatusTable)
1830     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1831     struct MutationStatusTbl {
1832         long id;
1833         long status;
1834         @ColumnName("exit_code")
1835         int exitCode;
1836 
1837         @ColumnName("compile_time_ms")
1838         long compileTimeMs;
1839 
1840         @ColumnName("test_time_ms")
1841         long testTimeMs;
1842 
1843         @ColumnName("test_cnt")
1844         long testCnt;
1845 
1846         @ColumnParam("")
1847         @ColumnName("update_ts")
1848         SysTime updated;
1849 
1850         @ColumnParam("")
1851         @ColumnName("added_ts")
1852         SysTime added;
1853 
1854         long checksum0;
1855         long checksum1;
1856 
1857         /// Priority of the mutant used when testing.
1858         long prio;
1859     }
1860 
1861     immutable newTbl = "new_" ~ mutationStatusTable;
1862     db.run(buildSchema!MutationStatusTbl("new_"));
1863 
1864     db.run(format("INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,test_cnt,update_ts,added_ts,checksum0,checksum1,prio)
1865         SELECT t.id,t.status,t.exit_code,t.compile_time_ms,t.test_time_ms,t.test_cnt,t.update_ts,t.added_ts,t.checksum0,t.checksum1,0
1866         FROM %s t", newTbl, mutationStatusTable));
1867     replaceTbl(db, newTbl, mutationStatusTable);
1868 }
1869 
1870 /// 2021-03-29
1871 void upgradeV36(ref Miniorm db) {
1872     import dextool.plugin.mutate.backend.type : Mutation;
1873 
1874     @TableName(mutationStatusTable)
1875     @TableConstraint("checksum UNIQUE (checksum0, checksum1)")
1876     struct MutationStatusTbl {
1877         long id;
1878 
1879         /// Mutation.Status
1880         long status;
1881 
1882         @ColumnName("exit_code")
1883         int exitCode;
1884 
1885         @ColumnName("compile_time_ms")
1886         long compileTimeMs;
1887 
1888         @ColumnName("test_time_ms")
1889         long testTimeMs;
1890 
1891         @ColumnParam("")
1892         @ColumnName("update_ts")
1893         SysTime updated;
1894 
1895         @ColumnParam("")
1896         @ColumnName("added_ts")
1897         SysTime added;
1898 
1899         long checksum0;
1900         long checksum1;
1901 
1902         /// Priority of the mutant used when testing.
1903         long prio;
1904     }
1905 
1906     immutable newTbl = "new_" ~ mutationStatusTable;
1907     db.run(buildSchema!MutationStatusTbl("new_"));
1908 
1909     db.run(format(
1910             "INSERT INTO %s (id,status,exit_code,compile_time_ms,test_time_ms,update_ts,added_ts,checksum0,checksum1,prio)
1911         SELECT t.id,t.status,t.exit_code,t.compile_time_ms,t.test_time_ms,t.update_ts,t.added_ts,t.checksum0,t.checksum1,t.prio
1912         FROM %s t", newTbl, mutationStatusTable));
1913     replaceTbl(db, newTbl, mutationStatusTable);
1914 }
1915 
1916 // 2021-04-18
1917 void upgradeV37(ref Miniorm db) {
1918     db.run("DELETE FROM " ~ schemataTable);
1919     db.run(buildSchema!(DextoolVersionTable));
1920 }
1921 
1922 // 2021-04-19
1923 void upgradeV38(ref Miniorm db) {
1924     @TableName(schemataTable)
1925     struct SchemataTable {
1926         long id;
1927 
1928         // number of fragments the schemata consist of.
1929         // used to detect if a fragment has been removed because its related file
1930         // was changed.
1931         long fragments;
1932     }
1933 
1934     db.run("DROP TABLE " ~ schemataTable);
1935     db.run(buildSchema!(SchemataTable));
1936 }
1937 
1938 // 2021-04-23
1939 void upgradeV39(ref Miniorm db) {
1940     // this is just to force a re-measure of the test suite.
1941     db.run(format("DELETE FROM %s", runtimeHistoryTable));
1942 }
1943 
1944 // 2021-05-06
1945 void upgradeV40(ref Miniorm db) {
1946     // force an upgrade because after this release all scheman will be zipped.
1947     db.run("DELETE FROM " ~ schemataFragmentTable);
1948     db.run("DELETE FROM " ~ schemataMutantTable);
1949     db.run("DELETE FROM " ~ schemataTable);
1950     db.run("DELETE FROM " ~ schemataUsedTable);
1951 }
1952 
1953 // 2021-05-09
1954 void upgradeV41(ref Miniorm db) {
1955     @TableName(testCmdOriginalTable)
1956     @TablePrimaryKey("checksum")
1957     @TableConstraint("unique_ UNIQUE (cmd)")
1958     struct TestCmdOriginalTable {
1959         long checksum;
1960         string cmd;
1961     }
1962 
1963     db.run(buildSchema!(TestCmdOriginalTable));
1964 }
1965 
1966 // 2021-05-23
1967 void upgradeV42(ref Miniorm db) {
1968     db.run(buildSchema!(TestCmdMutatedTable));
1969 }
1970 
1971 // 2021-06-07
1972 void upgradeV43(ref Miniorm db) {
1973     @TableName(mutationTable)
1974     @TableForeignKey("mp_id", KeyRef("mutation_point(id)"), KeyParam("ON DELETE CASCADE"))
1975     @TableForeignKey("st_id", KeyRef("mutation_status(id)"), KeyParam("ON DELETE CASCADE"))
1976     @TableConstraint("unique_ UNIQUE (mp_id, kind)")
1977     struct MutationTbl {
1978         long id;
1979 
1980         long mp_id;
1981 
1982         @ColumnParam("")
1983         long st_id;
1984 
1985         long kind;
1986     }
1987 
1988     immutable newTbl = "new_" ~ mutationTable;
1989     db.run(buildSchema!MutationTbl("new_"));
1990 
1991     db.run(format("INSERT INTO %s (id,mp_id,st_id,kind)
1992         SELECT id,mp_id,st_id,kind FROM %s WHERE st_id NOT NULL", newTbl, mutationTable));
1993     replaceTbl(db, newTbl, mutationTable);
1994 }
1995 
1996 // 2021-08-30
1997 void upgradeV44(ref Miniorm db) {
1998     @TableName(schemaMutantQTable)
1999     @TablePrimaryKey("kind")
2000     struct SchemaMutantKindQ {
2001         /// mutant subtype
2002         long kind;
2003 
2004         // max 100
2005         long probability;
2006     }
2007 
2008     @TableName(schemataUsedTable)
2009     @TableForeignKey("id", KeyRef("schemata(id)"), KeyParam("ON DELETE CASCADE"))
2010     struct SchemataUsedTable {
2011         enum SchemaStatus {
2012             /// no status exist
2013             none,
2014             /// the schema compiled and the test suite executed OK
2015             ok,
2016             /// either it failed to compile or the test suite failed
2017             broken,
2018             /// the schema only contain killed mutants
2019             allKilled,
2020         }
2021 
2022         long id;
2023 
2024         SchemaStatus status;
2025     }
2026 
2027     db.run("DROP TABLE " ~ schemataUsedTable);
2028     db.run(buildSchema!(SchemataUsedTable, SchemaMutantKindQ));
2029 }
2030 
2031 // 2021-08-30
2032 void upgradeV45(ref Miniorm db) {
2033     db.run("DROP TABLE " ~ schemaMutantQTable);
2034     db.run(buildSchema!(SchemaMutantKindQTable));
2035 }
2036 
2037 // 2021-08-30
2038 void upgradeV46(ref Miniorm db) {
2039     db.run(buildSchema!SchemaSizeQTable);
2040     // drop probability because max state where changed thus all previous values are now 10x off
2041     db.run("DELETE FROM " ~ schemaMutantQTable);
2042 }
2043 
2044 // 2021-11-01
2045 void upgradeV47(ref Miniorm db) {
2046     static immutable newTbl = "new_" ~ allTestCaseTable;
2047     db.run(buildSchema!AllTestCaseTbl("new_"));
2048 
2049     db.run("INSERT INTO " ~ newTbl ~ " (id,name,is_new) SELECT id,name,0 FROM " ~ allTestCaseTable);
2050     replaceTbl(db, newTbl, allTestCaseTable);
2051 }
2052 
2053 // 2021-11-02
2054 void upgradeV48(ref Miniorm db) {
2055     db.run(buildSchema!MutantMemOverloadtWorklistTbl);
2056 }
2057 
2058 // 2021-11-02
2059 void upgradeV49(ref Miniorm db) {
2060     static immutable newTbl = "new_" ~ mutantTimeoutWorklistTable;
2061     db.run(buildSchema!MutantTimeoutWorklistTbl("new_"));
2062 
2063     db.run("INSERT INTO " ~ newTbl ~ " (id,iter) SELECT id,0 FROM " ~ mutantTimeoutWorklistTable);
2064 
2065     replaceTbl(db, newTbl, mutantTimeoutWorklistTable);
2066 }
2067 
2068 // 2022-03-21
2069 void upgradeV50(ref Miniorm db) {
2070     db.run("DROP TABLE " ~ testCmdOriginalTable);
2071     db.run(buildSchema!(TestCmdRelMutantTable, TestCmdTable, TestCmdOriginalTable));
2072 }
2073 
2074 // 2022-06-14
2075 void upgradeV51(ref Miniorm db) {
2076     db.run("DROP TABLE " ~ srcCovInfoTable);
2077     db.run(buildSchema!CoverageInfoTable);
2078 }
2079 
2080 // 2022-06-21
2081 void upgradeV52(ref Miniorm db) {
2082     @TableName(mutationFileScoreHistoryTable)
2083     struct MutationFileScoreHistoryTable {
2084         long id;
2085 
2086         /// when the measurement was taken.
2087         @ColumnName("time_stamp")
2088         SysTime timeStamp;
2089 
2090         double score;
2091 
2092         @ColumnName("file_path")
2093         string filePath;
2094     }
2095 
2096     db.run(buildSchema!MutationFileScoreHistoryTable);
2097 }
2098 
2099 // 2022-08-10
2100 void upgradeV53(ref Miniorm db) {
2101     @TableName(mutationStatusTable)
2102     @TableConstraint("checksum UNIQUE (checksum)")
2103     struct MutationStatusTbl {
2104         long id;
2105 
2106         /// Mutation.Status
2107         long status;
2108 
2109         @ColumnName("exit_code")
2110         int exitCode;
2111 
2112         @ColumnName("compile_time_ms")
2113         long compileTimeMs;
2114 
2115         @ColumnName("test_time_ms")
2116         long testTimeMs;
2117 
2118         @ColumnParam("")
2119         @ColumnName("update_ts")
2120         SysTime updated;
2121 
2122         @ColumnParam("")
2123         @ColumnName("added_ts")
2124         SysTime added;
2125 
2126         long checksum;
2127 
2128         /// Priority of the mutant used when testing.
2129         long prio;
2130     }
2131 
2132     @TableName(markedMutantTable)
2133     @TablePrimaryKey("checksum")
2134     struct MarkedMutantTbl {
2135         // TODO: can be removed in the future considering mutationStatusId is the
2136         // checksum from v55+.
2137         /// Checksum of the mutant status the marking is related to.
2138         /// it is the mutationStatusId id.
2139         long checksum;
2140 
2141         /// updated each analyze.
2142         @ColumnName("st_id")
2143         long mutationStatusId;
2144 
2145         /// updated each analyze.
2146         @ColumnName("mut_id")
2147         long mutationId;
2148 
2149         uint line;
2150         uint column;
2151         string path;
2152 
2153         /// The status it should always be changed to.
2154         long toStatus;
2155 
2156         /// Time when the mutant where marked.
2157         SysTime time;
2158 
2159         string rationale;
2160 
2161         string mutText;
2162     }
2163 
2164     const newFilesTbl = "new_" ~ filesTable;
2165     db.run(buildSchema!FilesTbl("new_"));
2166     db.run("INSERT INTO " ~ newFilesTbl
2167             ~ " (id, path, lang, timestamp, root, checksum) SELECT id,path,lang,timestamp,root,checksum0 FROM "
2168             ~ filesTable);
2169     replaceTbl(db, newFilesTbl, filesTable);
2170 
2171     const newTestFilesTbl = "new_" ~ testFilesTable;
2172     db.run(buildSchema!TestFilesTable("new_"));
2173     db.run("INSERT INTO " ~ newTestFilesTbl
2174             ~ " (id,path,timestamp,checksum) SELECT id,path,timestamp,checksum0 FROM "
2175             ~ testFilesTable);
2176     replaceTbl(db, newTestFilesTbl, testFilesTable);
2177 
2178     const newMutationStatusTbl = "new_" ~ mutationStatusTable;
2179     db.run(buildSchema!MutationStatusTbl("new_"));
2180     db.run("INSERT INTO " ~ newMutationStatusTbl ~ " (id,status,exit_code,checksum,compile_time_ms,test_time_ms,prio) SELECT id,status,exit_code,checksum0,compile_time_ms,test_time_ms,prio FROM " ~ mutationStatusTable);
2181     replaceTbl(db, newMutationStatusTbl, mutationStatusTable);
2182 
2183     const newDepFileTbl = "new_" ~ depFileTable;
2184     db.run(buildSchema!DependencyFileTable("new_"));
2185     db.run("INSERT INTO " ~ newDepFileTbl
2186             ~ " (id, file, checksum) SELECT id,file,checksum0 FROM " ~ depFileTable);
2187     replaceTbl(db, newDepFileTbl, depFileTable);
2188 
2189     const newMarkedMutantTbl = "new_" ~ markedMutantTable;
2190     db.run(buildSchema!MarkedMutantTbl("new_"));
2191     db.run("INSERT INTO " ~ newMarkedMutantTbl ~ " (checksum,st_id,mut_id,line,column,path,toStatus,time,rationale,mutText) SELECT checksum0,st_id,mut_id,line,column,path,toStatus,time,rationale,mutText FROM " ~ markedMutantTable);
2192     replaceTbl(db, newMarkedMutantTbl, markedMutantTable);
2193 }
2194 
2195 // 2022-08-17
2196 void upgradeV54(ref Miniorm db) {
2197     static immutable newFileScoreTable = "new_" ~ mutationFileScoreHistoryTable;
2198     db.run(buildSchema!MutationFileScoreHistoryTable("new_"));
2199     db.run("INSERT INTO " ~ newFileScoreTable ~ " (id,time_stamp,score,file_path) SELECT id,strftime('%Y-%m-%d 00:00:00.0', time_stamp) AS time,score,file_path FROM " ~ mutationFileScoreHistoryTable ~ " GROUP BY date(time), file_path");
2200     replaceTbl(db, newFileScoreTable, mutationFileScoreHistoryTable);
2201 
2202     static immutable newMutationScoreTable = "new_" ~ mutationScoreHistoryTable;
2203     db.run(buildSchema!MutationScoreHistoryTable("new_"));
2204     db.run("INSERT INTO " ~ newMutationScoreTable
2205             ~ " (id,time,score) SELECT id,strftime('%Y-%m-%d 00:00:00.0', time),score FROM "
2206             ~ mutationScoreHistoryTable ~ " GROUP BY date(time)");
2207     replaceTbl(db, newMutationScoreTable, mutationScoreHistoryTable);
2208 }
2209 
2210 // 2022-08-23
2211 void upgradeV55(ref Miniorm db) {
2212     foreach (tbl; [
2213             srcMetadataTable, mutationTable, killedTestCaseTable,
2214             mutantWorklistTable, mutantMemOverloadWorklistTable,
2215             mutantTimeoutWorklistTable, schemataMutantTable, testCmdRelMutantTable
2216         ]) {
2217         db.run("DELETE FROM " ~ tbl);
2218     }
2219 
2220     db.run("DROP TABLE " ~ mutationStatusTable);
2221     db.run(buildSchema!MutationStatusTbl);
2222 }
2223 
2224 // 2022-08-23
2225 void upgradeV56(ref Miniorm db) {
2226     foreach (tbl; [nomutDataTable, markedMutantTable]) {
2227         db.run("DROP TABLE " ~ tbl);
2228     }
2229     db.run(buildSchema!(NomutDataTbl, MarkedMutantTbl));
2230 }
2231 
2232 // 2022-08-30
2233 void upgradeV57(ref Miniorm db) {
2234     // fejk upgrade to force recalculation of indexes
2235 }
2236 
2237 // 2022-09-02
2238 void upgradeV58(ref Miniorm db) {
2239     // clear out old junk values. those with zero mutants.
2240     db.run("DELETE FROM " ~ mutationFileScoreHistoryTable ~ " WHERE score=0");
2241 }
2242 
2243 // 2022-09-07
2244 void upgradeV59(ref Miniorm db) {
2245     db.run(buildSchema!(SchemaFragmentV2Table, SchemaMutantV2Table));
2246 }
2247 
2248 // 2022-10-09
2249 void upgradeV60(ref Miniorm db) {
2250     foreach (a; [
2251             schemataFragmentTable, schemataUsedTable, schemataMutantTable,
2252             schemataTable
2253         ])
2254         db.run("DROP TABLE " ~ a);
2255 }
2256 
2257 void upgradeV61(ref Miniorm db) {
2258     immutable newTbl = "new_" ~ mutationTable;
2259     db.run(buildSchema!MutationTbl("new_"));
2260 
2261     db.run("INSERT OR IGNORE INTO " ~ newTbl ~ " (id,mp_id,st_id,kind)
2262         SELECT id,mp_id,st_id,kind FROM " ~ mutationTable);
2263     replaceTbl(db, newTbl, mutationTable);
2264 }
2265 
2266 void upgradeV62(ref Miniorm db) {
2267     db.run(buildSchema!ConfigVersionTable);
2268 }
2269 
2270 void replaceTbl(ref Miniorm db, string src, string dst) {
2271     db.run("DROP TABLE " ~ dst);
2272     db.run("ALTER TABLE " ~ src ~ " RENAME TO " ~ dst);
2273 }
2274 
2275 struct UpgradeTable {
2276     alias UpgradeFunc = void function(ref Miniorm db);
2277     UpgradeFunc[long] tbl;
2278     alias tbl this;
2279 
2280     immutable long latestSchemaVersion;
2281 }
2282 
2283 /** Inspects a module for functions starting with upgradeV to create a table of
2284  * functions that can be used to upgrade a database.
2285  */
2286 UpgradeTable makeUpgradeTable() {
2287     import std.algorithm : sort, startsWith;
2288     import std.conv : to;
2289     import std.typecons : Tuple;
2290 
2291     immutable prefix = "upgradeV";
2292 
2293     alias Module = dextool.plugin.mutate.backend.database.schema;
2294 
2295     // the second parameter is the database version to upgrade FROM.
2296     alias UpgradeFx = Tuple!(UpgradeTable.UpgradeFunc, long);
2297 
2298     UpgradeFx[] upgradeFx;
2299     long last_from;
2300 
2301     static foreach (member; __traits(allMembers, Module)) {
2302         static if (member.startsWith(prefix))
2303             upgradeFx ~= UpgradeFx(&__traits(getMember, Module, member),
2304                     member[prefix.length .. $].to!long);
2305     }
2306 
2307     typeof(UpgradeTable.tbl) tbl;
2308     foreach (fn; upgradeFx.sort!((a, b) => a[1] < b[1])) {
2309         last_from = fn[1];
2310         tbl[last_from] = fn[0];
2311     }
2312 
2313     return UpgradeTable(tbl, last_from + 1);
2314 }