1 /**
2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.test_mutant;
11 
12 import core.time : Duration, dur;
13 import logger = std.experimental.logger;
14 import std.algorithm : map, filter, joiner, among;
15 import std.array : empty, array, appender;
16 import std.datetime : SysTime, Clock;
17 import std.datetime.stopwatch : StopWatch, AutoStart;
18 import std.exception : collectException;
19 import std.format : format;
20 import std.random : randomCover;
21 import std.typecons : Nullable, Tuple, Yes;
22 
23 import blob_model : Blob;
24 import my.container.vector;
25 import my.fsm : Fsm, next, act, get, TypeDataMap;
26 import my.hash : Checksum64;
27 import my.named_type;
28 import my.optional;
29 import my.set;
30 import proc : DrainElement;
31 import sumtype;
32 static import my.fsm;
33 
34 import dextool.plugin.mutate.backend.database : Database, MutationEntry,
35     NextMutationEntry, spinSql, TestFile, ChecksumTestCmdOriginal;
36 import dextool.plugin.mutate.backend.interface_ : FilesysIO;
37 import dextool.plugin.mutate.backend.test_mutant.common;
38 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner,
39     findExecutables, TestRunResult = TestResult;
40 import dextool.plugin.mutate.backend.type : Mutation, TestCase, ExitStatus;
41 import dextool.plugin.mutate.config;
42 import dextool.plugin.mutate.type : ShellCommand;
43 import dextool.type : AbsolutePath, ExitStatusType, Path;
44 
45 @safe:
46 
47 auto makeTestMutant() {
48     return BuildTestMutant();
49 }
50 
51 private:
52 
53 struct BuildTestMutant {
54 @safe:
55 
56     import dextool.plugin.mutate.type : MutationKind;
57 
58     private struct InternalData {
59         Mutation.Kind[] kinds;
60         FilesysIO filesys_io;
61         ConfigMutationTest config;
62         ConfigSchema schemaConf;
63         ConfigCoverage covConf;
64     }
65 
66     private InternalData data;
67 
68     auto config(ConfigMutationTest c) @trusted nothrow {
69         data.config = c;
70         return this;
71     }
72 
73     auto config(ConfigSchema c) @trusted nothrow {
74         data.schemaConf = c;
75         return this;
76     }
77 
78     auto config(ConfigCoverage c) @trusted nothrow {
79         data.covConf = c;
80         return this;
81     }
82 
83     auto mutations(MutationKind[] v) nothrow {
84         import dextool.plugin.mutate.backend.mutation_type : toInternal;
85 
86         logger.infof("mutation operators: %(%s, %)", v).collectException;
87         data.kinds = toInternal(v);
88         return this;
89     }
90 
91     ExitStatusType run(const AbsolutePath dbPath, FilesysIO fio) @trusted {
92         try {
93             auto db = Database.make(dbPath);
94             return internalRun(db, fio);
95         } catch (Exception e) {
96             logger.error(e.msg).collectException;
97         }
98 
99         return ExitStatusType.Errors;
100     }
101 
102     private ExitStatusType internalRun(ref Database db, FilesysIO fio) {
103         // trusted because the lifetime of the database is guaranteed to outlive any instances in this scope
104         auto dbPtr = () @trusted { return &db; }();
105 
106         auto cleanup = new AutoCleanup;
107         scope (exit)
108             cleanup.cleanup;
109 
110         auto test_driver = TestDriver(dbPtr, fio, data.kinds, cleanup,
111                 data.config, data.covConf, data.schemaConf);
112 
113         while (test_driver.isRunning) {
114             test_driver.execute;
115         }
116 
117         return test_driver.status;
118     }
119 }
120 
121 struct MeasureTestDurationResult {
122     bool ok;
123     Duration[] runtime;
124 }
125 
126 /** Measure the time it takes to run the test command.
127  *
128  * The runtime is the lowest of three executions. Anything else is assumed to
129  * be variations in the system.
130  *
131  * If the tests fail (exit code isn't 0) any time then they are too unreliable
132  * to use for mutation testing.
133  *
134  * Params:
135  *  runner = ?
136  *  samples = number of times to run the test suite
137  */
138 MeasureTestDurationResult measureTestCommand(ref TestRunner runner, int samples) @safe nothrow {
139     import std.algorithm : min;
140     import proc;
141 
142     if (runner.empty) {
143         collectException(logger.error("No test command(s) specified (--test-cmd)"));
144         return MeasureTestDurationResult(false);
145     }
146 
147     static struct Rval {
148         TestRunResult result;
149         Duration runtime;
150     }
151 
152     auto runTest() @safe {
153         auto sw = StopWatch(AutoStart.yes);
154         auto res = runner.run(999.dur!"hours");
155         return Rval(res, sw.peek);
156     }
157 
158     static void print(TestRunResult res) @trusted {
159         import std.stdio : stdout, write;
160 
161         foreach (kv; res.output.byKeyValue) {
162             logger.info("test_cmd: ", kv.key);
163             foreach (l; kv.value)
164                 write(l.byUTF8);
165         }
166 
167         stdout.flush;
168     }
169 
170     Duration[] runtimes;
171     bool failed;
172     for (int i; i < samples && !failed; ++i) {
173         try {
174             auto res = runTest;
175             final switch (res.result.status) with (TestRunResult) {
176             case Status.passed:
177                 runtimes ~= res.runtime;
178                 break;
179             case Status.failed:
180                 goto case;
181             case Status.timeout:
182                 goto case;
183             case Status.error:
184                 failed = true;
185                 print(res.result);
186                 logger.info("failing commands: ", res.result.output.byKey);
187                 logger.info("exit status: ", res.result.exitStatus.get);
188                 break;
189             }
190             logger.infof("%s: Measured test command runtime %s", i, res.runtime);
191         } catch (Exception e) {
192             logger.error(e.msg).collectException;
193             failed = true;
194         }
195     }
196 
197     return MeasureTestDurationResult(!failed, runtimes);
198 }
199 
200 struct TestDriver {
201     import std.datetime : SysTime;
202     import dextool.plugin.mutate.backend.database : SchemataId, MutationStatusId;
203     import dextool.plugin.mutate.backend.report.analyzers : EstimateScore;
204     import dextool.plugin.mutate.backend.test_mutant.source_mutant : MutationTestDriver;
205     import dextool.plugin.mutate.backend.test_mutant.timeout : calculateTimeout, TimeoutFsm;
206     import dextool.plugin.mutate.type : MutationOrder;
207 
208     Database* db;
209     FilesysIO filesysIO;
210     Mutation.Kind[] kinds;
211     AutoCleanup autoCleanup;
212 
213     ConfigMutationTest conf;
214     ConfigSchema schemaConf;
215     ConfigCoverage covConf;
216 
217     /// Runs the test commands.
218     TestRunner runner;
219 
220     ///
221     TestCaseAnalyzer testCaseAnalyzer;
222 
223     /// Stop conditions (most of them)
224     TestStopCheck stopCheck;
225 
226     /// assuming that there are no more than 100 instances running in
227     /// parallel.
228     uint maxParallelInstances;
229 
230     // need to use 10000 because in an untested code base it is not
231     // uncommon for mutants being in the thousands.
232     enum long unknownWeight = 10000;
233     // using a factor 1000 to make a pull request mutant very high prio
234     enum long pullRequestWeight = unknownWeight * 1000;
235 
236     TimeoutFsm timeoutFsm;
237 
238     /// The time it takes to execute the test suite when no mutant is injected.
239     Duration testSuiteRuntime;
240 
241     /// the next mutant to test, if there are any.
242     MutationEntry nextMutant;
243 
244     // when the user manually configure the timeout it means that the
245     // timeout algorithm should not be used.
246     bool hardcodedTimeout;
247 
248     /// Test commands to execute.
249     ShellCommand[] testCmds;
250 
251     /// mutation score estimation
252     EstimateScore estimate;
253 
254     // The order to test mutants. It is either affected by the user directly or if pull request mode is activated.
255     MutationOrder mutationOrder;
256 
257     static struct UpdateTimeoutData {
258         long lastTimeoutIter;
259     }
260 
261     static struct None {
262     }
263 
264     static struct Initialize {
265         bool halt;
266     }
267 
268     static struct PullRequest {
269     }
270 
271     static struct PullRequestData {
272         import dextool.plugin.mutate.type : TestConstraint;
273 
274         TestConstraint constraint;
275     }
276 
277     static struct SanityCheck {
278         bool sanityCheckFailed;
279     }
280 
281     static struct AnalyzeTestCmdForTestCase {
282         TestCase[][ShellCommand] foundTestCases;
283     }
284 
285     static struct UpdateAndResetAliveMutants {
286         TestCase[][ShellCommand] foundTestCases;
287     }
288 
289     static struct ResetOldMutant {
290     }
291 
292     static struct ResetOldMutantData {
293         /// Number of mutants that where reset.
294         long maxReset;
295         NamedType!(double, Tag!"OldMutantPercentage", double.init, TagStringable) resetPercentage;
296     }
297 
298     static struct Cleanup {
299     }
300 
301     static struct CheckMutantsLeft {
302         bool allMutantsTested;
303     }
304 
305     static struct SaveMutationScore {
306     }
307 
308     static struct ParseStdin {
309     }
310 
311     static struct PreCompileSut {
312         bool compilationError;
313     }
314 
315     static struct FindTestCmds {
316     }
317 
318     static struct ChooseMode {
319     }
320 
321     static struct MeasureTestSuite {
322         bool unreliableTestSuite;
323     }
324 
325     static struct MutationTest {
326         NamedType!(bool, Tag!"MutationError", bool.init, TagStringable) mutationError;
327         MutationTestResult[] result;
328     }
329 
330     static struct MutationTestData {
331         TestBinaryDb testBinaryDb;
332     }
333 
334     static struct CheckTimeout {
335         bool timeoutUnchanged;
336     }
337 
338     static struct NextSchemataData {
339         SchemataId[] schematas;
340         long totalSchematas;
341         long invalidSchematas;
342     }
343 
344     static struct NextSchemata {
345         NamedType!(bool, Tag!"HasSchema", bool.init, TagStringable, ImplicitConvertable) hasSchema;
346         SchemataId schemataId;
347 
348         /// stop mutation testing because the last schema has been used and the
349         /// user has configured that the testing should stop now.
350         NamedType!(bool, Tag!"StopTesting", bool.init, TagStringable, ImplicitConvertable) stop;
351     }
352 
353     static struct SchemataTest {
354         SchemataId id;
355         MutationTestResult[] result;
356         bool fatalError;
357     }
358 
359     static struct SchemataPruneUsed {
360     }
361 
362     static struct Done {
363     }
364 
365     static struct Error {
366     }
367 
368     static struct UpdateTimeout {
369     }
370 
371     static struct CheckPullRequestMutant {
372         NamedType!(bool, Tag!"NoUnknown", bool.init, TagStringable, ImplicitConvertable) noUnknownMutantsLeft;
373     }
374 
375     static struct CheckPullRequestMutantData {
376         long startWorklistCnt;
377         long stopAfter;
378     }
379 
380     static struct NextMutant {
381         NamedType!(bool, Tag!"NoUnknown", bool.init, TagStringable, ImplicitConvertable) noUnknownMutantsLeft;
382     }
383 
384     static struct HandleTestResult {
385         MutationTestResult[] result;
386     }
387 
388     static struct CheckStopCond {
389         bool halt;
390     }
391 
392     static struct LoadSchematas {
393     }
394 
395     static struct OverloadCheck {
396         bool sleep;
397     }
398 
399     static struct ContinuesCheckTestSuite {
400         bool ok;
401     }
402 
403     static struct ContinuesCheckTestSuiteData {
404         long lastWorklistCnt;
405         SysTime lastCheck;
406     }
407 
408     static struct Stop {
409     }
410 
411     static struct Coverage {
412         bool propagate;
413         bool fatalError;
414     }
415 
416     static struct PropagateCoverage {
417     }
418 
419     static struct ChecksumTestCmds {
420     }
421 
422     static struct SaveTestBinary {
423     }
424 
425     alias Fsm = my.fsm.Fsm!(None, Initialize, SanityCheck,
426             AnalyzeTestCmdForTestCase, UpdateAndResetAliveMutants, ResetOldMutant,
427             Cleanup, CheckMutantsLeft, PreCompileSut, MeasureTestSuite, NextMutant,
428             MutationTest, HandleTestResult, CheckTimeout, Done, Error,
429             UpdateTimeout, CheckStopCond, PullRequest, CheckPullRequestMutant, ParseStdin,
430             FindTestCmds, ChooseMode, NextSchemata, SchemataTest, LoadSchematas,
431             SchemataPruneUsed, Stop, SaveMutationScore, OverloadCheck,
432             Coverage, PropagateCoverage, ContinuesCheckTestSuite,
433             ChecksumTestCmds, SaveTestBinary);
434     alias LocalStateDataT = Tuple!(UpdateTimeoutData, CheckPullRequestMutantData, PullRequestData,
435             ResetOldMutantData, NextSchemataData, ContinuesCheckTestSuiteData, MutationTestData);
436 
437     private {
438         Fsm fsm;
439         TypeDataMap!(LocalStateDataT, UpdateTimeout, CheckPullRequestMutant, PullRequest,
440                 ResetOldMutant, NextSchemata, ContinuesCheckTestSuite, MutationTest) local;
441         bool isRunning_ = true;
442         bool isDone = false;
443     }
444 
445     this(Database* db, FilesysIO filesysIO, Mutation.Kind[] kinds, AutoCleanup autoCleanup,
446             ConfigMutationTest conf, ConfigCoverage coverage, ConfigSchema schema) {
447         this.db = db;
448         this.filesysIO = filesysIO;
449         this.kinds = kinds;
450         this.autoCleanup = autoCleanup;
451         this.conf = conf;
452         this.covConf = coverage;
453         this.schemaConf = schema;
454 
455         this.timeoutFsm = TimeoutFsm(kinds);
456         this.hardcodedTimeout = !conf.mutationTesterRuntime.isNull;
457         local.get!PullRequest.constraint = conf.constraint;
458         local.get!ResetOldMutant.maxReset = conf.oldMutantsNr;
459         local.get!ResetOldMutant.resetPercentage = conf.oldMutantPercentage;
460         this.testCmds = conf.mutationTester;
461         this.mutationOrder = conf.mutationOrder;
462 
463         this.runner.useEarlyStop(conf.useEarlyTestCmdStop);
464         this.runner = TestRunner.make(conf.testPoolSize);
465         this.runner.useEarlyStop(conf.useEarlyTestCmdStop);
466         this.runner.maxOutputCapture(
467                 TestRunner.MaxCaptureBytes(conf.maxTestCaseOutput.get * 1024 * 1024));
468         this.runner.put(conf.mutationTester);
469 
470         // TODO: allow a user, as is for test_cmd, to specify an array of
471         // external analyzers.
472         this.testCaseAnalyzer = TestCaseAnalyzer(conf.mutationTestCaseBuiltin,
473                 conf.mutationTestCaseAnalyze, autoCleanup);
474 
475         this.stopCheck = TestStopCheck(conf);
476 
477         this.maxParallelInstances = () {
478             if (mutationOrder.among(MutationOrder.random, MutationOrder.bySize))
479                 return 100;
480             return 1;
481         }();
482 
483         if (logger.globalLogLevel.among(logger.LogLevel.trace, logger.LogLevel.all))
484             fsm.logger = (string s) { logger.trace(s); };
485     }
486 
487     static void execute_(ref TestDriver self) @trusted {
488         // see test_mutant/basis.md and figures/test_mutant_fsm.pu for a
489         // graphical view of the state machine.
490 
491         self.fsm.next!((None a) => fsm(Initialize.init), (Initialize a) {
492             if (a.halt)
493                 return fsm(CheckStopCond.init);
494             return fsm(SanityCheck.init);
495         }, (SanityCheck a) {
496             if (a.sanityCheckFailed)
497                 return fsm(Error.init);
498             if (self.conf.unifiedDiffFromStdin)
499                 return fsm(ParseStdin.init);
500             return fsm(PreCompileSut.init);
501         }, (ParseStdin a) => fsm(PreCompileSut.init), (AnalyzeTestCmdForTestCase a) => fsm(
502                 UpdateAndResetAliveMutants(a.foundTestCases)),
503                 (UpdateAndResetAliveMutants a) => fsm(CheckMutantsLeft.init),
504                 (ResetOldMutant a) => fsm(UpdateTimeout.init), (Cleanup a) {
505             if (self.local.get!PullRequest.constraint.empty)
506                 return fsm(NextSchemata.init);
507             return fsm(CheckPullRequestMutant.init);
508         }, (CheckMutantsLeft a) {
509             if (a.allMutantsTested && self.conf.onOldMutants == ConfigMutationTest
510                 .OldMutant.nothing)
511                 return fsm(Done.init);
512             if (self.conf.testCmdChecksum.get)
513                 return fsm(ChecksumTestCmds.init);
514             return fsm(MeasureTestSuite.init);
515         }, (ChecksumTestCmds a) => MeasureTestSuite.init, (SaveMutationScore a) => SaveTestBinary.init,
516                 (SaveTestBinary a) => Stop.init, (PreCompileSut a) {
517             if (a.compilationError)
518                 return fsm(Error.init);
519             if (self.conf.testCommandDir.empty)
520                 return fsm(ChooseMode.init);
521             return fsm(FindTestCmds.init);
522         }, (FindTestCmds a) { return fsm(ChooseMode.init); }, (ChooseMode a) {
523             if (!self.local.get!PullRequest.constraint.empty)
524                 return fsm(PullRequest.init);
525             if (!self.conf.mutationTestCaseAnalyze.empty
526                 || !self.conf.mutationTestCaseBuiltin.empty)
527                 return fsm(AnalyzeTestCmdForTestCase.init);
528             return fsm(CheckMutantsLeft.init);
529         }, (PullRequest a) => fsm(CheckMutantsLeft.init), (MeasureTestSuite a) {
530             if (a.unreliableTestSuite)
531                 return fsm(Error.init);
532             if (self.covConf.use)
533                 return fsm(Coverage.init);
534             return fsm(LoadSchematas.init);
535         }, (Coverage a) {
536             if (a.fatalError)
537                 return fsm(Error.init);
538             if (a.propagate)
539                 return fsm(PropagateCoverage.init);
540             return fsm(LoadSchematas.init);
541         }, (PropagateCoverage a) => LoadSchematas.init,
542                 (LoadSchematas a) => fsm(ResetOldMutant.init), (CheckPullRequestMutant a) {
543             if (a.noUnknownMutantsLeft)
544                 return fsm(Done.init);
545             return fsm(NextMutant.init);
546         }, (NextSchemata a) {
547             if (a.hasSchema)
548                 return fsm(SchemataTest(a.schemataId));
549             if (a.stop)
550                 return fsm(Done.init);
551             return fsm(NextMutant.init);
552         }, (SchemataTest a) {
553             if (a.fatalError)
554                 return fsm(Error.init);
555             return fsm(CheckStopCond.init);
556         }, (NextMutant a) {
557             if (a.noUnknownMutantsLeft)
558                 return fsm(CheckTimeout.init);
559             return fsm(MutationTest.init);
560         }, (UpdateTimeout a) => fsm(OverloadCheck.init), (OverloadCheck a) {
561             if (a.sleep)
562                 return fsm(CheckStopCond.init);
563             return fsm(ContinuesCheckTestSuite.init);
564         }, (ContinuesCheckTestSuite a) {
565             if (a.ok)
566                 return fsm(Cleanup.init);
567             return fsm(Error.init);
568         }, (MutationTest a) {
569             if (a.mutationError)
570                 return fsm(Error.init);
571             return fsm(HandleTestResult(a.result));
572         }, (HandleTestResult a) => fsm(CheckStopCond.init), (CheckStopCond a) {
573             if (a.halt)
574                 return fsm(Done.init);
575             return fsm(UpdateTimeout.init);
576         }, (CheckTimeout a) {
577             if (a.timeoutUnchanged)
578                 return fsm(Done.init);
579             return fsm(UpdateTimeout.init);
580         }, (SchemataPruneUsed a) => SaveMutationScore.init,
581                 (Done a) => fsm(SchemataPruneUsed.init),
582                 (Error a) => fsm(Stop.init), (Stop a) => fsm(a));
583 
584         self.fsm.act!(self);
585     }
586 
587 nothrow:
588 
589     void execute() {
590         try {
591             execute_(this);
592         } catch (Exception e) {
593             logger.warning(e.msg).collectException;
594         }
595     }
596 
597     bool isRunning() {
598         return isRunning_;
599     }
600 
601     ExitStatusType status() {
602         if (isDone)
603             return ExitStatusType.Ok;
604         return ExitStatusType.Errors;
605     }
606 
607     void opCall(None data) {
608     }
609 
610     void opCall(ref Initialize data) {
611         logger.info("Initializing worklist").collectException;
612         spinSql!(() {
613             db.updateWorklist(kinds, Mutation.Status.unknown, unknownWeight, mutationOrder);
614         });
615 
616         // detect if the system is overloaded before trying to do something
617         // slow such as compiling the SUT.
618         if (conf.loadBehavior == ConfigMutationTest.LoadBehavior.halt && stopCheck.isHalt) {
619             data.halt = true;
620         }
621     }
622 
623     void opCall(Stop data) {
624         isRunning_ = false;
625     }
626 
627     void opCall(Done data) {
628         logger.info("Done!").collectException;
629         isDone = true;
630     }
631 
632     void opCall(Error data) {
633         autoCleanup.cleanup;
634     }
635 
636     void opCall(ref SanityCheck data) {
637         import core.sys.posix.sys.stat : S_IWUSR;
638         import std.path : buildPath;
639         import my.file : getAttrs;
640         import colorlog : color, Color;
641         import dextool.plugin.mutate.backend.utility : checksum, Checksum;
642 
643         logger.info("Sanity check of files to mutate").collectException;
644 
645         auto failed = appender!(string[])();
646         auto checksumFailed = appender!(string[])();
647         auto writePermissionFailed = appender!(string[])();
648         foreach (file; spinSql!(() { return db.getFiles; })) {
649             auto db_checksum = spinSql!(() { return db.getFileChecksum(file); });
650 
651             try {
652                 auto abs_f = AbsolutePath(buildPath(filesysIO.getOutputDir, file));
653                 auto f_checksum = checksum(filesysIO.makeInput(abs_f).content[]);
654                 if (db_checksum != f_checksum) {
655                     checksumFailed.put(abs_f);
656                 }
657 
658                 uint attrs;
659                 if (getAttrs(abs_f, attrs)) {
660                     if ((attrs & S_IWUSR) == 0) {
661                         writePermissionFailed.put(abs_f);
662                     }
663                 } else {
664                     writePermissionFailed.put(abs_f);
665                 }
666             } catch (Exception e) {
667                 failed.put(file);
668                 logger.warningf("%s: %s", file, e.msg).collectException;
669             }
670         }
671 
672         data.sanityCheckFailed = !failed.data.empty
673             || !checksumFailed.data.empty || !writePermissionFailed.data.empty;
674 
675         if (data.sanityCheckFailed) {
676             logger.info(!failed.data.empty,
677                     "Unknown error when checking the files").collectException;
678             foreach (f; failed.data)
679                 logger.info(f).collectException;
680 
681             logger.info(!checksumFailed.data.empty,
682                     "Detected that file(s) has changed since last analyze where done")
683                 .collectException;
684             logger.info(!checksumFailed.data.empty,
685                     "Either restore the file(s) or rerun the analyze").collectException;
686             foreach (f; checksumFailed.data)
687                 logger.info(f).collectException;
688 
689             logger.info(!writePermissionFailed.data.empty,
690                     "Files to mutate are not writable").collectException;
691             foreach (f; writePermissionFailed.data)
692                 logger.info(f).collectException;
693 
694             logger.info("Failed".color.fgRed).collectException;
695         } else {
696             logger.info("Ok".color.fgGreen).collectException;
697         }
698     }
699 
700     void opCall(ref OverloadCheck data) {
701         if (conf.loadBehavior == ConfigMutationTest.LoadBehavior.slowdown && stopCheck.isOverloaded) {
702             data.sleep = true;
703             logger.info(stopCheck.overloadToString).collectException;
704             stopCheck.pause;
705         }
706     }
707 
708     void opCall(ref ContinuesCheckTestSuite data) {
709         import std.algorithm : max;
710 
711         data.ok = true;
712 
713         if (!conf.contCheckTestSuite)
714             return;
715 
716         enum forceCheckEach = 1.dur!"hours";
717 
718         const wlist = spinSql!(() => db.getWorklistCount);
719         if (local.get!ContinuesCheckTestSuite.lastWorklistCnt == 0) {
720             // first time, just initialize.
721             local.get!ContinuesCheckTestSuite.lastWorklistCnt = wlist;
722             local.get!ContinuesCheckTestSuite.lastCheck = Clock.currTime + forceCheckEach;
723             return;
724         }
725 
726         const period = conf.contCheckTestSuitePeriod.get;
727         const diffCnt = local.get!ContinuesCheckTestSuite.lastWorklistCnt - wlist;
728         if (!(wlist % period == 0 || diffCnt >= period
729                 || Clock.currTime > local.get!ContinuesCheckTestSuite.lastCheck))
730             return;
731 
732         local.get!ContinuesCheckTestSuite.lastWorklistCnt = wlist;
733         local.get!ContinuesCheckTestSuite.lastCheck = Clock.currTime + forceCheckEach;
734 
735         compile(conf.mutationCompile, conf.buildCmdTimeout, PrintCompileOnFailure(true)).match!(
736                 (Mutation.Status a) { data.ok = false; }, (bool success) {
737             data.ok = success;
738         });
739 
740         if (data.ok) {
741             try {
742                 data.ok = measureTestCommand(runner, 1).ok;
743             } catch (Exception e) {
744                 logger.error(e.msg).collectException;
745                 data.ok = false;
746             }
747         }
748 
749         if (!data.ok) {
750             logger.warning("Continues sanity check of the test suite has failed.").collectException;
751             logger.infof("Rolling back the status of the last %s mutants to status unknown.",
752                     period).collectException;
753             foreach (a; spinSql!(() => db.getLatestMutants(kinds, max(diffCnt, period)))) {
754                 spinSql!(() => db.updateMutation(a.id, Mutation.Status.unknown,
755                         ExitStatus(0), MutantTimeProfile.init));
756             }
757         }
758     }
759 
760     void opCall(ParseStdin data) {
761         import dextool.plugin.mutate.backend.diff_parser : diffFromStdin;
762         import dextool.plugin.mutate.type : Line;
763 
764         try {
765             auto constraint = local.get!PullRequest.constraint;
766             foreach (pkv; diffFromStdin.toRange(filesysIO.getOutputDir)) {
767                 constraint.value[pkv.key] ~= pkv.value.toRange.map!(a => Line(a)).array;
768             }
769             local.get!PullRequest.constraint = constraint;
770         } catch (Exception e) {
771             logger.warning(e.msg).collectException;
772         }
773     }
774 
775     void opCall(ref AnalyzeTestCmdForTestCase data) {
776         TestCase[][ShellCommand] found;
777         try {
778             runner.captureAll(true);
779             scope (exit)
780                 runner.captureAll(false);
781 
782             // using an unreasonable timeout to make it possible to analyze for
783             // test cases and measure the test suite.
784             auto res = runTester(runner, 999.dur!"hours");
785 
786             foreach (testCmd; res.output.byKeyValue) {
787                 auto analyze = testCaseAnalyzer.analyze(testCmd.key, testCmd.value, Yes.allFound);
788 
789                 analyze.match!((TestCaseAnalyzer.Success a) {
790                     found[testCmd.key] = a.found;
791                 }, (TestCaseAnalyzer.Unstable a) {
792                     logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
793                     found[testCmd.key] = a.found;
794                 }, (TestCaseAnalyzer.Failed a) {
795                     logger.warning("The parser that analyze the output for test case(s) failed");
796                 });
797 
798             }
799 
800             warnIfConflictingTestCaseIdentifiers(found.byValue.joiner.array);
801         } catch (Exception e) {
802             logger.warning(e.msg).collectException;
803         }
804 
805         data.foundTestCases = found;
806     }
807 
808     void opCall(UpdateAndResetAliveMutants data) {
809         import std.traits : EnumMembers;
810 
811         // the test cases before anything has potentially changed.
812         auto old_tcs = spinSql!(() {
813             Set!string old_tcs;
814             foreach (tc; db.getDetectedTestCases)
815                 old_tcs.add(tc.name);
816             return old_tcs;
817         });
818 
819         void transaction() @safe {
820             final switch (conf.onRemovedTestCases) with (ConfigMutationTest.RemovedTestCases) {
821             case doNothing:
822                 db.addDetectedTestCases(data.foundTestCases.byValue.joiner.array);
823                 break;
824             case remove:
825                 bool update;
826                 // change all mutants which, if a test case is removed, no
827                 // longer has a test case that kills it to unknown status
828                 foreach (id; db.setDetectedTestCases(data.foundTestCases.byValue.joiner.array)) {
829                     if (!db.hasTestCases(id)) {
830                         update = true;
831                         db.updateMutationStatus(id, Mutation.Status.unknown, ExitStatus(0));
832                     }
833                 }
834                 if (update) {
835                     db.updateWorklist(kinds, Mutation.Status.unknown);
836                 }
837                 break;
838             }
839         }
840 
841         auto found_tcs = spinSql!(() @trusted {
842             auto tr = db.transaction;
843             transaction();
844 
845             Set!string found_tcs;
846             foreach (tc; db.getDetectedTestCases)
847                 found_tcs.add(tc.name);
848 
849             tr.commit;
850             return found_tcs;
851         });
852 
853         printDroppedTestCases(old_tcs, found_tcs);
854 
855         if (hasNewTestCases(old_tcs, found_tcs)
856                 && conf.onNewTestCases == ConfigMutationTest.NewTestCases.resetAlive) {
857             logger.info("Adding alive mutants to worklist").collectException;
858             spinSql!(() {
859                 db.updateWorklist(kinds, Mutation.Status.alive);
860                 // if these mutants are covered by the tests then they will be
861                 // removed from the worklist in PropagateCoverage.
862                 db.updateWorklist(kinds, Mutation.Status.noCoverage);
863             });
864         }
865     }
866 
867     void opCall(ResetOldMutant data) {
868         import std.range : enumerate;
869         import dextool.plugin.mutate.backend.database.type;
870 
871         void printStatus(T0)(T0 oldestMutant, SysTime newestTest, SysTime newestFile) {
872             logger.info("Tests last changed ", newestTest).collectException;
873             logger.info("Source code last changed ", newestFile).collectException;
874 
875             if (!oldestMutant.empty) {
876                 logger.info("The oldest mutant is ", oldestMutant[0].updated).collectException;
877             }
878         }
879 
880         if (conf.onOldMutants == ConfigMutationTest.OldMutant.nothing) {
881             return;
882         }
883         if (spinSql!(() { return db.getWorklistCount; }) != 0) {
884             // do not re-test any old mutants if there are still work to do in the worklist.
885             return;
886         }
887 
888         const oldestMutant = spinSql!(() => db.getOldestMutants(kinds, 1));
889         const newestTest = spinSql!(() => db.getNewestTestFile).orElse(TestFile.init).timeStamp;
890         const newestFile = spinSql!(() => db.getNewestFile).orElse(SysTime.init);
891         if (!oldestMutant.empty && oldestMutant[0].updated > newestTest
892                 && oldestMutant[0].updated > newestFile) {
893             // only re-test old mutants if needed.
894             logger.info("Mutation status is up to date").collectException;
895             printStatus(oldestMutant, newestTest, newestFile);
896             return;
897         } else {
898             logger.info("Mutation status is out of sync").collectException;
899             printStatus(oldestMutant, newestTest, newestFile);
900         }
901 
902         const long testCnt = () {
903             if (local.get!ResetOldMutant.resetPercentage.get == 0.0) {
904                 return local.get!ResetOldMutant.maxReset;
905             }
906 
907             const total = spinSql!(() { return db.totalSrcMutants(kinds).count; });
908             const rval = cast(long)(1 + total * local.get!ResetOldMutant.resetPercentage.get
909                     / 100.0);
910             return rval;
911         }();
912 
913         auto oldest = spinSql!(() { return db.getOldestMutants(kinds, testCnt); });
914 
915         logger.infof("Adding %s old mutants to worklist", oldest.length).collectException;
916         spinSql!(() {
917             foreach (const old; oldest.enumerate) {
918                 logger.infof("%s Last updated %s", old.index + 1,
919                     old.value.updated).collectException;
920                 db.addToWorklist(old.value.id);
921             }
922         });
923     }
924 
925     void opCall(Cleanup data) {
926         autoCleanup.cleanup;
927     }
928 
929     void opCall(ref CheckMutantsLeft data) {
930         spinSql!(() { timeoutFsm.execute(*db); });
931 
932         data.allMutantsTested = timeoutFsm.output.done;
933 
934         if (timeoutFsm.output.done) {
935             logger.info("All mutants are tested").collectException;
936         }
937     }
938 
939     void opCall(ChecksumTestCmds data) @trusted {
940         import std.file : exists;
941         import my.hash : Checksum64, makeCrc64Iso, checksum;
942         import dextool.plugin.mutate.backend.database.type : ChecksumTestCmdOriginal;
943 
944         auto previous = spinSql!(() => db.testCmdApi.original);
945 
946         try {
947             Set!Checksum64 current;
948 
949             foreach (testCmd; hashFiles(testCmds.filter!(a => !a.empty)
950                     .map!(a => a.value[0]))) {
951                 current.add(testCmd.cs);
952 
953                 if (testCmd.cs !in previous)
954                     spinSql!(() => db.testCmdApi.set(testCmd.file,
955                             ChecksumTestCmdOriginal(testCmd.cs)));
956             }
957 
958             foreach (a; previous.setDifference(current).toRange)
959                 spinSql!(() => db.testCmdApi.remove(ChecksumTestCmdOriginal(a)));
960 
961             local.get!MutationTest.testBinaryDb.original = current;
962         } catch (Exception e) {
963             logger.warning(e.msg).collectException;
964         }
965 
966         local.get!MutationTest.testBinaryDb.mutated = spinSql!(
967                 () @trusted => db.testCmdApi.mutated);
968     }
969 
970     void opCall(SaveMutationScore data) {
971         import dextool.plugin.mutate.backend.database.type : MutationScore;
972         import dextool.plugin.mutate.backend.report.analyzers : reportScore;
973 
974         if (spinSql!(() => db.unknownSrcMutants(kinds)).count != 0)
975             return;
976 
977         const score = reportScore(*db, kinds).score;
978 
979         // 10000 mutation scores is only ~80kbyte. Should be enough entries
980         // without taking up unresonable amount of space.
981         spinSql!(() @trusted {
982             auto t = db.transaction;
983             db.putMutationScore(MutationScore(Clock.currTime, typeof(MutationScore.score)(score)));
984             db.trimMutationScore(10000);
985             t.commit;
986         });
987     }
988 
989     void opCall(SaveTestBinary data) {
990         if (!local.get!MutationTest.testBinaryDb.empty)
991             saveTestBinaryDb(local.get!MutationTest.testBinaryDb);
992     }
993 
994     void opCall(ref PreCompileSut data) {
995         import proc;
996 
997         logger.info("Checking the build command").collectException;
998         compile(conf.mutationCompile, conf.buildCmdTimeout, PrintCompileOnFailure(true)).match!(
999                 (Mutation.Status a) { data.compilationError = true; }, (bool success) {
1000             data.compilationError = !success;
1001         });
1002     }
1003 
1004     void opCall(FindTestCmds data) {
1005         auto cmds = appender!(ShellCommand[])();
1006         foreach (root; conf.testCommandDir) {
1007             try {
1008                 cmds.put(findExecutables(root.AbsolutePath)
1009                         .map!(a => ShellCommand([a] ~ conf.testCommandDirFlag)));
1010             } catch (Exception e) {
1011                 logger.warning(e.msg).collectException;
1012             }
1013         }
1014 
1015         if (!cmds.data.empty) {
1016             testCmds ~= cmds.data;
1017             runner.put(this.testCmds);
1018             logger.infof("Found test commands in %s:", conf.testCommandDir).collectException;
1019             foreach (c; cmds.data) {
1020                 logger.info(c).collectException;
1021             }
1022         }
1023     }
1024 
1025     void opCall(ChooseMode data) {
1026     }
1027 
1028     void opCall(PullRequest data) {
1029         import std.algorithm : sort;
1030         import my.set;
1031         import dextool.plugin.mutate.backend.database : MutationStatusId;
1032         import dextool.plugin.mutate.backend.type : SourceLoc;
1033 
1034         // deterministic testing of mutants and prioritized by their size.
1035         mutationOrder = MutationOrder.bySize;
1036         maxParallelInstances = 1;
1037 
1038         // make sure they are unique.
1039         Set!MutationStatusId mutantIds;
1040 
1041         foreach (kv; local.get!PullRequest.constraint.value.byKeyValue) {
1042             const file_id = spinSql!(() => db.getFileId(kv.key));
1043             if (file_id.isNull) {
1044                 logger.infof("The file %s do not exist in the database. Skipping...",
1045                         kv.key).collectException;
1046                 continue;
1047             }
1048 
1049             foreach (l; kv.value) {
1050                 auto mutants = spinSql!(() {
1051                     return db.getMutationsOnLine(kinds, file_id.get, SourceLoc(l.value, 0));
1052                 });
1053 
1054                 const preCnt = mutantIds.length;
1055                 foreach (v; mutants) {
1056                     mutantIds.add(v);
1057                 }
1058 
1059                 logger.infof(mutantIds.length - preCnt > 0, "Found %s mutant(s) to test (%s:%s)",
1060                         mutantIds.length - preCnt, kv.key, l.value).collectException;
1061             }
1062         }
1063 
1064         logger.infof(!mutantIds.empty, "Found %s mutants in the diff",
1065                 mutantIds.length).collectException;
1066         spinSql!(() {
1067             foreach (id; mutantIds.toArray.sort)
1068                 db.addToWorklist(id, pullRequestWeight, MutationOrder.bySize);
1069         });
1070 
1071         local.get!CheckPullRequestMutant.startWorklistCnt = spinSql!(() => db.getWorklistCount);
1072         local.get!CheckPullRequestMutant.stopAfter = mutantIds.length;
1073 
1074         if (mutantIds.empty) {
1075             logger.warning("None of the locations specified with -L exists").collectException;
1076             logger.info("Available files are:").collectException;
1077             foreach (f; spinSql!(() => db.getFiles))
1078                 logger.info(f).collectException;
1079         }
1080     }
1081 
1082     void opCall(ref MeasureTestSuite data) {
1083         import std.algorithm : max, sum;
1084         import dextool.plugin.mutate.backend.database.type : TestCmdRuntime;
1085 
1086         if (!conf.mutationTesterRuntime.isNull) {
1087             testSuiteRuntime = conf.mutationTesterRuntime.get;
1088             return;
1089         }
1090 
1091         logger.infof("Measuring the runtime of the test command(s):\n%(%s\n%)",
1092                 testCmds).collectException;
1093 
1094         auto measures = spinSql!(() => db.getTestCmdRuntimes);
1095 
1096         const tester = () {
1097             try {
1098                 return measureTestCommand(runner, max(1, cast(int)(3 - measures.length)));
1099             } catch (Exception e) {
1100                 logger.error(e.msg).collectException;
1101                 return MeasureTestDurationResult(false);
1102             }
1103         }();
1104 
1105         if (tester.ok) {
1106             measures ~= tester.runtime.map!(a => TestCmdRuntime(Clock.currTime, a)).array;
1107             if (measures.length > 3) {
1108                 measures = measures[1 .. $]; // drop the oldest
1109             }
1110 
1111             auto mean = sum(measures.map!(a => a.runtime), Duration.zero) / measures.length;
1112 
1113             // The sampling of the test suite become too unreliable when the timeout is <1s.
1114             // This is a quick and dirty fix.
1115             // A proper fix requires an update of the sampler in runTester.
1116             auto t = mean < 1.dur!"seconds" ? 1.dur!"seconds" : mean;
1117             logger.info("Test command runtime: ", t).collectException;
1118             testSuiteRuntime = t;
1119 
1120             spinSql!(() @trusted {
1121                 auto t = db.transaction;
1122                 db.setTestCmdRuntimes(measures);
1123                 t.commit;
1124             });
1125         } else {
1126             data.unreliableTestSuite = true;
1127             logger.error("The test command is unreliable. It must return exit status '0' when no mutants are injected")
1128                 .collectException;
1129         }
1130     }
1131 
1132     void opCall(ref MutationTest data) {
1133         auto runnerPtr = () @trusted { return &runner; }();
1134         auto testBinaryDbPtr = () @trusted {
1135             return &local.get!MutationTest.testBinaryDb;
1136         }();
1137 
1138         try {
1139             auto g = MutationTestDriver.Global(filesysIO, db, nextMutant,
1140                     runnerPtr, testBinaryDbPtr);
1141             auto driver = MutationTestDriver(g,
1142                     MutationTestDriver.TestMutantData(!(conf.mutationTestCaseAnalyze.empty
1143                         && conf.mutationTestCaseBuiltin.empty),
1144                         conf.mutationCompile, conf.buildCmdTimeout),
1145                     MutationTestDriver.TestCaseAnalyzeData(&testCaseAnalyzer));
1146 
1147             while (driver.isRunning) {
1148                 driver.execute();
1149             }
1150 
1151             if (driver.stopBecauseError) {
1152                 data.mutationError.get = true;
1153             } else {
1154                 data.result = driver.result;
1155             }
1156         } catch (Exception e) {
1157             data.mutationError.get = true;
1158             logger.error(e.msg).collectException;
1159         }
1160     }
1161 
1162     void opCall(ref CheckTimeout data) {
1163         data.timeoutUnchanged = hardcodedTimeout || timeoutFsm.output.done;
1164     }
1165 
1166     void opCall(UpdateTimeout) {
1167         spinSql!(() { timeoutFsm.execute(*db); });
1168 
1169         const lastIter = local.get!UpdateTimeout.lastTimeoutIter;
1170 
1171         if (lastIter != timeoutFsm.output.iter) {
1172             logger.infof("Changed the timeout from %s to %s (iteration %s)",
1173                     calculateTimeout(lastIter, testSuiteRuntime),
1174                     calculateTimeout(timeoutFsm.output.iter, testSuiteRuntime),
1175                     timeoutFsm.output.iter).collectException;
1176             local.get!UpdateTimeout.lastTimeoutIter = timeoutFsm.output.iter;
1177         }
1178 
1179         runner.timeout = calculateTimeout(timeoutFsm.output.iter, testSuiteRuntime);
1180     }
1181 
1182     void opCall(ref CheckPullRequestMutant data) {
1183         const left = spinSql!(() { return db.getWorklistCount; });
1184         data.noUnknownMutantsLeft.get = (
1185                 local.get!CheckPullRequestMutant.startWorklistCnt - left) >= local
1186             .get!CheckPullRequestMutant.stopAfter;
1187 
1188         logger.infof(stopCheck.aliveMutants > 0, "Found %s/%s alive mutants",
1189                 stopCheck.aliveMutants, conf.maxAlive.get).collectException;
1190     }
1191 
1192     void opCall(ref NextMutant data) {
1193         nextMutant = MutationEntry.init;
1194 
1195         auto next = spinSql!(() {
1196             return db.nextMutation(kinds, maxParallelInstances);
1197         });
1198 
1199         data.noUnknownMutantsLeft.get = next.st == NextMutationEntry.Status.done;
1200 
1201         if (!next.entry.isNull)
1202             nextMutant = next.entry.get;
1203     }
1204 
1205     void opCall(HandleTestResult data) {
1206         saveTestResult(data.result);
1207         if (!local.get!MutationTest.testBinaryDb.empty)
1208             saveTestBinaryDb(local.get!MutationTest.testBinaryDb);
1209     }
1210 
1211     void opCall(ref CheckStopCond data) {
1212         const halt = stopCheck.isHalt;
1213         data.halt = halt != TestStopCheck.HaltReason.none;
1214 
1215         final switch (halt) with (TestStopCheck.HaltReason) {
1216         case none:
1217             break;
1218         case maxRuntime:
1219             logger.info(stopCheck.maxRuntimeToString).collectException;
1220             break;
1221         case aliveTested:
1222             logger.info("Alive mutants threshold reached").collectException;
1223             break;
1224         case overloaded:
1225             logger.info(stopCheck.overloadToString).collectException;
1226             break;
1227         }
1228         logger.warning(data.halt, "Halting").collectException;
1229     }
1230 
1231     void opCall(ref NextSchemata data) {
1232         auto schematas = local.get!NextSchemata.schematas;
1233 
1234         const threshold = schemataMutantsThreshold(schemaConf.minMutantsPerSchema.get,
1235                 local.get!NextSchemata.invalidSchematas, local.get!NextSchemata.totalSchematas);
1236 
1237         while (!schematas.empty && !data.hasSchema) {
1238             // TODO: replace with my.collection.vector
1239             const id = schematas[0];
1240             schematas = schematas[1 .. $];
1241             const mutants = spinSql!(() {
1242                 return db.schemataMutantsCount(id, kinds);
1243             });
1244 
1245             logger.infof("Schema %s has %s mutants (threshold %s)", id.get,
1246                     mutants, threshold).collectException;
1247 
1248             if (mutants >= threshold) {
1249                 data.hasSchema.get = true;
1250                 data.schemataId = id;
1251                 logger.infof("Use schema %s (%s/%s)", id.get, local.get!NextSchemata.totalSchematas - schematas.length,
1252                         local.get!NextSchemata.totalSchematas).collectException;
1253             }
1254         }
1255 
1256         local.get!NextSchemata.schematas = schematas;
1257 
1258         data.stop.get = !data.hasSchema && schemaConf.stopAfterLastSchema;
1259     }
1260 
1261     void opCall(ref SchemataTest data) {
1262         import dextool.plugin.mutate.backend.test_mutant.schemata;
1263 
1264         // only remove schemas that are of no further use.
1265         bool remove = true;
1266         void updateRemove(MutationTestResult[] result) {
1267             foreach (a; data.result) {
1268                 final switch (a.status) with (Mutation.Status) {
1269                 case unknown:
1270                     goto case;
1271                 case equivalent:
1272                     goto case;
1273                 case noCoverage:
1274                     goto case;
1275                 case alive:
1276                     remove = false;
1277                     return;
1278                 case killed:
1279                     goto case;
1280                 case timeout:
1281                     goto case;
1282                 case killedByCompiler:
1283                     break;
1284                 }
1285             }
1286         }
1287 
1288         void save(MutationTestResult[] result) {
1289             updateRemove(result);
1290             saveTestResult(result);
1291             logger.infof(result.length > 0, "Saving %s schemata mutant results",
1292                     result.length).collectException;
1293         }
1294 
1295         try {
1296             auto driver = SchemataTestDriver(filesysIO, &runner, db, &testCaseAnalyzer, schemaConf,
1297                     data.id, stopCheck, kinds, conf.mutationCompile, conf.buildCmdTimeout);
1298 
1299             const saveResultInterval = 20.dur!"minutes";
1300             auto nextSave = Clock.currTime + saveResultInterval;
1301             while (driver.isRunning) {
1302                 driver.execute;
1303 
1304                 // to avoid loosing results in case of a crash etc save them
1305                 // continuously
1306                 if (Clock.currTime > nextSave && !driver.hasFatalError) {
1307                     save(driver.popResult);
1308                     nextSave = Clock.currTime + saveResultInterval;
1309                 }
1310             }
1311 
1312             data.fatalError = driver.hasFatalError;
1313 
1314             if (driver.hasFatalError) {
1315                 // do nothing
1316             } else if (driver.isInvalidSchema) {
1317                 local.get!NextSchemata.invalidSchematas++;
1318             } else {
1319                 save(driver.popResult);
1320             }
1321 
1322             if (remove) {
1323                 spinSql!(() { db.markUsed(data.id); });
1324             }
1325 
1326         } catch (Exception e) {
1327             logger.info(e.msg).collectException;
1328             logger.warning("Failed executing schemata ", data.id).collectException;
1329         }
1330     }
1331 
1332     void opCall(SchemataPruneUsed data) {
1333         try {
1334             const removed = db.pruneUsedSchemas;
1335 
1336             if (removed != 0) {
1337                 logger.infof("Removed %s schemas from the database", removed);
1338                 // vacuum the database because schemas take up a significant
1339                 // amount of space.
1340                 db.vacuum;
1341             }
1342         } catch (Exception e) {
1343             logger.warning(e.msg).collectException;
1344         }
1345     }
1346 
1347     void opCall(LoadSchematas data) {
1348         if (!schemaConf.use) {
1349             return;
1350         }
1351 
1352         auto app = appender!(SchemataId[])();
1353         foreach (id; spinSql!(() { return db.getSchematas(); })) {
1354             if (spinSql!(() { return db.schemataMutantsCount(id, kinds); }) >= schemataMutantsThreshold(
1355                     schemaConf.minMutantsPerSchema.get, 0, 0)) {
1356                 app.put(id);
1357             }
1358         }
1359 
1360         logger.trace("Found schematas: ", app.data).collectException;
1361         // random reorder to reduce the chance that multipe instances of
1362         // dextool use the same schema
1363         local.get!NextSchemata.schematas = app.data.randomCover.array;
1364         local.get!NextSchemata.totalSchematas = app.data.length;
1365     }
1366 
1367     void opCall(ref Coverage data) {
1368         import dextool.plugin.mutate.backend.test_mutant.coverage;
1369 
1370         auto tracked = spinSql!(() => db.getLatestTimeStampOfTestOrSut).orElse(SysTime.init);
1371         auto covTimeStamp = spinSql!(() => db.getCoverageTimeStamp).orElse(SysTime.init);
1372 
1373         if (tracked < covTimeStamp) {
1374             logger.info("Coverage information is up to date").collectException;
1375             return;
1376         } else {
1377             logger.infof("Coverage is out of date with SUT/tests (%s < %s)",
1378                     covTimeStamp, tracked).collectException;
1379         }
1380 
1381         try {
1382             auto driver = CoverageDriver(filesysIO, db, &runner, covConf,
1383                     conf.mutationCompile, conf.buildCmdTimeout);
1384             while (driver.isRunning) {
1385                 driver.execute;
1386             }
1387             data.propagate = true;
1388             data.fatalError = driver.hasFatalError;
1389         } catch (Exception e) {
1390             logger.warning(e.msg).collectException;
1391             data.fatalError = true;
1392         }
1393 
1394         if (data.fatalError)
1395             logger.warning("Error detected when trying to gather coverage information")
1396                 .collectException;
1397     }
1398 
1399     void opCall(PropagateCoverage data) {
1400         void propagate() @trusted {
1401             auto trans = db.transaction;
1402 
1403             auto noCov = db.getNotCoveredMutants;
1404             foreach (id; noCov) {
1405                 db.updateMutationStatus(id, Mutation.Status.noCoverage, ExitStatus(0));
1406                 db.removeFromWorklist(id);
1407             }
1408 
1409             trans.commit;
1410             logger.infof("Marked %s mutants as alive because they where not covered by any test",
1411                     noCov.length);
1412         }
1413 
1414         spinSql!(() => propagate);
1415     }
1416 
1417     void saveTestResult(MutationTestResult[] results) @safe nothrow {
1418         void statusUpdate(MutationTestResult result) @safe {
1419             import dextool.plugin.mutate.backend.test_mutant.timeout : updateMutantStatus;
1420 
1421             estimate.update(result.status);
1422 
1423             updateMutantStatus(*db, result.id, result.status,
1424                     result.exitStatus, timeoutFsm.output.iter);
1425             db.updateMutation(result.id, result.profile);
1426             db.updateMutationTestCases(result.id, result.testCases);
1427             db.removeFromWorklist(result.id);
1428 
1429             if (result.status == Mutation.Status.alive)
1430                 stopCheck.incrAliveMutants;
1431         }
1432 
1433         const left = spinSql!(() @trusted {
1434             auto t = db.transaction;
1435             foreach (a; results) {
1436                 statusUpdate(a);
1437             }
1438 
1439             const left = db.getWorklistCount;
1440             t.commit;
1441             return left;
1442         });
1443 
1444         logger.infof("%s mutants left to test. Estimated mutation score %.3s (error %.3s)",
1445                 left, estimate.value.get, estimate.error.get).collectException;
1446     }
1447 
1448     void saveTestBinaryDb(ref TestBinaryDb testBinaryDb) @safe nothrow {
1449         import dextool.plugin.mutate.backend.database.type : ChecksumTestCmdMutated;
1450 
1451         spinSql!(() @trusted {
1452             auto t = db.transaction;
1453             foreach (a; testBinaryDb.added.byKeyValue) {
1454                 db.testCmdApi.add(ChecksumTestCmdMutated(a.key), a.value);
1455             }
1456             // magic number. about 10 Mbyte in the database (8+8+8)*20000
1457             db.testCmdApi.trimMutated(200000);
1458             t.commit;
1459         });
1460 
1461         testBinaryDb.clearAdded;
1462     }
1463 }
1464 
1465 private:
1466 
1467 /** A schemata must have at least this many mutants that have the status unknown
1468  * for it to be cost efficient to use schemata.
1469  *
1470  * The weights dynamically adjust with how many of the schemas that has failed
1471  * to compile.
1472  *
1473  * Params:
1474  *  checkSchemata = if the user has activated check_schemata that run all test cases before the schemata is used.
1475  */
1476 long schemataMutantsThreshold(const long minThreshold, const long invalidSchematas,
1477         const long totalSchematas) @safe pure nothrow @nogc {
1478     // "10" is a magic number that felt good but not too conservative. A future
1479     // improvement is to instead base it on the ratio between compilation time
1480     // and test suite execution time.
1481     if (totalSchematas > 0)
1482         return cast(long)(minThreshold + 10.0 * (
1483                 cast(double) invalidSchematas / cast(double) totalSchematas));
1484     return cast(long) minThreshold;
1485 }
1486 
1487 /** Compare the old test cases with those that have been found this run.
1488  *
1489  * TODO: the side effect that this function print to the console is NOT good.
1490  */
1491 bool hasNewTestCases(ref Set!string old_tcs, ref Set!string found_tcs) @safe nothrow {
1492     bool rval;
1493 
1494     auto new_tcs = found_tcs.setDifference(old_tcs);
1495     foreach (tc; new_tcs.toRange) {
1496         logger.info(!rval, "Found new test case(s):").collectException;
1497         logger.infof("%s", tc).collectException;
1498         rval = true;
1499     }
1500 
1501     return rval;
1502 }
1503 
1504 /** Compare old and new test cases to print those that have been removed.
1505  */
1506 void printDroppedTestCases(ref Set!string old_tcs, ref Set!string changed_tcs) @safe nothrow {
1507     auto diff = old_tcs.setDifference(changed_tcs);
1508     auto removed = diff.toArray;
1509 
1510     logger.info(removed.length != 0, "Detected test cases that has been removed:").collectException;
1511     foreach (tc; removed) {
1512         logger.infof("%s", tc).collectException;
1513     }
1514 }
1515 
1516 /// Returns: true if all tests cases have unique identifiers
1517 void warnIfConflictingTestCaseIdentifiers(TestCase[] found_tcs) @safe nothrow {
1518     Set!TestCase checked;
1519     bool conflict;
1520 
1521     foreach (tc; found_tcs) {
1522         if (checked.contains(tc)) {
1523             logger.info(!conflict,
1524                     "Found test cases that do not have global, unique identifiers")
1525                 .collectException;
1526             logger.info(!conflict,
1527                     "This make the report of test cases that has killed zero mutants unreliable")
1528                 .collectException;
1529             logger.info("%s", tc).collectException;
1530             conflict = true;
1531         }
1532     }
1533 }
1534 
1535 private:
1536 
1537 /**
1538 DESCRIPTION
1539 
1540      The getloadavg() function returns the number of processes in the system
1541      run queue averaged over various periods of time.  Up to nelem samples are
1542      retrieved and assigned to successive elements of loadavg[].  The system
1543      imposes a maximum of 3 samples, representing averages over the last 1, 5,
1544      and 15 minutes, respectively.
1545 
1546 
1547 DIAGNOSTICS
1548 
1549      If the load average was unobtainable, -1 is returned; otherwise, the num-
1550      ber of samples actually retrieved is returned.
1551  */
1552 extern (C) int getloadavg(double* loadavg, int nelem) nothrow;