1 /** 2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 */ 10 module dextool.plugin.mutate.backend.test_mutant; 11 12 import core.time : Duration, dur; 13 import logger = std.experimental.logger; 14 import std.algorithm : map, filter, joiner, among; 15 import std.array : empty, array, appender; 16 import std.datetime : SysTime, Clock; 17 import std.datetime.stopwatch : StopWatch, AutoStart; 18 import std.exception : collectException; 19 import std.format : format; 20 import std.random : randomCover; 21 import std.typecons : Nullable, Tuple, Yes; 22 23 import blob_model : Blob; 24 import my.container.vector; 25 import my.fsm : Fsm, next, act, get, TypeDataMap; 26 import my.hash : Checksum64; 27 import my.named_type; 28 import my.optional; 29 import my.set; 30 import proc : DrainElement; 31 import sumtype; 32 static import my.fsm; 33 34 import dextool.plugin.mutate.backend.database : Database, MutationEntry, 35 NextMutationEntry, spinSql, TestFile, ChecksumTestCmdOriginal; 36 import dextool.plugin.mutate.backend.interface_ : FilesysIO; 37 import dextool.plugin.mutate.backend.test_mutant.common; 38 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner, 39 findExecutables, TestRunResult = TestResult; 40 import dextool.plugin.mutate.backend.type : Mutation, TestCase, ExitStatus; 41 import dextool.plugin.mutate.config; 42 import dextool.plugin.mutate.type : ShellCommand; 43 import dextool.type : AbsolutePath, ExitStatusType, Path; 44 45 @safe: 46 47 auto makeTestMutant() { 48 return BuildTestMutant(); 49 } 50 51 private: 52 53 struct BuildTestMutant { 54 @safe: 55 56 import dextool.plugin.mutate.type : MutationKind; 57 58 private struct InternalData { 59 Mutation.Kind[] kinds; 60 FilesysIO filesys_io; 61 ConfigMutationTest config; 62 ConfigSchema schemaConf; 63 ConfigCoverage covConf; 64 } 65 66 private InternalData data; 67 68 auto config(ConfigMutationTest c) @trusted nothrow { 69 data.config = c; 70 return this; 71 } 72 73 auto config(ConfigSchema c) @trusted nothrow { 74 data.schemaConf = c; 75 return this; 76 } 77 78 auto config(ConfigCoverage c) @trusted nothrow { 79 data.covConf = c; 80 return this; 81 } 82 83 auto mutations(MutationKind[] v) nothrow { 84 import dextool.plugin.mutate.backend.mutation_type : toInternal; 85 86 logger.infof("mutation operators: %(%s, %)", v).collectException; 87 data.kinds = toInternal(v); 88 return this; 89 } 90 91 ExitStatusType run(const AbsolutePath dbPath, FilesysIO fio) @trusted { 92 try { 93 auto db = Database.make(dbPath); 94 return internalRun(db, fio); 95 } catch (Exception e) { 96 logger.error(e.msg).collectException; 97 } 98 99 return ExitStatusType.Errors; 100 } 101 102 private ExitStatusType internalRun(ref Database db, FilesysIO fio) { 103 // trusted because the lifetime of the database is guaranteed to outlive any instances in this scope 104 auto dbPtr = () @trusted { return &db; }(); 105 106 auto cleanup = new AutoCleanup; 107 scope (exit) 108 cleanup.cleanup; 109 110 auto test_driver = TestDriver(dbPtr, fio, data.kinds, cleanup, 111 data.config, data.covConf, data.schemaConf); 112 113 while (test_driver.isRunning) { 114 test_driver.execute; 115 } 116 117 return test_driver.status; 118 } 119 } 120 121 struct MeasureTestDurationResult { 122 bool ok; 123 Duration[] runtime; 124 } 125 126 /** Measure the time it takes to run the test command. 127 * 128 * The runtime is the lowest of three executions. Anything else is assumed to 129 * be variations in the system. 130 * 131 * If the tests fail (exit code isn't 0) any time then they are too unreliable 132 * to use for mutation testing. 133 * 134 * Params: 135 * runner = ? 136 * samples = number of times to run the test suite 137 */ 138 MeasureTestDurationResult measureTestCommand(ref TestRunner runner, int samples) @safe nothrow { 139 import std.algorithm : min; 140 import proc; 141 142 if (runner.empty) { 143 collectException(logger.error("No test command(s) specified (--test-cmd)")); 144 return MeasureTestDurationResult(false); 145 } 146 147 static struct Rval { 148 TestRunResult result; 149 Duration runtime; 150 } 151 152 auto runTest() @safe { 153 auto sw = StopWatch(AutoStart.yes); 154 auto res = runner.run(999.dur!"hours"); 155 return Rval(res, sw.peek); 156 } 157 158 static void print(TestRunResult res) @trusted { 159 import std.stdio : stdout, write; 160 161 foreach (kv; res.output.byKeyValue) { 162 logger.info("test_cmd: ", kv.key); 163 foreach (l; kv.value) 164 write(l.byUTF8); 165 } 166 167 stdout.flush; 168 } 169 170 Duration[] runtimes; 171 bool failed; 172 for (int i; i < samples && !failed; ++i) { 173 try { 174 auto res = runTest; 175 final switch (res.result.status) with (TestRunResult) { 176 case Status.passed: 177 runtimes ~= res.runtime; 178 break; 179 case Status.failed: 180 goto case; 181 case Status.timeout: 182 goto case; 183 case Status.error: 184 failed = true; 185 print(res.result); 186 logger.info("failing commands: ", res.result.output.byKey); 187 logger.info("exit status: ", res.result.exitStatus.get); 188 break; 189 } 190 logger.infof("%s: Measured test command runtime %s", i, res.runtime); 191 } catch (Exception e) { 192 logger.error(e.msg).collectException; 193 failed = true; 194 } 195 } 196 197 return MeasureTestDurationResult(!failed, runtimes); 198 } 199 200 struct TestDriver { 201 import std.datetime : SysTime; 202 import dextool.plugin.mutate.backend.database : SchemataId, MutationStatusId; 203 import dextool.plugin.mutate.backend.report.analyzers : EstimateScore; 204 import dextool.plugin.mutate.backend.test_mutant.source_mutant : MutationTestDriver; 205 import dextool.plugin.mutate.backend.test_mutant.timeout : calculateTimeout, TimeoutFsm; 206 import dextool.plugin.mutate.type : MutationOrder; 207 208 Database* db; 209 FilesysIO filesysIO; 210 Mutation.Kind[] kinds; 211 AutoCleanup autoCleanup; 212 213 ConfigMutationTest conf; 214 ConfigSchema schemaConf; 215 ConfigCoverage covConf; 216 217 /// Runs the test commands. 218 TestRunner runner; 219 220 /// 221 TestCaseAnalyzer testCaseAnalyzer; 222 223 /// Stop conditions (most of them) 224 TestStopCheck stopCheck; 225 226 /// assuming that there are no more than 100 instances running in 227 /// parallel. 228 uint maxParallelInstances; 229 230 // need to use 10000 because in an untested code base it is not 231 // uncommon for mutants being in the thousands. 232 enum long unknownWeight = 10000; 233 // using a factor 1000 to make a pull request mutant very high prio 234 enum long pullRequestWeight = unknownWeight * 1000; 235 236 TimeoutFsm timeoutFsm; 237 238 /// The time it takes to execute the test suite when no mutant is injected. 239 Duration testSuiteRuntime; 240 241 /// the next mutant to test, if there are any. 242 MutationEntry nextMutant; 243 244 // when the user manually configure the timeout it means that the 245 // timeout algorithm should not be used. 246 bool hardcodedTimeout; 247 248 /// Test commands to execute. 249 ShellCommand[] testCmds; 250 251 /// mutation score estimation 252 EstimateScore estimate; 253 254 // The order to test mutants. It is either affected by the user directly or if pull request mode is activated. 255 MutationOrder mutationOrder; 256 257 static struct UpdateTimeoutData { 258 long lastTimeoutIter; 259 } 260 261 static struct None { 262 } 263 264 static struct Initialize { 265 bool halt; 266 } 267 268 static struct PullRequest { 269 } 270 271 static struct PullRequestData { 272 import dextool.plugin.mutate.type : TestConstraint; 273 274 TestConstraint constraint; 275 } 276 277 static struct SanityCheck { 278 bool sanityCheckFailed; 279 } 280 281 static struct AnalyzeTestCmdForTestCase { 282 TestCase[][ShellCommand] foundTestCases; 283 } 284 285 static struct UpdateAndResetAliveMutants { 286 TestCase[][ShellCommand] foundTestCases; 287 } 288 289 static struct ResetOldMutant { 290 } 291 292 static struct ResetOldMutantData { 293 /// Number of mutants that where reset. 294 long maxReset; 295 NamedType!(double, Tag!"OldMutantPercentage", double.init, TagStringable) resetPercentage; 296 } 297 298 static struct Cleanup { 299 } 300 301 static struct CheckMutantsLeft { 302 bool allMutantsTested; 303 } 304 305 static struct SaveMutationScore { 306 } 307 308 static struct ParseStdin { 309 } 310 311 static struct PreCompileSut { 312 bool compilationError; 313 } 314 315 static struct FindTestCmds { 316 } 317 318 static struct ChooseMode { 319 } 320 321 static struct MeasureTestSuite { 322 bool unreliableTestSuite; 323 } 324 325 static struct MutationTest { 326 NamedType!(bool, Tag!"MutationError", bool.init, TagStringable) mutationError; 327 MutationTestResult[] result; 328 } 329 330 static struct MutationTestData { 331 TestBinaryDb testBinaryDb; 332 } 333 334 static struct CheckTimeout { 335 bool timeoutUnchanged; 336 } 337 338 static struct NextSchemataData { 339 SchemataId[] schematas; 340 long totalSchematas; 341 long invalidSchematas; 342 } 343 344 static struct NextSchemata { 345 NamedType!(bool, Tag!"HasSchema", bool.init, TagStringable, ImplicitConvertable) hasSchema; 346 SchemataId schemataId; 347 348 /// stop mutation testing because the last schema has been used and the 349 /// user has configured that the testing should stop now. 350 NamedType!(bool, Tag!"StopTesting", bool.init, TagStringable, ImplicitConvertable) stop; 351 } 352 353 static struct SchemataTest { 354 SchemataId id; 355 MutationTestResult[] result; 356 bool fatalError; 357 } 358 359 static struct SchemataPruneUsed { 360 } 361 362 static struct Done { 363 } 364 365 static struct Error { 366 } 367 368 static struct UpdateTimeout { 369 } 370 371 static struct CheckPullRequestMutant { 372 NamedType!(bool, Tag!"NoUnknown", bool.init, TagStringable, ImplicitConvertable) noUnknownMutantsLeft; 373 } 374 375 static struct CheckPullRequestMutantData { 376 long startWorklistCnt; 377 long stopAfter; 378 } 379 380 static struct NextMutant { 381 NamedType!(bool, Tag!"NoUnknown", bool.init, TagStringable, ImplicitConvertable) noUnknownMutantsLeft; 382 } 383 384 static struct HandleTestResult { 385 MutationTestResult[] result; 386 } 387 388 static struct CheckStopCond { 389 bool halt; 390 } 391 392 static struct LoadSchematas { 393 } 394 395 static struct OverloadCheck { 396 bool sleep; 397 } 398 399 static struct ContinuesCheckTestSuite { 400 bool ok; 401 } 402 403 static struct ContinuesCheckTestSuiteData { 404 long lastWorklistCnt; 405 SysTime lastCheck; 406 } 407 408 static struct Stop { 409 } 410 411 static struct Coverage { 412 bool propagate; 413 bool fatalError; 414 } 415 416 static struct PropagateCoverage { 417 } 418 419 static struct ChecksumTestCmds { 420 } 421 422 static struct SaveTestBinary { 423 } 424 425 alias Fsm = my.fsm.Fsm!(None, Initialize, SanityCheck, 426 AnalyzeTestCmdForTestCase, UpdateAndResetAliveMutants, ResetOldMutant, 427 Cleanup, CheckMutantsLeft, PreCompileSut, MeasureTestSuite, NextMutant, 428 MutationTest, HandleTestResult, CheckTimeout, Done, Error, 429 UpdateTimeout, CheckStopCond, PullRequest, CheckPullRequestMutant, ParseStdin, 430 FindTestCmds, ChooseMode, NextSchemata, SchemataTest, LoadSchematas, 431 SchemataPruneUsed, Stop, SaveMutationScore, OverloadCheck, 432 Coverage, PropagateCoverage, ContinuesCheckTestSuite, 433 ChecksumTestCmds, SaveTestBinary); 434 alias LocalStateDataT = Tuple!(UpdateTimeoutData, CheckPullRequestMutantData, PullRequestData, 435 ResetOldMutantData, NextSchemataData, ContinuesCheckTestSuiteData, MutationTestData); 436 437 private { 438 Fsm fsm; 439 TypeDataMap!(LocalStateDataT, UpdateTimeout, CheckPullRequestMutant, PullRequest, 440 ResetOldMutant, NextSchemata, ContinuesCheckTestSuite, MutationTest) local; 441 bool isRunning_ = true; 442 bool isDone = false; 443 } 444 445 this(Database* db, FilesysIO filesysIO, Mutation.Kind[] kinds, AutoCleanup autoCleanup, 446 ConfigMutationTest conf, ConfigCoverage coverage, ConfigSchema schema) { 447 this.db = db; 448 this.filesysIO = filesysIO; 449 this.kinds = kinds; 450 this.autoCleanup = autoCleanup; 451 this.conf = conf; 452 this.covConf = coverage; 453 this.schemaConf = schema; 454 455 this.timeoutFsm = TimeoutFsm(kinds); 456 this.hardcodedTimeout = !conf.mutationTesterRuntime.isNull; 457 local.get!PullRequest.constraint = conf.constraint; 458 local.get!ResetOldMutant.maxReset = conf.oldMutantsNr; 459 local.get!ResetOldMutant.resetPercentage = conf.oldMutantPercentage; 460 this.testCmds = conf.mutationTester; 461 this.mutationOrder = conf.mutationOrder; 462 463 this.runner.useEarlyStop(conf.useEarlyTestCmdStop); 464 this.runner = TestRunner.make(conf.testPoolSize); 465 this.runner.useEarlyStop(conf.useEarlyTestCmdStop); 466 this.runner.maxOutputCapture( 467 TestRunner.MaxCaptureBytes(conf.maxTestCaseOutput.get * 1024 * 1024)); 468 this.runner.put(conf.mutationTester); 469 470 // TODO: allow a user, as is for test_cmd, to specify an array of 471 // external analyzers. 472 this.testCaseAnalyzer = TestCaseAnalyzer(conf.mutationTestCaseBuiltin, 473 conf.mutationTestCaseAnalyze, autoCleanup); 474 475 this.stopCheck = TestStopCheck(conf); 476 477 this.maxParallelInstances = () { 478 if (mutationOrder.among(MutationOrder.random, MutationOrder.bySize)) 479 return 100; 480 return 1; 481 }(); 482 483 if (logger.globalLogLevel.among(logger.LogLevel.trace, logger.LogLevel.all)) 484 fsm.logger = (string s) { logger.trace(s); }; 485 } 486 487 static void execute_(ref TestDriver self) @trusted { 488 // see test_mutant/basis.md and figures/test_mutant_fsm.pu for a 489 // graphical view of the state machine. 490 491 self.fsm.next!((None a) => fsm(Initialize.init), (Initialize a) { 492 if (a.halt) 493 return fsm(CheckStopCond.init); 494 return fsm(SanityCheck.init); 495 }, (SanityCheck a) { 496 if (a.sanityCheckFailed) 497 return fsm(Error.init); 498 if (self.conf.unifiedDiffFromStdin) 499 return fsm(ParseStdin.init); 500 return fsm(PreCompileSut.init); 501 }, (ParseStdin a) => fsm(PreCompileSut.init), (AnalyzeTestCmdForTestCase a) => fsm( 502 UpdateAndResetAliveMutants(a.foundTestCases)), 503 (UpdateAndResetAliveMutants a) => fsm(CheckMutantsLeft.init), 504 (ResetOldMutant a) => fsm(UpdateTimeout.init), (Cleanup a) { 505 if (self.local.get!PullRequest.constraint.empty) 506 return fsm(NextSchemata.init); 507 return fsm(CheckPullRequestMutant.init); 508 }, (CheckMutantsLeft a) { 509 if (a.allMutantsTested && self.conf.onOldMutants == ConfigMutationTest 510 .OldMutant.nothing) 511 return fsm(Done.init); 512 if (self.conf.testCmdChecksum.get) 513 return fsm(ChecksumTestCmds.init); 514 return fsm(MeasureTestSuite.init); 515 }, (ChecksumTestCmds a) => MeasureTestSuite.init, (SaveMutationScore a) => SaveTestBinary.init, 516 (SaveTestBinary a) => Stop.init, (PreCompileSut a) { 517 if (a.compilationError) 518 return fsm(Error.init); 519 if (self.conf.testCommandDir.empty) 520 return fsm(ChooseMode.init); 521 return fsm(FindTestCmds.init); 522 }, (FindTestCmds a) { return fsm(ChooseMode.init); }, (ChooseMode a) { 523 if (!self.local.get!PullRequest.constraint.empty) 524 return fsm(PullRequest.init); 525 if (!self.conf.mutationTestCaseAnalyze.empty 526 || !self.conf.mutationTestCaseBuiltin.empty) 527 return fsm(AnalyzeTestCmdForTestCase.init); 528 return fsm(CheckMutantsLeft.init); 529 }, (PullRequest a) => fsm(CheckMutantsLeft.init), (MeasureTestSuite a) { 530 if (a.unreliableTestSuite) 531 return fsm(Error.init); 532 if (self.covConf.use) 533 return fsm(Coverage.init); 534 return fsm(LoadSchematas.init); 535 }, (Coverage a) { 536 if (a.fatalError) 537 return fsm(Error.init); 538 if (a.propagate) 539 return fsm(PropagateCoverage.init); 540 return fsm(LoadSchematas.init); 541 }, (PropagateCoverage a) => LoadSchematas.init, 542 (LoadSchematas a) => fsm(ResetOldMutant.init), (CheckPullRequestMutant a) { 543 if (a.noUnknownMutantsLeft) 544 return fsm(Done.init); 545 return fsm(NextMutant.init); 546 }, (NextSchemata a) { 547 if (a.hasSchema) 548 return fsm(SchemataTest(a.schemataId)); 549 if (a.stop) 550 return fsm(Done.init); 551 return fsm(NextMutant.init); 552 }, (SchemataTest a) { 553 if (a.fatalError) 554 return fsm(Error.init); 555 return fsm(CheckStopCond.init); 556 }, (NextMutant a) { 557 if (a.noUnknownMutantsLeft) 558 return fsm(CheckTimeout.init); 559 return fsm(MutationTest.init); 560 }, (UpdateTimeout a) => fsm(OverloadCheck.init), (OverloadCheck a) { 561 if (a.sleep) 562 return fsm(CheckStopCond.init); 563 return fsm(ContinuesCheckTestSuite.init); 564 }, (ContinuesCheckTestSuite a) { 565 if (a.ok) 566 return fsm(Cleanup.init); 567 return fsm(Error.init); 568 }, (MutationTest a) { 569 if (a.mutationError) 570 return fsm(Error.init); 571 return fsm(HandleTestResult(a.result)); 572 }, (HandleTestResult a) => fsm(CheckStopCond.init), (CheckStopCond a) { 573 if (a.halt) 574 return fsm(Done.init); 575 return fsm(UpdateTimeout.init); 576 }, (CheckTimeout a) { 577 if (a.timeoutUnchanged) 578 return fsm(Done.init); 579 return fsm(UpdateTimeout.init); 580 }, (SchemataPruneUsed a) => SaveMutationScore.init, 581 (Done a) => fsm(SchemataPruneUsed.init), 582 (Error a) => fsm(Stop.init), (Stop a) => fsm(a)); 583 584 self.fsm.act!(self); 585 } 586 587 nothrow: 588 589 void execute() { 590 try { 591 execute_(this); 592 } catch (Exception e) { 593 logger.warning(e.msg).collectException; 594 } 595 } 596 597 bool isRunning() { 598 return isRunning_; 599 } 600 601 ExitStatusType status() { 602 if (isDone) 603 return ExitStatusType.Ok; 604 return ExitStatusType.Errors; 605 } 606 607 void opCall(None data) { 608 } 609 610 void opCall(ref Initialize data) { 611 logger.info("Initializing worklist").collectException; 612 spinSql!(() { 613 db.updateWorklist(kinds, Mutation.Status.unknown, unknownWeight, mutationOrder); 614 }); 615 616 // detect if the system is overloaded before trying to do something 617 // slow such as compiling the SUT. 618 if (conf.loadBehavior == ConfigMutationTest.LoadBehavior.halt && stopCheck.isHalt) { 619 data.halt = true; 620 } 621 } 622 623 void opCall(Stop data) { 624 isRunning_ = false; 625 } 626 627 void opCall(Done data) { 628 logger.info("Done!").collectException; 629 isDone = true; 630 } 631 632 void opCall(Error data) { 633 autoCleanup.cleanup; 634 } 635 636 void opCall(ref SanityCheck data) { 637 import core.sys.posix.sys.stat : S_IWUSR; 638 import std.path : buildPath; 639 import my.file : getAttrs; 640 import colorlog : color, Color; 641 import dextool.plugin.mutate.backend.utility : checksum, Checksum; 642 643 logger.info("Sanity check of files to mutate").collectException; 644 645 auto failed = appender!(string[])(); 646 auto checksumFailed = appender!(string[])(); 647 auto writePermissionFailed = appender!(string[])(); 648 foreach (file; spinSql!(() { return db.getFiles; })) { 649 auto db_checksum = spinSql!(() { return db.getFileChecksum(file); }); 650 651 try { 652 auto abs_f = AbsolutePath(buildPath(filesysIO.getOutputDir, file)); 653 auto f_checksum = checksum(filesysIO.makeInput(abs_f).content[]); 654 if (db_checksum != f_checksum) { 655 checksumFailed.put(abs_f); 656 } 657 658 uint attrs; 659 if (getAttrs(abs_f, attrs)) { 660 if ((attrs & S_IWUSR) == 0) { 661 writePermissionFailed.put(abs_f); 662 } 663 } else { 664 writePermissionFailed.put(abs_f); 665 } 666 } catch (Exception e) { 667 failed.put(file); 668 logger.warningf("%s: %s", file, e.msg).collectException; 669 } 670 } 671 672 data.sanityCheckFailed = !failed.data.empty 673 || !checksumFailed.data.empty || !writePermissionFailed.data.empty; 674 675 if (data.sanityCheckFailed) { 676 logger.info(!failed.data.empty, 677 "Unknown error when checking the files").collectException; 678 foreach (f; failed.data) 679 logger.info(f).collectException; 680 681 logger.info(!checksumFailed.data.empty, 682 "Detected that file(s) has changed since last analyze where done") 683 .collectException; 684 logger.info(!checksumFailed.data.empty, 685 "Either restore the file(s) or rerun the analyze").collectException; 686 foreach (f; checksumFailed.data) 687 logger.info(f).collectException; 688 689 logger.info(!writePermissionFailed.data.empty, 690 "Files to mutate are not writable").collectException; 691 foreach (f; writePermissionFailed.data) 692 logger.info(f).collectException; 693 694 logger.info("Failed".color.fgRed).collectException; 695 } else { 696 logger.info("Ok".color.fgGreen).collectException; 697 } 698 } 699 700 void opCall(ref OverloadCheck data) { 701 if (conf.loadBehavior == ConfigMutationTest.LoadBehavior.slowdown && stopCheck.isOverloaded) { 702 data.sleep = true; 703 logger.info(stopCheck.overloadToString).collectException; 704 stopCheck.pause; 705 } 706 } 707 708 void opCall(ref ContinuesCheckTestSuite data) { 709 import std.algorithm : max; 710 711 data.ok = true; 712 713 if (!conf.contCheckTestSuite) 714 return; 715 716 enum forceCheckEach = 1.dur!"hours"; 717 718 const wlist = spinSql!(() => db.getWorklistCount); 719 if (local.get!ContinuesCheckTestSuite.lastWorklistCnt == 0) { 720 // first time, just initialize. 721 local.get!ContinuesCheckTestSuite.lastWorklistCnt = wlist; 722 local.get!ContinuesCheckTestSuite.lastCheck = Clock.currTime + forceCheckEach; 723 return; 724 } 725 726 const period = conf.contCheckTestSuitePeriod.get; 727 const diffCnt = local.get!ContinuesCheckTestSuite.lastWorklistCnt - wlist; 728 if (!(wlist % period == 0 || diffCnt >= period 729 || Clock.currTime > local.get!ContinuesCheckTestSuite.lastCheck)) 730 return; 731 732 local.get!ContinuesCheckTestSuite.lastWorklistCnt = wlist; 733 local.get!ContinuesCheckTestSuite.lastCheck = Clock.currTime + forceCheckEach; 734 735 compile(conf.mutationCompile, conf.buildCmdTimeout, PrintCompileOnFailure(true)).match!( 736 (Mutation.Status a) { data.ok = false; }, (bool success) { 737 data.ok = success; 738 }); 739 740 if (data.ok) { 741 try { 742 data.ok = measureTestCommand(runner, 1).ok; 743 } catch (Exception e) { 744 logger.error(e.msg).collectException; 745 data.ok = false; 746 } 747 } 748 749 if (!data.ok) { 750 logger.warning("Continues sanity check of the test suite has failed.").collectException; 751 logger.infof("Rolling back the status of the last %s mutants to status unknown.", 752 period).collectException; 753 foreach (a; spinSql!(() => db.getLatestMutants(kinds, max(diffCnt, period)))) { 754 spinSql!(() => db.updateMutation(a.id, Mutation.Status.unknown, 755 ExitStatus(0), MutantTimeProfile.init)); 756 } 757 } 758 } 759 760 void opCall(ParseStdin data) { 761 import dextool.plugin.mutate.backend.diff_parser : diffFromStdin; 762 import dextool.plugin.mutate.type : Line; 763 764 try { 765 auto constraint = local.get!PullRequest.constraint; 766 foreach (pkv; diffFromStdin.toRange(filesysIO.getOutputDir)) { 767 constraint.value[pkv.key] ~= pkv.value.toRange.map!(a => Line(a)).array; 768 } 769 local.get!PullRequest.constraint = constraint; 770 } catch (Exception e) { 771 logger.warning(e.msg).collectException; 772 } 773 } 774 775 void opCall(ref AnalyzeTestCmdForTestCase data) { 776 TestCase[][ShellCommand] found; 777 try { 778 runner.captureAll(true); 779 scope (exit) 780 runner.captureAll(false); 781 782 // using an unreasonable timeout to make it possible to analyze for 783 // test cases and measure the test suite. 784 auto res = runTester(runner, 999.dur!"hours"); 785 786 foreach (testCmd; res.output.byKeyValue) { 787 auto analyze = testCaseAnalyzer.analyze(testCmd.key, testCmd.value, Yes.allFound); 788 789 analyze.match!((TestCaseAnalyzer.Success a) { 790 found[testCmd.key] = a.found; 791 }, (TestCaseAnalyzer.Unstable a) { 792 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 793 found[testCmd.key] = a.found; 794 }, (TestCaseAnalyzer.Failed a) { 795 logger.warning("The parser that analyze the output for test case(s) failed"); 796 }); 797 798 } 799 800 warnIfConflictingTestCaseIdentifiers(found.byValue.joiner.array); 801 } catch (Exception e) { 802 logger.warning(e.msg).collectException; 803 } 804 805 data.foundTestCases = found; 806 } 807 808 void opCall(UpdateAndResetAliveMutants data) { 809 import std.traits : EnumMembers; 810 811 // the test cases before anything has potentially changed. 812 auto old_tcs = spinSql!(() { 813 Set!string old_tcs; 814 foreach (tc; db.getDetectedTestCases) 815 old_tcs.add(tc.name); 816 return old_tcs; 817 }); 818 819 void transaction() @safe { 820 final switch (conf.onRemovedTestCases) with (ConfigMutationTest.RemovedTestCases) { 821 case doNothing: 822 db.addDetectedTestCases(data.foundTestCases.byValue.joiner.array); 823 break; 824 case remove: 825 bool update; 826 // change all mutants which, if a test case is removed, no 827 // longer has a test case that kills it to unknown status 828 foreach (id; db.setDetectedTestCases(data.foundTestCases.byValue.joiner.array)) { 829 if (!db.hasTestCases(id)) { 830 update = true; 831 db.updateMutationStatus(id, Mutation.Status.unknown, ExitStatus(0)); 832 } 833 } 834 if (update) { 835 db.updateWorklist(kinds, Mutation.Status.unknown); 836 } 837 break; 838 } 839 } 840 841 auto found_tcs = spinSql!(() @trusted { 842 auto tr = db.transaction; 843 transaction(); 844 845 Set!string found_tcs; 846 foreach (tc; db.getDetectedTestCases) 847 found_tcs.add(tc.name); 848 849 tr.commit; 850 return found_tcs; 851 }); 852 853 printDroppedTestCases(old_tcs, found_tcs); 854 855 if (hasNewTestCases(old_tcs, found_tcs) 856 && conf.onNewTestCases == ConfigMutationTest.NewTestCases.resetAlive) { 857 logger.info("Adding alive mutants to worklist").collectException; 858 spinSql!(() { 859 db.updateWorklist(kinds, Mutation.Status.alive); 860 // if these mutants are covered by the tests then they will be 861 // removed from the worklist in PropagateCoverage. 862 db.updateWorklist(kinds, Mutation.Status.noCoverage); 863 }); 864 } 865 } 866 867 void opCall(ResetOldMutant data) { 868 import std.range : enumerate; 869 import dextool.plugin.mutate.backend.database.type; 870 871 void printStatus(T0)(T0 oldestMutant, SysTime newestTest, SysTime newestFile) { 872 logger.info("Tests last changed ", newestTest).collectException; 873 logger.info("Source code last changed ", newestFile).collectException; 874 875 if (!oldestMutant.empty) { 876 logger.info("The oldest mutant is ", oldestMutant[0].updated).collectException; 877 } 878 } 879 880 if (conf.onOldMutants == ConfigMutationTest.OldMutant.nothing) { 881 return; 882 } 883 if (spinSql!(() { return db.getWorklistCount; }) != 0) { 884 // do not re-test any old mutants if there are still work to do in the worklist. 885 return; 886 } 887 888 const oldestMutant = spinSql!(() => db.getOldestMutants(kinds, 1)); 889 const newestTest = spinSql!(() => db.getNewestTestFile).orElse(TestFile.init).timeStamp; 890 const newestFile = spinSql!(() => db.getNewestFile).orElse(SysTime.init); 891 if (!oldestMutant.empty && oldestMutant[0].updated > newestTest 892 && oldestMutant[0].updated > newestFile) { 893 // only re-test old mutants if needed. 894 logger.info("Mutation status is up to date").collectException; 895 printStatus(oldestMutant, newestTest, newestFile); 896 return; 897 } else { 898 logger.info("Mutation status is out of sync").collectException; 899 printStatus(oldestMutant, newestTest, newestFile); 900 } 901 902 const long testCnt = () { 903 if (local.get!ResetOldMutant.resetPercentage.get == 0.0) { 904 return local.get!ResetOldMutant.maxReset; 905 } 906 907 const total = spinSql!(() { return db.totalSrcMutants(kinds).count; }); 908 const rval = cast(long)(1 + total * local.get!ResetOldMutant.resetPercentage.get 909 / 100.0); 910 return rval; 911 }(); 912 913 auto oldest = spinSql!(() { return db.getOldestMutants(kinds, testCnt); }); 914 915 logger.infof("Adding %s old mutants to worklist", oldest.length).collectException; 916 spinSql!(() { 917 foreach (const old; oldest.enumerate) { 918 logger.infof("%s Last updated %s", old.index + 1, 919 old.value.updated).collectException; 920 db.addToWorklist(old.value.id); 921 } 922 }); 923 } 924 925 void opCall(Cleanup data) { 926 autoCleanup.cleanup; 927 } 928 929 void opCall(ref CheckMutantsLeft data) { 930 spinSql!(() { timeoutFsm.execute(*db); }); 931 932 data.allMutantsTested = timeoutFsm.output.done; 933 934 if (timeoutFsm.output.done) { 935 logger.info("All mutants are tested").collectException; 936 } 937 } 938 939 void opCall(ChecksumTestCmds data) @trusted { 940 import std.file : exists; 941 import my.hash : Checksum64, makeCrc64Iso, checksum; 942 import dextool.plugin.mutate.backend.database.type : ChecksumTestCmdOriginal; 943 944 auto previous = spinSql!(() => db.testCmdApi.original); 945 946 try { 947 Set!Checksum64 current; 948 949 foreach (testCmd; hashFiles(testCmds.filter!(a => !a.empty) 950 .map!(a => a.value[0]))) { 951 current.add(testCmd.cs); 952 953 if (testCmd.cs !in previous) 954 spinSql!(() => db.testCmdApi.set(testCmd.file, 955 ChecksumTestCmdOriginal(testCmd.cs))); 956 } 957 958 foreach (a; previous.setDifference(current).toRange) 959 spinSql!(() => db.testCmdApi.remove(ChecksumTestCmdOriginal(a))); 960 961 local.get!MutationTest.testBinaryDb.original = current; 962 } catch (Exception e) { 963 logger.warning(e.msg).collectException; 964 } 965 966 local.get!MutationTest.testBinaryDb.mutated = spinSql!( 967 () @trusted => db.testCmdApi.mutated); 968 } 969 970 void opCall(SaveMutationScore data) { 971 import dextool.plugin.mutate.backend.database.type : MutationScore; 972 import dextool.plugin.mutate.backend.report.analyzers : reportScore; 973 974 if (spinSql!(() => db.unknownSrcMutants(kinds)).count != 0) 975 return; 976 977 const score = reportScore(*db, kinds).score; 978 979 // 10000 mutation scores is only ~80kbyte. Should be enough entries 980 // without taking up unresonable amount of space. 981 spinSql!(() @trusted { 982 auto t = db.transaction; 983 db.putMutationScore(MutationScore(Clock.currTime, typeof(MutationScore.score)(score))); 984 db.trimMutationScore(10000); 985 t.commit; 986 }); 987 } 988 989 void opCall(SaveTestBinary data) { 990 if (!local.get!MutationTest.testBinaryDb.empty) 991 saveTestBinaryDb(local.get!MutationTest.testBinaryDb); 992 } 993 994 void opCall(ref PreCompileSut data) { 995 import proc; 996 997 logger.info("Checking the build command").collectException; 998 compile(conf.mutationCompile, conf.buildCmdTimeout, PrintCompileOnFailure(true)).match!( 999 (Mutation.Status a) { data.compilationError = true; }, (bool success) { 1000 data.compilationError = !success; 1001 }); 1002 } 1003 1004 void opCall(FindTestCmds data) { 1005 auto cmds = appender!(ShellCommand[])(); 1006 foreach (root; conf.testCommandDir) { 1007 try { 1008 cmds.put(findExecutables(root.AbsolutePath) 1009 .map!(a => ShellCommand([a] ~ conf.testCommandDirFlag))); 1010 } catch (Exception e) { 1011 logger.warning(e.msg).collectException; 1012 } 1013 } 1014 1015 if (!cmds.data.empty) { 1016 testCmds ~= cmds.data; 1017 runner.put(this.testCmds); 1018 logger.infof("Found test commands in %s:", conf.testCommandDir).collectException; 1019 foreach (c; cmds.data) { 1020 logger.info(c).collectException; 1021 } 1022 } 1023 } 1024 1025 void opCall(ChooseMode data) { 1026 } 1027 1028 void opCall(PullRequest data) { 1029 import std.algorithm : sort; 1030 import my.set; 1031 import dextool.plugin.mutate.backend.database : MutationStatusId; 1032 import dextool.plugin.mutate.backend.type : SourceLoc; 1033 1034 // deterministic testing of mutants and prioritized by their size. 1035 mutationOrder = MutationOrder.bySize; 1036 maxParallelInstances = 1; 1037 1038 // make sure they are unique. 1039 Set!MutationStatusId mutantIds; 1040 1041 foreach (kv; local.get!PullRequest.constraint.value.byKeyValue) { 1042 const file_id = spinSql!(() => db.getFileId(kv.key)); 1043 if (file_id.isNull) { 1044 logger.infof("The file %s do not exist in the database. Skipping...", 1045 kv.key).collectException; 1046 continue; 1047 } 1048 1049 foreach (l; kv.value) { 1050 auto mutants = spinSql!(() { 1051 return db.getMutationsOnLine(kinds, file_id.get, SourceLoc(l.value, 0)); 1052 }); 1053 1054 const preCnt = mutantIds.length; 1055 foreach (v; mutants) { 1056 mutantIds.add(v); 1057 } 1058 1059 logger.infof(mutantIds.length - preCnt > 0, "Found %s mutant(s) to test (%s:%s)", 1060 mutantIds.length - preCnt, kv.key, l.value).collectException; 1061 } 1062 } 1063 1064 logger.infof(!mutantIds.empty, "Found %s mutants in the diff", 1065 mutantIds.length).collectException; 1066 spinSql!(() { 1067 foreach (id; mutantIds.toArray.sort) 1068 db.addToWorklist(id, pullRequestWeight, MutationOrder.bySize); 1069 }); 1070 1071 local.get!CheckPullRequestMutant.startWorklistCnt = spinSql!(() => db.getWorklistCount); 1072 local.get!CheckPullRequestMutant.stopAfter = mutantIds.length; 1073 1074 if (mutantIds.empty) { 1075 logger.warning("None of the locations specified with -L exists").collectException; 1076 logger.info("Available files are:").collectException; 1077 foreach (f; spinSql!(() => db.getFiles)) 1078 logger.info(f).collectException; 1079 } 1080 } 1081 1082 void opCall(ref MeasureTestSuite data) { 1083 import std.algorithm : max, sum; 1084 import dextool.plugin.mutate.backend.database.type : TestCmdRuntime; 1085 1086 if (!conf.mutationTesterRuntime.isNull) { 1087 testSuiteRuntime = conf.mutationTesterRuntime.get; 1088 return; 1089 } 1090 1091 logger.infof("Measuring the runtime of the test command(s):\n%(%s\n%)", 1092 testCmds).collectException; 1093 1094 auto measures = spinSql!(() => db.getTestCmdRuntimes); 1095 1096 const tester = () { 1097 try { 1098 return measureTestCommand(runner, max(1, cast(int)(3 - measures.length))); 1099 } catch (Exception e) { 1100 logger.error(e.msg).collectException; 1101 return MeasureTestDurationResult(false); 1102 } 1103 }(); 1104 1105 if (tester.ok) { 1106 measures ~= tester.runtime.map!(a => TestCmdRuntime(Clock.currTime, a)).array; 1107 if (measures.length > 3) { 1108 measures = measures[1 .. $]; // drop the oldest 1109 } 1110 1111 auto mean = sum(measures.map!(a => a.runtime), Duration.zero) / measures.length; 1112 1113 // The sampling of the test suite become too unreliable when the timeout is <1s. 1114 // This is a quick and dirty fix. 1115 // A proper fix requires an update of the sampler in runTester. 1116 auto t = mean < 1.dur!"seconds" ? 1.dur!"seconds" : mean; 1117 logger.info("Test command runtime: ", t).collectException; 1118 testSuiteRuntime = t; 1119 1120 spinSql!(() @trusted { 1121 auto t = db.transaction; 1122 db.setTestCmdRuntimes(measures); 1123 t.commit; 1124 }); 1125 } else { 1126 data.unreliableTestSuite = true; 1127 logger.error("The test command is unreliable. It must return exit status '0' when no mutants are injected") 1128 .collectException; 1129 } 1130 } 1131 1132 void opCall(ref MutationTest data) { 1133 auto runnerPtr = () @trusted { return &runner; }(); 1134 auto testBinaryDbPtr = () @trusted { 1135 return &local.get!MutationTest.testBinaryDb; 1136 }(); 1137 1138 try { 1139 auto g = MutationTestDriver.Global(filesysIO, db, nextMutant, 1140 runnerPtr, testBinaryDbPtr); 1141 auto driver = MutationTestDriver(g, 1142 MutationTestDriver.TestMutantData(!(conf.mutationTestCaseAnalyze.empty 1143 && conf.mutationTestCaseBuiltin.empty), 1144 conf.mutationCompile, conf.buildCmdTimeout), 1145 MutationTestDriver.TestCaseAnalyzeData(&testCaseAnalyzer)); 1146 1147 while (driver.isRunning) { 1148 driver.execute(); 1149 } 1150 1151 if (driver.stopBecauseError) { 1152 data.mutationError.get = true; 1153 } else { 1154 data.result = driver.result; 1155 } 1156 } catch (Exception e) { 1157 data.mutationError.get = true; 1158 logger.error(e.msg).collectException; 1159 } 1160 } 1161 1162 void opCall(ref CheckTimeout data) { 1163 data.timeoutUnchanged = hardcodedTimeout || timeoutFsm.output.done; 1164 } 1165 1166 void opCall(UpdateTimeout) { 1167 spinSql!(() { timeoutFsm.execute(*db); }); 1168 1169 const lastIter = local.get!UpdateTimeout.lastTimeoutIter; 1170 1171 if (lastIter != timeoutFsm.output.iter) { 1172 logger.infof("Changed the timeout from %s to %s (iteration %s)", 1173 calculateTimeout(lastIter, testSuiteRuntime), 1174 calculateTimeout(timeoutFsm.output.iter, testSuiteRuntime), 1175 timeoutFsm.output.iter).collectException; 1176 local.get!UpdateTimeout.lastTimeoutIter = timeoutFsm.output.iter; 1177 } 1178 1179 runner.timeout = calculateTimeout(timeoutFsm.output.iter, testSuiteRuntime); 1180 } 1181 1182 void opCall(ref CheckPullRequestMutant data) { 1183 const left = spinSql!(() { return db.getWorklistCount; }); 1184 data.noUnknownMutantsLeft.get = ( 1185 local.get!CheckPullRequestMutant.startWorklistCnt - left) >= local 1186 .get!CheckPullRequestMutant.stopAfter; 1187 1188 logger.infof(stopCheck.aliveMutants > 0, "Found %s/%s alive mutants", 1189 stopCheck.aliveMutants, conf.maxAlive.get).collectException; 1190 } 1191 1192 void opCall(ref NextMutant data) { 1193 nextMutant = MutationEntry.init; 1194 1195 auto next = spinSql!(() { 1196 return db.nextMutation(kinds, maxParallelInstances); 1197 }); 1198 1199 data.noUnknownMutantsLeft.get = next.st == NextMutationEntry.Status.done; 1200 1201 if (!next.entry.isNull) 1202 nextMutant = next.entry.get; 1203 } 1204 1205 void opCall(HandleTestResult data) { 1206 saveTestResult(data.result); 1207 if (!local.get!MutationTest.testBinaryDb.empty) 1208 saveTestBinaryDb(local.get!MutationTest.testBinaryDb); 1209 } 1210 1211 void opCall(ref CheckStopCond data) { 1212 const halt = stopCheck.isHalt; 1213 data.halt = halt != TestStopCheck.HaltReason.none; 1214 1215 final switch (halt) with (TestStopCheck.HaltReason) { 1216 case none: 1217 break; 1218 case maxRuntime: 1219 logger.info(stopCheck.maxRuntimeToString).collectException; 1220 break; 1221 case aliveTested: 1222 logger.info("Alive mutants threshold reached").collectException; 1223 break; 1224 case overloaded: 1225 logger.info(stopCheck.overloadToString).collectException; 1226 break; 1227 } 1228 logger.warning(data.halt, "Halting").collectException; 1229 } 1230 1231 void opCall(ref NextSchemata data) { 1232 auto schematas = local.get!NextSchemata.schematas; 1233 1234 const threshold = schemataMutantsThreshold(schemaConf.minMutantsPerSchema.get, 1235 local.get!NextSchemata.invalidSchematas, local.get!NextSchemata.totalSchematas); 1236 1237 while (!schematas.empty && !data.hasSchema) { 1238 // TODO: replace with my.collection.vector 1239 const id = schematas[0]; 1240 schematas = schematas[1 .. $]; 1241 const mutants = spinSql!(() { 1242 return db.schemataMutantsCount(id, kinds); 1243 }); 1244 1245 logger.infof("Schema %s has %s mutants (threshold %s)", id.get, 1246 mutants, threshold).collectException; 1247 1248 if (mutants >= threshold) { 1249 data.hasSchema.get = true; 1250 data.schemataId = id; 1251 logger.infof("Use schema %s (%s/%s)", id.get, local.get!NextSchemata.totalSchematas - schematas.length, 1252 local.get!NextSchemata.totalSchematas).collectException; 1253 } 1254 } 1255 1256 local.get!NextSchemata.schematas = schematas; 1257 1258 data.stop.get = !data.hasSchema && schemaConf.stopAfterLastSchema; 1259 } 1260 1261 void opCall(ref SchemataTest data) { 1262 import dextool.plugin.mutate.backend.test_mutant.schemata; 1263 1264 // only remove schemas that are of no further use. 1265 bool remove = true; 1266 void updateRemove(MutationTestResult[] result) { 1267 foreach (a; data.result) { 1268 final switch (a.status) with (Mutation.Status) { 1269 case unknown: 1270 goto case; 1271 case equivalent: 1272 goto case; 1273 case noCoverage: 1274 goto case; 1275 case alive: 1276 remove = false; 1277 return; 1278 case killed: 1279 goto case; 1280 case timeout: 1281 goto case; 1282 case killedByCompiler: 1283 break; 1284 } 1285 } 1286 } 1287 1288 void save(MutationTestResult[] result) { 1289 updateRemove(result); 1290 saveTestResult(result); 1291 logger.infof(result.length > 0, "Saving %s schemata mutant results", 1292 result.length).collectException; 1293 } 1294 1295 try { 1296 auto driver = SchemataTestDriver(filesysIO, &runner, db, &testCaseAnalyzer, schemaConf, 1297 data.id, stopCheck, kinds, conf.mutationCompile, conf.buildCmdTimeout); 1298 1299 const saveResultInterval = 20.dur!"minutes"; 1300 auto nextSave = Clock.currTime + saveResultInterval; 1301 while (driver.isRunning) { 1302 driver.execute; 1303 1304 // to avoid loosing results in case of a crash etc save them 1305 // continuously 1306 if (Clock.currTime > nextSave && !driver.hasFatalError) { 1307 save(driver.popResult); 1308 nextSave = Clock.currTime + saveResultInterval; 1309 } 1310 } 1311 1312 data.fatalError = driver.hasFatalError; 1313 1314 if (driver.hasFatalError) { 1315 // do nothing 1316 } else if (driver.isInvalidSchema) { 1317 local.get!NextSchemata.invalidSchematas++; 1318 } else { 1319 save(driver.popResult); 1320 } 1321 1322 if (remove) { 1323 spinSql!(() { db.markUsed(data.id); }); 1324 } 1325 1326 } catch (Exception e) { 1327 logger.info(e.msg).collectException; 1328 logger.warning("Failed executing schemata ", data.id).collectException; 1329 } 1330 } 1331 1332 void opCall(SchemataPruneUsed data) { 1333 try { 1334 const removed = db.pruneUsedSchemas; 1335 1336 if (removed != 0) { 1337 logger.infof("Removed %s schemas from the database", removed); 1338 // vacuum the database because schemas take up a significant 1339 // amount of space. 1340 db.vacuum; 1341 } 1342 } catch (Exception e) { 1343 logger.warning(e.msg).collectException; 1344 } 1345 } 1346 1347 void opCall(LoadSchematas data) { 1348 if (!schemaConf.use) { 1349 return; 1350 } 1351 1352 auto app = appender!(SchemataId[])(); 1353 foreach (id; spinSql!(() { return db.getSchematas(); })) { 1354 if (spinSql!(() { return db.schemataMutantsCount(id, kinds); }) >= schemataMutantsThreshold( 1355 schemaConf.minMutantsPerSchema.get, 0, 0)) { 1356 app.put(id); 1357 } 1358 } 1359 1360 logger.trace("Found schematas: ", app.data).collectException; 1361 // random reorder to reduce the chance that multipe instances of 1362 // dextool use the same schema 1363 local.get!NextSchemata.schematas = app.data.randomCover.array; 1364 local.get!NextSchemata.totalSchematas = app.data.length; 1365 } 1366 1367 void opCall(ref Coverage data) { 1368 import dextool.plugin.mutate.backend.test_mutant.coverage; 1369 1370 auto tracked = spinSql!(() => db.getLatestTimeStampOfTestOrSut).orElse(SysTime.init); 1371 auto covTimeStamp = spinSql!(() => db.getCoverageTimeStamp).orElse(SysTime.init); 1372 1373 if (tracked < covTimeStamp) { 1374 logger.info("Coverage information is up to date").collectException; 1375 return; 1376 } else { 1377 logger.infof("Coverage is out of date with SUT/tests (%s < %s)", 1378 covTimeStamp, tracked).collectException; 1379 } 1380 1381 try { 1382 auto driver = CoverageDriver(filesysIO, db, &runner, covConf, 1383 conf.mutationCompile, conf.buildCmdTimeout); 1384 while (driver.isRunning) { 1385 driver.execute; 1386 } 1387 data.propagate = true; 1388 data.fatalError = driver.hasFatalError; 1389 } catch (Exception e) { 1390 logger.warning(e.msg).collectException; 1391 data.fatalError = true; 1392 } 1393 1394 if (data.fatalError) 1395 logger.warning("Error detected when trying to gather coverage information") 1396 .collectException; 1397 } 1398 1399 void opCall(PropagateCoverage data) { 1400 void propagate() @trusted { 1401 auto trans = db.transaction; 1402 1403 auto noCov = db.getNotCoveredMutants; 1404 foreach (id; noCov) { 1405 db.updateMutationStatus(id, Mutation.Status.noCoverage, ExitStatus(0)); 1406 db.removeFromWorklist(id); 1407 } 1408 1409 trans.commit; 1410 logger.infof("Marked %s mutants as alive because they where not covered by any test", 1411 noCov.length); 1412 } 1413 1414 spinSql!(() => propagate); 1415 } 1416 1417 void saveTestResult(MutationTestResult[] results) @safe nothrow { 1418 void statusUpdate(MutationTestResult result) @safe { 1419 import dextool.plugin.mutate.backend.test_mutant.timeout : updateMutantStatus; 1420 1421 estimate.update(result.status); 1422 1423 updateMutantStatus(*db, result.id, result.status, 1424 result.exitStatus, timeoutFsm.output.iter); 1425 db.updateMutation(result.id, result.profile); 1426 db.updateMutationTestCases(result.id, result.testCases); 1427 db.removeFromWorklist(result.id); 1428 1429 if (result.status == Mutation.Status.alive) 1430 stopCheck.incrAliveMutants; 1431 } 1432 1433 const left = spinSql!(() @trusted { 1434 auto t = db.transaction; 1435 foreach (a; results) { 1436 statusUpdate(a); 1437 } 1438 1439 const left = db.getWorklistCount; 1440 t.commit; 1441 return left; 1442 }); 1443 1444 logger.infof("%s mutants left to test. Estimated mutation score %.3s (error %.3s)", 1445 left, estimate.value.get, estimate.error.get).collectException; 1446 } 1447 1448 void saveTestBinaryDb(ref TestBinaryDb testBinaryDb) @safe nothrow { 1449 import dextool.plugin.mutate.backend.database.type : ChecksumTestCmdMutated; 1450 1451 spinSql!(() @trusted { 1452 auto t = db.transaction; 1453 foreach (a; testBinaryDb.added.byKeyValue) { 1454 db.testCmdApi.add(ChecksumTestCmdMutated(a.key), a.value); 1455 } 1456 // magic number. about 10 Mbyte in the database (8+8+8)*20000 1457 db.testCmdApi.trimMutated(200000); 1458 t.commit; 1459 }); 1460 1461 testBinaryDb.clearAdded; 1462 } 1463 } 1464 1465 private: 1466 1467 /** A schemata must have at least this many mutants that have the status unknown 1468 * for it to be cost efficient to use schemata. 1469 * 1470 * The weights dynamically adjust with how many of the schemas that has failed 1471 * to compile. 1472 * 1473 * Params: 1474 * checkSchemata = if the user has activated check_schemata that run all test cases before the schemata is used. 1475 */ 1476 long schemataMutantsThreshold(const long minThreshold, const long invalidSchematas, 1477 const long totalSchematas) @safe pure nothrow @nogc { 1478 // "10" is a magic number that felt good but not too conservative. A future 1479 // improvement is to instead base it on the ratio between compilation time 1480 // and test suite execution time. 1481 if (totalSchematas > 0) 1482 return cast(long)(minThreshold + 10.0 * ( 1483 cast(double) invalidSchematas / cast(double) totalSchematas)); 1484 return cast(long) minThreshold; 1485 } 1486 1487 /** Compare the old test cases with those that have been found this run. 1488 * 1489 * TODO: the side effect that this function print to the console is NOT good. 1490 */ 1491 bool hasNewTestCases(ref Set!string old_tcs, ref Set!string found_tcs) @safe nothrow { 1492 bool rval; 1493 1494 auto new_tcs = found_tcs.setDifference(old_tcs); 1495 foreach (tc; new_tcs.toRange) { 1496 logger.info(!rval, "Found new test case(s):").collectException; 1497 logger.infof("%s", tc).collectException; 1498 rval = true; 1499 } 1500 1501 return rval; 1502 } 1503 1504 /** Compare old and new test cases to print those that have been removed. 1505 */ 1506 void printDroppedTestCases(ref Set!string old_tcs, ref Set!string changed_tcs) @safe nothrow { 1507 auto diff = old_tcs.setDifference(changed_tcs); 1508 auto removed = diff.toArray; 1509 1510 logger.info(removed.length != 0, "Detected test cases that has been removed:").collectException; 1511 foreach (tc; removed) { 1512 logger.infof("%s", tc).collectException; 1513 } 1514 } 1515 1516 /// Returns: true if all tests cases have unique identifiers 1517 void warnIfConflictingTestCaseIdentifiers(TestCase[] found_tcs) @safe nothrow { 1518 Set!TestCase checked; 1519 bool conflict; 1520 1521 foreach (tc; found_tcs) { 1522 if (checked.contains(tc)) { 1523 logger.info(!conflict, 1524 "Found test cases that do not have global, unique identifiers") 1525 .collectException; 1526 logger.info(!conflict, 1527 "This make the report of test cases that has killed zero mutants unreliable") 1528 .collectException; 1529 logger.info("%s", tc).collectException; 1530 conflict = true; 1531 } 1532 } 1533 } 1534 1535 private: 1536 1537 /** 1538 DESCRIPTION 1539 1540 The getloadavg() function returns the number of processes in the system 1541 run queue averaged over various periods of time. Up to nelem samples are 1542 retrieved and assigned to successive elements of loadavg[]. The system 1543 imposes a maximum of 3 samples, representing averages over the last 1, 5, 1544 and 15 minutes, respectively. 1545 1546 1547 DIAGNOSTICS 1548 1549 If the load average was unobtainable, -1 is returned; otherwise, the num- 1550 ber of samples actually retrieved is returned. 1551 */ 1552 extern (C) int getloadavg(double* loadavg, int nelem) nothrow;