1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 Test a mutant by modifying the source code. 11 */ 12 module dextool.plugin.mutate.backend.test_mutant.source_mutant; 13 14 import logger = std.experimental.logger; 15 import std.array : empty; 16 import std.exception : collectException; 17 18 import sumtype; 19 import proc : DrainElement; 20 21 import dextool.fsm : Fsm, next, act, get, TypeDataMap; 22 import dextool.plugin.mutate.backend.database : Database, MutationEntry; 23 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 24 import dextool.plugin.mutate.backend.test_mutant.common; 25 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 26 import dextool.plugin.mutate.backend.type : Mutation, TestCase; 27 import dextool.plugin.mutate.config; 28 import dextool.plugin.mutate.type : ShellCommand; 29 import dextool.set; 30 import dextool.type : AbsolutePath, Path; 31 32 @safe: 33 34 /// The result of testing a mutant. 35 struct MutationTestResult { 36 import std.datetime : Duration; 37 import sumtype; 38 import proc : DrainElement; 39 import dextool.plugin.mutate.backend.database : MutationId; 40 import dextool.plugin.mutate.backend.type : TestCase; 41 42 static struct NoResult { 43 } 44 45 static struct StatusUpdate { 46 MutationId id; 47 Mutation.Status status; 48 Duration testTime; 49 TestCase[] testCases; 50 } 51 52 alias Value = SumType!(NoResult, StatusUpdate); 53 Value value; 54 55 void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc { 56 this.value = rhs.value; 57 } 58 59 void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc { 60 this.value = Value(rhs); 61 } 62 } 63 64 /** Drive the control flow when testing **a** mutant. 65 */ 66 struct MutationTestDriver { 67 import std.datetime.stopwatch : StopWatch; 68 import std.typecons : Tuple; 69 import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase; 70 71 static struct Global { 72 FilesysIO fio; 73 Database* db; 74 75 /// The mutant to apply. 76 MutationEntry mutp; 77 78 /// Runs the test commands. 79 TestRunner* runner; 80 81 /// File to mutate. 82 AbsolutePath mut_file; 83 84 /// The original file. 85 Blob original; 86 87 /// The result of running the test cases. 88 Mutation.Status mut_status; 89 90 /// Test cases that killed the mutant. 91 TestCase[] test_cases; 92 93 /// How long it took to do the mutation testing. 94 StopWatch sw; 95 } 96 97 static struct None { 98 } 99 100 static struct Initialize { 101 } 102 103 static struct MutateCode { 104 bool next; 105 bool filesysError; 106 bool mutationError; 107 } 108 109 static struct TestMutantData { 110 /// If the user has configured that the test cases should be analyzed. 111 bool hasTestCaseOutputAnalyzer; 112 ShellCommand compile_cmd; 113 } 114 115 static struct TestMutant { 116 bool hasTestOutput; 117 } 118 119 static struct RestoreCode { 120 bool next; 121 bool filesysError; 122 } 123 124 static struct TestCaseAnalyzeData { 125 TestCaseAnalyzer* testCaseAnalyzer; 126 DrainElement[] output; 127 } 128 129 static struct TestCaseAnalyze { 130 bool unstableTests; 131 } 132 133 static struct StoreResult { 134 } 135 136 static struct Done { 137 } 138 139 static struct FilesysError { 140 } 141 142 // happens when an error occurs during mutations testing but that do not 143 // prohibit testing of other mutants 144 static struct NoResultRestoreCode { 145 } 146 147 static struct NoResult { 148 } 149 150 alias Fsm = dextool.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode, 151 TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult); 152 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData); 153 154 private { 155 Fsm fsm; 156 Global global; 157 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local; 158 bool isRunning_ = true; 159 bool stopBecauseError_; 160 } 161 162 MutationTestResult result; 163 164 this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) { 165 this.global = global; 166 this.local = LocalStateDataT(l1, l2); 167 } 168 169 static void execute_(ref MutationTestDriver self) @trusted { 170 self.fsm.next!((None a) => fsm(Initialize.init), 171 (Initialize a) => fsm(MutateCode.init), (MutateCode a) { 172 if (a.next) 173 return fsm(TestMutant.init); 174 else if (a.filesysError) 175 return fsm(FilesysError.init); 176 else if (a.mutationError) 177 return fsm(NoResultRestoreCode.init); 178 return fsm(a); 179 }, (TestMutant a) { 180 if (self.global.mut_status == Mutation.Status.killed 181 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) { 182 return fsm(TestCaseAnalyze.init); 183 } 184 return fsm(RestoreCode.init); 185 }, (TestCaseAnalyze a) { 186 if (a.unstableTests) 187 return fsm(NoResultRestoreCode.init); 188 return fsm(RestoreCode.init); 189 }, (RestoreCode a) { 190 if (a.next) 191 return fsm(StoreResult.init); 192 else if (a.filesysError) 193 return fsm(FilesysError.init); 194 return fsm(a); 195 }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a), 196 (FilesysError a) => fsm(a), 197 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),); 198 199 debug logger.trace("state: ", self.fsm.logNext); 200 self.fsm.act!(self); 201 } 202 203 nothrow: 204 205 void execute() { 206 try { 207 execute_(this); 208 } catch (Exception e) { 209 logger.warning(e.msg).collectException; 210 } 211 } 212 213 /// Returns: true as long as the driver is processing a mutant. 214 bool isRunning() { 215 return isRunning_; 216 } 217 218 bool stopBecauseError() { 219 return stopBecauseError_; 220 } 221 222 void opCall(None data) { 223 } 224 225 void opCall(Initialize data) { 226 global.sw.start; 227 } 228 229 void opCall(Done data) { 230 isRunning_ = false; 231 } 232 233 void opCall(FilesysError data) { 234 logger.warning("Filesystem error").collectException; 235 isRunning_ = false; 236 stopBecauseError_ = true; 237 } 238 239 void opCall(NoResultRestoreCode data) { 240 RestoreCode tmp; 241 this.opCall(tmp); 242 } 243 244 void opCall(NoResult data) { 245 isRunning_ = false; 246 } 247 248 void opCall(ref MutateCode data) { 249 import dextool.plugin.mutate.backend.generate_mutant : generateMutant, 250 GenerateMutantResult, GenerateMutantStatus; 251 252 try { 253 global.mut_file = AbsolutePath(Path(global.mutp.file), global.fio.getOutputDir); 254 global.original = global.fio.makeInput(global.mut_file); 255 } catch (Exception e) { 256 logger.error(e.msg).collectException; 257 logger.warning("Unable to read ", global.mut_file).collectException; 258 data.filesysError = true; 259 return; 260 } 261 262 // mutate 263 try { 264 auto fout = global.fio.makeOutput(global.mut_file); 265 auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout); 266 267 final switch (mut_res.status) with (GenerateMutantStatus) { 268 case error: 269 data.mutationError = true; 270 break; 271 case filesysError: 272 data.filesysError = true; 273 break; 274 case databaseError: 275 // such as when the database is locked 276 data.mutationError = true; 277 break; 278 case checksumError: 279 data.filesysError = true; 280 break; 281 case noMutation: 282 data.mutationError = true; 283 break; 284 case ok: 285 data.next = true; 286 try { 287 logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id, 288 cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to, 289 global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column); 290 291 } catch (Exception e) { 292 logger.warning("Mutation ID", e.msg); 293 } 294 break; 295 } 296 } catch (Exception e) { 297 logger.warning(e.msg).collectException; 298 data.mutationError = true; 299 } 300 } 301 302 void opCall(ref TestMutant data) { 303 global.mut_status = Mutation.Status.unknown; 304 305 bool successCompile; 306 compile(local.get!TestMutant.compile_cmd).match!((Mutation.Status a) { 307 global.mut_status = a; 308 }, (bool success) { successCompile = success; },); 309 310 if (!successCompile) 311 return; 312 313 auto res = runTester(*global.runner); 314 global.mut_status = res.status; 315 data.hasTestOutput = !res.output.empty; 316 local.get!TestCaseAnalyze.output = res.output; 317 } 318 319 void opCall(ref TestCaseAnalyze data) { 320 global.test_cases = null; 321 322 try { 323 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze( 324 local.get!TestCaseAnalyze.output); 325 local.get!TestCaseAnalyze.output = null; 326 327 analyze.match!((TestCaseAnalyzer.Success a) { 328 global.test_cases = a.failed; 329 }, (TestCaseAnalyzer.Unstable a) { 330 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 331 logger.info( 332 "As configured the result is ignored which will force the mutant to be re-tested"); 333 data.unstableTests = true; 334 }, (TestCaseAnalyzer.Failed a) { 335 logger.warning("The parser that analyze the output from test case(s) failed"); 336 }); 337 } catch (Exception e) { 338 logger.warning(e.msg).collectException; 339 } 340 } 341 342 void opCall(StoreResult data) { 343 global.sw.stop; 344 result = MutationTestResult.StatusUpdate(global.mutp.id, 345 global.mut_status, global.sw.peek, global.test_cases); 346 } 347 348 void opCall(ref RestoreCode data) { 349 // restore the original file. 350 try { 351 global.fio.makeOutput(global.mut_file).write(global.original.content); 352 } catch (Exception e) { 353 logger.error(e.msg).collectException; 354 // fatal error because being unable to restore a file prohibit 355 // future mutations. 356 data.filesysError = true; 357 return; 358 } 359 360 data.next = true; 361 } 362 }