1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 Test a mutant by modifying the source code. 11 */ 12 module dextool.plugin.mutate.backend.test_mutant.source_mutant; 13 14 import core.time : Duration; 15 import logger = std.experimental.logger; 16 import std.array : empty; 17 import std.exception : collectException; 18 import std.path : buildPath; 19 20 import sumtype; 21 import proc : DrainElement; 22 import my.fsm : Fsm, next, act, get, TypeDataMap; 23 24 static import my.fsm; 25 26 import dextool.plugin.mutate.backend.database : Database, MutationEntry; 27 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 28 import dextool.plugin.mutate.backend.test_mutant.common; 29 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 30 import dextool.plugin.mutate.backend.type : Mutation, TestCase; 31 import dextool.plugin.mutate.config; 32 import dextool.plugin.mutate.type : ShellCommand; 33 import dextool.type : AbsolutePath, Path; 34 35 @safe: 36 37 /// The result of testing a mutant. 38 struct MutationTestResult { 39 import std.datetime : Duration; 40 import sumtype; 41 import proc : DrainElement; 42 import dextool.plugin.mutate.backend.database : MutationId; 43 import dextool.plugin.mutate.backend.type : TestCase; 44 45 static struct NoResult { 46 } 47 48 static struct StatusUpdate { 49 MutationId id; 50 Mutation.Status status; 51 Duration testTime; 52 TestCase[] testCases; 53 } 54 55 alias Value = SumType!(NoResult, StatusUpdate); 56 Value value; 57 58 void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc { 59 this.value = rhs.value; 60 } 61 62 void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc { 63 this.value = Value(rhs); 64 } 65 } 66 67 /** Drive the control flow when testing **a** mutant. 68 */ 69 struct MutationTestDriver { 70 import std.datetime.stopwatch : StopWatch; 71 import std.typecons : Tuple; 72 import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase; 73 74 static struct Global { 75 FilesysIO fio; 76 Database* db; 77 78 /// The mutant to apply. 79 MutationEntry mutp; 80 81 /// Runs the test commands. 82 TestRunner* runner; 83 84 /// File to mutate. 85 AbsolutePath mut_file; 86 87 /// The original file. 88 Blob original; 89 90 /// The result of running the test cases. 91 Mutation.Status mut_status; 92 93 /// Test cases that killed the mutant. 94 TestCase[] test_cases; 95 96 /// How long it took to do the mutation testing. 97 StopWatch sw; 98 } 99 100 static struct None { 101 } 102 103 static struct Initialize { 104 } 105 106 static struct MutateCode { 107 bool next; 108 bool filesysError; 109 bool mutationError; 110 } 111 112 static struct TestMutantData { 113 /// If the user has configured that the test cases should be analyzed. 114 bool hasTestCaseOutputAnalyzer; 115 ShellCommand buildCmd; 116 Duration buildCmdTimeout; 117 } 118 119 static struct TestMutant { 120 bool hasTestOutput; 121 } 122 123 static struct RestoreCode { 124 bool next; 125 bool filesysError; 126 } 127 128 static struct TestCaseAnalyzeData { 129 TestCaseAnalyzer* testCaseAnalyzer; 130 DrainElement[] output; 131 } 132 133 static struct TestCaseAnalyze { 134 bool unstableTests; 135 } 136 137 static struct StoreResult { 138 } 139 140 static struct Done { 141 } 142 143 static struct FilesysError { 144 } 145 146 // happens when an error occurs during mutations testing but that do not 147 // prohibit testing of other mutants 148 static struct NoResultRestoreCode { 149 } 150 151 static struct NoResult { 152 } 153 154 alias Fsm = my.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode, 155 TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult); 156 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData); 157 158 private { 159 Fsm fsm; 160 Global global; 161 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local; 162 bool isRunning_ = true; 163 bool stopBecauseError_; 164 } 165 166 MutationTestResult result; 167 168 this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) { 169 this.global = global; 170 this.local = LocalStateDataT(l1, l2); 171 } 172 173 static void execute_(ref MutationTestDriver self) @trusted { 174 self.fsm.next!((None a) => fsm(Initialize.init), 175 (Initialize a) => fsm(MutateCode.init), (MutateCode a) { 176 if (a.next) 177 return fsm(TestMutant.init); 178 else if (a.filesysError) 179 return fsm(FilesysError.init); 180 else if (a.mutationError) 181 return fsm(NoResultRestoreCode.init); 182 return fsm(a); 183 }, (TestMutant a) { 184 if (self.global.mut_status == Mutation.Status.killed 185 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) { 186 return fsm(TestCaseAnalyze.init); 187 } 188 return fsm(RestoreCode.init); 189 }, (TestCaseAnalyze a) { 190 if (a.unstableTests) 191 return fsm(NoResultRestoreCode.init); 192 return fsm(RestoreCode.init); 193 }, (RestoreCode a) { 194 if (a.next) 195 return fsm(StoreResult.init); 196 else if (a.filesysError) 197 return fsm(FilesysError.init); 198 return fsm(a); 199 }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a), 200 (FilesysError a) => fsm(a), 201 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),); 202 203 debug logger.trace("state: ", self.fsm.logNext); 204 self.fsm.act!(self); 205 } 206 207 nothrow: 208 209 void execute() { 210 try { 211 execute_(this); 212 } catch (Exception e) { 213 logger.warning(e.msg).collectException; 214 } 215 } 216 217 /// Returns: true as long as the driver is processing a mutant. 218 bool isRunning() { 219 return isRunning_; 220 } 221 222 bool stopBecauseError() { 223 return stopBecauseError_; 224 } 225 226 void opCall(None data) { 227 } 228 229 void opCall(Initialize data) { 230 global.sw.start; 231 } 232 233 void opCall(Done data) { 234 isRunning_ = false; 235 } 236 237 void opCall(FilesysError data) { 238 logger.warning("Filesystem error").collectException; 239 isRunning_ = false; 240 stopBecauseError_ = true; 241 } 242 243 void opCall(NoResultRestoreCode data) { 244 RestoreCode tmp; 245 this.opCall(tmp); 246 } 247 248 void opCall(NoResult data) { 249 isRunning_ = false; 250 } 251 252 void opCall(ref MutateCode data) { 253 import dextool.plugin.mutate.backend.generate_mutant : generateMutant, 254 GenerateMutantResult, GenerateMutantStatus; 255 256 try { 257 global.mut_file = AbsolutePath(buildPath(global.fio.getOutputDir, global.mutp.file)); 258 global.original = global.fio.makeInput(global.mut_file); 259 } catch (Exception e) { 260 logger.error(e.msg).collectException; 261 logger.warning("Unable to read ", global.mut_file).collectException; 262 data.filesysError = true; 263 return; 264 } 265 266 // mutate 267 try { 268 auto fout = global.fio.makeOutput(global.mut_file); 269 auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout); 270 271 final switch (mut_res.status) with (GenerateMutantStatus) { 272 case error: 273 data.mutationError = true; 274 break; 275 case filesysError: 276 data.filesysError = true; 277 break; 278 case databaseError: 279 // such as when the database is locked 280 data.mutationError = true; 281 break; 282 case checksumError: 283 data.filesysError = true; 284 break; 285 case noMutation: 286 data.mutationError = true; 287 break; 288 case ok: 289 data.next = true; 290 try { 291 logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id, 292 cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to, 293 global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column); 294 295 } catch (Exception e) { 296 logger.warning("Mutation ID", e.msg); 297 } 298 break; 299 } 300 } catch (Exception e) { 301 logger.warning(e.msg).collectException; 302 data.mutationError = true; 303 } 304 } 305 306 void opCall(ref TestMutant data) { 307 global.mut_status = Mutation.Status.unknown; 308 309 bool successCompile; 310 compile(local.get!TestMutant.buildCmd, local.get!TestMutant.buildCmdTimeout).match!( 311 (Mutation.Status a) { global.mut_status = a; }, (bool success) { 312 successCompile = success; 313 },); 314 315 if (!successCompile) 316 return; 317 318 auto res = runTester(*global.runner); 319 global.mut_status = res.status; 320 data.hasTestOutput = !res.output.empty; 321 local.get!TestCaseAnalyze.output = res.output; 322 } 323 324 void opCall(ref TestCaseAnalyze data) { 325 global.test_cases = null; 326 327 try { 328 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze( 329 local.get!TestCaseAnalyze.output); 330 local.get!TestCaseAnalyze.output = null; 331 332 analyze.match!((TestCaseAnalyzer.Success a) { 333 global.test_cases = a.failed; 334 }, (TestCaseAnalyzer.Unstable a) { 335 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 336 logger.info( 337 "As configured the result is ignored which will force the mutant to be re-tested"); 338 data.unstableTests = true; 339 }, (TestCaseAnalyzer.Failed a) { 340 logger.warning("The parser that analyze the output from test case(s) failed"); 341 }); 342 } catch (Exception e) { 343 logger.warning(e.msg).collectException; 344 } 345 } 346 347 void opCall(StoreResult data) { 348 global.sw.stop; 349 result = MutationTestResult.StatusUpdate(global.mutp.id, 350 global.mut_status, global.sw.peek, global.test_cases); 351 } 352 353 void opCall(ref RestoreCode data) { 354 // restore the original file. 355 try { 356 global.fio.makeOutput(global.mut_file).write(global.original.content); 357 } catch (Exception e) { 358 logger.error(e.msg).collectException; 359 // fatal error because being unable to restore a file prohibit 360 // future mutations. 361 data.filesysError = true; 362 return; 363 } 364 365 data.next = true; 366 } 367 }