1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 Test a mutant by modifying the source code. 11 */ 12 module dextool.plugin.mutate.backend.test_mutant.source_mutant; 13 14 import logger = std.experimental.logger; 15 import std.array : empty; 16 import std.exception : collectException; 17 18 import sumtype; 19 import process : DrainElement; 20 21 import dextool.fsm : Fsm, next, act, get, TypeDataMap; 22 import dextool.plugin.mutate.backend.database : Database, MutationEntry; 23 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 24 import dextool.plugin.mutate.backend.test_mutant.common; 25 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 26 import dextool.plugin.mutate.backend.type : Mutation, TestCase; 27 import dextool.plugin.mutate.config; 28 import dextool.plugin.mutate.type : ShellCommand; 29 import dextool.set; 30 import dextool.type : AbsolutePath, Path; 31 32 @safe: 33 34 /// The result of testing a mutant. 35 struct MutationTestResult { 36 import std.datetime : Duration; 37 import sumtype; 38 import process : DrainElement; 39 import dextool.plugin.mutate.backend.database : MutationId; 40 import dextool.plugin.mutate.backend.type : TestCase; 41 42 static struct NoResult { 43 } 44 45 static struct StatusUpdate { 46 MutationId id; 47 Mutation.Status status; 48 Duration testTime; 49 TestCase[] testCases; 50 DrainElement[] output; 51 } 52 53 alias Value = SumType!(NoResult, StatusUpdate); 54 Value value; 55 56 void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc { 57 this.value = rhs.value; 58 } 59 60 void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc { 61 this.value = Value(rhs); 62 } 63 } 64 65 /** Drive the control flow when testing **a** mutant. 66 */ 67 struct MutationTestDriver { 68 import std.datetime.stopwatch : StopWatch; 69 import std.typecons : Tuple; 70 import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase; 71 72 static struct Global { 73 FilesysIO fio; 74 Database* db; 75 76 /// The mutant to apply. 77 MutationEntry mutp; 78 79 /// Runs the test commands. 80 TestRunner* runner; 81 82 /// File to mutate. 83 AbsolutePath mut_file; 84 85 /// The original file. 86 Blob original; 87 88 /// The result of running the test cases. 89 Mutation.Status mut_status; 90 91 /// Test cases that killed the mutant. 92 TestCase[] test_cases; 93 94 /// How long it took to do the mutation testing. 95 StopWatch sw; 96 } 97 98 static struct None { 99 } 100 101 static struct Initialize { 102 } 103 104 static struct MutateCode { 105 bool next; 106 bool filesysError; 107 bool mutationError; 108 } 109 110 static struct TestMutantData { 111 /// If the user has configured that the test cases should be analyzed. 112 bool hasTestCaseOutputAnalyzer; 113 ShellCommand compile_cmd; 114 } 115 116 static struct TestMutant { 117 DrainElement[] output; 118 } 119 120 static struct RestoreCode { 121 bool next; 122 bool filesysError; 123 } 124 125 static struct TestCaseAnalyzeData { 126 TestCaseAnalyzer* testCaseAnalyzer; 127 } 128 129 static struct TestCaseAnalyze { 130 DrainElement[] output; 131 bool unstableTests; 132 } 133 134 static struct StoreResult { 135 } 136 137 static struct Done { 138 } 139 140 static struct FilesysError { 141 } 142 143 // happens when an error occurs during mutations testing but that do not 144 // prohibit testing of other mutants 145 static struct NoResultRestoreCode { 146 } 147 148 static struct NoResult { 149 } 150 151 alias Fsm = dextool.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode, 152 TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult); 153 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData); 154 155 private { 156 Fsm fsm; 157 Global global; 158 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local; 159 bool isRunning_ = true; 160 bool stopBecauseError_; 161 } 162 163 MutationTestResult result; 164 165 this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) { 166 this.global = global; 167 this.local = LocalStateDataT(l1, l2); 168 } 169 170 static void execute_(ref MutationTestDriver self) @trusted { 171 self.fsm.next!((None a) => fsm(Initialize.init), 172 (Initialize a) => fsm(MutateCode.init), (MutateCode a) { 173 if (a.next) 174 return fsm(TestMutant.init); 175 else if (a.filesysError) 176 return fsm(FilesysError.init); 177 else if (a.mutationError) 178 return fsm(NoResultRestoreCode.init); 179 return fsm(a); 180 }, (TestMutant a) { 181 if (self.global.mut_status == Mutation.Status.killed 182 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && !a.output.empty) 183 return fsm(TestCaseAnalyze(a.output)); 184 return fsm(RestoreCode.init); 185 }, (TestCaseAnalyze a) { 186 if (a.unstableTests) 187 return fsm(NoResultRestoreCode.init); 188 return fsm(RestoreCode.init); 189 }, (RestoreCode a) { 190 if (a.next) 191 return fsm(StoreResult.init); 192 else if (a.filesysError) 193 return fsm(FilesysError.init); 194 return fsm(a); 195 }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a), 196 (FilesysError a) => fsm(a), 197 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),); 198 199 self.fsm.act!(self); 200 } 201 202 nothrow: 203 204 void execute() { 205 try { 206 execute_(this); 207 } catch (Exception e) { 208 logger.warning(e.msg).collectException; 209 } 210 } 211 212 /// Returns: true as long as the driver is processing a mutant. 213 bool isRunning() { 214 return isRunning_; 215 } 216 217 bool stopBecauseError() { 218 return stopBecauseError_; 219 } 220 221 void opCall(None data) { 222 } 223 224 void opCall(Initialize data) { 225 global.sw.start; 226 } 227 228 void opCall(Done data) { 229 isRunning_ = false; 230 } 231 232 void opCall(FilesysError data) { 233 logger.warning("Filesystem error").collectException; 234 isRunning_ = false; 235 stopBecauseError_ = true; 236 } 237 238 void opCall(NoResultRestoreCode data) { 239 RestoreCode tmp; 240 this.opCall(tmp); 241 } 242 243 void opCall(NoResult data) { 244 isRunning_ = false; 245 } 246 247 void opCall(ref MutateCode data) { 248 import dextool.plugin.mutate.backend.generate_mutant : generateMutant, 249 GenerateMutantResult, GenerateMutantStatus; 250 251 try { 252 global.mut_file = AbsolutePath(Path(global.mutp.file), global.fio.getOutputDir); 253 global.original = global.fio.makeInput(global.mut_file); 254 } catch (Exception e) { 255 logger.error(e.msg).collectException; 256 logger.warning("Unable to read ", global.mut_file).collectException; 257 data.filesysError = true; 258 return; 259 } 260 261 // mutate 262 try { 263 auto fout = global.fio.makeOutput(global.mut_file); 264 auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout); 265 266 final switch (mut_res.status) with (GenerateMutantStatus) { 267 case error: 268 data.mutationError = true; 269 break; 270 case filesysError: 271 data.filesysError = true; 272 break; 273 case databaseError: 274 // such as when the database is locked 275 data.mutationError = true; 276 break; 277 case checksumError: 278 data.filesysError = true; 279 break; 280 case noMutation: 281 data.mutationError = true; 282 break; 283 case ok: 284 data.next = true; 285 try { 286 logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id, 287 cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to, 288 global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column); 289 290 } catch (Exception e) { 291 logger.warning("Mutation ID", e.msg); 292 } 293 break; 294 } 295 } catch (Exception e) { 296 logger.warning(e.msg).collectException; 297 data.mutationError = true; 298 } 299 } 300 301 void opCall(ref TestMutant data) { 302 global.mut_status = Mutation.Status.unknown; 303 304 bool successCompile; 305 compile(local.get!TestMutant.compile_cmd).match!((Mutation.Status a) { 306 global.mut_status = a; 307 }, (bool success) { successCompile = success; },); 308 309 if (!successCompile) 310 return; 311 312 auto res = runTester(*global.runner); 313 global.mut_status = res.status; 314 data.output = res.output; 315 } 316 317 void opCall(ref TestCaseAnalyze data) { 318 global.test_cases = null; 319 320 try { 321 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(data.output); 322 323 analyze.match!((TestCaseAnalyzer.Success a) { 324 global.test_cases = a.failed; 325 }, (TestCaseAnalyzer.Unstable a) { 326 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 327 logger.info( 328 "As configured the result is ignored which will force the mutant to be re-tested"); 329 data.unstableTests = true; 330 }, (TestCaseAnalyzer.Failed a) { 331 logger.warning("The parser that analyze the output from test case(s) failed"); 332 }); 333 } catch (Exception e) { 334 logger.warning(e.msg).collectException; 335 } 336 } 337 338 void opCall(StoreResult data) { 339 global.sw.stop; 340 result = MutationTestResult.StatusUpdate(global.mutp.id, 341 global.mut_status, global.sw.peek, global.test_cases); 342 } 343 344 void opCall(ref RestoreCode data) { 345 // restore the original file. 346 try { 347 global.fio.makeOutput(global.mut_file).write(global.original.content); 348 } catch (Exception e) { 349 logger.error(e.msg).collectException; 350 // fatal error because being unable to restore a file prohibit 351 // future mutations. 352 data.filesysError = true; 353 return; 354 } 355 356 data.next = true; 357 } 358 }