1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 Test a mutant by modifying the source code. 11 */ 12 module dextool.plugin.mutate.backend.test_mutant.source_mutant; 13 14 import core.time : Duration; 15 import logger = std.experimental.logger; 16 import std.algorithm : sort, map; 17 import std.array : empty, array; 18 import std.exception : collectException; 19 import std.path : buildPath; 20 21 import my.fsm : next, act, get, TypeDataMap; 22 import my.named_type; 23 import proc : DrainElement; 24 import sumtype; 25 26 static import my.fsm; 27 28 import dextool.plugin.mutate.backend.database : Database, MutationEntry; 29 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 30 import dextool.plugin.mutate.backend.test_mutant.common; 31 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner; 32 import dextool.plugin.mutate.backend.type : Mutation, TestCase; 33 import dextool.plugin.mutate.config; 34 import dextool.plugin.mutate.type : ShellCommand; 35 import dextool.type : AbsolutePath, Path; 36 37 @safe: 38 39 /** Drive the control flow when testing **a** mutant. 40 */ 41 struct MutationTestDriver { 42 import std.datetime.stopwatch : StopWatch; 43 import std.typecons : Tuple; 44 45 static struct Global { 46 FilesysIO fio; 47 Database* db; 48 49 /// The mutant to apply. 50 MutationEntry mutp; 51 52 /// Runs the test commands. 53 TestRunner* runner; 54 55 /// File to mutate. 56 AbsolutePath mutateFile; 57 58 /// The original file. 59 Blob original; 60 61 /// The result of running the test cases. 62 TestResult testResult; 63 64 /// Test cases that killed the mutant. 65 TestCase[] testCases; 66 67 /// How long it took to do the mutation testing. 68 StopWatch swCompile; 69 StopWatch swTest; 70 } 71 72 static struct None { 73 } 74 75 static struct Initialize { 76 } 77 78 static struct MutateCode { 79 NamedType!(bool, Tag!"FilesysError", bool.init, TagStringable, ImplicitConvertable) filesysError; 80 NamedType!(bool, Tag!"MutationError", bool.init, TagStringable, ImplicitConvertable) mutationError; 81 } 82 83 static struct TestMutantData { 84 /// If the user has configured that the test cases should be analyzed. 85 bool hasTestCaseOutputAnalyzer; 86 ShellCommand buildCmd; 87 Duration buildCmdTimeout; 88 } 89 90 static struct TestMutant { 91 NamedType!(bool, Tag!"HasTestOutput", bool.init, TagStringable, ImplicitConvertable) hasTestOutput; 92 } 93 94 static struct RestoreCode { 95 NamedType!(bool, Tag!"FilesysError", bool.init, TagStringable, ImplicitConvertable) filesysError; 96 } 97 98 static struct TestCaseAnalyzeData { 99 TestCaseAnalyzer* testCaseAnalyzer; 100 } 101 102 static struct TestCaseAnalyze { 103 bool unstableTests; 104 } 105 106 static struct StoreResult { 107 } 108 109 static struct Done { 110 } 111 112 static struct FilesysError { 113 } 114 115 // happens when an error occurs during mutations testing but that do not 116 // prohibit testing of other mutants 117 static struct NoResultRestoreCode { 118 } 119 120 static struct NoResult { 121 } 122 123 alias Fsm = my.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode, 124 TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult); 125 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData); 126 127 private { 128 Fsm fsm; 129 Global global; 130 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local; 131 bool isRunning_ = true; 132 bool stopBecauseError_; 133 } 134 135 MutationTestResult[] result; 136 137 this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) { 138 this.global = global; 139 this.local = LocalStateDataT(l1, l2); 140 141 if (logger.globalLogLevel == logger.LogLevel.trace) 142 fsm.logger = (string s) { logger.trace(s); }; 143 } 144 145 static void execute_(ref MutationTestDriver self) @trusted { 146 self.fsm.next!((None a) => fsm(Initialize.init), 147 (Initialize a) => fsm(MutateCode.init), (MutateCode a) { 148 if (a.filesysError) 149 return fsm(FilesysError.init); 150 else if (a.mutationError) 151 return fsm(NoResultRestoreCode.init); 152 return fsm(TestMutant.init); 153 }, (TestMutant a) { 154 if (self.global.testResult.status == Mutation.Status.killed 155 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) { 156 return fsm(TestCaseAnalyze.init); 157 } 158 return fsm(RestoreCode.init); 159 }, (TestCaseAnalyze a) { 160 if (a.unstableTests) 161 return fsm(NoResultRestoreCode.init); 162 return fsm(RestoreCode.init); 163 }, (RestoreCode a) { 164 if (a.filesysError) 165 return fsm(FilesysError.init); 166 return fsm(StoreResult.init); 167 }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a), 168 (FilesysError a) => fsm(a), 169 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),); 170 171 self.fsm.act!self; 172 } 173 174 nothrow: 175 176 void execute() { 177 try { 178 execute_(this); 179 } catch (Exception e) { 180 logger.warning(e.msg).collectException; 181 } 182 } 183 184 /// Returns: true as long as the driver is processing a mutant. 185 bool isRunning() { 186 return isRunning_; 187 } 188 189 bool stopBecauseError() { 190 return stopBecauseError_; 191 } 192 193 void opCall(None data) { 194 } 195 196 void opCall(Initialize data) { 197 global.swCompile.start; 198 } 199 200 void opCall(Done data) { 201 isRunning_ = false; 202 } 203 204 void opCall(FilesysError data) { 205 logger.warning("Filesystem error").collectException; 206 isRunning_ = false; 207 stopBecauseError_ = true; 208 } 209 210 void opCall(NoResultRestoreCode data) { 211 RestoreCode tmp; 212 this.opCall(tmp); 213 } 214 215 void opCall(NoResult data) { 216 isRunning_ = false; 217 } 218 219 void opCall(ref MutateCode data) { 220 import dextool.plugin.mutate.backend.generate_mutant : generateMutant, 221 GenerateMutantResult, GenerateMutantStatus; 222 223 try { 224 global.mutateFile = AbsolutePath(buildPath(global.fio.getOutputDir, global.mutp.file)); 225 global.original = global.fio.makeInput(global.mutateFile); 226 } catch (Exception e) { 227 logger.error(e.msg).collectException; 228 logger.warning("Unable to read ", global.mutateFile).collectException; 229 data.filesysError.get = true; 230 return; 231 } 232 233 // mutate 234 try { 235 auto fout = global.fio.makeOutput(global.mutateFile); 236 auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout); 237 238 final switch (mut_res.status) with (GenerateMutantStatus) { 239 case error: 240 data.mutationError.get = true; 241 break; 242 case filesysError: 243 data.filesysError.get = true; 244 break; 245 case databaseError: 246 // such as when the database is locked 247 data.mutationError.get = true; 248 break; 249 case checksumError: 250 data.filesysError.get = true; 251 break; 252 case noMutation: 253 data.mutationError.get = true; 254 break; 255 case ok: 256 try { 257 logger.infof("from '%s' to '%s' in %s:%s:%s", 258 cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to, 259 global.mutateFile, global.mutp.sloc.line, global.mutp.sloc.column); 260 logger.trace(global.mutp.id).collectException; 261 } catch (Exception e) { 262 logger.warningf("%s %s", global.mutp.id, e.msg); 263 } 264 break; 265 } 266 } catch (Exception e) { 267 logger.warning(e.msg).collectException; 268 data.mutationError.get = true; 269 } 270 } 271 272 void opCall(ref TestMutant data) { 273 bool successCompile; 274 compile(local.get!TestMutant.buildCmd, local.get!TestMutant.buildCmdTimeout).match!( 275 (Mutation.Status a) { global.testResult.status = a; }, (bool success) { 276 successCompile = success; 277 },); 278 279 global.swCompile.stop; 280 global.swTest.start; 281 282 if (!successCompile) 283 return; 284 285 global.testResult = runTester(*global.runner); 286 287 data.hasTestOutput.get = !global.testResult.output.empty; 288 } 289 290 void opCall(ref TestCaseAnalyze data) { 291 scope (exit) 292 global.testResult.output = null; 293 294 try { 295 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze( 296 global.testResult.output); 297 298 analyze.match!((TestCaseAnalyzer.Success a) { 299 global.testCases = a.failed; 300 global.testCases ~= global.testResult.testCmds.map!(a => TestCase(a.toShortString)) 301 .array; 302 }, (TestCaseAnalyzer.Unstable a) { 303 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 304 logger.info( 305 "As configured the result is ignored which will force the mutant to be re-tested"); 306 data.unstableTests = true; 307 }, (TestCaseAnalyzer.Failed a) { 308 logger.warning("The parser that analyze the output from test case(s) failed"); 309 }); 310 } catch (Exception e) { 311 logger.warning(e.msg).collectException; 312 } 313 } 314 315 void opCall(StoreResult data) { 316 import miniorm : spinSql; 317 318 const statusId = spinSql!(() => global.db.getMutationStatusId(global.mutp.id)); 319 320 global.swTest.stop; 321 auto profile = MutantTimeProfile(global.swCompile.peek, global.swTest.peek); 322 323 if (statusId.isNull) { 324 logger.trace("No MutationStatusId for ", global.mutp.id.get).collectException; 325 return; 326 } 327 328 result = [ 329 MutationTestResult(global.mutp.id, statusId.get, global.testResult.status, 330 profile, global.testCases, global.testResult.exitStatus) 331 ]; 332 333 logger.infof("%s:%s (%s)", global.testResult.status, 334 global.testResult.exitStatus.get, profile).collectException; 335 logger.infof(!global.testCases.empty, `killed by [%-(%s, %)]`, 336 global.testCases.sort.map!"a.name").collectException; 337 } 338 339 void opCall(ref RestoreCode data) { 340 // restore the original file. 341 try { 342 global.fio.makeOutput(global.mutateFile).write(global.original.content); 343 } catch (Exception e) { 344 logger.error(e.msg).collectException; 345 // fatal error because being unable to restore a file prohibit 346 // future mutations. 347 data.filesysError.get = true; 348 } 349 } 350 }