1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 */ 10 module dextool.plugin.mutate.backend.test_mutant.schemata; 11 12 import logger = std.experimental.logger; 13 import std.algorithm : sort, map; 14 import std.array : empty; 15 import std.conv : to; 16 import std.datetime : Duration; 17 import std.exception : collectException; 18 import std.typecons : Tuple; 19 20 import proc : DrainElement; 21 import sumtype; 22 23 import dextool.fsm : Fsm, next, act, get, TypeDataMap; 24 25 import dextool.plugin.mutate.backend.database : MutationStatusId, Database, spinSql; 26 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 27 import dextool.plugin.mutate.backend.test_mutant.common; 28 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 29 import dextool.plugin.mutate.backend.type : Mutation, TestCase, Checksum; 30 import dextool.plugin.mutate.type : TestCaseAnalyzeBuiltin, ShellCommand; 31 32 @safe: 33 34 struct MutationTestResult { 35 import std.datetime : Duration; 36 import dextool.plugin.mutate.backend.database : MutationStatusId; 37 import dextool.plugin.mutate.backend.type : TestCase; 38 39 MutationStatusId id; 40 Mutation.Status status; 41 Duration testTime; 42 TestCase[] testCases; 43 } 44 45 struct SchemataTestDriver { 46 private { 47 /// True as long as the schemata driver is running. 48 bool isRunning_ = true; 49 50 FilesysIO fio; 51 52 Database* db; 53 54 /// Runs the test commands. 55 TestRunner* runner; 56 57 /// Result of testing the mutants. 58 MutationTestResult[] result_; 59 } 60 61 static struct None { 62 } 63 64 static struct InitializeData { 65 MutationStatusId[] mutants; 66 } 67 68 static struct Initialize { 69 } 70 71 static struct Done { 72 } 73 74 static struct NextMutantData { 75 /// Mutants to test. 76 InjectIdResult mutants; 77 } 78 79 static struct NextMutant { 80 bool done; 81 InjectIdResult.InjectId inject; 82 } 83 84 static struct TestMutantData { 85 /// If the user has configured that the test cases should be analyzed. 86 bool hasTestCaseOutputAnalyzer; 87 } 88 89 static struct TestMutant { 90 InjectIdResult.InjectId inject; 91 92 MutationTestResult result; 93 bool hasTestOutput; 94 // if there are mutants status id's related to a file but the mutants 95 // have been removed. 96 bool mutantIdError; 97 } 98 99 static struct TestCaseAnalyzeData { 100 TestCaseAnalyzer* testCaseAnalyzer; 101 DrainElement[] output; 102 } 103 104 static struct TestCaseAnalyze { 105 MutationTestResult result; 106 bool unstableTests; 107 } 108 109 static struct StoreResult { 110 MutationTestResult result; 111 } 112 113 alias Fsm = dextool.fsm.Fsm!(None, Initialize, Done, NextMutant, 114 TestMutant, TestCaseAnalyze, StoreResult); 115 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData, 116 NextMutantData, InitializeData); 117 118 private { 119 Fsm fsm; 120 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze, NextMutant, Initialize) local; 121 } 122 123 this(FilesysIO fio, TestRunner* runner, Database* db, 124 TestCaseAnalyzer* testCaseAnalyzer, MutationStatusId[] mutants) { 125 this.fio = fio; 126 this.runner = runner; 127 this.db = db; 128 this.local.get!Initialize.mutants = mutants; 129 this.local.get!TestCaseAnalyze.testCaseAnalyzer = testCaseAnalyzer; 130 this.local.get!TestMutant.hasTestCaseOutputAnalyzer = !testCaseAnalyzer.empty; 131 } 132 133 static void execute_(ref SchemataTestDriver self) @trusted { 134 self.fsm.next!((None a) => fsm(Initialize.init), 135 (Initialize a) => fsm(NextMutant.init), (NextMutant a) { 136 if (a.done) 137 return fsm(Done.init); 138 return fsm(TestMutant(a.inject)); 139 }, (TestMutant a) { 140 if (a.mutantIdError) 141 return fsm(NextMutant.init); 142 if (a.result.status == Mutation.Status.killed 143 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) { 144 return fsm(TestCaseAnalyze(a.result)); 145 } 146 return fsm(StoreResult(a.result)); 147 }, (TestCaseAnalyze a) { 148 if (a.unstableTests) 149 return fsm(NextMutant.init); 150 return fsm(StoreResult(a.result)); 151 }, (StoreResult a) => fsm(NextMutant.init), (Done a) => fsm(a)); 152 153 debug logger.trace("state: ", self.fsm.logNext); 154 self.fsm.act!(self); 155 } 156 157 nothrow: 158 159 MutationTestResult[] result() { 160 return result_; 161 } 162 163 void execute() { 164 try { 165 execute_(this); 166 } catch (Exception e) { 167 logger.warning(e.msg).collectException; 168 } 169 } 170 171 bool isRunning() { 172 return isRunning_; 173 } 174 175 void opCall(None data) { 176 } 177 178 void opCall(Initialize data) { 179 scope (exit) 180 local.get!Initialize.mutants = null; 181 182 InjectIdBuilder builder; 183 foreach (mutant; local.get!Initialize.mutants) { 184 auto cs = spinSql!(() { return db.getChecksum(mutant); }); 185 if (!cs.isNull) { 186 builder.put(mutant, cs.get); 187 } 188 } 189 debug logger.trace(builder).collectException; 190 191 local.get!NextMutant.mutants = builder.finalize; 192 } 193 194 void opCall(Done data) { 195 isRunning_ = false; 196 } 197 198 void opCall(ref NextMutant data) { 199 data.done = local.get!NextMutant.mutants.empty; 200 201 if (!data.done) { 202 data.inject = local.get!NextMutant.mutants.front; 203 local.get!NextMutant.mutants.popFront; 204 } 205 } 206 207 void opCall(ref TestMutant data) { 208 import std.datetime.stopwatch : StopWatch, AutoStart; 209 import dextool.plugin.mutate.backend.analyze.pass_schemata : schemataMutantEnvKey, 210 checksumToId; 211 import dextool.plugin.mutate.backend.generate_mutant : makeMutationText; 212 213 data.result.id = data.inject.statusId; 214 215 auto id = spinSql!(() { return db.getMutationId(data.inject.statusId); }); 216 if (id.isNull) { 217 data.mutantIdError = true; 218 return; 219 } 220 auto entry_ = spinSql!(() { return db.getMutation(id.get); }); 221 if (entry_.isNull) { 222 data.mutantIdError = true; 223 return; 224 } 225 auto entry = entry_.get; 226 227 try { 228 const file = fio.toAbsoluteRoot(entry.file); 229 auto original = fio.makeInput(file); 230 auto txt = makeMutationText(original, entry.mp.offset, 231 entry.mp.mutations[0].kind, entry.lang); 232 debug logger.trace(entry); 233 logger.infof("%s from '%s' to '%s' in %s:%s:%s", data.inject.injectId, 234 txt.original, txt.mutation, file, entry.sloc.line, entry.sloc.column); 235 } catch (Exception e) { 236 logger.info(e.msg).collectException; 237 } 238 239 runner.env[schemataMutantEnvKey] = data.inject.injectId.to!string; 240 scope (exit) 241 runner.env.remove(schemataMutantEnvKey); 242 243 auto sw = StopWatch(AutoStart.yes); 244 auto res = runTester(*runner); 245 data.result.testTime = sw.peek; 246 247 data.result.status = res.status; 248 data.hasTestOutput = !res.output.empty; 249 local.get!TestCaseAnalyze.output = res.output; 250 251 logger.infof("%s %s (%s)", data.inject.injectId, data.result.status, 252 data.result.testTime).collectException; 253 } 254 255 void opCall(ref TestCaseAnalyze data) { 256 try { 257 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze( 258 local.get!TestCaseAnalyze.output); 259 local.get!TestCaseAnalyze.output = null; 260 261 analyze.match!((TestCaseAnalyzer.Success a) { 262 data.result.testCases = a.failed; 263 }, (TestCaseAnalyzer.Unstable a) { 264 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 265 logger.info( 266 "As configured the result is ignored which will force the mutant to be re-tested"); 267 data.unstableTests = true; 268 }, (TestCaseAnalyzer.Failed a) { 269 logger.warning("The parser that analyze the output from test case(s) failed"); 270 }); 271 272 logger.infof(!data.result.testCases.empty, `%s killed by [%-(%s, %)]`, 273 data.result.id, data.result.testCases.sort.map!"a.name").collectException; 274 } catch (Exception e) { 275 logger.warning(e.msg).collectException; 276 } 277 } 278 279 void opCall(StoreResult data) { 280 result_ ~= data.result; 281 } 282 } 283 284 /** Generate schemata injection IDs (32bit) from mutant checksums (128bit). 285 * 286 * There is a possibility that an injection ID result in a collision because 287 * they are only 32 bit. If that happens the mutant is discarded as unfeasable 288 * to use for schemata. 289 * 290 * TODO: if this is changed to being order dependent then it can handle all 291 * mutants. But I can't see how that can be done easily both because of how the 292 * schemas are generated and how the database is setup. 293 */ 294 struct InjectIdBuilder { 295 import dextool.set; 296 297 private { 298 alias InjectId = InjectIdResult.InjectId; 299 300 InjectId[uint] result; 301 Set!uint collisions; 302 } 303 304 void put(MutationStatusId id, Checksum cs) @safe pure nothrow { 305 import dextool.plugin.mutate.backend.analyze.pass_schemata : checksumToId; 306 307 const injectId = checksumToId(cs); 308 debug logger.tracef("%s %s %s", id, cs, injectId).collectException; 309 310 if (injectId in collisions) { 311 } else if (injectId in result) { 312 collisions.add(injectId); 313 result.remove(injectId); 314 } else { 315 result[injectId] = InjectId(id, injectId); 316 } 317 } 318 319 InjectIdResult finalize() @safe pure nothrow { 320 import std.array : array; 321 322 return InjectIdResult(result.byValue.array); 323 } 324 } 325 326 struct InjectIdResult { 327 alias InjectId = Tuple!(MutationStatusId, "statusId", uint, "injectId"); 328 InjectId[] ids; 329 330 InjectId front() @safe pure nothrow { 331 assert(!empty, "Can't get front of an empty range"); 332 return ids[0]; 333 } 334 335 void popFront() @safe pure nothrow { 336 assert(!empty, "Can't pop front of an empty range"); 337 ids = ids[1 .. $]; 338 } 339 340 bool empty() @safe pure nothrow const @nogc { 341 return ids.empty; 342 } 343 } 344 345 @("shall detect a collision and make sure it is never part of the result") 346 unittest { 347 InjectIdBuilder builder; 348 builder.put(MutationStatusId(1), Checksum(1, 2)); 349 builder.put(MutationStatusId(2), Checksum(3, 4)); 350 builder.put(MutationStatusId(3), Checksum(1, 2)); 351 auto r = builder.finalize; 352 353 assert(r.front.statusId == MutationStatusId(2)); 354 r.popFront; 355 assert(r.empty); 356 }