1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 */ 10 module dextool.plugin.mutate.backend.test_mutant.schemata; 11 12 import logger = std.experimental.logger; 13 import std.algorithm : sort, map; 14 import std.array : empty; 15 import std.conv : to; 16 import std.datetime : Duration; 17 import std.exception : collectException; 18 import std.typecons : Tuple; 19 20 import proc : DrainElement; 21 import sumtype; 22 23 import my.fsm : Fsm, next, act, get, TypeDataMap; 24 static import my.fsm; 25 26 import dextool.plugin.mutate.backend.database : MutationStatusId, Database, spinSql; 27 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob; 28 import dextool.plugin.mutate.backend.test_mutant.common; 29 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 30 import dextool.plugin.mutate.backend.type : Mutation, TestCase, Checksum; 31 import dextool.plugin.mutate.type : TestCaseAnalyzeBuiltin, ShellCommand; 32 33 @safe: 34 35 struct MutationTestResult { 36 import std.datetime : Duration; 37 import dextool.plugin.mutate.backend.database : MutationStatusId; 38 import dextool.plugin.mutate.backend.type : TestCase; 39 40 MutationStatusId id; 41 Mutation.Status status; 42 Duration testTime; 43 TestCase[] testCases; 44 } 45 46 struct SchemataTestDriver { 47 private { 48 /// True as long as the schemata driver is running. 49 bool isRunning_ = true; 50 51 FilesysIO fio; 52 53 Database* db; 54 55 /// Runs the test commands. 56 TestRunner* runner; 57 58 /// Result of testing the mutants. 59 MutationTestResult[] result_; 60 } 61 62 static struct None { 63 } 64 65 static struct InitializeData { 66 MutationStatusId[] mutants; 67 } 68 69 static struct Initialize { 70 } 71 72 static struct Done { 73 } 74 75 static struct NextMutantData { 76 /// Mutants to test. 77 InjectIdResult mutants; 78 } 79 80 static struct NextMutant { 81 bool done; 82 InjectIdResult.InjectId inject; 83 } 84 85 static struct TestMutantData { 86 /// If the user has configured that the test cases should be analyzed. 87 bool hasTestCaseOutputAnalyzer; 88 } 89 90 static struct TestMutant { 91 InjectIdResult.InjectId inject; 92 93 MutationTestResult result; 94 bool hasTestOutput; 95 // if there are mutants status id's related to a file but the mutants 96 // have been removed. 97 bool mutantIdError; 98 } 99 100 static struct TestCaseAnalyzeData { 101 TestCaseAnalyzer* testCaseAnalyzer; 102 DrainElement[] output; 103 } 104 105 static struct TestCaseAnalyze { 106 MutationTestResult result; 107 bool unstableTests; 108 } 109 110 static struct StoreResult { 111 MutationTestResult result; 112 } 113 114 alias Fsm = my.fsm.Fsm!(None, Initialize, Done, NextMutant, TestMutant, 115 TestCaseAnalyze, StoreResult); 116 alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData, 117 NextMutantData, InitializeData); 118 119 private { 120 Fsm fsm; 121 TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze, NextMutant, Initialize) local; 122 } 123 124 this(FilesysIO fio, TestRunner* runner, Database* db, 125 TestCaseAnalyzer* testCaseAnalyzer, MutationStatusId[] mutants) { 126 this.fio = fio; 127 this.runner = runner; 128 this.db = db; 129 this.local.get!Initialize.mutants = mutants; 130 this.local.get!TestCaseAnalyze.testCaseAnalyzer = testCaseAnalyzer; 131 this.local.get!TestMutant.hasTestCaseOutputAnalyzer = !testCaseAnalyzer.empty; 132 } 133 134 static void execute_(ref SchemataTestDriver self) @trusted { 135 self.fsm.next!((None a) => fsm(Initialize.init), 136 (Initialize a) => fsm(NextMutant.init), (NextMutant a) { 137 if (a.done) 138 return fsm(Done.init); 139 return fsm(TestMutant(a.inject)); 140 }, (TestMutant a) { 141 if (a.mutantIdError) 142 return fsm(NextMutant.init); 143 if (a.result.status == Mutation.Status.killed 144 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) { 145 return fsm(TestCaseAnalyze(a.result)); 146 } 147 return fsm(StoreResult(a.result)); 148 }, (TestCaseAnalyze a) { 149 if (a.unstableTests) 150 return fsm(NextMutant.init); 151 return fsm(StoreResult(a.result)); 152 }, (StoreResult a) => fsm(NextMutant.init), (Done a) => fsm(a)); 153 154 debug logger.trace("state: ", self.fsm.logNext); 155 self.fsm.act!(self); 156 } 157 158 nothrow: 159 160 MutationTestResult[] result() { 161 return result_; 162 } 163 164 void execute() { 165 try { 166 execute_(this); 167 } catch (Exception e) { 168 logger.warning(e.msg).collectException; 169 } 170 } 171 172 bool isRunning() { 173 return isRunning_; 174 } 175 176 void opCall(None data) { 177 } 178 179 void opCall(Initialize data) { 180 scope (exit) 181 local.get!Initialize.mutants = null; 182 183 InjectIdBuilder builder; 184 foreach (mutant; local.get!Initialize.mutants) { 185 auto cs = spinSql!(() { return db.getChecksum(mutant); }); 186 if (!cs.isNull) { 187 builder.put(mutant, cs.get); 188 } 189 } 190 debug logger.trace(builder).collectException; 191 192 local.get!NextMutant.mutants = builder.finalize; 193 } 194 195 void opCall(Done data) { 196 isRunning_ = false; 197 } 198 199 void opCall(ref NextMutant data) { 200 data.done = local.get!NextMutant.mutants.empty; 201 202 if (!data.done) { 203 data.inject = local.get!NextMutant.mutants.front; 204 local.get!NextMutant.mutants.popFront; 205 } 206 } 207 208 void opCall(ref TestMutant data) { 209 import std.datetime.stopwatch : StopWatch, AutoStart; 210 import dextool.plugin.mutate.backend.analyze.pass_schemata : schemataMutantEnvKey, 211 checksumToId; 212 import dextool.plugin.mutate.backend.generate_mutant : makeMutationText; 213 214 data.result.id = data.inject.statusId; 215 216 auto id = spinSql!(() { return db.getMutationId(data.inject.statusId); }); 217 if (id.isNull) { 218 data.mutantIdError = true; 219 return; 220 } 221 auto entry_ = spinSql!(() { return db.getMutation(id.get); }); 222 if (entry_.isNull) { 223 data.mutantIdError = true; 224 return; 225 } 226 auto entry = entry_.get; 227 228 try { 229 const file = fio.toAbsoluteRoot(entry.file); 230 auto original = fio.makeInput(file); 231 auto txt = makeMutationText(original, entry.mp.offset, 232 entry.mp.mutations[0].kind, entry.lang); 233 debug logger.trace(entry); 234 logger.infof("%s from '%s' to '%s' in %s:%s:%s", data.inject.injectId, 235 txt.original, txt.mutation, file, entry.sloc.line, entry.sloc.column); 236 } catch (Exception e) { 237 logger.info(e.msg).collectException; 238 } 239 240 runner.env[schemataMutantEnvKey] = data.inject.injectId.to!string; 241 scope (exit) 242 runner.env.remove(schemataMutantEnvKey); 243 244 auto sw = StopWatch(AutoStart.yes); 245 auto res = runTester(*runner); 246 data.result.testTime = sw.peek; 247 248 data.result.status = res.status; 249 data.hasTestOutput = !res.output.empty; 250 local.get!TestCaseAnalyze.output = res.output; 251 252 logger.infof("%s %s (%s)", data.inject.injectId, data.result.status, 253 data.result.testTime).collectException; 254 } 255 256 void opCall(ref TestCaseAnalyze data) { 257 try { 258 auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze( 259 local.get!TestCaseAnalyze.output); 260 local.get!TestCaseAnalyze.output = null; 261 262 analyze.match!((TestCaseAnalyzer.Success a) { 263 data.result.testCases = a.failed; 264 }, (TestCaseAnalyzer.Unstable a) { 265 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable); 266 logger.info( 267 "As configured the result is ignored which will force the mutant to be re-tested"); 268 data.unstableTests = true; 269 }, (TestCaseAnalyzer.Failed a) { 270 logger.warning("The parser that analyze the output from test case(s) failed"); 271 }); 272 273 logger.infof(!data.result.testCases.empty, `%s killed by [%-(%s, %)]`, 274 data.result.id, data.result.testCases.sort.map!"a.name").collectException; 275 } catch (Exception e) { 276 logger.warning(e.msg).collectException; 277 } 278 } 279 280 void opCall(StoreResult data) { 281 result_ ~= data.result; 282 } 283 } 284 285 /** Generate schemata injection IDs (32bit) from mutant checksums (128bit). 286 * 287 * There is a possibility that an injection ID result in a collision because 288 * they are only 32 bit. If that happens the mutant is discarded as unfeasable 289 * to use for schemata. 290 * 291 * TODO: if this is changed to being order dependent then it can handle all 292 * mutants. But I can't see how that can be done easily both because of how the 293 * schemas are generated and how the database is setup. 294 */ 295 struct InjectIdBuilder { 296 import my.set; 297 298 private { 299 alias InjectId = InjectIdResult.InjectId; 300 301 InjectId[uint] result; 302 Set!uint collisions; 303 } 304 305 void put(MutationStatusId id, Checksum cs) @safe pure nothrow { 306 import dextool.plugin.mutate.backend.analyze.pass_schemata : checksumToId; 307 308 const injectId = checksumToId(cs); 309 debug logger.tracef("%s %s %s", id, cs, injectId).collectException; 310 311 if (injectId in collisions) { 312 } else if (injectId in result) { 313 collisions.add(injectId); 314 result.remove(injectId); 315 } else { 316 result[injectId] = InjectId(id, injectId); 317 } 318 } 319 320 InjectIdResult finalize() @safe pure nothrow { 321 import std.array : array; 322 323 return InjectIdResult(result.byValue.array); 324 } 325 } 326 327 struct InjectIdResult { 328 alias InjectId = Tuple!(MutationStatusId, "statusId", uint, "injectId"); 329 InjectId[] ids; 330 331 InjectId front() @safe pure nothrow { 332 assert(!empty, "Can't get front of an empty range"); 333 return ids[0]; 334 } 335 336 void popFront() @safe pure nothrow { 337 assert(!empty, "Can't pop front of an empty range"); 338 ids = ids[1 .. $]; 339 } 340 341 bool empty() @safe pure nothrow const @nogc { 342 return ids.empty; 343 } 344 } 345 346 @("shall detect a collision and make sure it is never part of the result") 347 unittest { 348 InjectIdBuilder builder; 349 builder.put(MutationStatusId(1), Checksum(1, 2)); 350 builder.put(MutationStatusId(2), Checksum(3, 4)); 351 builder.put(MutationStatusId(3), Checksum(1, 2)); 352 auto r = builder.finalize; 353 354 assert(r.front.statusId == MutationStatusId(2)); 355 r.popFront; 356 assert(r.empty); 357 }