1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 Test a mutant by modifying the source code.
11 */
12 module dextool.plugin.mutate.backend.test_mutant.source_mutant;
13 
14 import logger = std.experimental.logger;
15 import std.array : empty;
16 import std.exception : collectException;
17 
18 import sumtype;
19 import proc : DrainElement;
20 
21 import dextool.fsm : Fsm, next, act, get, TypeDataMap;
22 import dextool.plugin.mutate.backend.database : Database, MutationEntry;
23 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
24 import dextool.plugin.mutate.backend.test_mutant.common;
25 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
26 import dextool.plugin.mutate.backend.type : Mutation, TestCase;
27 import dextool.plugin.mutate.config;
28 import dextool.plugin.mutate.type : ShellCommand;
29 import dextool.set;
30 import dextool.type : AbsolutePath, Path;
31 
32 @safe:
33 
34 /// The result of testing a mutant.
35 struct MutationTestResult {
36     import std.datetime : Duration;
37     import sumtype;
38     import proc : DrainElement;
39     import dextool.plugin.mutate.backend.database : MutationId;
40     import dextool.plugin.mutate.backend.type : TestCase;
41 
42     static struct NoResult {
43     }
44 
45     static struct StatusUpdate {
46         MutationId id;
47         Mutation.Status status;
48         Duration testTime;
49         TestCase[] testCases;
50     }
51 
52     alias Value = SumType!(NoResult, StatusUpdate);
53     Value value;
54 
55     void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc {
56         this.value = rhs.value;
57     }
58 
59     void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc {
60         this.value = Value(rhs);
61     }
62 }
63 
64 /** Drive the control flow when testing **a** mutant.
65  */
66 struct MutationTestDriver {
67     import std.datetime.stopwatch : StopWatch;
68     import std.typecons : Tuple;
69     import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase;
70 
71     static struct Global {
72         FilesysIO fio;
73         Database* db;
74 
75         /// The mutant to apply.
76         MutationEntry mutp;
77 
78         /// Runs the test commands.
79         TestRunner* runner;
80 
81         /// File to mutate.
82         AbsolutePath mut_file;
83 
84         /// The original file.
85         Blob original;
86 
87         /// The result of running the test cases.
88         Mutation.Status mut_status;
89 
90         /// Test cases that killed the mutant.
91         TestCase[] test_cases;
92 
93         /// How long it took to do the mutation testing.
94         StopWatch sw;
95     }
96 
97     static struct None {
98     }
99 
100     static struct Initialize {
101     }
102 
103     static struct MutateCode {
104         bool next;
105         bool filesysError;
106         bool mutationError;
107     }
108 
109     static struct TestMutantData {
110         /// If the user has configured that the test cases should be analyzed.
111         bool hasTestCaseOutputAnalyzer;
112         ShellCommand compile_cmd;
113     }
114 
115     static struct TestMutant {
116         bool hasTestOutput;
117     }
118 
119     static struct RestoreCode {
120         bool next;
121         bool filesysError;
122     }
123 
124     static struct TestCaseAnalyzeData {
125         TestCaseAnalyzer* testCaseAnalyzer;
126         DrainElement[] output;
127     }
128 
129     static struct TestCaseAnalyze {
130         bool unstableTests;
131     }
132 
133     static struct StoreResult {
134     }
135 
136     static struct Done {
137     }
138 
139     static struct FilesysError {
140     }
141 
142     // happens when an error occurs during mutations testing but that do not
143     // prohibit testing of other mutants
144     static struct NoResultRestoreCode {
145     }
146 
147     static struct NoResult {
148     }
149 
150     alias Fsm = dextool.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode,
151             TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult);
152     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData);
153 
154     private {
155         Fsm fsm;
156         Global global;
157         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local;
158         bool isRunning_ = true;
159         bool stopBecauseError_;
160     }
161 
162     MutationTestResult result;
163 
164     this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) {
165         this.global = global;
166         this.local = LocalStateDataT(l1, l2);
167     }
168 
169     static void execute_(ref MutationTestDriver self) @trusted {
170         self.fsm.next!((None a) => fsm(Initialize.init),
171                 (Initialize a) => fsm(MutateCode.init), (MutateCode a) {
172             if (a.next)
173                 return fsm(TestMutant.init);
174             else if (a.filesysError)
175                 return fsm(FilesysError.init);
176             else if (a.mutationError)
177                 return fsm(NoResultRestoreCode.init);
178             return fsm(a);
179         }, (TestMutant a) {
180             if (self.global.mut_status == Mutation.Status.killed
181                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) {
182                 return fsm(TestCaseAnalyze.init);
183             }
184             return fsm(RestoreCode.init);
185         }, (TestCaseAnalyze a) {
186             if (a.unstableTests)
187                 return fsm(NoResultRestoreCode.init);
188             return fsm(RestoreCode.init);
189         }, (RestoreCode a) {
190             if (a.next)
191                 return fsm(StoreResult.init);
192             else if (a.filesysError)
193                 return fsm(FilesysError.init);
194             return fsm(a);
195         }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a),
196                 (FilesysError a) => fsm(a),
197                 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),);
198 
199         debug logger.trace("state: ", self.fsm.logNext);
200         self.fsm.act!(self);
201     }
202 
203 nothrow:
204 
205     void execute() {
206         try {
207             execute_(this);
208         } catch (Exception e) {
209             logger.warning(e.msg).collectException;
210         }
211     }
212 
213     /// Returns: true as long as the driver is processing a mutant.
214     bool isRunning() {
215         return isRunning_;
216     }
217 
218     bool stopBecauseError() {
219         return stopBecauseError_;
220     }
221 
222     void opCall(None data) {
223     }
224 
225     void opCall(Initialize data) {
226         global.sw.start;
227     }
228 
229     void opCall(Done data) {
230         isRunning_ = false;
231     }
232 
233     void opCall(FilesysError data) {
234         logger.warning("Filesystem error").collectException;
235         isRunning_ = false;
236         stopBecauseError_ = true;
237     }
238 
239     void opCall(NoResultRestoreCode data) {
240         RestoreCode tmp;
241         this.opCall(tmp);
242     }
243 
244     void opCall(NoResult data) {
245         isRunning_ = false;
246     }
247 
248     void opCall(ref MutateCode data) {
249         import dextool.plugin.mutate.backend.generate_mutant : generateMutant,
250             GenerateMutantResult, GenerateMutantStatus;
251 
252         try {
253             global.mut_file = AbsolutePath(Path(global.mutp.file), global.fio.getOutputDir);
254             global.original = global.fio.makeInput(global.mut_file);
255         } catch (Exception e) {
256             logger.error(e.msg).collectException;
257             logger.warning("Unable to read ", global.mut_file).collectException;
258             data.filesysError = true;
259             return;
260         }
261 
262         // mutate
263         try {
264             auto fout = global.fio.makeOutput(global.mut_file);
265             auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout);
266 
267             final switch (mut_res.status) with (GenerateMutantStatus) {
268             case error:
269                 data.mutationError = true;
270                 break;
271             case filesysError:
272                 data.filesysError = true;
273                 break;
274             case databaseError:
275                 // such as when the database is locked
276                 data.mutationError = true;
277                 break;
278             case checksumError:
279                 data.filesysError = true;
280                 break;
281             case noMutation:
282                 data.mutationError = true;
283                 break;
284             case ok:
285                 data.next = true;
286                 try {
287                     logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id,
288                             cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to,
289                             global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column);
290 
291                 } catch (Exception e) {
292                     logger.warning("Mutation ID", e.msg);
293                 }
294                 break;
295             }
296         } catch (Exception e) {
297             logger.warning(e.msg).collectException;
298             data.mutationError = true;
299         }
300     }
301 
302     void opCall(ref TestMutant data) {
303         global.mut_status = Mutation.Status.unknown;
304 
305         bool successCompile;
306         compile(local.get!TestMutant.compile_cmd).match!((Mutation.Status a) {
307             global.mut_status = a;
308         }, (bool success) { successCompile = success; },);
309 
310         if (!successCompile)
311             return;
312 
313         auto res = runTester(*global.runner);
314         global.mut_status = res.status;
315         data.hasTestOutput = !res.output.empty;
316         local.get!TestCaseAnalyze.output = res.output;
317     }
318 
319     void opCall(ref TestCaseAnalyze data) {
320         global.test_cases = null;
321 
322         try {
323             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(
324                     local.get!TestCaseAnalyze.output);
325             local.get!TestCaseAnalyze.output = null;
326 
327             analyze.match!((TestCaseAnalyzer.Success a) {
328                 global.test_cases = a.failed;
329             }, (TestCaseAnalyzer.Unstable a) {
330                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
331                 logger.info(
332                     "As configured the result is ignored which will force the mutant to be re-tested");
333                 data.unstableTests = true;
334             }, (TestCaseAnalyzer.Failed a) {
335                 logger.warning("The parser that analyze the output from test case(s) failed");
336             });
337         } catch (Exception e) {
338             logger.warning(e.msg).collectException;
339         }
340     }
341 
342     void opCall(StoreResult data) {
343         global.sw.stop;
344         result = MutationTestResult.StatusUpdate(global.mutp.id,
345                 global.mut_status, global.sw.peek, global.test_cases);
346     }
347 
348     void opCall(ref RestoreCode data) {
349         // restore the original file.
350         try {
351             global.fio.makeOutput(global.mut_file).write(global.original.content);
352         } catch (Exception e) {
353             logger.error(e.msg).collectException;
354             // fatal error because being unable to restore a file prohibit
355             // future mutations.
356             data.filesysError = true;
357             return;
358         }
359 
360         data.next = true;
361     }
362 }