1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 Test a mutant by modifying the source code.
11 */
12 module dextool.plugin.mutate.backend.test_mutant.source_mutant;
13 
14 import logger = std.experimental.logger;
15 import std.array : empty;
16 import std.exception : collectException;
17 
18 import sumtype;
19 import process : DrainElement;
20 
21 import dextool.fsm : Fsm, next, act, get, TypeDataMap;
22 import dextool.plugin.mutate.backend.database : Database, MutationEntry;
23 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
24 import dextool.plugin.mutate.backend.test_mutant.common;
25 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
26 import dextool.plugin.mutate.backend.type : Mutation, TestCase;
27 import dextool.plugin.mutate.config;
28 import dextool.plugin.mutate.type : ShellCommand;
29 import dextool.set;
30 import dextool.type : AbsolutePath, Path;
31 
32 @safe:
33 
34 /// The result of testing a mutant.
35 struct MutationTestResult {
36     import std.datetime : Duration;
37     import sumtype;
38     import process : DrainElement;
39     import dextool.plugin.mutate.backend.database : MutationId;
40     import dextool.plugin.mutate.backend.type : TestCase;
41 
42     static struct NoResult {
43     }
44 
45     static struct StatusUpdate {
46         MutationId id;
47         Mutation.Status status;
48         Duration testTime;
49         TestCase[] testCases;
50         DrainElement[] output;
51     }
52 
53     alias Value = SumType!(NoResult, StatusUpdate);
54     Value value;
55 
56     void opAssign(MutationTestResult rhs) @trusted pure nothrow @nogc {
57         this.value = rhs.value;
58     }
59 
60     void opAssign(StatusUpdate rhs) @trusted pure nothrow @nogc {
61         this.value = Value(rhs);
62     }
63 }
64 
65 /** Drive the control flow when testing **a** mutant.
66  */
67 struct MutationTestDriver {
68     import std.datetime.stopwatch : StopWatch;
69     import std.typecons : Tuple;
70     import dextool.plugin.mutate.backend.test_mutant.interface_ : GatherTestCase;
71 
72     static struct Global {
73         FilesysIO fio;
74         Database* db;
75 
76         /// The mutant to apply.
77         MutationEntry mutp;
78 
79         /// Runs the test commands.
80         TestRunner* runner;
81 
82         /// File to mutate.
83         AbsolutePath mut_file;
84 
85         /// The original file.
86         Blob original;
87 
88         /// The result of running the test cases.
89         Mutation.Status mut_status;
90 
91         /// Test cases that killed the mutant.
92         TestCase[] test_cases;
93 
94         /// How long it took to do the mutation testing.
95         StopWatch sw;
96     }
97 
98     static struct None {
99     }
100 
101     static struct Initialize {
102     }
103 
104     static struct MutateCode {
105         bool next;
106         bool filesysError;
107         bool mutationError;
108     }
109 
110     static struct TestMutantData {
111         /// If the user has configured that the test cases should be analyzed.
112         bool hasTestCaseOutputAnalyzer;
113         ShellCommand compile_cmd;
114     }
115 
116     static struct TestMutant {
117         DrainElement[] output;
118     }
119 
120     static struct RestoreCode {
121         bool next;
122         bool filesysError;
123     }
124 
125     static struct TestCaseAnalyzeData {
126         TestCaseAnalyzer* testCaseAnalyzer;
127     }
128 
129     static struct TestCaseAnalyze {
130         DrainElement[] output;
131         bool unstableTests;
132     }
133 
134     static struct StoreResult {
135     }
136 
137     static struct Done {
138     }
139 
140     static struct FilesysError {
141     }
142 
143     // happens when an error occurs during mutations testing but that do not
144     // prohibit testing of other mutants
145     static struct NoResultRestoreCode {
146     }
147 
148     static struct NoResult {
149     }
150 
151     alias Fsm = dextool.fsm.Fsm!(None, Initialize, MutateCode, TestMutant, RestoreCode,
152             TestCaseAnalyze, StoreResult, Done, FilesysError, NoResultRestoreCode, NoResult);
153     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData);
154 
155     private {
156         Fsm fsm;
157         Global global;
158         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze) local;
159         bool isRunning_ = true;
160         bool stopBecauseError_;
161     }
162 
163     MutationTestResult result;
164 
165     this(Global global, TestMutantData l1, TestCaseAnalyzeData l2) {
166         this.global = global;
167         this.local = LocalStateDataT(l1, l2);
168     }
169 
170     static void execute_(ref MutationTestDriver self) @trusted {
171         self.fsm.next!((None a) => fsm(Initialize.init),
172                 (Initialize a) => fsm(MutateCode.init), (MutateCode a) {
173             if (a.next)
174                 return fsm(TestMutant.init);
175             else if (a.filesysError)
176                 return fsm(FilesysError.init);
177             else if (a.mutationError)
178                 return fsm(NoResultRestoreCode.init);
179             return fsm(a);
180         }, (TestMutant a) {
181             if (self.global.mut_status == Mutation.Status.killed
182                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && !a.output.empty)
183                 return fsm(TestCaseAnalyze(a.output));
184             return fsm(RestoreCode.init);
185         }, (TestCaseAnalyze a) {
186             if (a.unstableTests)
187                 return fsm(NoResultRestoreCode.init);
188             return fsm(RestoreCode.init);
189         }, (RestoreCode a) {
190             if (a.next)
191                 return fsm(StoreResult.init);
192             else if (a.filesysError)
193                 return fsm(FilesysError.init);
194             return fsm(a);
195         }, (StoreResult a) { return fsm(Done.init); }, (Done a) => fsm(a),
196                 (FilesysError a) => fsm(a),
197                 (NoResultRestoreCode a) => fsm(NoResult.init), (NoResult a) => fsm(a),);
198 
199         self.fsm.act!(self);
200     }
201 
202 nothrow:
203 
204     void execute() {
205         try {
206             execute_(this);
207         } catch (Exception e) {
208             logger.warning(e.msg).collectException;
209         }
210     }
211 
212     /// Returns: true as long as the driver is processing a mutant.
213     bool isRunning() {
214         return isRunning_;
215     }
216 
217     bool stopBecauseError() {
218         return stopBecauseError_;
219     }
220 
221     void opCall(None data) {
222     }
223 
224     void opCall(Initialize data) {
225         global.sw.start;
226     }
227 
228     void opCall(Done data) {
229         isRunning_ = false;
230     }
231 
232     void opCall(FilesysError data) {
233         logger.warning("Filesystem error").collectException;
234         isRunning_ = false;
235         stopBecauseError_ = true;
236     }
237 
238     void opCall(NoResultRestoreCode data) {
239         RestoreCode tmp;
240         this.opCall(tmp);
241     }
242 
243     void opCall(NoResult data) {
244         isRunning_ = false;
245     }
246 
247     void opCall(ref MutateCode data) {
248         import dextool.plugin.mutate.backend.generate_mutant : generateMutant,
249             GenerateMutantResult, GenerateMutantStatus;
250 
251         try {
252             global.mut_file = AbsolutePath(Path(global.mutp.file), global.fio.getOutputDir);
253             global.original = global.fio.makeInput(global.mut_file);
254         } catch (Exception e) {
255             logger.error(e.msg).collectException;
256             logger.warning("Unable to read ", global.mut_file).collectException;
257             data.filesysError = true;
258             return;
259         }
260 
261         // mutate
262         try {
263             auto fout = global.fio.makeOutput(global.mut_file);
264             auto mut_res = generateMutant(*global.db, global.mutp, global.original, fout);
265 
266             final switch (mut_res.status) with (GenerateMutantStatus) {
267             case error:
268                 data.mutationError = true;
269                 break;
270             case filesysError:
271                 data.filesysError = true;
272                 break;
273             case databaseError:
274                 // such as when the database is locked
275                 data.mutationError = true;
276                 break;
277             case checksumError:
278                 data.filesysError = true;
279                 break;
280             case noMutation:
281                 data.mutationError = true;
282                 break;
283             case ok:
284                 data.next = true;
285                 try {
286                     logger.infof("%s from '%s' to '%s' in %s:%s:%s", global.mutp.id,
287                             cast(const(char)[]) mut_res.from, cast(const(char)[]) mut_res.to,
288                             global.mut_file, global.mutp.sloc.line, global.mutp.sloc.column);
289 
290                 } catch (Exception e) {
291                     logger.warning("Mutation ID", e.msg);
292                 }
293                 break;
294             }
295         } catch (Exception e) {
296             logger.warning(e.msg).collectException;
297             data.mutationError = true;
298         }
299     }
300 
301     void opCall(ref TestMutant data) {
302         global.mut_status = Mutation.Status.unknown;
303 
304         bool successCompile;
305         compile(local.get!TestMutant.compile_cmd).match!((Mutation.Status a) {
306             global.mut_status = a;
307         }, (bool success) { successCompile = success; },);
308 
309         if (!successCompile)
310             return;
311 
312         auto res = runTester(*global.runner);
313         global.mut_status = res.status;
314         data.output = res.output;
315     }
316 
317     void opCall(ref TestCaseAnalyze data) {
318         global.test_cases = null;
319 
320         try {
321             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(data.output);
322 
323             analyze.match!((TestCaseAnalyzer.Success a) {
324                 global.test_cases = a.failed;
325             }, (TestCaseAnalyzer.Unstable a) {
326                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
327                 logger.info(
328                     "As configured the result is ignored which will force the mutant to be re-tested");
329                 data.unstableTests = true;
330             }, (TestCaseAnalyzer.Failed a) {
331                 logger.warning("The parser that analyze the output from test case(s) failed");
332             });
333         } catch (Exception e) {
334             logger.warning(e.msg).collectException;
335         }
336     }
337 
338     void opCall(StoreResult data) {
339         global.sw.stop;
340         result = MutationTestResult.StatusUpdate(global.mutp.id,
341                 global.mut_status, global.sw.peek, global.test_cases);
342     }
343 
344     void opCall(ref RestoreCode data) {
345         // restore the original file.
346         try {
347             global.fio.makeOutput(global.mut_file).write(global.original.content);
348         } catch (Exception e) {
349             logger.error(e.msg).collectException;
350             // fatal error because being unable to restore a file prohibit
351             // future mutations.
352             data.filesysError = true;
353             return;
354         }
355 
356         data.next = true;
357     }
358 }