1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 A runner that can execute all the users test commands. These can be either
11 manually specified or automatically detected.
12 */
13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
14 
15 import logger = std.experimental.logger;
16 import std.algorithm : filter;
17 import std.array : appender, Appender, empty;
18 import std.datetime : Duration;
19 import std.exception : collectException;
20 import std.parallelism : TaskPool, Task, task;
21 
22 import process;
23 
24 import dextool.plugin.mutate.type : ShellCommand;
25 import dextool.type;
26 
27 version (unittest) {
28     import unit_threaded.assertions;
29 }
30 
31 @safe:
32 
33 struct TestRunner {
34     private {
35         alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]);
36         TaskPool pool;
37         Duration timeout_;
38     }
39 
40     /// Commands that execute the test cases.
41     ShellCommand[] commands;
42 
43     /// Environment to set when executing either binaries or the command.
44     string[string] env;
45 
46     static auto make(int poolSize) {
47         auto pool = () {
48             if (poolSize == 0) {
49                 return new TaskPool;
50             }
51             return new TaskPool(poolSize);
52         }();
53         pool.isDaemon = true;
54         return TestRunner(pool);
55     }
56 
57     ~this() {
58         pool.stop;
59     }
60 
61     bool empty() @safe pure nothrow const @nogc {
62         return commands.length == 0;
63     }
64 
65     void timeout(Duration timeout) pure nothrow @nogc {
66         this.timeout_ = timeout;
67     }
68 
69     void put(ShellCommand sh) pure nothrow {
70         commands ~= sh;
71     }
72 
73     void put(ShellCommand[] sh) pure nothrow {
74         commands ~= sh;
75     }
76 
77     TestResult run() {
78         return this.run(timeout_);
79     }
80 
81     TestResult run(Duration timeout) {
82         import core.thread : Thread;
83         import core.time : dur;
84         import std.range : enumerate;
85 
86         static TestTask* findDone(ref TestTask*[] tasks) {
87             foreach (t; tasks.enumerate.filter!(a => a.value.done)) {
88                 tasks[t.index] = tasks[$ - 1];
89                 tasks = tasks[0 .. $ - 1];
90                 return t.value;
91             }
92             return null;
93         }
94 
95         static void processDone(TestTask* t, ref TestResult result,
96                 ref Appender!(DrainElement[]) output) {
97             auto res = t.yieldForce;
98             final switch (res.status) {
99             case RunResult.Status.normal:
100                 if (result.status == TestResult.Status.passed && res.exitStatus != 0) {
101                     result.status = TestResult.Status.failed;
102                 }
103                 output.put(res.output);
104                 break;
105             case RunResult.Status.timeout:
106                 result.status = TestResult.Status.timeout;
107                 break;
108             case RunResult.Status.error:
109                 result.status = TestResult.Status.error;
110                 break;
111             }
112         }
113 
114         TestTask*[] tasks = startTests(timeout);
115         TestResult rval;
116         auto output = appender!(DrainElement[])();
117         while (!tasks.empty) {
118             auto t = findDone(tasks);
119             if (t !is null) {
120                 processDone(t, rval, output);
121             }
122             () @trusted { Thread.sleep(50.dur!"msecs"); }();
123         }
124 
125         rval.output = output.data;
126         return rval;
127     }
128 
129     private auto startTests(Duration timeout) @trusted {
130         auto tasks = appender!(TestTask*[])();
131 
132         foreach (c; commands) {
133             auto t = task!spawnRunTest(c.value, timeout, env);
134             tasks.put(t);
135             pool.put(t);
136         }
137         return tasks.data;
138     }
139 }
140 
141 /// The result of running the tests.
142 struct TestResult {
143     enum Status {
144         /// All test commands exited with exit status zero.
145         passed,
146         /// At least one test command indicated that an error where found by exit status not zero.
147         failed,
148         /// At least one test command timed out.
149         timeout,
150         /// Something happend when the test command executed thus the result should not be used.
151         error,
152     }
153 
154     Status status;
155 
156     /// Output from all the test binaries and command.
157     DrainElement[] output;
158 }
159 
160 /// Finds all executables in a directory tree.
161 string[] findExecutables(AbsolutePath root) @trusted {
162     import core.sys.posix.sys.stat;
163     import std.file : getAttributes;
164     import std.file : dirEntries, SpanMode;
165 
166     static bool isExecutable(string p) nothrow {
167         try {
168             return (getAttributes(p) & S_IXUSR) != 0;
169         } catch (Exception e) {
170         }
171         return false;
172     }
173 
174     auto app = appender!(string[])();
175     foreach (f; dirEntries(root, SpanMode.breadth).filter!(a => a.isFile)
176             .filter!(a => isExecutable(a.name))) {
177         app.put([f.name]);
178     }
179 
180     return app.data;
181 }
182 
183 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) @trusted nothrow {
184     import std.algorithm : copy;
185     static import std.process;
186 
187     RunResult rval;
188 
189     try {
190         auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).scopeKill;
191         auto output = appender!(DrainElement[])();
192         p.process.drain.copy(output);
193 
194         if (p.timeoutTriggered) {
195             rval.status = RunResult.Status.timeout;
196         }
197 
198         rval.exitStatus = p.wait;
199         rval.output = output.data;
200     } catch (Exception e) {
201         logger.warning(e.msg).collectException;
202         rval.status = RunResult.Status.error;
203     }
204 
205     return rval;
206 }
207 
208 private:
209 
210 struct RunResult {
211     enum Status {
212         /// the test command successfully executed.
213         normal,
214         /// Something happend when the test command executed thus the result should not be used.
215         error,
216         /// The test command timed out.
217         timeout,
218     }
219 
220     Status status;
221     ///
222     int exitStatus;
223     ///
224     DrainElement[] output;
225 }
226 
227 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) {
228     import core.sys.posix.sys.stat;
229     import std.file : getAttributes, setAttributes, thisExePath;
230     import std.stdio : File;
231     import std.path : baseName;
232     import std.conv : to;
233 
234     immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh";
235 
236     File(fname, "w").writeln(`#!/bin/bash
237 echo $1
238 if [[ "$3" = "timeout" ]]; then
239     sleep 10m
240 fi
241 exit $2`);
242     setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH);
243     return fname;
244 }
245 
246 version (unittest) {
247     import core.time : dur;
248     import std.algorithm : count;
249     import std.array : array, empty;
250     import std.file : remove, exists;
251     import std.string : strip;
252 }
253 
254 @("shall collect the result from the test commands when running them")
255 unittest {
256     immutable script = makeUnittestScript("script_");
257     scope (exit)
258         () {
259         if (exists(script))
260             remove(script);
261     }();
262 
263     auto runner = TestRunner.make(0);
264     runner.put([script, "foo", "0"].ShellCommand);
265     runner.put([script, "foo", "0"].ShellCommand);
266     auto res = runner.run(5.dur!"seconds");
267 
268     res.output.filter!(a => !a.empty).count.shouldEqual(2);
269     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
270     res.status.shouldEqual(TestResult.Status.passed);
271 }
272 
273 @("shall set the status to failed when one of test command fail")
274 unittest {
275     immutable script = makeUnittestScript("script_");
276     scope (exit)
277         () {
278         if (exists(script))
279             remove(script);
280     }();
281 
282     auto runner = TestRunner.make(0);
283     runner.put([script, "foo", "0"].ShellCommand);
284     runner.put([script, "foo", "1"].ShellCommand);
285     auto res = runner.run(5.dur!"seconds");
286 
287     res.output.filter!(a => !a.empty).count.shouldEqual(2);
288     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
289     res.status.shouldEqual(TestResult.Status.failed);
290 }
291 
292 @("shall set the status to timeout when one of the tests commands reach the timeout limit")
293 unittest {
294     immutable script = makeUnittestScript("script_");
295     scope (exit)
296         () {
297         if (exists(script))
298             remove(script);
299     }();
300 
301     auto runner = TestRunner.make(0);
302     runner.put([script, "foo", "0"].ShellCommand);
303     runner.put([script, "foo", "0", "timeout"].ShellCommand);
304     auto res = runner.run(1.dur!"seconds");
305 
306     res.output.filter!(a => !a.empty).count.shouldEqual(1);
307     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1);
308     res.status.shouldEqual(TestResult.Status.timeout);
309 }