1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 A runner that can execute all the users test commands. These can be either
11 manually specified or automatically detected.
12 */
13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
14 
15 import logger = std.experimental.logger;
16 import std.algorithm : filter;
17 import std.array : appender, Appender, empty;
18 import std.datetime : Duration, dur;
19 import std.exception : collectException;
20 import std.parallelism : TaskPool, Task, task;
21 
22 import process;
23 
24 import dextool.plugin.mutate.type : ShellCommand;
25 import dextool.type;
26 
27 version (unittest) {
28     import unit_threaded.assertions;
29 }
30 
31 @safe:
32 
33 struct TestRunner {
34     private {
35         alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]);
36         TaskPool pool;
37         Duration timeout_;
38     }
39 
40     /// Commands that execute the test cases.
41     ShellCommand[] commands;
42 
43     /// Environment to set when executing either binaries or the command.
44     string[string] env;
45 
46     static auto make(int poolSize) {
47         return TestRunner(poolSize);
48     }
49 
50     this(int poolSize_) {
51         this.poolSize(poolSize_);
52     }
53 
54     ~this() {
55         pool.stop;
56     }
57 
58     bool empty() @safe pure nothrow const @nogc {
59         return commands.length == 0;
60     }
61 
62     void poolSize(const int s) @safe {
63         if (pool !is null) {
64             pool.stop;
65         }
66         if (s == 0) {
67             pool = new TaskPool;
68         } else {
69             pool = new TaskPool(s);
70         }
71         pool.isDaemon = true;
72     }
73 
74     void timeout(Duration timeout) pure nothrow @nogc {
75         this.timeout_ = timeout;
76     }
77 
78     void put(ShellCommand sh) pure nothrow {
79         commands ~= sh;
80     }
81 
82     void put(ShellCommand[] sh) pure nothrow {
83         commands ~= sh;
84     }
85 
86     TestResult run(string[string] localEnv = null) {
87         return this.run(timeout_, localEnv);
88     }
89 
90     TestResult run(Duration timeout, string[string] localEnv = null) {
91         import core.thread : Thread;
92         import core.time : dur;
93         import std.range : enumerate;
94 
95         static TestTask* findDone(ref TestTask*[] tasks) {
96             bool found;
97             size_t idx;
98             foreach (t; tasks.enumerate.filter!(a => a.value.done)) {
99                 idx = t.index;
100                 found = true;
101                 break;
102             }
103 
104             if (found) {
105                 auto t = tasks[idx];
106                 tasks[idx] = tasks[$ - 1];
107                 tasks = tasks[0 .. $ - 1];
108                 return t;
109             }
110             return null;
111         }
112 
113         static void processDone(TestTask* t, ref TestResult result,
114                 ref Appender!(DrainElement[]) output) {
115             auto res = t.yieldForce;
116             final switch (res.status) {
117             case RunResult.Status.normal:
118                 if (result.status == TestResult.Status.passed && res.exitStatus != 0) {
119                     result.status = TestResult.Status.failed;
120                 }
121                 output.put(res.output);
122                 break;
123             case RunResult.Status.timeout:
124                 result.status = TestResult.Status.timeout;
125                 break;
126             case RunResult.Status.error:
127                 result.status = TestResult.Status.error;
128                 break;
129             }
130         }
131 
132         auto env_ = env;
133         foreach (kv; localEnv.byKeyValue) {
134             env_[kv.key] = kv.value;
135         }
136 
137         TestTask*[] tasks = startTests(timeout, env_);
138         TestResult rval;
139         auto output = appender!(DrainElement[])();
140         while (!tasks.empty) {
141             auto t = findDone(tasks);
142             if (t !is null) {
143                 processDone(t, rval, output);
144                 .destroy(t);
145             }
146             () @trusted { Thread.sleep(50.dur!"msecs"); }();
147         }
148 
149         rval.output = output.data;
150         return rval;
151     }
152 
153     private auto startTests(Duration timeout, string[string] env) @trusted {
154         auto tasks = appender!(TestTask*[])();
155 
156         foreach (c; commands) {
157             auto t = task!spawnRunTest(c.value, timeout, env);
158             tasks.put(t);
159             pool.put(t);
160         }
161         return tasks.data;
162     }
163 }
164 
165 /// The result of running the tests.
166 struct TestResult {
167     enum Status {
168         /// All test commands exited with exit status zero.
169         passed,
170         /// At least one test command indicated that an error where found by exit status not zero.
171         failed,
172         /// At least one test command timed out.
173         timeout,
174         /// Something happend when the test command executed thus the result should not be used.
175         error,
176     }
177 
178     Status status;
179 
180     /// Output from all the test binaries and command.
181     DrainElement[] output;
182 }
183 
184 /// Finds all executables in a directory tree.
185 string[] findExecutables(AbsolutePath root) @trusted {
186     import core.sys.posix.sys.stat;
187     import std.file : getAttributes;
188     import std.file : dirEntries, SpanMode;
189 
190     static bool isExecutable(string p) nothrow {
191         try {
192             return (getAttributes(p) & S_IXUSR) != 0;
193         } catch (Exception e) {
194         }
195         return false;
196     }
197 
198     auto app = appender!(string[])();
199     foreach (f; dirEntries(root, SpanMode.breadth).filter!(a => a.isFile)
200             .filter!(a => isExecutable(a.name))) {
201         app.put([f.name]);
202     }
203 
204     return app.data;
205 }
206 
207 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) @trusted nothrow {
208     import std.algorithm : copy;
209     static import std.process;
210 
211     RunResult rval;
212 
213     try {
214         auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).scopeKill;
215         auto output = appender!(DrainElement[])();
216         p.process.drain(200.dur!"msecs").copy(output);
217 
218         if (p.timeoutTriggered) {
219             rval.status = RunResult.Status.timeout;
220         }
221 
222         rval.exitStatus = p.wait;
223         rval.output = output.data;
224     } catch (Exception e) {
225         logger.warning(e.msg).collectException;
226         rval.status = RunResult.Status.error;
227     }
228 
229     return rval;
230 }
231 
232 private:
233 
234 struct RunResult {
235     enum Status {
236         /// the test command successfully executed.
237         normal,
238         /// Something happend when the test command executed thus the result should not be used.
239         error,
240         /// The test command timed out.
241         timeout,
242     }
243 
244     Status status;
245     ///
246     int exitStatus;
247     ///
248     DrainElement[] output;
249 }
250 
251 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) {
252     import core.sys.posix.sys.stat;
253     import std.file : getAttributes, setAttributes, thisExePath;
254     import std.stdio : File;
255     import std.path : baseName;
256     import std.conv : to;
257 
258     immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh";
259 
260     File(fname, "w").writeln(`#!/bin/bash
261 echo $1
262 if [[ "$3" = "timeout" ]]; then
263     sleep 10m
264 fi
265 exit $2`);
266     setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH);
267     return fname;
268 }
269 
270 version (unittest) {
271     import core.time : dur;
272     import std.algorithm : count;
273     import std.array : array, empty;
274     import std.file : remove, exists;
275     import std.string : strip;
276 }
277 
278 @("shall collect the result from the test commands when running them")
279 unittest {
280     immutable script = makeUnittestScript("script_");
281     scope (exit)
282         () {
283         if (exists(script))
284             remove(script);
285     }();
286 
287     auto runner = TestRunner.make(0);
288     runner.put([script, "foo", "0"].ShellCommand);
289     runner.put([script, "foo", "0"].ShellCommand);
290     auto res = runner.run(5.dur!"seconds");
291 
292     res.output.filter!(a => !a.empty).count.shouldEqual(2);
293     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
294     res.status.shouldEqual(TestResult.Status.passed);
295 }
296 
297 @("shall set the status to failed when one of test command fail")
298 unittest {
299     immutable script = makeUnittestScript("script_");
300     scope (exit)
301         () {
302         if (exists(script))
303             remove(script);
304     }();
305 
306     auto runner = TestRunner.make(0);
307     runner.put([script, "foo", "0"].ShellCommand);
308     runner.put([script, "foo", "1"].ShellCommand);
309     auto res = runner.run(5.dur!"seconds");
310 
311     res.output.filter!(a => !a.empty).count.shouldEqual(2);
312     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
313     res.status.shouldEqual(TestResult.Status.failed);
314 }
315 
316 @("shall set the status to timeout when one of the tests commands reach the timeout limit")
317 unittest {
318     immutable script = makeUnittestScript("script_");
319     scope (exit)
320         () {
321         if (exists(script))
322             remove(script);
323     }();
324 
325     auto runner = TestRunner.make(0);
326     runner.put([script, "foo", "0"].ShellCommand);
327     runner.put([script, "foo", "0", "timeout"].ShellCommand);
328     auto res = runner.run(1.dur!"seconds");
329 
330     res.output.filter!(a => !a.empty).count.shouldEqual(1);
331     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1);
332     res.status.shouldEqual(TestResult.Status.timeout);
333 }