1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 
10 A runner that can execute all the users test commands. These can be either
11 manually specified or automatically detected.
12 */
13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
14 
15 import logger = std.experimental.logger;
16 import std.algorithm : filter;
17 import std.array : appender, Appender, empty;
18 import std.datetime : Duration;
19 import std.exception : collectException;
20 import std.parallelism : TaskPool, Task, task;
21 
22 import process;
23 
24 import dextool.plugin.mutate.type : ShellCommand;
25 import dextool.type;
26 
27 version (unittest) {
28     import unit_threaded.assertions;
29 }
30 
31 @safe:
32 
33 struct TestRunner {
34     private {
35         alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]);
36         TaskPool pool;
37         Duration timeout_;
38     }
39 
40     /// Commands that execute the test cases.
41     ShellCommand[] commands;
42 
43     /// Environment to set when executing either binaries or the command.
44     string[string] env;
45 
46     static auto make() {
47         auto pool = new TaskPool;
48         pool.isDaemon = true;
49         return TestRunner(pool);
50     }
51 
52     ~this() {
53         pool.stop;
54     }
55 
56     bool empty() @safe pure nothrow const @nogc {
57         return commands.length == 0;
58     }
59 
60     void timeout(Duration timeout) pure nothrow @nogc {
61         this.timeout_ = timeout;
62     }
63 
64     void put(ShellCommand sh) pure nothrow {
65         commands ~= sh;
66     }
67 
68     void put(ShellCommand[] sh) pure nothrow {
69         commands ~= sh;
70     }
71 
72     TestResult run() {
73         return this.run(timeout_);
74     }
75 
76     TestResult run(Duration timeout) {
77         import core.thread : Thread;
78         import core.time : dur;
79         import std.range : enumerate;
80 
81         static TestTask* findDone(ref TestTask*[] tasks) {
82             foreach (t; tasks.enumerate.filter!(a => a.value.done)) {
83                 tasks[t.index] = tasks[$ - 1];
84                 tasks = tasks[0 .. $ - 1];
85                 return t.value;
86             }
87             return null;
88         }
89 
90         static void processDone(TestTask* t, ref TestResult result,
91                 ref Appender!(DrainElement[]) output) {
92             auto res = t.yieldForce;
93             final switch (res.status) {
94             case RunResult.Status.normal:
95                 if (result.status == TestResult.Status.passed && res.exitStatus != 0) {
96                     result.status = TestResult.Status.failed;
97                 }
98                 output.put(res.output);
99                 break;
100             case RunResult.Status.timeout:
101                 result.status = TestResult.Status.timeout;
102                 break;
103             case RunResult.Status.error:
104                 result.status = TestResult.Status.error;
105                 break;
106             }
107         }
108 
109         TestTask*[] tasks = startTests(timeout);
110         TestResult rval;
111         auto output = appender!(DrainElement[])();
112         while (!tasks.empty) {
113             auto t = findDone(tasks);
114             if (t !is null) {
115                 processDone(t, rval, output);
116             }
117             () @trusted { Thread.sleep(50.dur!"msecs"); }();
118         }
119 
120         rval.output = output.data;
121         return rval;
122     }
123 
124     private auto startTests(Duration timeout) @trusted {
125         auto tasks = appender!(TestTask*[])();
126 
127         foreach (c; commands) {
128             auto t = task!spawnRunTest(c.value, timeout, env);
129             tasks.put(t);
130             pool.put(t);
131         }
132         return tasks.data;
133     }
134 }
135 
136 /// The result of running the tests.
137 struct TestResult {
138     enum Status {
139         /// All test commands exited with exit status zero.
140         passed,
141         /// At least one test command indicated that an error where found by exit status not zero.
142         failed,
143         /// At least one test command timed out.
144         timeout,
145         /// Something happend when the test command executed thus the result should not be used.
146         error,
147     }
148 
149     Status status;
150 
151     /// Output from all the test binaries and command.
152     DrainElement[] output;
153 }
154 
155 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) nothrow {
156     import std.algorithm : copy;
157     static import std.process;
158 
159     RunResult rval;
160 
161     try {
162         auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).raii;
163         auto output = appender!(DrainElement[])();
164         p.drain.copy(output);
165 
166         if (p.timeoutTriggered) {
167             rval.status = RunResult.Status.timeout;
168         }
169 
170         rval.exitStatus = p.wait;
171         rval.output = output.data;
172     } catch (Exception e) {
173         logger.warning(e.msg).collectException;
174         rval.status = RunResult.Status.error;
175     }
176 
177     return rval;
178 }
179 
180 private:
181 
182 struct RunResult {
183     enum Status {
184         /// the test command successfully executed.
185         normal,
186         /// Something happend when the test command executed thus the result should not be used.
187         error,
188         /// The test command timed out.
189         timeout,
190     }
191 
192     Status status;
193     ///
194     int exitStatus;
195     ///
196     DrainElement[] output;
197 }
198 
199 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) {
200     import core.sys.posix.sys.stat;
201     import std.file : getAttributes, setAttributes, thisExePath;
202     import std.stdio : File;
203     import std.path : baseName;
204     import std.conv : to;
205 
206     immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh";
207 
208     File(fname, "w").writeln(`#!/bin/bash
209 echo $1
210 if [[ "$3" = "timeout" ]]; then
211     sleep 10m
212 fi
213 exit $2`);
214     setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH);
215     return fname;
216 }
217 
218 version (unittest) {
219     import core.time : dur;
220     import std.algorithm : count;
221     import std.array : array, empty;
222     import std.file : remove, exists;
223     import std.string : strip;
224 }
225 
226 @("shall collect the result from the test commands when running them")
227 unittest {
228     immutable script = makeUnittestScript("script_");
229     scope (exit)
230         () {
231         if (exists(script))
232             remove(script);
233     }();
234 
235     auto runner = TestRunner.make;
236     runner.put([script, "foo", "0"].ShellCommand);
237     runner.put([script, "foo", "0"].ShellCommand);
238     auto res = runner.run(5.dur!"seconds");
239 
240     res.output.filter!(a => !a.empty).count.shouldEqual(2);
241     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
242     res.status.shouldEqual(TestResult.Status.passed);
243 }
244 
245 @("shall set the status to failed when one of test command fail")
246 unittest {
247     immutable script = makeUnittestScript("script_");
248     scope (exit)
249         () {
250         if (exists(script))
251             remove(script);
252     }();
253 
254     auto runner = TestRunner.make;
255     runner.put([script, "foo", "0"].ShellCommand);
256     runner.put([script, "foo", "1"].ShellCommand);
257     auto res = runner.run(5.dur!"seconds");
258 
259     res.output.filter!(a => !a.empty).count.shouldEqual(2);
260     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2);
261     res.status.shouldEqual(TestResult.Status.failed);
262 }
263 
264 @("shall set the status to timeout when one of the tests commands reach the timeout limit")
265 unittest {
266     immutable script = makeUnittestScript("script_");
267     scope (exit)
268         () {
269         if (exists(script))
270             remove(script);
271     }();
272 
273     auto runner = TestRunner.make;
274     runner.put([script, "foo", "0"].ShellCommand);
275     runner.put([script, "foo", "0", "timeout"].ShellCommand);
276     auto res = runner.run(1.dur!"seconds");
277 
278     res.output.filter!(a => !a.empty).count.shouldEqual(1);
279     res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1);
280     res.status.shouldEqual(TestResult.Status.timeout);
281 }