1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 A runner that can execute all the users test commands. These can be either 11 manually specified or automatically detected. 12 */ 13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 14 15 import logger = std.experimental.logger; 16 import std.algorithm : filter; 17 import std.array : appender, Appender, empty; 18 import std.datetime : Duration; 19 import std.exception : collectException; 20 import std.parallelism : TaskPool, Task, task; 21 22 import process; 23 24 import dextool.plugin.mutate.type : ShellCommand; 25 import dextool.type; 26 27 version (unittest) { 28 import unit_threaded.assertions; 29 } 30 31 @safe: 32 33 struct TestRunner { 34 private { 35 alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]); 36 TaskPool pool; 37 Duration timeout_; 38 } 39 40 /// Commands that execute the test cases. 41 ShellCommand[] commands; 42 43 /// Environment to set when executing either binaries or the command. 44 string[string] env; 45 46 static auto make() { 47 auto pool = new TaskPool; 48 pool.isDaemon = true; 49 return TestRunner(pool); 50 } 51 52 ~this() { 53 pool.stop; 54 } 55 56 bool empty() @safe pure nothrow const @nogc { 57 return commands.length == 0; 58 } 59 60 void timeout(Duration timeout) pure nothrow @nogc { 61 this.timeout_ = timeout; 62 } 63 64 void put(ShellCommand sh) pure nothrow { 65 commands ~= sh; 66 } 67 68 void put(ShellCommand[] sh) pure nothrow { 69 commands ~= sh; 70 } 71 72 TestResult run() { 73 return this.run(timeout_); 74 } 75 76 TestResult run(Duration timeout) { 77 import core.thread : Thread; 78 import core.time : dur; 79 import std.range : enumerate; 80 81 static TestTask* findDone(ref TestTask*[] tasks) { 82 foreach (t; tasks.enumerate.filter!(a => a.value.done)) { 83 tasks[t.index] = tasks[$ - 1]; 84 tasks = tasks[0 .. $ - 1]; 85 return t.value; 86 } 87 return null; 88 } 89 90 static void processDone(TestTask* t, ref TestResult result, 91 ref Appender!(DrainElement[]) output) { 92 auto res = t.yieldForce; 93 final switch (res.status) { 94 case RunResult.Status.normal: 95 if (result.status == TestResult.Status.passed && res.exitStatus != 0) { 96 result.status = TestResult.Status.failed; 97 } 98 output.put(res.output); 99 break; 100 case RunResult.Status.timeout: 101 result.status = TestResult.Status.timeout; 102 break; 103 case RunResult.Status.error: 104 result.status = TestResult.Status.error; 105 break; 106 } 107 } 108 109 TestTask*[] tasks = startTests(timeout); 110 TestResult rval; 111 auto output = appender!(DrainElement[])(); 112 while (!tasks.empty) { 113 auto t = findDone(tasks); 114 if (t !is null) { 115 processDone(t, rval, output); 116 } 117 () @trusted { Thread.sleep(50.dur!"msecs"); }(); 118 } 119 120 rval.output = output.data; 121 return rval; 122 } 123 124 private auto startTests(Duration timeout) @trusted { 125 auto tasks = appender!(TestTask*[])(); 126 127 foreach (c; commands) { 128 auto t = task!spawnRunTest(c.value, timeout, env); 129 tasks.put(t); 130 pool.put(t); 131 } 132 return tasks.data; 133 } 134 } 135 136 /// The result of running the tests. 137 struct TestResult { 138 enum Status { 139 /// All test commands exited with exit status zero. 140 passed, 141 /// At least one test command indicated that an error where found by exit status not zero. 142 failed, 143 /// At least one test command timed out. 144 timeout, 145 /// Something happend when the test command executed thus the result should not be used. 146 error, 147 } 148 149 Status status; 150 151 /// Output from all the test binaries and command. 152 DrainElement[] output; 153 } 154 155 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) nothrow { 156 import std.algorithm : copy; 157 static import std.process; 158 159 RunResult rval; 160 161 try { 162 auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).raii; 163 auto output = appender!(DrainElement[])(); 164 p.drain.copy(output); 165 166 if (p.timeoutTriggered) { 167 rval.status = RunResult.Status.timeout; 168 } 169 170 rval.exitStatus = p.wait; 171 rval.output = output.data; 172 } catch (Exception e) { 173 logger.warning(e.msg).collectException; 174 rval.status = RunResult.Status.error; 175 } 176 177 return rval; 178 } 179 180 private: 181 182 struct RunResult { 183 enum Status { 184 /// the test command successfully executed. 185 normal, 186 /// Something happend when the test command executed thus the result should not be used. 187 error, 188 /// The test command timed out. 189 timeout, 190 } 191 192 Status status; 193 /// 194 int exitStatus; 195 /// 196 DrainElement[] output; 197 } 198 199 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) { 200 import core.sys.posix.sys.stat; 201 import std.file : getAttributes, setAttributes, thisExePath; 202 import std.stdio : File; 203 import std.path : baseName; 204 import std.conv : to; 205 206 immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh"; 207 208 File(fname, "w").writeln(`#!/bin/bash 209 echo $1 210 if [[ "$3" = "timeout" ]]; then 211 sleep 10m 212 fi 213 exit $2`); 214 setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH); 215 return fname; 216 } 217 218 version (unittest) { 219 import core.time : dur; 220 import std.algorithm : count; 221 import std.array : array, empty; 222 import std.file : remove, exists; 223 import std.string : strip; 224 } 225 226 @("shall collect the result from the test commands when running them") 227 unittest { 228 immutable script = makeUnittestScript("script_"); 229 scope (exit) 230 () { 231 if (exists(script)) 232 remove(script); 233 }(); 234 235 auto runner = TestRunner.make; 236 runner.put([script, "foo", "0"].ShellCommand); 237 runner.put([script, "foo", "0"].ShellCommand); 238 auto res = runner.run(5.dur!"seconds"); 239 240 res.output.filter!(a => !a.empty).count.shouldEqual(2); 241 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 242 res.status.shouldEqual(TestResult.Status.passed); 243 } 244 245 @("shall set the status to failed when one of test command fail") 246 unittest { 247 immutable script = makeUnittestScript("script_"); 248 scope (exit) 249 () { 250 if (exists(script)) 251 remove(script); 252 }(); 253 254 auto runner = TestRunner.make; 255 runner.put([script, "foo", "0"].ShellCommand); 256 runner.put([script, "foo", "1"].ShellCommand); 257 auto res = runner.run(5.dur!"seconds"); 258 259 res.output.filter!(a => !a.empty).count.shouldEqual(2); 260 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 261 res.status.shouldEqual(TestResult.Status.failed); 262 } 263 264 @("shall set the status to timeout when one of the tests commands reach the timeout limit") 265 unittest { 266 immutable script = makeUnittestScript("script_"); 267 scope (exit) 268 () { 269 if (exists(script)) 270 remove(script); 271 }(); 272 273 auto runner = TestRunner.make; 274 runner.put([script, "foo", "0"].ShellCommand); 275 runner.put([script, "foo", "0", "timeout"].ShellCommand); 276 auto res = runner.run(1.dur!"seconds"); 277 278 res.output.filter!(a => !a.empty).count.shouldEqual(1); 279 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1); 280 res.status.shouldEqual(TestResult.Status.timeout); 281 }