1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 A runner that can execute all the users test commands. These can be either 11 manually specified or automatically detected. 12 */ 13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 14 15 import logger = std.experimental.logger; 16 import std.algorithm : filter; 17 import std.array : appender, Appender, empty; 18 import std.datetime : Duration; 19 import std.exception : collectException; 20 import std.parallelism : TaskPool, Task, task; 21 22 import process; 23 24 import dextool.plugin.mutate.type : ShellCommand; 25 import dextool.type; 26 27 version (unittest) { 28 import unit_threaded.assertions; 29 } 30 31 @safe: 32 33 struct TestRunner { 34 private { 35 alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]); 36 TaskPool pool; 37 Duration timeout_; 38 } 39 40 /// Commands that execute the test cases. 41 ShellCommand[] commands; 42 43 /// Environment to set when executing either binaries or the command. 44 string[string] env; 45 46 static auto make(int poolSize) { 47 auto pool = () { 48 if (poolSize == 0) { 49 return new TaskPool; 50 } 51 return new TaskPool(poolSize); 52 }(); 53 pool.isDaemon = true; 54 return TestRunner(pool); 55 } 56 57 ~this() { 58 pool.stop; 59 } 60 61 bool empty() @safe pure nothrow const @nogc { 62 return commands.length == 0; 63 } 64 65 void timeout(Duration timeout) pure nothrow @nogc { 66 this.timeout_ = timeout; 67 } 68 69 void put(ShellCommand sh) pure nothrow { 70 commands ~= sh; 71 } 72 73 void put(ShellCommand[] sh) pure nothrow { 74 commands ~= sh; 75 } 76 77 TestResult run() { 78 return this.run(timeout_); 79 } 80 81 TestResult run(Duration timeout) { 82 import core.thread : Thread; 83 import core.time : dur; 84 import std.range : enumerate; 85 86 static TestTask* findDone(ref TestTask*[] tasks) { 87 foreach (t; tasks.enumerate.filter!(a => a.value.done)) { 88 tasks[t.index] = tasks[$ - 1]; 89 tasks = tasks[0 .. $ - 1]; 90 return t.value; 91 } 92 return null; 93 } 94 95 static void processDone(TestTask* t, ref TestResult result, 96 ref Appender!(DrainElement[]) output) { 97 auto res = t.yieldForce; 98 final switch (res.status) { 99 case RunResult.Status.normal: 100 if (result.status == TestResult.Status.passed && res.exitStatus != 0) { 101 result.status = TestResult.Status.failed; 102 } 103 output.put(res.output); 104 break; 105 case RunResult.Status.timeout: 106 result.status = TestResult.Status.timeout; 107 break; 108 case RunResult.Status.error: 109 result.status = TestResult.Status.error; 110 break; 111 } 112 } 113 114 TestTask*[] tasks = startTests(timeout); 115 TestResult rval; 116 auto output = appender!(DrainElement[])(); 117 while (!tasks.empty) { 118 auto t = findDone(tasks); 119 if (t !is null) { 120 processDone(t, rval, output); 121 } 122 () @trusted { Thread.sleep(50.dur!"msecs"); }(); 123 } 124 125 rval.output = output.data; 126 return rval; 127 } 128 129 private auto startTests(Duration timeout) @trusted { 130 auto tasks = appender!(TestTask*[])(); 131 132 foreach (c; commands) { 133 auto t = task!spawnRunTest(c.value, timeout, env); 134 tasks.put(t); 135 pool.put(t); 136 } 137 return tasks.data; 138 } 139 } 140 141 /// The result of running the tests. 142 struct TestResult { 143 enum Status { 144 /// All test commands exited with exit status zero. 145 passed, 146 /// At least one test command indicated that an error where found by exit status not zero. 147 failed, 148 /// At least one test command timed out. 149 timeout, 150 /// Something happend when the test command executed thus the result should not be used. 151 error, 152 } 153 154 Status status; 155 156 /// Output from all the test binaries and command. 157 DrainElement[] output; 158 } 159 160 /// Finds all executables in a directory tree. 161 string[] findExecutables(AbsolutePath root) @trusted { 162 import core.sys.posix.sys.stat; 163 import std.file : getAttributes; 164 import std.file : dirEntries, SpanMode; 165 166 static bool isExecutable(string p) nothrow { 167 try { 168 return (getAttributes(p) & S_IXUSR) != 0; 169 } catch (Exception e) { 170 } 171 return false; 172 } 173 174 auto app = appender!(string[])(); 175 foreach (f; dirEntries(root, SpanMode.breadth).filter!(a => a.isFile) 176 .filter!(a => isExecutable(a.name))) { 177 app.put([f.name]); 178 } 179 180 return app.data; 181 } 182 183 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) @trusted nothrow { 184 import std.algorithm : copy; 185 static import std.process; 186 187 RunResult rval; 188 189 try { 190 auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).scopeKill; 191 auto output = appender!(DrainElement[])(); 192 p.process.drain.copy(output); 193 194 if (p.timeoutTriggered) { 195 rval.status = RunResult.Status.timeout; 196 } 197 198 rval.exitStatus = p.wait; 199 rval.output = output.data; 200 } catch (Exception e) { 201 logger.warning(e.msg).collectException; 202 rval.status = RunResult.Status.error; 203 } 204 205 return rval; 206 } 207 208 private: 209 210 struct RunResult { 211 enum Status { 212 /// the test command successfully executed. 213 normal, 214 /// Something happend when the test command executed thus the result should not be used. 215 error, 216 /// The test command timed out. 217 timeout, 218 } 219 220 Status status; 221 /// 222 int exitStatus; 223 /// 224 DrainElement[] output; 225 } 226 227 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) { 228 import core.sys.posix.sys.stat; 229 import std.file : getAttributes, setAttributes, thisExePath; 230 import std.stdio : File; 231 import std.path : baseName; 232 import std.conv : to; 233 234 immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh"; 235 236 File(fname, "w").writeln(`#!/bin/bash 237 echo $1 238 if [[ "$3" = "timeout" ]]; then 239 sleep 10m 240 fi 241 exit $2`); 242 setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH); 243 return fname; 244 } 245 246 version (unittest) { 247 import core.time : dur; 248 import std.algorithm : count; 249 import std.array : array, empty; 250 import std.file : remove, exists; 251 import std.string : strip; 252 } 253 254 @("shall collect the result from the test commands when running them") 255 unittest { 256 immutable script = makeUnittestScript("script_"); 257 scope (exit) 258 () { 259 if (exists(script)) 260 remove(script); 261 }(); 262 263 auto runner = TestRunner.make(0); 264 runner.put([script, "foo", "0"].ShellCommand); 265 runner.put([script, "foo", "0"].ShellCommand); 266 auto res = runner.run(5.dur!"seconds"); 267 268 res.output.filter!(a => !a.empty).count.shouldEqual(2); 269 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 270 res.status.shouldEqual(TestResult.Status.passed); 271 } 272 273 @("shall set the status to failed when one of test command fail") 274 unittest { 275 immutable script = makeUnittestScript("script_"); 276 scope (exit) 277 () { 278 if (exists(script)) 279 remove(script); 280 }(); 281 282 auto runner = TestRunner.make(0); 283 runner.put([script, "foo", "0"].ShellCommand); 284 runner.put([script, "foo", "1"].ShellCommand); 285 auto res = runner.run(5.dur!"seconds"); 286 287 res.output.filter!(a => !a.empty).count.shouldEqual(2); 288 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 289 res.status.shouldEqual(TestResult.Status.failed); 290 } 291 292 @("shall set the status to timeout when one of the tests commands reach the timeout limit") 293 unittest { 294 immutable script = makeUnittestScript("script_"); 295 scope (exit) 296 () { 297 if (exists(script)) 298 remove(script); 299 }(); 300 301 auto runner = TestRunner.make(0); 302 runner.put([script, "foo", "0"].ShellCommand); 303 runner.put([script, "foo", "0", "timeout"].ShellCommand); 304 auto res = runner.run(1.dur!"seconds"); 305 306 res.output.filter!(a => !a.empty).count.shouldEqual(1); 307 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1); 308 res.status.shouldEqual(TestResult.Status.timeout); 309 }