1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 10 A runner that can execute all the users test commands. These can be either 11 manually specified or automatically detected. 12 */ 13 module dextool.plugin.mutate.backend.test_mutant.test_cmd_runner; 14 15 import logger = std.experimental.logger; 16 import std.algorithm : filter; 17 import std.array : appender, Appender, empty; 18 import std.datetime : Duration, dur; 19 import std.exception : collectException; 20 import std.parallelism : TaskPool, Task, task; 21 22 import process; 23 24 import dextool.plugin.mutate.type : ShellCommand; 25 import dextool.type; 26 27 version (unittest) { 28 import unit_threaded.assertions; 29 } 30 31 @safe: 32 33 struct TestRunner { 34 private { 35 alias TestTask = Task!(spawnRunTest, string[], Duration, string[string]); 36 TaskPool pool; 37 Duration timeout_; 38 } 39 40 /// Commands that execute the test cases. 41 ShellCommand[] commands; 42 43 /// Environment to set when executing either binaries or the command. 44 string[string] env; 45 46 static auto make(int poolSize) { 47 return TestRunner(poolSize); 48 } 49 50 this(int poolSize_) { 51 this.poolSize(poolSize_); 52 } 53 54 ~this() { 55 pool.stop; 56 } 57 58 bool empty() @safe pure nothrow const @nogc { 59 return commands.length == 0; 60 } 61 62 void poolSize(const int s) @safe { 63 if (pool !is null) { 64 pool.stop; 65 } 66 if (s == 0) { 67 pool = new TaskPool; 68 } else { 69 pool = new TaskPool(s); 70 } 71 pool.isDaemon = true; 72 } 73 74 void timeout(Duration timeout) pure nothrow @nogc { 75 this.timeout_ = timeout; 76 } 77 78 void put(ShellCommand sh) pure nothrow { 79 commands ~= sh; 80 } 81 82 void put(ShellCommand[] sh) pure nothrow { 83 commands ~= sh; 84 } 85 86 TestResult run(string[string] localEnv = null) { 87 return this.run(timeout_, localEnv); 88 } 89 90 TestResult run(Duration timeout, string[string] localEnv = null) { 91 import core.thread : Thread; 92 import core.time : dur; 93 import std.range : enumerate; 94 95 static TestTask* findDone(ref TestTask*[] tasks) { 96 bool found; 97 size_t idx; 98 foreach (t; tasks.enumerate.filter!(a => a.value.done)) { 99 idx = t.index; 100 found = true; 101 break; 102 } 103 104 if (found) { 105 auto t = tasks[idx]; 106 tasks[idx] = tasks[$ - 1]; 107 tasks = tasks[0 .. $ - 1]; 108 return t; 109 } 110 return null; 111 } 112 113 static void processDone(TestTask* t, ref TestResult result, 114 ref Appender!(DrainElement[]) output) { 115 auto res = t.yieldForce; 116 final switch (res.status) { 117 case RunResult.Status.normal: 118 if (result.status == TestResult.Status.passed && res.exitStatus != 0) { 119 result.status = TestResult.Status.failed; 120 } 121 output.put(res.output); 122 break; 123 case RunResult.Status.timeout: 124 result.status = TestResult.Status.timeout; 125 break; 126 case RunResult.Status.error: 127 result.status = TestResult.Status.error; 128 break; 129 } 130 } 131 132 auto env_ = env; 133 foreach (kv; localEnv.byKeyValue) { 134 env_[kv.key] = kv.value; 135 } 136 137 TestTask*[] tasks = startTests(timeout, env_); 138 TestResult rval; 139 auto output = appender!(DrainElement[])(); 140 while (!tasks.empty) { 141 auto t = findDone(tasks); 142 if (t !is null) { 143 processDone(t, rval, output); 144 .destroy(t); 145 } 146 () @trusted { Thread.sleep(50.dur!"msecs"); }(); 147 } 148 149 rval.output = output.data; 150 return rval; 151 } 152 153 private auto startTests(Duration timeout, string[string] env) @trusted { 154 auto tasks = appender!(TestTask*[])(); 155 156 foreach (c; commands) { 157 auto t = task!spawnRunTest(c.value, timeout, env); 158 tasks.put(t); 159 pool.put(t); 160 } 161 return tasks.data; 162 } 163 } 164 165 /// The result of running the tests. 166 struct TestResult { 167 enum Status { 168 /// All test commands exited with exit status zero. 169 passed, 170 /// At least one test command indicated that an error where found by exit status not zero. 171 failed, 172 /// At least one test command timed out. 173 timeout, 174 /// Something happend when the test command executed thus the result should not be used. 175 error, 176 } 177 178 Status status; 179 180 /// Output from all the test binaries and command. 181 DrainElement[] output; 182 } 183 184 /// Finds all executables in a directory tree. 185 string[] findExecutables(AbsolutePath root) @trusted { 186 import core.sys.posix.sys.stat; 187 import std.file : getAttributes; 188 import std.file : dirEntries, SpanMode; 189 190 static bool isExecutable(string p) nothrow { 191 try { 192 return (getAttributes(p) & S_IXUSR) != 0; 193 } catch (Exception e) { 194 } 195 return false; 196 } 197 198 auto app = appender!(string[])(); 199 foreach (f; dirEntries(root, SpanMode.breadth).filter!(a => a.isFile) 200 .filter!(a => isExecutable(a.name))) { 201 app.put([f.name]); 202 } 203 204 return app.data; 205 } 206 207 RunResult spawnRunTest(string[] cmd, Duration timeout, string[string] env) @trusted nothrow { 208 import std.algorithm : copy; 209 static import std.process; 210 211 RunResult rval; 212 213 try { 214 auto p = pipeProcess(cmd, std.process.Redirect.all, env).sandbox.timeout(timeout).scopeKill; 215 auto output = appender!(DrainElement[])(); 216 p.process.drain(200.dur!"msecs").copy(output); 217 218 if (p.timeoutTriggered) { 219 rval.status = RunResult.Status.timeout; 220 } 221 222 rval.exitStatus = p.wait; 223 rval.output = output.data; 224 } catch (Exception e) { 225 logger.warning(e.msg).collectException; 226 rval.status = RunResult.Status.error; 227 } 228 229 return rval; 230 } 231 232 private: 233 234 struct RunResult { 235 enum Status { 236 /// the test command successfully executed. 237 normal, 238 /// Something happend when the test command executed thus the result should not be used. 239 error, 240 /// The test command timed out. 241 timeout, 242 } 243 244 Status status; 245 /// 246 int exitStatus; 247 /// 248 DrainElement[] output; 249 } 250 251 string makeUnittestScript(string script, string file = __FILE__, uint line = __LINE__) { 252 import core.sys.posix.sys.stat; 253 import std.file : getAttributes, setAttributes, thisExePath; 254 import std.stdio : File; 255 import std.path : baseName; 256 import std.conv : to; 257 258 immutable fname = thisExePath ~ "_" ~ script ~ file.baseName ~ line.to!string ~ ".sh"; 259 260 File(fname, "w").writeln(`#!/bin/bash 261 echo $1 262 if [[ "$3" = "timeout" ]]; then 263 sleep 10m 264 fi 265 exit $2`); 266 setAttributes(fname, getAttributes(fname) | S_IXUSR | S_IXGRP | S_IXOTH); 267 return fname; 268 } 269 270 version (unittest) { 271 import core.time : dur; 272 import std.algorithm : count; 273 import std.array : array, empty; 274 import std.file : remove, exists; 275 import std.string : strip; 276 } 277 278 @("shall collect the result from the test commands when running them") 279 unittest { 280 immutable script = makeUnittestScript("script_"); 281 scope (exit) 282 () { 283 if (exists(script)) 284 remove(script); 285 }(); 286 287 auto runner = TestRunner.make(0); 288 runner.put([script, "foo", "0"].ShellCommand); 289 runner.put([script, "foo", "0"].ShellCommand); 290 auto res = runner.run(5.dur!"seconds"); 291 292 res.output.filter!(a => !a.empty).count.shouldEqual(2); 293 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 294 res.status.shouldEqual(TestResult.Status.passed); 295 } 296 297 @("shall set the status to failed when one of test command fail") 298 unittest { 299 immutable script = makeUnittestScript("script_"); 300 scope (exit) 301 () { 302 if (exists(script)) 303 remove(script); 304 }(); 305 306 auto runner = TestRunner.make(0); 307 runner.put([script, "foo", "0"].ShellCommand); 308 runner.put([script, "foo", "1"].ShellCommand); 309 auto res = runner.run(5.dur!"seconds"); 310 311 res.output.filter!(a => !a.empty).count.shouldEqual(2); 312 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(2); 313 res.status.shouldEqual(TestResult.Status.failed); 314 } 315 316 @("shall set the status to timeout when one of the tests commands reach the timeout limit") 317 unittest { 318 immutable script = makeUnittestScript("script_"); 319 scope (exit) 320 () { 321 if (exists(script)) 322 remove(script); 323 }(); 324 325 auto runner = TestRunner.make(0); 326 runner.put([script, "foo", "0"].ShellCommand); 327 runner.put([script, "foo", "0", "timeout"].ShellCommand); 328 auto res = runner.run(1.dur!"seconds"); 329 330 res.output.filter!(a => !a.empty).count.shouldEqual(1); 331 res.output.filter!(a => a.byUTF8.array.strip == "foo").count.shouldEqual(1); 332 res.status.shouldEqual(TestResult.Status.timeout); 333 }