1 /** 2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 */ 10 module dextool_test.golden; 11 12 import scriptlike; 13 14 struct BuildCompare { 15 import std.typecons : Yes, No; 16 17 private { 18 string outdir_; 19 20 Flag!"sortLines" sort_lines = No.sortLines; 21 Flag!"skipComments" skip_comments = Yes.skipComments; 22 23 /// if the output from running the command should be yapped via scriptlike 24 bool yap_output = true; 25 26 /// Throw an exception if a compare failes 27 bool throw_on_failed_compare_ = true; 28 29 GoldResult[] gold_results; 30 } 31 32 private static struct GoldResult { 33 Path gold; 34 Path result; 35 } 36 37 this(string outdir) { 38 this.outdir_ = outdir; 39 } 40 41 Path outdir() { 42 return Path(outdir_); 43 } 44 45 auto addCompare(Path gold, string result_file) { 46 this.gold_results ~= GoldResult(gold, buildPath(outdir_, result_file).Path); 47 return this; 48 } 49 50 auto sortLines(bool v) { 51 sort_lines = cast(Flag!"sortLines") v; 52 return this; 53 } 54 55 auto skipComments(bool v) { 56 skip_comments = cast(Flag!"skipComments") v; 57 return this; 58 } 59 60 auto throwOnFailure(bool v) { 61 this.throw_on_failed_compare_ = v; 62 return this; 63 } 64 65 auto run() { 66 CompareResult res; 67 68 foreach (const ref gr; gold_results) { 69 res = compare(gr.gold, gr.result, sort_lines, skip_comments); 70 if (!res.status) { 71 break; 72 } 73 } 74 75 if (!res.status && yap_output) { 76 File(nextFreeLogfile(outdir_), "w").writef("%s", res); 77 } 78 79 if (!res.status && throw_on_failed_compare_) { 80 throw new ErrorLevelException(1, res.errorMsg); 81 } 82 83 return res; 84 } 85 } 86 87 /** Sorted compare of gold and result. 88 * 89 * max_diff is arbitrarily chosen to 5. 90 * The purpose is to limit the amount of text that is dumped. 91 * The reasoning is that it is better to give more than one line as feedback. 92 */ 93 private CompareResult compare(const Path gold, const Path result, 94 Flag!"sortLines" sortLines, Flag!"skipComments" skipComments) { 95 import std.format : format; 96 import std.stdio : File; 97 import dextool_test.utils : escapePath, removeJunk; 98 99 CompareResult res; 100 101 res.msg ~= "Comparing gold:" ~ gold.raw; 102 res.msg ~= " result:" ~ result.raw; 103 104 File goldf; 105 File resultf; 106 107 try { 108 goldf = File(gold.escapePath); 109 resultf = File(result.escapePath); 110 } 111 catch (ErrnoException ex) { 112 res.errorMsg = ex.msg; 113 res.status = false; 114 return res; 115 } 116 117 auto maybeSort(T)(T lines) { 118 import std.array : array; 119 import std.algorithm : sort; 120 121 if (sortLines) { 122 return sort!((a, b) => a[1] < b[1])(lines.array()).array(); 123 } 124 125 return lines.array(); 126 } 127 128 bool diff_detected = false; 129 immutable max_diff = 5; 130 int accumulated_diff; 131 // dfmt off 132 foreach (g, r; 133 lockstep(maybeSort(goldf 134 .byLineCopy() 135 .enumerate 136 .removeJunk(skipComments)), 137 maybeSort(resultf 138 .byLineCopy() 139 .enumerate 140 .removeJunk(skipComments)) 141 )) { 142 if (g[1] != r[1] && accumulated_diff < max_diff) { 143 // +1 of index because editors start counting lines from 1 144 res.lineDiff ~= format("Line %s gold: %s", g[0] + 1, g[1]); 145 res.lineDiff ~= format("Line %s out: %s", r[0] + 1, r[1]); 146 diff_detected = true; 147 ++accumulated_diff; 148 } 149 } 150 // dfmt on 151 152 res.status = !diff_detected; 153 154 if (diff_detected) { 155 res.errorMsg = "Output is different from reference file (gold): " ~ gold.escapePath; 156 } 157 158 return res; 159 } 160 161 struct CompareResult { 162 import std.ascii : newline; 163 import std.format : FormatSpec; 164 165 // true if the golden file and result are _equal_. 166 bool status; 167 168 string errorMsg; 169 string[] msg; 170 string[] lineDiff; 171 172 void toString(Writer, Char)(scope Writer w, FormatSpec!Char fmt) const { 173 import std.algorithm : each; 174 import std.format : formattedWrite; 175 import std.range.primitives : put; 176 177 formattedWrite(w, "status: %s\n", status); 178 179 if (errorMsg.length) { 180 put(w, errorMsg); 181 put(w, newline); 182 } 183 184 this.msg.each!((a) { put(w, a); put(w, newline); }); 185 this.lineDiff.each!((a) { put(w, a); put(w, newline); }); 186 } 187 188 string toString() @safe pure const { 189 import std.exception : assumeUnique; 190 import std.format : FormatSpec; 191 192 char[] buf; 193 buf.reserve(100); 194 auto fmt = FormatSpec!char("%s"); 195 toString((const(char)[] s) { buf ~= s; }, fmt); 196 auto trustedUnique(T)(T t) @trusted { 197 return assumeUnique(t); 198 } 199 200 return trustedUnique(buf); 201 } 202 } 203 204 private auto nextFreeLogfile(string outdir) { 205 import std.file : exists; 206 import std.path : baseName; 207 import std..string : format; 208 209 int idx; 210 string f; 211 do { 212 f = buildPath(outdir, format("run_compare%s.log", idx)); 213 ++idx; 214 } 215 while (exists(f)); 216 217 return f; 218 }