ci.sh (6814B)
1 #!/usr/bin/env bash 2 3 4 stderr() { 5 echo "$@" 1>&2 6 } 7 8 usage() { 9 b=$(basename "$0") 10 echo $b: ERROR: "$@" 1>&2 11 12 cat 1>&2 <<EOF 13 14 DESCRIPTION 15 16 $(basename "$0") is the script to run continuous integration commands for 17 go-toml on unix. 18 19 Requires Go and Git to be available in the PATH. Expects to be ran from the 20 root of go-toml's Git repository. 21 22 USAGE 23 24 $b COMMAND [OPTIONS...] 25 26 COMMANDS 27 28 benchmark [OPTIONS...] [BRANCH] 29 30 Run benchmarks. 31 32 ARGUMENTS 33 34 BRANCH Optional. Defines which Git branch to use when running 35 benchmarks. 36 37 OPTIONS 38 39 -d Compare benchmarks of HEAD with BRANCH using benchstats. In 40 this form the BRANCH argument is required. 41 42 -a Compare benchmarks of HEAD against go-toml v1 and 43 BurntSushi/toml. 44 45 -html When used with -a, emits the output as HTML, ready to be 46 embedded in the README. 47 48 coverage [OPTIONS...] [BRANCH] 49 50 Generates code coverage. 51 52 ARGUMENTS 53 54 BRANCH Optional. Defines which Git branch to use when reporting 55 coverage. Defaults to HEAD. 56 57 OPTIONS 58 59 -d Compare coverage of HEAD with the one of BRANCH. In this form, 60 the BRANCH argument is required. Exit code is non-zero when 61 coverage percentage decreased. 62 EOF 63 exit 1 64 } 65 66 cover() { 67 branch="${1}" 68 dir="$(mktemp -d)" 69 70 stderr "Executing coverage for ${branch} at ${dir}" 71 72 if [ "${branch}" = "HEAD" ]; then 73 cp -r . "${dir}/" 74 else 75 git worktree add "$dir" "$branch" 76 fi 77 78 pushd "$dir" 79 go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./... 80 cat coverage.out.tmp | grep -v fuzz | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out 81 go tool cover -func=coverage.out 82 popd 83 84 if [ "${branch}" != "HEAD" ]; then 85 git worktree remove --force "$dir" 86 fi 87 } 88 89 coverage() { 90 case "$1" in 91 -d) 92 shift 93 target="${1?Need to provide a target branch argument}" 94 95 output_dir="$(mktemp -d)" 96 target_out="${output_dir}/target.txt" 97 head_out="${output_dir}/head.txt" 98 99 cover "${target}" > "${target_out}" 100 cover "HEAD" > "${head_out}" 101 102 cat "${target_out}" 103 cat "${head_out}" 104 105 echo "" 106 107 target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')" 108 head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')" 109 echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" 110 111 delta_pct=$(echo "$head_pct - $target_pct" | bc -l) 112 echo "Delta: ${delta_pct}" 113 114 if [[ $delta_pct = \-* ]]; then 115 echo "Regression!"; 116 117 target_diff="${output_dir}/target.diff.txt" 118 head_diff="${output_dir}/head.diff.txt" 119 cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}" 120 cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}" 121 122 diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}" 123 return 1 124 fi 125 return 0 126 ;; 127 esac 128 129 cover "${1-HEAD}" 130 } 131 132 bench() { 133 branch="${1}" 134 out="${2}" 135 replace="${3}" 136 dir="$(mktemp -d)" 137 138 stderr "Executing benchmark for ${branch} at ${dir}" 139 140 if [ "${branch}" = "HEAD" ]; then 141 cp -r . "${dir}/" 142 else 143 git worktree add "$dir" "$branch" 144 fi 145 146 pushd "$dir" 147 148 if [ "${replace}" != "" ]; then 149 find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \; 150 go get "${replace}" 151 fi 152 153 export GOMAXPROCS=2 154 nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}" 155 popd 156 157 if [ "${branch}" != "HEAD" ]; then 158 git worktree remove --force "$dir" 159 fi 160 } 161 162 fmktemp() { 163 if mktemp --version|grep GNU >/dev/null; then 164 mktemp --suffix=-$1; 165 else 166 mktemp -t $1; 167 fi 168 } 169 170 benchstathtml() { 171 python3 - $1 <<'EOF' 172 import sys 173 174 lines = [] 175 stop = False 176 177 with open(sys.argv[1]) as f: 178 for line in f.readlines(): 179 line = line.strip() 180 if line == "": 181 stop = True 182 if not stop: 183 lines.append(line.split(',')) 184 185 results = [] 186 for line in reversed(lines[1:]): 187 v2 = float(line[1]) 188 results.append([ 189 line[0].replace("-32", ""), 190 "%.1fx" % (float(line[3])/v2), # v1 191 "%.1fx" % (float(line[5])/v2), # bs 192 ]) 193 # move geomean to the end 194 results.append(results[0]) 195 del results[0] 196 197 198 def printtable(data): 199 print(""" 200 <table> 201 <thead> 202 <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr> 203 </thead> 204 <tbody>""") 205 206 for r in data: 207 print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r)) 208 209 print(""" </tbody> 210 </table>""") 211 212 213 def match(x): 214 return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0] 215 216 above = [x for x in results if match(x)] 217 below = [x for x in results if not match(x)] 218 219 printtable(above) 220 print("<details><summary>See more</summary>") 221 print("""<p>The table above has the results of the most common use-cases. The table below 222 contains the results of all benchmarks, including unrealistic ones. It is 223 provided for completeness.</p>""") 224 printtable(below) 225 print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>') 226 print("</details>") 227 228 EOF 229 } 230 231 benchmark() { 232 case "$1" in 233 -d) 234 shift 235 target="${1?Need to provide a target branch argument}" 236 237 old=`fmktemp ${target}` 238 bench "${target}" "${old}" 239 240 new=`fmktemp HEAD` 241 bench HEAD "${new}" 242 243 benchstat "${old}" "${new}" 244 return 0 245 ;; 246 -a) 247 shift 248 249 v2stats=`fmktemp go-toml-v2` 250 bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2" 251 v1stats=`fmktemp go-toml-v1` 252 bench HEAD "${v1stats}" "github.com/pelletier/go-toml" 253 bsstats=`fmktemp bs-toml` 254 bench HEAD "${bsstats}" "github.com/BurntSushi/toml" 255 256 cp "${v2stats}" go-toml-v2.txt 257 cp "${v1stats}" go-toml-v1.txt 258 cp "${bsstats}" bs-toml.txt 259 260 if [ "$1" = "-html" ]; then 261 tmpcsv=`fmktemp csv` 262 benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv 263 benchstathtml $tmpcsv 264 else 265 benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt 266 fi 267 268 rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt 269 return $? 270 esac 271 272 bench "${1-HEAD}" `mktemp` 273 } 274 275 case "$1" in 276 coverage) shift; coverage $@;; 277 benchmark) shift; benchmark $@;; 278 *) usage "bad argument $1";; 279 esac