1 #!/bin/sh 2 # 3 # Benchmarking harness for json-c 4 # 5 # Use this to compare the behavior of two different versions of the library, 6 # such as json-c-0.14 release vs master, master vs local changes, etc... 7 # 8 9 set -e 10 11 trap 'echo "FAILED!"' EXIT 12 13 RUNDIR=$(dirname "$0") 14 RUNDIR=$(cd "$RUNDIR" && pwd) 15 16 TOP=$(cd "$RUNDIR/.." && pwd) 17 18 usage() 19 { 20 exitval=$1 21 errmsg=$2 22 if [ $exitval -ne 0 ] ; then 23 exec 1>&2 24 fi 25 if [ ! -z "$errmsg" ] ; then 26 echo "ERROR: $errmsg" 1>&2 27 fi 28 cat <<EOF 29 Usage: $0 [-h] [-v] [--build] [--run] [--compare] ...XAX... 30 EOF 31 32 exit $extival 33 } 34 35 before_arg= 36 after_arg= 37 do_all=1 38 do_build=0 39 do_run=0 40 do_compare=0 41 42 while [ $# -gt 0 ] ; do 43 case "$1" in 44 --before) 45 before_arg=$2 46 shift 47 ;; 48 --after) 49 after_arg=$2 50 shift 51 ;; 52 --build) 53 do_all=0 54 do_build=1 55 ;; 56 --run) 57 do_all=0 58 do_run=1 59 ;; 60 --compare) 61 do_all=0 62 do_compare=1 63 ;; 64 -h) 65 usage 0 "" 66 ;; 67 -v) 68 set -x 69 ;; 70 *) 71 usage 1 "Unknown args: $*" 72 ;; 73 esac 74 shift 75 done 76 77 WORK="${RUNDIR}/work" 78 mkdir -p "${WORK}" 79 80 DATA="${RUNDIR}/data" 81 mkdir -p "${DATA}" 82 83 for file in citm_catalog.json twitter.json canada.json ; do 84 if [ ! -r "${DATA}/${file}" ] ; then 85 echo "Fetching ${file} from github.com/mloskot/json_benchmark" 86 URL="https://github.com/mloskot/json_benchmark/raw/master/data/${file}" 87 curl -s -L -o "${DATA}/${file}" "$URL" 88 fi 89 done 90 echo 91 92 # Identify "after" commit hash, in order of preference 93 if [ ! -z "$after_arg" -a -d "$after_arg" ] ; then 94 # Use provided directory 95 after_src_dir="$after_arg" 96 after_commit= 97 echo "Using provided directory [$after_arg] as 'after'" 98 else 99 _commit= 100 if [ ! -z "$after_arg" ] ; then 101 # Use provided commit hash 102 _commit=$(git rev-parse --verify "$after_arg") 103 fi 104 if [ ! -z "$_commit" ] ;then 105 after_src_dir= # i.e. current tree 106 after_commit="$_commit" 107 echo "Using provided commit [$after_arg => $_commit] as 'after'" 108 else 109 # Local changes in current working directory 110 # ${cur_branch} 111 after_src_dir=$TOP 112 after_commit= 113 echo "Using local changes in $TOP as 'after'" 114 fi 115 fi 116 117 # Identify "before" commit hash, in order of preference 118 if [ ! -z "$before_arg" -a -d "$before_arg" ] ; then 119 # Use provided directory 120 before_src_dir="$before_arg" 121 before_commit= 122 echo "Using provided directory [$before_arg] as 'before'" 123 else 124 _commit= 125 if [ ! -z "$before_arg" ] ; then 126 # Use provided commit hash 127 _commit=$(git rev-parse --verify "$before_arg") 128 fi 129 if [ ! -z "$_commit" ] ;then 130 before_src_dir= # i.e. current tree 131 before_commit="$_commit" 132 echo "Using provided commit [$before_arg => $_commit] as 'before'" 133 else 134 # Use origin/${cur_branch}, if different from ${after_commit} 135 _cur_branch=$(git rev-parse --abbrev-ref HEAD) 136 _commit= 137 if [ ! -z "${_cur_branch}" ] ; then 138 _commit=$(git rev-parse --verify "origin/${_cur_branch}") 139 echo "Using origin/${_cur_branch} [$_commit] as 'before'" 140 fi 141 if [ "$_commit" = "${after_commit}" ] ; then 142 _commit= 143 fi 144 fi 145 146 if [ ! -z "$_commit" ] ; then 147 before_src_dir= # i.e. current tree 148 before_commit="$_commit" 149 else 150 # Use previous release 151 before_src_dir= # i.e. current tree 152 before_commit="$(git tag | sort | tail -1)" 153 echo "Using previous release [$before_commit] as 'before'" 154 fi 155 fi 156 157 echo 158 159 compile_benchmark() 160 { 161 local bname=$1 162 local src_dir="$2" 163 local src_commit="$3" 164 165 local build_dir="${WORK}/$bname/build" 166 local inst_dir="${WORK}/$bname/install" 167 local bench_dir="${WORK}/$bname/bench" 168 169 echo 170 echo "=========== $bname ===========" 171 echo 172 173 mkdir -p "${build_dir}" 174 mkdir -p "${inst_dir}" 175 mkdir -p "${bench_dir}" 176 177 if [ ! -z "$src_commit" ] ; then 178 # Resolve the short hash, tag or branch name to full hash 179 src_commit=$(git rev-parse $src_commit) 180 fi 181 182 # No src dir specified, clone and checkout $src_commit 183 if [ -z "$src_dir" ] ; then 184 src_dir="${WORK}/$bname/src" 185 echo "=== Using sources in $src_dir" 186 mkdir -p "$src_dir" 187 at_commit=$(git --git-dir="$src_dir/.git" rev-parse HEAD 2> /dev/null || true) 188 echo "at_commit: $at_commit" 189 if [ -z "$at_commit" ] ; then 190 # Assume it's an empty dir 191 git clone -n "$TOP" "$src_dir" 192 fi 193 git -C "$src_dir" --git-dir="$src_dir/.git" checkout "$src_commit" 194 fi 195 # else, use the provided $src_dir 196 197 if [ -e "${src_dir}/CMakeLists.txt" ] ; then 198 cd "${build_dir}" 199 cmake -DCMAKE_INSTALL_PREFIX="${inst_dir}" "${src_dir}" 200 else 201 # Old versions of json-c used automake/autoconf 202 cd "${src_dir}" 203 sh autogen.sh # always run it, configure doesn't always work 204 cd "${build_dir}" 205 "${src_dir}/configure" --prefix="${inst_dir}" 206 fi 207 make all install 208 209 cd "${bench_dir}" 210 cmake -DCMAKE_PREFIX_PATH="${inst_dir}" "${TOP}/apps" 211 make all 212 } 213 214 # XXX TODO: name "after" and "before" uniquely using the dir & commit 215 216 if [ $do_all -ne 0 -o $do_build -ne 0 ] ; then 217 sleep 5 # Wait slightly, to allow the human to read the message 218 # about what exactly we're doing to benchmark. 219 compile_benchmark "after" "${after_src_dir}" "${after_commit}" 220 compile_benchmark "before" "${before_src_dir}" "${before_commit}" 221 fi 222 223 run_benchmark() 224 { 225 local bname=$1 226 local inst_dir="${WORK}/$bname/install" 227 local bench_dir="${WORK}/$bname/bench" 228 229 local INPUT=${DATA}/canada.json 230 231 cd "${bench_dir}" 232 mkdir -p results 233 (time ./json_parse -n "${INPUT}") > results/basic_timing.out 2>&1 234 valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n "${INPUT}" 235 ms_print massif.out > results/ms_print.out 236 heaptrack -o heaptrack_out ./json_parse -n "${INPUT}" 237 heaptrack --analyze heaptrack_out.gz -H histogram.out > results/heaptrack.out 238 awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out > results/histogram2.out 239 240 # XXX stamp some info about what was built & run into ./results/. 241 242 echo "DONE with $bname" 243 } 244 245 if [ $do_all -ne 0 -o $do_run -ne 0 ] ; then 246 run_benchmark "after" 247 run_benchmark "before" 248 fi 249 250 if [ $do_compare -ne 0 ] ; then 251 # XXX this needs better analysis 252 cd "${WORK}" 253 diff -udr before/bench/results after/bench/results || true 254 else 255 echo "To compare results, run:" 256 echo "$0 --compare" 257 fi 258 259 trap '' EXIT 260 261 :<<=cut 262 263 Benchmarks to run: 264 265 * Parse JSON strings, of various sizes and characteristics 266 * Flags: STRICT vs. non-STRICT, validate UTF8 267 268 * Serialization time 269 * plain, spaces, pretty 270 271 * json_c_visit tests 272 * JSON pointer tests 273 274 Things to record and compare: 275 276 * Running time 277 * Peak memory usage 278 * Useful bytes vs. overhead for memory allocations 279 * Total number of allocations 280 * Average allocation size 281 * Log of all allocation sizes 282 283 =cut 284
This page was automatically generated by LXR 0.3.1. • OpenWrt