[PATCH] proof of concept: systemtap/git differential code-coverage

[Brian Chrisman <brchrisman at gmail dot com>]

Using systemtap and a git-diff hook, I implemented a poor
man's differential code coverage tool (roughly extracted from
a tool I built for use internally).  The script
testsuite/testAndRun generates and launches a systemtap script
defining userspace probes and then executes 'make check'.
At the end of the test, a report is output with identifiers
showing whether a added/modified line is:
a) coverage not available (no DWARF point)
b) covered but not executed
c) covered and executed by some test

example output from a recent commit I moved up to HEAD:
...
if test -n ""; then mail  < systemtap.sum; fi
make[1]: Leaving directory `/root/systemtap/testsuite'
cover_na       tapsets.cxx    1112 :                   || fi->descriptor) // ppc opd (and also undefined symbols)
cover_na       tapsets.cxx    1113 :                 continue;
cover_exec     tapsets.cxx    1111 :               if (!null_die(&fi->die) // already handled in query_module_dwarf()
cover_exec     tapsets.cxx    1114 :               if (dw.function_name_matches_pattern(fi->name, function_str_val))
---
 testsuite/testAndCover |   72 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 72 insertions(+), 0 deletions(-)
 create mode 100755 testsuite/testAndCover

diff --git a/testsuite/testAndCover b/testsuite/testAndCover
new file mode 100755
index 0000000..00bdd96
--- /dev/null
+++ b/testsuite/testAndCover
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+mkdir -p coverage
+
+# probediff
+[[ $0 =~ probediff ]] &&
+{
+    read path oldFile oldHex oldMode newFile newHex newMode< <(echo $*)
+    [[ $path =~ \.cxx$ || $path =~ \.cpp$ || $path =~ \.h$ ]] || exit 0
+    (( $# == 7 )) && diff \
+        --new-line-format="process(\"$stapBinary\").statement(\"*@$(pwd)/$path:%dn\")%c'\012'" \
+        --old-line-format='' \
+        --unchanged-line-format='' \
+        $oldFile $newFile
+    exit 0
+}
+
+function coverageAvailable() { (( $(wc -l < coverage/diffCoverage.stp) > 1 )); }
+
+# generate stap script from git output, launch stap, run tests, killoff stap
+(
+    #export stapBinary=${PWD%/*}/stap
+    export stapBinary=/usr/local/bin/stap
+    export GIT_EXTERNAL_DIFF=${PWD}/probediff
+    git diff HEAD^                          |
+        sort -u                             |
+        tee coverage/all_lines              |
+        sed -e 's/"/\\"/g'                  |
+        xargs -n 1 -P 16 /usr/bin/stap -l   |
+        sort -u                             |
+        tee coverage/valid_lines            |
+        sed -e 's/"/\\"/g'                  |
+        xargs -n 1 -P 16 /usr/bin/stap -l   |
+        sort -u                             |
+        awk 'BEGIN { print "global tracker" } { print "probe " $0 " { if (tracker[" NR "]++ == 0) println(pp()) }" }' \
+            > coverage/diffCoverage.stp
+    
+    if coverageAvailable; then
+        stapPID=$(/usr/bin/stap -o coverage/covered_lines -F coverage/diffCoverage.stp)
+        [[ $? -ne 0 ]] || trap "kill $stapPID" EXIT
+    	[[ -n $stapPID ]] || exit 1
+    fi
+    #runtest systemtap.base/statement.exp
+    make check
+) || exit 1
+
+coverageAvailable || exit 0
+
+# report on coverage
+
+# all_lines will not have fully resolved function names, yet valid_lines and covered_lines will, so blow away everything up to '@'
+# binaries shouldn't be named with '@' symbols?
+# SEDDY!!!!
+function seddy() { sed -e 's/.*@//;' "$@" | sort -u ; }
+
+(
+    cd coverage
+    comm -23 <(seddy all_lines) <(seddy valid_lines) | 
+        tr '"):' '   ' |
+        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_na $(basename $file) $line; sed -n -e "${line}p" $file; done; }
+
+    comm -12 <(seddy valid_lines) <(seddy covered_lines) |
+        tr '"):' '   ' |
+        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_exec $(basename $file) $line; sed -n -e "${line}p" $file ; done; }
+
+    comm -23 <(seddy valid_lines) <(seddy covered_lines) |
+        tr '"):' '   ' |
+        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_no_exec $(basename $file) $line; sed -n -e "${line}p" $file ; done; }
+
+        #{ while read file line; do sed -n -e "${line}p" $file; done; } |
+        #sed -e 's/^/cover_no_ex:/'
+) | tee coverage/coverage.log
-- 
1.7.1