A new pinning system for the yaml-test-suite

The new system uses 2 branches instead of 3.
They are `master` and `run-test-suite`.

Pinning is no longer in tsv file.

There are pin files for each supported master commit in the `env` dir.
A pin file names the yaml-test-suite repo and data branch commit to use.
It also lets you blacklist/whitelist which tests to use.

The pin files can be symlinks.
They can also include (source) and override other files.

To specify pinning info for a PR you put it in your commit message.
See ReadMe.md for more details.

.gitignore

 /data/
-/test/

Makefile

-SHELL := bash
+SHELL = bash
 
@@ -2,3 +2,3 @@
 
-PINNED_COMMITS := $(shell LIBYAML_DEBUG_PIN=$(LIBYAML_DEBUG_PIN) ./bin/pin)
+export ENV := $(shell pwd)/env
 
@@ -4,5 +4,5 @@
 
-ifeq ($(PINNED_COMMITS),)
-    $(error ./bin/pin failed)
+ifdef env
+    export LIBYAML_TEST_SUITE_ENV := $(env)
 endif
 
@@ -7,7 +7,11 @@
 endif
 
-LIBYAML_TEST_SUITE_CODE_COMMIT ?= $(word 2, $(PINNED_COMMITS))
-LIBYAML_TEST_SUITE_DATA_COMMIT ?= $(word 3, $(PINNED_COMMITS))
-
-default: help
+.ONESHELL:
+.PHONY: test
+test: data
+	@set -ex
+	[[ "$(debug)" ]] && export LIBYAML_TEST_SUITE_DEBUG=1
+	export LIBYAML_TEST_SUITE_ENV=$$(./lookup env)
+	[[ $$LIBYAML_TEST_SUITE_ENV ]] || exit 1
+	prove -v test/
 
@@ -13,6 +17,4 @@
 
-help:
-	@echo 'test  - Run the tests'
-	@echo 'clean - Remove generated files'
-	@echo 'help  - Show help'
+test-all:
+	prove -v test/test-all.sh
 
@@ -18,6 +20,11 @@
 
-.PHONY: test
-test: data code
-	prove -lv test
+data:
+	@set -ex
+	[[ "$(debug)" ]] && export LIBYAML_TEST_SUITE_DEBUG=1
+	data=$$(./lookup data); repo=$${data%\ *}; commit=$${data#*\ }
+	[[ $$data ]] || exit 1
+	echo "repo=$$repo commit=$$commit"
+	git clone $$repo $@
+	(cd $@ && git reset --hard $$commit)
 
 clean:
@@ -22,15 +29,4 @@
 
 clean:
-	rm -fr data test
-	git worktree prune
-
-data:
-	git clone https://github.com/yaml/yaml-test-suite $@ --branch=$@
-	(cd $@ && git reset --hard $(LIBYAML_TEST_SUITE_DATA_COMMIT))
-
-code:
-	-git branch --track run-test-suite-code origin/run-test-suite-code
-	git worktree prune
-	[[ -d test ]] || \
-	    git worktree add test run-test-suite-code
-	(cd test && git reset --hard $(LIBYAML_TEST_SUITE_CODE_COMMIT))
+	rm -fr data
+	rm -f env/tmp-*

ReadMe.md

@@ -26,5 +26,5 @@
 
 * https://github.com/yaml/yaml-test-suite
 
-# Pinning
+# Pinning a Test Environment
 
@@ -30,3 +30,10 @@
 
-You can test older versions of master simply by checking out a version of master and running `make test-suite`.
+By default, the test system will look for a file called `env/pin-<commit>` where `<commit>` is a git ref of the libyaml source code being tested (usually master).
+When a commit is pushed to master, an entry should be added to the `env` directory for that commit.
+The pin file contains information about how the tests should be run and will be sourced by the test runner.
+
+Specifically, the pin file names the test suite repo, the commit to use, and which tests to use (blacklist/whitelist info).
+The test suite repo is usually https://github.com/yaml/yaml-test-suite/, but could be a fork if you need to try something.
+
+There are 2 commands: `whitelist` and `blacklist` which can be used to identify the appropriate tests to run.
 
@@ -32,5 +39,10 @@
 
-The HEAD commit of your master is checked against `conf/pin.tsv`.
-The first row in the tsv file whose master-commit is found in your master history is used.
-This will pin to the correct yaml-test-suite commit and the correct run-test-suite-code branch commit.
+Here is an example pin file:
+```
+# First set these 2 variables:
+LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
+LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
+
+# Start by including all the tests:
+whitelist all
 
@@ -36,3 +48,12 @@
 
-The test code and whitelists are stored in the libyaml branch `run-test-suite-code`.
+# Next, blacklist or whitelist for `emitter`, `parser` and `parser-error`:
+
+# Args can be a multiline string of IDs and descriptions:
+blacklist parser "\
+XXX1: A test
+XXX2: Another test
+"
+
+# Use a predefined set of blacklists:
+source "$ENV/blacklist-123"
 
@@ -38,3 +59,6 @@
 
-NOTE: If no pinning is found, you will get a warning and the HEAD commit will be used for the data and test code commits.
+# Args can also be a list of IDs:
+whitelist emitter AAA1 AAA2 AAA3
+whitelist parser-error XXX8 XXX9
+```
 
@@ -40,3 +64,13 @@
 
-## Pin Overrides
+Here's a pin file for just running a single parser test:
+```
+LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
+LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
+whitelist parser AAA1
+```
+
+## Overriding the Pin File
+
+There are a few ways to specify which pinning to use.
+They involve setting the `LIBYAML_TEST_SUITE_ENV` variable.
 
@@ -42,3 +76,12 @@
 
-You can set the following environment variables to override the pinning manually:
+To use a specific pin file for testing, use an absolute path to some file:
+```
+make test-suite LIBYAML_TEST_SUITE_ENV=/tmp/my-pin-file
+```
+
+NOTE: You can use `env` instead of `LIBYAML_TEST_SUITE_ENV` with a `make test` command.
+
+To use a specific pin file, name the file that lives in the `env` directory:
+```
+make test-suite env=pin-0032321756cee86a67171de425267c1d0d406092
 ```
@@ -44,5 +87,7 @@
 ```
-make test \
-    LIBYAML_TEST_SUITE_CODE_COMMIT=origin/run-test-suite-code \
-    LIBYAML_TEST_SUITE_DATA_COMMIT=data-2020-02-11
+
+## Specifying the Pin Info for a Pull Request
+
+If you are submitting a PR, you can put the pin file content in your log message.
+Add this exact line to your log message:
 ```
@@ -48,4 +93,8 @@
 ```
+*** yaml-test-suite
+```
+
+Everything after that will be placed into a pin file.
 
 ## Pin Debugging
 
@@ -49,12 +98,5 @@
 
 ## Pin Debugging
 
-You can turn on debugging info for the pin resolver:
-```
-make test LIBYAML_DEBUG_PIN=1
-```
-
-# Test Runner Usage
-
-Print parse events for a YAML file (or stdin):
+You can turn on debugging info for the pin `lookup` utility:
 ```
@@ -60,5 +102,3 @@
 ```
-../run-parser-test-suite file.yaml
-../run-parser-test-suite < file.yaml
-cat file.yaml | ../run-parser-test-suite
+make test-suite debug=1
 ```
@@ -64,9 +104,1 @@
 ```
-
-Print the YAML for a libyaml-parser events file (or stdin):
-```
-../run-emitter-test-suite file.events
-../run-emitter-test-suite < file.events
-cat file.events | ../libyaml-run-test-suite
-```
-

bin/pin

-#!/bin/bash
-
-set -e -u -o pipefail
-
-debug() {
-  if [[ ${LIBYAML_DEBUG_PIN-} ]]; then
-    echo "[DEBUG] $*" >&2
-  fi
-}
-
-warn() {
-  echo "$*" >&2
-}
-
-tail -n+2 conf/pin.tsv |
-tac |
-(
-  while read -r commit_master commit_code commit_data; do
-    line="$commit_master $commit_code $commit_data"
-
-    debug "Line '$line'"
-    debug "Checking commit '$commit_master'"
-    debug "cd ../..; git merge-base --is-ancestor $commit_master HEAD"
-
-    if (cd ../..; git merge-base --is-ancestor "$commit_master" HEAD 2>/dev/null); then
-      debug "Found pinning: '$line'"
-      echo "$line"
-      exit
-    else
-      debug "No pinning found"
-    fi
-  done
-
-  warn 'WARNING: No repository pinning found for libyaml/yaml-test-suite testing.'
-  warn 'WARNING: Defaulting to HEAD...'
-
-  echo -e "HEAD\tHEAD\tHEAD"
-)

conf/pin.tsv

-master commit...........................	run-test-suite-code commit..............	yaml-test-suite data branch commit......
-e4aee06ff401668404de21b4af4c6a90be0f35a6	75e4367a4c85df0b642c2db0e07c0f155e2d8fdc	data-2019-09-17
-94ecadc50ebd19299c96ee96417d252a417c7816	75c79f6b642a33468fba6cb54ca0524cee38d3a0	data-2020-02-11
-3d5b7e65dbd57a4d6f8508658efd3ad38d5125ce	6009568eec9878d19e7a37259c87cea18c6ea3bc	data-2020-02-11
-3d5b7e65dbd57a4d6f8508658efd3ad38d5125ce	67365dd2af265f17ba6ee0832c1411e4fd80ac0b	data-2020-02-11

env/blacklist-001

+#!/bin/bash
+
+whitelist all
+
+blacklist parser "\
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+5MUD: Colon and adjacent value on next line
+6BCT: Spec Example 6.3. Separation Spaces
+6LVF: Spec Example 6.13. Reserved Directives
+6M2F: Aliases in Explicit Block Mapping
+7Z25: Bare document after document end marker
+8XYN: Anchor with unicode character
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9SA2: Multiline double quoted flow mapping key
+A2M4: Spec Example 6.2. Indentation Indicators
+BEC7: Spec Example 6.14. “YAML” directive
+DBG4: Spec Example 7.10. Plain Characters
+DK3J: Zero indented block scalar with line that looks like a comment
+FP8R: Zero indented block scalar
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+HWV9: Document-end marker
+K3WX: Colon and adjacent value after comment on next line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+M7A3: Spec Example 9.3. Bare Documents
+NHX8: Empty Lines at End of Document
+NJ66: Multiline plain flow mapping key
+Q5MG: Tab at beginning of line followed by a flow mapping
+QT73: Comment and document-end marker
+R4YG: Spec Example 8.2. Block Indentation Indicator
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+UT92: Spec Example 9.4. Explicit Documents
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+Y2GN: Anchor with colon in the middle
+"
+
+blacklist parser-error "\
+9C9N: Wrong indented flow sequence
+9HCY: Need document footer before directives
+9JBA: Invalid comment after end of flow sequence
+CVW2: Invalid comment after comma
+EB22: Missing document-end marker before directive
+QB6E: Wrong indented multiline quoted scalar
+RHX7: YAML directive without document end marker
+S98Z: Block scalar with more spaces than first content line
+SU5Z: Comment without whitespace after doublequoted scalar
+X4QW: Comment without whitespace after block scalar indicator
+"
+
+blacklist emitter "\
+26DV: Whitespace around colon in mappings
+2EBW: Allowed characters in keys
+2JQS: Block Mapping with Missing Keys
+2LFX: Spec Example 6.13. Reserved Directives [1.3]
+2SXE: Anchors With Colon in Name
+2XXW: Spec Example 2.25. Unordered Sets
+3MYT: Plain Scalar looking like key, comment, anchor and tag
+4ABK: Spec Example 7.17. Flow Mapping Separate Values
+4MUZ: Flow mapping colon on line after key
+4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3]
+52DL: Explicit Non-Specific Tag [1.3]
+565N: Construct Binary
+5TYM: Spec Example 6.21. Local Tag Prefix
+5WE3: Spec Example 8.17. Explicit Block Mapping Entries
+6CK3: Spec Example 6.26. Tag Shorthands
+6FWR: Block Scalar Keep
+6KGN: Anchor for empty node
+6M2F: Aliases in Explicit Block Mapping
+6PBE: Zero-indented sequences in explicit mapping keys
+6SLA: Allowed characters in quoted mapping key
+6WLZ: Spec Example 6.18. Primary Tag Handle [1.3]
+6WPF: Spec Example 6.8. Flow Folding [1.3]
+6XDY: Two document start markers
+6ZKB: Spec Example 9.6. Stream
+7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines
+7W2P: Block Mapping with Missing Values
+7Z25: Bare document after document end marker
+8KB6: Multiline plain flow mapping key without value
+8XYN: Anchor with unicode character
+9BXH: Multiline doublequoted flow mapping key without value
+8MK2: Explicit Non-Specific Tag
+9DXL: Spec Example 9.6. Stream [1.3]
+9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
+9TFX: Spec Example 7.6. Double Quoted Lines [1.3]
+B3HG: Spec Example 8.9. Folded Scalar [1.3]
+C2DT: Spec Example 7.18. Flow Mapping Adjacent Values
+DFF7: Spec Example 7.16. Flow Mapping Entries
+E76Z: Aliases in Implicit Block Mapping
+EX5H: Multiline Scalar at Top Level [1.3]
+EXG3: Three dashes and content without space [1.3]
+F8F9: Spec Example 8.5. Chomping Trailing Lines
+FBC9: Allowed characters in plain scalars
+FH7J: Tags on Empty Scalars
+FRK4: Spec Example 7.3. Completely Empty Flow Nodes
+J3BT: Spec Example 5.12. Tabs and Spaces
+JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3]
+JTV5: Block Mapping with Multiline Scalars
+K54U: Tab after document header
+K858: Spec Example 8.6. Empty Scalar Chomping
+KK5P: Various combinations of explicit block mappings
+KSS4: Scalars on --- line
+KZN9: Spec Example 7.21. Single Pair Implicit Entries
+LE5A: Spec Example 7.24. Flow Nodes
+M7A3: Spec Example 9.3. Bare Documents
+M9B4: Spec Example 8.7. Literal Scalar
+NAT4: Various empty or newline only quoted strings
+NHX8: Empty Lines at End of Document
+PUW8: Document start on last line
+PW8X: Anchors on Empty Scalars
+Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3]
+S3PD: Spec Example 8.18. Implicit Block Mapping Entries
+S4JQ: Spec Example 6.28. Non-Specific Tags
+T26H: Spec Example 8.8. Literal Content [1.3]
+T4YY: Spec Example 7.9. Single Quoted Lines [1.3]
+T5N4: Spec Example 8.7. Literal Scalar [1.3]
+UT92: Spec Example 9.4. Explicit Documents
+W42U: Spec Example 8.15. Block Sequence Entry Types
+W4TN: Spec Example 9.5. Directives Documents
+W5VH: Allowed characters in alias
+WZ62: Spec Example 7.2. Empty Content
+X38W: Aliases in Flow Objects
+XLQ9: Multiline scalar that looks like a YAML directive
+Y2GN: Anchor with colon in the middle
+ZWK4: Key with anchor after missing explicit mapping value
+"

env/pin-0032321756cee86a67171de425267c1d0d406092

+#!/bin/bash
+
+LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
+LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
+
+source "$ENV/blacklist-001"

env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400

+pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file

env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095

+pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file

env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b

+pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file

env/pin-sample

+#!/bin/bash
+
+# This pinning file can be used for development (before committing to master)
+# like this:
+#
+#    make test-suite LIBYAML_TEST_SUITE_ENV=pin
+
+# yaml-test-suite info:
+LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
+LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
+
+# Start with a base blacklist (optional):
+source "$ENV/blacklist-001"
+
+# Add to blacklists:
+blacklist parser XXX1 XXX2
+blacklist parser-error XXX3 XXX4
+blacklist emitter XXX5 XXX6
+
+# Whitelist (un-blacklist) certain tests:
+whitelist parser AAA1 AAA2
+whitelist parser-error AAA3 AAA4
+whitelist emitter AAA5 AAA6
+
+whitelist parser 2JQS

lookup

+#!/usr/bin/env bash
+
+# shellcheck disable=1090
+
+set -e -u -o pipefail
+
+main() {
+  lookup=$1
+
+  # default_repo=https://github.com/yaml/yaml-test-suite
+  # default_commit=origin/data
+  repo=
+  commit=
+  env=
+
+  if [[ $lookup == data ]]; then
+    debug "Looking up test-suite data repo/commit"
+  elif [[ $lookup == env ]]; then
+    debug "Looking up test-suite env"
+  fi
+
+  base=$(cd ../.. && pwd)
+  head=$(get-head-commit)
+
+  if [[ ${LIBYAML_TEST_SUITE_ENV-} ]]; then
+    lookup-env
+  else
+    [[ $env ]] || lookup-log
+    [[ $env ]] || lookup-local
+  fi
+
+  if [[ $lookup == data ]]; then
+    [[ $repo && $commit ]] ||
+      die "Can't find yaml-test-suite env to use"
+    echo "$repo $commit"
+  elif [[ $lookup == env ]]; then
+    [[ $env ]] ||
+      die "Can't find yaml-test-suite env to use"
+    echo "$env"
+  fi
+}
+
+get-head-commit() {
+  debug "Getting libyaml HEAD commit"
+
+  [[ -d $base/.git ]] ||
+    die "'$base' is not a git repository"
+
+  head=$(cd "$base" && git rev-parse HEAD)
+
+  [[ $head ]] ||
+    die "Can't find HEAD commit in '$base'"
+
+  debug "HEAD commit is '$head'"
+
+  echo "$head"
+}
+
+lookup-env() {
+  local lookup=$LIBYAML_TEST_SUITE_ENV
+
+  debug "Trying LIBYAML_TEST_SUITE_ENV=$lookup"
+
+  case $lookup in
+    /*)
+      env=$lookup
+      source "$env"
+      found
+      ;;
+    pin-*)
+      env=env/$lookup
+      [[ -e $env ]] ||
+        die "Specified LIBYAML_TEST_SUITE_ENV=$lookup but '$env' not found"
+      source "$env"
+      found
+      ;;
+    time)
+      die "LIBYAML_TEST_SUITE_ENV=time not yet implemented"
+      ;;
+    *)
+      die "Unsupported value 'LIBYAML_TEST_SUITE_ENV=$lookup'"
+      ;;
+  esac
+}
+
+lookup-log() {
+  debug "Looking for '*** yaml-test-suite' in most recent 'git log' message"
+
+  found=$(
+    cd "$base" || exit
+    git log --format=%B -n1 |
+      (grep -A999999 '^\*\*\*\ yaml-test-suite' || true) |
+      tail -n+2
+  )
+  if [[ $found ]]; then
+    env=env/tmp-$head
+    echo "$found" > "$env"
+    source "$env"
+    found
+  fi
+}
+
+lookup-local() {
+  debug "Looking for local env for '$head'"
+
+  local e=env/pin-$head
+
+  if [[ -e $e ]]; then
+    env=$e
+    source "./env/pin-$head"
+    found
+  fi
+}
+
+found() {
+  debug "Found '$env'"
+
+  repo=$LIBYAML_TEST_SUITE_DATA_REPO
+  commit=$LIBYAML_TEST_SUITE_DATA_COMMIT
+}
+
+whitelist() (:)
+blacklist() (:)
+die() { echo "Died: $*" >&2; exit 1; }
+warn() ( echo "$*" >&2 )
+debug() (
+  if [[ ${LIBYAML_TEST_SUITE_DEBUG-} ]]; then
+    warn "[DEBUG] $*"
+  fi
+)
+
+main "$@"

test/libyaml-emitter.t

+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+source "$root"/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  [[ " ${emitter_list[*]} " == *\ $id\ * ]] || return 1
+  [[ -e $t/error ]] && return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=true
+
+  want=$dir/out.yaml
+  [[ -e $want ]] || want="$dir/in.yaml"
+
+  ../../tests/run-emitter-test-suite "$dir/test.event" > /tmp/test.out || {
+    (
+      cat "$dir/test.event"
+      cat "$want"
+    ) | sed 's/^/# /'
+  }
+
+  output="$(${DIFF:-diff} -u "$want" /tmp/test.out)" || ok=false
+}
+
+run-tests "$@"

test/libyaml-parser-error.t

+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+source "$root"/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  [[ " ${parser_error_list[*]} " == *\ $id\ * ]] || return 1
+  [[ -e $t/error ]] || return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=false
+
+  ../../tests/run-parser-test-suite "$dir/in.yaml" > /tmp/test.out 2>&1 || ok=true
+
+  $ok || output=$(< /tmp/test.out)
+}
+
+run-tests "$@"

test/libyaml-parser.t

+#!/usr/bin/env bash
+
+# shellcheck disable=1090,2034
+
+root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+source "$root"/test-runner.bash
+
+check-test() {
+  id=$1
+  t=data/$id
+
+  [[ " ${parser_list[*]} " == *\ $id\ * ]] || return 1
+  [[ -e $t/error ]] && return 1
+
+  return 0
+}
+
+run-test() {
+  dir=$1
+  ok=true
+
+  ../../tests/run-parser-test-suite "$dir/in.yaml" > /tmp/test.out || {
+    (
+      cat "$dir/in.yaml"
+      cat "$dir/test.event"
+    ) | sed 's/^/# /'
+  }
+
+  output=$(${DIFF:-diff} -u "$dir/test.event" /tmp/test.out) || ok=false
+}
+
+run-tests "$@"

test/test-runner.bash

+# shellcheck disable=2001,2154,2207
+
+set -e -u -o pipefail
+
+run-tests() (
+  root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+  source "$LIBYAML_TEST_SUITE_ENV"
+
+  cd "$root/.."
+
+  if [[ $# -gt 0 ]]; then
+    ids=("$@")
+  else
+    ids=($(cd data; printf "%s\n" * | grep '[0-9]'))
+  fi
+
+  count=0
+  for id in "${ids[@]}"; do
+    check-test "$id" || continue
+
+    dir=data/$id
+    label="$id: $(< "$dir/===")"
+    [[ -e $dir/in.yaml ]] || continue
+
+    run-test "$dir"
+
+    if $ok; then
+      echo "ok $((++count)) $label"
+    else
+      echo "not ok $((++count)) $label"
+      echo "$output" | sed 's/^/# /'
+    fi
+  done
+
+  echo "1..$count"
+)
+
+declare -a parser_list parser_error_list emitter_list
+whitelist() {
+  if [[ $# -eq 1 && $1 == all ]]; then
+    all=( $(ls data | sort) )
+    parser_list=( ${all[@]} )
+    parser_error_list=( ${all[@]} )
+    emitter_list=( ${all[@]} )
+    return
+  fi
+
+  test_type=$1; shift
+  list_name=${test_type/-/_}_list
+  declare -n list=$list_name
+
+  local all="${list[*]}"
+
+  if [[ $1 == *:\ * ]]; then
+    white=( $(echo "$1" | cut -d: -f1) )
+  else
+    white=( "$@" )
+  fi
+
+  for id in ${white[@]}; do
+    blacklist "$test_type" "$id"
+    all="$all $id"
+  done
+
+  eval "$list_name=( $all )"
+}
+
+blacklist() {
+  if [[ $# -eq 1 && $1 == all ]]; then
+    parser_list=()
+    parser_error_list=()
+    emitter_list=()
+    return
+  fi
+
+  test_type=$1; shift
+  list_name=${test_type/-/_}_list
+  declare -n list=$list_name
+
+  local all="${list[*]}"
+
+  if [[ $1 == *:\ * ]]; then
+    black=( $(echo "$1" | cut -d: -f1) )
+  else
+    black=( "$@" )
+  fi
+
+  for id in ${black[@]}; do
+    all=${all/$id/}
+  done
+
+  eval "$list_name=( $all )"
+}
+
+die() { echo "$*" >&2; exit 1; }