Skip to content
Snippets Groups Projects
Commit a6c063a1f462 authored by Ingy döt Net's avatar Ingy döt Net
Browse files

Support a default environment 

The env files were renamed to be more obvious.

Now pin-* files are symlinks to env-* files.

If a pinning can't be found using the current mainline commit, we
use env/default default which is a symlink to the most recent.

This will probably work most of the time.

A big warning box will be printed showing what happened and where to get
more info.

To try this out:

  git checkout ingy.test-suite-support
  export LIBYAML_TEST_SUITE_RUN_BRANCH=run-test-suite-2
  ln -fs .makefile GNUmakefile
  make distclean
  make test-suite
parent 5108f8dc0501
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 205 additions and 168 deletions
Makefile
+ 5
− 1
View file @ a6c063a1
......@@ -13,7 +13,11 @@
[[ "$(debug)" ]] && export LIBYAML_TEST_SUITE_DEBUG=1
export LIBYAML_TEST_SUITE_ENV=$$(./lookup env)
[[ $$LIBYAML_TEST_SUITE_ENV ]] || exit 1
prove -v test/
set +ex
(set -x; prove -v test/)
if [[ $$LIBYAML_TEST_SUITE_ENV == env/default ]]; then
./lookup default-warning
fi
test-all:
prove -v test/test-all.sh
......
env/blacklist-001 deleted 100644 → 0
+ 0
− 130
View file @ 5108f8dc
#!/bin/bash
whitelist all
blacklist parser "\
2JQS: Block Mapping with Missing Keys
2LFX: Spec Example 6.13. Reserved Directives [1.3]
2SXE: Anchors With Colon in Name
4ABK: Spec Example 7.17. Flow Mapping Separate Values
4MUZ: Flow mapping colon on line after key
5MUD: Colon and adjacent value on next line
6BCT: Spec Example 6.3. Separation Spaces
6LVF: Spec Example 6.13. Reserved Directives
6M2F: Aliases in Explicit Block Mapping
7Z25: Bare document after document end marker
8XYN: Anchor with unicode character
9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
9SA2: Multiline double quoted flow mapping key
A2M4: Spec Example 6.2. Indentation Indicators
BEC7: Spec Example 6.14. “YAML” directive
DBG4: Spec Example 7.10. Plain Characters
DK3J: Zero indented block scalar with line that looks like a comment
FP8R: Zero indented block scalar
FRK4: Spec Example 7.3. Completely Empty Flow Nodes
HWV9: Document-end marker
K3WX: Colon and adjacent value after comment on next line
KZN9: Spec Example 7.21. Single Pair Implicit Entries
M7A3: Spec Example 9.3. Bare Documents
NHX8: Empty Lines at End of Document
NJ66: Multiline plain flow mapping key
Q5MG: Tab at beginning of line followed by a flow mapping
QT73: Comment and document-end marker
R4YG: Spec Example 8.2. Block Indentation Indicator
S3PD: Spec Example 8.18. Implicit Block Mapping Entries
UT92: Spec Example 9.4. Explicit Documents
W4TN: Spec Example 9.5. Directives Documents
W5VH: Allowed characters in alias
WZ62: Spec Example 7.2. Empty Content
Y2GN: Anchor with colon in the middle
"
blacklist parser-error "\
9C9N: Wrong indented flow sequence
9HCY: Need document footer before directives
9JBA: Invalid comment after end of flow sequence
CVW2: Invalid comment after comma
EB22: Missing document-end marker before directive
QB6E: Wrong indented multiline quoted scalar
RHX7: YAML directive without document end marker
S98Z: Block scalar with more spaces than first content line
SU5Z: Comment without whitespace after doublequoted scalar
X4QW: Comment without whitespace after block scalar indicator
"
blacklist emitter "\
26DV: Whitespace around colon in mappings
2EBW: Allowed characters in keys
2JQS: Block Mapping with Missing Keys
2LFX: Spec Example 6.13. Reserved Directives [1.3]
2SXE: Anchors With Colon in Name
2XXW: Spec Example 2.25. Unordered Sets
3MYT: Plain Scalar looking like key, comment, anchor and tag
4ABK: Spec Example 7.17. Flow Mapping Separate Values
4MUZ: Flow mapping colon on line after key
4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3]
52DL: Explicit Non-Specific Tag [1.3]
565N: Construct Binary
5TYM: Spec Example 6.21. Local Tag Prefix
5WE3: Spec Example 8.17. Explicit Block Mapping Entries
6CK3: Spec Example 6.26. Tag Shorthands
6FWR: Block Scalar Keep
6KGN: Anchor for empty node
6M2F: Aliases in Explicit Block Mapping
6PBE: Zero-indented sequences in explicit mapping keys
6SLA: Allowed characters in quoted mapping key
6WLZ: Spec Example 6.18. Primary Tag Handle [1.3]
6WPF: Spec Example 6.8. Flow Folding [1.3]
6XDY: Two document start markers
6ZKB: Spec Example 9.6. Stream
7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines
7W2P: Block Mapping with Missing Values
7Z25: Bare document after document end marker
8KB6: Multiline plain flow mapping key without value
8XYN: Anchor with unicode character
9BXH: Multiline doublequoted flow mapping key without value
8MK2: Explicit Non-Specific Tag
9DXL: Spec Example 9.6. Stream [1.3]
9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
9TFX: Spec Example 7.6. Double Quoted Lines [1.3]
B3HG: Spec Example 8.9. Folded Scalar [1.3]
C2DT: Spec Example 7.18. Flow Mapping Adjacent Values
DFF7: Spec Example 7.16. Flow Mapping Entries
E76Z: Aliases in Implicit Block Mapping
EX5H: Multiline Scalar at Top Level [1.3]
EXG3: Three dashes and content without space [1.3]
F8F9: Spec Example 8.5. Chomping Trailing Lines
FBC9: Allowed characters in plain scalars
FH7J: Tags on Empty Scalars
FRK4: Spec Example 7.3. Completely Empty Flow Nodes
J3BT: Spec Example 5.12. Tabs and Spaces
JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3]
JTV5: Block Mapping with Multiline Scalars
K54U: Tab after document header
K858: Spec Example 8.6. Empty Scalar Chomping
KK5P: Various combinations of explicit block mappings
KSS4: Scalars on --- line
KZN9: Spec Example 7.21. Single Pair Implicit Entries
LE5A: Spec Example 7.24. Flow Nodes
M7A3: Spec Example 9.3. Bare Documents
M9B4: Spec Example 8.7. Literal Scalar
NAT4: Various empty or newline only quoted strings
NHX8: Empty Lines at End of Document
PUW8: Document start on last line
PW8X: Anchors on Empty Scalars
Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3]
S3PD: Spec Example 8.18. Implicit Block Mapping Entries
S4JQ: Spec Example 6.28. Non-Specific Tags
T26H: Spec Example 8.8. Literal Content [1.3]
T4YY: Spec Example 7.9. Single Quoted Lines [1.3]
T5N4: Spec Example 8.7. Literal Scalar [1.3]
UT92: Spec Example 9.4. Explicit Documents
W42U: Spec Example 8.15. Block Sequence Entry Types
W4TN: Spec Example 9.5. Directives Documents
W5VH: Allowed characters in alias
WZ62: Spec Example 7.2. Empty Content
X38W: Aliases in Flow Objects
XLQ9: Multiline scalar that looks like a YAML directive
Y2GN: Anchor with colon in the middle
ZWK4: Key with anchor after missing explicit mapping value
"
env/default 0 → 120000
+ 1
− 0
View file @ a6c063a1
env-2020-05-24-test-suite-9fe5172ca31
\ No newline at end of file
env/env-2020-05-24-test-suite-9fe5172ca31 0 → 100644
+ 138
− 0
View file @ a6c063a1
#!/bin/bash
LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
LIBYAML_TEST_SUITE_DATA_COMMIT=9fe5172ca313349ad5b8200f8168de0b0a48ea5d
whitelist all
blacklist emitter "\
26DV: Whitespace around colon in mappings
2EBW: Allowed characters in keys
2JQS: Block Mapping with Missing Keys
2LFX: Spec Example 6.13. Reserved Directives [1.3]
2SXE: Anchors With Colon in Name
2XXW: Spec Example 2.25. Unordered Sets
3MYT: Plain Scalar looking like key, comment, anchor and tag
4ABK: Spec Example 7.17. Flow Mapping Separate Values
4MUZ: Flow mapping colon on line after key
4QFQ: Spec Example 8.2. Block Indentation Indicator [1.3]
52DL: Explicit Non-Specific Tag [1.3]
565N: Construct Binary
5T43: Colon at the beginning of adjacent flow scalar
5TYM: Spec Example 6.21. Local Tag Prefix
5WE3: Spec Example 8.17. Explicit Block Mapping Entries
6CK3: Spec Example 6.26. Tag Shorthands
6FWR: Block Scalar Keep
6KGN: Anchor for empty node
6M2F: Aliases in Explicit Block Mapping
6PBE: Zero-indented sequences in explicit mapping keys
6SLA: Allowed characters in quoted mapping key
6WLZ: Spec Example 6.18. Primary Tag Handle [1.3]
6WPF: Spec Example 6.8. Flow Folding [1.3]
6XDY: Two document start markers
6ZKB: Spec Example 9.6. Stream
7T8X: Spec Example 8.10. Folded Lines - 8.13. Final Empty Lines
7W2P: Block Mapping with Missing Values
7Z25: Bare document after document end marker
8KB6: Multiline plain flow mapping key without value
8XYN: Anchor with unicode character
9BXH: Multiline doublequoted flow mapping key without value
8MK2: Explicit Non-Specific Tag
9DXL: Spec Example 9.6. Stream [1.3]
9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
9TFX: Spec Example 7.6. Double Quoted Lines [1.3]
B3HG: Spec Example 8.9. Folded Scalar [1.3]
C2DT: Spec Example 7.18. Flow Mapping Adjacent Values
DFF7: Spec Example 7.16. Flow Mapping Entries
E76Z: Aliases in Implicit Block Mapping
EX5H: Multiline Scalar at Top Level [1.3]
EXG3: Three dashes and content without space [1.3]
F8F9: Spec Example 8.5. Chomping Trailing Lines
FBC9: Allowed characters in plain scalars
FH7J: Tags on Empty Scalars
FRK4: Spec Example 7.3. Completely Empty Flow Nodes
J3BT: Spec Example 5.12. Tabs and Spaces
JDH8: Plain Scalar looking like key, comment, anchor and tag [1.3]
JTV5: Block Mapping with Multiline Scalars
K54U: Tab after document header
K858: Spec Example 8.6. Empty Scalar Chomping
KK5P: Various combinations of explicit block mappings
KSS4: Scalars on --- line
KZN9: Spec Example 7.21. Single Pair Implicit Entries
LE5A: Spec Example 7.24. Flow Nodes
M7A3: Spec Example 9.3. Bare Documents
M9B4: Spec Example 8.7. Literal Scalar
NAT4: Various empty or newline only quoted strings
NHX8: Empty Lines at End of Document
PUW8: Document start on last line
PW8X: Anchors on Empty Scalars
Q8AD: Spec Example 7.5. Double Quoted Line Breaks [1.3]
S3PD: Spec Example 8.18. Implicit Block Mapping Entries
S4JQ: Spec Example 6.28. Non-Specific Tags
T26H: Spec Example 8.8. Literal Content [1.3]
T4YY: Spec Example 7.9. Single Quoted Lines [1.3]
T5N4: Spec Example 8.7. Literal Scalar [1.3]
UT92: Spec Example 9.4. Explicit Documents
W42U: Spec Example 8.15. Block Sequence Entry Types
W4TN: Spec Example 9.5. Directives Documents
W5VH: Allowed characters in alias
WZ62: Spec Example 7.2. Empty Content
X38W: Aliases in Flow Objects
XLQ9: Multiline scalar that looks like a YAML directive
Y2GN: Anchor with colon in the middle
ZWK4: Key with anchor after missing explicit mapping value
"
blacklist parser "\
2JQS: Block Mapping with Missing Keys
2LFX: Spec Example 6.13. Reserved Directives [1.3]
2SXE: Anchors With Colon in Name
4ABK: Spec Example 7.17. Flow Mapping Separate Values
4MUZ: Flow mapping colon on line after key
5MUD: Colon and adjacent value on next line
5T43: Colon at the beginning of adjacent flow scalar
6BCT: Spec Example 6.3. Separation Spaces
6LVF: Spec Example 6.13. Reserved Directives
6M2F: Aliases in Explicit Block Mapping
7Z25: Bare document after document end marker
8XYN: Anchor with unicode character
9MMW: Spec Example 7.21. Single Pair Implicit Entries [1.3
9SA2: Multiline double quoted flow mapping key
A2M4: Spec Example 6.2. Indentation Indicators
BEC7: Spec Example 6.14. “YAML” directive
DBG4: Spec Example 7.10. Plain Characters
DK3J: Zero indented block scalar with line that looks like a comment
FP8R: Zero indented block scalar
FRK4: Spec Example 7.3. Completely Empty Flow Nodes
HWV9: Document-end marker
JR7V: Question marks in scalars
K3WX: Colon and adjacent value after comment on next line
KZN9: Spec Example 7.21. Single Pair Implicit Entries
M7A3: Spec Example 9.3. Bare Documents
NHX8: Empty Lines at End of Document
NJ66: Multiline plain flow mapping key
Q5MG: Tab at beginning of line followed by a flow mapping
QT73: Comment and document-end marker
R4YG: Spec Example 8.2. Block Indentation Indicator
S3PD: Spec Example 8.18. Implicit Block Mapping Entries
UT92: Spec Example 9.4. Explicit Documents
W4TN: Spec Example 9.5. Directives Documents
W5VH: Allowed characters in alias
WZ62: Spec Example 7.2. Empty Content
Y2GN: Anchor with colon in the middle
"
blacklist parser-error "\
9C9N: Wrong indented flow sequence
9HCY: Need document footer before directives
9JBA: Invalid comment after end of flow sequence
CVW2: Invalid comment after comma
EB22: Missing document-end marker before directive
QB6E: Wrong indented multiline quoted scalar
RHX7: YAML directive without document end marker
S98Z: Block scalar with more spaces than first content line
5T43: Colon at the beginning of adjacent flow scalar
SU5Z: Comment without whitespace after doublequoted scalar
U99R: Invalid comma in tag
X4QW: Comment without whitespace after block scalar indicator
"
env/pin-0.2.4-72e2f75277ff88ad1fb8a81cf422df06bf371578 0 → 120000
+ 1
− 0
View file @ a6c063a1
env-2020-05-24-test-suite-9fe5172ca31
\ No newline at end of file
env/pin-0032321756cee86a67171de425267c1d0d406092 deleted 100644 → 0
+ 0
− 6
View file @ 5108f8dc
#!/bin/bash
LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
source "$ENV/blacklist-001"
env/pin-0092c82664d2e7f2f0faf21de660a7f1d1e54400 deleted 120000 → 0
+ 0
− 1
View file @ 5108f8dc
pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file
env/pin-55a405c42069b05b20c1e53d3ed8c099d4815095 deleted 120000 → 0
+ 0
− 1
View file @ 5108f8dc
pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file
env/pin-9deee01508bb3b8ad25822f53d91d9f54eae9a0b deleted 120000 → 0
+ 0
− 1
View file @ 5108f8dc
pin-0032321756cee86a67171de425267c1d0d406092
\ No newline at end of file
env/pin-master-9deee01508bb3b8ad25822f53d91d9f54eae9a0b 0 → 120000
+ 1
− 0
View file @ a6c063a1
env-2020-05-24-test-suite-9fe5172ca31
\ No newline at end of file
env/pin-sample deleted 100644 → 0
+ 0
− 25
View file @ 5108f8dc
#!/bin/bash
# This pinning file can be used for development (before committing to master)
# like this:
#
# make test-suite LIBYAML_TEST_SUITE_ENV=pin
# yaml-test-suite info:
LIBYAML_TEST_SUITE_DATA_REPO=https://github.com/yaml/yaml-test-suite
LIBYAML_TEST_SUITE_DATA_COMMIT=data-2019-09-17
# Start with a base blacklist (optional):
source "$ENV/blacklist-001"
# Add to blacklists:
blacklist parser XXX1 XXX2
blacklist parser-error XXX3 XXX4
blacklist emitter XXX5 XXX6
# Whitelist (un-blacklist) certain tests:
whitelist parser AAA1 AAA2
whitelist parser-error AAA3 AAA4
whitelist emitter AAA5 AAA6
whitelist parser 2JQS
lookup
+ 59
− 3
View file @ a6c063a1
......@@ -22,8 +22,13 @@
base=$(cd ../.. && pwd)
head=$(get-head-commit)
if [[ $lookup == default-warning ]]; then
default-warning
exit
fi
if [[ ${LIBYAML_TEST_SUITE_ENV-} ]]; then
lookup-env
else
[[ $env ]] || lookup-log
[[ $env ]] || lookup-local
......@@ -25,8 +30,9 @@
if [[ ${LIBYAML_TEST_SUITE_ENV-} ]]; then
lookup-env
else
[[ $env ]] || lookup-log
[[ $env ]] || lookup-local
[[ $env ]] || lookup-default
fi
if [[ $lookup == data ]]; then
......@@ -103,7 +109,7 @@
lookup-local() {
debug "Looking for local env for '$head'"
local e=env/pin-$head
e=$(echo env/pin-*-$head)
if [[ -e $e ]]; then
env=$e
......@@ -107,8 +113,8 @@
if [[ -e $e ]]; then
env=$e
source "./env/pin-$head"
source "$env"
found
fi
}
......@@ -111,7 +117,17 @@
found
fi
}
lookup-default() {
debug "Using default environment 'env/default'"
[[ $lookup == env ]] && default-warning
env=env/default
source "$env"
found
}
found() {
debug "Found '$env'"
......@@ -119,8 +135,37 @@
commit=$LIBYAML_TEST_SUITE_DATA_COMMIT
}
default-warning() (
main_branch=$(cd "$base" && git rev-parse --abbrev-ref HEAD)
test_branch=$(git rev-parse --abbrev-ref HEAD)
default=env/default
while [[ -h $default ]]; do
default=$(readlink "$default")
done
message-in-a-box "
WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING
Using the default test-suite env:
env/$default
No test-suite environment info was found for the HEAD of '$main_branch':
$head
This info is kept in tests/run-test-suite/env/
For more information, see:
* https://github.com/yaml/libyaml/blob/$test_branch/ReadMe.md
* https://github.com/yaml/libyaml/tree/$test_branch/env
"
)
whitelist() (:)
blacklist() (:)
die() { echo "Died: $*" >&2; exit 1; }
warn() ( echo "$*" >&2 )
debug() (
......@@ -122,9 +167,9 @@
whitelist() (:)
blacklist() (:)
die() { echo "Died: $*" >&2; exit 1; }
warn() ( echo "$*" >&2 )
debug() (
if [[ ${LIBYAML_TEST_SUITE_DEBUG-} ]]; then
if [[ ${LIBYAML_TEST_SUITE_DEBUG-} && $lookup != default-warning ]]; then
warn "[DEBUG] $*"
fi
)
......@@ -128,5 +173,16 @@
warn "[DEBUG] $*"
fi
)
message-in-a-box() (
len=${COLUMNS:-$(tput cols)}
len=${len:-80}
line=$(eval "printf '*%.0s' {1..$len}")
(
echo "$line"
echo "* ${1//$'\n'/$'\n'* }"
echo "$line"
) >&2
)
main "$@"
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment