Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#!/usr/bin/env python3
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import io
import lzma
import os
from pathlib import Path
from tabulate import tabulate
import orjson
dirname = os.path.join(os.path.dirname(__file__), "data")
LIBRARIES = ["orjson", "ujson", "rapidjson", "simplejson", "json"]
def read_fixture_bytes(filename, subdir=None):
if subdir is None:
parts = (dirname, filename)
else:
parts = (dirname, subdir, filename)
path = Path(*parts)
if path.suffix == ".xz":
contents = lzma.decompress(path.read_bytes())
else:
contents = path.read_bytes()
return contents
PARSING = {
filename: read_fixture_bytes(filename, "parsing")
for filename in os.listdir("data/parsing")
}
JSONCHECKER = {
filename: read_fixture_bytes(filename, "jsonchecker")
for filename in os.listdir("data/jsonchecker")
}
RESULTS = collections.defaultdict(dict)
def read_fixture(filename, subdir=None):
if not filename in BYTES_CACHE:
BYTES_CACHE[filename] = read_fixture_bytes(filename, subdir)
return BYTES_CACHE[filename]
def test_passed(library, fixture):
passed = []
try:
passed.append(library.loads(fixture) == orjson.loads(fixture))
passed.append(
library.loads(fixture.decode("utf-8"))
== orjson.loads(fixture.decode("utf-8"))
)
except Exception:
passed.append(False)
return all(passed)
def test_failed(library, fixture):
rejected_as_bytes = False
try:
library.loads(fixture)
except Exception:
rejected_as_bytes = True
rejected_as_str = False
try:
library.loads(fixture.decode("utf-8"))
except Exception:
rejected_as_str = True
return rejected_as_bytes and rejected_as_str
MISTAKEN_PASSES = {key: 0 for key in LIBRARIES}
MISTAKEN_FAILS = {key: 0 for key in LIBRARIES}
PASS_WHITELIST = ("fail01.json", "fail18.json")
def should_pass(filename):
return (
filename.startswith("y_")
or filename.startswith("pass")
or filename in PASS_WHITELIST
)
def should_fail(filename):
return (
filename.startswith("n_")
or filename.startswith("i_string")
or filename.startswith("i_object")
or filename.startswith("fail")
) and not filename in PASS_WHITELIST
for libname in LIBRARIES:
library = __import__(libname)
for fixture_set in (PARSING, JSONCHECKER):
for filename, fixture in fixture_set.items():
if should_pass(filename):
res = test_passed(library, fixture)
RESULTS[filename][libname] = res
if not res:
MISTAKEN_PASSES[libname] += 1
elif should_fail(filename):
res = test_failed(library, fixture)
RESULTS[filename][libname] = res
if not res:
MISTAKEN_FAILS[libname] += 1
elif filename.startswith("i_"):
continue
else:
raise NotImplementedError
FILENAMES = sorted(list(PARSING.keys()) + list(JSONCHECKER.keys()))
tab_results = []
for filename in FILENAMES:
entry = [
filename,
]
for libname in LIBRARIES:
try:
entry.append("ok" if RESULTS[filename][libname] else "fail")
except KeyError:
continue
tab_results.append(entry)
buf = io.StringIO()
buf.write(tabulate(tab_results, ["Fixture"] + LIBRARIES, tablefmt="github"))
buf.write("\n")
print(buf.getvalue())
failure_results = [
[libname, MISTAKEN_FAILS[libname], MISTAKEN_PASSES[libname]]
for libname in LIBRARIES
]
buf = io.StringIO()
buf.write(
tabulate(
failure_results,
[
"Library",
"Invalid JSON documents not rejected",
"Valid JSON documents not deserialized",
],
tablefmt="github",
)
)
buf.write("\n")
print(buf.getvalue())
num_results = len([each for each in tab_results if len(each) > 1])