diff --git a/CHANGES.txt b/CHANGES.txt
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_Q0hBTkdFUy50eHQ=..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_Q0hBTkdFUy50eHQ= 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,3 +1,12 @@
+Version 3.1.0 released XXXX-XX-XX
+
+* Improve error messages for certain kinds of truncated input
+ http://bugs.python.org/issue16009
+* Moved JSONDecodeError to json.scanner (still available for import
+ from json.decoder)
+* Changed scanner to use JSONDecodeError directly rather than
+ StopIteration to improve error messages
+
Version 3.0.9 released 2013-02-21
* Fix an off-by-one error in the colno property of JSONDecodeError
diff --git a/conf.py b/conf.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_Y29uZi5weQ==..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_Y29uZi5weQ== 100644
--- a/conf.py
+++ b/conf.py
@@ -36,9 +36,9 @@
# General substitutions.
project = 'simplejson'
-copyright = '2012, Bob Ippolito'
+copyright = '2013, Bob Ippolito'
# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
@@ -40,7 +40,7 @@
# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
# The short X.Y version.
-version = '3.0'
+version = '3.1'
# The full version, including alpha/beta/rc tags.
@@ -46,5 +46,5 @@
# The full version, including alpha/beta/rc tags.
-release = '3.0.9'
+release = '3.1'
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
diff --git a/setup.py b/setup.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2V0dXAucHk=..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2V0dXAucHk= 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
DistutilsPlatformError
IS_PYPY = hasattr(sys, 'pypy_translation_info')
-VERSION = '3.0.9'
+VERSION = '3.1.0'
DESCRIPTION = "Simple, fast, extensible JSON encoder/decoder for Python"
with open('README.rst', 'r') as f:
diff --git a/simplejson/__init__.py b/simplejson/__init__.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2ltcGxlanNvbi9fX2luaXRfXy5weQ==..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2ltcGxlanNvbi9fX2luaXRfXy5weQ== 100644
--- a/simplejson/__init__.py
+++ b/simplejson/__init__.py
@@ -99,7 +99,7 @@
Expecting property name: line 1 column 3 (char 2)
"""
from __future__ import absolute_import
-__version__ = '3.0.9'
+__version__ = '3.1.0'
__all__ = [
'dump', 'dumps', 'load', 'loads',
'JSONDecoder', 'JSONDecodeError', 'JSONEncoder',
@@ -110,7 +110,8 @@
from decimal import Decimal
-from .decoder import JSONDecoder, JSONDecodeError
+from .scanner import JSONDecodeError
+from .decoder import JSONDecoder
from .encoder import JSONEncoder, JSONEncoderForHTML
def _import_OrderedDict():
import collections
diff --git a/simplejson/_speedups.c b/simplejson/_speedups.c
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2ltcGxlanNvbi9fc3BlZWR1cHMuYw==..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2ltcGxlanNvbi9fc3BlZWR1cHMuYw== 100644
--- a/simplejson/_speedups.c
+++ b/simplejson/_speedups.c
@@ -1,3 +1,4 @@
+/* -*- mode: C; c-file-style: "python"; c-basic-offset: 4 -*- */
#include "Python.h"
#include "structmember.h"
@@ -109,6 +110,21 @@
static void
JSON_Accu_Destroy(JSON_Accu *acc);
+#define ERR_EXPECTING_VALUE "Expecting value"
+#define ERR_ARRAY_DELIMITER "Expecting ',' delimiter or ']'"
+#define ERR_ARRAY_VALUE_FIRST "Expecting value or ']'"
+#define ERR_OBJECT_DELIMITER "Expecting ',' delimiter or '}'"
+#define ERR_OBJECT_PROPERTY "Expecting property name enclosed in double quotes"
+#define ERR_OBJECT_PROPERTY_FIRST "Expecting property name enclosed in double quotes or '}'"
+#define ERR_OBJECT_PROPERTY_DELIMITER "Expecting ':' delimiter"
+#define ERR_STRING_UNTERMINATED "Unterminated string starting at"
+#define ERR_STRING_CONTROL "Invalid control character %r at"
+#define ERR_STRING_ESC1 "Invalid \\X escape sequence %r"
+#define ERR_STRING_ESC4 "Invalid \\uXXXX escape sequence"
+#define ERR_STRING_SURROGATE "Invalid \\uXXXX\\uXXXX surrogate pair"
+#define ERR_STRING_HIGH_SURROGATE "Unpaired high surrogate"
+#define ERR_STRING_LOW_SURROGATE "Unpaired low surrogate"
+
typedef struct _PyScannerObject {
PyObject_HEAD
PyObject *encoding;
@@ -746,8 +762,7 @@
static void
raise_errmsg(char *msg, PyObject *s, Py_ssize_t end)
{
- /* Use the Python function simplejson.decoder.errmsg to raise a nice
- looking ValueError exception */
+ /* Use JSONDecodeError exception to raise a nice looking ValueError subclass */
static PyObject *JSONDecodeError = NULL;
PyObject *exc;
if (JSONDecodeError == NULL) {
@@ -751,6 +766,6 @@
static PyObject *JSONDecodeError = NULL;
PyObject *exc;
if (JSONDecodeError == NULL) {
- PyObject *decoder = PyImport_ImportModule("simplejson.decoder");
- if (decoder == NULL)
+ PyObject *scanner = PyImport_ImportModule("simplejson.scanner");
+ if (scanner == NULL)
return;
@@ -756,6 +771,6 @@
return;
- JSONDecodeError = PyObject_GetAttrString(decoder, "JSONDecodeError");
- Py_DECREF(decoder);
+ JSONDecodeError = PyObject_GetAttrString(scanner, "JSONDecodeError");
+ Py_DECREF(scanner);
if (JSONDecodeError == NULL)
return;
}
@@ -873,7 +888,7 @@
PyObject *strchunk = NULL;
if (len == end) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
else if (end < 0 || len < end) {
@@ -889,7 +904,7 @@
break;
}
else if (strict && c <= 0x1f) {
- raise_errmsg("Invalid control character at", pystr, next);
+ raise_errmsg(ERR_STRING_CONTROL, pystr, next);
goto bail;
}
else if (c > 0x7f) {
@@ -897,7 +912,7 @@
}
}
if (!(c == '"' || c == '\\')) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
/* Pick up this chunk if it's not zero length */
@@ -936,7 +951,7 @@
break;
}
if (next == len) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
c = buf[next];
@@ -955,7 +970,7 @@
default: c = 0;
}
if (c == 0) {
- raise_errmsg("Invalid \\escape", pystr, end - 2);
+ raise_errmsg(ERR_STRING_ESC1, pystr, end - 2);
goto bail;
}
}
@@ -964,7 +979,7 @@
next++;
end = next + 4;
if (end >= len) {
- raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
+ raise_errmsg(ERR_STRING_ESC4, pystr, next - 1);
goto bail;
}
/* Decode 4 hex digits */
@@ -982,7 +997,7 @@
case 'F':
c |= (digit - 'A' + 10); break;
default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+ raise_errmsg(ERR_STRING_ESC4, pystr, end - 5);
goto bail;
}
}
@@ -991,7 +1006,7 @@
if ((c & 0xfc00) == 0xd800) {
JSON_UNICHR c2 = 0;
if (end + 6 >= len) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
if (buf[next++] != '\\' || buf[next++] != 'u') {
@@ -995,7 +1010,7 @@
goto bail;
}
if (buf[next++] != '\\' || buf[next++] != 'u') {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
end += 6;
@@ -1014,8 +1029,8 @@
case 'F':
c2 |= (digit - 'A' + 10); break;
default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+ raise_errmsg(ERR_STRING_ESC4, pystr, end - 5);
goto bail;
}
}
if ((c2 & 0xfc00) != 0xdc00) {
@@ -1018,10 +1033,10 @@
goto bail;
}
}
if ((c2 & 0xfc00) != 0xdc00) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
}
else if ((c & 0xfc00) == 0xdc00) {
@@ -1023,9 +1038,9 @@
goto bail;
}
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
}
else if ((c & 0xfc00) == 0xdc00) {
- raise_errmsg("Unpaired low surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_LOW_SURROGATE, pystr, end - 5);
goto bail;
}
#endif /* PY_MAJOR_VERSION >= 3 || Py_UNICODE_WIDE */
@@ -1102,7 +1117,7 @@
PyObject *chunk = NULL;
if (len == end) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
else if (end < 0 || len < end) {
@@ -1118,8 +1133,8 @@
break;
}
else if (strict && c <= 0x1f) {
- raise_errmsg("Invalid control character at", pystr, next);
+ raise_errmsg(ERR_STRING_CONTROL, pystr, next);
goto bail;
}
}
if (!(c == '"' || c == '\\')) {
@@ -1122,8 +1137,8 @@
goto bail;
}
}
if (!(c == '"' || c == '\\')) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
/* Pick up this chunk if it's not zero length */
@@ -1144,7 +1159,7 @@
break;
}
if (next == len) {
- raise_errmsg("Unterminated string starting at", pystr, begin);
+ raise_errmsg(ERR_STRING_UNTERMINATED, pystr, begin);
goto bail;
}
c = PyUnicode_READ(kind, buf, next);
@@ -1163,7 +1178,7 @@
default: c = 0;
}
if (c == 0) {
- raise_errmsg("Invalid \\escape", pystr, end - 2);
+ raise_errmsg(ERR_STRING_ESC1, pystr, end - 2);
goto bail;
}
}
@@ -1172,7 +1187,7 @@
next++;
end = next + 4;
if (end >= len) {
- raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
+ raise_errmsg(ERR_STRING_ESC4, pystr, next - 1);
goto bail;
}
/* Decode 4 hex digits */
@@ -1190,7 +1205,7 @@
case 'F':
c |= (digit - 'A' + 10); break;
default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+ raise_errmsg(ERR_STRING_ESC4, pystr, end - 5);
goto bail;
}
}
@@ -1199,8 +1214,8 @@
if ((c & 0xfc00) == 0xd800) {
JSON_UNICHR c2 = 0;
if (end + 6 >= len) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
if (PyUnicode_READ(kind, buf, next++) != '\\' ||
PyUnicode_READ(kind, buf, next++) != 'u') {
@@ -1203,8 +1218,8 @@
goto bail;
}
if (PyUnicode_READ(kind, buf, next++) != '\\' ||
PyUnicode_READ(kind, buf, next++) != 'u') {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
end += 6;
@@ -1223,8 +1238,8 @@
case 'F':
c2 |= (digit - 'A' + 10); break;
default:
- raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+ raise_errmsg(ERR_STRING_ESC4, pystr, end - 5);
goto bail;
}
}
if ((c2 & 0xfc00) != 0xdc00) {
@@ -1227,10 +1242,10 @@
goto bail;
}
}
if ((c2 & 0xfc00) != 0xdc00) {
- raise_errmsg("Unpaired high surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_HIGH_SURROGATE, pystr, end - 5);
goto bail;
}
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
}
else if ((c & 0xfc00) == 0xdc00) {
@@ -1232,9 +1247,9 @@
goto bail;
}
c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
}
else if ((c & 0xfc00) == 0xdc00) {
- raise_errmsg("Unpaired low surrogate", pystr, end - 5);
+ raise_errmsg(ERR_STRING_LOW_SURROGATE, pystr, end - 5);
goto bail;
}
#endif
@@ -1405,6 +1420,7 @@
char *encoding = JSON_ASCII_AS_STRING(s->encoding);
int strict = PyObject_IsTrue(s->strict);
int has_pairs_hook = (s->pairs_hook != Py_None);
+ int did_parse = 0;
Py_ssize_t next_idx;
if (has_pairs_hook) {
pairs = PyList_New(0);
@@ -1422,5 +1438,6 @@
/* only loop if the object is non-empty */
if (idx <= end_idx && str[idx] != '}') {
+ int trailing_delimiter = 0;
while (idx <= end_idx) {
PyObject *memokey;
@@ -1425,5 +1442,6 @@
while (idx <= end_idx) {
PyObject *memokey;
+ trailing_delimiter = 0;
/* read key */
if (str[idx] != '"') {
@@ -1427,9 +1445,7 @@
/* read key */
if (str[idx] != '"') {
- raise_errmsg(
- "Expecting property name enclosed in double quotes",
- pystr, idx);
+ raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx);
goto bail;
}
key = scanstring_str(pystr, idx + 1, encoding, strict, &next_idx);
@@ -1450,7 +1466,7 @@
/* skip whitespace between key and : delimiter, read :, skip whitespace */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
if (idx > end_idx || str[idx] != ':') {
- raise_errmsg("Expecting ':' delimiter", pystr, idx);
+ raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx);
goto bail;
}
idx++;
@@ -1485,8 +1501,9 @@
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
/* bail if the object is closed or we didn't get the , delimiter */
+ did_parse = 1;
if (idx > end_idx) break;
if (str[idx] == '}') {
break;
}
else if (str[idx] != ',') {
@@ -1488,12 +1505,12 @@
if (idx > end_idx) break;
if (str[idx] == '}') {
break;
}
else if (str[idx] != ',') {
- raise_errmsg("Expecting ',' delimiter", pystr, idx);
+ raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx);
goto bail;
}
idx++;
/* skip whitespace after , delimiter */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
@@ -1494,7 +1511,8 @@
goto bail;
}
idx++;
/* skip whitespace after , delimiter */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
+ trailing_delimiter = 1;
}
@@ -1500,4 +1518,8 @@
}
+ if (trailing_delimiter) {
+ raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx);
+ goto bail;
+ }
}
/* verify that idx < end_idx, str[idx] should be '}' */
if (idx > end_idx || str[idx] != '}') {
@@ -1501,7 +1523,11 @@
}
/* verify that idx < end_idx, str[idx] should be '}' */
if (idx > end_idx || str[idx] != '}') {
- raise_errmsg("Expecting object", pystr, end_idx);
+ if (did_parse) {
+ raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx);
+ } else {
+ raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx);
+ }
goto bail;
}
@@ -1555,6 +1581,7 @@
PyObject *val = NULL;
int strict = PyObject_IsTrue(s->strict);
int has_pairs_hook = (s->pairs_hook != Py_None);
+ int did_parse = 0;
Py_ssize_t next_idx;
if (has_pairs_hook) {
@@ -1573,5 +1600,6 @@
/* only loop if the object is non-empty */
if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') {
+ int trailing_delimiter = 0;
while (idx <= end_idx) {
PyObject *memokey;
@@ -1576,5 +1604,6 @@
while (idx <= end_idx) {
PyObject *memokey;
+ trailing_delimiter = 0;
/* read key */
if (PyUnicode_READ(kind, str, idx) != '"') {
@@ -1578,9 +1607,7 @@
/* read key */
if (PyUnicode_READ(kind, str, idx) != '"') {
- raise_errmsg(
- "Expecting property name enclosed in double quotes",
- pystr, idx);
+ raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx);
goto bail;
}
key = scanstring_unicode(pystr, idx + 1, strict, &next_idx);
@@ -1602,7 +1629,7 @@
whitespace */
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;
if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') {
- raise_errmsg("Expecting ':' delimiter", pystr, idx);
+ raise_errmsg(ERR_OBJECT_PROPERTY_DELIMITER, pystr, idx);
goto bail;
}
idx++;
@@ -1638,8 +1665,9 @@
/* bail if the object is closed or we didn't get the ,
delimiter */
+ did_parse = 1;
if (idx > end_idx) break;
if (PyUnicode_READ(kind, str, idx) == '}') {
break;
}
else if (PyUnicode_READ(kind, str, idx) != ',') {
@@ -1641,12 +1669,12 @@
if (idx > end_idx) break;
if (PyUnicode_READ(kind, str, idx) == '}') {
break;
}
else if (PyUnicode_READ(kind, str, idx) != ',') {
- raise_errmsg("Expecting ',' delimiter", pystr, idx);
+ raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx);
goto bail;
}
idx++;
/* skip whitespace after , delimiter */
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;
@@ -1647,7 +1675,8 @@
goto bail;
}
idx++;
/* skip whitespace after , delimiter */
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;
+ trailing_delimiter = 1;
}
@@ -1653,5 +1682,9 @@
}
+ if (trailing_delimiter) {
+ raise_errmsg(ERR_OBJECT_PROPERTY, pystr, idx);
+ goto bail;
+ }
}
/* verify that idx < end_idx, str[idx] should be '}' */
if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') {
@@ -1654,8 +1687,12 @@
}
/* verify that idx < end_idx, str[idx] should be '}' */
if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') {
- raise_errmsg("Expecting object", pystr, end_idx);
+ if (did_parse) {
+ raise_errmsg(ERR_OBJECT_DELIMITER, pystr, idx);
+ } else {
+ raise_errmsg(ERR_OBJECT_PROPERTY_FIRST, pystr, idx);
+ }
goto bail;
}
@@ -1712,4 +1749,5 @@
/* only loop if the array is non-empty */
if (idx <= end_idx && str[idx] != ']') {
+ int trailing_delimiter = 0;
while (idx <= end_idx) {
@@ -1715,5 +1753,5 @@
while (idx <= end_idx) {
-
+ trailing_delimiter = 0;
/* read any JSON term and de-tuplefy the (rval, idx) */
val = scan_once_str(s, pystr, idx, &next_idx);
if (val == NULL) {
@@ -1717,10 +1755,6 @@
/* read any JSON term and de-tuplefy the (rval, idx) */
val = scan_once_str(s, pystr, idx, &next_idx);
if (val == NULL) {
- if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
- PyErr_Clear();
- raise_errmsg("Expecting object", pystr, idx);
- }
goto bail;
}
@@ -1739,10 +1773,10 @@
break;
}
else if (str[idx] != ',') {
- raise_errmsg("Expecting ',' delimiter", pystr, idx);
+ raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx);
goto bail;
}
idx++;
/* skip whitespace after , */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
@@ -1743,7 +1777,8 @@
goto bail;
}
idx++;
/* skip whitespace after , */
while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++;
+ trailing_delimiter = 1;
}
@@ -1749,5 +1784,9 @@
}
+ if (trailing_delimiter) {
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
+ goto bail;
+ }
}
/* verify that idx < end_idx, str[idx] should be ']' */
if (idx > end_idx || str[idx] != ']') {
@@ -1750,8 +1789,12 @@
}
/* verify that idx < end_idx, str[idx] should be ']' */
if (idx > end_idx || str[idx] != ']') {
- raise_errmsg("Expecting object", pystr, end_idx);
+ if (PyList_GET_SIZE(rval)) {
+ raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx);
+ } else {
+ raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx);
+ }
goto bail;
}
*next_idx_ptr = idx + 1;
@@ -1787,4 +1830,5 @@
/* only loop if the array is non-empty */
if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') {
+ int trailing_delimiter = 0;
while (idx <= end_idx) {
@@ -1790,5 +1834,5 @@
while (idx <= end_idx) {
-
+ trailing_delimiter = 0;
/* read any JSON term */
val = scan_once_unicode(s, pystr, idx, &next_idx);
if (val == NULL) {
@@ -1792,10 +1836,6 @@
/* read any JSON term */
val = scan_once_unicode(s, pystr, idx, &next_idx);
if (val == NULL) {
- if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
- PyErr_Clear();
- raise_errmsg("Expecting object", pystr, idx);
- }
goto bail;
}
@@ -1814,10 +1854,10 @@
break;
}
else if (PyUnicode_READ(kind, str, idx) != ',') {
- raise_errmsg("Expecting ',' delimiter", pystr, idx);
+ raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx);
goto bail;
}
idx++;
/* skip whitespace after , */
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;
@@ -1818,7 +1858,8 @@
goto bail;
}
idx++;
/* skip whitespace after , */
while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++;
+ trailing_delimiter = 1;
}
@@ -1824,5 +1865,9 @@
}
+ if (trailing_delimiter) {
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
+ goto bail;
+ }
}
/* verify that idx < end_idx, str[idx] should be ']' */
if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') {
@@ -1825,8 +1870,12 @@
}
/* verify that idx < end_idx, str[idx] should be ']' */
if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') {
- raise_errmsg("Expecting object", pystr, end_idx);
+ if (PyList_GET_SIZE(rval)) {
+ raise_errmsg(ERR_ARRAY_DELIMITER, pystr, idx);
+ } else {
+ raise_errmsg(ERR_ARRAY_VALUE_FIRST, pystr, idx);
+ }
goto bail;
}
*next_idx_ptr = idx + 1;
@@ -1886,8 +1935,7 @@
/* read a sign if it's there, make sure it's not the end of the string */
if (str[idx] == '-') {
- idx++;
- if (idx > end_idx) {
- PyErr_SetNone(PyExc_StopIteration);
+ if (idx >= end_idx) {
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
@@ -1892,5 +1940,6 @@
return NULL;
}
+ idx++;
}
/* read as many integer digits as we find as long as it doesn't start with 0 */
@@ -1904,7 +1953,7 @@
}
/* no integer digits, error */
else {
- PyErr_SetNone(PyExc_StopIteration);
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
@@ -1993,8 +2042,7 @@
/* read a sign if it's there, make sure it's not the end of the string */
if (PyUnicode_READ(kind, str, idx) == '-') {
- idx++;
- if (idx > end_idx) {
- PyErr_SetNone(PyExc_StopIteration);
+ if (idx >= end_idx) {
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
@@ -1999,5 +2047,6 @@
return NULL;
}
+ idx++;
}
/* read as many integer digits as we find as long as it doesn't start with 0 */
@@ -2014,7 +2063,7 @@
}
else {
/* no integer digits, error */
- PyErr_SetNone(PyExc_StopIteration);
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
@@ -2097,7 +2146,7 @@
PyObject *rval = NULL;
int fallthrough = 0;
if (idx >= length) {
- PyErr_SetNone(PyExc_StopIteration);
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
switch (str[idx]) {
@@ -2205,7 +2254,7 @@
PyObject *rval = NULL;
int fallthrough = 0;
if (idx >= length) {
- PyErr_SetNone(PyExc_StopIteration);
+ raise_errmsg(ERR_EXPECTING_VALUE, pystr, idx);
return NULL;
}
switch (PyUnicode_READ(kind, str, idx)) {
diff --git a/simplejson/decoder.py b/simplejson/decoder.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2ltcGxlanNvbi9kZWNvZGVyLnB5..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2ltcGxlanNvbi9kZWNvZGVyLnB5 100644
--- a/simplejson/decoder.py
+++ b/simplejson/decoder.py
@@ -5,7 +5,8 @@
import sys
import struct
from .compat import fromhex, b, u, text_type, binary_type, PY3, unichr
-from .scanner import make_scanner
+from .scanner import make_scanner, JSONDecodeError
+
def _import_c_scanstring():
try:
from ._speedups import scanstring
@@ -14,6 +15,8 @@
return None
c_scanstring = _import_c_scanstring()
+# NOTE (3.1.0): JSONDecodeError may still be imported from this module for
+# compatibility, but it was never in the __all__
__all__ = ['JSONDecoder']
FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
@@ -29,57 +32,6 @@
NaN, PosInf, NegInf = _floatconstants()
-
-class JSONDecodeError(ValueError):
- """Subclass of ValueError with the following additional properties:
-
- msg: The unformatted error message
- doc: The JSON document being parsed
- pos: The start index of doc where parsing failed
- end: The end index of doc where parsing failed (may be None)
- lineno: The line corresponding to pos
- colno: The column corresponding to pos
- endlineno: The line corresponding to end (may be None)
- endcolno: The column corresponding to end (may be None)
-
- """
- def __init__(self, msg, doc, pos, end=None):
- ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
- self.msg = msg
- self.doc = doc
- self.pos = pos
- self.end = end
- self.lineno, self.colno = linecol(doc, pos)
- if end is not None:
- self.endlineno, self.endcolno = linecol(doc, end)
- else:
- self.endlineno, self.endcolno = None, None
-
-
-def linecol(doc, pos):
- lineno = doc.count('\n', 0, pos) + 1
- if lineno == 1:
- colno = pos + 1
- else:
- colno = pos - doc.rindex('\n', 0, pos)
- return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
- # Note that this function is called from _speedups
- lineno, colno = linecol(doc, pos)
- if end is None:
- #fmt = '{0}: line {1} column {2} (char {3})'
- #return fmt.format(msg, lineno, colno, pos)
- fmt = '%s: line %d column %d (char %d)'
- return fmt % (msg, lineno, colno, pos)
- endlineno, endcolno = linecol(doc, end)
- #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
- #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
- fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
- return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
-
-
_CONSTANTS = {
'-Infinity': NegInf,
'Infinity': PosInf,
@@ -128,8 +80,7 @@
break
elif terminator != '\\':
if strict:
- msg = "Invalid control character %r at" % (terminator,)
- #msg = "Invalid control character {0!r} at".format(terminator)
+ msg = "Invalid control character %r at"
raise JSONDecodeError(msg, s, end)
else:
_append(terminator)
@@ -144,8 +95,8 @@
try:
char = _b[esc]
except KeyError:
- msg = "Invalid \\escape: " + repr(esc)
+ msg = "Invalid \\X escape sequence %r"
raise JSONDecodeError(msg, s, end)
end += 1
else:
# Unicode escape sequence
@@ -148,7 +99,8 @@
raise JSONDecodeError(msg, s, end)
end += 1
else:
# Unicode escape sequence
+ msg = "Invalid \\uXXXX escape sequence"
esc = s[end + 1:end + 5]
next_end = end + 5
if len(esc) != 4:
@@ -152,8 +104,7 @@
esc = s[end + 1:end + 5]
next_end = end + 5
if len(esc) != 4:
- msg = "Invalid \\uXXXX escape"
raise JSONDecodeError(msg, s, end)
try:
uni = int(esc, 16)
except ValueError:
@@ -156,10 +107,9 @@
raise JSONDecodeError(msg, s, end)
try:
uni = int(esc, 16)
except ValueError:
- msg = "Invalid \\uXXXX escape"
raise JSONDecodeError(msg, s, end)
# Check for surrogate pair on UCS-4 systems
if _maxunicode > 65535:
unimask = uni & 0xfc00
if unimask == 0xd800:
@@ -161,9 +111,9 @@
raise JSONDecodeError(msg, s, end)
# Check for surrogate pair on UCS-4 systems
if _maxunicode > 65535:
unimask = uni & 0xfc00
if unimask == 0xd800:
- msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
+ msg = "Unpaired high surrogate"
if not s[end + 5:end + 7] == '\\u':
raise JSONDecodeError(msg, s, end)
esc2 = s[end + 7:end + 11]
@@ -174,7 +124,6 @@
except ValueError:
raise JSONDecodeError(msg, s, end)
if uni2 & 0xfc00 != 0xdc00:
- msg = "Unpaired high surrogate"
raise JSONDecodeError(msg, s, end)
uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
next_end += 6
@@ -246,10 +195,7 @@
except IndexError:
pass
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise JSONDecodeError("Expecting object", s, end)
+ value, end = scan_once(s, end)
pairs.append((key, value))
try:
@@ -264,7 +210,7 @@
if nextchar == '}':
break
elif nextchar != ',':
- raise JSONDecodeError("Expecting ',' delimiter", s, end - 1)
+ raise JSONDecodeError("Expecting ',' delimiter or '}'", s, end - 1)
try:
nextchar = s[end]
@@ -301,5 +247,7 @@
# Look-ahead for trivial empty array
if nextchar == ']':
return values, end + 1
+ elif nextchar == '':
+ raise JSONDecodeError("Expecting value or ']'", s, end)
_append = values.append
while True:
@@ -304,9 +252,6 @@
_append = values.append
while True:
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise JSONDecodeError("Expecting object", s, end)
+ value, end = scan_once(s, end)
_append(value)
nextchar = s[end:end + 1]
if nextchar in _ws:
@@ -316,7 +261,7 @@
if nextchar == ']':
break
elif nextchar != ',':
- raise JSONDecodeError("Expecting ',' delimiter", s, end)
+ raise JSONDecodeError("Expecting ',' delimiter or ']'", s, end - 1)
try:
if s[end] in _ws:
@@ -445,8 +390,4 @@
"""
if _PY3 and not isinstance(s, text_type):
raise TypeError("Input string must be text, not bytes")
- try:
- obj, end = self.scan_once(s, idx=_w(s, idx).end())
- except StopIteration:
- raise JSONDecodeError("No JSON object could be decoded", s, idx)
- return obj, end
+ return self.scan_once(s, idx=_w(s, idx).end())
diff --git a/simplejson/scanner.py b/simplejson/scanner.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2ltcGxlanNvbi9zY2FubmVyLnB5..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2ltcGxlanNvbi9zY2FubmVyLnB5 100644
--- a/simplejson/scanner.py
+++ b/simplejson/scanner.py
@@ -9,9 +9,9 @@
return None
c_make_scanner = _import_c_make_scanner()
-__all__ = ['make_scanner']
+__all__ = ['make_scanner', 'JSONDecodeError']
NUMBER_RE = re.compile(
r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
(re.VERBOSE | re.MULTILINE | re.DOTALL))
@@ -13,8 +13,55 @@
NUMBER_RE = re.compile(
r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
(re.VERBOSE | re.MULTILINE | re.DOTALL))
+class JSONDecodeError(ValueError):
+ """Subclass of ValueError with the following additional properties:
+
+ msg: The unformatted error message
+ doc: The JSON document being parsed
+ pos: The start index of doc where parsing failed
+ end: The end index of doc where parsing failed (may be None)
+ lineno: The line corresponding to pos
+ colno: The column corresponding to pos
+ endlineno: The line corresponding to end (may be None)
+ endcolno: The column corresponding to end (may be None)
+
+ """
+ # Note that this exception is used from _speedups
+ def __init__(self, msg, doc, pos, end=None):
+ ValueError.__init__(self, errmsg(msg, doc, pos, end=end))
+ self.msg = msg
+ self.doc = doc
+ self.pos = pos
+ self.end = end
+ self.lineno, self.colno = linecol(doc, pos)
+ if end is not None:
+ self.endlineno, self.endcolno = linecol(doc, end)
+ else:
+ self.endlineno, self.endcolno = None, None
+
+
+def linecol(doc, pos):
+ lineno = doc.count('\n', 0, pos) + 1
+ if lineno == 1:
+ colno = pos + 1
+ else:
+ colno = pos - doc.rindex('\n', 0, pos)
+ return lineno, colno
+
+
+def errmsg(msg, doc, pos, end=None):
+ lineno, colno = linecol(doc, pos)
+ msg = msg.replace('%r', repr(doc[pos:pos + 1]))
+ if end is None:
+ fmt = '%s: line %d column %d (char %d)'
+ return fmt % (msg, lineno, colno, pos)
+ endlineno, endcolno = linecol(doc, end)
+ fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
+ return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
+
+
def py_make_scanner(context):
parse_object = context.parse_object
parse_array = context.parse_array
@@ -30,6 +77,7 @@
memo = context.memo
def _scan_once(string, idx):
+ errmsg = 'Expecting value'
try:
nextchar = string[idx]
except IndexError:
@@ -33,7 +81,7 @@
try:
nextchar = string[idx]
except IndexError:
- raise StopIteration
+ raise JSONDecodeError(errmsg, string, idx)
if nextchar == '"':
return parse_string(string, idx + 1, encoding, strict)
@@ -64,7 +112,7 @@
elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
return parse_constant('-Infinity'), idx + 9
else:
- raise StopIteration
+ raise JSONDecodeError(errmsg, string, idx)
def scan_once(string, idx):
try:
diff --git a/simplejson/tests/test_fail.py b/simplejson/tests/test_fail.py
index 08df7633a87bc3079ba438dfaf5e6eeae2a56048_c2ltcGxlanNvbi90ZXN0cy90ZXN0X2ZhaWwucHk=..4d9b1fc3c56a8dda8b8a7d9d67b6f3159acdd98b_c2ltcGxlanNvbi90ZXN0cy90ZXN0X2ZhaWwucHk= 100644
--- a/simplejson/tests/test_fail.py
+++ b/simplejson/tests/test_fail.py
@@ -99,7 +99,6 @@
except json.JSONDecodeError:
pass
else:
- #self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc))
self.fail("Expected failure for fail%d.json: %r" % (idx, doc))
def test_array_decoder_issue46(self):
@@ -117,3 +116,43 @@
self.fail("Unexpected exception raised %r %s" % (e, e))
else:
self.fail("Unexpected success parsing '[,]'")
+
+ def test_truncated_input(self):
+ test_cases = [
+ ('', 'Expecting value', 0),
+ ('[', "Expecting value or ']'", 1),
+ ('[42', "Expecting ',' delimiter", 3),
+ ('[42,', 'Expecting value', 4),
+ ('["', 'Unterminated string starting at', 1),
+ ('["spam', 'Unterminated string starting at', 1),
+ ('["spam"', "Expecting ',' delimiter", 7),
+ ('["spam",', 'Expecting value', 8),
+ ('{', 'Expecting property name enclosed in double quotes', 1),
+ ('{"', 'Unterminated string starting at', 1),
+ ('{"spam', 'Unterminated string starting at', 1),
+ ('{"spam"', "Expecting ':' delimiter", 7),
+ ('{"spam":', 'Expecting value', 8),
+ ('{"spam":42', "Expecting ',' delimiter", 10),
+ ('{"spam":42,', 'Expecting property name enclosed in double quotes',
+ 11),
+ ('"', 'Unterminated string starting at', 0),
+ ('"spam', 'Unterminated string starting at', 0),
+ ('[,', "Expecting value", 1),
+ ]
+ for data, msg, idx in test_cases:
+ try:
+ json.loads(data)
+ except json.JSONDecodeError:
+ e = sys.exc_info()[1]
+ self.assertEqual(
+ e.msg[:len(msg)],
+ msg,
+ "%r doesn't start with %r for %r" % (e.msg, msg, data))
+ self.assertEqual(
+ e.pos, idx,
+ "pos %r != %r for %r" % (e.pos, idx, data))
+ except Exception:
+ e = sys.exc_info()[1]
+ self.fail("Unexpected exception raised %r %s" % (e, e))
+ else:
+ self.fail("Unexpected success parsing '%r'" % (data,))