# HG changeset patch
# User Ilya Etingof <etingof@gmail.com>
# Date 1569915411 -7200
# Tue Oct 01 09:36:51 2019 +0200
# Node ID dc19e59c48b1c73a34b7608d44abe1383494df9d
# Parent 5a19d79a302d6cc51cf8445636b33532aeb388ea
Optimize `streaming` objects access for performance
diff --git a/pyasn1/codec/ber/decoder.py b/pyasn1/codec/ber/decoder.py
--- a/pyasn1/codec/ber/decoder.py
+++ b/pyasn1/codec/ber/decoder.py
@@ -8,8 +8,11 @@
from pyasn1 import debug
from pyasn1 import error
-from pyasn1.codec import streaming
from pyasn1.codec.ber import eoo
+from pyasn1.codec.streaming import asSeekableStream
+from pyasn1.codec.streaming import isEndOfStream
+from pyasn1.codec.streaming import peekIntoStream
+from pyasn1.codec.streaming import readFromStream
from pyasn1.compat.integer import from_bytes
from pyasn1.compat.octets import oct2int, octs2ints, ints2octs, null
from pyasn1.error import PyAsn1Error
@@ -20,7 +23,6 @@
from pyasn1.type import univ
from pyasn1.type import useful
-
__all__ = ['StreamingDecoder', 'Decoder', 'decode']
LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER)
@@ -64,7 +66,7 @@
class AbstractSimplePayloadDecoder(AbstractPayloadDecoder):
@staticmethod
def substrateCollector(asn1Object, substrate, length, options):
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
yield chunk
def _createComponent(self, asn1Spec, tagSet, value, **options):
@@ -112,14 +114,12 @@
for value in decodeFun(
substrate, asn1Spec, tagSet, length,
allowEoo=True, **options):
+
if value is eoo.endOfOctets:
- break
+ return
yield value
- if value is eoo.endOfOctets:
- break
-
rawPayloadDecoder = RawPayloadDecoder()
@@ -135,7 +135,7 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -175,7 +175,7 @@
if not length:
raise error.PyAsn1Error('Empty BIT STRING substrate')
- for chunk in streaming.isEndOfStream(substrate):
+ for chunk in isEndOfStream(substrate):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -184,7 +184,7 @@
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
- for trailingBits in streaming.read(substrate, 1, options):
+ for trailingBits in readFromStream(substrate, 1, options):
if isinstance(trailingBits, SubstrateUnderrunError):
yield trailingBits
@@ -194,7 +194,7 @@
'Trailing bits overflow %s' % trailingBits
)
- for chunk in streaming.read(substrate, length - 1, options):
+ for chunk in readFromStream(substrate, length - 1, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -263,12 +263,12 @@
substrate, self.protoComponent, substrateFun=substrateFun,
allowEoo=True, **options):
+ if component is eoo.endOfOctets:
+ break
+
if isinstance(component, SubstrateUnderrunError):
yield component
- if component is eoo.endOfOctets:
- break
-
if component is eoo.endOfOctets:
break
@@ -303,7 +303,7 @@
return
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -383,7 +383,7 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -405,7 +405,7 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -465,7 +465,7 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -663,7 +663,7 @@
if substrate.tell() < original_position + length:
if LOG:
- for trailing in streaming.read(substrate, context=options):
+ for trailing in readFromStream(substrate, context=options):
if isinstance(trailing, SubstrateUnderrunError):
yield trailing
@@ -791,7 +791,7 @@
for pos, containerElement in enumerate(
containerValue):
- stream = streaming.asSeekableStream(containerValue[pos].asOctets())
+ stream = asSeekableStream(containerValue[pos].asOctets())
for component in decodeFun(stream, asn1Spec=openType, **options):
if isinstance(component, SubstrateUnderrunError):
@@ -800,7 +800,7 @@
containerValue[pos] = component
else:
- stream = streaming.asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
+ stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
for component in decodeFun(stream, asn1Spec=openType, **options):
if isinstance(component, SubstrateUnderrunError):
@@ -995,7 +995,7 @@
for pos, containerElement in enumerate(
containerValue):
- stream = streaming.asSeekableStream(containerValue[pos].asOctets())
+ stream = asSeekableStream(containerValue[pos].asOctets())
for component in decodeFun(stream, asn1Spec=openType,
**dict(options, allowEoo=True)):
@@ -1008,7 +1008,7 @@
containerValue[pos] = component
else:
- stream = streaming.asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
+ stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
for component in decodeFun(stream, asn1Spec=openType,
**dict(options, allowEoo=True)):
if isinstance(component, SubstrateUnderrunError):
@@ -1229,7 +1229,7 @@
length += currentPosition - fullPosition
if LOG:
- for chunk in streaming.peek(substrate, length):
+ for chunk in peekIntoStream(substrate, length):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
LOG('decoding as untagged ANY, substrate '
@@ -1243,7 +1243,7 @@
return
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -1275,7 +1275,7 @@
currentPosition = substrate.tell()
substrate.seek(fullPosition, os.SEEK_SET)
- for chunk in streaming.read(substrate, currentPosition - fullPosition, options):
+ for chunk in readFromStream(substrate, currentPosition - fullPosition, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
@@ -1476,7 +1476,7 @@
# Look for end-of-octets sentinel
if allowEoo and self.supportIndefLength:
- for eoo_candidate in streaming.read(substrate, 2, options):
+ for eoo_candidate in readFromStream(substrate, 2, options):
if isinstance(eoo_candidate, SubstrateUnderrunError):
yield eoo_candidate
@@ -1504,7 +1504,7 @@
# Decode tag
isShortTag = True
- for firstByte in streaming.read(substrate, 1, options):
+ for firstByte in readFromStream(substrate, 1, options):
if isinstance(firstByte, SubstrateUnderrunError):
yield firstByte
@@ -1525,7 +1525,7 @@
tagId = 0
while True:
- for integerByte in streaming.read(substrate, 1, options):
+ for integerByte in readFromStream(substrate, 1, options):
if isinstance(integerByte, SubstrateUnderrunError):
yield integerByte
@@ -1572,7 +1572,7 @@
if state is stDecodeLength:
# Decode length
- for firstOctet in streaming.read(substrate, 1, options):
+ for firstOctet in readFromStream(substrate, 1, options):
if isinstance(firstOctet, SubstrateUnderrunError):
yield firstOctet
@@ -1584,7 +1584,7 @@
elif firstOctet > 128:
size = firstOctet & 0x7F
# encoded in size bytes
- for encodedLength in streaming.read(substrate, size, options):
+ for encodedLength in readFromStream(substrate, size, options):
if isinstance(encodedLength, SubstrateUnderrunError):
yield encodedLength
encodedLength = list(encodedLength)
@@ -1873,7 +1873,7 @@
SINGLE_ITEM_DECODER = SingleItemDecoder
def __init__(self, substrate, asn1Spec=None, **kwargs):
- self._substrate = streaming.asSeekableStream(substrate)
+ self._substrate = asSeekableStream(substrate)
self._asn1Spec = asn1Spec
self._options = kwargs
self._decoder = self.SINGLE_ITEM_DECODER()
@@ -1884,7 +1884,7 @@
self._substrate, self._asn1Spec, **self._options):
yield asn1Object
- for chunk in streaming.isEndOfStream(self._substrate):
+ for chunk in isEndOfStream(self._substrate):
if isinstance(chunk, SubstrateUnderrunError):
yield
@@ -1960,14 +1960,14 @@
1 2 3
"""
- substrate = streaming.asSeekableStream(substrate)
+ substrate = asSeekableStream(substrate)
for asn1Object in cls.STREAMING_DECODER(substrate, asn1Spec, **kwargs):
if isinstance(asn1Object, SubstrateUnderrunError):
raise error.SubstrateUnderrunError('Short substrate on input')
try:
- tail = next(streaming.read(substrate))
+ tail = next(readFromStream(substrate))
except error.EndOfStreamError:
tail = null
diff --git a/pyasn1/codec/cer/decoder.py b/pyasn1/codec/cer/decoder.py
--- a/pyasn1/codec/cer/decoder.py
+++ b/pyasn1/codec/cer/decoder.py
@@ -5,7 +5,7 @@
# License: http://snmplabs.com/pyasn1/license.html
#
from pyasn1 import error
-from pyasn1.codec import streaming
+from pyasn1.codec.streaming import readFromStream
from pyasn1.codec.ber import decoder
from pyasn1.compat.octets import oct2int
from pyasn1.type import univ
@@ -26,7 +26,7 @@
if length != 1:
raise error.PyAsn1Error('Not single-octet Boolean payload')
- for chunk in streaming.read(substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
if isinstance(chunk, SubstrateUnderrunError):
yield chunk
diff --git a/pyasn1/codec/streaming.py b/pyasn1/codec/streaming.py
--- a/pyasn1/codec/streaming.py
+++ b/pyasn1/codec/streaming.py
@@ -154,8 +154,8 @@
yield not received
-def peek(substrate, size=-1):
- """Peek the stream.
+def peekIntoStream(substrate, size=-1):
+ """Peek into stream.
Parameters
----------
@@ -183,14 +183,14 @@
else:
current_position = substrate.tell()
try:
- for chunk in read(substrate, size):
+ for chunk in readFromStream(substrate, size):
yield chunk
finally:
substrate.seek(current_position)
-def read(substrate, size=-1, context=None):
+def readFromStream(substrate, size=-1, context=None):
"""Read from the stream.
Parameters
diff --git a/tests/codec/ber/test_decoder.py b/tests/codec/ber/test_decoder.py
--- a/tests/codec/ber/test_decoder.py
+++ b/tests/codec/ber/test_decoder.py
@@ -142,13 +142,13 @@
def testDefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((35, 8, 3, 2, 0, 169, 3, 2, 1, 138)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138, 0, 0)), str2octs(''))
def testTypeChecking(self):
@@ -186,14 +186,14 @@
assert decoder.decode(
ints2octs(
(36, 23, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((36, 128, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111,
120, 0, 0)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120, 0, 0)), str2octs(''))
@@ -246,7 +246,7 @@
def testDefModeSubst(self):
assert decoder.decode(
ints2octs((101, 17, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)), str2octs(''))
def testIndefModeSubst(self):
@@ -254,7 +254,7 @@
ints2octs((
101, 128, 36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0,
0, 0, 0)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0, 0, 0, 0)), str2octs(''))
@@ -682,13 +682,13 @@
def testWithOptionalAndDefaultedDefModeSubst(self):
assert decoder.decode(
ints2octs((48, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((48, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
@@ -1168,13 +1168,13 @@
def testWithOptionalAndDefaultedDefModeSubst(self):
assert decoder.decode(
ints2octs((49, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((49, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
@@ -1500,14 +1500,14 @@
assert decoder.decode(
ints2octs((4, 3, 102, 111, 120)),
asn1Spec=self.s,
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 3, 102, 111, 120)), str2octs(''))
def testTaggedExSubst(self):
assert decoder.decode(
ints2octs((164, 5, 4, 3, 102, 111, 120)),
asn1Spec=self.s,
- substrateFun=lambda a, b, c, d: streaming.read(b, c)
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((164, 5, 4, 3, 102, 111, 120)), str2octs(''))