diff --git a/CHANGES.rst b/CHANGES.rst
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_Q0hBTkdFUy5yc3Q=..429a07287d1b98eda26bd821c0e3006b0a59de28_Q0hBTkdFUy5yc3Q= 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,26 @@
+
+Revision 0.5.0, released XX-09-2019
+-----------------------------------
+
+- Make BER/CER/DER decodersstreaming and suspendible
+
+ The goal of this change is to make the decoder yielding on input
+ data starvation and resuming from where it stopped whenever the
+ caller decides to try again (hopefully making sure that some more
+ input becomes available).
+
+ This change makes it possible for the decoder to operate on streams
+ of data (meaning that the entire DER blob might not be immediately
+ available on input).
+
+ On top of that, the decoder yields partially reconstructed ASN.1
+ object on input starvation making it possible for the caller to
+ inspect what has been decoded so far and possibly consume partial
+ ASN.1 data.
+
+ All these new feature are natively available through
+ `StreamingDecoder` class. Previously published API is implemented
+ as a thin wrapper on top of that ensuring backward compatibility.
Revision 0.4.9, released XX-11-2019
-----------------------------------
diff --git a/README.md b/README.md
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_UkVBRE1FLm1k..429a07287d1b98eda26bd821c0e3006b0a59de28_UkVBRE1FLm1k 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,7 @@
* Generic implementation of ASN.1 types (X.208)
* Standards compliant BER/CER/DER codecs
+* Can operate on streams of serialized data
* Dumps/loads ASN.1 structures from Python types
* 100% Python, works with Python 2.7 and 3.5+
* MT-safe
diff --git a/pyasn1/codec/ber/decoder.py b/pyasn1/codec/ber/decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Jlci9kZWNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Jlci9kZWNvZGVyLnB5 100644
--- a/pyasn1/codec/ber/decoder.py
+++ b/pyasn1/codec/ber/decoder.py
@@ -4,6 +4,8 @@
# Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com>
# License: http://snmplabs.com/pyasn1/license.html
#
+import os
+
from pyasn1 import debug
from pyasn1 import error
from pyasn1.codec.ber import eoo
@@ -7,5 +9,9 @@
from pyasn1 import debug
from pyasn1 import error
from pyasn1.codec.ber import eoo
+from pyasn1.codec.streaming import asSeekableStream
+from pyasn1.codec.streaming import isEndOfStream
+from pyasn1.codec.streaming import peekIntoStream
+from pyasn1.codec.streaming import readFromStream
from pyasn1.compat.integer import from_bytes
from pyasn1.compat.octets import oct2int, octs2ints, ints2octs, null
@@ -10,5 +16,6 @@
from pyasn1.compat.integer import from_bytes
from pyasn1.compat.octets import oct2int, octs2ints, ints2octs, null
+from pyasn1.error import PyAsn1Error
from pyasn1.type import base
from pyasn1.type import char
from pyasn1.type import tag
@@ -16,9 +23,9 @@
from pyasn1.type import univ
from pyasn1.type import useful
-__all__ = ['decode']
+__all__ = ['StreamingDecoder', 'Decoder', 'decode']
LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER)
noValue = base.noValue
@@ -20,13 +27,15 @@
LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER)
noValue = base.noValue
-
-class AbstractDecoder(object):
+SubstrateUnderrunError = error.SubstrateUnderrunError
+
+
+class AbstractPayloadDecoder(object):
protoComponent = None
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
@@ -27,12 +36,16 @@
protoComponent = None
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- raise error.PyAsn1Error('Decoder not implemented for %s' % (tagSet,))
+ """Decode value with fixed byte length.
+
+ The decoder is allowed to consume as many bytes as necessary.
+ """
+ raise error.PyAsn1Error('SingleItemDecoder not implemented for %s' % (tagSet,)) # TODO: Seems more like an NotImplementedError?
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
@@ -34,10 +47,12 @@
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,))
-
-
-class AbstractSimpleDecoder(AbstractDecoder):
+ """Decode value with undefined length.
+
+ The decoder is allowed to consume as many bytes as necessary.
+ """
+ raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) # TODO: Seems more like an NotImplementedError?
+
@staticmethod
@@ -43,6 +58,16 @@
@staticmethod
- def substrateCollector(asn1Object, substrate, length):
- return substrate[:length], substrate[length:]
+ def _passAsn1Object(asn1Object, options):
+ if 'asn1Object' not in options:
+ options['asn1Object'] = asn1Object
+
+ return options
+
+
+class AbstractSimplePayloadDecoder(AbstractPayloadDecoder):
+ @staticmethod
+ def substrateCollector(asn1Object, substrate, length, options):
+ for chunk in readFromStream(substrate, length, options):
+ yield chunk
def _createComponent(self, asn1Spec, tagSet, value, **options):
if options.get('native'):
@@ -55,7 +80,7 @@
return asn1Spec.clone(value)
-class ExplicitTagDecoder(AbstractSimpleDecoder):
+class RawPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.Any('')
def valueDecoder(self, substrate, asn1Spec,
@@ -63,23 +88,18 @@
decodeFun=None, substrateFun=None,
**options):
if substrateFun:
- return substrateFun(
- self._createComponent(asn1Spec, tagSet, '', **options),
- substrate, length
- )
-
- head, tail = substrate[:length], substrate[length:]
-
- value, _ = decodeFun(head, asn1Spec, tagSet, length, **options)
-
- if LOG:
- LOG('explicit tag container carries %d octets of trailing payload '
- '(will be lost!): %s' % (len(_), debug.hexdump(_)))
-
- return value, tail
+ asn1Object = self._createComponent(asn1Spec, tagSet, '', **options)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
+
+ for value in decodeFun(substrate, asn1Spec, tagSet, length, **options):
+ yield value
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
if substrateFun:
@@ -80,28 +100,31 @@
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
if substrateFun:
- return substrateFun(
- self._createComponent(asn1Spec, tagSet, '', **options),
- substrate, length
- )
-
- value, substrate = decodeFun(substrate, asn1Spec, tagSet, length, **options)
-
- eooMarker, substrate = decodeFun(substrate, allowEoo=True, **options)
-
- if eooMarker is eoo.endOfOctets:
- return value, substrate
- else:
- raise error.PyAsn1Error('Missing end-of-octets terminator')
-
-
-explicitTagDecoder = ExplicitTagDecoder()
-
-
-class IntegerDecoder(AbstractSimpleDecoder):
+ asn1Object = self._createComponent(asn1Spec, tagSet, '', **options)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
+
+ while True:
+ for value in decodeFun(
+ substrate, asn1Spec, tagSet, length,
+ allowEoo=True, **options):
+
+ if value is eoo.endOfOctets:
+ return
+
+ yield value
+
+
+rawPayloadDecoder = RawPayloadDecoder()
+
+
+class IntegerPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.Integer(0)
def valueDecoder(self, substrate, asn1Spec,
@@ -112,17 +135,20 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- head, tail = substrate[:length], substrate[length:]
-
- if not head:
- return self._createComponent(asn1Spec, tagSet, 0, **options), tail
-
- value = from_bytes(head, signed=True)
-
- return self._createComponent(asn1Spec, tagSet, value, **options), tail
-
-
-class BooleanDecoder(IntegerDecoder):
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ if chunk:
+ value = from_bytes(chunk, signed=True)
+
+ else:
+ value = 0
+
+ yield self._createComponent(asn1Spec, tagSet, value, **options)
+
+
+class BooleanPayloadDecoder(IntegerPayloadDecoder):
protoComponent = univ.Boolean(0)
def _createComponent(self, asn1Spec, tagSet, value, **options):
@@ -126,7 +152,7 @@
protoComponent = univ.Boolean(0)
def _createComponent(self, asn1Spec, tagSet, value, **options):
- return IntegerDecoder._createComponent(
+ return IntegerPayloadDecoder._createComponent(
self, asn1Spec, tagSet, value and 1 or 0, **options)
@@ -130,7 +156,7 @@
self, asn1Spec, tagSet, value and 1 or 0, **options)
-class BitStringDecoder(AbstractSimpleDecoder):
+class BitStringPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.BitString(())
supportConstructedForm = True
@@ -138,6 +164,5 @@
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- head, tail = substrate[:length], substrate[length:]
if substrateFun:
@@ -142,10 +167,21 @@
if substrateFun:
- return substrateFun(self._createComponent(
- asn1Spec, tagSet, noValue, **options), substrate, length)
-
- if not head:
+ asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
+
+ if not length:
+ raise error.PyAsn1Error('Empty BIT STRING substrate')
+
+ for chunk in isEndOfStream(substrate):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ if chunk:
raise error.PyAsn1Error('Empty BIT STRING substrate')
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
@@ -148,10 +184,14 @@
raise error.PyAsn1Error('Empty BIT STRING substrate')
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
- trailingBits = oct2int(head[0])
+ for trailingBits in readFromStream(substrate, 1, options):
+ if isinstance(trailingBits, SubstrateUnderrunError):
+ yield trailingBits
+
+ trailingBits = ord(trailingBits)
if trailingBits > 7:
raise error.PyAsn1Error(
'Trailing bits overflow %s' % trailingBits
)
@@ -153,6 +193,10 @@
if trailingBits > 7:
raise error.PyAsn1Error(
'Trailing bits overflow %s' % trailingBits
)
+ for chunk in readFromStream(substrate, length - 1, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
value = self.protoComponent.fromOctetString(
@@ -158,7 +202,9 @@
value = self.protoComponent.fromOctetString(
- head[1:], internalFormat=True, padding=trailingBits)
-
- return self._createComponent(asn1Spec, tagSet, value, **options), tail
+ chunk, internalFormat=True, padding=trailingBits)
+
+ yield self._createComponent(asn1Spec, tagSet, value, **options)
+
+ return
if not self.supportConstructedForm:
raise error.PyAsn1Error('Constructed encoding form prohibited '
@@ -172,9 +218,14 @@
bitString = self.protoComponent.fromOctetString(null, internalFormat=True)
- while head:
- component, head = decodeFun(head, self.protoComponent,
- substrateFun=substrateFun, **options)
+ current_position = substrate.tell()
+
+ while substrate.tell() - current_position < length:
+ for component in decodeFun(
+ substrate, self.protoComponent, substrateFun=substrateFun,
+ **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
trailingBits = oct2int(component[0])
if trailingBits > 7:
@@ -187,7 +238,7 @@
prepend=bitString, padding=trailingBits
)
- return self._createComponent(asn1Spec, tagSet, bitString, **options), tail
+ yield self._createComponent(asn1Spec, tagSet, bitString, **options)
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -195,10 +246,15 @@
**options):
if substrateFun:
- return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), substrate, length)
+ asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
# All inner fragments are of the same type, treat them as octet string
substrateFun = self.substrateCollector
bitString = self.protoComponent.fromOctetString(null, internalFormat=True)
@@ -199,13 +255,21 @@
# All inner fragments are of the same type, treat them as octet string
substrateFun = self.substrateCollector
bitString = self.protoComponent.fromOctetString(null, internalFormat=True)
- while substrate:
- component, substrate = decodeFun(substrate, self.protoComponent,
- substrateFun=substrateFun,
- allowEoo=True, **options)
+ while True: # loop over fragments
+
+ for component in decodeFun(
+ substrate, self.protoComponent, substrateFun=substrateFun,
+ allowEoo=True, **options):
+
+ if component is eoo.endOfOctets:
+ break
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
if component is eoo.endOfOctets:
break
@@ -220,13 +284,10 @@
prepend=bitString, padding=trailingBits
)
- else:
- raise error.SubstrateUnderrunError('No EOO seen before substrate ends')
-
- return self._createComponent(asn1Spec, tagSet, bitString, **options), substrate
-
-
-class OctetStringDecoder(AbstractSimpleDecoder):
+ yield self._createComponent(asn1Spec, tagSet, bitString, **options)
+
+
+class OctetStringPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.OctetString('')
supportConstructedForm = True
@@ -234,6 +295,4 @@
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- head, tail = substrate[:length], substrate[length:]
-
if substrateFun:
@@ -239,5 +298,9 @@
if substrateFun:
- return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options),
- substrate, length)
+ asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
@@ -242,6 +305,12 @@
if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check?
- return self._createComponent(asn1Spec, tagSet, head, **options), tail
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ yield self._createComponent(asn1Spec, tagSet, chunk, **options)
+
+ return
if not self.supportConstructedForm:
raise error.PyAsn1Error('Constructed encoding form prohibited at %s' % self.__class__.__name__)
@@ -254,9 +323,14 @@
header = null
- while head:
- component, head = decodeFun(head, self.protoComponent,
- substrateFun=substrateFun,
- **options)
+ original_position = substrate.tell()
+ # head = popSubstream(substrate, length)
+ while substrate.tell() - original_position < length:
+ for component in decodeFun(
+ substrate, self.protoComponent, substrateFun=substrateFun,
+ **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
header += component
@@ -261,6 +335,6 @@
header += component
- return self._createComponent(asn1Spec, tagSet, header, **options), tail
+ yield self._createComponent(asn1Spec, tagSet, header, **options)
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -268,10 +342,14 @@
**options):
if substrateFun and substrateFun is not self.substrateCollector:
asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options)
- return substrateFun(asn1Object, substrate, length)
+
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
# All inner fragments are of the same type, treat them as octet string
substrateFun = self.substrateCollector
header = null
@@ -272,16 +350,23 @@
# All inner fragments are of the same type, treat them as octet string
substrateFun = self.substrateCollector
header = null
- while substrate:
- component, substrate = decodeFun(substrate,
- self.protoComponent,
- substrateFun=substrateFun,
- allowEoo=True, **options)
+ while True: # loop over fragments
+
+ for component in decodeFun(
+ substrate, self.protoComponent, substrateFun=substrateFun,
+ allowEoo=True, **options):
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
+
if component is eoo.endOfOctets:
break
header += component
@@ -283,17 +368,12 @@
if component is eoo.endOfOctets:
break
header += component
- else:
- raise error.SubstrateUnderrunError(
- 'No EOO seen before substrate ends'
- )
-
- return self._createComponent(asn1Spec, tagSet, header, **options), substrate
-
-
-class NullDecoder(AbstractSimpleDecoder):
+ yield self._createComponent(asn1Spec, tagSet, header, **options)
+
+
+class NullPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.Null('')
def valueDecoder(self, substrate, asn1Spec,
@@ -304,7 +384,9 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- head, tail = substrate[:length], substrate[length:]
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
component = self._createComponent(asn1Spec, tagSet, '', **options)
@@ -308,6 +390,6 @@
component = self._createComponent(asn1Spec, tagSet, '', **options)
- if head:
+ if chunk:
raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length)
@@ -312,9 +394,9 @@
raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length)
- return component, tail
-
-
-class ObjectIdentifierDecoder(AbstractSimpleDecoder):
+ yield component
+
+
+class ObjectIdentifierPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.ObjectIdentifier(())
def valueDecoder(self, substrate, asn1Spec,
@@ -324,7 +406,10 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- head, tail = substrate[:length], substrate[length:]
- if not head:
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ if not chunk:
raise error.PyAsn1Error('Empty substrate')
@@ -329,6 +414,6 @@
raise error.PyAsn1Error('Empty substrate')
- head = octs2ints(head)
+ chunk = octs2ints(chunk)
oid = ()
index = 0
@@ -332,5 +417,5 @@
oid = ()
index = 0
- substrateLen = len(head)
+ substrateLen = len(chunk)
while index < substrateLen:
@@ -336,5 +421,5 @@
while index < substrateLen:
- subId = head[index]
+ subId = chunk[index]
index += 1
if subId < 128:
oid += (subId,)
@@ -348,7 +433,7 @@
raise error.SubstrateUnderrunError(
'Short substrate for sub-OID past %s' % (oid,)
)
- nextSubId = head[index]
+ nextSubId = chunk[index]
index += 1
oid += ((subId << 7) + nextSubId,)
elif subId == 128:
@@ -366,12 +451,12 @@
elif oid[0] >= 80:
oid = (2, oid[0] - 80) + oid[1:]
else:
- raise error.PyAsn1Error('Malformed first OID octet: %s' % head[0])
-
- return self._createComponent(asn1Spec, tagSet, oid, **options), tail
-
-
-class RealDecoder(AbstractSimpleDecoder):
+ raise error.PyAsn1Error('Malformed first OID octet: %s' % chunk[0])
+
+ yield self._createComponent(asn1Spec, tagSet, oid, **options)
+
+
+class RealPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.Real()
def valueDecoder(self, substrate, asn1Spec,
@@ -381,11 +466,14 @@
if tagSet[0].tagFormat != tag.tagFormatSimple:
raise error.PyAsn1Error('Simple tag format expected')
- head, tail = substrate[:length], substrate[length:]
-
- if not head:
- return self._createComponent(asn1Spec, tagSet, 0.0, **options), tail
-
- fo = oct2int(head[0])
- head = head[1:]
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ if not chunk:
+ yield self._createComponent(asn1Spec, tagSet, 0.0, **options)
+ return
+
+ fo = oct2int(chunk[0])
+ chunk = chunk[1:]
if fo & 0x80: # binary encoding
@@ -391,5 +479,5 @@
if fo & 0x80: # binary encoding
- if not head:
+ if not chunk:
raise error.PyAsn1Error("Incomplete floating-point value")
if LOG:
@@ -398,12 +486,12 @@
n = (fo & 0x03) + 1
if n == 4:
- n = oct2int(head[0])
- head = head[1:]
-
- eo, head = head[:n], head[n:]
-
- if not eo or not head:
+ n = oct2int(chunk[0])
+ chunk = chunk[1:]
+
+ eo, chunk = chunk[:n], chunk[n:]
+
+ if not eo or not chunk:
raise error.PyAsn1Error('Real exponent screwed')
e = oct2int(eo[0]) & 0x80 and -1 or 0
@@ -425,5 +513,5 @@
e *= 4
p = 0
- while head: # value
+ while chunk: # value
p <<= 8
@@ -429,6 +517,6 @@
p <<= 8
- p |= oct2int(head[0])
- head = head[1:]
+ p |= oct2int(chunk[0])
+ chunk = chunk[1:]
if fo & 0x40: # sign bit
p = -p
@@ -444,7 +532,7 @@
value = fo & 0x01 and '-inf' or 'inf'
elif fo & 0xc0 == 0: # character encoding
- if not head:
+ if not chunk:
raise error.PyAsn1Error("Incomplete floating-point value")
if LOG:
@@ -452,6 +540,6 @@
try:
if fo & 0x3 == 0x1: # NR1
- value = (int(head), 10, 0)
+ value = (int(chunk), 10, 0)
elif fo & 0x3 == 0x2: # NR2
@@ -456,5 +544,5 @@
elif fo & 0x3 == 0x2: # NR2
- value = float(head)
+ value = float(chunk)
elif fo & 0x3 == 0x3: # NR3
@@ -459,6 +547,6 @@
elif fo & 0x3 == 0x3: # NR3
- value = float(head)
+ value = float(chunk)
else:
raise error.SubstrateUnderrunError(
@@ -475,10 +563,10 @@
'Unknown encoding (tag %s)' % fo
)
- return self._createComponent(asn1Spec, tagSet, value, **options), tail
-
-
-class AbstractConstructedDecoder(AbstractDecoder):
+ yield self._createComponent(asn1Spec, tagSet, value, **options)
+
+
+class AbstractConstructedPayloadDecoder(AbstractPayloadDecoder):
protoComponent = None
@@ -482,7 +570,7 @@
protoComponent = None
-class UniversalConstructedTypeDecoder(AbstractConstructedDecoder):
+class ConstructedPayloadDecoderBase(AbstractConstructedPayloadDecoder):
protoRecordComponent = None
protoSequenceComponent = None
@@ -492,7 +580,12 @@
def _getComponentPositionByType(self, asn1Object, tagSet, idx):
raise NotImplementedError()
- def _decodeComponents(self, substrate, tagSet=None, decodeFun=None, **options):
+ def _decodeComponentsSchemaless(
+ self, substrate, tagSet=None, decodeFun=None,
+ length=None, **options):
+
+ asn1Object = None
+
components = []
componentTypes = set()
@@ -496,11 +589,16 @@
components = []
componentTypes = set()
- while substrate:
- component, substrate = decodeFun(substrate, **options)
- if component is eoo.endOfOctets:
+ original_position = substrate.tell()
+
+ while length == -1 or substrate.tell() < original_position + length:
+ for component in decodeFun(substrate, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if length == -1 and component is eoo.endOfOctets:
break
components.append(component)
componentTypes.add(component.tagSet)
@@ -502,23 +600,23 @@
break
components.append(component)
componentTypes.add(component.tagSet)
- # Now we have to guess is it SEQUENCE/SET or SEQUENCE OF/SET OF
- # The heuristics is:
- # * 1+ components of different types -> likely SEQUENCE/SET
- # * otherwise -> likely SEQUENCE OF/SET OF
- if len(componentTypes) > 1:
- protoComponent = self.protoRecordComponent
-
- else:
- protoComponent = self.protoSequenceComponent
-
- asn1Object = protoComponent.clone(
- # construct tagSet from base tag from prototype ASN.1 object
- # and additional tags recovered from the substrate
- tagSet=tag.TagSet(protoComponent.tagSet.baseTag, *tagSet.superTags)
- )
+ # Now we have to guess is it SEQUENCE/SET or SEQUENCE OF/SET OF
+ # The heuristics is:
+ # * 1+ components of different types -> likely SEQUENCE/SET
+ # * otherwise -> likely SEQUENCE OF/SET OF
+ if len(componentTypes) > 1:
+ protoComponent = self.protoRecordComponent
+
+ else:
+ protoComponent = self.protoSequenceComponent
+
+ asn1Object = protoComponent.clone(
+ # construct tagSet from base tag from prototype ASN.1 object
+ # and additional tags recovered from the substrate
+ tagSet=tag.TagSet(protoComponent.tagSet.baseTag, *tagSet.superTags)
+ )
if LOG:
LOG('guessed %r container type (pass `asn1Spec` to guide the '
@@ -531,7 +629,7 @@
matchTags=False, matchConstraints=False
)
- return asn1Object, substrate
+ yield asn1Object
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -540,9 +638,9 @@
if tagSet[0].tagFormat != tag.tagFormatConstructed:
raise error.PyAsn1Error('Constructed tag format expected')
- head, tail = substrate[:length], substrate[length:]
-
- if substrateFun is not None:
+ original_position = substrate.tell()
+
+ if substrateFun:
if asn1Spec is not None:
asn1Object = asn1Spec.clone()
@@ -552,6 +650,9 @@
else:
asn1Object = self.protoRecordComponent, self.protoSequenceComponent
- return substrateFun(asn1Object, substrate, length)
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
if asn1Spec is None:
@@ -556,8 +657,10 @@
if asn1Spec is None:
- asn1Object, trailing = self._decodeComponents(
- head, tagSet=tagSet, decodeFun=decodeFun, **options
- )
-
- if trailing:
+ for asn1Object in self._decodeComponentsSchemaless(
+ substrate, tagSet=tagSet, decodeFun=decodeFun,
+ length=length, **options):
+ if isinstance(asn1Object, SubstrateUnderrunError):
+ yield asn1Object
+
+ if substrate.tell() < original_position + length:
if LOG:
@@ -563,4 +666,8 @@
if LOG:
+ for trailing in readFromStream(substrate, context=options):
+ if isinstance(trailing, SubstrateUnderrunError):
+ yield trailing
+
LOG('Unused trailing %d octets encountered: %s' % (
len(trailing), debug.hexdump(trailing)))
@@ -564,8 +671,10 @@
LOG('Unused trailing %d octets encountered: %s' % (
len(trailing), debug.hexdump(trailing)))
- return asn1Object, tail
+ yield asn1Object
+
+ return
asn1Object = asn1Spec.clone()
asn1Object.clear()
@@ -568,7 +677,9 @@
asn1Object = asn1Spec.clone()
asn1Object.clear()
+ options = self._passAsn1Object(asn1Object, options)
+
if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId):
namedTypes = asn1Spec.componentType
@@ -583,7 +694,7 @@
seenIndices = set()
idx = 0
- while head:
+ while substrate.tell() - original_position < length:
if not namedTypes:
componentType = None
@@ -606,7 +717,9 @@
'Excessive components decoded at %r' % (asn1Spec,)
)
- component, head = decodeFun(head, componentType, **options)
+ for component in decodeFun(substrate, componentType, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
if not isDeterministic and namedTypes:
if isSetType:
@@ -693,11 +806,12 @@
for pos, containerElement in enumerate(
containerValue):
- component, rest = decodeFun(
- containerValue[pos].asOctets(),
- asn1Spec=openType, **options
- )
+ stream = asSeekableStream(containerValue[pos].asOctets())
+
+ for component in decodeFun(stream, asn1Spec=openType, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
containerValue[pos] = component
else:
@@ -700,11 +814,12 @@
containerValue[pos] = component
else:
- component, rest = decodeFun(
- asn1Object.getComponentByPosition(idx).asOctets(),
- asn1Spec=openType, **options
- )
+ stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
+
+ for component in decodeFun(stream, asn1Spec=openType, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
asn1Object.setComponentByPosition(idx, component)
@@ -714,9 +829,6 @@
raise inconsistency
else:
- asn1Object = asn1Spec.clone()
- asn1Object.clear()
-
componentType = asn1Spec.componentType
if LOG:
@@ -724,8 +836,11 @@
idx = 0
- while head:
- component, head = decodeFun(head, componentType, **options)
+ while substrate.tell() - original_position < length:
+ for component in decodeFun(substrate, componentType, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
asn1Object.setComponentByPosition(
idx, component,
verifyConstraints=False,
@@ -734,7 +849,7 @@
idx += 1
- return asn1Object, tail
+ yield asn1Object
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -753,6 +868,9 @@
else:
asn1Object = self.protoRecordComponent, self.protoSequenceComponent
- return substrateFun(asn1Object, substrate, length)
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
if asn1Spec is None:
@@ -757,10 +875,15 @@
if asn1Spec is None:
- return self._decodeComponents(
- substrate, tagSet=tagSet, decodeFun=decodeFun,
- **dict(options, allowEoo=True)
- )
+ for asn1Object in self._decodeComponentsSchemaless(
+ substrate, tagSet=tagSet, decodeFun=decodeFun,
+ length=length, **dict(options, allowEoo=True)):
+ if isinstance(asn1Object, SubstrateUnderrunError):
+ yield asn1Object
+
+ yield asn1Object
+
+ return
asn1Object = asn1Spec.clone()
asn1Object.clear()
@@ -763,7 +886,9 @@
asn1Object = asn1Spec.clone()
asn1Object.clear()
+ options = self._passAsn1Object(asn1Object, options)
+
if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId):
namedTypes = asn1Object.componentType
@@ -777,4 +902,5 @@
asn1Spec))
seenIndices = set()
+
idx = 0
@@ -780,5 +906,6 @@
idx = 0
- while substrate:
+
+ while True: # loop over components
if len(namedTypes) <= idx:
asn1Spec = None
@@ -801,10 +928,17 @@
'Excessive components decoded at %r' % (asn1Object,)
)
- component, substrate = decodeFun(substrate, asn1Spec, allowEoo=True, **options)
+ for component in decodeFun(substrate, asn1Spec, allowEoo=True, **options):
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
+
if component is eoo.endOfOctets:
break
if not isDeterministic and namedTypes:
if isSetType:
idx = namedTypes.getPositionByType(component.effectiveTagSet)
@@ -805,9 +939,10 @@
if component is eoo.endOfOctets:
break
if not isDeterministic and namedTypes:
if isSetType:
idx = namedTypes.getPositionByType(component.effectiveTagSet)
+
elif namedTypes[idx].isOptional or namedTypes[idx].isDefaulted:
idx = namedTypes.getPositionNearType(component.effectiveTagSet, idx)
@@ -820,13 +955,8 @@
seenIndices.add(idx)
idx += 1
- else:
- raise error.SubstrateUnderrunError(
- 'No EOO seen before substrate ends'
- )
-
if LOG:
LOG('seen component indices %s' % seenIndices)
if namedTypes:
if not namedTypes.requiredComponents.issubset(seenIndices):
@@ -828,9 +958,11 @@
if LOG:
LOG('seen component indices %s' % seenIndices)
if namedTypes:
if not namedTypes.requiredComponents.issubset(seenIndices):
- raise error.PyAsn1Error('ASN.1 object %s has uninitialized components' % asn1Object.__class__.__name__)
+ raise error.PyAsn1Error(
+ 'ASN.1 object %s has uninitialized '
+ 'components' % asn1Object.__class__.__name__)
if namedTypes.hasOpenTypes:
@@ -892,11 +1024,16 @@
for pos, containerElement in enumerate(
containerValue):
- component, rest = decodeFun(
- containerValue[pos].asOctets(),
- asn1Spec=openType, **dict(options, allowEoo=True)
- )
+ stream = asSeekableStream(containerValue[pos].asOctets())
+
+ for component in decodeFun(stream, asn1Spec=openType,
+ **dict(options, allowEoo=True)):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
containerValue[pos] = component
else:
@@ -899,13 +1036,16 @@
containerValue[pos] = component
else:
- component, rest = decodeFun(
- asn1Object.getComponentByPosition(idx).asOctets(),
- asn1Spec=openType, **dict(options, allowEoo=True)
- )
-
- if component is not eoo.endOfOctets:
+ stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets())
+ for component in decodeFun(stream, asn1Spec=openType,
+ **dict(options, allowEoo=True)):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
+
asn1Object.setComponentByPosition(idx, component)
else:
@@ -914,9 +1054,6 @@
raise inconsistency
else:
- asn1Object = asn1Spec.clone()
- asn1Object.clear()
-
componentType = asn1Spec.componentType
if LOG:
@@ -924,8 +1061,16 @@
idx = 0
- while substrate:
- component, substrate = decodeFun(substrate, componentType, allowEoo=True, **options)
+ while True:
+
+ for component in decodeFun(
+ substrate, componentType, allowEoo=True, **options):
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
if component is eoo.endOfOctets:
break
@@ -938,16 +1083,11 @@
idx += 1
- else:
- raise error.SubstrateUnderrunError(
- 'No EOO seen before substrate ends'
- )
-
- return asn1Object, substrate
-
-
-class SequenceOrSequenceOfDecoder(UniversalConstructedTypeDecoder):
+ yield asn1Object
+
+
+class SequenceOrSequenceOfPayloadDecoder(ConstructedPayloadDecoderBase):
protoRecordComponent = univ.Sequence()
protoSequenceComponent = univ.SequenceOf()
@@ -950,8 +1090,8 @@
protoRecordComponent = univ.Sequence()
protoSequenceComponent = univ.SequenceOf()
-class SequenceDecoder(SequenceOrSequenceOfDecoder):
+class SequencePayloadDecoder(SequenceOrSequenceOfPayloadDecoder):
protoComponent = univ.Sequence()
@@ -955,7 +1095,7 @@
protoComponent = univ.Sequence()
-class SequenceOfDecoder(SequenceOrSequenceOfDecoder):
+class SequenceOfPayloadDecoder(SequenceOrSequenceOfPayloadDecoder):
protoComponent = univ.SequenceOf()
@@ -959,8 +1099,8 @@
protoComponent = univ.SequenceOf()
-class SetOrSetOfDecoder(UniversalConstructedTypeDecoder):
+class SetOrSetOfPayloadDecoder(ConstructedPayloadDecoderBase):
protoRecordComponent = univ.Set()
protoSequenceComponent = univ.SetOf()
@@ -963,8 +1103,8 @@
protoRecordComponent = univ.Set()
protoSequenceComponent = univ.SetOf()
-class SetDecoder(SetOrSetOfDecoder):
+class SetPayloadDecoder(SetOrSetOfPayloadDecoder):
protoComponent = univ.Set()
@@ -968,8 +1108,7 @@
protoComponent = univ.Set()
-
-class SetOfDecoder(SetOrSetOfDecoder):
+class SetOfPayloadDecoder(SetOrSetOfPayloadDecoder):
protoComponent = univ.SetOf()
@@ -973,10 +1112,10 @@
protoComponent = univ.SetOf()
-class ChoiceDecoder(AbstractConstructedDecoder):
+class ChoicePayloadDecoder(ConstructedPayloadDecoderBase):
protoComponent = univ.Choice()
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
@@ -977,11 +1116,9 @@
protoComponent = univ.Choice()
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- head, tail = substrate[:length], substrate[length:]
-
if asn1Spec is None:
asn1Object = self.protoComponent.clone(tagSet=tagSet)
@@ -989,9 +1126,14 @@
asn1Object = asn1Spec.clone()
if substrateFun:
- return substrateFun(asn1Object, substrate, length)
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
+
+ options = self._passAsn1Object(asn1Object, options)
if asn1Object.tagSet == tagSet:
if LOG:
LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,))
@@ -993,13 +1135,14 @@
if asn1Object.tagSet == tagSet:
if LOG:
LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,))
- component, head = decodeFun(
- head, asn1Object.componentTagMap, **options
- )
+ for component in decodeFun(
+ substrate, asn1Object.componentTagMap, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
else:
if LOG:
LOG('decoding %s as untagged CHOICE' % (tagSet,))
@@ -1001,12 +1144,13 @@
else:
if LOG:
LOG('decoding %s as untagged CHOICE' % (tagSet,))
- component, head = decodeFun(
- head, asn1Object.componentTagMap,
- tagSet, length, state, **options
- )
+ for component in decodeFun(
+ substrate, asn1Object.componentTagMap, tagSet, length,
+ state, **options):
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
effectiveTagSet = component.effectiveTagSet
@@ -1020,7 +1164,7 @@
innerFlag=False
)
- return asn1Object, tail
+ yield asn1Object
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -1028,7 +1172,8 @@
**options):
if asn1Spec is None:
asn1Object = self.protoComponent.clone(tagSet=tagSet)
+
else:
asn1Object = asn1Spec.clone()
if substrateFun:
@@ -1031,34 +1176,14 @@
else:
asn1Object = asn1Spec.clone()
if substrateFun:
- return substrateFun(asn1Object, substrate, length)
-
- if asn1Object.tagSet == tagSet:
- if LOG:
- LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,))
-
- component, substrate = decodeFun(
- substrate, asn1Object.componentType.tagMapUnique, **options
- )
-
- # eat up EOO marker
- eooMarker, substrate = decodeFun(
- substrate, allowEoo=True, **options
- )
-
- if eooMarker is not eoo.endOfOctets:
- raise error.PyAsn1Error('No EOO seen before substrate ends')
-
- else:
- if LOG:
- LOG('decoding %s as untagged CHOICE' % (tagSet,))
-
- component, substrate = decodeFun(
- substrate, asn1Object.componentType.tagMapUnique,
- tagSet, length, state, **options
- )
-
- effectiveTagSet = component.effectiveTagSet
+ for chunk in substrateFun(asn1Object, substrate, length, options):
+ yield chunk
+
+ return
+
+ options = self._passAsn1Object(asn1Object, options)
+
+ isTagged = asn1Object.tagSet == tagSet
if LOG:
@@ -1063,18 +1188,51 @@
if LOG:
- LOG('decoded component %s, effective tag set %s' % (component, effectiveTagSet))
-
- asn1Object.setComponentByType(
- effectiveTagSet, component,
- verifyConstraints=False,
- matchTags=False, matchConstraints=False,
- innerFlag=False
- )
-
- return asn1Object, substrate
-
-
-class AnyDecoder(AbstractSimpleDecoder):
+ LOG('decoding %s as %stagged CHOICE' % (
+ tagSet, isTagged and 'explicitly ' or 'un'))
+
+ while True:
+
+ if isTagged:
+ iterator = decodeFun(
+ substrate, asn1Object.componentType.tagMapUnique,
+ **dict(options, allowEoo=True))
+
+ else:
+ iterator = decodeFun(
+ substrate, asn1Object.componentType.tagMapUnique,
+ tagSet, length, state, **dict(options, allowEoo=True))
+
+ for component in iterator:
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
+
+ effectiveTagSet = component.effectiveTagSet
+
+ if LOG:
+ LOG('decoded component %s, effective tag set '
+ '%s' % (component, effectiveTagSet))
+
+ asn1Object.setComponentByType(
+ effectiveTagSet, component,
+ verifyConstraints=False,
+ matchTags=False, matchConstraints=False,
+ innerFlag=False
+ )
+
+ if not isTagged:
+ break
+
+ if not isTagged or component is eoo.endOfOctets:
+ break
+
+ yield asn1Object
+
+
+class AnyPayloadDecoder(AbstractSimplePayloadDecoder):
protoComponent = univ.Any()
def valueDecoder(self, substrate, asn1Spec,
@@ -1091,10 +1249,10 @@
isUntagged = tagSet != asn1Spec.tagSet
if isUntagged:
- fullSubstrate = options['fullSubstrate']
-
- # untagged Any container, recover inner header substrate
- length += len(fullSubstrate) - len(substrate)
- substrate = fullSubstrate
+ fullPosition = substrate.markedPosition
+ currentPosition = substrate.tell()
+
+ substrate.seek(fullPosition, os.SEEK_SET)
+ length += currentPosition - fullPosition
if LOG:
@@ -1099,5 +1257,9 @@
if LOG:
- LOG('decoding as untagged ANY, substrate %s' % debug.hexdump(substrate))
+ for chunk in peekIntoStream(substrate, length):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+ LOG('decoding as untagged ANY, substrate '
+ '%s' % debug.hexdump(chunk))
if substrateFun:
@@ -1102,11 +1264,17 @@
if substrateFun:
- return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options),
- substrate, length)
-
- head, tail = substrate[:length], substrate[length:]
-
- return self._createComponent(asn1Spec, tagSet, head, **options), tail
+ for chunk in substrateFun(
+ self._createComponent(asn1Spec, tagSet, noValue, **options),
+ substrate, length, options):
+ yield chunk
+
+ return
+
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ yield self._createComponent(asn1Spec, tagSet, chunk, **options)
def indefLenValueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
@@ -1123,9 +1291,9 @@
if isTagged:
# tagged Any type -- consume header substrate
- header = null
+ chunk = null
if LOG:
LOG('decoding as tagged ANY')
else:
@@ -1127,11 +1295,15 @@
if LOG:
LOG('decoding as tagged ANY')
else:
- fullSubstrate = options['fullSubstrate']
-
- # untagged Any, recover header substrate
- header = fullSubstrate[:-len(substrate)]
+ # TODO: Seems not to be tested
+ fullPosition = substrate.markedPosition
+ currentPosition = substrate.tell()
+
+ substrate.seek(fullPosition, os.SEEK_SET)
+ for chunk in readFromStream(substrate, currentPosition - fullPosition, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
if LOG:
@@ -1136,8 +1308,8 @@
if LOG:
- LOG('decoding as untagged ANY, header substrate %s' % debug.hexdump(header))
+ LOG('decoding as untagged ANY, header substrate %s' % debug.hexdump(chunk))
# Any components do not inherit initial tag
asn1Spec = self.protoComponent
if substrateFun and substrateFun is not self.substrateCollector:
@@ -1139,10 +1311,16 @@
# Any components do not inherit initial tag
asn1Spec = self.protoComponent
if substrateFun and substrateFun is not self.substrateCollector:
- asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options)
- return substrateFun(asn1Object, header + substrate, length + len(header))
+ asn1Object = self._createComponent(
+ asn1Spec, tagSet, noValue, **options)
+
+ for chunk in substrateFun(
+ asn1Object, chunk + substrate, length + len(chunk), options):
+ yield chunk
+
+ return
if LOG:
LOG('assembling constructed serialization')
@@ -1150,10 +1328,18 @@
# All inner fragments are of the same type, treat them as octet string
substrateFun = self.substrateCollector
- while substrate:
- component, substrate = decodeFun(substrate, asn1Spec,
- substrateFun=substrateFun,
- allowEoo=True, **options)
+ while True: # loop over fragments
+
+ for component in decodeFun(
+ substrate, asn1Spec, substrateFun=substrateFun,
+ allowEoo=True, **options):
+
+ if isinstance(component, SubstrateUnderrunError):
+ yield component
+
+ if component is eoo.endOfOctets:
+ break
+
if component is eoo.endOfOctets:
break
@@ -1157,6 +1343,9 @@
if component is eoo.endOfOctets:
break
- header += component
+ chunk += component
+
+ if substrateFun:
+ yield chunk # TODO: Weird
else:
@@ -1161,14 +1350,6 @@
else:
- raise error.SubstrateUnderrunError(
- 'No EOO seen before substrate ends'
- )
-
- if substrateFun:
- return header, substrate
-
- else:
- return self._createComponent(asn1Spec, tagSet, header, **options), substrate
+ yield self._createComponent(asn1Spec, tagSet, chunk, **options)
# character string types
@@ -1172,7 +1353,7 @@
# character string types
-class UTF8StringDecoder(OctetStringDecoder):
+class UTF8StringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.UTF8String()
@@ -1176,7 +1357,7 @@
protoComponent = char.UTF8String()
-class NumericStringDecoder(OctetStringDecoder):
+class NumericStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.NumericString()
@@ -1180,7 +1361,7 @@
protoComponent = char.NumericString()
-class PrintableStringDecoder(OctetStringDecoder):
+class PrintableStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.PrintableString()
@@ -1184,7 +1365,7 @@
protoComponent = char.PrintableString()
-class TeletexStringDecoder(OctetStringDecoder):
+class TeletexStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.TeletexString()
@@ -1188,7 +1369,7 @@
protoComponent = char.TeletexString()
-class VideotexStringDecoder(OctetStringDecoder):
+class VideotexStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.VideotexString()
@@ -1192,7 +1373,7 @@
protoComponent = char.VideotexString()
-class IA5StringDecoder(OctetStringDecoder):
+class IA5StringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.IA5String()
@@ -1196,7 +1377,7 @@
protoComponent = char.IA5String()
-class GraphicStringDecoder(OctetStringDecoder):
+class GraphicStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.GraphicString()
@@ -1200,7 +1381,7 @@
protoComponent = char.GraphicString()
-class VisibleStringDecoder(OctetStringDecoder):
+class VisibleStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.VisibleString()
@@ -1204,7 +1385,7 @@
protoComponent = char.VisibleString()
-class GeneralStringDecoder(OctetStringDecoder):
+class GeneralStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.GeneralString()
@@ -1208,7 +1389,7 @@
protoComponent = char.GeneralString()
-class UniversalStringDecoder(OctetStringDecoder):
+class UniversalStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.UniversalString()
@@ -1212,8 +1393,8 @@
protoComponent = char.UniversalString()
-class BMPStringDecoder(OctetStringDecoder):
+class BMPStringPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = char.BMPString()
# "useful" types
@@ -1216,8 +1397,8 @@
protoComponent = char.BMPString()
# "useful" types
-class ObjectDescriptorDecoder(OctetStringDecoder):
+class ObjectDescriptorPayloadDecoder(OctetStringPayloadDecoder):
protoComponent = useful.ObjectDescriptor()
@@ -1221,7 +1402,7 @@
protoComponent = useful.ObjectDescriptor()
-class GeneralizedTimeDecoder(OctetStringDecoder):
+class GeneralizedTimePayloadDecoder(OctetStringPayloadDecoder):
protoComponent = useful.GeneralizedTime()
@@ -1225,7 +1406,7 @@
protoComponent = useful.GeneralizedTime()
-class UTCTimeDecoder(OctetStringDecoder):
+class UTCTimePayloadDecoder(OctetStringPayloadDecoder):
protoComponent = useful.UTCTime()
@@ -1229,16 +1410,16 @@
protoComponent = useful.UTCTime()
-tagMap = {
- univ.Integer.tagSet: IntegerDecoder(),
- univ.Boolean.tagSet: BooleanDecoder(),
- univ.BitString.tagSet: BitStringDecoder(),
- univ.OctetString.tagSet: OctetStringDecoder(),
- univ.Null.tagSet: NullDecoder(),
- univ.ObjectIdentifier.tagSet: ObjectIdentifierDecoder(),
- univ.Enumerated.tagSet: IntegerDecoder(),
- univ.Real.tagSet: RealDecoder(),
- univ.Sequence.tagSet: SequenceOrSequenceOfDecoder(), # conflicts with SequenceOf
- univ.Set.tagSet: SetOrSetOfDecoder(), # conflicts with SetOf
- univ.Choice.tagSet: ChoiceDecoder(), # conflicts with Any
+TAG_MAP = {
+ univ.Integer.tagSet: IntegerPayloadDecoder(),
+ univ.Boolean.tagSet: BooleanPayloadDecoder(),
+ univ.BitString.tagSet: BitStringPayloadDecoder(),
+ univ.OctetString.tagSet: OctetStringPayloadDecoder(),
+ univ.Null.tagSet: NullPayloadDecoder(),
+ univ.ObjectIdentifier.tagSet: ObjectIdentifierPayloadDecoder(),
+ univ.Enumerated.tagSet: IntegerPayloadDecoder(),
+ univ.Real.tagSet: RealPayloadDecoder(),
+ univ.Sequence.tagSet: SequenceOrSequenceOfPayloadDecoder(), # conflicts with SequenceOf
+ univ.Set.tagSet: SetOrSetOfPayloadDecoder(), # conflicts with SetOf
+ univ.Choice.tagSet: ChoicePayloadDecoder(), # conflicts with Any
# character string types
@@ -1244,13 +1425,13 @@
# character string types
- char.UTF8String.tagSet: UTF8StringDecoder(),
- char.NumericString.tagSet: NumericStringDecoder(),
- char.PrintableString.tagSet: PrintableStringDecoder(),
- char.TeletexString.tagSet: TeletexStringDecoder(),
- char.VideotexString.tagSet: VideotexStringDecoder(),
- char.IA5String.tagSet: IA5StringDecoder(),
- char.GraphicString.tagSet: GraphicStringDecoder(),
- char.VisibleString.tagSet: VisibleStringDecoder(),
- char.GeneralString.tagSet: GeneralStringDecoder(),
- char.UniversalString.tagSet: UniversalStringDecoder(),
- char.BMPString.tagSet: BMPStringDecoder(),
+ char.UTF8String.tagSet: UTF8StringPayloadDecoder(),
+ char.NumericString.tagSet: NumericStringPayloadDecoder(),
+ char.PrintableString.tagSet: PrintableStringPayloadDecoder(),
+ char.TeletexString.tagSet: TeletexStringPayloadDecoder(),
+ char.VideotexString.tagSet: VideotexStringPayloadDecoder(),
+ char.IA5String.tagSet: IA5StringPayloadDecoder(),
+ char.GraphicString.tagSet: GraphicStringPayloadDecoder(),
+ char.VisibleString.tagSet: VisibleStringPayloadDecoder(),
+ char.GeneralString.tagSet: GeneralStringPayloadDecoder(),
+ char.UniversalString.tagSet: UniversalStringPayloadDecoder(),
+ char.BMPString.tagSet: BMPStringPayloadDecoder(),
# useful types
@@ -1256,7 +1437,7 @@
# useful types
- useful.ObjectDescriptor.tagSet: ObjectDescriptorDecoder(),
- useful.GeneralizedTime.tagSet: GeneralizedTimeDecoder(),
- useful.UTCTime.tagSet: UTCTimeDecoder()
+ useful.ObjectDescriptor.tagSet: ObjectDescriptorPayloadDecoder(),
+ useful.GeneralizedTime.tagSet: GeneralizedTimePayloadDecoder(),
+ useful.UTCTime.tagSet: UTCTimePayloadDecoder()
}
# Type-to-codec map for ambiguous ASN.1 types
@@ -1260,13 +1441,13 @@
}
# Type-to-codec map for ambiguous ASN.1 types
-typeMap = {
- univ.Set.typeId: SetDecoder(),
- univ.SetOf.typeId: SetOfDecoder(),
- univ.Sequence.typeId: SequenceDecoder(),
- univ.SequenceOf.typeId: SequenceOfDecoder(),
- univ.Choice.typeId: ChoiceDecoder(),
- univ.Any.typeId: AnyDecoder()
+TYPE_MAP = {
+ univ.Set.typeId: SetPayloadDecoder(),
+ univ.SetOf.typeId: SetOfPayloadDecoder(),
+ univ.Sequence.typeId: SequencePayloadDecoder(),
+ univ.SequenceOf.typeId: SequenceOfPayloadDecoder(),
+ univ.Choice.typeId: ChoicePayloadDecoder(),
+ univ.Any.typeId: AnyPayloadDecoder()
}
# Put in non-ambiguous types for faster codec lookup
@@ -1270,6 +1451,6 @@
}
# Put in non-ambiguous types for faster codec lookup
-for typeDecoder in tagMap.values():
+for typeDecoder in TAG_MAP.values():
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
@@ -1274,7 +1455,7 @@
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
- if typeId is not None and typeId not in typeMap:
- typeMap[typeId] = typeDecoder
+ if typeId is not None and typeId not in TYPE_MAP:
+ TYPE_MAP[typeId] = typeDecoder
(stDecodeTag,
@@ -1289,6 +1470,9 @@
stStop) = [x for x in range(10)]
-class Decoder(object):
+EOO_SENTINEL = ints2octs((0, 0))
+
+
+class SingleItemDecoder(object):
defaultErrorState = stErrorCondition
#defaultErrorState = stDumpRawValue
@@ -1293,5 +1477,6 @@
defaultErrorState = stErrorCondition
#defaultErrorState = stDumpRawValue
- defaultRawDecoder = AnyDecoder()
+ defaultRawDecoder = AnyPayloadDecoder()
+
supportIndefLength = True
@@ -1296,7 +1481,10 @@
supportIndefLength = True
- # noinspection PyDefaultArgument
- def __init__(self, tagMap, typeMap={}):
- self.__tagMap = tagMap
- self.__typeMap = typeMap
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+ def __init__(self, **options):
+ self._tagMap = options.get('tagMap', self.TAG_MAP)
+ self._typeMap = options.get('typeMap', self.TYPE_MAP)
+
# Tag & TagSet objects caches
@@ -1302,10 +1490,9 @@
# Tag & TagSet objects caches
- self.__tagCache = {}
- self.__tagSetCache = {}
- self.__eooSentinel = ints2octs((0, 0))
+ self._tagCache = {}
+ self._tagSetCache = {}
def __call__(self, substrate, asn1Spec=None,
tagSet=None, length=None, state=stDecodeTag,
decodeFun=None, substrateFun=None,
**options):
@@ -1306,7 +1493,9 @@
def __call__(self, substrate, asn1Spec=None,
tagSet=None, length=None, state=stDecodeTag,
decodeFun=None, substrateFun=None,
**options):
+ allowEoo = options.pop('allowEoo', False)
+
if LOG:
@@ -1312,7 +1501,7 @@
if LOG:
- LOG('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate)))
-
- allowEoo = options.pop('allowEoo', False)
+ LOG('decoder called at scope %s with state %d, working with up '
+ 'to %s octets of substrate: '
+ '%s' % (debug.scope, state, length, substrate))
# Look for end-of-octets sentinel
if allowEoo and self.supportIndefLength:
@@ -1316,6 +1505,11 @@
# Look for end-of-octets sentinel
if allowEoo and self.supportIndefLength:
- if substrate[:2] == self.__eooSentinel:
+
+ for eoo_candidate in readFromStream(substrate, 2, options):
+ if isinstance(eoo_candidate, SubstrateUnderrunError):
+ yield eoo_candidate
+
+ if eoo_candidate == EOO_SENTINEL:
if LOG:
LOG('end-of-octets sentinel found')
@@ -1320,6 +1514,15 @@
if LOG:
LOG('end-of-octets sentinel found')
- return eoo.endOfOctets, substrate[2:]
+ yield eoo.endOfOctets
+ return
+
+ else:
+ substrate.seek(-2, os.SEEK_CUR)
+
+ tagMap = self._tagMap
+ typeMap = self._typeMap
+ tagCache = self._tagCache
+ tagSetCache = self._tagSetCache
value = noValue
@@ -1323,13 +1526,8 @@
value = noValue
- tagMap = self.__tagMap
- typeMap = self.__typeMap
- tagCache = self.__tagCache
- tagSetCache = self.__tagSetCache
-
- fullSubstrate = substrate
+ substrate.markedPosition = substrate.tell()
while state is not stStop:
if state is stDecodeTag:
@@ -1332,11 +1530,6 @@
while state is not stStop:
if state is stDecodeTag:
- if not substrate:
- raise error.SubstrateUnderrunError(
- 'Short octet stream on tag decoding'
- )
-
# Decode tag
isShortTag = True
@@ -1341,9 +1534,13 @@
# Decode tag
isShortTag = True
- firstOctet = substrate[0]
- substrate = substrate[1:]
+
+ for firstByte in readFromStream(substrate, 1, options):
+ if isinstance(firstByte, SubstrateUnderrunError):
+ yield firstByte
+
+ firstOctet = ord(firstByte)
try:
lastTag = tagCache[firstOctet]
except KeyError:
@@ -1345,9 +1542,9 @@
try:
lastTag = tagCache[firstOctet]
except KeyError:
- integerTag = oct2int(firstOctet)
+ integerTag = firstOctet
tagClass = integerTag & 0xC0
tagFormat = integerTag & 0x20
tagId = integerTag & 0x1F
@@ -1357,21 +1554,23 @@
lengthOctetIdx = 0
tagId = 0
- try:
- while True:
- integerTag = oct2int(substrate[lengthOctetIdx])
- lengthOctetIdx += 1
- tagId <<= 7
- tagId |= (integerTag & 0x7F)
- if not integerTag & 0x80:
- break
-
- substrate = substrate[lengthOctetIdx:]
-
- except IndexError:
- raise error.SubstrateUnderrunError(
- 'Short octet stream on long tag decoding'
- )
+ while True:
+ for integerByte in readFromStream(substrate, 1, options):
+ if isinstance(integerByte, SubstrateUnderrunError):
+ yield integerByte
+
+ if not integerByte:
+ raise error.SubstrateUnderrunError(
+ 'Short octet stream on long tag decoding'
+ )
+
+ integerTag = ord(integerByte)
+ lengthOctetIdx += 1
+ tagId <<= 7
+ tagId |= (integerTag & 0x7F)
+
+ if not integerTag & 0x80:
+ break
lastTag = tag.Tag(
tagClass=tagClass, tagFormat=tagFormat, tagId=tagId
@@ -1403,11 +1602,10 @@
if state is stDecodeLength:
# Decode length
- if not substrate:
- raise error.SubstrateUnderrunError(
- 'Short octet stream on length decoding'
- )
-
- firstOctet = oct2int(substrate[0])
+ for firstOctet in readFromStream(substrate, 1, options):
+ if isinstance(firstOctet, SubstrateUnderrunError):
+ yield firstOctet
+
+ firstOctet = ord(firstOctet)
if firstOctet < 128:
@@ -1412,8 +1610,7 @@
if firstOctet < 128:
- size = 1
length = firstOctet
elif firstOctet > 128:
size = firstOctet & 0x7F
# encoded in size bytes
@@ -1415,9 +1612,12 @@
length = firstOctet
elif firstOctet > 128:
size = firstOctet & 0x7F
# encoded in size bytes
- encodedLength = octs2ints(substrate[1:size + 1])
+ for encodedLength in readFromStream(substrate, size, options):
+ if isinstance(encodedLength, SubstrateUnderrunError):
+ yield encodedLength
+ encodedLength = list(encodedLength)
# missing check on maximum size, which shouldn't be a
# problem, we can handle more than is possible
if len(encodedLength) != size:
@@ -1428,6 +1628,6 @@
length = 0
for lengthOctet in encodedLength:
length <<= 8
- length |= lengthOctet
+ length |= oct2int(lengthOctet)
size += 1
@@ -1432,6 +1632,5 @@
size += 1
- else:
- size = 1
+ else: # 128 means indefinite
length = -1
@@ -1436,15 +1635,8 @@
length = -1
- substrate = substrate[size:]
-
- if length == -1:
- if not self.supportIndefLength:
- raise error.PyAsn1Error('Indefinite length encoding not supported by this codec')
-
- else:
- if len(substrate) < length:
- raise error.SubstrateUnderrunError('%d-octet short' % (length - len(substrate)))
+ if length == -1 and not self.supportIndefLength:
+ raise error.PyAsn1Error('Indefinite length encoding not supported by this codec')
state = stGetValueDecoder
if LOG:
@@ -1447,8 +1639,8 @@
state = stGetValueDecoder
if LOG:
- LOG('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length])))
+ LOG('value length decoded into %d' % length)
if state is stGetValueDecoder:
if asn1Spec is None:
@@ -1567,6 +1759,6 @@
if not options.get('recursiveFlag', True) and not substrateFun: # deprecate this
substrateFun = lambda a, b, c: (a, b[:c])
- options.update(fullSubstrate=fullSubstrate)
+ original_position = substrate.tell()
if length == -1: # indef length
@@ -1571,10 +1763,10 @@
if length == -1: # indef length
- value, substrate = concreteDecoder.indefLenValueDecoder(
- substrate, asn1Spec,
- tagSet, length, stGetValueDecoder,
- self, substrateFun,
- **options
- )
+ for value in concreteDecoder.indefLenValueDecoder(
+ substrate, asn1Spec,
+ tagSet, length, stGetValueDecoder,
+ self, substrateFun, **options):
+ if isinstance(value, SubstrateUnderrunError):
+ yield value
else:
@@ -1579,10 +1771,15 @@
else:
- value, substrate = concreteDecoder.valueDecoder(
- substrate, asn1Spec,
- tagSet, length, stGetValueDecoder,
- self, substrateFun,
- **options
- )
+ for value in concreteDecoder.valueDecoder(
+ substrate, asn1Spec,
+ tagSet, length, stGetValueDecoder,
+ self, substrateFun, **options):
+ if isinstance(value, SubstrateUnderrunError):
+ yield value
+
+ bytesRead = substrate.tell() - original_position
+ if bytesRead != length:
+ raise PyAsn1Error(
+ "Read %s bytes instead of expected %s." % (bytesRead, length))
if LOG:
@@ -1587,6 +1784,8 @@
if LOG:
- LOG('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, isinstance(value, base.Asn1Item) and value.prettyPrint() or value, substrate and debug.hexdump(substrate) or '<none>'))
+ LOG('codec %s yields type %s, value:\n%s\n...' % (
+ concreteDecoder.__class__.__name__, value.__class__.__name__,
+ isinstance(value, base.Asn1Item) and value.prettyPrint() or value))
state = stStop
break
@@ -1596,7 +1795,7 @@
tagSet[0].tagFormat == tag.tagFormatConstructed and
tagSet[0].tagClass != tag.tagClassUniversal):
# Assume explicit tagging
- concreteDecoder = explicitTagDecoder
+ concreteDecoder = rawPayloadDecoder
state = stDecodeValue
else:
@@ -1623,7 +1822,190 @@
debug.scope.pop()
LOG('decoder left scope %s, call completed' % debug.scope)
- return value, substrate
+ yield value
+
+
+class StreamingDecoder(object):
+ """Create an iterator that turns BER/CER/DER byte stream into ASN.1 objects.
+
+ On each iteration, consume whatever BER/CER/DER serialization is
+ available in the `substrate` stream-like object and turns it into
+ one or more, possibly nested, ASN.1 objects.
+
+ Parameters
+ ----------
+ substrate: :py:class:`file`, :py:class:`io.BytesIO`
+ BER/CER/DER serialization in form of a byte stream
+
+ Keyword Args
+ ------------
+ asn1Spec: :py:class:`~pyasn1.type.base.PyAsn1Item`
+ A pyasn1 type object to act as a template guiding the decoder.
+ Depending on the ASN.1 structure being decoded, `asn1Spec` may
+ or may not be required. One of the reasons why `asn1Spec` may
+ me required is that ASN.1 structure is encoded in the *IMPLICIT*
+ tagging mode.
+
+ Yields
+ ------
+ : :py:class:`~pyasn1.type.base.PyAsn1Item`, :py:class:`~pyasn1.error.SubstrateUnderrunError`
+ Decoded ASN.1 object (possibly, nested) or
+ :py:class:`~pyasn1.error.SubstrateUnderrunError` object indicating
+ insufficient BER/CER/DER serialization on input to fully recover ASN.1
+ objects from it.
+
+ In the latter case the caller is advised to ensure some more data in
+ the input stream, then call the iterator again. The decoder will resume
+ the decoding process using the newly arrived data.
+
+ The `context` property of :py:class:`~pyasn1.error.SubstrateUnderrunError`
+ object might hold a reference to the partially populated ASN.1 object
+ being reconstructed.
+
+ Raises
+ ------
+ ~pyasn1.error.PyAsn1Error, ~pyasn1.error.EndOfStreamError
+ `PyAsn1Error` on deserialization error, `EndOfStreamError` on
+ premature stream closure.
+
+ Examples
+ --------
+ Decode BER serialisation without ASN.1 schema
+
+ .. code-block:: pycon
+
+ >>> stream = io.BytesIO(
+ ... b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03')
+ >>>
+ >>> for asn1Object in StreamingDecoder(stream):
+ ... print(asn1Object)
+ >>>
+ SequenceOf:
+ 1 2 3
+
+ Decode BER serialisation with ASN.1 schema
+
+ .. code-block:: pycon
+
+ >>> stream = io.BytesIO(
+ ... b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03')
+ >>>
+ >>> schema = SequenceOf(componentType=Integer())
+ >>>
+ >>> decoder = StreamingDecoder(stream, asn1Spec=schema)
+ >>> for asn1Object in decoder:
+ ... print(asn1Object)
+ >>>
+ SequenceOf:
+ 1 2 3
+ """
+
+ SINGLE_ITEM_DECODER = SingleItemDecoder
+
+ def __init__(self, substrate, asn1Spec=None, **options):
+ self._singleItemDecoder = self.SINGLE_ITEM_DECODER(**options)
+ self._substrate = asSeekableStream(substrate)
+ self._asn1Spec = asn1Spec
+ self._options = options
+
+ def __iter__(self):
+ while True:
+ for asn1Object in self._singleItemDecoder(
+ self._substrate, self._asn1Spec, **self._options):
+ yield asn1Object
+
+ for chunk in isEndOfStream(self._substrate):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield
+
+ break
+
+ if chunk:
+ break
+
+
+class Decoder(object):
+ """Create a BER decoder object.
+
+ Parse BER/CER/DER octet-stream into one, possibly nested, ASN.1 object.
+ """
+ STREAMING_DECODER = StreamingDecoder
+
+ @classmethod
+ def __call__(cls, substrate, asn1Spec=None, **options):
+ """Turns BER/CER/DER octet stream into an ASN.1 object.
+
+ Takes BER/CER/DER octet-stream in form of :py:class:`bytes` (Python 3)
+ or :py:class:`str` (Python 2) and decode it into an ASN.1 object
+ (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) which
+ may be a scalar or an arbitrary nested structure.
+
+ Parameters
+ ----------
+ substrate: :py:class:`bytes` (Python 3) or :py:class:`str` (Python 2)
+ BER/CER/DER octet-stream to parse
+
+ Keyword Args
+ ------------
+ asn1Spec: :py:class:`~pyasn1.type.base.PyAsn1Item`
+ A pyasn1 type object (:py:class:`~pyasn1.type.base.PyAsn1Item`
+ derivative) to act as a template guiding the decoder.
+ Depending on the ASN.1 structure being decoded, `asn1Spec` may or
+ may not be required. Most common reason for it to require is that
+ ASN.1 structure is encoded in *IMPLICIT* tagging mode.
+
+ Returns
+ -------
+ : :py:class:`tuple`
+ A tuple of :py:class:`~pyasn1.type.base.PyAsn1Item` object
+ recovered from BER/CER/DER substrate and the unprocessed trailing
+ portion of the `substrate` (may be empty)
+
+ Raises
+ ------
+ : :py:class:`~pyasn1.error.PyAsn1Error`
+ :py:class:`~pyasn1.error.SubstrateUnderrunError` on insufficient
+ input or :py:class:`~pyasn1.error.PyAsn1Error` on decoding error.
+
+ Examples
+ --------
+ Decode BER/CER/DER serialisation without ASN.1 schema
+
+ .. code-block:: pycon
+
+ >>> s, unprocessed = decode(b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03')
+ >>> str(s)
+ SequenceOf:
+ 1 2 3
+
+ Decode BER/CER/DER serialisation with ASN.1 schema
+
+ .. code-block:: pycon
+
+ >>> seq = SequenceOf(componentType=Integer())
+ >>> s, unprocessed = decode(
+ b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03', asn1Spec=seq)
+ >>> str(s)
+ SequenceOf:
+ 1 2 3
+
+ """
+ substrate = asSeekableStream(substrate)
+
+ streamingDecoder = cls.STREAMING_DECODER(
+ substrate, asn1Spec, **options)
+
+ for asn1Object in streamingDecoder:
+ if isinstance(asn1Object, SubstrateUnderrunError):
+ raise error.SubstrateUnderrunError('Short substrate on input')
+
+ try:
+ tail = next(readFromStream(substrate))
+
+ except error.EndOfStreamError:
+ tail = null
+
+ return asn1Object, tail
#: Turns BER octet stream into an ASN.1 object.
@@ -1655,6 +2037,11 @@
#: ~pyasn1.error.PyAsn1Error, ~pyasn1.error.SubstrateUnderrunError
#: On decoding errors
#:
+#: Notes
+#: -----
+#: This function is deprecated. Please use :py:class:`Decoder` or
+#: :py:class:`StreamingDecoder` class instance.
+#:
#: Examples
#: --------
#: Decode BER serialisation without ASN.1 schema
@@ -1676,7 +2063,4 @@
#: SequenceOf:
#: 1 2 3
#:
-decode = Decoder(tagMap, typeMap)
-
-# XXX
-# non-recursive decoding; return position rather than substrate
+decode = Decoder()
diff --git a/pyasn1/codec/ber/encoder.py b/pyasn1/codec/ber/encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Jlci9lbmNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Jlci9lbmNvZGVyLnB5 100644
--- a/pyasn1/codec/ber/encoder.py
+++ b/pyasn1/codec/ber/encoder.py
@@ -17,7 +17,7 @@
from pyasn1.type import univ
from pyasn1.type import useful
-__all__ = ['encode']
+__all__ = ['Encoder', 'encode']
LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_ENCODER)
@@ -706,7 +706,7 @@
return value, not options.get('defMode', True), True
-tagMap = {
+TAG_MAP = {
eoo.endOfOctets.tagSet: EndOfOctetsEncoder(),
univ.Boolean.tagSet: BooleanEncoder(),
univ.Integer.tagSet: IntegerEncoder(),
@@ -739,7 +739,7 @@
}
# Put in ambiguous & non-ambiguous types for faster codec lookup
-typeMap = {
+TYPE_MAP = {
univ.Boolean.typeId: BooleanEncoder(),
univ.Integer.typeId: IntegerEncoder(),
univ.BitString.typeId: BitStringEncoder(),
@@ -774,7 +774,7 @@
}
-class Encoder(object):
+class SingleItemEncoder(object):
fixedDefLengthMode = None
fixedChunkSize = None
@@ -778,10 +778,12 @@
fixedDefLengthMode = None
fixedChunkSize = None
- # noinspection PyDefaultArgument
- def __init__(self, tagMap, typeMap={}):
- self.__tagMap = tagMap
- self.__typeMap = typeMap
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+ def __init__(self, **options):
+ self._tagMap = options.get('tagMap', self.TAG_MAP)
+ self._typeMap = options.get('typeMap', self.TYPE_MAP)
def __call__(self, value, asn1Spec=None, **options):
try:
@@ -795,8 +797,11 @@
'and "asn1Spec" not given' % (value,))
if LOG:
- LOG('encoder called in %sdef mode, chunk size %s for '
- 'type %s, value:\n%s' % (not options.get('defMode', True) and 'in' or '', options.get('maxChunkSize', 0), asn1Spec is None and value.prettyPrintType() or asn1Spec.prettyPrintType(), value))
+ LOG('encoder called in %sdef mode, chunk size %s for type %s, '
+ 'value:\n%s' % (not options.get('defMode', True) and 'in' or '',
+ options.get('maxChunkSize', 0),
+ asn1Spec is None and value.prettyPrintType() or
+ asn1Spec.prettyPrintType(), value))
if self.fixedDefLengthMode is not None:
options.update(defMode=self.fixedDefLengthMode)
@@ -804,5 +809,4 @@
if self.fixedChunkSize is not None:
options.update(maxChunkSize=self.fixedChunkSize)
-
try:
@@ -808,4 +812,4 @@
try:
- concreteEncoder = self.__typeMap[typeId]
+ concreteEncoder = self._typeMap[typeId]
if LOG:
@@ -810,6 +814,7 @@
if LOG:
- LOG('using value codec %s chosen by type ID %s' % (concreteEncoder.__class__.__name__, typeId))
+ LOG('using value codec %s chosen by type ID '
+ '%s' % (concreteEncoder.__class__.__name__, typeId))
except KeyError:
if asn1Spec is None:
@@ -821,9 +826,9 @@
baseTagSet = tag.TagSet(tagSet.baseTag, tagSet.baseTag)
try:
- concreteEncoder = self.__tagMap[baseTagSet]
+ concreteEncoder = self._tagMap[baseTagSet]
except KeyError:
raise error.PyAsn1Error('No encoder for %r (%s)' % (value, tagSet))
if LOG:
@@ -825,10 +830,11 @@
except KeyError:
raise error.PyAsn1Error('No encoder for %r (%s)' % (value, tagSet))
if LOG:
- LOG('using value codec %s chosen by tagSet %s' % (concreteEncoder.__class__.__name__, tagSet))
+ LOG('using value codec %s chosen by tagSet '
+ '%s' % (concreteEncoder.__class__.__name__, tagSet))
substrate = concreteEncoder.encode(value, asn1Spec, self, **options)
if LOG:
@@ -831,8 +837,10 @@
substrate = concreteEncoder.encode(value, asn1Spec, self, **options)
if LOG:
- LOG('codec %s built %s octets of substrate: %s\nencoder completed' % (concreteEncoder, len(substrate), debug.hexdump(substrate)))
+ LOG('codec %s built %s octets of substrate: %s\nencoder '
+ 'completed' % (concreteEncoder, len(substrate),
+ debug.hexdump(substrate)))
return substrate
@@ -836,6 +844,18 @@
return substrate
+
+class Encoder(object):
+ SINGLE_ITEM_ENCODER = SingleItemEncoder
+
+ def __init__(self, **options):
+ self._singleItemEncoder = self.SINGLE_ITEM_ENCODER(**options)
+
+ def __call__(self, pyObject, asn1Spec=None, **options):
+ return self._singleItemEncoder(
+ pyObject, asn1Spec=asn1Spec, **options)
+
+
#: Turns ASN.1 object into BER octet stream.
#:
#: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative)
@@ -887,4 +907,4 @@
#: >>> encode(seq)
#: b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03'
#:
-encode = Encoder(tagMap, typeMap)
+encode = Encoder()
diff --git a/pyasn1/codec/cer/decoder.py b/pyasn1/codec/cer/decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Nlci9kZWNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Nlci9kZWNvZGVyLnB5 100644
--- a/pyasn1/codec/cer/decoder.py
+++ b/pyasn1/codec/cer/decoder.py
@@ -5,7 +5,8 @@
# License: http://snmplabs.com/pyasn1/license.html
#
from pyasn1 import error
+from pyasn1.codec.streaming import readFromStream
from pyasn1.codec.ber import decoder
from pyasn1.compat.octets import oct2int
from pyasn1.type import univ
@@ -8,7 +9,9 @@
from pyasn1.codec.ber import decoder
from pyasn1.compat.octets import oct2int
from pyasn1.type import univ
-__all__ = ['decode']
+__all__ = ['decode', 'StreamingDecoder']
+
+SubstrateUnderrunError = error.SubstrateUnderrunError
@@ -13,9 +16,9 @@
-class BooleanDecoder(decoder.AbstractSimpleDecoder):
+class BooleanPayloadDecoder(decoder.AbstractSimplePayloadDecoder):
protoComponent = univ.Boolean(0)
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
@@ -16,9 +19,9 @@
protoComponent = univ.Boolean(0)
def valueDecoder(self, substrate, asn1Spec,
tagSet=None, length=None, state=None,
decodeFun=None, substrateFun=None,
**options):
- head, tail = substrate[:length], substrate[length:]
- if not head or length != 1:
+
+ if length != 1:
raise error.PyAsn1Error('Not single-octet Boolean payload')
@@ -24,7 +27,13 @@
raise error.PyAsn1Error('Not single-octet Boolean payload')
- byte = oct2int(head[0])
+
+ for chunk in readFromStream(substrate, length, options):
+ if isinstance(chunk, SubstrateUnderrunError):
+ yield chunk
+
+ byte = oct2int(chunk[0])
+
# CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while
# BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1
# in https://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
if byte == 0xff:
value = 1
@@ -26,7 +35,8 @@
# CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while
# BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1
# in https://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf
if byte == 0xff:
value = 1
+
elif byte == 0x00:
value = 0
@@ -31,4 +41,5 @@
elif byte == 0x00:
value = 0
+
else:
raise error.PyAsn1Error('Unexpected Boolean payload: %s' % byte)
@@ -33,5 +44,7 @@
else:
raise error.PyAsn1Error('Unexpected Boolean payload: %s' % byte)
- return self._createComponent(asn1Spec, tagSet, value, **options), tail
+
+ yield self._createComponent(asn1Spec, tagSet, value, **options)
+
# TODO: prohibit non-canonical encoding
@@ -36,6 +49,6 @@
# TODO: prohibit non-canonical encoding
-BitStringDecoder = decoder.BitStringDecoder
-OctetStringDecoder = decoder.OctetStringDecoder
-RealDecoder = decoder.RealDecoder
+BitStringPayloadDecoder = decoder.BitStringPayloadDecoder
+OctetStringPayloadDecoder = decoder.OctetStringPayloadDecoder
+RealPayloadDecoder = decoder.RealPayloadDecoder
@@ -41,9 +54,9 @@
-tagMap = decoder.tagMap.copy()
-tagMap.update(
- {univ.Boolean.tagSet: BooleanDecoder(),
- univ.BitString.tagSet: BitStringDecoder(),
- univ.OctetString.tagSet: OctetStringDecoder(),
- univ.Real.tagSet: RealDecoder()}
+TAG_MAP = decoder.TAG_MAP.copy()
+TAG_MAP.update(
+ {univ.Boolean.tagSet: BooleanPayloadDecoder(),
+ univ.BitString.tagSet: BitStringPayloadDecoder(),
+ univ.OctetString.tagSet: OctetStringPayloadDecoder(),
+ univ.Real.tagSet: RealPayloadDecoder()}
)
@@ -48,5 +61,5 @@
)
-typeMap = decoder.typeMap.copy()
+TYPE_MAP = decoder.TYPE_MAP.copy()
# Put in non-ambiguous types for faster codec lookup
@@ -51,5 +64,5 @@
# Put in non-ambiguous types for faster codec lookup
-for typeDecoder in tagMap.values():
+for typeDecoder in TAG_MAP.values():
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
@@ -54,7 +67,20 @@
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
- if typeId is not None and typeId not in typeMap:
- typeMap[typeId] = typeDecoder
+ if typeId is not None and typeId not in TYPE_MAP:
+ TYPE_MAP[typeId] = typeDecoder
+
+
+class SingleItemDecoder(decoder.SingleItemDecoder):
+ __doc__ = decoder.SingleItemDecoder.__doc__
+
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+
+class StreamingDecoder(decoder.StreamingDecoder):
+ __doc__ = decoder.StreamingDecoder.__doc__
+
+ SINGLE_ITEM_DECODER = SingleItemDecoder
class Decoder(decoder.Decoder):
@@ -58,7 +84,9 @@
class Decoder(decoder.Decoder):
- pass
+ __doc__ = decoder.Decoder.__doc__
+
+ STREAMING_DECODER = StreamingDecoder
#: Turns CER octet stream into an ASN.1 object.
@@ -111,4 +139,4 @@
#: SequenceOf:
#: 1 2 3
#:
-decode = Decoder(tagMap, decoder.typeMap)
+decode = Decoder()
diff --git a/pyasn1/codec/cer/encoder.py b/pyasn1/codec/cer/encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Nlci9lbmNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Nlci9lbmNvZGVyLnB5 100644
--- a/pyasn1/codec/cer/encoder.py
+++ b/pyasn1/codec/cer/encoder.py
@@ -10,7 +10,7 @@
from pyasn1.type import univ
from pyasn1.type import useful
-__all__ = ['encode']
+__all__ = ['Encoder', 'encode']
class BooleanEncoder(encoder.IntegerEncoder):
@@ -234,8 +234,9 @@
omitEmptyOptionals = True
-tagMap = encoder.tagMap.copy()
-tagMap.update({
+TAG_MAP = encoder.TAG_MAP.copy()
+
+TAG_MAP.update({
univ.Boolean.tagSet: BooleanEncoder(),
univ.Real.tagSet: RealEncoder(),
useful.GeneralizedTime.tagSet: GeneralizedTimeEncoder(),
@@ -245,8 +246,9 @@
univ.Sequence.typeId: SequenceEncoder()
})
-typeMap = encoder.typeMap.copy()
-typeMap.update({
+TYPE_MAP = encoder.TYPE_MAP.copy()
+
+TYPE_MAP.update({
univ.Boolean.typeId: BooleanEncoder(),
univ.Real.typeId: RealEncoder(),
useful.GeneralizedTime.typeId: GeneralizedTimeEncoder(),
@@ -259,7 +261,7 @@
})
-class Encoder(encoder.Encoder):
+class SingleItemEncoder(encoder.SingleItemEncoder):
fixedDefLengthMode = False
fixedChunkSize = 1000
@@ -263,6 +265,14 @@
fixedDefLengthMode = False
fixedChunkSize = 1000
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+
+class Encoder(encoder.Encoder):
+ SINGLE_ITEM_ENCODER = SingleItemEncoder
+
+
#: Turns ASN.1 object into CER octet stream.
#:
#: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative)
@@ -308,6 +318,6 @@
#: >>> encode(seq)
#: b'0\x80\x02\x01\x01\x02\x01\x02\x02\x01\x03\x00\x00'
#:
-encode = Encoder(tagMap, typeMap)
+encode = Encoder()
# EncoderFactory queries class instance and builds a map of tags -> encoders
diff --git a/pyasn1/codec/der/decoder.py b/pyasn1/codec/der/decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Rlci9kZWNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Rlci9kZWNvZGVyLnB5 100644
--- a/pyasn1/codec/der/decoder.py
+++ b/pyasn1/codec/der/decoder.py
@@ -7,6 +7,6 @@
from pyasn1.codec.cer import decoder
from pyasn1.type import univ
-__all__ = ['decode']
+__all__ = ['decode', 'StreamingDecoder']
@@ -11,6 +11,10 @@
-class BitStringDecoder(decoder.BitStringDecoder):
+class BitStringPayloadDecoder(decoder.BitStringPayloadDecoder):
+ supportConstructedForm = False
+
+
+class OctetStringPayloadDecoder(decoder.OctetStringPayloadDecoder):
supportConstructedForm = False
@@ -14,7 +18,4 @@
supportConstructedForm = False
-class OctetStringDecoder(decoder.OctetStringDecoder):
- supportConstructedForm = False
-
# TODO: prohibit non-canonical encoding
@@ -20,3 +21,3 @@
# TODO: prohibit non-canonical encoding
-RealDecoder = decoder.RealDecoder
+RealPayloadDecoder = decoder.RealPayloadDecoder
@@ -22,8 +23,8 @@
-tagMap = decoder.tagMap.copy()
-tagMap.update(
- {univ.BitString.tagSet: BitStringDecoder(),
- univ.OctetString.tagSet: OctetStringDecoder(),
- univ.Real.tagSet: RealDecoder()}
+TAG_MAP = decoder.TAG_MAP.copy()
+TAG_MAP.update(
+ {univ.BitString.tagSet: BitStringPayloadDecoder(),
+ univ.OctetString.tagSet: OctetStringPayloadDecoder(),
+ univ.Real.tagSet: RealPayloadDecoder()}
)
@@ -28,5 +29,5 @@
)
-typeMap = decoder.typeMap.copy()
+TYPE_MAP = decoder.TYPE_MAP.copy()
# Put in non-ambiguous types for faster codec lookup
@@ -31,5 +32,5 @@
# Put in non-ambiguous types for faster codec lookup
-for typeDecoder in tagMap.values():
+for typeDecoder in TAG_MAP.values():
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
@@ -34,7 +35,22 @@
if typeDecoder.protoComponent is not None:
typeId = typeDecoder.protoComponent.__class__.typeId
- if typeId is not None and typeId not in typeMap:
- typeMap[typeId] = typeDecoder
+ if typeId is not None and typeId not in TYPE_MAP:
+ TYPE_MAP[typeId] = typeDecoder
+
+
+class SingleItemDecoder(decoder.SingleItemDecoder):
+ __doc__ = decoder.SingleItemDecoder.__doc__
+
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+ supportIndefLength = False
+
+
+class StreamingDecoder(decoder.StreamingDecoder):
+ __doc__ = decoder.StreamingDecoder.__doc__
+
+ SINGLE_ITEM_DECODER = SingleItemDecoder
class Decoder(decoder.Decoder):
@@ -38,7 +54,9 @@
class Decoder(decoder.Decoder):
- supportIndefLength = False
+ __doc__ = decoder.Decoder.__doc__
+
+ STREAMING_DECODER = StreamingDecoder
#: Turns DER octet stream into an ASN.1 object.
@@ -91,4 +109,4 @@
#: SequenceOf:
#: 1 2 3
#:
-decode = Decoder(tagMap, typeMap)
+decode = Decoder()
diff --git a/pyasn1/codec/der/encoder.py b/pyasn1/codec/der/encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL2Rlci9lbmNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL2Rlci9lbmNvZGVyLnB5 100644
--- a/pyasn1/codec/der/encoder.py
+++ b/pyasn1/codec/der/encoder.py
@@ -8,7 +8,7 @@
from pyasn1.codec.cer import encoder
from pyasn1.type import univ
-__all__ = ['encode']
+__all__ = ['Encoder', 'encode']
class SetEncoder(encoder.SetEncoder):
@@ -42,9 +42,11 @@
else:
return compType.tagSet
-tagMap = encoder.tagMap.copy()
-tagMap.update({
+
+TAG_MAP = encoder.TAG_MAP.copy()
+
+TAG_MAP.update({
# Set & SetOf have same tags
univ.Set.tagSet: SetEncoder()
})
@@ -47,11 +49,12 @@
# Set & SetOf have same tags
univ.Set.tagSet: SetEncoder()
})
-typeMap = encoder.typeMap.copy()
-typeMap.update({
+TYPE_MAP = encoder.TYPE_MAP.copy()
+
+TYPE_MAP.update({
# Set & SetOf have same tags
univ.Set.typeId: SetEncoder()
})
@@ -53,9 +56,9 @@
# Set & SetOf have same tags
univ.Set.typeId: SetEncoder()
})
-class Encoder(encoder.Encoder):
+class SingleItemEncoder(encoder.SingleItemEncoder):
fixedDefLengthMode = True
fixedChunkSize = 0
@@ -59,6 +62,14 @@
fixedDefLengthMode = True
fixedChunkSize = 0
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+
+class Encoder(encoder.Encoder):
+ SINGLE_ITEM_ENCODER = SingleItemEncoder
+
+
#: Turns ASN.1 object into DER octet stream.
#:
#: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative)
@@ -104,4 +115,4 @@
#: >>> encode(seq)
#: b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03'
#:
-encode = Encoder(tagMap, typeMap)
+encode = Encoder()
diff --git a/pyasn1/codec/native/decoder.py b/pyasn1/codec/native/decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL25hdGl2ZS9kZWNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL25hdGl2ZS9kZWNvZGVyLnB5 100644
--- a/pyasn1/codec/native/decoder.py
+++ b/pyasn1/codec/native/decoder.py
@@ -17,8 +17,8 @@
LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER)
-class AbstractScalarDecoder(object):
+class AbstractScalarPayloadDecoder(object):
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
return asn1Spec.clone(pyObject)
@@ -21,9 +21,9 @@
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
return asn1Spec.clone(pyObject)
-class BitStringDecoder(AbstractScalarDecoder):
+class BitStringPayloadDecoder(AbstractScalarPayloadDecoder):
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
return asn1Spec.clone(univ.BitString.fromBinaryString(pyObject))
@@ -26,8 +26,8 @@
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
return asn1Spec.clone(univ.BitString.fromBinaryString(pyObject))
-class SequenceOrSetDecoder(object):
+class SequenceOrSetPayloadDecoder(object):
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
asn1Value = asn1Spec.clone()
@@ -40,7 +40,7 @@
return asn1Value
-class SequenceOfOrSetOfDecoder(object):
+class SequenceOfOrSetOfPayloadDecoder(object):
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
asn1Value = asn1Spec.clone()
@@ -50,7 +50,7 @@
return asn1Value
-class ChoiceDecoder(object):
+class ChoicePayloadDecoder(object):
def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
asn1Value = asn1Spec.clone()
@@ -64,16 +64,16 @@
return asn1Value
-tagMap = {
- univ.Integer.tagSet: AbstractScalarDecoder(),
- univ.Boolean.tagSet: AbstractScalarDecoder(),
- univ.BitString.tagSet: BitStringDecoder(),
- univ.OctetString.tagSet: AbstractScalarDecoder(),
- univ.Null.tagSet: AbstractScalarDecoder(),
- univ.ObjectIdentifier.tagSet: AbstractScalarDecoder(),
- univ.Enumerated.tagSet: AbstractScalarDecoder(),
- univ.Real.tagSet: AbstractScalarDecoder(),
- univ.Sequence.tagSet: SequenceOrSetDecoder(), # conflicts with SequenceOf
- univ.Set.tagSet: SequenceOrSetDecoder(), # conflicts with SetOf
- univ.Choice.tagSet: ChoiceDecoder(), # conflicts with Any
+TAG_MAP = {
+ univ.Integer.tagSet: AbstractScalarPayloadDecoder(),
+ univ.Boolean.tagSet: AbstractScalarPayloadDecoder(),
+ univ.BitString.tagSet: BitStringPayloadDecoder(),
+ univ.OctetString.tagSet: AbstractScalarPayloadDecoder(),
+ univ.Null.tagSet: AbstractScalarPayloadDecoder(),
+ univ.ObjectIdentifier.tagSet: AbstractScalarPayloadDecoder(),
+ univ.Enumerated.tagSet: AbstractScalarPayloadDecoder(),
+ univ.Real.tagSet: AbstractScalarPayloadDecoder(),
+ univ.Sequence.tagSet: SequenceOrSetPayloadDecoder(), # conflicts with SequenceOf
+ univ.Set.tagSet: SequenceOrSetPayloadDecoder(), # conflicts with SetOf
+ univ.Choice.tagSet: ChoicePayloadDecoder(), # conflicts with Any
# character string types
@@ -79,13 +79,13 @@
# character string types
- char.UTF8String.tagSet: AbstractScalarDecoder(),
- char.NumericString.tagSet: AbstractScalarDecoder(),
- char.PrintableString.tagSet: AbstractScalarDecoder(),
- char.TeletexString.tagSet: AbstractScalarDecoder(),
- char.VideotexString.tagSet: AbstractScalarDecoder(),
- char.IA5String.tagSet: AbstractScalarDecoder(),
- char.GraphicString.tagSet: AbstractScalarDecoder(),
- char.VisibleString.tagSet: AbstractScalarDecoder(),
- char.GeneralString.tagSet: AbstractScalarDecoder(),
- char.UniversalString.tagSet: AbstractScalarDecoder(),
- char.BMPString.tagSet: AbstractScalarDecoder(),
+ char.UTF8String.tagSet: AbstractScalarPayloadDecoder(),
+ char.NumericString.tagSet: AbstractScalarPayloadDecoder(),
+ char.PrintableString.tagSet: AbstractScalarPayloadDecoder(),
+ char.TeletexString.tagSet: AbstractScalarPayloadDecoder(),
+ char.VideotexString.tagSet: AbstractScalarPayloadDecoder(),
+ char.IA5String.tagSet: AbstractScalarPayloadDecoder(),
+ char.GraphicString.tagSet: AbstractScalarPayloadDecoder(),
+ char.VisibleString.tagSet: AbstractScalarPayloadDecoder(),
+ char.GeneralString.tagSet: AbstractScalarPayloadDecoder(),
+ char.UniversalString.tagSet: AbstractScalarPayloadDecoder(),
+ char.BMPString.tagSet: AbstractScalarPayloadDecoder(),
# useful types
@@ -91,7 +91,7 @@
# useful types
- useful.ObjectDescriptor.tagSet: AbstractScalarDecoder(),
- useful.GeneralizedTime.tagSet: AbstractScalarDecoder(),
- useful.UTCTime.tagSet: AbstractScalarDecoder()
+ useful.ObjectDescriptor.tagSet: AbstractScalarPayloadDecoder(),
+ useful.GeneralizedTime.tagSet: AbstractScalarPayloadDecoder(),
+ useful.UTCTime.tagSet: AbstractScalarPayloadDecoder()
}
# Put in ambiguous & non-ambiguous types for faster codec lookup
@@ -95,13 +95,13 @@
}
# Put in ambiguous & non-ambiguous types for faster codec lookup
-typeMap = {
- univ.Integer.typeId: AbstractScalarDecoder(),
- univ.Boolean.typeId: AbstractScalarDecoder(),
- univ.BitString.typeId: BitStringDecoder(),
- univ.OctetString.typeId: AbstractScalarDecoder(),
- univ.Null.typeId: AbstractScalarDecoder(),
- univ.ObjectIdentifier.typeId: AbstractScalarDecoder(),
- univ.Enumerated.typeId: AbstractScalarDecoder(),
- univ.Real.typeId: AbstractScalarDecoder(),
+TYPE_MAP = {
+ univ.Integer.typeId: AbstractScalarPayloadDecoder(),
+ univ.Boolean.typeId: AbstractScalarPayloadDecoder(),
+ univ.BitString.typeId: BitStringPayloadDecoder(),
+ univ.OctetString.typeId: AbstractScalarPayloadDecoder(),
+ univ.Null.typeId: AbstractScalarPayloadDecoder(),
+ univ.ObjectIdentifier.typeId: AbstractScalarPayloadDecoder(),
+ univ.Enumerated.typeId: AbstractScalarPayloadDecoder(),
+ univ.Real.typeId: AbstractScalarPayloadDecoder(),
# ambiguous base types
@@ -107,8 +107,8 @@
# ambiguous base types
- univ.Set.typeId: SequenceOrSetDecoder(),
- univ.SetOf.typeId: SequenceOfOrSetOfDecoder(),
- univ.Sequence.typeId: SequenceOrSetDecoder(),
- univ.SequenceOf.typeId: SequenceOfOrSetOfDecoder(),
- univ.Choice.typeId: ChoiceDecoder(),
- univ.Any.typeId: AbstractScalarDecoder(),
+ univ.Set.typeId: SequenceOrSetPayloadDecoder(),
+ univ.SetOf.typeId: SequenceOfOrSetOfPayloadDecoder(),
+ univ.Sequence.typeId: SequenceOrSetPayloadDecoder(),
+ univ.SequenceOf.typeId: SequenceOfOrSetOfPayloadDecoder(),
+ univ.Choice.typeId: ChoicePayloadDecoder(),
+ univ.Any.typeId: AbstractScalarPayloadDecoder(),
# character string types
@@ -114,13 +114,13 @@
# character string types
- char.UTF8String.typeId: AbstractScalarDecoder(),
- char.NumericString.typeId: AbstractScalarDecoder(),
- char.PrintableString.typeId: AbstractScalarDecoder(),
- char.TeletexString.typeId: AbstractScalarDecoder(),
- char.VideotexString.typeId: AbstractScalarDecoder(),
- char.IA5String.typeId: AbstractScalarDecoder(),
- char.GraphicString.typeId: AbstractScalarDecoder(),
- char.VisibleString.typeId: AbstractScalarDecoder(),
- char.GeneralString.typeId: AbstractScalarDecoder(),
- char.UniversalString.typeId: AbstractScalarDecoder(),
- char.BMPString.typeId: AbstractScalarDecoder(),
+ char.UTF8String.typeId: AbstractScalarPayloadDecoder(),
+ char.NumericString.typeId: AbstractScalarPayloadDecoder(),
+ char.PrintableString.typeId: AbstractScalarPayloadDecoder(),
+ char.TeletexString.typeId: AbstractScalarPayloadDecoder(),
+ char.VideotexString.typeId: AbstractScalarPayloadDecoder(),
+ char.IA5String.typeId: AbstractScalarPayloadDecoder(),
+ char.GraphicString.typeId: AbstractScalarPayloadDecoder(),
+ char.VisibleString.typeId: AbstractScalarPayloadDecoder(),
+ char.GeneralString.typeId: AbstractScalarPayloadDecoder(),
+ char.UniversalString.typeId: AbstractScalarPayloadDecoder(),
+ char.BMPString.typeId: AbstractScalarPayloadDecoder(),
# useful types
@@ -126,7 +126,7 @@
# useful types
- useful.ObjectDescriptor.typeId: AbstractScalarDecoder(),
- useful.GeneralizedTime.typeId: AbstractScalarDecoder(),
- useful.UTCTime.typeId: AbstractScalarDecoder()
+ useful.ObjectDescriptor.typeId: AbstractScalarPayloadDecoder(),
+ useful.GeneralizedTime.typeId: AbstractScalarPayloadDecoder(),
+ useful.UTCTime.typeId: AbstractScalarPayloadDecoder()
}
@@ -130,5 +130,5 @@
}
-class Decoder(object):
+class SingleItemDecoder(object):
@@ -134,10 +134,12 @@
- # noinspection PyDefaultArgument
- def __init__(self, tagMap, typeMap):
- self.__tagMap = tagMap
- self.__typeMap = typeMap
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+ def __init__(self, **options):
+ self._tagMap = options.get('tagMap', self.TAG_MAP)
+ self._typeMap = options.get('typeMap', self.TYPE_MAP)
def __call__(self, pyObject, asn1Spec, **options):
if LOG:
debug.scope.push(type(pyObject).__name__)
@@ -139,8 +141,9 @@
def __call__(self, pyObject, asn1Spec, **options):
if LOG:
debug.scope.push(type(pyObject).__name__)
- LOG('decoder called at scope %s, working with type %s' % (debug.scope, type(pyObject).__name__))
+ LOG('decoder called at scope %s, working with '
+ 'type %s' % (debug.scope, type(pyObject).__name__))
if asn1Spec is None or not isinstance(asn1Spec, base.Asn1Item):
@@ -145,5 +148,7 @@
if asn1Spec is None or not isinstance(asn1Spec, base.Asn1Item):
- raise error.PyAsn1Error('asn1Spec is not valid (should be an instance of an ASN.1 Item, not %s)' % asn1Spec.__class__.__name__)
+ raise error.PyAsn1Error(
+ 'asn1Spec is not valid (should be an instance of an ASN.1 '
+ 'Item, not %s)' % asn1Spec.__class__.__name__)
try:
@@ -148,9 +153,9 @@
try:
- valueDecoder = self.__typeMap[asn1Spec.typeId]
+ valueDecoder = self._typeMap[asn1Spec.typeId]
except KeyError:
# use base type for codec lookup to recover untagged types
baseTagSet = tag.TagSet(asn1Spec.tagSet.baseTag, asn1Spec.tagSet.baseTag)
try:
@@ -151,11 +156,12 @@
except KeyError:
# use base type for codec lookup to recover untagged types
baseTagSet = tag.TagSet(asn1Spec.tagSet.baseTag, asn1Spec.tagSet.baseTag)
try:
- valueDecoder = self.__tagMap[baseTagSet]
+ valueDecoder = self._tagMap[baseTagSet]
+
except KeyError:
raise error.PyAsn1Error('Unknown ASN.1 tag %s' % asn1Spec.tagSet)
if LOG:
@@ -158,9 +164,11 @@
except KeyError:
raise error.PyAsn1Error('Unknown ASN.1 tag %s' % asn1Spec.tagSet)
if LOG:
- LOG('calling decoder %s on Python type %s <%s>' % (type(valueDecoder).__name__, type(pyObject).__name__, repr(pyObject)))
+ LOG('calling decoder %s on Python type %s '
+ '<%s>' % (type(valueDecoder).__name__,
+ type(pyObject).__name__, repr(pyObject)))
value = valueDecoder(pyObject, asn1Spec, self, **options)
if LOG:
@@ -163,10 +171,12 @@
value = valueDecoder(pyObject, asn1Spec, self, **options)
if LOG:
- LOG('decoder %s produced ASN.1 type %s <%s>' % (type(valueDecoder).__name__, type(value).__name__, repr(value)))
+ LOG('decoder %s produced ASN.1 type %s '
+ '<%s>' % (type(valueDecoder).__name__,
+ type(value).__name__, repr(value)))
debug.scope.pop()
return value
@@ -168,8 +178,18 @@
debug.scope.pop()
return value
+class Decoder(object):
+ SINGLE_ITEM_DECODER = SingleItemDecoder
+
+ def __init__(self, **options):
+ self._singleItemDecoder = self.SINGLE_ITEM_DECODER(**options)
+
+ def __call__(self, pyObject, asn1Spec=None, **kwargs):
+ return self._singleItemDecoder(pyObject, asn1Spec=asn1Spec, **kwargs)
+
+
#: Turns Python objects of built-in types into ASN.1 objects.
#:
#: Takes Python objects of built-in types and turns them into a tree of
@@ -210,4 +230,4 @@
#: SequenceOf:
#: 1 2 3
#:
-decode = Decoder(tagMap, typeMap)
+decode = Decoder()
diff --git a/pyasn1/codec/native/encoder.py b/pyasn1/codec/native/encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2NvZGVjL25hdGl2ZS9lbmNvZGVyLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL25hdGl2ZS9lbmNvZGVyLnB5 100644
--- a/pyasn1/codec/native/encoder.py
+++ b/pyasn1/codec/native/encoder.py
@@ -103,7 +103,7 @@
return value.asOctets()
-tagMap = {
+TAG_MAP = {
univ.Boolean.tagSet: BooleanEncoder(),
univ.Integer.tagSet: IntegerEncoder(),
univ.BitString.tagSet: BitStringEncoder(),
@@ -136,7 +136,7 @@
# Put in ambiguous & non-ambiguous types for faster codec lookup
-typeMap = {
+TYPE_MAP = {
univ.Boolean.typeId: BooleanEncoder(),
univ.Integer.typeId: IntegerEncoder(),
univ.BitString.typeId: BitStringEncoder(),
@@ -171,5 +171,5 @@
}
-class Encoder(object):
+class SingleItemEncoder(object):
@@ -175,8 +175,10 @@
- # noinspection PyDefaultArgument
- def __init__(self, tagMap, typeMap={}):
- self.__tagMap = tagMap
- self.__typeMap = typeMap
+ TAG_MAP = TAG_MAP
+ TYPE_MAP = TYPE_MAP
+
+ def __init__(self, **options):
+ self._tagMap = options.get('tagMap', self.TAG_MAP)
+ self._typeMap = options.get('typeMap', self.TYPE_MAP)
def __call__(self, value, **options):
if not isinstance(value, base.Asn1Item):
@@ -180,7 +182,8 @@
def __call__(self, value, **options):
if not isinstance(value, base.Asn1Item):
- raise error.PyAsn1Error('value is not valid (should be an instance of an ASN.1 Item)')
+ raise error.PyAsn1Error(
+ 'value is not valid (should be an instance of an ASN.1 Item)')
if LOG:
debug.scope.push(type(value).__name__)
@@ -184,8 +187,9 @@
if LOG:
debug.scope.push(type(value).__name__)
- LOG('encoder called for type %s <%s>' % (type(value).__name__, value.prettyPrint()))
+ LOG('encoder called for type %s '
+ '<%s>' % (type(value).__name__, value.prettyPrint()))
tagSet = value.tagSet
try:
@@ -188,8 +192,8 @@
tagSet = value.tagSet
try:
- concreteEncoder = self.__typeMap[value.typeId]
+ concreteEncoder = self._typeMap[value.typeId]
except KeyError:
# use base type for codec lookup to recover untagged types
@@ -193,6 +197,7 @@
except KeyError:
# use base type for codec lookup to recover untagged types
- baseTagSet = tag.TagSet(value.tagSet.baseTag, value.tagSet.baseTag)
+ baseTagSet = tag.TagSet(
+ value.tagSet.baseTag, value.tagSet.baseTag)
try:
@@ -197,8 +202,8 @@
try:
- concreteEncoder = self.__tagMap[baseTagSet]
+ concreteEncoder = self._tagMap[baseTagSet]
except KeyError:
raise error.PyAsn1Error('No encoder for %s' % (value,))
if LOG:
@@ -200,10 +205,11 @@
except KeyError:
raise error.PyAsn1Error('No encoder for %s' % (value,))
if LOG:
- LOG('using value codec %s chosen by %s' % (concreteEncoder.__class__.__name__, tagSet))
+ LOG('using value codec %s chosen by '
+ '%s' % (concreteEncoder.__class__.__name__, tagSet))
pyObject = concreteEncoder.encode(value, self, **options)
if LOG:
@@ -206,10 +212,11 @@
pyObject = concreteEncoder.encode(value, self, **options)
if LOG:
- LOG('encoder %s produced: %s' % (type(concreteEncoder).__name__, repr(pyObject)))
+ LOG('encoder %s produced: '
+ '%s' % (type(concreteEncoder).__name__, repr(pyObject)))
debug.scope.pop()
return pyObject
@@ -211,8 +218,19 @@
debug.scope.pop()
return pyObject
+class Encoder(object):
+ SINGLE_ITEM_ENCODER = SingleItemEncoder
+
+ def __init__(self, **options):
+ self._singleItemEncoder = self.SINGLE_ITEM_ENCODER(**options)
+
+ def __call__(self, pyObject, asn1Spec=None, **options):
+ return self._singleItemEncoder(
+ pyObject, asn1Spec=asn1Spec, **options)
+
+
#: Turns ASN.1 object into a Python built-in type object(s).
#:
#: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative)
@@ -248,4 +266,4 @@
#: >>> encode(seq)
#: [1, 2, 3]
#:
-encode = Encoder(tagMap, typeMap)
+encode = SingleItemEncoder()
diff --git a/pyasn1/codec/streaming.py b/pyasn1/codec/streaming.py
new file mode 100644
index 0000000000000000000000000000000000000000..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2NvZGVjL3N0cmVhbWluZy5weQ==
--- /dev/null
+++ b/pyasn1/codec/streaming.py
@@ -0,0 +1,243 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com>
+# License: http://snmplabs.com/pyasn1/license.html
+#
+import io
+import os
+import sys
+
+from pyasn1 import error
+from pyasn1.type import univ
+
+_PY2 = sys.version_info < (3,)
+
+
+class CachingStreamWrapper(io.IOBase):
+ """Wrapper around non-seekable streams.
+
+ Note that the implementation is tied to the decoder,
+ not checking for dangerous arguments for the sake
+ of performance.
+
+ The read bytes are kept in an internal cache until
+ setting _markedPosition which may reset the cache.
+ """
+ def __init__(self, raw):
+ self._raw = raw
+ self._cache = io.BytesIO()
+ self._markedPosition = 0
+
+ def peek(self, n):
+ result = self.read(n)
+ self._cache.seek(-len(result), os.SEEK_CUR)
+ return result
+
+ def seekable(self):
+ return True
+
+ def seek(self, n=-1, whence=os.SEEK_SET):
+ # Note that this not safe for seeking forward.
+ return self._cache.seek(n, whence)
+
+ def read(self, n=-1):
+ read_from_cache = self._cache.read(n)
+ if n != -1:
+ n -= len(read_from_cache)
+ if not n: # 0 bytes left to read
+ return read_from_cache
+
+ read_from_raw = self._raw.read(n)
+
+ self._cache.write(read_from_raw)
+
+ return read_from_cache + read_from_raw
+
+ @property
+ def markedPosition(self):
+ """Position where the currently processed element starts.
+
+ This is used for back-tracking in SingleItemDecoder.__call__
+ and (indefLen)ValueDecoder and should not be used for other purposes.
+ The client is not supposed to ever seek before this position.
+ """
+ return self._markedPosition
+
+ @markedPosition.setter
+ def markedPosition(self, value):
+ # By setting the value, we ensure we won't seek back before it.
+ # `value` should be the same as the current position
+ # We don't check for this for performance reasons.
+ self._markedPosition = value
+
+ # Whenever we set _marked_position, we know for sure
+ # that we will not return back, and thus it is
+ # safe to drop all cached data.
+ if self._cache.tell() > io.DEFAULT_BUFFER_SIZE:
+ self._cache = io.BytesIO(self._cache.read())
+ self._markedPosition = 0
+
+ def tell(self):
+ return self._cache.tell()
+
+
+def asSeekableStream(substrate):
+ """Convert object to seekable byte-stream.
+
+ Parameters
+ ----------
+ substrate: :py:class:`bytes` or :py:class:`io.IOBase` or :py:class:`univ.OctetString`
+
+ Returns
+ -------
+ : :py:class:`io.IOBase`
+
+ Raises
+ ------
+ : :py:class:`~pyasn1.error.PyAsn1Error`
+ If the supplied substrate cannot be converted to a seekable stream.
+ """
+ if isinstance(substrate, io.BytesIO):
+ return substrate
+
+ elif isinstance(substrate, bytes):
+ return io.BytesIO(substrate)
+
+ elif isinstance(substrate, univ.OctetString):
+ return io.BytesIO(substrate.asOctets())
+
+ try:
+ # Special case: impossible to set attributes on `file` built-in
+ if _PY2 and isinstance(substrate, file):
+ return io.BufferedReader(substrate)
+
+ elif substrate.seekable(): # Will fail for most invalid types
+ return substrate
+
+ else:
+ return CachingStreamWrapper(substrate)
+
+ except AttributeError:
+ raise error.UnsupportedSubstrateError(
+ "Cannot convert " + substrate.__class__.__name__ +
+ " to a seekable bit stream.")
+
+
+def isEndOfStream(substrate):
+ """Check whether we have reached the end of a stream.
+
+ Although it is more effective to read and catch exceptions, this
+ function
+
+ Parameters
+ ----------
+ substrate: :py:class:`IOBase`
+ Stream to check
+
+ Returns
+ -------
+ : :py:class:`bool`
+ """
+ if isinstance(substrate, io.BytesIO):
+ cp = substrate.tell()
+ substrate.seek(0, os.SEEK_END)
+ result = substrate.tell() == cp
+ substrate.seek(cp, os.SEEK_SET)
+ yield result
+
+ else:
+ received = substrate.read(1)
+ if received is None:
+ yield
+
+ if received:
+ substrate.seek(-1, os.SEEK_CUR)
+
+ yield not received
+
+
+def peekIntoStream(substrate, size=-1):
+ """Peek into stream.
+
+ Parameters
+ ----------
+ substrate: :py:class:`IOBase`
+ Stream to read from.
+
+ size: :py:class:`int`
+ How many bytes to peek (-1 = all available)
+
+ Returns
+ -------
+ : :py:class:`bytes` or :py:class:`str`
+ The return type depends on Python major version
+ """
+ if hasattr(substrate, "peek"):
+ received = substrate.peek(size)
+ if received is None:
+ yield
+
+ while len(received) < size:
+ yield
+
+ yield received
+
+ else:
+ current_position = substrate.tell()
+ try:
+ for chunk in readFromStream(substrate, size):
+ yield chunk
+
+ finally:
+ substrate.seek(current_position)
+
+
+def readFromStream(substrate, size=-1, context=None):
+ """Read from the stream.
+
+ Parameters
+ ----------
+ substrate: :py:class:`IOBase`
+ Stream to read from.
+
+ Keyword parameters
+ ------------------
+ size: :py:class:`int`
+ How many bytes to read (-1 = all available)
+
+ context: :py:class:`dict`
+ Opaque caller context will be attached to exception objects created
+ by this function.
+
+ Yields
+ ------
+ : :py:class:`bytes` or :py:class:`str` or :py:class:`SubstrateUnderrunError`
+ Read data or :py:class:`~pyasn1.error.SubstrateUnderrunError`
+ object if no `size` bytes is readily available in the stream. The
+ data type depends on Python major version
+
+ Raises
+ ------
+ : :py:class:`~pyasn1.error.EndOfStreamError`
+ Input stream is exhausted
+ """
+ while True:
+ # this will block unless stream is non-blocking
+ received = substrate.read(size)
+ if received is None: # non-blocking stream can do this
+ yield error.SubstrateUnderrunError(context=context)
+
+ elif not received and size != 0: # end-of-stream
+ raise error.EndOfStreamError(context=context)
+
+ elif len(received) < size:
+ substrate.seek(-len(received), os.SEEK_CUR)
+
+ # behave like a non-blocking stream
+ yield error.SubstrateUnderrunError(context=context)
+
+ else:
+ break
+
+ yield received
diff --git a/pyasn1/error.py b/pyasn1/error.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_cHlhc24xL2Vycm9yLnB5..429a07287d1b98eda26bd821c0e3006b0a59de28_cHlhc24xL2Vycm9yLnB5 100644
--- a/pyasn1/error.py
+++ b/pyasn1/error.py
@@ -12,4 +12,15 @@
`PyAsn1Error` is the base exception class (based on
:class:`Exception`) that represents all possible ASN.1 related
errors.
+
+ Parameters
+ ----------
+ args:
+ Opaque positional parameters
+
+ Keyword Args
+ ------------
+ kwargs:
+ Opaque keyword parameters
+
"""
@@ -15,4 +26,22 @@
"""
+ def __init__(self, *args, **kwargs):
+ self._args = args
+ self._kwargs = kwargs
+
+ @property
+ def context(self):
+ """Return exception context
+
+ When exception object is created, the caller can supply some opaque
+ context for the upper layers to better understand the cause of the
+ exception.
+
+ Returns
+ -------
+ : :py:class:`dict`
+ Dict holding context specific data
+ """
+ return self._kwargs.get('context', {})
class ValueConstraintError(PyAsn1Error):
@@ -34,6 +63,18 @@
"""
+class EndOfStreamError(SubstrateUnderrunError):
+ """ASN.1 data structure deserialization error
+
+ The `EndOfStreamError` exception indicates the condition of the input
+ stream has been closed.
+ """
+
+
+class UnsupportedSubstrateError(PyAsn1Error):
+ """Unsupported substrate type to parse as ASN.1 data."""
+
+
class PyAsn1UnicodeError(PyAsn1Error, UnicodeError):
"""Unicode text processing error
diff --git a/tests/codec/__main__.py b/tests/codec/__main__.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_dGVzdHMvY29kZWMvX19tYWluX18ucHk=..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvX19tYWluX18ucHk= 100644
--- a/tests/codec/__main__.py
+++ b/tests/codec/__main__.py
@@ -7,7 +7,8 @@
import unittest
suite = unittest.TestLoader().loadTestsFromNames(
- ['tests.codec.ber.__main__.suite',
+ ['tests.codec.test_streaming.suite',
+ 'tests.codec.ber.__main__.suite',
'tests.codec.cer.__main__.suite',
'tests.codec.der.__main__.suite',
'tests.codec.native.__main__.suite']
diff --git a/tests/codec/ber/test_decoder.py b/tests/codec/ber/test_decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_dGVzdHMvY29kZWMvYmVyL3Rlc3RfZGVjb2Rlci5weQ==..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvYmVyL3Rlc3RfZGVjb2Rlci5weQ== 100644
--- a/tests/codec/ber/test_decoder.py
+++ b/tests/codec/ber/test_decoder.py
@@ -4,4 +4,7 @@
# Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com>
# License: http://snmplabs.com/pyasn1/license.html
#
+import gzip
+import io
+import os
import sys
@@ -7,2 +10,3 @@
import sys
+import tempfile
import unittest
@@ -8,4 +12,5 @@
import unittest
+import zipfile
from tests.base import BaseTestCase
@@ -14,6 +19,7 @@
from pyasn1.type import opentype
from pyasn1.type import univ
from pyasn1.type import char
+from pyasn1.codec import streaming
from pyasn1.codec.ber import decoder
from pyasn1.codec.ber import eoo
from pyasn1.compat.octets import ints2octs, str2octs, null
@@ -17,7 +23,7 @@
from pyasn1.codec.ber import decoder
from pyasn1.codec.ber import eoo
from pyasn1.compat.octets import ints2octs, str2octs, null
-from pyasn1.error import PyAsn1Error
+from pyasn1 import error
class LargeTagDecoderTestCase(BaseTestCase):
@@ -69,7 +75,7 @@
decoder.decode(
ints2octs((2, 1, 12)), asn1Spec=univ.Null()
) == (12, null)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong asn1Spec worked out'
@@ -80,7 +86,7 @@
def testTagFormat(self):
try:
decoder.decode(ints2octs((34, 1, 12)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -102,7 +108,7 @@
def testTagFormat(self):
try:
decoder.decode(ints2octs((33, 1, 1)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -132,9 +138,9 @@
def testDefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((35, 8, 3, 2, 0, 169, 3, 2, 1, 138)),
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)),
@@ -136,11 +142,11 @@
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)),
- substrateFun=lambda a, b, c: (b, str2octs(''))
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138, 0, 0)), str2octs(''))
def testTypeChecking(self):
try:
decoder.decode(ints2octs((35, 4, 2, 2, 42, 42)))
@@ -142,9 +148,9 @@
) == (ints2octs((3, 2, 0, 169, 3, 2, 1, 138, 0, 0)), str2octs(''))
def testTypeChecking(self):
try:
decoder.decode(ints2octs((35, 4, 2, 2, 42, 42)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'accepted mis-encoded bit-string constructed out of an integer'
@@ -176,10 +182,10 @@
assert decoder.decode(
ints2octs(
(36, 23, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)),
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((36, 128, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111,
120, 0, 0)),
@@ -180,10 +186,10 @@
) == (ints2octs((4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120)), str2octs(''))
def testIndefModeChunkedSubst(self):
assert decoder.decode(
ints2octs((36, 128, 4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111,
120, 0, 0)),
- substrateFun=lambda a, b, c: (b, str2octs(''))
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(4, 4, 81, 117, 105, 99, 4, 4, 107, 32, 98, 114, 4, 4, 111, 119, 110, 32, 4, 3, 102, 111, 120, 0, 0)), str2octs(''))
@@ -236,7 +242,7 @@
def testDefModeSubst(self):
assert decoder.decode(
ints2octs((101, 17, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)),
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120)), str2octs(''))
def testIndefModeSubst(self):
@@ -244,7 +250,7 @@
ints2octs((
101, 128, 36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0,
0, 0, 0)),
- substrateFun=lambda a, b, c: (b, str2octs(''))
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(36, 128, 4, 15, 81, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 32, 102, 111, 120, 0, 0, 0, 0)), str2octs(''))
@@ -256,7 +262,7 @@
def testTagFormat(self):
try:
decoder.decode(ints2octs((37, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -325,7 +331,7 @@
decoder.decode(
ints2octs((6, 5, 85, 4, 128, 129, 0))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'Leading 0x80 tolerated'
@@ -335,7 +341,7 @@
decoder.decode(
ints2octs((6, 7, 1, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7F))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'Leading 0x80 tolerated'
@@ -345,7 +351,7 @@
decoder.decode(
ints2octs((6, 2, 0x80, 1))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'Leading 0x80 tolerated'
@@ -355,7 +361,7 @@
decoder.decode(
ints2octs((6, 2, 0x80, 0x7F))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'Leading 0x80 tolerated'
@@ -363,7 +369,7 @@
def testTagFormat(self):
try:
decoder.decode(ints2octs((38, 1, 239)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -371,7 +377,7 @@
def testZeroLength(self):
try:
decoder.decode(ints2octs((6, 0, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'zero length tolerated'
@@ -379,7 +385,7 @@
def testIndefiniteLength(self):
try:
decoder.decode(ints2octs((6, 128, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'indefinite length tolerated'
@@ -387,7 +393,7 @@
def testReservedLength(self):
try:
decoder.decode(ints2octs((6, 255, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'reserved length tolerated'
@@ -464,7 +470,7 @@
def testTagFormat(self):
try:
decoder.decode(ints2octs((41, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -472,7 +478,7 @@
def testShortEncoding(self):
try:
decoder.decode(ints2octs((9, 1, 131)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'accepted too-short real'
@@ -671,9 +677,9 @@
def testWithOptionalAndDefaultedDefModeSubst(self):
assert decoder.decode(
ints2octs((48, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((48, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
@@ -675,9 +681,9 @@
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((48, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
- substrateFun=lambda a, b, c: (b, str2octs(''))
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
@@ -686,7 +692,7 @@
decoder.decode(
ints2octs((16, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -868,7 +874,7 @@
decodeOpenTypes=True
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
@@ -1007,7 +1013,7 @@
decodeOpenTypes=True
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
@@ -1157,9 +1163,9 @@
def testWithOptionalAndDefaultedDefModeSubst(self):
assert decoder.decode(
ints2octs((49, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)),
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((49, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
@@ -1161,9 +1167,9 @@
) == (ints2octs((5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1)), str2octs(''))
def testWithOptionalAndDefaultedIndefModeSubst(self):
assert decoder.decode(
ints2octs((49, 128, 5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)),
- substrateFun=lambda a, b, c: (b, str2octs(''))
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs(
(5, 0, 36, 128, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 0, 0, 2, 1, 1, 0, 0)), str2octs(''))
@@ -1172,7 +1178,7 @@
decoder.decode(
ints2octs((16, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1))
)
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'wrong tagFormat worked out'
@@ -1489,10 +1495,10 @@
assert decoder.decode(
ints2octs((4, 3, 102, 111, 120)),
asn1Spec=self.s,
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((4, 3, 102, 111, 120)), str2octs(''))
def testTaggedExSubst(self):
assert decoder.decode(
ints2octs((164, 5, 4, 3, 102, 111, 120)),
asn1Spec=self.s,
@@ -1493,10 +1499,10 @@
) == (ints2octs((4, 3, 102, 111, 120)), str2octs(''))
def testTaggedExSubst(self):
assert decoder.decode(
ints2octs((164, 5, 4, 3, 102, 111, 120)),
asn1Spec=self.s,
- substrateFun=lambda a, b, c: (b, b[c:])
+ substrateFun=lambda a, b, c, d: streaming.readFromStream(b, c)
) == (ints2octs((164, 5, 4, 3, 102, 111, 120)), str2octs(''))
@@ -1504,7 +1510,7 @@
def testUnexpectedEoo(self):
try:
decoder.decode(ints2octs((0, 0)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'end-of-contents octets accepted at top level'
@@ -1517,7 +1523,7 @@
def testDefiniteNoEoo(self):
try:
decoder.decode(ints2octs((0x23, 0x02, 0x00, 0x00)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'end-of-contents octets accepted inside definite-length encoding'
@@ -1529,7 +1535,7 @@
def testNoLongFormEoo(self):
try:
decoder.decode(ints2octs((0x23, 0x80, 0x00, 0x81, 0x00)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'end-of-contents octets accepted with invalid long-form length'
@@ -1537,7 +1543,7 @@
def testNoConstructedEoo(self):
try:
decoder.decode(ints2octs((0x23, 0x80, 0x20, 0x00)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'end-of-contents octets accepted with invalid constructed encoding'
@@ -1545,7 +1551,7 @@
def testNoEooData(self):
try:
decoder.decode(ints2octs((0x23, 0x80, 0x00, 0x01, 0x00)))
- except PyAsn1Error:
+ except error.PyAsn1Error:
pass
else:
assert 0, 'end-of-contents octets accepted with unexpected data'
@@ -1568,7 +1574,8 @@
self.substrate = ints2octs([48, 18, 5, 0, 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110, 2, 1, 1])
def testOctetString(self):
- s, _ = decoder.decode(univ.OctetString(self.substrate), asn1Spec=self.s)
- assert self.s == s
+ s = list(decoder.StreamingDecoder(
+ univ.OctetString(self.substrate), asn1Spec=self.s))
+ assert [self.s] == s
def testAny(self):
@@ -1573,9 +1580,10 @@
def testAny(self):
- s, _ = decoder.decode(univ.Any(self.substrate), asn1Spec=self.s)
- assert self.s == s
+ s = list(decoder.StreamingDecoder(
+ univ.Any(self.substrate), asn1Spec=self.s))
+ assert [self.s] == s
class ErrorOnDecodingTestCase(BaseTestCase):
def testErrorCondition(self):
@@ -1577,8 +1585,11 @@
class ErrorOnDecodingTestCase(BaseTestCase):
def testErrorCondition(self):
- decode = decoder.Decoder(decoder.tagMap, decoder.typeMap)
+ decode = decoder.SingleItemDecoder(
+ tagMap=decoder.TAG_MAP, typeMap=decoder.TYPE_MAP)
+ substrate = ints2octs((00, 1, 2))
+ stream = streaming.asSeekableStream(substrate)
try:
@@ -1583,4 +1594,4 @@
try:
- asn1Object, rest = decode(str2octs('abc'))
+ asn1Object = next(decode(stream))
@@ -1586,3 +1597,3 @@
- except PyAsn1Error:
+ except error.PyAsn1Error:
exc = sys.exc_info()[1]
@@ -1588,8 +1599,8 @@
exc = sys.exc_info()[1]
- assert isinstance(exc, PyAsn1Error), (
+ assert isinstance(exc, error.PyAsn1Error), (
'Unexpected exception raised %r' % (exc,))
else:
assert False, 'Unexpected decoder result %r' % (asn1Object,)
def testRawDump(self):
@@ -1590,8 +1601,12 @@
'Unexpected exception raised %r' % (exc,))
else:
assert False, 'Unexpected decoder result %r' % (asn1Object,)
def testRawDump(self):
- decode = decoder.Decoder(decoder.tagMap, decoder.typeMap)
+ substrate = ints2octs((31, 8, 2, 1, 1, 131, 3, 2, 1, 12))
+ stream = streaming.asSeekableStream(substrate)
+
+ class SingleItemEncoder(decoder.SingleItemDecoder):
+ defaultErrorState = decoder.stDumpRawValue
@@ -1597,3 +1612,4 @@
- decode.defaultErrorState = decoder.stDumpRawValue
+ class StreamingDecoder(decoder.StreamingDecoder):
+ SINGLE_ITEM_DECODER = SingleItemEncoder
@@ -1599,6 +1615,10 @@
- asn1Object, rest = decode(ints2octs(
- (31, 8, 2, 1, 1, 131, 3, 2, 1, 12)))
+ class OneShotDecoder(decoder.Decoder):
+ STREAMING_DECODER = StreamingDecoder
+
+ d = OneShotDecoder()
+
+ asn1Object, rest = d(stream)
assert isinstance(asn1Object, univ.Any), (
'Unexpected raw dump type %r' % (asn1Object,))
@@ -1608,6 +1628,218 @@
'Unexpected rest of substrate after raw dump %r' % rest)
+class BinaryFileTestCase(BaseTestCase):
+ """Assure that decode works on open binary files."""
+ def testOneObject(self):
+ _, path = tempfile.mkstemp()
+ try:
+ with open(path, "wb") as out:
+ out.write(ints2octs((2, 1, 12)))
+
+ with open(path, "rb") as source:
+ values = list(decoder.StreamingDecoder(source))
+
+ assert values == [12]
+ finally:
+ os.remove(path)
+
+ def testMoreObjects(self):
+ _, path = tempfile.mkstemp()
+ try:
+ with open(path, "wb") as out:
+ out.write(ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)))
+
+ with open(path, "rb") as source:
+ values = list(decoder.StreamingDecoder(source))
+
+ assert values == [12, (1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1)]
+
+ finally:
+ os.remove(path)
+
+ def testInvalidFileContent(self):
+ _, path = tempfile.mkstemp()
+ try:
+ with open(path, "wb") as out:
+ out.write(ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0, 7)))
+
+ with open(path, "rb") as source:
+ list(decoder.StreamingDecoder(source))
+
+ except error.EndOfStreamError:
+ pass
+
+ finally:
+ os.remove(path)
+
+
+class BytesIOTestCase(BaseTestCase):
+ def testRead(self):
+ source = ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0))
+ stream = io.BytesIO(source)
+ values = list(decoder.StreamingDecoder(stream))
+ assert values == [12, (1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1)]
+
+
+class UnicodeTestCase(BaseTestCase):
+ def testFail(self):
+ # This ensures that unicode objects in Python 2 & str objects in Python 3.7 cannot be parsed.
+ source = ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)).decode("latin-1")
+ try:
+ next(decoder.StreamingDecoder(source))
+
+ except error.UnsupportedSubstrateError:
+ pass
+
+ else:
+ assert False, 'Tolerated parsing broken unicode strings'
+
+
+class RestartableDecoderTestCase(BaseTestCase):
+
+ class NonBlockingStream(io.BytesIO):
+ block = False
+
+ def read(self, size=-1):
+ self.block = not self.block
+ if self.block:
+ return # this is what non-blocking streams sometimes do
+
+ return io.BytesIO.read(self, size)
+
+ def setUp(self):
+ BaseTestCase.setUp(self)
+
+ self.s = univ.SequenceOf(componentType=univ.OctetString())
+ self.s.setComponentByPosition(0, univ.OctetString('quick brown'))
+ source = ints2octs(
+ (48, 26,
+ 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110,
+ 4, 11, 113, 117, 105, 99, 107, 32, 98, 114, 111, 119, 110))
+ self.stream = self.NonBlockingStream(source)
+
+ def testPartialReadingFromNonBlockingStream(self):
+ iterator = iter(decoder.StreamingDecoder(self.stream, asn1Spec=self.s))
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' not in res.context
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' not in res.context
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 0
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 0
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 0
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 1
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 1
+
+ res = next(iterator)
+
+ assert isinstance(res, error.SubstrateUnderrunError)
+ assert 'asn1Object' in res.context
+ assert isinstance(res.context['asn1Object'], univ.SequenceOf)
+ assert res.context['asn1Object'].isValue
+ assert len(res.context['asn1Object']) == 1
+
+ res = next(iterator)
+
+ assert isinstance(res, univ.SequenceOf)
+ assert res.isValue
+ assert len(res) == 2
+
+ try:
+ next(iterator)
+
+ except StopIteration:
+ pass
+
+ else:
+ assert False, 'End of stream not raised'
+
+
+class CompressedFilesTestCase(BaseTestCase):
+ def testGzip(self):
+ _, path = tempfile.mkstemp(suffix=".gz")
+ try:
+ with gzip.open(path, "wb") as out:
+ out.write(ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)))
+
+ with gzip.open(path, "rb") as source:
+ values = list(decoder.StreamingDecoder(source))
+
+ assert values == [12, (1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1)]
+
+ finally:
+ os.remove(path)
+
+ def testZipfile(self):
+ # File from ZIP archive is a good example of non-seekable stream in Python 2.7
+ # In Python 3.7, it is a seekable stream.
+ _, path = tempfile.mkstemp(suffix=".zip")
+ try:
+ with zipfile.ZipFile(path, "w") as myzip:
+ myzip.writestr("data", ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)))
+
+ with zipfile.ZipFile(path, "r") as myzip:
+ with myzip.open("data", "r") as source:
+ values = list(decoder.StreamingDecoder(source))
+ assert values == [12, (1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1)]
+ finally:
+ os.remove(path)
+
+ def testZipfileMany(self):
+ _, path = tempfile.mkstemp(suffix=".zip")
+ try:
+ with zipfile.ZipFile(path, "w") as myzip:
+ #for i in range(100):
+ myzip.writestr("data", ints2octs((2, 1, 12, 35, 128, 3, 2, 0, 169, 3, 2, 1, 138, 0, 0)) * 1000)
+
+ with zipfile.ZipFile(path, "r") as myzip:
+ with myzip.open("data", "r") as source:
+ values = list(decoder.StreamingDecoder(source))
+ assert values == [12, (1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1)] * 1000
+ finally:
+ os.remove(path)
+
+
suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
if __name__ == '__main__':
diff --git a/tests/codec/ber/test_encoder.py b/tests/codec/ber/test_encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_dGVzdHMvY29kZWMvYmVyL3Rlc3RfZW5jb2Rlci5weQ==..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvYmVyL3Rlc3RfZW5jb2Rlci5weQ== 100644
--- a/tests/codec/ber/test_encoder.py
+++ b/tests/codec/ber/test_encoder.py
@@ -377,7 +377,7 @@
def testBin3(self):
# change binEncBase in the RealEncoder instance => for all further Real
- binEncBase, encoder.typeMap[univ.Real.typeId].binEncBase = encoder.typeMap[univ.Real.typeId].binEncBase, 16
+ binEncBase, encoder.TYPE_MAP[univ.Real.typeId].binEncBase = encoder.TYPE_MAP[univ.Real.typeId].binEncBase, 16
assert encoder.encode(
univ.Real((0.00390625, 2, 0)) # check encbase = 16
) == ints2octs((9, 3, 160, 254, 1))
@@ -381,7 +381,7 @@
assert encoder.encode(
univ.Real((0.00390625, 2, 0)) # check encbase = 16
) == ints2octs((9, 3, 160, 254, 1))
- encoder.typeMap[univ.Real.typeId].binEncBase = binEncBase
+ encoder.TYPE_MAP[univ.Real.typeId].binEncBase = binEncBase
def testBin4(self):
# choose binEncBase automatically for all further Real (testBin[4-7])
@@ -385,7 +385,7 @@
def testBin4(self):
# choose binEncBase automatically for all further Real (testBin[4-7])
- binEncBase, encoder.typeMap[univ.Real.typeId].binEncBase = encoder.typeMap[univ.Real.typeId].binEncBase, None
+ binEncBase, encoder.TYPE_MAP[univ.Real.typeId].binEncBase = encoder.TYPE_MAP[univ.Real.typeId].binEncBase, None
assert encoder.encode(
univ.Real((1, 2, 0)) # check exponent = 0
) == ints2octs((9, 3, 128, 0, 1))
@@ -389,7 +389,7 @@
assert encoder.encode(
univ.Real((1, 2, 0)) # check exponent = 0
) == ints2octs((9, 3, 128, 0, 1))
- encoder.typeMap[univ.Real.typeId].binEncBase = binEncBase
+ encoder.TYPE_MAP[univ.Real.typeId].binEncBase = binEncBase
def testBin5(self):
assert encoder.encode(
diff --git a/tests/codec/cer/test_decoder.py b/tests/codec/cer/test_decoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_dGVzdHMvY29kZWMvY2VyL3Rlc3RfZGVjb2Rlci5weQ==..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvY2VyL3Rlc3RfZGVjb2Rlci5weQ== 100644
--- a/tests/codec/cer/test_decoder.py
+++ b/tests/codec/cer/test_decoder.py
@@ -37,6 +37,7 @@
except PyAsn1Error:
pass
+
class BitStringDecoderTestCase(BaseTestCase):
def testShortMode(self):
assert decoder.decode(
diff --git a/tests/codec/cer/test_encoder.py b/tests/codec/cer/test_encoder.py
index 89d6f575bf98eccab40ce442b3e56ec3a3dffe45_dGVzdHMvY29kZWMvY2VyL3Rlc3RfZW5jb2Rlci5weQ==..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvY2VyL3Rlc3RfZW5jb2Rlci5weQ== 100644
--- a/tests/codec/cer/test_encoder.py
+++ b/tests/codec/cer/test_encoder.py
@@ -80,7 +80,6 @@
else:
assert 0, 'Missing timezone tolerated'
-
def testDecimalCommaPoint(self):
try:
assert encoder.encode(
diff --git a/tests/codec/test_streaming.py b/tests/codec/test_streaming.py
new file mode 100644
index 0000000000000000000000000000000000000000..429a07287d1b98eda26bd821c0e3006b0a59de28_dGVzdHMvY29kZWMvdGVzdF9zdHJlYW1pbmcucHk=
--- /dev/null
+++ b/tests/codec/test_streaming.py
@@ -0,0 +1,75 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com>
+# License: http://snmplabs.com/pyasn1/license.html
+#
+import io
+import sys
+
+try:
+ import unittest2 as unittest
+
+except ImportError:
+ import unittest
+
+from tests.base import BaseTestCase
+
+from pyasn1.codec import streaming
+
+
+class CachingStreamWrapperTestCase(BaseTestCase):
+ def setUp(self):
+ self.shortText = b"abcdefghij"
+ self.longText = self.shortText * (io.DEFAULT_BUFFER_SIZE * 5)
+ self.shortStream = io.BytesIO(self.shortText)
+ self.longStream = io.BytesIO(self.longText)
+
+ def testReadJustFromCache(self):
+ wrapper = streaming.CachingStreamWrapper(self.shortStream)
+ wrapper.read(6)
+ wrapper.seek(3)
+ assert wrapper.read(1) == b"d"
+ assert wrapper.read(1) == b"e"
+ assert wrapper.tell() == 5
+
+ def testReadFromCacheAndStream(self):
+ wrapper = streaming.CachingStreamWrapper(self.shortStream)
+ wrapper.read(6)
+ wrapper.seek(3)
+ assert wrapper.read(4) == b"defg"
+ assert wrapper.tell() == 7
+
+ def testReadJustFromStream(self):
+ wrapper = streaming.CachingStreamWrapper(self.shortStream)
+ assert wrapper.read(6) == b"abcdef"
+ assert wrapper.tell() == 6
+
+ def testPeek(self):
+ wrapper = streaming.CachingStreamWrapper(self.longStream)
+ read_bytes = wrapper.peek(io.DEFAULT_BUFFER_SIZE + 73)
+ assert len(read_bytes) == io.DEFAULT_BUFFER_SIZE + 73
+ assert read_bytes.startswith(b"abcdefg")
+ assert wrapper.tell() == 0
+ assert wrapper.read(4) == b"abcd"
+
+ def testMarkedPositionResets(self):
+ wrapper = streaming.CachingStreamWrapper(self.longStream)
+ wrapper.read(10)
+ wrapper.markedPosition = wrapper.tell()
+ assert wrapper.markedPosition == 10
+
+ # Reach the maximum capacity of cache
+ wrapper.read(io.DEFAULT_BUFFER_SIZE)
+ assert wrapper.tell() == 10 + io.DEFAULT_BUFFER_SIZE
+
+ # The following should clear the cache
+ wrapper.markedPosition = wrapper.tell()
+ assert wrapper.markedPosition == 0
+ assert len(wrapper._cache.getvalue()) == 0
+
+
+suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+
+if __name__ == '__main__':
+ unittest.TextTestRunner(verbosity=2).run(suite)