Merge remote-tracking branch 'origin/master'

Author: Leif Johansson

.travis.yml

@@ -1,7 +1,11 @@
 sudo: false
+dist: xenial
 language: python
 python:
   - 2.7
+  - 3.5
+  - 3.6
+  - 3.7
 
 addons:
   apt:

README.rst

@@ -1,8 +1,8 @@
 python XML Security
 ===================
 
-.. image:: https://img.shields.io/travis/leifj/pyXMLSecurity.svg
-   :target: https://travis-ci.org/leifj/pyXMLSecurity
+.. image:: https://img.shields.io/travis/IdentityPython/pyXMLSecurity.svg
+   :target: https://travis-ci.org/IdentityPython/pyXMLSecurity
    :alt: Travis Build
 .. image:: https://img.shields.io/coveralls/leifj/pyXMLSecurity.svg
    :target: https://coveralls.io/r/leifj/pyXMLSecurity?branch=master

setup.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python
 from setuptools import setup, find_packages
-import sys, os
+import sys
+import os
 from distutils import versionpredicate
 
 here = os.path.abspath(os.path.dirname(__file__))
 README = open(os.path.join(here, 'README.rst')).read()
 NEWS = open(os.path.join(here, 'NEWS.txt')).read()
 
 
-version = '0.19dev0'
+version = '0.19dev1'
 
 install_requires = [
-    'idna==2.5', 'defusedxml', 'lxml', 'pyconfig', 'requests', 'cryptography'
+    'idna==2.5', 'defusedxml', 'lxml', 'pyconfig', 'requests', 'cryptography', 'six'
 ]
 
 # Let some other project depend on 'xmlsec[PKCS11]'

src/xmlsec/__init__.py

@@ -5,14 +5,15 @@
 
 __author__ = 'leifj'
 
+import six
 from defusedxml import lxml
 from lxml import etree as etree
 import logging
 import copy
 from lxml.builder import ElementMaker
 from xmlsec.exceptions import XMLSigException
 from xmlsec import constants
-from xmlsec.utils import parse_xml, pem2b64, unescape_xml_entities, delete_elt, root_elt, b64d, b64e
+from xmlsec.utils import parse_xml, pem2b64, unescape_xml_entities, delete_elt, root_elt, b64d, b64e, etree_to_string
 import xmlsec.crypto
 import pyconfig
 
@@ -83,9 +84,9 @@ def _signed_value(data, key_size, do_pad, hash_alg):  # TODO Do proper asn1 CMS
     if do_pad:
         # Pad to "one octet shorter than the RSA modulus" [RSA-SHA1]
         # WARNING: key size is in bits, not bytes!
-        padded_size = key_size / 8 - 1
+        padded_size = key_size // 8 - 1
         pad_size = padded_size - len(asn_digest) - 2
-        pad = '\x01' + '\xFF' * pad_size + '\x00'
+        pad = b'\x01' + b'\xFF' * pad_size + b'\x00'
         return pad + asn_digest
     else:
         return asn_digest
@@ -153,7 +154,7 @@ def _process_references(t, sig, verify_mode=True, sig_path=".//{%s}Signature" %
 
         if config.debug_write_to_files:
             with open("/tmp/foo-pre-transform.xml", "w") as fd:
-                fd.write(etree.tostring(obj))
+                fd.write(etree_to_string(obj))
 
         for tr in ref.findall(".//{%s}Transform" % NS['ds']):
             obj = _transform(_alg(tr), obj, tr=tr, sig_path=sig_path)
@@ -164,14 +165,16 @@ def _process_references(t, sig, verify_mode=True, sig_path=".//{%s}Signature" %
                     if nsprefix in r.nsmap:
                         obj_copy.nsmap[nsprefix] = r.nsmap[nsprefix]
 
-        if not isinstance(obj, basestring):
+        if not isinstance(obj, six.string_types):
             if config.debug_write_to_files:
                 with open("/tmp/foo-pre-serialize.xml", "w") as fd:
-                    fd.write(etree.tostring(obj))
+                    fd.write(etree_to_string(obj))
             obj = _transform(constants.TRANSFORM_C14N_INCLUSIVE, obj)
 
         if config.debug_write_to_files:
             with open("/tmp/foo-obj.xml", "w") as fd:
+                if six.PY2:
+                    obj = obj.encode('utf-8')
                 fd.write(obj)
 
         hash_alg = _ref_digest(ref)
@@ -217,7 +220,7 @@ def _enveloped_signature(t, sig_path=".//{%s}Signature" % NS['ds']):
         delete_elt(sig)
     if config.debug_write_to_files:
         with open("/tmp/foo-env.xml", "w") as fd:
-            fd.write(etree.tostring(t))
+            fd.write(etree_to_string(t))
     return t
 
 
@@ -233,15 +236,17 @@ def _c14n(t, exclusive, with_comments, inclusive_prefix_list=None, schema=None):
     """
     doc = t
     if root_elt(doc).getparent() is not None:
-        xml_str = etree.tostring(doc, encoding=unicode)
+        xml_str = etree_to_string(doc)
         doc = parse_xml(xml_str, remove_whitespace=config.c14n_strip_ws, remove_comments=not with_comments, schema=schema)
         del xml_str
 
-    buf = etree.tostring(doc,
-                         method='c14n',
-                         exclusive=exclusive,
-                         with_comments=with_comments,
-                         inclusive_ns_prefixes=inclusive_prefix_list)
+    buf = six.text_type(
+        etree.tostring(doc,
+                       method='c14n',
+                       exclusive=exclusive,
+                       with_comments=with_comments,
+                       inclusive_ns_prefixes=inclusive_prefix_list),
+        'utf-8')
     #u = unescape_xml_entities(buf.decode("utf8", 'strict')).encode("utf8").strip()
     assert buf[0] == '<'
     assert buf[-1] == '>'
@@ -294,7 +299,7 @@ def _verify(t, keyspec, sig_path=".//{%s}Signature" % NS['ds'], drop_signature=F
     """
     if config.debug_write_to_files:
         with open("/tmp/foo-sig.xml", "w") as fd:
-            fd.write(etree.tostring(root_elt(t)))
+            fd.write(etree_to_string(t))
 
     validated = []
     for sig in t.findall(sig_path):
@@ -332,7 +337,7 @@ def _verify(t, keyspec, sig_path=".//{%s}Signature" % NS['ds'], drop_signature=F
                 if not this_cert.verify(b64d(sv), actual, sig_digest_alg):
                     raise XMLSigException("Failed to validate {!s} using sig digest {!s} and cm {!s}".format(etree.tostring(sig), sig_digest_alg, cm_alg))
                 validated.append(obj)
-        except XMLSigException, ex:
+        except XMLSigException as ex:
             log.error(ex)
 
     if not validated:
@@ -435,8 +440,8 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
                                       public.keysize, private.keysize))
             # This might be incorrect for PKCS#11 tokens if we have no public key
             log.debug("Using {!s} bit key".format(private.keysize))
-
-    templates = filter(_is_template, t.findall(sig_path))
+    sig_paths = t.findall(sig_path)
+    templates = list(filter(_is_template, sig_paths))
     if not templates:
         tmpl = add_enveloped_signature(t, reference_uri=reference_uri, pos=insert_index)
         templates = [tmpl]
@@ -445,7 +450,7 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
 
     if config.debug_write_to_files:
         with open("/tmp/sig-ref.xml", "w") as fd:
-            fd.write(etree.tostring(root_elt(t)))
+            fd.write(etree_to_string(root_elt(t)))
 
     for sig in templates:
         log.debug("processing sig template: %s" % etree.tostring(sig))
@@ -471,6 +476,8 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
 
         signed = private.sign(tbs, sig_alg)
         signature = b64e(signed)
+        if isinstance(signature, six.binary_type):
+            signature = six.text_type(signature, 'utf-8')
         log.debug("SignatureValue: %s" % signature)
         sv = sig.find(".//{%s}SignatureValue" % NS['ds'])
         if sv is None:

src/xmlsec/constants.py

@@ -21,15 +21,14 @@
 ALGORITHM_SIGNATURE_RSA_SHA384 = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha384"
 ALGORITHM_SIGNATURE_RSA_SHA512 = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha512"
 
-
 # ASN.1 BER SHA1 algorithm designator prefixes (RFC3447)
 ASN1_BER_ALG_DESIGNATOR_PREFIX = {
     # disabled 'md2': '\x30\x20\x30\x0c\x06\x08\x2a\x86\x48\x86\xf7\x0d\x02\x02\x05\x00\x04\x10',
     # disabled 'md5': '\x30\x20\x30\x0c\x06\x08\x2a\x86\x48\x86\xf7\x0d\x02\x05\x05\x00\x04\x10',
-    _SHA1_INTERNAL: '\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14',
-    _SHA256_INTERNAL: '\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20',
-    _SHA384_INTERNAL: '\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x02\x05\x00\x04\x30',
-    _SHA512_INTERNAL: '\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x03\x05\x00\x04\x40',
+    _SHA1_INTERNAL: b'\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14',
+    _SHA256_INTERNAL: b'\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20',
+    _SHA384_INTERNAL: b'\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x02\x05\x00\x04\x30',
+    _SHA512_INTERNAL: b'\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x03\x05\x00\x04\x40',
 }
 
 sign_alg_xmldsig_digest_to_internal_d = {

src/xmlsec/crypto.py

@@ -3,15 +3,20 @@
 import base64
 import logging
 import threading
+import six
 from binascii import hexlify
-from UserDict import DictMixin
 from xmlsec.exceptions import XMLSigException
+from xmlsec.utils import unicode_to_bytes
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization, hashes
 from cryptography.hazmat.primitives.asymmetric import rsa, padding, utils
 from cryptography.x509 import load_pem_x509_certificate, load_der_x509_certificate, Certificate
 
+if six.PY2:
+    from UserDict import DictMixin
+else:
+    from collections import MutableMapping as DictMixin
 
 NS = {'ds': 'http://www.w3.org/2000/09/xmldsig#'}
 
@@ -85,6 +90,8 @@ def __init__(self, source, do_padding, private, do_digest=True):
 
     def sign(self, data, hash_alg, pad_alg="PKCS1v15"):
         if self.is_private:
+            if not isinstance(data, six.binary_type):
+                data = unicode_to_bytes(data)
             hasher = getattr(hashes, hash_alg)
             padder = getattr(padding, pad_alg)
             return self.key.sign(data, padder(), hasher())
@@ -93,6 +100,8 @@ def sign(self, data, hash_alg, pad_alg="PKCS1v15"):
 
     def verify(self, signature, msg, hash_alg, pad_alg="PKCS1v15"):
         if not self.is_private:
+            if not isinstance(msg, six.binary_type):
+                msg = unicode_to_bytes(msg)
             try:
                 hasher = getattr(hashes, hash_alg)
                 padder = getattr(padding, pad_alg)
@@ -221,7 +230,7 @@ def sign(self, data, hash_alg=None):
             if not 'signed' in msg:
                 raise ValueError("Missing signed data in response message")
             return msg['signed'].decode('base64')
-        except Exception, ex:
+        except Exception as ex:
             from traceback import print_exc
             print_exc(ex)
             raise XMLSigException(ex)
@@ -230,7 +239,7 @@ def sign(self, data, hash_alg=None):
 def _load_keyspec(keyspec, private=False, signature_element=None):
     if private and hasattr(keyspec, '__call__'):
         return XMLSecCryptoCallable(keyspec)
-    if isinstance(keyspec, basestring):
+    if isinstance(keyspec, six.string_types):
         if os.path.isfile(keyspec):
             return XMLSecCryptoFile(keyspec, private)
         elif private and keyspec.startswith("pkcs11://"):
@@ -274,6 +283,13 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         del self.certs[key]
 
+    def __len__(self):
+        return len(self.certs)
+
+    def __iter__(self):
+        for item in self.certs:
+            yield item
+
     def _get_cert_by_fp(self, fp):
         """
         Get the cryptography.x509.Certificate representation.
@@ -320,6 +336,7 @@ def _find_cert_by_fingerprint(t, fp):
 
     return cert.public_bytes(encoding=serialization.Encoding.PEM)
 
+
 def _digest(data, hash_alg):
     """
     Calculate a hash digest of algorithm hash_alg and return the result base64 encoded.
@@ -330,5 +347,7 @@ def _digest(data, hash_alg):
     """
     h = getattr(hashes, hash_alg)
     d = hashes.Hash(h(), backend=default_backend())
+    if not isinstance(data, six.binary_type):
+        data = unicode_to_bytes(data)
     d.update(data)
     return base64.b64encode(d.finalize())

src/xmlsec/pk11.py

@@ -4,7 +4,7 @@
 __author__ = 'leifj'
 
 from xmlsec.exceptions import XMLSigException
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 import os
 import logging
 from xmlsec.utils import b642pem
@@ -17,7 +17,7 @@
 except ImportError:
     raise XMLSigException("pykcs11 is required for PKCS#11 keys - cf README.rst")
 
-all_attributes = PyKCS11.CKA.keys()
+all_attributes = list(PyKCS11.CKA.keys())
 
 # remove the CKR_ATTRIBUTE_SENSITIVE attributes since we can't get
 all_attributes.remove(PyKCS11.LowLevel.CKA_PRIVATE_EXPONENT)
@@ -81,7 +81,7 @@ def parse_uri(pk11_uri):
 
 
 def _intarray2bytes(x):
-    return ''.join(chr(i) for i in x)
+    return bytearray(x)
 
 
 def _close_session(session):

src/xmlsec/test/case.py

@@ -7,8 +7,8 @@
 import os
 import pkg_resources
 from defusedxml import lxml
-import lxml.etree as etree
-from StringIO import StringIO
+from lxml import etree
+from six.moves import StringIO
 import xmlsec