Fixes for python 2 and 3 compatibility

Author: johanlundberg

setup.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python
 from setuptools import setup, find_packages
-import sys, os
+import sys
+import os
 from distutils import versionpredicate
 
 here = os.path.abspath(os.path.dirname(__file__))
 README = open(os.path.join(here, 'README.rst')).read()
 NEWS = open(os.path.join(here, 'NEWS.txt')).read()
 
 
-version = '0.19dev0'
+version = '0.19dev1'
 
 install_requires = [
-    'defusedxml', 'lxml', 'pyconfig', 'requests', 'cryptography'
+    'defusedxml', 'lxml', 'pyconfig', 'requests', 'cryptography', 'six'
 ]
 
 # Let some other project depend on 'xmlsec[PKCS11]'

src/xmlsec/__init__.py

@@ -5,14 +5,15 @@
 
 __author__ = 'leifj'
 
+import six
 from defusedxml import lxml
 from lxml import etree as etree
 import logging
 import copy
 from lxml.builder import ElementMaker
 from xmlsec.exceptions import XMLSigException
 from xmlsec import constants
-from xmlsec.utils import parse_xml, pem2b64, unescape_xml_entities, delete_elt, root_elt, b64d, b64e
+from xmlsec.utils import parse_xml, pem2b64, unescape_xml_entities, delete_elt, root_elt, b64d, b64e, etree_to_string
 import xmlsec.crypto
 import pyconfig
 
@@ -38,7 +39,7 @@ class Config(object):
     default_signature_alg = pyconfig.setting("xmlsec.default_signature_alg", constants.ALGORITHM_SIGNATURE_RSA_SHA256)
     default_digest_alg = pyconfig.setting("xmlsec.default_digest_alg", constants.ALGORITHM_DIGEST_SHA256)
     default_c14n_alg = pyconfig.setting("xmlsec.default_c14n_alg", constants.TRANSFORM_C14N_EXCLUSIVE)
-    debug_write_to_files = pyconfig.setting("xmlsec.config.debug_write_to_files", False)
+    debug_write_to_files = pyconfig.setting("xmlsec.config.debug_write_to_files", True)
     same_document_is_root = pyconfig.setting("xmlsec.same_document_is_root", False)
     id_attributes = pyconfig.setting("xmlsec.id_attributes", ['ID', 'id'])
     c14n_strip_ws = pyconfig.setting("xmlsec.c14n_strip_ws", False)
@@ -81,9 +82,9 @@ def _signed_value(data, key_size, do_pad, hash_alg):  # TODO Do proper asn1 CMS
     if do_pad:
         # Pad to "one octet shorter than the RSA modulus" [RSA-SHA1]
         # WARNING: key size is in bits, not bytes!
-        padded_size = key_size / 8 - 1
+        padded_size = key_size // 8 - 1
         pad_size = padded_size - len(asn_digest) - 2
-        pad = '\x01' + '\xFF' * pad_size + '\x00'
+        pad = b'\x01' + b'\xFF' * pad_size + b'\x00'
         return pad + asn_digest
     else:
         return asn_digest
@@ -151,7 +152,7 @@ def _process_references(t, sig, verify_mode=True, sig_path=".//{%s}Signature" %
 
         if config.debug_write_to_files:
             with open("/tmp/foo-pre-transform.xml", "w") as fd:
-                fd.write(etree.tostring(obj))
+                fd.write(etree_to_string(obj))
 
         for tr in ref.findall(".//{%s}Transform" % NS['ds']):
             obj = _transform(_alg(tr), obj, tr=tr, sig_path=sig_path)
@@ -162,14 +163,16 @@ def _process_references(t, sig, verify_mode=True, sig_path=".//{%s}Signature" %
                     if nsprefix in r.nsmap:
                         obj_copy.nsmap[nsprefix] = r.nsmap[nsprefix]
 
-        if not isinstance(obj, basestring):
+        if not isinstance(obj, six.string_types):
             if config.debug_write_to_files:
                 with open("/tmp/foo-pre-serialize.xml", "w") as fd:
-                    fd.write(etree.tostring(obj))
+                    fd.write(etree_to_string(obj))
             obj = _transform(constants.TRANSFORM_C14N_INCLUSIVE, obj)
 
         if config.debug_write_to_files:
             with open("/tmp/foo-obj.xml", "w") as fd:
+                if six.PY2:
+                    obj = obj.encode('utf-8')
                 fd.write(obj)
 
         hash_alg = _ref_digest(ref)
@@ -215,7 +218,7 @@ def _enveloped_signature(t, sig_path=".//{%s}Signature" % NS['ds']):
         delete_elt(sig)
     if config.debug_write_to_files:
         with open("/tmp/foo-env.xml", "w") as fd:
-            fd.write(etree.tostring(t))
+            fd.write(etree_to_string(t))
     return t
 
 
@@ -231,15 +234,17 @@ def _c14n(t, exclusive, with_comments, inclusive_prefix_list=None, schema=None):
     """
     doc = t
     if root_elt(doc).getparent() is not None:
-        xml_str = etree.tostring(doc, encoding=unicode)
+        xml_str = etree_to_string(doc)
         doc = parse_xml(xml_str, remove_whitespace=config.c14n_strip_ws, remove_comments=not with_comments, schema=schema)
         del xml_str
 
-    buf = etree.tostring(doc,
-                         method='c14n',
-                         exclusive=exclusive,
-                         with_comments=with_comments,
-                         inclusive_ns_prefixes=inclusive_prefix_list)
+    buf = six.text_type(
+        etree.tostring(doc,
+                       method='c14n',
+                       exclusive=exclusive,
+                       with_comments=with_comments,
+                       inclusive_ns_prefixes=inclusive_prefix_list),
+        'utf-8')
     #u = unescape_xml_entities(buf.decode("utf8", 'strict')).encode("utf8").strip()
     assert buf[0] == '<'
     assert buf[-1] == '>'
@@ -292,7 +297,7 @@ def _verify(t, keyspec, sig_path=".//{%s}Signature" % NS['ds'], drop_signature=F
     """
     if config.debug_write_to_files:
         with open("/tmp/foo-sig.xml", "w") as fd:
-            fd.write(etree.tostring(root_elt(t)))
+            fd.write(etree_to_string(t))
 
     validated = []
     for sig in t.findall(sig_path):
@@ -330,7 +335,7 @@ def _verify(t, keyspec, sig_path=".//{%s}Signature" % NS['ds'], drop_signature=F
                 if not this_cert.verify(b64d(sv), actual, sig_digest_alg):
                     raise XMLSigException("Failed to validate {!s} using sig digest {!s} and cm {!s}".format(etree.tostring(sig), sig_digest_alg, cm_alg))
                 validated.append(obj)
-        except XMLSigException, ex:
+        except XMLSigException as ex:
             logging.error(ex)
 
     if not validated:
@@ -433,8 +438,8 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
                                       public.keysize, private.keysize))
             # This might be incorrect for PKCS#11 tokens if we have no public key
             logging.debug("Using {!s} bit key".format(private.keysize))
-
-    templates = filter(_is_template, t.findall(sig_path))
+    sig_paths = t.findall(sig_path)
+    templates = list(filter(_is_template, sig_paths))
     if not templates:
         tmpl = add_enveloped_signature(t, reference_uri=reference_uri, pos=insert_index)
         templates = [tmpl]
@@ -443,7 +448,7 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
 
     if config.debug_write_to_files:
         with open("/tmp/sig-ref.xml", "w") as fd:
-            fd.write(etree.tostring(root_elt(t)))
+            fd.write(etree_to_string(root_elt(t)))
 
     for sig in templates:
         logging.debug("processing sig template: %s" % etree.tostring(sig))
@@ -469,6 +474,8 @@ def sign(t, key_spec, cert_spec=None, reference_uri='', insert_index=0, sig_path
 
         signed = private.sign(tbs, sig_alg)
         signature = b64e(signed)
+        if isinstance(signature, six.binary_type):
+            signature = six.text_type(signature, 'utf-8')
         logging.debug("SignatureValue: %s" % signature)
         sv = sig.find(".//{%s}SignatureValue" % NS['ds'])
         if sv is None:

src/xmlsec/constants.py

@@ -21,15 +21,14 @@
 ALGORITHM_SIGNATURE_RSA_SHA384 = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha384"
 ALGORITHM_SIGNATURE_RSA_SHA512 = "http://www.w3.org/2001/04/xmldsig-more#rsa-sha512"
 
-
 # ASN.1 BER SHA1 algorithm designator prefixes (RFC3447)
 ASN1_BER_ALG_DESIGNATOR_PREFIX = {
     # disabled 'md2': '\x30\x20\x30\x0c\x06\x08\x2a\x86\x48\x86\xf7\x0d\x02\x02\x05\x00\x04\x10',
     # disabled 'md5': '\x30\x20\x30\x0c\x06\x08\x2a\x86\x48\x86\xf7\x0d\x02\x05\x05\x00\x04\x10',
-    _SHA1_INTERNAL: '\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14',
-    _SHA256_INTERNAL: '\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20',
-    _SHA384_INTERNAL: '\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x02\x05\x00\x04\x30',
-    _SHA512_INTERNAL: '\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x03\x05\x00\x04\x40',
+    _SHA1_INTERNAL: b'\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14',
+    _SHA256_INTERNAL: b'\x30\x31\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20',
+    _SHA384_INTERNAL: b'\x30\x41\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x02\x05\x00\x04\x30',
+    _SHA512_INTERNAL: b'\x30\x51\x30\x0d\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x03\x05\x00\x04\x40',
 }
 
 sign_alg_xmldsig_digest_to_internal_d = {

src/xmlsec/crypto.py

@@ -3,15 +3,20 @@
 import base64
 import logging
 import threading
+import six
 from binascii import hexlify
-from UserDict import DictMixin
 from xmlsec.exceptions import XMLSigException
+from xmlsec.utils import unicode_to_bytes
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization, hashes
 from cryptography.hazmat.primitives.asymmetric import rsa, padding, utils
 from cryptography.x509 import load_pem_x509_certificate, load_der_x509_certificate, Certificate
 
+if six.PY2:
+    from UserDict import DictMixin
+else:
+    from collections import MutableMapping as DictMixin
 
 NS = {'ds': 'http://www.w3.org/2000/09/xmldsig#'}
 
@@ -85,6 +90,8 @@ def __init__(self, source, do_padding, private, do_digest=True):
 
     def sign(self, data, hash_alg, pad_alg="PKCS1v15"):
         if self.is_private:
+            if not isinstance(data, six.binary_type):
+                data = unicode_to_bytes(data)
             hasher = getattr(hashes, hash_alg)
             padder = getattr(padding, pad_alg)
             return self.key.sign(data, padder(), hasher())
@@ -93,6 +100,8 @@ def sign(self, data, hash_alg, pad_alg="PKCS1v15"):
 
     def verify(self, signature, msg, hash_alg, pad_alg="PKCS1v15"):
         if not self.is_private:
+            if not isinstance(msg, six.binary_type):
+                msg = unicode_to_bytes(msg)
             try:
                 hasher = getattr(hashes, hash_alg)
                 padder = getattr(padding, pad_alg)
@@ -221,7 +230,7 @@ def sign(self, data, hash_alg=None):
             if not 'signed' in msg:
                 raise ValueError("Missing signed data in response message")
             return msg['signed'].decode('base64')
-        except Exception, ex:
+        except Exception as ex:
             from traceback import print_exc
             print_exc(ex)
             raise XMLSigException(ex)
@@ -230,7 +239,7 @@ def sign(self, data, hash_alg=None):
 def _load_keyspec(keyspec, private=False, signature_element=None):
     if private and hasattr(keyspec, '__call__'):
         return XMLSecCryptoCallable(keyspec)
-    if isinstance(keyspec, basestring):
+    if isinstance(keyspec, six.string_types):
         if os.path.isfile(keyspec):
             return XMLSecCryptoFile(keyspec, private)
         elif private and keyspec.startswith("pkcs11://"):
@@ -274,6 +283,13 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         del self.certs[key]
 
+    def __len__(self):
+        return len(self.certs)
+
+    def __iter__(self):
+        for item in self.certs:
+            yield item
+
     def _get_cert_by_fp(self, fp):
         """
         Get the cryptography.x509.Certificate representation.
@@ -320,6 +336,7 @@ def _find_cert_by_fingerprint(t, fp):
 
     return cert.public_bytes(encoding=serialization.Encoding.PEM)
 
+
 def _digest(data, hash_alg):
     """
     Calculate a hash digest of algorithm hash_alg and return the result base64 encoded.
@@ -330,5 +347,7 @@ def _digest(data, hash_alg):
     """
     h = getattr(hashes, hash_alg)
     d = hashes.Hash(h(), backend=default_backend())
+    if not isinstance(data, six.binary_type):
+        data = unicode_to_bytes(data)
     d.update(data)
     return base64.b64encode(d.finalize())

src/xmlsec/pk11.py

@@ -4,7 +4,7 @@
 __author__ = 'leifj'
 
 from xmlsec.exceptions import XMLSigException
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 import os
 import logging
 from xmlsec.utils import b642pem
@@ -17,7 +17,7 @@
 except ImportError:
     raise XMLSigException("pykcs11 is required for PKCS#11 keys - cf README.rst")
 
-all_attributes = PyKCS11.CKA.keys()
+all_attributes = list(PyKCS11.CKA.keys())
 
 # remove the CKR_ATTRIBUTE_SENSITIVE attributes since we can't get
 all_attributes.remove(PyKCS11.LowLevel.CKA_PRIVATE_EXPONENT)
@@ -81,7 +81,7 @@ def parse_uri(pk11_uri):
 
 
 def _intarray2bytes(x):
-    return ''.join(chr(i) for i in x)
+    return bytearray(x)
 
 
 def _close_session(session):

src/xmlsec/test/case.py

@@ -7,8 +7,8 @@
 import os
 import pkg_resources
 from defusedxml import lxml
-import lxml.etree as etree
-from StringIO import StringIO
+from lxml import etree
+from six.moves import StringIO
 import xmlsec
 
 

src/xmlsec/test/p11_test.py

@@ -21,6 +21,7 @@
 from xmlsec.test import run_cmd
 
 from xmlsec.test.case import load_test_data
+from xmlsec.exceptions import XMLSigException
 
 try:
     from PyKCS11 import PyKCS11Error
@@ -29,8 +30,8 @@
     raise unittest.SkipTest("PyKCS11 not installed")
 
 try:
-    import xmlsec.pk11 as pk11
-except Exception:
+    from xmlsec import pk11
+except (ImportError, XMLSigException):
     raise unittest.SkipTest("PyKCS11 not installed")
 
 
@@ -211,9 +212,9 @@ def setup():
                  '--pin', 'secret1'], softhsm_conf=softhsm_conf)
 
     except Exception as ex:
-        print "-" * 64
+        print("-" * 64)
         traceback.print_exc()
-        print "-" * 64
+        print("-" * 64)
         logging.warning("PKCS11 tests disabled: unable to initialize test token: %s" % ex)
 
 
@@ -253,7 +254,7 @@ def test_open_session(self):
             os.environ['SOFTHSM2_CONF'] = softhsm_conf
             session = pk11._session(component_path['P11_MODULE'], pk11_uri="pkcs11://%s/test?pin=secret1" % component_path['P11_MODULE'])
             assert session is not None
-        except Exception, ex:
+        except Exception as ex:
             traceback.print_exc()
             raise ex
         finally:
@@ -268,7 +269,7 @@ def test_open_session_no_pin(self):
             os.environ['SOFTHSM2_CONF'] = softhsm_conf
             session = pk11._session(component_path['P11_MODULE'], pk11_uri="pkcs11://%s/test" % component_path['P11_MODULE'])
             assert session is not None
-        except Exception, ex:
+        except Exception as ex:
             traceback.print_exc()
             raise ex
         finally:
@@ -288,7 +289,7 @@ def test_two_sessions(self):
             assert session1 != session2
             assert session1 is not None
             assert session2 is not None
-        except Exception, ex:
+        except Exception as ex:
             raise ex
         finally:
             if session1 is not None:
@@ -303,7 +304,7 @@ def test_bad_login(self):
         try:
             session = pk11._session(component_path['P11_MODULE'], pk11_uri="pkcs11://%s/test?pin=wrong" % component_path['P11_MODULE'])
             assert False, "We should have failed the last login"
-        except PyKCS11Error, ex:
+        except PyKCS11Error as ex:
             assert ex.value == CKR_PIN_INCORRECT
             pass
 
@@ -317,8 +318,8 @@ def test_find_key(self):
             key, cert = pk11._find_key(session, "test")
             assert key is not None
             assert cert is not None
-            assert cert.strip() == open(signer_cert_pem).read().strip()
-        except Exception, ex:
+            assert cert.strip() == open(signer_cert_pem).read().strip().encode('utf-8')
+        except Exception as ex:
             raise ex
         finally:
             if session is not None: