Port python bindings to Python 3

python/google/protobuf/internal/cpp_message.py

@@ -39,7 +39,21 @@
 from google.protobuf.internal import _net_proto2___python
 from google.protobuf.internal import enum_type_wrapper
 from google.protobuf import message
-
+from google.protobuf.internal.utils import bytestr_to_string, iteritems, PY2
+
+if PY2:
+  import copy_reg
+  def is_sequence(other):
+    return operator.isSequenceType(other)
+  def copy_reg_pickle(type, function):
+    return copy_reg.pickle(type,function)
+else:
+  import collections
+  import copyreg
+  def is_sequence(other):
+    return isinstance(other, collections.Sequence)
+  def copy_reg_pickle(type, function):
+    return copyreg.pickle(type,function)
 
 _LABEL_REPEATED = _net_proto2___python.LABEL_REPEATED
 _LABEL_OPTIONAL = _net_proto2___python.LABEL_OPTIONAL
@@ -146,7 +160,7 @@ def __len__(self):
   def __eq__(self, other):
     if self is other:
       return True
-    if not operator.isSequenceType(other):
+    if not is_sequence(other):
       raise TypeError(
           'Can only compare repeated scalar fields against sequences.')
     # We are presumably comparing against some other sequence type.
@@ -385,7 +399,7 @@ def InitMessage(message_descriptor, cls):
   _AddInitMethod(message_descriptor, cls)
   _AddMessageMethods(message_descriptor, cls)
   _AddPropertiesForExtensions(message_descriptor, cls)
-  copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
+  copy_reg_pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
 
 
 def _AddDescriptors(message_descriptor, dictionary):
@@ -610,7 +624,10 @@ def FindInitializationErrors(self):
     return self._cmsg.FindInitializationErrors()
 
   def __str__(self):
-    return str(self._cmsg)
+    if PY2:
+      return str(self._cmsg)
+    else:
+      return bytestr_to_string(self._cmsg)
 
   def __eq__(self, other):
     if self is other:
@@ -628,10 +645,10 @@ def __hash__(self):
   def __unicode__(self):
     # Lazy import to prevent circular import when text_format imports this file.
     from google.protobuf import text_format
-    return text_format.MessageToString(self, as_utf8=True).decode('utf-8')
+    return bytestr_to_string(text_format.MessageToString(self, as_utf8=True))
 
   # Attach the local methods to the message class.
-  for key, value in locals().copy().iteritems():
+  for key, value in iteritems(locals().copy()):
     if key not in ('key', 'value', '__builtins__', '__name__', '__doc__'):
       setattr(cls, key, value)
 
@@ -658,6 +675,6 @@ def FromString(string):
 def _AddPropertiesForExtensions(message_descriptor, cls):
   """Adds properties for all fields in this protocol message type."""
   extension_dict = message_descriptor.extensions_by_name
-  for extension_name, extension_field in extension_dict.iteritems():
+  for extension_name, extension_field in iteritems(extension_dict):
     constant_name = extension_name.upper() + '_FIELD_NUMBER'
     setattr(cls, constant_name, extension_field.number)

python/google/protobuf/internal/decoder.py

@@ -85,18 +85,17 @@
 __author__ = 'kenton@google.com (Kenton Varda)'
 
 import struct
-import sys  ##PY25
-_PY2 = sys.version_info[0] < 3  ##PY25
 from google.protobuf.internal import encoder
 from google.protobuf.internal import wire_format
+from google.protobuf.internal.utils import bytestr_to_string, b
 from google.protobuf import message
 
 
 # This will overflow and thus become IEEE-754 "infinity".  We would use
 # "float('inf')" but it doesn't work on Windows pre-Python-2.6.
-_POS_INF = 1e10000
-_NEG_INF = -_POS_INF
-_NAN = _POS_INF * 0
+_POS_INF = float('inf')
+_NEG_INF = float('-inf')
+_NAN = float('nan')
 
 
 # This is not for optimization, but rather to avoid conflicts with local
@@ -114,14 +113,11 @@ def _VarintDecoder(mask, result_type):
   decoder returns a (value, new_pos) pair.
   """
 
-  local_ord = ord
-  py2 = _PY2  ##PY25
-##!PY25  py2 = str is bytes
   def DecodeVarint(buffer, pos):
     result = 0
     shift = 0
     while 1:
-      b = local_ord(buffer[pos]) if py2 else buffer[pos]
+      b = ord(buffer[pos:pos+1])
       result |= ((b & 0x7f) << shift)
       pos += 1
       if not (b & 0x80):
@@ -137,14 +133,11 @@ def DecodeVarint(buffer, pos):
 def _SignedVarintDecoder(mask, result_type):
   """Like _VarintDecoder() but decodes signed values."""
 
-  local_ord = ord
-  py2 = _PY2  ##PY25
-##!PY25  py2 = str is bytes
   def DecodeVarint(buffer, pos):
     result = 0
     shift = 0
     while 1:
-      b = local_ord(buffer[pos]) if py2 else buffer[pos]
+      b = ord(buffer[pos:pos+1])
       result |= ((b & 0x7f) << shift)
       pos += 1
       if not (b & 0x80):
@@ -183,10 +176,8 @@ def ReadTag(buffer, pos):
   use that, but not in Python.
   """
 
-  py2 = _PY2  ##PY25
-##!PY25  py2 = str is bytes
   start = pos
-  while (ord(buffer[pos]) if py2 else buffer[pos]) & 0x80:
+  while ord(buffer[pos:pos+1]) & 0x80:
     pos += 1
   pos += 1
   return (buffer[start:pos], pos)
@@ -301,7 +292,6 @@ def _FloatDecoder():
   """
 
   local_unpack = struct.unpack
-  b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1')  ##PY25
 
   def InnerDecode(buffer, pos):
     # We expect a 32-bit value in little-endian byte order.  Bit 1 is the sign
@@ -312,17 +302,13 @@ def InnerDecode(buffer, pos):
     # If this value has all its exponent bits set, then it's non-finite.
     # In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
     # To avoid that, we parse it specially.
-    if ((float_bytes[3:4] in b('\x7F\xFF'))  ##PY25
-##!PY25    if ((float_bytes[3:4] in b'\x7F\xFF')
-        and (float_bytes[2:3] >= b('\x80'))):  ##PY25
-##!PY25        and (float_bytes[2:3] >= b'\x80')):
+    if ((float_bytes[3] in b('\x7F\xFF'))
+        and (float_bytes[2] >= 128)):
       # If at least one significand bit is set...
-      if float_bytes[0:3] != b('\x00\x00\x80'):  ##PY25
-##!PY25      if float_bytes[0:3] != b'\x00\x00\x80':
+      if float_bytes[0:3] != b('\x00\x00\x80'):
         return (_NAN, new_pos)
       # If sign bit is set...
-      if float_bytes[3:4] == b('\xFF'):  ##PY25
-##!PY25      if float_bytes[3:4] == b'\xFF':
+      if float_bytes[3] == b('\xFF'):
         return (_NEG_INF, new_pos)
       return (_POS_INF, new_pos)
 
@@ -341,7 +327,6 @@ def _DoubleDecoder():
   """
 
   local_unpack = struct.unpack
-  b = (lambda x:x) if _PY2 else lambda x:x.encode('latin1')  ##PY25
 
   def InnerDecode(buffer, pos):
     # We expect a 64-bit value in little-endian byte order.  Bit 1 is the sign
@@ -352,12 +337,9 @@ def InnerDecode(buffer, pos):
     # If this value has all its exponent bits set and at least one significand
     # bit set, it's not a number.  In Python 2.4, struct.unpack will treat it
     # as inf or -inf.  To avoid that, we treat it specially.
-##!PY25    if ((double_bytes[7:8] in b'\x7F\xFF')
-##!PY25        and (double_bytes[6:7] >= b'\xF0')
-##!PY25        and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')):
-    if ((double_bytes[7:8] in b('\x7F\xFF'))  ##PY25
-        and (double_bytes[6:7] >= b('\xF0'))  ##PY25
-        and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))):  ##PY25
+    if ((double_bytes[7] in b('\x7F\xFF'))
+        and (double_bytes[6] >= 240)
+        and (double_bytes[0:7] != b('\x00\x00\x00\x00\x00\x00\xF0'))):
       return (_NAN, new_pos)
 
     # Note that we expect someone up-stack to catch struct.error and convert
@@ -480,15 +462,6 @@ def StringDecoder(field_number, is_repeated, is_packed, key, new_default):
   """Returns a decoder for a string field."""
 
   local_DecodeVarint = _DecodeVarint
-  local_unicode = unicode
-
-  def _ConvertToUnicode(byte_str):
-    try:
-      return local_unicode(byte_str, 'utf-8')
-    except UnicodeDecodeError, e:
-      # add more information to the error message and re-raise it.
-      e.reason = '%s in field: %s' % (e, key.full_name)
-      raise
 
   assert not is_packed
   if is_repeated:
@@ -504,7 +477,7 @@ def DecodeRepeatedField(buffer, pos, end, message, field_dict):
         new_pos = pos + size
         if new_pos > end:
           raise _DecodeError('Truncated string.')
-        value.append(_ConvertToUnicode(buffer[pos:new_pos]))
+        value.append(bytestr_to_string(buffer[pos:new_pos]))
         # Predict that the next tag is another copy of the same repeated field.
         pos = new_pos + tag_len
         if buffer[new_pos:pos] != tag_bytes or new_pos == end:
@@ -517,7 +490,7 @@ def DecodeField(buffer, pos, end, message, field_dict):
       new_pos = pos + size
       if new_pos > end:
         raise _DecodeError('Truncated string.')
-      field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
+      field_dict[key] = bytestr_to_string(buffer[pos:new_pos])
       return new_pos
     return DecodeField
 

python/google/protobuf/internal/encoder.py

@@ -71,15 +71,13 @@
 __author__ = 'kenton@google.com (Kenton Varda)'
 
 import struct
-import sys  ##PY25
-_PY2 = sys.version_info[0] < 3  ##PY25
 from google.protobuf.internal import wire_format
+from google.protobuf.internal.utils import string_to_bytestr, bytestr, b
 
 
-# This will overflow and thus become IEEE-754 "infinity".  We would use
-# "float('inf')" but it doesn't work on Windows pre-Python-2.6.
-_POS_INF = 1e10000
-_NEG_INF = -_POS_INF
+# These constants won't work on Windows pre-Python-2.6.
+_POS_INF = float('inf')
+_NEG_INF = float('-inf')
 
 
 def _VarintSize(value):
@@ -346,16 +344,14 @@ def FieldSize(value):
 def _VarintEncoder():
   """Return an encoder for a basic varint value (does not include tag)."""
 
-  local_chr = _PY2 and chr or (lambda x: bytes((x,)))  ##PY25
-##!PY25  local_chr = chr if bytes is str else lambda x: bytes((x,))
   def EncodeVarint(write, value):
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(local_chr(0x80|bits))
+      write(bytestr(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(local_chr(bits))
+    return write(bytestr(bits))
 
   return EncodeVarint
 
@@ -364,18 +360,16 @@ def _SignedVarintEncoder():
   """Return an encoder for a basic signed varint value (does not include
   tag)."""
 
-  local_chr = _PY2 and chr or (lambda x: bytes((x,)))  ##PY25
-##!PY25  local_chr = chr if bytes is str else lambda x: bytes((x,))
   def EncodeSignedVarint(write, value):
     if value < 0:
       value += (1 << 64)
     bits = value & 0x7f
     value >>= 7
     while value:
-      write(local_chr(0x80|bits))
+      write(bytestr(0x80|bits))
       bits = value & 0x7f
       value >>= 7
-    return write(local_chr(bits))
+    return write(bytestr(bits))
 
   return EncodeSignedVarint
 
@@ -390,8 +384,7 @@ def _VarintBytes(value):
 
   pieces = []
   _EncodeVarint(pieces.append, value)
-  return "".encode("latin1").join(pieces)  ##PY25
-##!PY25  return b"".join(pieces)
+  return b("").join(pieces)
 
 
 def TagBytes(field_number, wire_type):
@@ -529,35 +522,28 @@ def _FloatingPointEncoder(wire_type, format):
       format:  The format string to pass to struct.pack().
   """
 
-  b = _PY2 and (lambda x:x) or (lambda x:x.encode('latin1'))  ##PY25
   value_size = struct.calcsize(format)
   if value_size == 4:
     def EncodeNonFiniteOrRaise(write, value):
       # Remember that the serialized form uses little-endian byte order.
       if value == _POS_INF:
-        write(b('\x00\x00\x80\x7F'))  ##PY25
-##!PY25        write(b'\x00\x00\x80\x7F')
+        write(b('\x00\x00\x80\x7F'))
       elif value == _NEG_INF:
-        write(b('\x00\x00\x80\xFF'))  ##PY25
-##!PY25        write(b'\x00\x00\x80\xFF')
+        write(b('\x00\x00\x80\xFF'))
       elif value != value:           # NaN
-        write(b('\x00\x00\xC0\x7F'))  ##PY25
-##!PY25        write(b'\x00\x00\xC0\x7F')
+        write(b('\x00\x00\xC0\x7F'))
       else:
-        raise
+        raise BaseException()
   elif value_size == 8:
     def EncodeNonFiniteOrRaise(write, value):
       if value == _POS_INF:
-        write(b('\x00\x00\x00\x00\x00\x00\xF0\x7F'))  ##PY25
-##!PY25        write(b'\x00\x00\x00\x00\x00\x00\xF0\x7F')
+        write(b('\x00\x00\x00\x00\x00\x00\xF0\x7F'))
       elif value == _NEG_INF:
-        write(b('\x00\x00\x00\x00\x00\x00\xF0\xFF'))  ##PY25
-##!PY25        write(b'\x00\x00\x00\x00\x00\x00\xF0\xFF')
+        write(b('\x00\x00\x00\x00\x00\x00\xF0\xFF'))
       elif value != value:                         # NaN
-        write(b('\x00\x00\x00\x00\x00\x00\xF8\x7F'))  ##PY25
-##!PY25        write(b'\x00\x00\x00\x00\x00\x00\xF8\x7F')
+        write(b('\x00\x00\x00\x00\x00\x00\xF8\x7F'))
       else:
-        raise
+        raise BaseException()
   else:
     raise ValueError('Can\'t encode floating-point values that are '
                      '%d bytes long (only 4 or 8)' % value_size)
@@ -620,21 +606,19 @@ def EncodeField(write, value):
 # formats, they will also have the same size across all platforms (as opposed
 # to without the prefix, where their sizes depend on the C compiler's basic
 # type sizes).
-Fixed32Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I')
-Fixed64Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q')
-SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i')
-SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q')
-FloatEncoder    = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f')
-DoubleEncoder   = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d')
+Fixed32Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, b('<I'))
+Fixed64Encoder  = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, b('<Q'))
+SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, b('<i'))
+SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, b('<q'))
+FloatEncoder    = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, b('<f'))
+DoubleEncoder   = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, b('<d'))
 
 
 def BoolEncoder(field_number, is_repeated, is_packed):
   """Returns an encoder for a boolean field."""
 
-##!PY25  false_byte = b'\x00'
-##!PY25  true_byte = b'\x01'
-  false_byte = '\x00'.encode('latin1')  ##PY25
-  true_byte = '\x01'.encode('latin1')  ##PY25
+  false_byte = b('\x00')
+  true_byte = b('\x01')
   if is_packed:
     tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED)
     local_EncodeVarint = _EncodeVarint
@@ -677,14 +661,14 @@ def StringEncoder(field_number, is_repeated, is_packed):
   if is_repeated:
     def EncodeRepeatedField(write, value):
       for element in value:
-        encoded = element.encode('utf-8')
+        encoded = string_to_bytestr(element)
         write(tag)
         local_EncodeVarint(write, local_len(encoded))
         write(encoded)
     return EncodeRepeatedField
   else:
     def EncodeField(write, value):
-      encoded = value.encode('utf-8')
+      encoded = string_to_bytestr(value)
       write(tag)
       local_EncodeVarint(write, local_len(encoded))
       return write(encoded)
@@ -770,8 +754,7 @@ def MessageSetItemEncoder(field_number):
       }
     }
   """
-  start_bytes = "".encode("latin1").join([  ##PY25
-##!PY25  start_bytes = b"".join([
+  start_bytes = b("").join([
       TagBytes(1, wire_format.WIRETYPE_START_GROUP),
       TagBytes(2, wire_format.WIRETYPE_VARINT),
       _VarintBytes(field_number),