Skip to content

Commit 4faf21c

Browse files
committed
Zero-copy integer parsing
Common method to parse integer was very inefficient: - read method has been called to create a copy of couple bytes; - separate method has been called just to call struct.unpack; - for one-byte integers always failing type check happend in byte2int method; - for three-byte integers there were extra bytes concatenation. This change makes all integer parsing with no copies and with minimal method calls number.
1 parent 618f844 commit 4faf21c

1 file changed

Lines changed: 41 additions & 10 deletions

File tree

pymysql/connections.py

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -291,23 +291,54 @@ def get_bytes(self, position, length=1):
291291
"""
292292
return self._data[position:(position+length)]
293293

294+
if PY2:
295+
def read_uint8(self):
296+
result = ord(self._data[self._position])
297+
self._position += 1
298+
return result
299+
else:
300+
def read_uint8(self):
301+
result = self._data[self._position]
302+
self._position += 1
303+
return result
304+
305+
def read_uint16(self):
306+
result = struct.unpack_from('<H', self._data, self._position)[0]
307+
self._position += 2
308+
return result
309+
310+
def read_uint24(self):
311+
low, high = struct.unpack_from('<HB', self._data, self._position)
312+
self._position += 3
313+
return low + high << 16
314+
315+
def read_uint32(self):
316+
result = struct.unpack_from('<I', self._data, self._position)[0]
317+
self._position += 4
318+
return result
319+
320+
def read_uint64(self):
321+
result = struct.unpack_from('<Q', self._data, self._position)[0]
322+
self._position += 8
323+
return result
324+
294325
def read_length_encoded_integer(self):
295326
"""Read a 'Length Coded Binary' number from the data buffer.
296327
297328
Length coded numbers can be anywhere from 1 to 9 bytes depending
298329
on the value of the first byte.
299330
"""
300-
c = ord(self.read(1))
331+
c = self.read_uint8()
301332
if c == NULL_COLUMN:
302333
return None
303334
if c < UNSIGNED_CHAR_COLUMN:
304335
return c
305336
elif c == UNSIGNED_SHORT_COLUMN:
306-
return unpack_uint16(self.read(UNSIGNED_SHORT_LENGTH))
337+
return self.read_uint16()
307338
elif c == UNSIGNED_INT24_COLUMN:
308-
return unpack_int24(self.read(UNSIGNED_INT24_LENGTH))
339+
return self.read_uint24()
309340
elif c == UNSIGNED_INT64_COLUMN:
310-
return unpack_int64(self.read(UNSIGNED_INT64_LENGTH))
341+
return self.read_uint64()
311342

312343
def read_length_coded_string(self):
313344
"""Read a 'Length Coded String' from the data buffer.
@@ -344,7 +375,7 @@ def check_error(self):
344375
if self.is_error_packet():
345376
self.rewind()
346377
self.advance(1) # field_count == error (we already know that)
347-
errno = unpack_uint16(self.read(2))
378+
errno = self.read_uint16()
348379
if DEBUG: print("errno =", errno)
349380
raise_mysql_exception(self._data)
350381

@@ -375,11 +406,11 @@ def __parse_field_descriptor(self, encoding):
375406
self.name = self.read_length_coded_string().decode(encoding)
376407
self.org_name = self.read_length_coded_string().decode(encoding)
377408
self.advance(1) # non-null filler
378-
self.charsetnr = struct.unpack('<H', self.read(2))[0]
379-
self.length = struct.unpack('<I', self.read(4))[0]
380-
self.type_code = byte2int(self.read(1))
381-
self.flags = struct.unpack('<H', self.read(2))[0]
382-
self.scale = byte2int(self.read(1)) # "decimals"
409+
self.charsetnr = self.read_uint16()
410+
self.length = self.read_uint32()
411+
self.type_code = self.read_uint8()
412+
self.flags = self.read_uint16()
413+
self.scale = self.read_uint8() # "decimals"
383414
self.advance(2) # filler (always 0x00)
384415
# 'default' is a length coded binary and is still in the buffer?
385416
# not used for normal result sets...

0 commit comments

Comments
 (0)