How to add bytes together and validate them against the checksum in python

user_11343161 0
Viewed 371

How do you calculate the checksum of incoming bytes to see if it is a valid packet? Currently I am reading the bytes and decoding them and receiving the information but before I do that I would like to be able to validate it against the checksum to make sure I do not get any invalid/corrupt packets.

Here is what I currently have

def batteryConnect(port, baudrate):
    # establishing a serial connection
    ser = serial.Serial(port=port, baudrate= baudrate, bytesize=serial.EIGHTBITS, parity=serial.PARITY_NONE,timeout=3)
    return ser

class VictronBmv700:

    def __init__(self,port,baudrate):
        self.port = port
        self.baudrate = baudrate
        self.parameterDict = dict.fromkeys(["PID","V","I","P","CE","SOC","TTG","Alarm","Relay","AR","BMV","FW","H1","H2","H3",
                                  "H4","H5","H6","H7","H8","H9","H10","H11","H12","H17","H18"])


    def getBMVInfo(self):

        bmvdata = batteryConnect(self.port,self.baudrate)

        #getting the data
        bmvdata.flushInput()

        #getting the message then splitting the key, value pairs
        while True:
            print(bmvdata.readline())

            message = bmvdata.readline().decode(encoding='utf-8',errors='ignore')

            #splitting on tabs
            message_parts = message.split('\t')

            if len(message_parts) > 1:
                key = message_parts[0]
                value = message_parts[1].rstrip()  #stripping \r\n after the value


                #updating deictionary based on keys and their values.
                self.parameterDict[key] = value

if __name__ == "__main__":
    print("BATTERY MONITOR")
    bmv700 = VictronBmv700("COM17", 19200)
    bmv700.getBMVInfo()

Which starts outputting the following (from the print(bmvdata.readline()) )

b'\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd7\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd6\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'

How would I check the incoming bytes against the checksum then continue to decode after validation?

EDIT Sometimes I get different checksum values as below. Here are two additional times I ran the code. 

b'\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd7\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd8\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'
b'PID\t0x203\r\n'
b'I\t0\r\n'
b'CE\t0\r\n'
b'TTG\t-1\r\n'
b'Relay\tOFF\r\n'
b'BMV\t700\r\n'
b'Checksum\t\xd0\r\n'
b'H2\t0\r\n'
b'H4\t0\r\n'
b'H6\t-9001\r\n'
b'H8\t28403\r\n'
b'H10\t0\r\n'
b'H12\t0\r\n'
b'H18\t87\r\n'

Second run

b'\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25549\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25549\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'
b'H1\t-6254\r\n'
b'H3\t0\r\n'
b'H5\t0\r\n'
b'H7\t-10\r\n'
b'H9\t0\r\n'
b'H11\t0\r\n'
b'H17\t20\r\n'
b'Checksum\tT\r\n'
b'V\t25548\r\n'
b'P\t0\r\n'
b'SOC\t1000\r\n'
b'Alarm\tOFF\r\n'
b'AR\t0\r\n'
b'FW\t0310\r\n'

EDIT Ran the following code

current_block = []
# it's unfortunate that lines start with `\r\n` rather than end with it
# we will read the first empty line separatey, and then include these 
# 2 characters in the last line that is read
message = bmvdata.readline()
while True:
    # Note that we cannot decode yet, since the Checksum value may not be a valid utf8 character
    message = bmvdata.readline()

    # we save the message in the current block before any preprocessing
    current_block.append(message)

    #splitting on tabs (as bytes !!)
    message_parts = message.split(b'\t')

    if len(message_parts) > 1:
        key = message_parts[0].decode('utf8')  # here it is safe to decode the key
        value = message_parts[1].rstrip()  #stripping \r\n after the value

    if key == 'Checksum':
        block_chars = b''.join(current_block)
        checksum = sum(block_chars) % 256
        print('Retrieved  checksum', value[-1])
        print('Calculated checksum', checksum)
        print('----')

        # reset the block
        current_block = []

output

Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 219
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 220
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 219
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 219
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 220
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 220
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 220
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
Retrieved checksum 219
Calculated checksum 0
-------
Retrieved checksum 84
Calculated checksum 0
-------
checksum pyserial python-3.x python
2 Answers

It seems that the poster is trying to read data from a BMV700 battery monitor. Reading the papers here, we see that this communicates over the Serial interface using a text protocol:

2 TextProtocol

When no VE.Direct queries are sent to the device, the charger periodically sends human readable (TEXT) data to the serial port. See the "VE.Direct Protocol" document for a detailed description of the contents and availability of the information

Excerpt from the HEX protocol (BMV-7xx-HEX-Protocol-public.pdf)

Looking into the specification of the TEXT protocol (VE.Direct-Protocol-3.28.pdf), we find:

Message format

The device transmits blocks of data at 1 second intervals. Each field is sent using the following format:

<Newline><Field-Label><Tab><Field-Value>

The identifiers are defined as follows:

+---------------+--------------------------------------------------------------------------------------+
| Identifier    | Meaning                                                                              |
+===============+======================================================================================+
| <Newline>     | A carriage return followed by a line feed (0x0D, 0x0A).                              |
+---------------+--------------------------------------------------------------------------------------+
| <Field-Label> | An arbitrary length label that identifies the field.                                 |
|               | Where applicable, this will be the same as the label that is used on the LCD.        |
+---------------+--------------------------------------------------------------------------------------+
| <Tab>         | A horizontal tab (0x09).                                                             |
+---------------+--------------------------------------------------------------------------------------+
| <Field-Value> | The ASCII formatted value of this field.                                             |
|               | The number of characters transmitted depends on the magnitude and sign of the value. |
+---------------+--------------------------------------------------------------------------------------+

This corrsponds to the data you're printing, with one exception: the line starts with \r\b, it doesn't end with them.

Then, the file gives details about the Checksum:

Data integrity

The statistics are grouped in blocks with a checksum appended. The last field in a block will always be “Checksum”. The value is a single byte, and will not necessarily be a printable ASCII character. The modulo 256 sum of all bytes in a block will equal 0 if there were no transmission errors. Multiple blocks are sent containing different fields.

So, in the excerpt that you posted as output, you have two complete blocks, and an incomplete one, as the last block is missing the Checksum.

block = (b'\r\nH2\t0\r\n'
    b'H4\t0\r\n'
    b'H6\t-9001\r\n'
    b'H8\t28403\r\n'
    b'H10\t0\r\n'
    b'H12\t0\r\n'
    b'H18\t87\r\n'
    b'PID\t0x203\r\n'
    b'I\t0\r\n'
    b'CE\t0\r\n'
    b'TTG\t-1\r\n'
    b'Relay\tOFF\r\n'
    b'BMV\t700\r\n'
    b'Checksum\t\xd6')

Note that I have added the \r\n at the beginning, and removed them from the end of the block, such that the last byte is the block's checksum.

Now, we can calculate the checksum of the block:

>>> sum(block) % 256
213

This should have been zero. So, there may be some transmission problems, or they may be calculating the checksum differently from what they say in the docs.

Edit after new data.

Here is the code that I have used to inspect all the blocks that you have posted:

current_block = []
# it's unfortunate that lines start with `\r\n` rather than end with it
# we will read the first empty line separatey, and then include these 
# 2 characters in the last line that is read
message = bmvdata.readline()
while True:
    # Note that we cannot decode yet, since the Checksum value may not be a valid utf8 character
    message = bmvdata.readline()

    # we save the message in the current block before any preprocessing
    current_block.append(message)

    #splitting on tabs (as bytes !!)
    message_parts = message.split(b'\t')

    if len(message_parts) > 1:
        key = message_parts[0].decode('utf8')  # here it is safe to decode the key
        value = message_parts[1].rstrip()  #stripping \r\n after the value

    if key == 'Checksum':
        block_chars = b''.join(current_block)
        checksum = sum(block_chars) % 256
        print('Retrieved  checksum', value[-1])
        print('Calculated checksum', checksum)
        print('----')

        # reset the block
        current_block = []

It is weird that for the first run you always get the calculated value equal retrieved + 1, but not in the second run. So, there may be some transmission problems.

user_7865590

Below code compare two files in this case checksum as asker mentioned:

 bmvdata = batteryConnect(self.port,self.baudrate)
 bmvdata.flushInput()
 print(bmvdata.readline())

It has been done by the aid of hashlib

import hashlib
#File 1 = checksum
hasher1 = hashlib.md5()
#afile1 = open('checksum', 'rb')
buf1 = bmvdata.read()
#buf1 = afile1.read()
a = hasher1.update(buf1)
md5_a=(str(hasher1.hexdigest()))
#File 2
hasher2 = hashlib.md5()
afile2 = open('incoming-byte', 'rb')
buf2 = afile2.read()
b = hasher2.update(buf2)
md5_b=(str(hasher2.hexdigest()))
#Compare md5
if(md5_a==md5_b):
    print("Yes")
else:
    print("No")
user_22927670
Related