Python socket server script

user_14216337 0
Viewed 473

I have a problem with setting up a continuously running python written socket server on a RasPi (1st gen) running the latest Raspbian build. What I wanted to do is to set up the RasPi as a data server for remote temp. sensors which uses ESP32 with WiFi for comms. The problem I'm having is with my server freezes which happen on an unregular basis. What the server does (or at least should be doing) is, it accepts a remote connection, gets data, slices it into chunks and saves data to a file. The script starts of running great, with everything running as expected, but over time it freezes and for the love of the Coding Gods I have no idea why. Maybe someone here could help me?

Server code below:

import socket
import datetime

PARAMS=['T','P','A','H','B','C']
sensors={'id':0}

for param in PARAMS:
    sensors[param]={'value':0.0,'index_start':0,'index_end':0}


s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('192.168.0.10', 7543 ))
s.listen(1)

while True:
    print("****Waiting for connection")
    conn, addr = s.accept()
    #create index for data slicing
    contents=[]
    with conn:
        print('***Incoming connection from: ', addr)
        while conn: # change here!
            print("****What do you have for me? ")
            content = conn.recv(1024)
            print("***Received: ")
            print(content)
            
            contents.append(content)
            print("****Data put into the pool")
            if not content:
                print("****No more data")
                break
            #conn.sendall(data)


    print("***Data passed for processing: ") 
    print(contents)
    
    now=datetime.datetime.now()
    timestamp=now.strftime("%Y-%m-%d %H:%M:%S")
    temp_list=[]
    for c in contents:
       
        clean_c=str(c).lstrip('b\'').rstrip('\'')
        temp_list.append(clean_c)
    text=''.join(temp_list)
    print(text)
    
    for i,param in enumerate(PARAMS):
        sensors[param]['index_start']=text.index(param)+2
        if param != 'T':
            prev_param=PARAMS[i-1]
            sensors[prev_param]['index_end']=text.index(param)
        if param == 'C':
            sensors[param]['index_end']=len(text)-1
    
    sensors['id']=text[:sensors['T']['index_start']-2]
    for param in PARAMS:
        sensors[param]['value']=text[sensors[param]['index_start']:sensors[param]['index_end']]
        if param == 'C':
            sensors[param]['value']=text[sensors[param]['index_start']:]



    value_tuple=(timestamp,sensors['id'],)
    for param in PARAMS:
        value_tuple+=(sensors[param]['value'].replace('.',','),)
        
    text='\t'.join(value_tuple)
    
    with open('/home/pi/Desktop/sensors.csv','a+') as file:
        file.write(text+'\n')    
    
    

    print("****Closing connection")
    print(timestamp)
    #conn.close()

The freeze always happens on the

content = conn.recv(1024)

I know that this function locks my program, but I don't know why it even goes there if there's no data? And what to do to make it .... well .. not do that? I close the connection on the client side and from what I know the with function should respond to socet closure, right?

Any help would be greatly appreciated.

EDIT1:

So I've been asked about stack errors but no. After I Ctr+C I get a standard response, ONLY with the content = conn.recv(1024) reference:

Traceback (most recent call last):
  File "/home/pi/My_Python/Server2.py", line 27, in <module>
    conn, addr = s.accept()
  File "/usr/lib/python3.7/socket.py", line 212, in accept
    fd, addr = self._accept()
KeyboardInterrupt: Execution interrupted

As for the client code:

#define ver 0.21
#define date "2020.09.03"
#include <displayVersion.h>
displayVersion caseVersion;

#define DEEP_SLEEP_TIME 60  
#define numberOfMeasurements 10

#include <WiFi.h>

#define ONBOARD_LED  2


int sensorID = 1001;
bool debug=true;
float temperature,pressure,altitude,humidity;
float AVGtemperature,AVGpressure,AVGaltitude,AVGhumidity;
int batteryLevel;
int AVGbatteryLevel;
bool batteryCharging;

 
const char* ssid = "xxxx";
const char* password =  "xxxx";
const uint16_t port = 7543;
const char * host = "192.168.0.10";

bool wifiConnectionStatus;

void setup()
{
 // setup onboard IOs
  Serial.begin(115200);
  caseVersion.display(ver, date, 1000);      
  
  pinMode(ONBOARD_LED,OUTPUT);
  pinMode(BATTERY_LEVEL_PIN,INPUT);
  pinMode(BATTERY_CHARGING_PIN,INPUT);

 
// setup WiFi 
  bool status;
  Serial.println("Starting operation...");
  WiFi.begin(ssid, password);
  Serial.println("Connecting to WiFi...");
  while (WiFi.status() != WL_CONNECTED) {
    delay(100);
    Serial.print(".");
  }
  Serial.println("");
  Serial.print("WiFi connected with IP: ");
  Serial.println(WiFi.localIP());

}
 
void loop()
{

    // checks i f its connected to WiFi 
    if (WiFi.status()==3) 
    {
      digitalWrite(ONBOARD_LED,HIGH);
      wifiConnectionStatus=true;
    }
    
    else 
    {
      digitalWrite(ONBOARD_LED,LOW);
      wifiConnectionStatus=false;
    }


  
    
    // gets sensor data a set number of times
    
    Serial.println ("Starting measurements...");
    for (int i=0;i<numberOfMeasurements; i++)
    {
      temperature= 4;
      AVGtemperature=AVGtemperature+temperature;
      
      pressure=5;
      AVGpressure=AVGpressure+pressure;
      
      altitude=6;
      AVGaltitude=AVGaltitude+altitude;
      
      humidity= 7;
      AVGhumidity=AVGhumidity+humidity;
      
        // gets battery data
      batteryLevel  = analogRead(BATTERY_LEVEL_PIN);
      AVGbatteryLevel=AVGbatteryLevel+batteryLevel;
      
      Serial.printf("Measurement No: %d\n", i+1);
      delay(300); // to be uncommented later on
    
      }
  
  AVGtemperature=AVGtemperature/numberOfMeasurements;
  AVGpressure=AVGpressure/numberOfMeasurements;
  AVGaltitude=AVGaltitude/numberOfMeasurements;
  AVGhumidity=AVGhumidity/numberOfMeasurements;
  AVGbatteryLevel=AVGbatteryLevel/numberOfMeasurements;

int temp      = analogRead(BATTERY_CHARGING_PIN);
if (temp > 1000)  batteryCharging = 1;
else              batteryCharging = 0;

    // while in debug
if (debug)
  {
  Serial.print("**Sensors** ");  
  Serial.printf("Temperature: %f ", AVGtemperature);
  Serial.printf("Pressure: %f ", AVGpressure);
  Serial.printf("Altitude: %f ", AVGaltitude);
  Serial.printf("Humidity: %f ", AVGhumidity);
  Serial.print("**Status** ");
  Serial.printf("Battery level: %d % ", AVGbatteryLevel);
  Serial.printf("Battery charging status: %d \n", batteryCharging);
  Serial.printf("Data to be sent: \n");
  Serial.printf("\n%dT:%2fP:%2fA:2%fH:2%fB:%dC:%d",sensorID,AVGtemperature,AVGpressure,AVGaltitude,AVGhumidity,AVGbatteryLevel,batteryCharging);

  }

   
    
    // connects to a sever
    if (wifiConnectionStatus)
    {
     WiFiClient client; 
      if (!client.connect(host, port)) 
      {
 
        if (debug) Serial.println("Connection to host failed");
        //delay(500);
        //return; // removed to save energy
        goToDeepSleep();
      }
        if (debug) Serial.println("Connected to server successful!");
 
    // sends out data
    client.printf("%dT:%2fP:%2fA:%2fH:%2fB:%dC:%d",sensorID,AVGtemperature,AVGpressure,AVGaltitude,AVGhumidity,AVGbatteryLevel,batteryCharging);

        
    // dissconnects
        if (debug) 
        {
          Serial.println("Disconnecting...");
        }
      client.stop();
      if (debug) 
        {
          Serial.println("DONE...");
        }
        delay(5);
    }
    // goes to sleep
      goToDeepSleep();
    //wakes up and -> get temp data
     
}


void goToDeepSleep()
{
Serial.println("Going to sleep...");  

  esp_sleep_enable_timer_wakeup (DEEP_SLEEP_TIME*1000000);
  esp_deep_sleep_start();
  
  }

EDIT 2:

Ok. So first things first. I don't know how one should format their posts on stackoverflow. This is actually my first time posting. That's why I don't know if I should carry out the discussion in the comments or here or ....? I also don't know if I should post my comments and findings at the bottom of this post, at the top or in a separate post. The thing I want to point out is NOT an answer to my problem it's just a finding. That being said, here it is:

I've been thinking about what @Mark Setchell and @bnaecker have said about reading a set number of bytes and it got me thinking. What exactly is a stop?
Apart from terminating a connection locally it also sends out a message. And I think I'm getting that message.... See, whenever I run my server and print out my received data, I get 2 streams. First is my data stream and the other one is:

(...)
***Received: 
b''
****Data put into the pool
(...)

could this be the EOF from the stop() function? And if so, would there be a chance that my server is somehow not receiving this message sometimes? This would make it freeze. But if that's the case I would need to have a way to see if a socket is open or not. If my client closes a connection that means that the socket is done for and the server should detect it. I could work with that. But how to get that? I was thinking of suing the 'conn' from my script but as I've just literally started to use this language it's still overwhelming to me. Then is 'conn' an object? A variable? When I print it, I get huge amount of data from it. So, is it a string? I'm at a loss here...

EDIT3:

This is getting ridiculous with all those post-edits, but I guess these are the rules.

First of all, I would like to give out a HUGE 'thank You!' to each and every one who has been helping me so far. You guys rock! :D It took me a while to answer because I've been troubleshooting. I guess this is one of the things which I'm not particularly loving about Python. There's no repository with a complete description of functions. For example, everywhere I've looked there’s this statement that “socket.recv() is a blocking function”. And (to quote Alec Baldwin in a commercial) yeah, it's a blocking function except, no it isn't. What you can do is conn.recv(59,socket.MSG_DONTWAIT) What it does is, it gets the incoming buffer and that's it. No waiting. I would guess it should be the same with conn.recv() I was so happy to try it and it's been running ok until I got [Errno 11] Resource temporarily unavailable which basically says that there was nothing to read. I thought that it may have been the problem with getting the EOF signal - judging from when has the exception been thrown - even though I send/read a set number of bytes. This means that the problem is consistent with what I’ve been experiencing so far. Cool So then I've decided to use Your advice and put a timeout on the socket. This has been the longest run so far! 2h! I was so happy! Until it crashed. On that timeout. Ugh… So now I'm running my next test with exception handling, like @glory9211 and @urban had suggested. I'll let You guys know how it goes. Fingers crossed.

Cheers, Rafał

serversocket raspbian raspberry-pi server python
2 Answers

tl;dr:

  1. set timeout on the conn to avoid issues that could originate on the client
  2. try/except all exceptions around content = conn.recv(1024), log them and close socket
  3. contents.append(content) happens before check for conn close so you always end up with [..., b''] at the end of input
  4. focus on the client and maybe use tcpdump/wireshark to investigate more

I have tested the following minimal server loop with netcat and telnet:

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('127.0.0.1', 7543 ))
s.listen(1)

while True:
    print("****Waiting for connection")
    conn, addr = s.accept()
    #create index for data slicing
    contents=[]
    with conn:
        print('***Incoming connection from: ', addr)
        while conn: # change here!
            print("****What do you have for me? ")
            content = conn.recv(1024)
            print("***Received: ")
            print(content)
            
            contents.append(content)
            print("****Data put into the pool")
            if not content:
                print("****No more data")
                break
            #conn.sendall(data)


    print("***Data passed for processing: ") 
    print(contents)

Observation 1:

The server kept on working as expected through multiple tests with both netcat and telnet. With netcat I Ctrl+C which on my system puts the connection on TIME_WAIT but the server kept on working (as expected). Killing either client (with kill) also had no effect on the server.

The one case I could cause some harm to the server was when I forcefully kill the client TCP connection with:

$ sudo ss -K dst 127.0.0.1 dport = 7543

In this case the server breaks with:

Traceback (most recent call last):
  File "./tests.py", line 24, in <module>
    content = conn.recv(1024)
ConnectionResetError: [Errno 104] Connection reset by peer

This should be easy to handle with try/except around receive. I am suggesting this cause I see you put the client to deep sleep. I am far from expert on how this works and I cannot see how this case can happen but better safe than sry :)

Observation 2:

As per comments above, EOF is always sent to the socket when the connection closes. In your code you first append data to the contents and later you check for EOF! As a result you have the EOF in the contents. Example:

***Incoming connection from:  ('127.0.0.1', 49272)
****What do you have for me? 
***Received: 
b'asdasd\r\n'
****Data put into the pool
****What do you have for me? 
***Received: 
b'asdasd\r\n'
****Data put into the pool
****What do you have for me? 
***Received: 
b'sadasd\r\n'
****Data put into the pool
****What do you have for me? 
***Received: 
b'asdsda\r\n'
****Data put into the pool
****What do you have for me? 
***Received: 
b''  # <<<<<< HERE you receive the "close"
****Data put into the pool
****No more data
***Data passed for processing: 
[b'asdasd\r\n', b'asdasd\r\n', b'sadasd\r\n', b'asdsda\r\n', b'']  # <<<<<< Here you see you added it in contents

The above behaviour is standard. You are not actually getting 2 streams as in your question but 2 "messages" (in TCP the correct term I think is Segments) in the same stream. The first is data sent by the client and the second (EOF) is send to the application when a socket is terminated (FIN, ACK+FIN, ACK). A random result from google you can read more about. I believe that the EOF here is not sent by the client but by the local TCP/IP stack (kernel) to your application to indicate that the termination sequence occured in the TCP socket so you cannot "miss it" (not 100% sure on this). If you miss something is the FIN sequence.

Observation 3: socket timeouts and block on recv()

You are asking what conn is: it is a socket object specific to a particular client. Thats what accept() returns.

If you suspect that the client connects and then sends nothing for a long time (yes, this blocks on recv() which blocks your server), I would suggest you try to set the timeout on the conn socket. I am saying this because from what I gather the client connects, sends, closes in each loop. If for some reason this does not happen and the client just connects and waits then it keeps your server loop busy because it just accepted the client and waits for data on the conn socket!

This is the only case I think the server would block on recv() and is most probably an issue with the client not sending... In any case, your server shouldn't really be "vulnerable" to that so I would attempt conn.settimeout(5) giving 5 secs to the client to transmit. You will need to handle timeout Exception raised when the time passed and the client did not send anything.

I have tried to reprod this case with the following steps:

  • Start server script
  • Start netcat client and send some data thru
  • Block the dest port on the firewall with sudo iptables -A INPUT -p tcp --destination-port 7543 -j DROP - no more data go from netcat->server
  • Ctrl+C the client which now attemptes to send a FIN but it is dropped

Server indeed hangs on:

***Incoming connection from:  ('127.0.0.1', 51194)
****What do you have for me? 
***Received: 
b'asdasd\n'
****Data put into the pool
****What do you have for me? 
***Received: 
b'asdasd\n'
****Data put into the pool
****What do you have for me?
  • Remove the firewall rule (sudo iptables -D INPUT 1)
  • Attempt to connect to the server again server is blocked
  • Ctrl+C show block on recv:
^CTraceback (most recent call last):
  File "./tests.py", line 24, in <module>
    content = conn.recv(1024)
KeyboardInterrupt

Proposed "Solution":

NOTE: This does not root-cause or solve the problem, just potentially works around it (and logs it)!

  • Adding timeout of 5 sec
  • Connecting with telnet
  • Send some data within 5 sec
  • Wait for >5 secs

The server output is (note it detected timeout):

****Waiting for connection
***Incoming connection from:  ('127.0.0.1', 51858)
****What do you have for me? 
***Received: 
b'asd\r\n'
****Data put into the pool
****What do you have for me? 
!!! ERROR while receiving: timed out
***Data passed for processing: 
[b'asd\r\n']
****Waiting for connection

The client is also notified when the server closes the connection:

$ telnet localhost 7543
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
asd  # << This was received before timeout
Connection closed by foreign host. ## << This is the server hanging up

The code:

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('127.0.0.1', 7543 ))
s.listen(1)

while True:
    print("****Waiting for connection")
    conn, addr = s.accept()
    #create index for data slicing
    contents=[]
    with conn:
        conn.settimeout(5)  # << NEW!
        try:                # << NEW!
            print('***Incoming connection from: ', addr)
            while conn: # change here!
                print("****What do you have for me? ")
                content = conn.recv(1024)
                print("***Received: ")
                print(content)
                
                contents.append(content)
                print("****Data put into the pool")
                if not content:
                    print("****No more data")
                    break
                #conn.sendall(data)
        except Exception as e:
            print("!!! ERROR while receiving:", str(e))


    print("***Data passed for processing: ") 
    print(contents)

As for the root-cause, you would need to either collect timestamped logs or wireshark/tcpdump traces to see what happens. This might be complex if the issue only happens randomly and after long time...

user_37270500

So let me give a simple answer here:

When you say conn.recv(x) you're saying 'don't return until you've read x bytes from the socket'. This is called "blocking I/O". The program will wait to receive x bytes (freeze in your case).

So there are many solutions to this (Kindly comment which method work for you under this answers for others help).

  1. Replace conn.recv(1024) with conn.recv(). This will return value everytime new data arrives.

  2. Replace conn.recv(1024) with conn.recv(x).Where 'x' is the size of data from your board's client.printf statement. This will return value every time new data of size x arrives. I would recommend this since the data you are sending has a fixed size.

  3. Another way is to use a timeout: place conn.settimeout(3) before calling conn.recv(x) which will wait for only 3 secs (instead of freezing) and if no data is received it will throw an exception which you can handle via python try/except block.

  4. A bit tricky one, but you can make your socket non-blocking for conn.recv() (For info you can find good tutorials on internet, just sharing it here in case you are a curious person)

Also a little tip regarding your client code:

Don't create WiFiClient client inside the loop and then close it. Instead declare it globally (alongside ssid, temprature variables etc.). Inside void loop () only call client.connected() to get connection status and send data.

Hopefully, Solution 1,2,3 or combinations of these will work for you.

user_116723520
Related