Your new topic does not fit any of the above??? Check first. Then post here. Thanks.

Moderator: igrr

User avatar
By cjcharles
#65857 Hi all,

Apologies for yet another topic on this but I have tried all sorts of things and cant reliably get this working quickly so any tips or pointers would be much appreciated.

Summary: I have a server device in my house with a TCP API on a custom port, and if you send it JSON then it will respond with different JSON data. I have written an ESP8266 bridge so that you can send the JSON through a normal webpage (with an argument), it will take the argument and make the request from the API before passing back the response to the requester. This seems to work ok on small JSON responses from the API server (~5kB), but if this JSON response goes up to 20kB then it selectively doesn't finish the request. I have tried setting various noDelay flags, changing the timeout and also changing the packet size but have so far failed. I even tried creating a 20kB char array and downloading the whole response from server before passing it back to the requester, which worked ok, but then obviously fails when you reach 21kB!

I am using about 30% of storage and 45% of dynamic memory so should be plenty of space, hence I'd love to hear your thoughts on speeding it up.

Code: Select all#include <ESP8266WiFi.h>
#include <WiFiClient.h>
#include <WiFiUdp.h>
#include <ESP8266WebServer.h>
#include <ESP8266HTTPUpdateServer.h>
#include <WiFiManager.h>
#include <ArduinoOTA.h>
#include <EEPROM.h>
//No longer needed as reading JsonChunkSize bytes at a time
//#include <PrintEx.h>

#define JsonChunkSize 2920

char jsontext[JsonChunkSize];

#define JsonHeaderText "HTTP/1.1 200 OK\r\nContent-Type: application/json;charset=utf-8\r\nServer: Hub Bridge\r\nConnection: close\r\n\r\n"


//////////////////////////////////////////////////////////////////////////////////

ESP8266WebServer server(80);
ESP8266HTTPUpdateServer httpUpdater;


void handleRoot() {
  //This returns the 'homepage' with links to each other main page
 
  char szTmp[1024];
  sprintf(szTmp, "<html>Web Commands<br><a href='/relay'>Hub Relay</a></html>");
  server.send(200, "text/html", szTmp);
}

void handleRelay() {
    //Start building the response with the original request and then add in the hub response
    char commandchararray[500];
    char devicechararray[50];
    int commandlength;
    bool has_device_arg = false;
 
    if (server.hasArg("command")){
        //If we have a command then lets log it for manipulation
        strncpy(commandchararray, server.arg("command").c_str(), 499);
        //Ensure the ending is null char
        commandchararray[499] = '\0';
        Serial.println(commandchararray);
        commandlength = strlen(commandchararray);
        if (server.hasArg("device")) {
            has_device_arg = true;
            strncpy(devicechararray, server.arg("device").c_str(), 49);
            devicechararray[49] = '\0';
            }
    }
    else {
        //No command so send a response to say so
        Serial.println("No command received");
        WiFiClient client;
        client = server.client();
        client.print(JsonHeaderText);
        client.print("{\"No command received\":\"Add /?command=blahblahblah\"}");
        client.stop();
        return;
    }

    strcpy(jsontext, "{\"relaycommand\": ");
    strcat(jsontext, commandchararray);
    strcat(jsontext, ",\r\n");
    if (has_device_arg) {
        strcat(jsontext, "\"relaydevice\": \"");
        strcat(jsontext, devicechararray);
        strcat(jsontext, "\",\r\n");
    }
    strcat(jsontext, "\"relayresult\":");
   

    WiFiClient client; //This is main for the requester
    WiFiClient hubsrv; //This is for the Hub

   //Tried all of these with different varieties with no luck
    //client.setNoDelay(true);
    //hubsrv.setNoDelay(true);
    //client.setTimeout(200);
    //hubsrv.setTimeout(200);
   
    if (hubsrv.connect("192.168.1.100", 424)) {
        Serial.println("Connected to hub");
    }
    else {
        Serial.println("Connection to hub failed");
        return;
    }
 
    //Send command with the \0 otherwise the server rejects the command
    hubsrv.write((const uint8_t *)commandchararray, commandlength+1);
   
    //Now set up the client for printing the return
    client = server.client();
    client.print(JsonHeaderText);
    client.print(jsontext);

    //Just have this wait in case the response is not back from hub yet
    unsigned long timeout = millis();
    while (hubsrv.available() == 0) {
        if (millis() - timeout > 8000) {
            Serial.println(">>> Client Timeout !");
            hubsrv.stop();
            client.stop();
            return;
        }
    }

    while(hubsrv.available()){
        hubsrv.readBytes(jsontext, JsonChunkSize-1);
        jsontext[JsonChunkSize-1] = '\0';
        client.print(jsontext);
    }
   
    client.print("}\r\n");
   
    //Now we should close the connection to the client and hub
    hubsrv.stop();
    client.stop();
}


void handleNotFound(){
  String message = "File Not Found\n\n";
  message += "URI: ";
  message += server.uri();
  message += "\nMethod: ";
  message += (server.method() == HTTP_GET)?"GET":"POST";
  message += "\nArguments: ";
  message += server.args();
  message += "\n";
  for (uint8_t i=0; i<server.args(); i++){
    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
  }
  server.send(404, "text/plain", message);
}

void setup(void){

    Serial.begin(9600); //enable Serial if we need it for debugging

    WiFiManager wifiManager;
    //reset settings - for testing
    //wifiManager.resetSettings();
    wifiManager.setTimeout(180);
   
    //fetches ssid and pass and tries to connect
    //if it does not connect it starts an access point with the specified name
    //and goes into a blocking loop awaiting configuration
    if(!wifiManager.autoConnect("HubRelayBridge")) {
        Serial.println("failed to connect and hit timeout");
        delay(3000);
        //reset and try again, or maybe put it to deep sleep
        ESP.reset();
        delay(5000);
    }


    Serial.println("WiFi Connected - Local IP:");
    Serial.println(WiFi.localIP());
 
    server.on("/", handleRoot);
    server.on("/relay", handleRelay);
    //server.on("/config", HTTP_POST, handleConfig);
   

    //Link the HTTP Updater with the web sesrver
    httpUpdater.setup(&server);
    //Start listening for OTA update requests
    ArduinoOTA.begin();
    //Use default password
    ArduinoOTA.setPassword((const char *)"123");
 
    server.onNotFound(handleNotFound);
    server.begin();

    Serial.println("WiFi Server Started");
}


void loop(void){
    ArduinoOTA.handle();
    server.handleClient();
}
User avatar
By cjcharles
#66609 Does anybody have any ideas or comments that I could try out please?

The most obvious delays are:
1) After the last byte has been loaded it takes a while for the page spinner to stop spinning - maybe 2-3seconds.
2) It takes a bit of time to get the first request but presumably this is just the time to connect to the various servers
3) You can see the individual chunks of JSON data appear one by one, with a delay of maybe 200ms between them.
The overall page load is 6-7seconds with a 5kb JSON, though based on the actual page loading I would say it only takes less than 1s for the data to appear out of the 6-7s total time, based on the chunks from point 3.

Here is the Chrome breakdown of loading time:
Queueing 4.00 ms

Connection Start TIME
Stalled 247.00 ms
DNS Lookup 0
Initial connection 1.61 s

Request/Response TIME
Request sent 0
Waiting (TTFB) 10.00 ms
Content Download 4.44 s

Total 6.32s

Thank you again