-->
Page 1 of 2

MQTT does not work if payload is over 107 characters

PostPosted: Sat Sep 02, 2017 2:30 pm
by taikapanu
Hi,

I'm suffering with a small problem with my esp chip. I cannot seem to be able to send over 107 character message to mosquitto MQTT server running on raspberry. I'm using arduino ide to program esp and PubSubClient library to communicate to MQTT. The point is to gather all information from one place to one JSON to be processed in Home Assistant.

I know the fault is with the ESP as I can send quite a lot longer message from Home Assistant software.

This is the piece of code I'm using to send the data. It is just an array as I was trying to figure out from there how long message I'm able to send. data array is big enough, I can print it to serial and it's fine. But MQTT is not. Any advice what to do next?
Code: Select all strcpy(data, "{\"lock\":{\"state\":\"LOCK\"},\"door\":{\"state\":\"closed\"},\"temperature\":{\"warehouse\":\"20\",\"outside\":\"21\",\"sauna\":\"");//22\"},\"humidity\":{\"warehouse\":\"23\",\"outside\":\"24\",\"sauna\":\"25\"}}");
client.publish(MQTT_SENSOR_TOPIC_WAREHOUSE, data, true);

Re: MQTT does not work if payload is over 107 characters

PostPosted: Sun Sep 03, 2017 12:48 am
by taikapanu
No error. Just fail..

I did some more reading and noticed that the library just drops the message. Then I felt a bit embarrassing to read from github page:
The maximum message size, including header, is 128 bytes by default. This is configurable via MQTT_MAX_PACKET_SIZE in PubSubClient.h.

woops. Anyhow, ran into next problem. It didn't work even if I included this.
Code: Select all#define MQTT_MAX_PACKET_SIZE 500

It was after the #include so of course wouldn't work but it didn't work the other way around either. Then I increased the value directly in the arduino library .h file. Then it works as I expected.
It is defined like this in the .h file. It was 128 but I have increased it.
Code: Select all// MQTT_MAX_PACKET_SIZE : Maximum packet size
#ifndef MQTT_MAX_PACKET_SIZE
#define MQTT_MAX_PACKET_SIZE 328
#endif

Why it didn't work with the #define before #include?

The ESP is reading JSON from Arduino. Basically it is just message handler between Arduino and MQTT server. Any comments on my code appreciated!
Code: Select all#define MQTT_VERSION MQTT_VERSION_3_1_1
#define MQTT_MAX_PACKET_SIZE 500

#include <ESP8266WiFi.h>
#include <PubSubClient.h>
#include <ArduinoJson.h>

// OTA updates start
#include <ESP8266mDNS.h>
#include <WiFiUdp.h>
#include <ArduinoOTA.h>
// OTA updates end

// Wifi: SSID and password
const char* WIFI_SSID = "XX";
const char* WIFI_PASSWORD = "XX";

// MQTT: ID, server IP, port, username and password
const PROGMEM char* MQTT_CLIENT_ID = "XX";
const PROGMEM char* MQTT_SERVER_IP = "XX";
const PROGMEM uint16_t MQTT_SERVER_PORT = XX;
const PROGMEM char* MQTT_USER = "XX";
const PROGMEM char* MQTT_PASSWORD = "XX";

// MQTT: topic
const PROGMEM char* MQTT_SENSOR_TOPIC_WAREHOUSE = "warehouse/json";

// Leds
#define LEDRED 16
#define LEDGREEN 12

// Serial communication
String inputString = "";         // a string to hold incoming data

// Wifi and MQTT clients
WiFiClient wifiClient;
PubSubClient client(wifiClient);


// function called when a MQTT message arrived
void callback(char* p_topic, byte* p_payload, unsigned int p_length) {
}

void reconnect() {
  // Loop until we're reconnected
  while (!client.connected()) {
    //Serial.print("INFO: Attempting MQTT connection...");
    // Attempt to connect
    if (client.connect(MQTT_CLIENT_ID, MQTT_USER, MQTT_PASSWORD)) {
      //Serial.println("INFO: connected");
    } else {
      //Serial.print("ERROR: failed, rc=");
      //Serial.print(client.state());
      //Serial.println("DEBUG: try again in 5 seconds");
      // Wait 5 seconds before retrying
      yield();
      delay(5000);
    }
  }
}

void setup() {
  // init the serial
  Serial.begin(9600);
    inputString.reserve(360); // reserve 200 bytes for the inputString:

  pinMode(LEDRED, OUTPUT);
  pinMode(LEDGREEN, OUTPUT);

  digitalWrite(LEDRED, LOW);
  digitalWrite(LEDGREEN, LOW);

  digitalWrite(LEDRED, HIGH);
  digitalWrite(LEDGREEN, LOW);
 
  // init the WiFi connection
  //Serial.println();
  //Serial.println();
  //Serial.print("INFO: Connecting to ");
  WiFi.mode(WIFI_STA);
  //Serial.println(WIFI_SSID);
  WiFi.begin(WIFI_SSID, WIFI_PASSWORD);

  unsigned long connect_check = millis();
  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    //Serial.print(".");
     if ( connect_check + 10000 < millis() )
    {
        //Serial.println(".");
        //Serial.println("Attempting to reconnect wifi");
        WiFi.disconnect();
        delay(1000);
        WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
        connect_check = millis();
        yield();
    }
  }

  Serial.println("");
  Serial.println("INFO: WiFi connected");
  Serial.println("INFO: IP address: ");
  Serial.println(WiFi.localIP());

  // init the MQTT connection
  client.setServer(MQTT_SERVER_IP, MQTT_SERVER_PORT);
  client.setCallback(callback);

  // OTA updates
  ArduinoOTA.setHostname("xx");
  ArduinoOTA.setPassword((const char *)"xx");
  ArduinoOTA.onStart([]() {
    Serial.println("Start");
  });
  ArduinoOTA.onEnd([]() {
    Serial.println("\nEnd");
  });
  ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) {
    Serial.printf("Progress: %u%%\r", (progress / (total / 100)));
  });
  ArduinoOTA.onError([](ota_error_t error) {
    Serial.printf("Error[%u]: ", error);
    if (error == OTA_AUTH_ERROR) Serial.println("Auth Failed");
    else if (error == OTA_BEGIN_ERROR) Serial.println("Begin Failed");
    else if (error == OTA_CONNECT_ERROR) Serial.println("Connect Failed");
    else if (error == OTA_RECEIVE_ERROR) Serial.println("Receive Failed");
    else if (error == OTA_END_ERROR) Serial.println("End Failed");
  });
  ArduinoOTA.begin();

  digitalWrite(LEDRED, LOW);
  digitalWrite(LEDGREEN, HIGH);
}

void loop() {
    if (!client.connected()) {
    reconnect();
    }
    client.loop();
    ArduinoOTA.handle();
   
   

    yield();

   
 
    checkSerialEvent();

}

void checkSerialEvent() {
  while (Serial.available()) {
    // get the new byte:
    char inChar = (char)Serial.read();
    // add it to the inputString:
    inputString += inChar;
    // if the incoming character is a newline, set a flag
    // so the main loop can do something about it:
    if (inChar == '\n')
    {
        // Message has ended. Decode it.
        decodeMessage(); 
        // Decode done. Clear the input buffer
        inputString = "";
    }
  }
}

void decodeMessage()
{
    digitalWrite(LEDGREEN, LOW);
    Serial.println(inputString);
    char data[300];

    /*StaticJsonBuffer<370> jsonBuffer;
    JsonObject& root = jsonBuffer.parseObject(inputString);
    */
    const size_t bufferSize = 2*JSON_OBJECT_SIZE(1) + 2*JSON_OBJECT_SIZE(3) + JSON_OBJECT_SIZE(4) + 140;
    DynamicJsonBuffer jsonBuffer(bufferSize);

    JsonObject& root = jsonBuffer.parseObject(inputString);
   
    if (!root.success())
    {
        digitalWrite(LEDGREEN, LOW);
        digitalWrite(LEDRED, HIGH);
        // Return if parsing fails
   
        strcpy(data, "{\"parse\":\"failed\"}");
        client.publish(MQTT_SENSOR_TOPIC_WAREHOUSE, data, true);
        return;
    }
    else
    {
        digitalWrite(LEDGREEN, HIGH);
        digitalWrite(LEDRED, LOW);
        // Send JSON directly to MQTT
        root.printTo(data, root.measureLength() + 1);

        //strcpy(data, "{\"lock\":{\"state\":\"LOCK\"},\"door\":{\"state\":\"closed\"},\"temperature\":{\"warehouse\":\"20\",\"outside\":\"21\",\"sauna\":\"22\"},\"humidity\":{\"warehouse\":\"23\",\"outside\":\"24\",\"sauna\":\"25\"}}");
        Serial.println(client.publish(MQTT_SENSOR_TOPIC_WAREHOUSE, data, true));
        //sendMQTT(data);
    }
}

void sendMQTT(char* data)
{
    //client.publish(MQTT_SENSOR_TOPIC_WAREHOUSE, data, true);
}

Re: MQTT does not work if payload is over 107 characters

PostPosted: Sun Sep 03, 2017 10:28 am
by QuickFix
taikapanu wrote:Why it didn't work with the #define before #include?

I observed the same thing with another library (and a totally different project); apparently it isn't possible to re-declare a constant or definition.

In other languages, like for instance Pascal, it also isn't allowed, but then you'll get a hard error from the compiler and the code just doesn't work.

Re: MQTT does not work if payload is over 107 characters

PostPosted: Wed Jun 12, 2019 7:17 am
by JohnDoe23232
Hello all,

I'm quite late to the party. Anyway here goes nothing:
You would have to first un-define it:

Code: Select all#undef  MQTT_MAX_PACKET_SIZE // un-define max packet size
#define MQTT_MAX_PACKET_SIZE 500  // fix for MQTT client dropping messages over 128B


Thanks for the work beforehand!