Issues with reading a file from web and parsing it

The use of the ESP8266 in the world of IoT
 Reply

User avatar

Issues with reading a file from web and parsing it

By mars000
- Mon Dec 12, 2016 1:56 am #59386 Hi
I'm trying to read a simple json format file from the web and parsing it. Ideally I'd like to read the entire file but haven't been able to sort out the memory issues so I just read in the first 2000 bytes and parse it. The problem is that sometimes client.available() sometimes returns 1418 and sometimes over 2000 (I don't know why) - can someone help ? Here is my code and outputs:

Code: Select all#include <ESP8266WiFi.h>
#include <WiFiClient.h>
// #include <ArduinoJson.h>
#include <SimpleTimer.h>

#define MAX_BYTES_TO_READ 2000

int n = 0;
 
// WiFi information
const char ssid[] = "xxxxx";
const char pass[] = "xxxxx";
 
// Remote site information
const char http_site[] = "www.bom.gov.au";
const int http_port = 80;
 
// Pin definitions
const int LED_PIN = 5;
 
// Global variables
WiFiClient client;

// char line[5000];
// String stringOne;
SimpleTimer timer;
int counter = 0;

void setup() {
 
  // Set up serial console to read web page
  Serial.begin(115200);
  Serial.println("Starting Thing GET Example");
 
  // Set up LED for debugging
 // pinMode(LED_PIN, OUTPUT);
 
  // Connect to WiFi
  connectWiFi();
 
  // Attempt to connect to website
 /*if ( !getPage() ) {
    Serial.println("GET request failed");
  } else { Serial.println ("Get request passed"); } */
  // get_observed_temp();
  timer.setInterval (10000L, get_observed_temp);
}


void get_observed_temp()
{
 char c;
 String stringOne = "";
   if (!getPage() ) {
    Serial.println("GET request failed");
  } else { Serial.println ("Get request passed");
  n = 0;
  // Lets read the json web file !
  Serial.print("Client available = ");
  Serial.println(client.available());

   while ((client.available()) and (n < MAX_BYTES_TO_READ)) {
  // while (n < MAX_BYTES_TO_READ) {
    c = client.read();
    Serial.print(c);
  //read char by char HTTP request - just read first 2000 characters only
    if (n < MAX_BYTES_TO_READ) {
    stringOne = stringOne + c;
    n++; }
  }  // end while loop - finished reading json file (at least the part we are interested in

  // ======== If the server has disconnected, stop the client and WiFi =======
  if (!client.connected() ) {    // should do this at end of everything
    Serial.println("Client disconnected !");
   
    // Close socket and wait for disconnect from WiFi
    client.stop();
    if ( WiFi.status() != WL_DISCONNECTED ) {
      WiFi.disconnect(); }
   
    // Do nothing
    Serial.println("Finished Thing GET test");
    Serial.println("*********************************");
    Serial.println ("here n = ");
    Serial.println (n);   
   
   
  //  return (-20);
  } // end server disconnected code
 
   if  ((client.available()) and (n == MAX_BYTES_TO_READ)) {
   // if  (n == MAX_BYTES_TO_READ) {
    Serial.println();
    Serial.println ("===============================================");
    Serial.println ("===============================================");
 
     if (stringOne.indexOf("Essendon Airport") > 0)
     {
     
      Serial.println ("Essendon Airport file !");
      int string_position_data = stringOne.indexOf("ID");     // get string position of term "ID"
      Serial.print ("Postion of ID in array = ");
      Serial.println (string_position_data);
   
      int string_position_data_1 = stringOne.indexOf("air_temp"); // get string position of term "Main ID"
      Serial.print ("Postion of air_temp in array = ");
      Serial.println (string_position_data_1);

      String air_temp;
      air_temp = stringOne.substring(string_position_data_1+11, string_position_data_1+18);
   
      Serial.print ("air_temp = ");
      Serial.println (air_temp);
 
      int air_temp_int = air_temp.toInt();

      Serial.print ("Air temp as an Integer = ");
      Serial.println(air_temp_int);
   
      Serial.println ("===============================================");
      Serial.println ("===============================================");
      Serial.println ("===============================================");

     }
     else { Serial.println ("Essendon Airport not found in file"); }
   }
    client.stop();
    Serial.println();
    Serial.print ("COUNTER = "); 
    Serial.println (counter++);
    Serial.print ("N = "); 
    Serial.println (n);
  }
}



void loop() {
   
   timer.run();
}

 
// Attempt to connect to WiFi
void connectWiFi() {
 
  byte led_status = 0;
 
   WiFi.begin(ssid, pass);                           // Connect to WiFi network
    Serial.println("Waiting to connect to Wifi:");
    while (WiFi.status() != WL_CONNECTED) {           // Wait for board to connect to WiFi network
      delay(500);                                   
      Serial.print("."); 
  }
}

 
// Perform an HTTP GET request to a remote page

bool getPage() {

    if (!client.connect(http_site, http_port)) {
    Serial.println("Connection failed :-(");
    return false;
  }

  Serial.println("Connected to host - sending request...");
 
  client.print(String("GET /fwo/IDV60901/IDV60901.95866.json HTTP/1.1\r\n") +
               "Host: " + http_site + "\r\n" +
               "Connection: close\r\n\r\n");
 
  Serial.println("Request sent - waiting for reply...");
  int timeout = millis() + 5000;
  while(client.available() == 0) {
    if (timeout - millis() < 0) {
        Serial.println(">>> Client Timeout !");
        client.stop();
        return false;
    } // end if
  } // end while
  return true;
}


SAMPLE OUTPUT:
Code: Select allConnected to host - sending request...
Request sent - waiting for reply...
Get request passed
Client available = 1418
HTTP/1.1 200 OK
Server: Apache/2.2.3 (Red Hat)
Last-Modified: Mon, 12 Dec 2016 06:51:27 GMT
ETag: "17d8263-20671-543708860d9c0"
Content-Type: application/json
Date: Mon, 12 Dec 2016 06:56:22 GMT
Transfer-Encoding:  chunked
Connection: close
Connection: Transfer-Encoding

00006000
{
   "observations": {
      "notice": [
         {
            "copyright": "Copyright Commonwealth of Australia 2016, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
            "copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
            "disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
            "feedback_url": "http://www.bom.gov.au/other/feedback"
         }
      ],
      "header": [
         {
            "refresh_message": "Issued at  5:51 pm EDT Monday 12 December 2016",
            "ID": "IDV60901",
            "main_ID": "IDV60900",
            "name": "Essendon Airport",
            "state_time_zone": "VIC",
            "time_zone": "EDT",
            "product_name": "Capital City Observations",
            "state": "Victoria"
         }
      ],
      "data": [
         {
            "sort_order": 0,
            "wmo": 95866,
            "name": "Essendon Airport",
            "history_product": "IDV60901",
            "local_date_time": "12/05:30pm",
            "local_date_time_full": "20161212173000",
            "aifstime_utc": "20161212063000",
            "lat": -37.7,
            "lon": 144.9,
            "apparent_t": 27.3,
            "cloud": "-",
            "cloud_base_m": null,
            "cloud_oktas": nu
COUNTER = 1257
N = 1418
 Reply
Return to IoT (Internet of Things)