reading json web file and parsing issue
Posted: Mon Dec 12, 2016 12:33 amHi
I want to read a json file from a website and parse the contents. Ideally I want to read all the file however I've settled to just being ok in reading first 2000 characters at it contains the info I need (and I haven't been able to work out how to read the entire file). I then simply search for "air_temp" value as per the code below. What I have found is that the value returned from client.available() varies from over 2400 to 1418 and it appears to be random (from what I can gather). I don't know why it would be doing this - hoping someone can help me out. And while at it is there an elegant way I can read the entire contents into a String or char[] that I can parse. Code below:
WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN NOT ENOUGH READ IN:
WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN ALL 2000 BYTES READ IN:
I want to read a json file from a website and parse the contents. Ideally I want to read all the file however I've settled to just being ok in reading first 2000 characters at it contains the info I need (and I haven't been able to work out how to read the entire file). I then simply search for "air_temp" value as per the code below. What I have found is that the value returned from client.available() varies from over 2400 to 1418 and it appears to be random (from what I can gather). I don't know why it would be doing this - hoping someone can help me out. And while at it is there an elegant way I can read the entire contents into a String or char[] that I can parse. Code below:
Code: Select all
#include <ESP8266WiFi.h>
#include <WiFiClient.h>
// #include <ArduinoJson.h>
#include <SimpleTimer.h>
#define MAX_BYTES_TO_READ 2000
int n = 0;
// WiFi information
const char ssid[] = "xxxxx";
const char pass[] = "xxxxx";
// Remote site information
const char http_site[] = "www.bom.gov.au";
const int http_port = 80;
// Pin definitions
const int LED_PIN = 5;
// Global variables
WiFiClient client;
// char line[5000];
// String stringOne;
SimpleTimer timer;
int counter = 0;
void setup() {
// Set up serial console to read web page
Serial.begin(115200);
Serial.println("Starting Thing GET Example");
// Set up LED for debugging
// pinMode(LED_PIN, OUTPUT);
// Connect to WiFi
connectWiFi();
// Attempt to connect to website
// get_observed_temp();
timer.setInterval (10000L, get_observed_temp);
}
void get_observed_temp()
{
char c;
String stringOne = "";
if (!getPage() ) {
Serial.println("GET request failed");
} else { Serial.println ("Get request passed");
n = 0;
// Lets read the json web file !
Serial.print("Client available = ");
Serial.println(client.available());
while ((client.available()) and (n < MAX_BYTES_TO_READ)) {
// while (n < MAX_BYTES_TO_READ) {
c = client.read();
Serial.print(c);
//read char by char HTTP request - just read first 2000 characters only
if (n < MAX_BYTES_TO_READ) {
stringOne = stringOne + c;
n++; }
} // end while loop - finished reading json file (at least the part we are interested in
// ======== If the server has disconnected, stop the client and WiFi =======
if (!client.connected() ) { // should do this at end of everything
Serial.println("Client disconnected !");
// Close socket and wait for disconnect from WiFi
client.stop();
if ( WiFi.status() != WL_DISCONNECTED ) {
WiFi.disconnect(); }
// Do nothing
Serial.println("Finished Thing GET test");
Serial.println("*********************************");
Serial.println ("here n = ");
Serial.println (n);
// return (-20);
} // end server disconnected code
if ((client.available()) and (n == MAX_BYTES_TO_READ)) {
// if (n == MAX_BYTES_TO_READ) {
Serial.println();
Serial.println ("===============================================");
Serial.println ("===============================================");
if (stringOne.indexOf("Essendon Airport") > 0)
{
Serial.println ("Essendon Airport file !");
int string_position_data = stringOne.indexOf("ID"); // get string position of term "ID"
Serial.print ("Postion of ID in array = ");
Serial.println (string_position_data);
int string_position_data_1 = stringOne.indexOf("air_temp"); // get string position of term "Main ID"
Serial.print ("Postion of air_temp in array = ");
Serial.println (string_position_data_1);
String air_temp;
air_temp = stringOne.substring(string_position_data_1+11, string_position_data_1+18);
Serial.print ("air_temp = ");
Serial.println (air_temp);
int air_temp_int = air_temp.toInt();
Serial.print ("Air temp as an Integer = ");
Serial.println(air_temp_int);
Serial.println ("===============================================");
Serial.println ("===============================================");
Serial.println ("===============================================");
}
else { Serial.println ("Essendon Airport not found in file"); }
}
client.stop();
Serial.println();
Serial.print ("COUNTER = ");
Serial.println (counter++);
Serial.print ("N = ");
Serial.println (n);
}
}
void loop() {
timer.run();
}
// Attempt to connect to WiFi
void connectWiFi() {
byte led_status = 0;
WiFi.begin(ssid, pass); // Connect to WiFi network
Serial.println("Waiting to connect to Wifi:");
while (WiFi.status() != WL_CONNECTED) { // Wait for board to connect to WiFi network
delay(500);
Serial.print(".");
}
}
// Perform an HTTP GET request to a remote page
bool getPage() {
if (!client.connect(http_site, http_port)) {
Serial.println("Connection failed :-(");
return false;
}
Serial.println("Connected to host - sending request...");
client.print(String("GET /fwo/IDV60901/IDV60901.95866.json HTTP/1.1\r\n") +
"Host: " + http_site + "\r\n" +
"Connection: close\r\n\r\n");
Serial.println("Request sent - waiting for reply...");
int timeout = millis() + 5000;
while(client.available() == 0) {
if (timeout - millis() < 0) {
Serial.println(">>> Client Timeout !");
client.stop();
return false;
} // end if
} // end while
return true;
}
WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN NOT ENOUGH READ IN:
Code: Select all
Connected to host - sending request...
Request sent - waiting for reply...
Get request passed
Client available = 1418
HTTP/1.1 200 OK
Server: Apache/2.2.3 (Red Hat)
Last-Modified: Mon, 12 Dec 2016 05:21:30 GMT
ETag: "17d8273-20d58-5436f46b12a80"
Content-Type: application/json
Date: Mon, 12 Dec 2016 05:28:12 GMT
Transfer-Encoding: chunked
Connection: close
Connection: Transfer-Encoding
00006000
{
"observations": {
"notice": [
{
"copyright": "Copyright Commonwealth of Australia 2016, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
"copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
"disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
"feedback_url": "http://www.bom.gov.au/other/feedback"
}
],
"header": [
{
"refresh_message": "Issued at 4:21 pm EDT Monday 12 December 2016",
"ID": "IDV60901",
"main_ID": "IDV60900",
"name": "Essendon Airport",
"state_time_zone": "VIC",
"time_zone": "EDT",
"product_name": "Capital City Observations",
"state": "Victoria"
}
],
"data": [
{
"sort_order": 0,
"wmo": 95866,
"name": "Essendon Airport",
"history_product": "IDV60901",
"local_date_time": "12/04:00pm",
"local_date_time_full": "20161212160000",
"aifstime_utc": "20161212050000",
"lat": -37.7,
"lon": 144.9,
"apparent_t": 26.0,
"cloud": "-",
"cloud_base_m": null,
"cloud_oktas": nu
COUNTER = 728
N = 1418
WHAT SERIAL THE OUTPUT LOOKS LIKE WHEN ALL 2000 BYTES READ IN:
Code: Select all
Connected to host - sending request...
Request sent - waiting for reply...
Get request passed
Client available = 5672
HTTP/1.1 200 OK
Server: Apache/2.2.3 (Red Hat)
Last-Modified: Mon, 12 Dec 2016 05:21:30 GMT
ETag: "17d8273-20d58-5436f46b12a80"
Content-Type: application/json
Date: Mon, 12 Dec 2016 05:27:22 GMT
Transfer-Encoding: chunked
Connection: close
Connection: Transfer-Encoding
00006000
{
"observations": {
"notice": [
{
"copyright": "Copyright Commonwealth of Australia 2016, Bureau of Meteorology. For more information see: http://www.bom.gov.au/other/copyright.shtml http://www.bom.gov.au/other/disclaimer.shtml",
"copyright_url": "http://www.bom.gov.au/other/copyright.shtml",
"disclaimer_url": "http://www.bom.gov.au/other/disclaimer.shtml",
"feedback_url": "http://www.bom.gov.au/other/feedback"
}
],
"header": [
{
"refresh_message": "Issued at 4:21 pm EDT Monday 12 December 2016",
"ID": "IDV60901",
"main_ID": "IDV60900",
"name": "Essendon Airport",
"state_time_zone": "VIC",
"time_zone": "EDT",
"product_name": "Capital City Observations",
"state": "Victoria"
}
],
"data": [
{
"sort_order": 0,
"wmo": 95866,
"name": "Essendon Airport",
"history_product": "IDV60901",
"local_date_time": "12/04:00pm",
"local_date_time_full": "20161212160000",
"aifstime_utc": "20161212050000",
"lat": -37.7,
"lon": 144.9,
"apparent_t": 26.0,
"cloud": "-",
"cloud_base_m": null,
"cloud_oktas": null,
"cloud_type_id": null,
"cloud_type": "-",
"delta_t": 15.4,
"gust_kmh": 37,
"gust_kt": 20,
"air_temp": 32.3,
"dewpt": 1.4,
"press": 1014.4,
"press_qnh": 1014.5,
"press_msl": 1014.4,
"press_tend": "-",
"rain_trace": "0.0",
"rel_hum": 14,
"sea_state": "-",
"swell_dir_worded": "-",
"swell_height": null,
"swell_period": null,
"vis_km": "10",
"weather": "-",
"wind_dir": "NNW",
"wind_spd_kmh": 24,
"wind_spd_kt": 13
},
{
"sort_order": 1,
"wmo": 95866,
"name": "Essendon Airpor
===============================================
===============================================
Essendon Airport file !
Postion of ID in array = 836
Postion of air_temp in array = 1537
air_temp = 32.3,
Air temp as an Integer = 32
===============================================
===============================================
===============================================
COUNTER = 723
N = 2000