ESP8266 Community Forum

- Mon Mar 20, 2017 12:57 pm #64015 Oh and by the way, I ran into similar problems using the SD server code with the large files. Sometimes they don't finish. I tinkered with some ways of using the javascript to pre-load stuff and re-try on failures (from a small footprint start point) but that's a really cludgy way of doing it. I heard that some of the advanced server code such as the async servers don't have as much of a problem with failures and incompletes. I haven't learned enough about the coding to try implementing one though.

- Tue Mar 21, 2017 11:00 pm #64074 By the way, I'm not sure what the status was of the web server code when you wrote this, but using your loadFromFlash as is was creating duplicate headers for Content-Length. The code on my system at the moment allows you to set the content length prior to the send() call then it conditionally builds the content length either from a prior set value (with setContentLength(size_t) ) or from the length of the string sent.
So I replaced your server.sendHeader("Content-Length", String(len)) with server.setContentLength(len)

- Tue Mar 21, 2017 11:08 pm #64076 I ended up throwing in my gzipped, minified files and got it to work like a champ with a few modifications to the header file when it was done. (I modified the script initially to gzip -cd the file before passing it through the 'file' utility to try to determine the mimetype, but it seems to fail on the minified css and just sees it as text. So I'll play with that some further. But I can confirm the code also works by including the full server path [relative to server root] for the filename if the change the path.endsWidth(filename) condition to path == filename.
Mind you, for anyone trying this it means that along with using the setContentLength(len) I mentioned above, you also need to make sure the mime-type represents the original Content type before compressing for the browser to handle it properly, then add another header line for the encoding so it knows it's compressed data:

Code: Select allbool loadFromFlash(String path) {
   if(path.endsWith("/")) path += "index.htm";

   int NumFiles = sizeof(files)/sizeof(struct t_websitefiles);
   
   for(int i=0; i<NumFiles; i++) {
      if(path == String(files[i].filename)) {
         _FLASH_ARRAY<uint8_t>* filecontent;
         String dataType = "text/plain";
         unsigned int len = 0;
         
         dataType = files[i].mime;
         len = files[i].len;

         webServer.sendHeader("Content-Encoding", "gzip");
         webServer.setContentLength(len);
         webServer.send(200, files[i].mime, "");
         
         filecontent = (_FLASH_ARRAY<uint8_t>*)files[i].content;
         
         filecontent->open();
         
         WiFiClient client = webServer.client();
         client.write(*filecontent, 100);
         
         return true;
      }
   }
   
   return false;
}


This is what my test header file looks like including the gzipped binary data for the minified /index.htm and the minified /css/main.css files. (I had an image in there also but took it out for the moment while testing so you'll notice the broken link if you try using this data)

Code: Select all//
// converted websites to mainly flash variables
//

#include "Flash.h"

FLASH_ARRAY(uint8_t, file_0,
  0x1f, 0x8b, 0x08, 0x08, 0x25, 0x38, 0xcf, 0x58, 0x02, 0x03, 0x69, 0x6e,
  0x64, 0x65, 0x78, 0x2e, 0x6d, 0x69, 0x6e, 0x2e, 0x68, 0x74, 0x6d, 0x00,
  0x5d, 0x8d, 0xb1, 0x6a, 0xc4, 0x30, 0x10, 0x44, 0x7b, 0x7d, 0xc5, 0xe6,
  0x03, 0xce, 0x4a, 0xc0, 0x98, 0x14, 0x2b, 0x35, 0x97, 0x40, 0xba, 0x4b,
  0x61, 0x08, 0x29, 0x65, 0x7b, 0x2d, 0x2d, 0xb7, 0xb2, 0x8d, 0xb4, 0xc9,
  0x91, 0xbf, 0x8f, 0x42, 0xba, 0x83, 0x61, 0x18, 0x1e, 0x33, 0x0c, 0x3e,
  0xbc, 0x5c, 0xce, 0xe3, 0xe7, 0xfb, 0x2b, 0x24, 0xcd, 0xe2, 0xf1, 0xcf,
  0x41, 0xc2, 0x16, 0x1d, 0x6d, 0x1e, 0x33, 0x69, 0x80, 0x39, 0x85, 0x52,
  0x49, 0xdd, 0x97, 0xae, 0xa7, 0x67, 0x8f, 0xca, 0x2a, 0xe4, 0xdf, 0x48,
  0x64, 0x07, 0xf3, 0xb1, 0x17, 0x59, 0xd0, 0xfe, 0x33, 0x14, 0xde, 0xae,
  0x90, 0x0a, 0xad, 0x6e, 0xae, 0xd5, 0xe6, 0xc0, 0x5b, 0xd7, 0x02, 0x14,
  0x12, 0x57, 0xf5, 0x47, 0xa8, 0x26, 0x22, 0x6d, 0x1f, 0x4f, 0x77, 0xf3,
  0x06, 0xf0, 0xf0, 0x63, 0xe2, 0x0a, 0x4d, 0x01, 0x6e, 0x34, 0xc1, 0x11,
  0x22, 0x75, 0x68, 0x0f, 0x8f, 0x9c, 0x23, 0x4c, 0x7b, 0x59, 0xa8, 0xb8,
  0x47, 0x48, 0xc4, 0x31, 0xa9, 0x1b, 0x7a, 0x30, 0xb5, 0xcc, 0x8e, 0x73,
  0xab, 0x55, 0xfb, 0x3d, 0x45, 0xda, 0x87, 0xbe, 0x8b, 0xbc, 0xc2, 0x8d,
  0x17, 0x4d, 0xad, 0xe0, 0x8d, 0xf9, 0x05, 0x3d, 0x2a, 0xb3, 0xe0, 0xdd,
  0x00, 0x00, 0x00
);

FLASH_ARRAY(uint8_t, file_1,
  0x1f, 0x8b, 0x08, 0x08, 0x48, 0x37, 0xcf, 0x58, 0x02, 0x03, 0x6d, 0x61,
  0x69, 0x6e, 0x2e, 0x6d, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x00, 0x75,
  0x52, 0xcb, 0x8e, 0xdb, 0x30, 0x0c, 0xbc, 0xef, 0x8f, 0xec, 0x65, 0x02,
  0x74, 0xdb, 0x6e, 0x0f, 0xce, 0xd7, 0x50, 0x16, 0x6d, 0xb1, 0x91, 0x25,
  0x45, 0xa2, 0x0c, 0x04, 0x46, 0xfe, 0xbd, 0xb4, 0x9b, 0x2c, 0x76, 0x0b,
  0x14, 0x90, 0xcc, 0x31, 0x29, 0xf1, 0x31, 0x23, 0x02, 0x39, 0x57, 0x41,
  0x63, 0xcd, 0xe9, 0xb6, 0x80, 0xbc, 0xaf, 0xdc, 0x1a, 0xa8, 0x94, 0xc8,
  0x0a, 0xaa, 0x2a, 0x63, 0x64, 0x50, 0x13, 0x6f, 0xdf, 0xee, 0x25, 0xc3,
  0xc1, 0xc9, 0x0c, 0x17, 0xf3, 0x78, 0xb9, 0xf6, 0xac, 0x0c, 0x97, 0xfd,
  0x0d, 0x23, 0xa5, 0x95, 0x9a, 0x99, 0xa2, 0x92, 0x13, 0x46, 0x4e, 0xca,
  0x15, 0xa3, 0x58, 0x7c, 0xcc, 0x76, 0xd7, 0x7b, 0x78, 0x8e, 0xb6, 0x95,
  0x24, 0x36, 0xf8, 0x29, 0xc1, 0xcb, 0x0a, 0x6f, 0x2e, 0x05, 0x2f, 0xb6,
  0x1c, 0x7b, 0x4c, 0xc2, 0xd1, 0x37, 0x2b, 0x3c, 0xc9, 0xfc, 0x4c, 0x65,
  0xb0, 0x57, 0xc6, 0x94, 0xf3, 0x9e, 0x71, 0xca, 0x75, 0x41, 0x78, 0x43,
  0xf8, 0x8e, 0xf0, 0x03, 0xe1, 0x27, 0xc2, 0x3b, 0xc2, 0x2f, 0x04, 0x26,
  0x6f, 0xd1, 0x30, 0xd7, 0xdc, 0x0b, 0x82, 0x2e, 0x11, 0x02, 0x99, 0x2a,
  0x2d, 0x0c, 0x59, 0x66, 0x48, 0x6a, 0xb8, 0x38, 0x8f, 0x48, 0xce, 0xba,
  0x88, 0x3c, 0x73, 0xb2, 0x1f, 0xc1, 0x42, 0xf5, 0x82, 0x85, 0x53, 0x47,
  0xa2, 0x15, 0xd9, 0xfd, 0xe6, 0x51, 0x91, 0x23, 0x72, 0xd7, 0xd2, 0x15,
  0x05, 0xc5, 0x4a, 0x5f, 0x51, 0xbb, 0xbb, 0xa1, 0xa1, 0xd1, 0x52, 0xd0,
  0xec, 0xc8, 0xde, 0x57, 0x5b, 0x28, 0x46, 0xb4, 0x42, 0x06, 0xb5, 0xca,
  0x85, 0x77, 0x93, 0xd3, 0x8c, 0xd6, 0x9d, 0xed, 0xc5, 0x52, 0xdb, 0x1d,
  0xeb, 0x46, 0xc9, 0x19, 0x85, 0x7a, 0xd0, 0xa4, 0x1e, 0xba, 0x4f, 0x02,
  0x0d, 0xb6, 0xac, 0x69, 0xa8, 0x58, 0x8b, 0x5a, 0xa1, 0x8a, 0x8e, 0x1e,
  0xb1, 0x52, 0xc5, 0x6a, 0x6c, 0xe7, 0xcd, 0x12, 0xcc, 0x92, 0x86, 0x6f,
  0xe7, 0x62, 0xaa, 0x48, 0x9a, 0x0d, 0xb9, 0x5c, 0x6d, 0x4c, 0x03, 0x53,
  0x4e, 0x3a, 0x48, 0x0a, 0x5c, 0x45, 0xcf, 0x2b, 0xef, 0x32, 0x51, 0x3c,
  0x51, 0x94, 0x39, 0x0d, 0x8e, 0x1a, 0x47, 0x49, 0x7c, 0xff, 0xaa, 0xde,
  0x93, 0xfa, 0xff, 0x52, 0xfb, 0x95, 0xc3, 0x0f, 0x52, 0x1e, 0xf3, 0x6e,
  0x5e, 0x5a, 0x89, 0x74, 0x1b, 0x0e, 0xe5, 0xef, 0xfb, 0x34, 0xdb, 0x5e,
  0xe5, 0x14, 0x58, 0xe6, 0xa0, 0xc3, 0xdb, 0xdd, 0x68, 0xeb, 0xd1, 0x7c,
  0x4d, 0x4f, 0x4d, 0x6f, 0x91, 0x87, 0x94, 0xad, 0x89, 0x4f, 0x0f, 0xe5,
  0xba, 0x1d, 0xb6, 0xfd, 0x1b, 0x18, 0x68, 0xda, 0xeb, 0x7f, 0x72, 0x38,
  0x36, 0x9d, 0xed, 0xc2, 0x23, 0x72, 0x7d, 0x38, 0xb6, 0xd1, 0xc6, 0xb6,
  0x97, 0x35, 0xbc, 0xbe, 0x9e, 0x9f, 0xf0, 0xc8, 0x75, 0x50, 0xbc, 0xfd,
  0x65, 0xe7, 0x34, 0xe6, 0x18, 0xa9, 0x34, 0x1e, 0x9e, 0xe0, 0x41, 0xdb,
  0xc9, 0xb4, 0x1a, 0x0f, 0x1e, 0xef, 0x2f, 0x2f, 0x7f, 0x00, 0xfa, 0xbf,
  0x99, 0x56, 0xf8, 0x02, 0x00, 0x00
);

struct t_websitefiles {
  const char* filename;
  const char* mime;
  const unsigned int len;
  const _FLASH_ARRAY<uint8_t>* content;
} files[] = {
  {
    .filename = "/index.htm",
    .mime = "text/html",
    .len = 207,
    .content = &file_0
  },
  {
    .filename = "/css/main.css",
    .mime = "text/css",
    .len = 450,
    .content = &file_1
  },
};


So now I'm off to try to figure out how to get some command line minifiers for htm/css/js files so I can modify the script some more to compress down a whole website tree into the header file. I'll ultimately replace your 'ls -1' with a find . -type f instead and modify the later code to parse the full path to fill in the values automatically. I'm thinking a number of case/switch type conditions for certain file types otherwise handle them default with your existing checks. Woo hoo - thank you for this example code. It saves me a ton of time with what I was trying to do and opens up a ton of possibilities without needing to use an external memory chip or tf card.

- Wed Mar 22, 2017 5:45 pm #64111 One more follow up for anyone that might find this useful. I modified the struct a bit so it reflects 'path' rather than 'filename' and I added a string for encoding ('enc') so I can mark when a file is gzipped or not. I went back to perl for writing the script to generate the header file as there were existing minifier libraries on cpan (although not the greatest I've see - I may play around with the node.js command line minifier code)

The perl script is crude (I used to be a perl expert but haven't used in in more than 6 years presently) and relised on the HTML, CSS and JavaScript Packer libraries for minifying those three file types. After minifying I use Faster::Gzip to compress them. I also compress down anything that shows up at text/plain.

I decided not to do anything fancy with the mime type detection and it instead expects you are using standard file extensions and determines the mime type from the file name extension using MIME::Types. File reads/writes are done with File::Slurp calls with the exception of the streaming input from the xxd command (the perl xxd library didn't have the -i formatting and I didn't find anything else right away that was as clean and suitable to the c++ needs)

I ended up doing the file collection with a simple streaming open() handle to the unix find command. Syntax should be similar in cygwin. It seeks out files specifically in the $webdir folder. A match is any file with a a standard 1+ character filename with a 2-4 character extension after the '.'
The sub folder name is chopped off of the find results to create the resulting path used by the code on the esp.

hexify.pl:

Code: Select all#!/usr/bin/perl

use CSS::Packer;
use JavaScript::Packer;
use HTML::Packer;
use File::Slurp;
use Gzip::Faster;
use MIME::Types;


my $outfile = "webfiles.h";
my $webdir = "www";
my $find_exe = "/usr/bin/find";
my $xxdexe = "/usr/bin/xxd -i";
my $findcmd = $find_exe . " " . $webdir . " -type f ";

my $mt = MIME::Types->new();
my $files = open(FINDSTRM, "$findcmd|") || die $!;
my @filelist;
while($line = <FINDSTRM>) {
   chomp($line);
   if($line =~ /^$webdir(.*)\/([^\.\/]+)\.(\w{2,4})$/) {
      my %fileinfo;
      $fileinfo{path} = "$1/$2.$3";
      $fileinfo{dir} = $1;
      $fileinfo{name} = $2;
      $fileinfo{ext} = $3;
      my $type =  $mt->mimeTypeOf($fileinfo{ext});
      $fileinfo{type} = $type->{MT_type};

      my $bindat;
      if($type->{MT_type} eq "text/html") {
         print "html minifying $line\n";
         my $html = read_file($line);
         my $packer = HTML::Packer->init();
         my $htmin = $packer->minify( \$html, { remove_newlines => 1, remove_comments => 1, do_javascript => 'best', do_stylesheet => 'minify' } );
         $bindat = gzip($htmin);
      }
      if($type->{MT_type} eq "text/css") {
         print "css minifying $line\n";
         my $css = read_file($line);
         my $packer = CSS::Packer->init();
         my $cssmin = $packer->minify(\$css, { compress => 'minify', remove_comments => 1 } );
         $bindat = gzip($cssmin);
      }
      if($type->{MT_type} eq "application/javascript") {
         print "javascript minifying $line\n";
         my $js = read_file($line);
         my $packer = JavaScript::Packer->init();
         my $jsmin = $packer->minify(\$js, { compress => 'best', remove_comments => 1, remove_copyright => 1 } );
         $bindat = gzip($jsmin);
      }

      # other text based formats
      if( $type->{MT_type} eq "text/plain" ) {
         print "compressing text in $line\n";
         my $asc = read_file($line);
         $bindat = gzip($asc);
      }

      if(length($bindat) == 0) {
         print "raw reading $line\n";
         $bindat = read_file($line, { binmode => ':raw' });
         $fileinfo{enc} = '';
      } else {
         $fileinfo{enc} = 'gzip';
      }

      {
         use bytes;
         $fileinfo{len} = length($bindat);
      }
      $tfile = "/tmp/tmpfile";
      write_file( $tfile, {binmode => ':raw'}, $bindat );
      open(INXXD, "$xxdexe < $tfile |") || die $!;
      $fileinfo{hexdat} = do { local $/; <INXXD> };
      close INXXD;
      $fileinfo{hexdat} =~ s/^\s+|\s+$//g;
      push(@filelist,\%fileinfo);
  }
}
close FINDSTRM;

open(OUTFILE, ">", $outfile) || die $!;
print OUTFILE "\n#include \"Flash.h\"\n\n";

$count = 1;
foreach $f (@filelist) {
   print OUTFILE "FLASH_ARRAY(uint8_t, file_" . $count . ",\n";
   print OUTFILE "  " . $f->{hexdat};
   print OUTFILE ");\n\n";
   $count++;
}

print OUTFILE "struct t_websitefiles {\n";
print OUTFILE "  const char* path;\n";
print OUTFILE "  const char* mime;\n";
print OUTFILE "  const unsigned int len;\n";
print OUTFILE "  const char* enc;\n";
print OUTFILE "  const _FLASH_ARRAY<uint8_t>* content;\n";
print OUTFILE "} files[] = {\n";

$count = 1;
foreach $f (@filelist) {
   print OUTFILE "  {\n";
   print OUTFILE "    .path = \"" . $f->{path} . "\",\n";
   print OUTFILE "    .mime = \"" . $f->{type} . "\",\n";
   print OUTFILE "    .len = " . $f->{len} . ",\n";
   print OUTFILE "    .enc = \"" . $f->{enc} . "\",\n";
   print OUTFILE "    .content = \&file_" . $count . ",\n";
   print OUTFILE "  },\n";
   $count++;
}

print OUTFILE "};\n";
close OUTFILE;


Since I changed the one variable from filename to path in the struct and added the enc type, there are a couple other mods to the c++ loadFromFlash function also:

Code: Select allbool loadFromFlash(String path) {
   if(path.endsWith("/")) path += "index.htm";

   int NumFiles = sizeof(files)/sizeof(struct t_websitefiles);
   
   for(int i=0; i<NumFiles; i++) {
      if(path == String(files[i].path)) {
         _FLASH_ARRAY<uint8_t>* filecontent;
         String dataType = "text/plain";
         unsigned int len = 0;
         
         dataType = files[i].mime;
         len = files[i].len;

            if(files[i].enc != "")
            webServer.sendHeader("Content-Encoding", files[i].enc);
         webServer.setContentLength(len);
         webServer.send(200, files[i].mime, "");
         
         filecontent = (_FLASH_ARRAY<uint8_t>*)files[i].content;
         
         filecontent->open();
         
         WiFiClient client = webServer.client();
         client.write(*filecontent, 100);
         
         return true;
      }
   }
   
   return false;
}