Trying out transparent decompression using heatshrink... doesn't completely work on the esp8166 itself yet.

include/httpdconfig.h

@@ -0,0 +1,6 @@
+
+//Define this if you want to be able to use Heatshrink-compressed espfs images.
+#define EFS_HEATSHRINK
+
+//Pos of esp fs in flash
+#define ESPFS_POS 0x12000

user/espfs.c

@@ -4,14 +4,37 @@ mkespfsimg tool, and can use that block to do abstracted operations on the files
 It's written for use with httpd, but doesn't need to be used as such.
 */
 
+#ifdef __ets__
+//esp build
 #include "driver/uart.h"
 #include "c_types.h"
 #include "user_interface.h"
 #include "espconn.h"
 #include "mem.h"
 #include "osapi.h"
+#else
+//Test build
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#define os_malloc malloc
+#define os_free free
+#define os_memcpy memcpy
+#define os_strncmp strncmp
+#define os_strcmp strcmp
+#define os_strcpy strcpy
+#define os_printf printf
+#define ICACHE_FLASH_ATTR
+extern char* espFsData;
+#endif
+
 #include "../mkespfsimage/espfsformat.h"
 #include "espfs.h"
+#include "httpdconfig.h"
+#ifdef EFS_HEATSHRINK
+#include "heatshrink_decoder.h"
+#endif
 
 struct EspFsFile {
 	EspFsHeader *header;
@@ -25,7 +48,8 @@ struct EspFsFile {
 /*
 Available locations, at least in my flash, with boundaries partially guessed:
 0x00000 (0x10000): Code/data (RAM data?)
-0x10000 (0x30000): Free (filled with zeroes) (parts used by ESPCloud and maybe SSL)
+0x10000 (0x02000): Gets erased by something?
+0x12000 (0x2E000): Free (filled with zeroes) (parts used by ESPCloud and maybe SSL)
 0x40000 (0x20000): Code/data (ROM data?)
 0x60000 (0x1C000): Free
 0x7c000 (0x04000): Param store
@@ -56,7 +80,11 @@ void ICACHE_FLASH_ATTR memcpyAligned(char *dst, char *src, int len) {
 
 
 EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
+#ifdef __ets__
 	char *p=(char *)(ESPFS_POS+0x40200000);
+#else
+	char *p=espFsData;
+#endif
 	char *hpos;
 	char namebuf[256];
 	EspFsHeader h;
@@ -67,14 +95,17 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
 	while(1) {
 		hpos=p;
 		os_memcpy(&h, p, sizeof(EspFsHeader));
-		//ToDo: check magic
+		if (h.magic!=0x73665345) {
+			os_printf("Magic mismatch. EspFS image broken.\n");
+			return NULL;
+		}
 		if (h.flags&FLAG_LASTFILE) {
-//			os_printf("End of archive.\n");
+			os_printf("End of image.\n");
 			return NULL;
 		}
 		p+=sizeof(EspFsHeader);
 		os_memcpy(namebuf, p, sizeof(namebuf));
-//		os_printf("Found file %s . Namelen=%x fileLen=%x\n", namebuf, h.nameLen,h.fileLenComp);
+		os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", namebuf, h.nameLen,h.fileLenComp,h.compression,h.flags);
 		if (os_strcmp(namebuf, fileName)==0) {
 			p+=h.nameLen;
 			r=(EspFsFile *)os_malloc(sizeof(EspFsFile));
@@ -84,8 +115,22 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
 			r->posComp=p;
 			r->posStart=p;
 			r->posDecomp=0;
+			if (h.compression==COMPRESS_NONE) {
 				r->decompData=NULL;
-			//ToDo: Init any decompressor
+#ifdef EFS_HEATSHRINK
+			} else if (h.compression==COMPRESS_HEATSHRINK) {
+				char parm;
+				//Decoder params are stored in 1st byte.
+				memcpyAligned(&parm, r->posComp, 1);
+				r->posComp++;
+				os_printf("Heatshrink compressed file; decode parms = %x\n", parm);
+				r->decompData=(heatshrink_decoder *)heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf);
+				os_printf("Decompressor allocated.\n");
+#endif
+			} else {
+				os_printf("Invalid compression: %d\n", h.compression);
+				return NULL;
+			}
 			return r;
 		}
 		//Skip name and file
@@ -97,22 +142,63 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
 
 
 int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) {
+	int flen;
 	if (fh==NULL) return 0;
+		memcpyAligned((char*)&flen, (char*)&fh->header->fileLenComp, 4);
 	if (fh->decompressor==COMPRESS_NONE) {
 		int toRead;
-		toRead=fh->header->fileLenComp-(fh->posComp-fh->posStart);
+		toRead=flen-(fh->posComp-fh->posStart);
 		if (len>toRead) len=toRead;
-//		os_printf("Reading %d bytes from %x\n", len, fh->posComp);
+		os_printf("Reading %d bytes from %x\n", len, fh->posComp);
 		memcpyAligned(buff, fh->posComp, len);
 		fh->posDecomp+=len;
 		fh->posComp+=len;
 //		os_printf("Done reading %d bytes, pos=%x\n", len, fh->posComp);
 		return len;
+#ifdef EFS_HEATSHRINK
+	} else if (fh->decompressor==COMPRESS_HEATSHRINK) {
+		int decoded=0;
+		int elen, rlen, r;
+		char ebuff[16];
+		os_printf("heatshrink: reading\n");
+		heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData;
+		while(decoded<len) {
+			//Feed data into the decompressor
+			elen=flen-(fh->posComp - fh->posStart);
+			if (elen==0) return decoded; //file is read
+			if (elen>0) {
+				os_printf("heatshrink: feeding decoder (%d comp bytes left)\n", elen);
+				memcpyAligned(ebuff, fh->posComp, 16);
+				for (r=0; r<16; r++) os_printf("%02hhx ", ebuff[r]);
+				os_printf("\n");
+				r=heatshrink_decoder_sink(dec, ebuff, (elen>16)?16:elen, &rlen);
+				os_printf("heatshrink: decoder ate %d bytes (code %d)\n", rlen, r);
+				fh->posComp+=rlen;
+				if (rlen==elen) {
+					os_printf("heatshrink: finish\n");
+					heatshrink_decoder_finish(dec);
 				}
+			}
+			//Grab decompressed data and put into buff
+			r=heatshrink_decoder_poll(dec, buff, len-decoded, &rlen);
+			os_printf("heatshrink: decoder emitted %d bytes (code %d)\n", rlen, r);
+			fh->posDecomp+=rlen;
+			buff+=rlen;
+			decoded+=rlen;
+		}
+		return len;
+#endif
+	}
+	return 0;
 }
 
 void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) {
 	if (fh==NULL) return;
+#ifdef EFS_HEATSHRINK
+	if (fh->decompressor==COMPRESS_HEATSHRINK) {
+		heatshrink_decoder_free((heatshrink_decoder*)fh->decompData);
+	}
+#endif
 	os_free(fh);
 }
 

user/espfs.h

@@ -1,8 +1,6 @@
 #ifndef ESPFS_H
 #define ESPFS_H
 
-//Pos of esp fs in flash
-#define ESPFS_POS 0x20000
 
 
 typedef struct EspFsFile EspFsFile;