From f9561b153d161e8551245038d797a36a6f6df880 Mon Sep 17 00:00:00 2001 From: Jeroen Domburg Date: Sat, 11 Oct 2014 22:15:37 +0200 Subject: [PATCH] Added heatshrink encoding to mkespfsimage. No decoding routines yet. --- .gitmodules | 3 + Makefile | 5 +- lib/heatshrink | 1 + mkespfsimage/Makefile | 5 +- mkespfsimage/espfsformat.h | 2 +- mkespfsimage/heatshrink_encoder.c | 3 + mkespfsimage/main.c | 112 +++++++++++++++++++++++++++--- 7 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 .gitmodules create mode 160000 lib/heatshrink create mode 100644 mkespfsimage/heatshrink_encoder.c diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..772d424 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lib/heatshrink"] + path = lib/heatshrink + url = https://github.com/atomicobject/heatshrink.git diff --git a/Makefile b/Makefile index 9c02f4d..16a517a 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ ESPTOOL ?= esptool.py ESPPORT ?= /dev/ttyUSB0 # name for the target project -TARGET = ting +TARGET = httpd # which modules (subdirectories) of the project to include in compiling MODULES = driver user @@ -141,7 +141,8 @@ mkespfsimage/mkespfsimage: mkespfsimage/ make -C mkespfsimage htmlflash: webpages.espfs - -$(ESPTOOL) --port $(ESPPORT) write_flash 0x20000 webpages.espfs + if [ $$(stat -c '%s' webpages.espfs) -gt $$(( 0x30000 )) ]; then echo "webpages.espfs too big!"; false; fi + -$(ESPTOOL) --port $(ESPPORT) write_flash 0x10000 webpages.espfs clean: $(Q) rm -f $(APP_AR) diff --git a/lib/heatshrink b/lib/heatshrink new file mode 160000 index 0000000..555f7cf --- /dev/null +++ b/lib/heatshrink @@ -0,0 +1 @@ +Subproject commit 555f7cf0b0a508c2f804d4fdf6c1fd0d92f9a798 diff --git a/mkespfsimage/Makefile b/mkespfsimage/Makefile index 07b61a8..71741f3 100644 --- a/mkespfsimage/Makefile +++ b/mkespfsimage/Makefile @@ -1,4 +1,5 @@ +CFLAGS=-I../lib/heatshrink -std=gnu99 -mkespfsimage: main.o - $(CC) -o mkespfsimage $< +mkespfsimage: main.o heatshrink_encoder.o + $(CC) -o $@ $^ diff --git a/mkespfsimage/espfsformat.h b/mkespfsimage/espfsformat.h index 10a7d2b..964d71c 100644 --- a/mkespfsimage/espfsformat.h +++ b/mkespfsimage/espfsformat.h @@ -17,7 +17,7 @@ with the FLAG_LASTFILE flag set. #define FLAG_LASTFILE (1<<0) #define COMPRESS_NONE 0 -#define COMPRESS_LZF 1 +#define COMPRESS_HEATSHRINK 1 typedef struct { int32_t magic; diff --git a/mkespfsimage/heatshrink_encoder.c b/mkespfsimage/heatshrink_encoder.c new file mode 100644 index 0000000..2c49a2d --- /dev/null +++ b/mkespfsimage/heatshrink_encoder.c @@ -0,0 +1,3 @@ +//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir + +#include "../lib/heatshrink/heatshrink_encoder.c" diff --git a/mkespfsimage/main.c b/mkespfsimage/main.c index d6a6caf..63be2b2 100644 --- a/mkespfsimage/main.c +++ b/mkespfsimage/main.c @@ -1,5 +1,5 @@ - #include +#include #include #include #include @@ -10,6 +10,12 @@ #include #include "espfsformat.h" +//Heatshrink +#include "heatshrink_common.h" +#include "heatshrink_config.h" +#include "heatshrink_encoder.h" + + //Routines to convert host format to the endianness used in the xtensa short htoxs(short in) { char r[2]; @@ -27,38 +33,95 @@ int htoxl(int in) { return *((int *)r); } +size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) { + char *inp=in; + char *outp=out; + int len; + int ws[]={13, 11, 8, 6, 5}; + int ls[]={4, 4, 4, 3, 3}; + HSE_poll_res pres; + HSE_sink_res sres; + size_t r=0; + if (level==-1) level=8; + level=(level-1)/2; //level is now 0, 1, 2, 3, 4 + heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]); + if (enc==NULL) { + perror("allocating mem for heatshrink"); + exit(1); + } + do { + if (insize>0) { + sres=heatshrink_encoder_sink(enc, inp, insize, &len); + if (sres!=HSER_SINK_OK) break; + inp+=len; insize-=len; + if (insize==0) heatshrink_encoder_finish(enc); + } + do { + pres=heatshrink_encoder_poll(enc, outp, outsize, &len); + if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break; + outp+=len; outsize-=len; + r+=len; + } while (pres==HSER_POLL_MORE); + } while (insize!=0); -void handleFile(int f, char *name) { - char *fdat; - off_t size; + if (insize!=0) { + fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres); + exit(1); + } + + heatshrink_encoder_free(enc); + return r; +} + +int handleFile(int f, char *name, int compression, int level) { + char *fdat, *cdat; + off_t size, csize; EspFsHeader h; int nameLen; size=lseek(f, 0, SEEK_END); fdat=mmap(NULL, size, PROT_READ, MAP_SHARED, f, 0); if (fdat==MAP_FAILED) { perror("mmap"); - return; + return 0; } + if (compression==COMPRESS_NONE) { + csize=size; + cdat=fdat; + } else if (compression==COMPRESS_HEATSHRINK) { + cdat=malloc(size*2); + csize=compressHeatshrink(fdat, size, cdat, size*2, level); + } else { + fprintf(stderr, "Unknown compression - %d\n", compression); + exit(1); + } + + if (csize>size) { + //Compressing enbiggened this file. Revert to uncompressed store. + csize=size; + cdat=fdat; + } + //Fill header data h.magic=('E'<<0)+('S'<<8)+('f'<<16)+('s'<<24); h.flags=0; - h.compression=COMPRESS_NONE; + h.compression=compression; nameLen=strlen(name)+1; if (nameLen&3) nameLen+=4-(nameLen&3); //Round to next 32bit boundary h.nameLen=htoxs(nameLen); - h.fileLenComp=htoxl(size); + h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); write(1, &h, sizeof(EspFsHeader)); write(1, name, nameLen); //ToDo: this can eat up a few bytes after the buffer. - write(1, fdat, size); + write(1, cdat, csize); //Pad out to 32bit boundary while (size&3) { write(1, "\000", 1); size++; } munmap(fdat, size); + return (csize*100)/size; } //Write final dummy header with FLAG_LASTFILE set. @@ -75,11 +138,37 @@ void finishArchive() { int main(int argc, char **argv) { - int f; + int f, x; char fileName[1024]; char *realName; struct stat statBuf; int serr; + int rate; + int err=0; + int compType=1; //default compression type - heatshrink + int compLvl=-1; + + for (x=1; x=x-2) { + compType=atoi(argv[x=1]); + x++; + } else if (strcmp(argv[x], "-l")==0 && argc>=x-2) { + compLvl=atoi(argv[x=1]); + if (compLvl<1 || compLvl>9) err=1; + x++; + } else { + err=1; + } + } + + if (err) { + fprintf(stderr, "%s - Program to create espfs images\n", argv[0]); + fprintf(stderr, "Usage: \nfind | %s [-c compressor] [-l compression_level] > out.espfs\n", argv[0]); + fprintf(stderr, "Compressors:\n0 - None\n1 - Heatshrink(defautl\n"); + fprintf(stderr, "Compression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n"); + exit(0); + } + while(fgets(fileName, sizeof(fileName), stdin)) { //Kill off '\n' at the end fileName[strlen(fileName)-1]=0; @@ -92,8 +181,8 @@ int main(int argc, char **argv) { if (realName[0]=='/') realName++; f=open(fileName, O_RDONLY); if (f>0) { - handleFile(f, realName); - fprintf(stderr, "%s\n", realName); + rate=handleFile(f, realName, compType, compLvl); + fprintf(stderr, "%s (%d%%)\n", realName, rate); close(f); } else { perror(fileName); @@ -105,5 +194,6 @@ int main(int argc, char **argv) { } } finishArchive(); + return 0; }