From e8964de1d3e3af1a3953a6ff53905b71ef9c8efd Mon Sep 17 00:00:00 2001
From: Thorsten von Eicken <tve@voneicken.com>
Date: Mon, 6 Jan 2020 22:24:59 -0800
Subject: [PATCH] generate espfs image and compile as C file

---
 mkespfs.py      | 87 ++++++++++++++++++++++++++++---------------------
 pio-genespfs.py | 23 +++++++++++++
 platformio.ini  | 12 ++++---
 src/main.c      |  8 ++---
 4 files changed, 82 insertions(+), 48 deletions(-)
 create mode 100644 pio-genespfs.py

diff --git a/mkespfs.py b/mkespfs.py
index 5c9d4a2..5368b70 100755
--- a/mkespfs.py
+++ b/mkespfs.py
@@ -1,43 +1,56 @@
 #! /usr/bin/python3
+# Given a directory name as argument, this script generates an espfs filesystem image
+# with all the html, js, css, png, and ico files found in the directory tree.
+# Each file is compressed with gzip so it can be served up in compressed form from the esp8266.
 
 from sys import argv, exit, stderr, stdout
 from pathlib import Path
 import re, gzip, struct
 
-MAGIC = 0x73665345
-FL_GZIP = 2 # gzipped file flag
-FL_LAST = 1 # last entry file flag
-
-if len(argv) != 2 or not Path(argv[1]).is_dir():
-    print("Usage: {} directory".format(argv[0]), file=stderr)
-    exit(1)
-dir = argv[1]
-
-f_html = list(Path(dir).rglob('*.html'))
-f_css = list(Path(dir).rglob('*.css'))
-f_js = list(Path(dir).rglob('*.js'))
-f_img = list(Path(dir).rglob('*.ico')) + list(Path(dir).rglob('*.png'))
-f_all = f_html + f_css + f_js + f_img
-
-for fn in f_all:
-    if not fn.is_file(): continue
-    out_path = fn.relative_to(dir).as_posix().encode('ascii')
-    info = fn.stat()
-    data_un = fn.read_bytes()
-    data_comp = gzip.compress(data_un)
-
-    print("Processing {} -> {}[{}], {}->{} bytes".format(fn, out_path, len(out_path), info.st_size,
-        len(data_comp)), file=stderr)
-
-    header = struct.pack('<IBBHII', MAGIC, FL_GZIP, 0, len(out_path), len(data_comp), info.st_size)
-    stdout.buffer.write(header)
-    stdout.buffer.write(out_path)
-    if len(out_path)%4 != 0:
-        stdout.buffer.write(bytes(4)[:4-len(out_path)%4])
-    stdout.buffer.write(data_comp)
-    if len(data_comp)%4 != 0:
-        stdout.buffer.write(bytes(4)[:4-len(data_comp)%4])
-
-trailer = struct.pack('<IBBHII', MAGIC, FL_LAST, 0, 0, 0, 0)
-
-print("DONE", file=stderr)
+def mkespfs(dir, outbuf):
+    MAGIC = 0x73665345
+    FL_GZIP = 2 # gzipped file flag
+    FL_LAST = 1 # last entry file flag
+
+    f_html = list(Path(dir).rglob('*.html'))
+    f_css = list(Path(dir).rglob('*.css'))
+    f_js = list(Path(dir).rglob('*.js'))
+    f_img = list(Path(dir).rglob('*.ico')) + list(Path(dir).rglob('*.png'))
+    f_all = f_html + f_css + f_js + f_img
+    sz = 0
+
+    for fn in f_all:
+        if not fn.is_file(): continue
+        out_path = fn.relative_to(dir).as_posix().encode('ascii')
+        info = fn.stat()
+        data_un = fn.read_bytes()
+        data_comp = gzip.compress(data_un)
+
+        #print("Processing {} -> {}[{}], {}->{} bytes".format(fn, out_path, len(out_path), info.st_size,
+        #    len(data_comp)), file=stderr)
+
+        header = struct.pack('<IBBHII', MAGIC, FL_GZIP, 0, len(out_path), len(data_comp), info.st_size)
+        outbuf.write(header)
+        sz += len(header)
+        outbuf.write(out_path)
+        sz = (sz+len(out_path)+3) // 4 * 4
+        if len(out_path)%4 != 0:
+            outbuf.write(bytes(4)[:4-len(out_path)%4])
+        outbuf.write(data_comp)
+        sz += len(data_comp)
+        if len(data_comp)%4 != 0:
+            outbuf.write(bytes(4)[:4-len(data_comp)%4])
+        sz = (sz+3) // 4 * 4
+
+    trailer = struct.pack('<IBBHII', MAGIC, FL_LAST, 0, 0, 0, 0)
+    outbuf.write(trailer)
+    sz += len(trailer)
+
+    print("espfs image: {} bytes".format(sz), file=stderr)
+
+if __name__ == "__main__":
+    if len(argv) != 2 or not Path(argv[1]).is_dir():
+        print("Usage: {} directory".format(argv[0]), file=stderr)
+        exit(1)
+    mkespfs(argv[1], stdout.buffer)
+
diff --git a/pio-genespfs.py b/pio-genespfs.py
new file mode 100644
index 0000000..4aff19f
--- /dev/null
+++ b/pio-genespfs.py
@@ -0,0 +1,23 @@
+# pio-genespfs is a platformio extra script that uses mkespfs to generate a filesystem image
+# that it then converts into a C file so all the data can be included into the flash image.
+
+Import("env")
+#Import("projenv")
+
+from mkespfs import mkespfs
+from io import BytesIO
+from pathlib import Path
+
+dir = "html"
+espfile = "src/espfs_img.c"
+
+buf = BytesIO()
+espfsimg = mkespfs(dir, buf)
+
+fd = Path(espfile).open(mode='w')
+fd.write("unsigned char espfs_image[] __attribute__((section(\".irom.text\"))) = {");
+for i, b in enumerate(buf.getbuffer()):
+    if i%16 == 0: fd.write("\n")
+    fd.write(" 0x{:02x},".format(b))
+fd.write("\n};\n");
+fd.close()
diff --git a/platformio.ini b/platformio.ini
index 5477fe3..e5cafb4 100644
--- a/platformio.ini
+++ b/platformio.ini
@@ -2,13 +2,10 @@
 env_default = esp12e
 data_dir = data/spiffs
 
-[env:esp12e]
+[env]
 platform = espressif8266
-board = esp12e
 framework = esp8266-nonos-sdk
 src_filter = +<*> -<*/test/> -<*/*/test/>
-monitor_speed = 115200
-upload_resetmethod = ck
 build_flags =
     -Isrc/include -I.
     -Wl,-Tld/eagle.flash.4m1m.app1.ld
@@ -20,8 +17,13 @@ build_flags =
     -DMQTT -DREST -DSOCKET -DWEBSERVER -DSYSLOG
     -DCHANGE_TO_STA=yes
     -DESP_HOSTNAME=esp-link-v3
+extra_scripts = pre:pio-genespfs.py
 
-
+[env:esp12e]
+board = esp12e
+monitor_speed = 115200
+upload_resetmethod = ck
 upload_speed = 460800
+upload_port = /dev/ttyUSB0
 #upload_speed = 230400
 
diff --git a/src/main.c b/src/main.c
index 3eb0180..09a4805 100644
--- a/src/main.c
+++ b/src/main.c
@@ -131,9 +131,7 @@ static void ICACHE_FLASH_ATTR prHeapTimerCb(void *arg) {
 char* esp_link_version = VERS_STR(VERSION);
 
 // address of espfs binary blob
-#if PIO_HACK
-extern uint32_t _binary_espfs_img_start;
-#endif
+extern uint8_t espfs_image[];
 
 extern void app_init(void);
 extern void mqtt_client_init(void);
@@ -186,10 +184,8 @@ user_init(void) {
   serledInit();
   // Wifi
   wifiInit();
-#if PIO_HACK
   // init the flash filesystem with the html stuff
-  espFsInit(espLinkCtx, &_binary_espfs_img_start, ESPFS_MEMORY);
-#endif
+  espFsInit(espLinkCtx, espfs_image, ESPFS_MEMORY);
 
   //EspFsInitResult res = espFsInit(&_binary_espfs_img_start);
   //os_printf("espFsInit %s\n", res?"ERR":"ok");