From c7e8d19deeba7c3c99494df97cd12b7f39f7da7f Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 13 Jul 2015 10:25:29 -0700 Subject: [PATCH 01/23] reduce size of icons --- html/favicon.ico | Bin 622 -> 874 bytes html/home.html | 2 +- html/jl-200x55.png | Bin 2751 -> 0 bytes html/style.css | 7 + html/ui.js | 358 --------------------------------------------- 5 files changed, 8 insertions(+), 359 deletions(-) delete mode 100755 html/jl-200x55.png delete mode 100644 html/ui.js diff --git a/html/favicon.ico b/html/favicon.ico index ea4eca11c5ea71459cf3dcecea6e9c4c6bf90fa3..fe24a7b535b3d0acfd1db7f56d08b424ef11f880 100755 GIT binary patch literal 874 zcmV-w1C{(oNk%w1VL$*t0P6q%|NsC0{{UlaYpoa|=HA|SZf>NktgRs>r5G6Aq^!kV zV*l^&R#sN6EiJ7W7^MspA^8LW00031EC2ui06+jh000F4Fvv-(wIb^+uK!?I6(nbp zg{ZEh>Aoo|Tg+?U-OMrt?w>kU4OcK2kA|Kv__-AcfT(l|AS7?h%FVE#dc6w+Hf!U4 z&tI`Crc-hX==j;aVYBEOs64Kp64o*=U3+K*3||Cz87wD+az$v0D~*nE0)1bSA$V|= z01B83nn0Vsd5Sf0|x^NqKAZkbsj?+ot_Q50}oP3m1-Lo zCZ16V2)wn zS4jY|&=Txy^$gv*0*c4URS~k3zN^KqjE4Lefdvb$f5En_*)gyK;$ocs4mq#|tPmh@ z;H7t1;Nl2OmA7<)OSgW#s5a`g@h;QUxWfkj9i4D@04Ai#)J#D84ImF56h!oi1NkVB zKmrO}lM+Y3*+kNXAchzsCbVQ_*8jg_yR@t#1B&+Nf3(Q9PEVR)wlH3m^RQsz9+IH(L1rY!MJGW+g AmH+?% literal 622 zcmZ?wbhEHbRA5kGSj)ih|NsAg|Nf<=r>~MzDkv=69~QZC)vC>#H$Qs(_~y-W_-g;e`3n{tO-Zk;s@iI0ckI})UmriddGqGh z+)qmoN^8pc?NdZa4UcGWFSz5y!S41)Ps2yzJm~o!NdHTtYwwY~5 zE9EXQh_$6j|5P)1Q6iAa$eUg5=EKrlEF_mEB~Z+f9qv=#puj9H#I9Jv>)XM^){w@> zEW;qiE!dY{$>iEu)F8;rEX^q|xm`4!dx~?xQby73LLw5|m<5YB1Tl3N?_u62#>l!| za*~g;H_H*}?Sc{_LVLNdr^os;_;4MOVdi6$=M>&9E3qV<>+U-y9T^Fa4+lDb0$Sv;b6n2XlXCSAL=Z98!MNyiEub191P`SJ=B)_ z?TkjFP-MR4gbNK!TcZmqnHEQtPHEvxG|yRlVL|ha{mG}DgbQAZTO_49Tr}0rm4C!A zS&aW>a75!4|F%hcBu%%7$l2x{nZP(Dt+&BBa6`_+M^C#0wbsT;M_yVo_s#9FCI$v; E07DSuRsaA1 diff --git a/html/home.html b/html/home.html index f16a0e0..308b4df 100644 --- a/html/home.html +++ b/html/home.html @@ -1,6 +1,6 @@
-
+
JEELABS

esp-link

diff --git a/html/jl-200x55.png b/html/jl-200x55.png deleted file mode 100755 index 3bc4a42cdfe781b8e1b0a9e2dc48e6b6e8577a4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2751 zcmV;w3PAOVP) zPly}W9ml^;?Ze8jU9tsd-LX> z+}Q5}Vl^{QZ+t06Na=I)FySFdAvfA(2F7Pm-M# zbv+N@1MDh7SULbsR}7<-luQyy91>BGqONaZU@4O1YQ->4C)}q*5{g&~Qq=W3h;A)J zX;lp4aLVV9NT@_okfN?H0(c`?Q36@27{=n5&nbDt39TSSUH@97ou+9x@xlv2eg5_1 zk71goNYh@$Fb)WHDv>0Bu!0nIJqzFtuT2=GU{SBvajjktYIdMv7`<30l1Ky)QV=IQ zb&GX;?(A6{IdUX1p#Y%SY@%AL@tPc#*)B=i1S$wmIdZuiUcY!TaiQESudHBeYm0SQ zDv2ajzQsGpQ;t%pgrC0nVq$_UT)c>HPflWXcGk21=+UG2ZZ3yrvq}5d{12agx_0yC zP3dk)QsJ?H<5cBXU0ucO%uJ$!^!t6Lo0iMvu)4a+`Y1Opl1SpfR1j9S$|#qSnV3ja zh-S0tvm7Lui3u3xGWD2fW=$dq3spgix=y`abME4S{QdRcFppJhHCz;oWmqyewHb*d zQCJq_0=c79Dn+&&t)*pj-g_S#Pdtf@C!VA#i)i1Oi3yZSCEB5J7cM1_06qnAhL3p2 z<#MrNj(`3B9pvp@I?fcu?xcY;`OgD_={Q2{~eMMbo z7BnT2u#grcOE#dnGGX}glTSU{ZtJmLE_F=XNI{Ca zK1UvOU%XN1efGI;U-lP2AHCn&+VaF7_qVq_Z42-IG4LET8H>7pDvX>G2@AIc@kP>c z$5%C8dDXMM-xcV$Zr{fA^z>-E^WOWu&mVm5*}!u!WMpRrB$5y?3x)2BH!5VneeB&o zjNY%WuX|z$GUk~2{`Z8>lgVVrT~opzmPj}-6-3jtSRK<+yZeb$F)wv2;$98W&YTe! zOd<)vUVt8dFalsUEr--%bCL*5*6E+Lg0f ztKr0n6H!k2vnU^ugu_(S0%iA(PSu#&icjq(wG3|990?)VuI1NrBPUH}uoDQcOqq(B)oiPi@aa47Dq`ql3=LXhfvO<=en0XMliS_@ zdbXdr{3afE`dTlSJ+%*VoR&Kc+-x>6Jv|+JsEZ^u$UOeg#c9BMN56vvm$)?*brppB ztl1W81Zh0`QN5lDhoRm3=6$uesGhH~%wyv6r$~t@jzy;UwHO;ns@!l|t)esbh z3$GeZPOB9uB*v`f%hgWOgJnI2G>GlAGFm+ZB$%9=hsd>X!AbLa;;Ny7{E79cukPK$ zm!EvN|p_Gq`;NvyNQT~tdF4O3BlvrS2h7$Tz-mni^uOhx5RanNk* zu%jp<8Nq^+1OTw>VpTyz{Oxudxm*st&p!91V$b~N-@aaHwGVeS9N8& z!yWh!sTf98xPyd9!PTKw+#fK$%#ffvbPFM35#D#{q?g@y`n;*Ai)4n1(&rimvB$8V zIZ)EZaHXgp0|44Gp612|0I?CZfIt3lTLf}i?cX?)sG=~GRgx~w@R~TEFC-y{W3kqV z-Jys5Xtq_=Z0i6+87}~w!HS&K4DpvA9UHk?Uwrrxbz&25TJ0Afe&lKEE+j^QCo2t8 zQE!DX`Pais^w1J_VGLH)JqiVpMy8?;hmZ)hchLj9S*!&R=5uR%8?!Sr*grGlnap&5 zdz)JUcW16>(`rdtOUu*&i!9tO9?iW1$JcZfnQobi+5vD*v#o}ysHZS^I+lkZvrY6} z5N#`%$(jn$XGbJcB-$S-AI*A*xRjKPRzc){=g64_-7V@+`k2FRLSfU%Sb5UB&bTX| z;yUiwu7XqyV~t!7Yk7GYM!AfaZd?zX{_4g(Fp)Q{HVJ42!ir&#<`)3fSI%Phi=)vA zDNdL>q=-L09Dypr5`aa4cB1`}@{!iVH}aK12i~dNo0Pt3gta!F3b=FO(0KI~ z;TzJ$^2H)U3vlY6!D}Jxo1F9o@IljRiJ#jJmX45E9QZ5;|N)I*4=sJ;u$8KMyg z?K`B^QDSm{Ecj`*)pLIDM##Q8Zkb61@jdG7x0fyf*uD6lf4l6fR~&rqS-f=Px=)Ej zrq#w_C>sIyqISNtLttBn4uC^p=QBLy%n{q7)?-MA*iK?fbQOc&X%DX&uG!Xc=Qlr1 zk;6?314Z*(KfXIEZJpbfPoBid2xGJK`FWH|rNkGy2AB`pkx!tKBnmBy<-5SH)$8c* z>_j?28p_e%*B`xh2OrzH#^r`0A2LzaeoESn^0m_aM=@dqLV z%_EPBK2I+)Rt#gT1NJ132D}I8P~jzSkA+^V7)H3@NFoV==wqOnN&*a?`jqrz z+4?&>L5TsW9Fo)#4s4yp<(LJYOeTZUrAx7wEbVr?SYBB{zu)IIJNF=lvr8U5LS=r6 zx;_QqZcw{?J`e588DVAVcDpb)HqdGbO#(b1V-Crlh;1mX{th3^k76vP-T&);}T&71Nkx0Ht{14hL^?zhM5i|e*002ovPDHLk FV1fk~Lhb+n diff --git a/html/style.css b/html/style.css index 4fa393a..29d15a3 100644 --- a/html/style.css +++ b/html/style.css @@ -114,6 +114,13 @@ div.tt { .header h1 .esp { font-size: 1.25em; } +.jl { + font: normal 800 1.5em sans-serif; + position: relative; + bottom: 19px; + color: #9d1414; + margin-left: 3px; +} .content-subhead { margin: 50px 0 20px 0; diff --git a/html/ui.js b/html/ui.js deleted file mode 100644 index b67f077..0000000 --- a/html/ui.js +++ /dev/null @@ -1,358 +0,0 @@ -//===== Collection of small utilities - -/* - * Bind/Unbind events - * - * Usage: - * var el = document.getElementyById('#container'); - * bnd(el, 'click', function() { - * console.log('clicked'); - * }); - */ - -var bnd = function( - d, // a DOM element - e, // an event name such as "click" - f // a handler function -){ - d.addEventListener(e, f, false); -} - -/* - * Create DOM element - * - * Usage: - * var el = m('

Hello

'); - * document.body.appendChild(el); - * - * Copyright (C) 2011 Jed Schmidt - WTFPL - * More: https://gist.github.com/966233 - */ - -var m = function( - a, // an HTML string - b, // placeholder - c // placeholder -){ - b = document; // get the document, - c = b.createElement("p"); // create a container element, - c.innerHTML = a; // write the HTML to it, and - a = b.createDocumentFragment(); // create a fragment. - - while ( // while - b = c.firstChild // the container element has a first child - ) a.appendChild(b); // append the child to the fragment, - - return a // and then return the fragment. -} - -/* - * DOM selector - * - * Usage: - * $('div'); - * $('#name'); - * $('.name'); - * - * Copyright (C) 2011 Jed Schmidt - WTFPL - * More: https://gist.github.com/991057 - */ - -var $ = function( - a, // take a simple selector like "name", "#name", or ".name", and - b // an optional context, and -){ - a = a.match(/^(\W)?(.*)/); // split the selector into name and symbol. - return( // return an element or list, from within the scope of - b // the passed context - || document // or document, - )[ - "getElement" + ( // obtained by the appropriate method calculated by - a[1] - ? a[1] == "#" - ? "ById" // the node by ID, - : "sByClassName" // the nodes by class name, or - : "sByTagName" // the nodes by tag name, - ) - ]( - a[2] // called with the name. - ) -} - -/* - * Get cross browser xhr object - * - * Copyright (C) 2011 Jed Schmidt - * More: https://gist.github.com/993585 - */ - -var j = function( - a // cursor placeholder -){ - for( // for all a - a=0; // from 0 - a<4; // to 4, - a++ // incrementing - ) try { // try - return a // returning - ? new ActiveXObject( // a new ActiveXObject - [ // reflecting - , // (elided) - "Msxml2", // the various - "Msxml3", // working - "Microsoft" // options - ][a] + // for Microsoft implementations, and - ".XMLHTTP" // the appropriate suffix, - ) // but make sure to - : new XMLHttpRequest // try the w3c standard first, and - } - - catch(e){} // ignore when it fails. -} - -// createElement short-hand - -e = function(a) { return document.createElement(a); } - -// chain onload handlers - -function onLoad(f) { - var old = window.onload; - if (typeof old != 'function') { - window.onload = f; - } else { - window.onload = function() { - old(); - f(); - } - } -} - -//===== helpers to add/remove/toggle HTML element classes - -function addClass(el, cl) { - el.className += ' ' + cl; -} -function removeClass(el, cl) { - var cls = el.className.split(/\s+/), - l = cls.length; - for (var i=0; i= 200 && xhr.status < 300) { - console.log("XHR done:", method, url, "->", xhr.status); - ok_cb(xhr.responseText); - } else { - console.log("XHR ERR :", method, url, "->", xhr.status, xhr.responseText, xhr); - err_cb(xhr.status, xhr.responseText); - } - } - console.log("XHR send:", method, url); - try { - xhr.send(); - } catch(err) { - console.log("XHR EXC :", method, url, "->", err); - err_cb(599, err); - } -} - -function dispatchJson(resp, ok_cb, err_cb) { - var j; - try { j = JSON.parse(resp); } - catch(err) { - console.log("JSON parse error: " + err + ". In: " + resp); - err_cb(500, "JSON parse error: " + err); - return; - } - ok_cb(j); -} - -function ajaxJson(method, url, ok_cb, err_cb) { - ajaxReq(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); -} - -function ajaxSpin(method, url, ok_cb, err_cb) { - $("#spinner").removeAttribute('hidden'); - ajaxReq(method, url, function(resp) { - $("#spinner").setAttribute('hidden', ''); - ok_cb(resp); - }, function(status, statusText) { - $("#spinner").setAttribute('hidden', ''); - //showWarning("Error: " + statusText); - err_cb(status, statusText); - }); -} - -function ajaxJsonSpin(method, url, ok_cb, err_cb) { - ajaxSpin(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); -} - -//===== main menu, header spinner and notification boxes - -onLoad(function() { - var l = $("#layout"); - var o = l.childNodes[0]; - // spinner - l.insertBefore(m(''), o); - // notification boxes - l.insertBefore(m( - '
'), o); - // menu hamburger button - l.insertBefore(m(''), o); - // menu left-pane - var mm = m( - '\ - '); - l.insertBefore(mm, o); - - // make hamburger button pull out menu - var ml = $('#menuLink'), mm = $('#menu'); - bnd(ml, 'click', function (e) { - console.log("hamburger time"); - var active = 'active'; - e.preventDefault(); - toggleClass(l, active); - toggleClass(mm, active); - toggleClass(ml, active); - }); - - // populate menu via ajax call - var getMenu = function() { - ajaxJson("GET", "/menu", function(data) { - var html = "", path = window.location.pathname; - for (var i=0; i" + - "" + - data.menu[i] + ""); - } - $("#menu-list").innerHTML = html; - - v = $("#version"); - if (v != null) { v.innerHTML = data.version; } - }, function() { setTimeout(getMenu, 1000); }); - }; - getMenu(); -}); - -//===== Wifi info - -function showWifiInfo(data) { - Object.keys(data).forEach(function(v) { - el = $("#wifi-" + v); - if (el != null) { - if (el.nodeName === "INPUT") el.value = data[v]; - else el.innerHTML = data[v]; - } - }); - var dhcp = $('#dhcp-r'+data.dhcp); - if (dhcp) dhcp.click(); - $("#wifi-spinner").setAttribute("hidden", ""); - $("#wifi-table").removeAttribute("hidden"); - currAp = data.ssid; -} - -function getWifiInfo() { - ajaxJson('GET', "/wifi/info", showWifiInfo, - function(s, st) { window.setTimeout(getWifiInfo, 1000); }); -} - -//===== Notifications - -function showWarning(text) { - var el = $("#warning"); - el.innerHTML = text; - el.removeAttribute('hidden'); -} -function hideWarning() { - el = $("#warning").setAttribute('hidden', ''); -} -var notifTimeout = null; -function showNotification(text) { - var el = $("#notification"); - el.innerHTML = text; - el.removeAttribute('hidden'); - if (notifTimeout != null) clearTimeout(notifTimeout); - notifTimout = setTimeout(function() { - el.setAttribute('hidden', ''); - notifTimout = null; - }, 4000); -} - -//===== GPIO Pin mux card - -var currPin; -// pin={reset:12, isp:13, LED_conn:0, LED_ser:2} -function createInputForPin(pin) { - var input = document.createElement("input"); - input.type = "radio"; - input.name = "pins"; - input.data = pin.name; - input.className = "pin-input"; - input.value= pin.value; - input.id = "opt-" + pin.value; - if (currPin == pin.name) input.checked = "1"; - - var descr = m('"); - var div = document.createElement("div"); - div.appendChild(input); - div.appendChild(descr); - return div; -} - -function displayPins(resp) { - var po = $("#pin-mux"); - po.innerHTML = ""; - currPin = resp.curr; - resp.map.forEach(function(v) { - po.appendChild(createInputForPin(v)); - }); - var i, inputs = $(".pin-input"); - for (i=0; i Date: Tue, 14 Jul 2015 00:09:21 -0700 Subject: [PATCH 02/23] oops, readd mistakenly deleted file --- html/ui.js | 358 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 html/ui.js diff --git a/html/ui.js b/html/ui.js new file mode 100644 index 0000000..5de2f4c --- /dev/null +++ b/html/ui.js @@ -0,0 +1,358 @@ +//===== Collection of small utilities + +/* + * Bind/Unbind events + * + * Usage: + * var el = document.getElementyById('#container'); + * bnd(el, 'click', function() { + * console.log('clicked'); + * }); + */ + +var bnd = function( + d, // a DOM element + e, // an event name such as "click" + f // a handler function +){ + d.addEventListener(e, f, false); +} + +/* + * Create DOM element + * + * Usage: + * var el = m('

Hello

'); + * document.body.appendChild(el); + * + * Copyright (C) 2011 Jed Schmidt - WTFPL + * More: https://gist.github.com/966233 + */ + +var m = function( + a, // an HTML string + b, // placeholder + c // placeholder +){ + b = document; // get the document, + c = b.createElement("p"); // create a container element, + c.innerHTML = a; // write the HTML to it, and + a = b.createDocumentFragment(); // create a fragment. + + while ( // while + b = c.firstChild // the container element has a first child + ) a.appendChild(b); // append the child to the fragment, + + return a // and then return the fragment. +} + +/* + * DOM selector + * + * Usage: + * $('div'); + * $('#name'); + * $('.name'); + * + * Copyright (C) 2011 Jed Schmidt - WTFPL + * More: https://gist.github.com/991057 + */ + +var $ = function( + a, // take a simple selector like "name", "#name", or ".name", and + b // an optional context, and +){ + a = a.match(/^(\W)?(.*)/); // split the selector into name and symbol. + return( // return an element or list, from within the scope of + b // the passed context + || document // or document, + )[ + "getElement" + ( // obtained by the appropriate method calculated by + a[1] + ? a[1] == "#" + ? "ById" // the node by ID, + : "sByClassName" // the nodes by class name, or + : "sByTagName" // the nodes by tag name, + ) + ]( + a[2] // called with the name. + ) +} + +/* + * Get cross browser xhr object + * + * Copyright (C) 2011 Jed Schmidt + * More: https://gist.github.com/993585 + */ + +var j = function( + a // cursor placeholder +){ + for( // for all a + a=0; // from 0 + a<4; // to 4, + a++ // incrementing + ) try { // try + return a // returning + ? new ActiveXObject( // a new ActiveXObject + [ // reflecting + , // (elided) + "Msxml2", // the various + "Msxml3", // working + "Microsoft" // options + ][a] + // for Microsoft implementations, and + ".XMLHTTP" // the appropriate suffix, + ) // but make sure to + : new XMLHttpRequest // try the w3c standard first, and + } + + catch(e){} // ignore when it fails. +} + +// createElement short-hand + +e = function(a) { return document.createElement(a); } + +// chain onload handlers + +function onLoad(f) { + var old = window.onload; + if (typeof old != 'function') { + window.onload = f; + } else { + window.onload = function() { + old(); + f(); + } + } +} + +//===== helpers to add/remove/toggle HTML element classes + +function addClass(el, cl) { + el.className += ' ' + cl; +} +function removeClass(el, cl) { + var cls = el.className.split(/\s+/), + l = cls.length; + for (var i=0; i= 200 && xhr.status < 300) { + console.log("XHR done:", method, url, "->", xhr.status); + ok_cb(xhr.responseText); + } else { + console.log("XHR ERR :", method, url, "->", xhr.status, xhr.responseText, xhr); + err_cb(xhr.status, xhr.responseText); + } + } + console.log("XHR send:", method, url); + try { + xhr.send(); + } catch(err) { + console.log("XHR EXC :", method, url, "->", err); + err_cb(599, err); + } +} + +function dispatchJson(resp, ok_cb, err_cb) { + var j; + try { j = JSON.parse(resp); } + catch(err) { + console.log("JSON parse error: " + err + ". In: " + resp); + err_cb(500, "JSON parse error: " + err); + return; + } + ok_cb(j); +} + +function ajaxJson(method, url, ok_cb, err_cb) { + ajaxReq(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); +} + +function ajaxSpin(method, url, ok_cb, err_cb) { + $("#spinner").removeAttribute('hidden'); + ajaxReq(method, url, function(resp) { + $("#spinner").setAttribute('hidden', ''); + ok_cb(resp); + }, function(status, statusText) { + $("#spinner").setAttribute('hidden', ''); + //showWarning("Error: " + statusText); + err_cb(status, statusText); + }); +} + +function ajaxJsonSpin(method, url, ok_cb, err_cb) { + ajaxSpin(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); +} + +//===== main menu, header spinner and notification boxes + +onLoad(function() { + var l = $("#layout"); + var o = l.childNodes[0]; + // spinner + l.insertBefore(m(''), o); + // notification boxes + l.insertBefore(m( + '
'), o); + // menu hamburger button + l.insertBefore(m(''), o); + // menu left-pane + var mm = m( + '\ + '); + l.insertBefore(mm, o); + + // make hamburger button pull out menu + var ml = $('#menuLink'), mm = $('#menu'); + bnd(ml, 'click', function (e) { + console.log("hamburger time"); + var active = 'active'; + e.preventDefault(); + toggleClass(l, active); + toggleClass(mm, active); + toggleClass(ml, active); + }); + + // populate menu via ajax call + var getMenu = function() { + ajaxJson("GET", "/menu", function(data) { + var html = "", path = window.location.pathname; + for (var i=0; i" + + "" + + data.menu[i] + ""); + } + $("#menu-list").innerHTML = html; + + v = $("#version"); + if (v != null) { v.innerHTML = data.version; } + }, function() { setTimeout(getMenu, 1000); }); + }; + getMenu(); +}); + +//===== Wifi info + +function showWifiInfo(data) { + Object.keys(data).forEach(function(v) { + el = $("#wifi-" + v); + if (el != null) { + if (el.nodeName === "INPUT") el.value = data[v]; + else el.innerHTML = data[v]; + } + }); + var dhcp = $('#dhcp-r'+data.dhcp); + if (dhcp) dhcp.click(); + $("#wifi-spinner").setAttribute("hidden", ""); + $("#wifi-table").removeAttribute("hidden"); + currAp = data.ssid; +} + +function getWifiInfo() { + ajaxJson('GET', "/wifi/info", showWifiInfo, + function(s, st) { window.setTimeout(getWifiInfo, 1000); }); +} + +//===== Notifications + +function showWarning(text) { + var el = $("#warning"); + el.innerHTML = text; + el.removeAttribute('hidden'); +} +function hideWarning() { + el = $("#warning").setAttribute('hidden', ''); +} +var notifTimeout = null; +function showNotification(text) { + var el = $("#notification"); + el.innerHTML = text; + el.removeAttribute('hidden'); + if (notifTimeout != null) clearTimeout(notifTimeout); + notifTimout = setTimeout(function() { + el.setAttribute('hidden', ''); + notifTimout = null; + }, 4000); +} + +//===== GPIO Pin mux card + +var currPin; +// pin={reset:12, isp:13, LED_conn:0, LED_ser:2} +function createInputForPin(pin) { + var input = document.createElement("input"); + input.type = "radio"; + input.name = "pins"; + input.data = pin.name; + input.className = "pin-input"; + input.value= pin.value; + input.id = "opt-" + pin.value; + if (currPin == pin.name) input.checked = "1"; + + var descr = m('"); + var div = document.createElement("div"); + div.appendChild(input); + div.appendChild(descr); + return div; +} + +function displayPins(resp) { + var po = $("#pin-mux"); + po.innerHTML = ""; + currPin = resp.curr; + resp.map.forEach(function(v) { + po.appendChild(createInputForPin(v)); + }); + var i, inputs = $(".pin-input"); + for (i=0; i Date: Tue, 14 Jul 2015 23:38:18 -0700 Subject: [PATCH 03/23] make esp-12 with 4MB flash work --- Makefile | 55 +++++++++++++++++++++++++++++++----------------- user/cgiflash.c | 4 +++- user/user_main.c | 1 + 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index cad8624..99d5617 100644 --- a/Makefile +++ b/Makefile @@ -29,11 +29,28 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- +# Pick your flash size: "512KB" or "4MB" +FLASH_SIZE ?= 4MB + +ifeq ("$(FLASH_SIZE)","512KB") +# Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 ESP_SPI_SIZE ?= 0 # 0->512KB ESP_FLASH_MODE ?= 0 # 0->QIO ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +else +# Winbond 25Q32 4MB flash, typ for esp-12 +# Here we're using two partitions of approx 0.5MB because that's what's easily available in terms +# of linker scripts in the SDK. Ideally we'd use two partitions of approx 1MB, the remaining 2MB +# cannot be used for code. +ESP_SPI_SIZE ?= 4 # 6->4MB (1MB+1MB) or 4->4MB (512KB+512KB) +ESP_FLASH_MODE ?= 0 # 0->QIO, 2->DIO +ESP_FLASH_FREQ_DIV ?= 15 # 15->80Mhz +ESP_FLASH_MAX ?= 503808 # max bin file for 512KB flash partition: 492KB +#ESP_FLASH_MAX ?= 1028096 # max bin file for 1MB flash partition: 1004KB +endif + # hostname or IP address for wifi flashing ESP_HOSTNAME ?= esp-link @@ -163,7 +180,6 @@ SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.c)) OBJ := $(patsubst %.c,$(BUILD_BASE)/%.o,$(SRC)) $(BUILD_BASE)/espfs_img.o LIBS := $(addprefix -l,$(LIBS)) APP_AR := $(addprefix $(BUILD_BASE)/,$(TARGET)_app.a) -TARGET_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).out) USER1_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).user1.out) USER2_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).user2.out) @@ -202,16 +218,11 @@ endef .PHONY: all checkdirs clean webpages.espfs wiflash -all: echo_version checkdirs $(FW_BASE) firmware/user1.bin firmware/user2.bin +all: echo_version checkdirs $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin echo_version: @echo VERSION: $(VERSION) -$(TARGET_OUT): $(APP_AR) $(LD_SCRIPT) - $(vecho) "LD $@" - $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ -# $(OBJDP) -x $(TARGET_OUT) | egrep '(espfs_img)' - $(USER1_OUT): $(APP_AR) $(LD_SCRIPT1) $(vecho) "LD $@" $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT1) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ @@ -224,12 +235,11 @@ $(USER2_OUT): $(APP_AR) $(LD_SCRIPT2) $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT2) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ # $(Q) $(OBJDP) -x $(TARGET_OUT) | egrep espfs_img -$(FW_BASE): $(TARGET_OUT) +$(FW_BASE): $(vecho) "FW $@" $(Q) mkdir -p $@ - $(Q) $(ESPTOOL) elf2image $(TARGET_OUT) --output $@/ -firmware/user1.bin: $(USER1_OUT) +$(FW_BASE)/user1.bin: $(USER1_OUT) $(FW_BASE) $(Q) $(OBJCP) --only-section .text -O binary $(USER1_OUT) eagle.app.v6.text.bin $(Q) $(OBJCP) --only-section .data -O binary $(USER1_OUT) eagle.app.v6.data.bin $(Q) $(OBJCP) --only-section .rodata -O binary $(USER1_OUT) eagle.app.v6.rodata.bin @@ -241,7 +251,7 @@ firmware/user1.bin: $(USER1_OUT) @echo "** user1.bin uses $$(stat -c '%s' $@) bytes of" $(ESP_FLASH_MAX) "available" $(Q) if [ $$(stat -c '%s' $@) -gt $$(( $(ESP_FLASH_MAX) )) ]; then echo "$@ too big!"; false; fi -firmware/user2.bin: $(USER2_OUT) +$(FW_BASE)/user2.bin: $(USER2_OUT) $(FW_BASE) $(Q) $(OBJCP) --only-section .text -O binary $(USER2_OUT) eagle.app.v6.text.bin $(Q) $(OBJCP) --only-section .data -O binary $(USER2_OUT) eagle.app.v6.data.bin $(Q) $(OBJCP) --only-section .rodata -O binary $(USER2_OUT) eagle.app.v6.rodata.bin @@ -251,7 +261,6 @@ firmware/user2.bin: $(USER2_OUT) $(Q) mv eagle.app.flash.bin $@ $(Q) if [ $$(stat -c '%s' $@) -gt $$(( $(ESP_FLASH_MAX) )) ]; then echo "$@ too big!"; false; fi - $(APP_AR): $(OBJ) $(vecho) "AR $@" $(Q) $(AR) cru $@ $^ @@ -262,7 +271,7 @@ $(BUILD_DIR): $(Q) mkdir -p $@ wiflash: all - ./wiflash $(ESP_HOSTNAME) firmware/user1.bin firmware/user2.bin + ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin flash: all $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash \ @@ -302,24 +311,32 @@ endif # we also adjust the sizes of the segments 'cause we need more irom0 # in the end the only thing that matters wrt size is that the whole shebang fits into the # 236KB available (in a 512KB flash) -build/eagle.esphttpd.v6.ld: $(SDK_LDDIR)/eagle.app.v6.ld - $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ - $(SDK_LDDIR)/eagle.app.v6.ld >$@ +ifeq ("$(FLASH_SIZE)","512KB") build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ +else +build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app1.ld + $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ + -e '/^ irom0_0_seg/ s/6B000/7C000/' \ + $(SDK_LDDIR)/eagle.app.v6.new.1024.app1.ld >$@ +build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld + $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ + -e '/^ irom0_0_seg/ s/6B000/7C000/' \ + $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld >$@ +endif espfs/mkespfsimage/mkespfsimage: espfs/mkespfsimage/ $(Q) $(MAKE) -C espfs/mkespfsimage USE_HEATSHRINK="$(USE_HEATSHRINK)" GZIP_COMPRESSION="$(GZIP_COMPRESSION)" release: all $(Q) rm -rf release; mkdir -p release/esp-link - $(Q) cp firmware/user1.bin firmware/user2.bin $(SDK_BASE)/bin/blank.bin \ + $(Q) cp $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin $(SDK_BASE)/bin/blank.bin \ "$(SDK_BASE)/bin/boot_v1.4(b1).bin" wiflash release/esp-link $(Q) tar zcf esp-link.tgz -C release esp-link $(Q) rm -rf release diff --git a/user/cgiflash.c b/user/cgiflash.c index 2c5dbd4..85aaf97 100644 --- a/user/cgiflash.c +++ b/user/cgiflash.c @@ -22,8 +22,10 @@ Some flash handling cgi routines. Used for reading the existing flash and updati // Check that the header of the firmware blob looks like actual firmware... static char* ICACHE_FLASH_ATTR check_header(void *buf) { uint8_t *cd = (uint8_t *)buf; + uint32_t *buf32 = buf; + os_printf("%p: %08lX %08lX %08lX %08lX\n", buf, buf32[0], buf32[1], buf32[2], buf32[3]); if (cd[0] != 0xEA) return "IROM magic missing"; - if (cd[1] != 4 || cd[2] > 3 || cd[3] > 0x40) return "bad flash header"; + if (cd[1] != 4 || cd[2] > 3 || (cd[3]>>4) > 6) return "bad flash header"; if (((uint16_t *)buf)[3] != 0x4010) return "Invalid entry addr"; if (((uint32_t *)buf)[2] != 0) return "Invalid start offset"; return NULL; diff --git a/user/user_main.c b/user/user_main.c index 57443ae..645d215 100644 --- a/user/user_main.c +++ b/user/user_main.c @@ -156,6 +156,7 @@ void user_init(void) { os_printf("exccause=%d epc1=0x%x epc2=0x%x epc3=0x%x excvaddr=0x%x depc=0x%x\n", rst_info->exccause, rst_info->epc1, rst_info->epc2, rst_info->epc3, rst_info->excvaddr, rst_info->depc); + os_printf("Flash map %d, chip %08X\n", system_get_flash_size_map(), spi_flash_get_id()); os_printf("** esp-link ready\n"); } From b31c0a557c23001af2e8003845d09194a010a350 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Sat, 1 Aug 2015 23:43:01 -0700 Subject: [PATCH 04/23] add support for wifi-link-12 with swapped uart pins Conflicts: serial/serbridge.c --- serial/serbridge.c | 13 ++++++++++++- user/cgipins.c | 20 ++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/serial/serbridge.c b/serial/serbridge.c index 6d45610..e18ac82 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -322,7 +322,18 @@ serbridgeUartCb(char *buf, int length) { void ICACHE_FLASH_ATTR serbridgeInitPins() { mcu_reset_pin = flashConfig.reset_pin; mcu_isp_pin = flashConfig.isp_pin; - os_printf("Serbridge pins: reset=%d isp=%d\n", mcu_reset_pin, mcu_isp_pin); + os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", + mcu_reset_pin, mcu_isp_pin, flashConfig.swap_uart); + + if (flashConfig.swap_uart) { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); + system_uart_swap(); + } else { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, 0); + system_uart_de_swap(); + } // set both pins to 1 before turning them on so we don't cause a reset if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); diff --git a/user/cgipins.c b/user/cgipins.c index ae6b26c..a5940a3 100644 --- a/user/cgipins.c +++ b/user/cgipins.c @@ -23,15 +23,13 @@ static const int num_map_func = sizeof(map_func)/sizeof(char*); // Cgi to return choice of pin assignments int ICACHE_FLASH_ATTR cgiPinsGet(HttpdConnData *connData) { - char buff[1024]; - int len; + if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted - if (connData->conn==NULL) { - return HTTPD_CGI_DONE; // Connection aborted - } + char buff[2048]; + int len; // figure out current mapping - int curr = 99; + int curr = 0; for (int i=0; i= 0) len += os_sprintf(buff+len, " %s:gpio%d", map_func[f], p); - else len += os_sprintf(buff+len, " %s:n/a", map_func[f]); + if (f == 4) + len += os_sprintf(buff+len, " %s:%s", map_func[f], p?"yes":"no"); + else if (p >= 0) + len += os_sprintf(buff+len, " %s:gpio%d", map_func[f], p); + else + len += os_sprintf(buff+len, " %s:n/a", map_func[f]); } len += os_sprintf(buff+len, "\" }"); } @@ -87,6 +90,7 @@ int ICACHE_FLASH_ATTR cgiPinsSet(HttpdConnData *connData) { flashConfig.isp_pin = map[1]; flashConfig.conn_led_pin = map[2]; flashConfig.ser_led_pin = map[3]; + flashConfig.swap_uart = map[4]; serbridgeInitPins(); serledInit(); From 1ed85714da7d13cd2c2406e6044dc7d09a41cd68 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 3 Aug 2015 20:56:39 -0700 Subject: [PATCH 05/23] clear bssid before reassociating --- user/cgiwifi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/user/cgiwifi.c b/user/cgiwifi.c index 7292aa4..0d96e96 100644 --- a/user/cgiwifi.c +++ b/user/cgiwifi.c @@ -261,6 +261,7 @@ static ETSTimer reassTimer; static void ICACHE_FLASH_ATTR reassTimerCb(void *arg) { os_printf("Wifi changing association\n"); wifi_station_disconnect(); + stconf.bssid_set = 0; wifi_station_set_config(&stconf); wifi_station_connect(); // Schedule check From 08ab8b58db919254985024dad33f85a1f723eebc Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 14:13:25 -0700 Subject: [PATCH 06/23] fix swap-uart --- serial/serbridge.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/serial/serbridge.c b/serial/serbridge.c index e18ac82..0d6dc57 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -328,6 +328,8 @@ void ICACHE_FLASH_ATTR serbridgeInitPins() { if (flashConfig.swap_uart) { PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); + PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTCK_U); + PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTDO_U); system_uart_swap(); } else { PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); From 08ac33dd56cac5c16f5ee3aec578897ecd760252 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 13:39:01 -0700 Subject: [PATCH 07/23] tweak \n handling on console page --- serial/console.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/serial/console.c b/serial/console.c index d38e73a..6fa4374 100644 --- a/serial/console.c +++ b/serial/console.c @@ -34,16 +34,18 @@ console_write(char c) { } } +#if 0 // return previous character in console, 0 if at start static char ICACHE_FLASH_ATTR console_prev(void) { if (console_wr == console_rd) return 0; return console_buf[(console_wr-1+BUF_MAX)%BUF_MAX]; } +#endif void ICACHE_FLASH_ATTR console_write_char(char c) { - if (c == '\n' && console_prev() != '\r') console_write('\r'); + //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good console_write(c); } @@ -112,6 +114,8 @@ ajaxConsole(HttpdConnData *connData) { if (c == '\\' || c == '"') { buff[len++] = '\\'; buff[len++] = c; + } else if (c == '\r') { + // this is crummy, but browsers display a newline for \r\n sequences } else if (c < ' ') { len += os_sprintf(buff+len, "\\u%04x", c); } else { From a41f8916956b0feae66acca8f42aea625fca49de Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 14:43:06 -0700 Subject: [PATCH 08/23] add missing pieces for swap-uart; switch to SDK 1.3 --- Makefile | 4 ++-- user/cgipins.c | 20 +++++++++++--------- user/config.h | 1 + 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 99d5617..ec7203d 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ XTENSA_TOOLS_ROOT ?= $(abspath ../esp-open-sdk/xtensa-lx106-elf/bin)/ # Base directory of the ESP8266 SDK package, absolute # Typically you'll download from Espressif's BBS, http://bbs.espressif.com/viewforum.php?f=5 -SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.2.0) +SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.3.0) # Esptool.py path and port, only used for 1-time serial flashing # Typically you'll use https://github.com/themadinventor/esptool @@ -30,7 +30,7 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- # Pick your flash size: "512KB" or "4MB" -FLASH_SIZE ?= 4MB +FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 diff --git a/user/cgipins.c b/user/cgipins.c index a5940a3..ab6fdc0 100644 --- a/user/cgipins.c +++ b/user/cgipins.c @@ -8,15 +8,16 @@ #include "serbridge.h" static char *map_names[] = { - "esp-bridge", "jn-esp-v2", "esp-01(AVR)", "esp-01(ARM)", "esp-br-rev", + "esp-bridge", "jn-esp-v2", "esp-01(AVR)", "esp-01(ARM)", "esp-br-rev", "wifi-link-12", }; -static char* map_func[] = { "reset", "isp", "conn_led", "ser_led" }; -static int8_t map_asn[][4] = { - { 12, 13, 0, 14 }, // esp-bridge - { 12, 13, 0, 2 }, // jn-esp-v2 - { 0, -1, 2, -1 }, // esp-01(AVR) - { 0, 2, -1, -1 }, // esp-01(ARM) - { 13, 12, 14, 0 }, // esp-br-rev -- for test purposes +static char* map_func[] = { "reset", "isp", "conn_led", "ser_led", "swap_uart" }; +static int8_t map_asn[][5] = { + { 12, 13, 0, 14, 0 }, // esp-bridge + { 12, 13, 0, 2, 0 }, // jn-esp-v2 + { 0, -1, 2, -1, 0 }, // esp-01(AVR) + { 0, 2, -1, -1, 0 }, // esp-01(ARM) + { 13, 12, 14, 0, 0 }, // esp-br-rev -- for test purposes + { 3, 1, 0, 2, 1 }, // esp-link-12 }; static const int num_map_names = sizeof(map_names)/sizeof(char*); static const int num_map_func = sizeof(map_func)/sizeof(char*); @@ -33,7 +34,8 @@ int ICACHE_FLASH_ATTR cgiPinsGet(HttpdConnData *connData) { for (int i=0; i Date: Wed, 19 Aug 2015 19:13:30 -0700 Subject: [PATCH 09/23] better support for 4MB flash, add 1MB flash --- Makefile | 30 +++++++++++++++++++++++------- user/config.c | 8 +++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index ec7203d..5d222a5 100644 --- a/Makefile +++ b/Makefile @@ -34,21 +34,37 @@ FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 -ESP_SPI_SIZE ?= 0 # 0->512KB -ESP_FLASH_MODE ?= 0 # 0->QIO -ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz -ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +ESP_SPI_SIZE ?= 0 # 0->512KB (256KB+256KB) +ESP_FLASH_MODE ?= 0 # 0->QIO +ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz +ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +ET_FS ?= 4m # 4Mbit flash size in esptool flash command +ET_FF ?= 40m # 40Mhz flash speed in esptool flash command +ET_BLANK ?= 0x7E000 # where to flash blank.bin to erase wireless settings + +else ifeq ("$(FLASH_SIZE)","1MB") +# ESP-01E +ESP_SPI_SIZE ?= 2 # 2->1MB (512KB+512KB) +ESP_FLASH_MODE ?= 0 # 0->QIO +ESP_FLASH_FREQ_DIV ?= 15 # 15->80MHz +ESP_FLASH_MAX ?= 503808 # max bin file for 1MB flash: 492KB +ET_FS ?= 8m # 8Mbit flash size in esptool flash command +ET_FF ?= 80m # 80Mhz flash speed in esptool flash command +ET_BLANK ?= 0xFE000 # where to flash blank.bin to erase wireless settings else # Winbond 25Q32 4MB flash, typ for esp-12 # Here we're using two partitions of approx 0.5MB because that's what's easily available in terms # of linker scripts in the SDK. Ideally we'd use two partitions of approx 1MB, the remaining 2MB -# cannot be used for code. +# cannot be used for code (esp8266 limitation). ESP_SPI_SIZE ?= 4 # 6->4MB (1MB+1MB) or 4->4MB (512KB+512KB) ESP_FLASH_MODE ?= 0 # 0->QIO, 2->DIO ESP_FLASH_FREQ_DIV ?= 15 # 15->80Mhz ESP_FLASH_MAX ?= 503808 # max bin file for 512KB flash partition: 492KB #ESP_FLASH_MAX ?= 1028096 # max bin file for 1MB flash partition: 1004KB +ET_FS ?= 32m # 32Mbit flash size in esptool flash command +ET_FF ?= 80m # 80Mhz flash speed in esptool flash command +ET_BLANK ?= 0x3FE000 # where to flash blank.bin to erase wireless settings endif # hostname or IP address for wifi flashing @@ -274,9 +290,9 @@ wiflash: all ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin flash: all - $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash \ + $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \ 0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \ - 0x7E000 $(SDK_BASE)/bin/blank.bin + $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui diff --git a/user/config.c b/user/config.c index b840c8f..0200919 100644 --- a/user/config.c +++ b/user/config.c @@ -24,10 +24,16 @@ typedef union { uint8_t block[128]; } FlashFull; +// magic number to recognize thet these are our flash settings as opposed to some random stuff #define FLASH_MAGIC (0xaa55) -#define FLASH_ADDR (0x3E000) +// size of the setting sector #define FLASH_SECT (4096) + +// address where to flash the settings: there are 16KB of reserved space at the end of the first +// flash partition, we use the upper 8KB (2 sectors) +#define FLASH_ADDR (FLASH_SECT + FIRMWARE_SIZE + 2*FLASH_SECT) + static int flash_pri; // primary flash sector (0 or 1, or -1 for error) #if 0 From 0d2eed195727251f5fbfe8b0822751fbb3c3edc3 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:13:55 -0700 Subject: [PATCH 10/23] cgiflash cleanup --- user/cgiflash.c | 24 ------------------------ user/config.c | 4 ++-- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/user/cgiflash.c b/user/cgiflash.c index 85aaf97..57c45b4 100644 --- a/user/cgiflash.c +++ b/user/cgiflash.c @@ -31,30 +31,6 @@ static char* ICACHE_FLASH_ATTR check_header(void *buf) { return NULL; } -#if 0 -//===== Cgi that reads the SPI flash. Assumes 512KByte flash. -int ICACHE_FLASH_ATTR cgiReadFlash(HttpdConnData *connData) { - int *pos=(int *)&connData->cgiData; - if (connData->conn==NULL) { - //Connection aborted. Clean up. - return HTTPD_CGI_DONE; - } - - if (*pos==0) { - os_printf("Start flash download.\n"); - httpdStartResponse(connData, 200); - httpdHeader(connData, "Content-Type", "application/bin"); - httpdEndHeaders(connData); - *pos=0x40200000; - return HTTPD_CGI_MORE; - } - //Send 1K of flash per call. We will get called again if we haven't sent 512K yet. - espconn_sent(connData->conn, (uint8 *)(*pos), 1024); - *pos+=1024; - if (*pos>=0x40200000+(512*1024)) return HTTPD_CGI_DONE; else return HTTPD_CGI_MORE; -} -#endif - //===== Cgi to query which firmware needs to be uploaded next int ICACHE_FLASH_ATTR cgiGetFirmwareNext(HttpdConnData *connData) { if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. diff --git a/user/config.c b/user/config.c index 0200919..6c6cf91 100644 --- a/user/config.c +++ b/user/config.c @@ -90,7 +90,7 @@ void ICACHE_FLASH_ATTR configWipe(void) { spi_flash_erase_sector((FLASH_ADDR+FLASH_SECT)>>12); } -static uint32_t ICACHE_FLASH_ATTR selectPrimary(FlashFull *fc0, FlashFull *fc1); +static int ICACHE_FLASH_ATTR selectPrimary(FlashFull *fc0, FlashFull *fc1); bool ICACHE_FLASH_ATTR configRestore(void) { FlashFull ff0, ff1; @@ -112,7 +112,7 @@ bool ICACHE_FLASH_ATTR configRestore(void) { return true; } -static uint32_t ICACHE_FLASH_ATTR selectPrimary(FlashFull *ff0, FlashFull *ff1) { +static int ICACHE_FLASH_ATTR selectPrimary(FlashFull *ff0, FlashFull *ff1) { // check CRC of ff0 uint16_t crc = ff0->fc.crc; ff0->fc.crc = 0; From dcadb70df870bdc793fa8911462210d875e9d756 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:14:10 -0700 Subject: [PATCH 11/23] correct reset codes --- user/user_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/user_main.c b/user/user_main.c index 645d215..f454c17 100644 --- a/user/user_main.c +++ b/user/user_main.c @@ -111,7 +111,7 @@ void user_rf_pre_init(void) { extern uint32_t _binary_espfs_img_start; static char *rst_codes[] = { - "normal", "wdt reset", "exception", "soft wdt", "restart", "deep sleep", "???", + "normal", "wdt reset", "exception", "soft wdt", "restart", "deep sleep", "external", }; # define VERS_STR_STR(V) #V From cd0754071b869aa3ce4fe95028ca51fdc93cfa9d Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:30:59 -0700 Subject: [PATCH 12/23] small flash config improvement plus more comments --- user/config.c | 11 +++++++---- user/config.h | 4 ++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/user/config.c b/user/config.c index 6c6cf91..b8122be 100644 --- a/user/config.c +++ b/user/config.c @@ -17,6 +17,7 @@ FlashConfig flashDefault = { "esp-link\0 ", // hostname 0, 0x00ffffff, 0, // static ip, netmask, gateway 0, // log mode + 0, // swap uart (don't by default) }; typedef union { @@ -46,7 +47,6 @@ static void memDump(void *addr, int len) { #endif bool ICACHE_FLASH_ATTR configSave(void) { - FlashFull ff; memset(&ff, 0, sizeof(ff)); memcpy(&ff, &flashConfig, sizeof(FlashConfig)); @@ -54,7 +54,7 @@ bool ICACHE_FLASH_ATTR configSave(void) { // erase secondary uint32_t addr = FLASH_ADDR + (1-flash_pri)*FLASH_SECT; if (spi_flash_erase_sector(addr>>12) != SPI_FLASH_RESULT_OK) - return false; // no harm done, give up + goto fail; // no harm done, give up // calculate CRC ff.fc.seq = seq; ff.fc.magic = FLASH_MAGIC; @@ -66,11 +66,11 @@ bool ICACHE_FLASH_ATTR configSave(void) { // write primary with incorrect seq ff.fc.seq = 0xffffffff; if (spi_flash_write(addr, (void *)&ff, sizeof(ff)) != SPI_FLASH_RESULT_OK) - return false; // no harm done, give up + goto fail; // no harm done, give up // fill in correct seq ff.fc.seq = seq; if (spi_flash_write(addr, (void *)&ff, sizeof(uint32_t)) != SPI_FLASH_RESULT_OK) - return false; // most likely failed, but no harm if successful + goto fail; // most likely failed, but no harm if successful // now that we have safely written the new version, erase old primary addr = FLASH_ADDR + flash_pri*FLASH_SECT; flash_pri = 1-flash_pri; @@ -83,6 +83,9 @@ bool ICACHE_FLASH_ATTR configSave(void) { ff.fc.seq = seq; spi_flash_write(addr, (void *)&ff, sizeof(uint32_t)); return true; +fail: + os_printf("*** Failed to save config ***\n"); + return false; } void ICACHE_FLASH_ATTR configWipe(void) { diff --git a/user/config.h b/user/config.h index f451f26..d92e7f2 100644 --- a/user/config.h +++ b/user/config.h @@ -1,6 +1,10 @@ #ifndef CONFIG_H #define CONFIG_H +// Flash configuration settings. When adding new items always add them at the end and formulate +// them such that a value of zero is an appropriate default or backwards compatible. Existing +// modules that are upgraded will have zero in the new fields. This ensures that an upgrade does +// not wipe out the old settings. typedef struct { uint32_t seq; // flash write sequence number uint16_t magic, crc; From b48614c37328e4607c0aa9ad49dade17d70d7c34 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 21:43:08 -0700 Subject: [PATCH 13/23] makefile tweak --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d222a5..709a0a1 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- -# Pick your flash size: "512KB" or "4MB" +# Pick your flash size: "512KB", "1MB", or "4MB" FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") From 5cec16c3fdebed65480954e42426d5e64018f9fb Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Fri, 21 Aug 2015 17:52:40 -0500 Subject: [PATCH 14/23] Added windows build support --- .gitignore | 5 + Makefile | 5 +- esp-link.sln | 22 + esp-link.vcxproj | 175 +++ espfs/espfs.c | 21 +- espfs/heatshrink/LICENSE | 14 + espfs/heatshrink/Makefile | 49 + espfs/heatshrink/README.md | 49 + espfs/heatshrink/dec_sm.dot | 52 + espfs/heatshrink/enc_sm.dot | 51 + espfs/heatshrink/greatest.h | 591 +++++++++++ espfs/heatshrink/heatshrink.c | 446 ++++++++ espfs/heatshrink/heatshrink_common.h | 20 + espfs/heatshrink/heatshrink_config.h | 24 + espfs/heatshrink/heatshrink_decoder.c | 382 +++++++ espfs/heatshrink/heatshrink_decoder.h | 101 ++ espfs/heatshrink/heatshrink_encoder.c | 650 ++++++++++++ espfs/heatshrink/heatshrink_encoder.h | 109 ++ espfs/heatshrink/test_heatshrink_dynamic.c | 999 ++++++++++++++++++ .../test_heatshrink_dynamic_theft.c | 521 +++++++++ espfs/heatshrink/test_heatshrink_static.c | 167 +++ espfs/heatshrink_decoder.c | 2 +- espfs/mkespfsimage/Makefile | 36 +- espfs/mkespfsimage/Makefile.linux | 24 + espfs/mkespfsimage/Makefile.windows | 33 + espfs/mkespfsimage/build-linux.sh | 4 + espfs/mkespfsimage/build-win32.sh | 7 + espfs/mkespfsimage/espfsformat.h | 33 + espfs/mkespfsimage/heatshrink_encoder.c | 2 +- espfs/mkespfsimage/main.c | 35 +- espfs/mkespfsimage/mman-win32/Makefile | 48 + espfs/mkespfsimage/mman-win32/config.mak | 11 + espfs/mkespfsimage/mman-win32/configure | 157 +++ espfs/mkespfsimage/mman-win32/mman.c | 180 ++++ espfs/mkespfsimage/mman-win32/mman.h | 55 + espfs/mkespfsimage/mman-win32/test.c | 235 ++++ espmake.cmd | 7 + 37 files changed, 5285 insertions(+), 37 deletions(-) create mode 100644 esp-link.sln create mode 100644 esp-link.vcxproj create mode 100644 espfs/heatshrink/LICENSE create mode 100644 espfs/heatshrink/Makefile create mode 100644 espfs/heatshrink/README.md create mode 100644 espfs/heatshrink/dec_sm.dot create mode 100644 espfs/heatshrink/enc_sm.dot create mode 100644 espfs/heatshrink/greatest.h create mode 100644 espfs/heatshrink/heatshrink.c create mode 100644 espfs/heatshrink/heatshrink_common.h create mode 100644 espfs/heatshrink/heatshrink_config.h create mode 100644 espfs/heatshrink/heatshrink_decoder.c create mode 100644 espfs/heatshrink/heatshrink_decoder.h create mode 100644 espfs/heatshrink/heatshrink_encoder.c create mode 100644 espfs/heatshrink/heatshrink_encoder.h create mode 100644 espfs/heatshrink/test_heatshrink_dynamic.c create mode 100644 espfs/heatshrink/test_heatshrink_dynamic_theft.c create mode 100644 espfs/heatshrink/test_heatshrink_static.c create mode 100644 espfs/mkespfsimage/Makefile.linux create mode 100644 espfs/mkespfsimage/Makefile.windows create mode 100644 espfs/mkespfsimage/build-linux.sh create mode 100644 espfs/mkespfsimage/build-win32.sh create mode 100644 espfs/mkespfsimage/espfsformat.h create mode 100644 espfs/mkespfsimage/mman-win32/Makefile create mode 100644 espfs/mkespfsimage/mman-win32/config.mak create mode 100644 espfs/mkespfsimage/mman-win32/configure create mode 100644 espfs/mkespfsimage/mman-win32/mman.c create mode 100644 espfs/mkespfsimage/mman-win32/mman.h create mode 100644 espfs/mkespfsimage/mman-win32/test.c create mode 100644 espmake.cmd diff --git a/.gitignore b/.gitignore index 7b7ee90..899b7c3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ html_compressed/ esp-link.tgz tve-patch/ yui +espfs/mkespfsimage/mman-win32/mman.o +esp-link.opensdf +esp-link.sdf +espfs/mkespfsimage/mman-win32/libmman.a +.localhistory/ diff --git a/Makefile b/Makefile index 709a0a1..297e4fe 100644 --- a/Makefile +++ b/Makefile @@ -289,6 +289,9 @@ $(BUILD_DIR): wiflash: all ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin +baseflash: all + $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash 0x01000 $(FW_BASE)/user1.bin + flash: all $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \ 0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \ @@ -296,7 +299,7 @@ flash: all yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui - cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) ifeq ("$(COMPRESS_W_YUI)","yes") $(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR) diff --git a/esp-link.sln b/esp-link.sln new file mode 100644 index 0000000..fb7b468 --- /dev/null +++ b/esp-link.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "esp-link", "esp-link.vcxproj", "{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM = Debug|ARM + Release|ARM = Release|ARM + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.ActiveCfg = Debug|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.Build.0 = Debug|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.ActiveCfg = Release|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.Build.0 = Release|ARM + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/esp-link.vcxproj b/esp-link.vcxproj new file mode 100644 index 0000000..45dd2ae --- /dev/null +++ b/esp-link.vcxproj @@ -0,0 +1,175 @@ + + + + + Debug + ARM + + + Release + ARM + + + + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F} + MakeFileProj + + + + Makefile + + + + + + + + + + + __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ + .\mqtt\include;.\serial;.\user;.\espfs;.\httpd;.\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include;c:\Espressif\sdk\include + + + + + + + bin + build + + + espmake flash + espmake clean all + espmake clean + + + espmake baseflash + espmake clean all + espmake clean + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/espfs/espfs.c b/espfs/espfs.c index 3da6692..1f503d5 100644 --- a/espfs/espfs.c +++ b/espfs/espfs.c @@ -7,9 +7,9 @@ It's written for use with httpd, but doesn't need to be used as such. /* * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): - * Jeroen Domburg wrote this file. As long as you retain - * this notice you can do whatever you want with this stuff. If we meet some day, - * and you think this stuff is worth it, you can buy me a beer in return. + * Jeroen Domburg wrote this file. As long as you retain + * this notice you can do whatever you want with this stuff. If we meet some day, + * and you think this stuff is worth it, you can buy me a beer in return. * ---------------------------------------------------------------------------- */ @@ -42,11 +42,12 @@ It's written for use with httpd, but doesn't need to be used as such. #ifdef ESPFS_HEATSHRINK #include "heatshrink_config_custom.h" -#include "heatshrink_decoder.h" +#include "heatshrink/heatshrink_decoder.h" #endif static char* espFsData = NULL; + struct EspFsFile { EspFsHeader *header; char decompressor; @@ -146,19 +147,19 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { return NULL; } if (h.flags&FLAG_LASTFILE) { - //os_printf("End of image.\n"); + os_printf("End of image.\n"); return NULL; } //Grab the name of the file. - p+=sizeof(EspFsHeader); + p+=sizeof(EspFsHeader); os_memcpy(namebuf, p, sizeof(namebuf)); -// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", +// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", // namebuf, (unsigned int)h.nameLen, (unsigned int)h.fileLenComp, h.compression, h.flags); if (os_strcmp(namebuf, fileName)==0) { //Yay, this is the file we need! p+=h.nameLen; //Skip to content. r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem - //os_printf("Alloc %p[%d]\n", r, sizeof(EspFsFile)); +// os_printf("Alloc %p\n", r); if (r==NULL) return NULL; r->header=(EspFsHeader *)hpos; r->decompressor=h.compression; @@ -175,7 +176,7 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { //Decoder params are stored in 1st byte. memcpyAligned(&parm, r->posComp, 1); r->posComp++; - //os_printf("Heatshrink compressed file; decode parms = %x\n", parm); + os_printf("Heatshrink compressed file; decode parms = %x\n", parm); dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf); r->decompData=dec; #endif @@ -265,7 +266,7 @@ void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) { // os_printf("Freed %p\n", dec); } #endif - //os_printf("Freed %p\n", fh); +// os_printf("Freed %p\n", fh); os_free(fh); } diff --git a/espfs/heatshrink/LICENSE b/espfs/heatshrink/LICENSE new file mode 100644 index 0000000..31ec3df --- /dev/null +++ b/espfs/heatshrink/LICENSE @@ -0,0 +1,14 @@ +Copyright (c) 2013, Scott Vokes +All rights reserved. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/espfs/heatshrink/Makefile b/espfs/heatshrink/Makefile new file mode 100644 index 0000000..d8e6875 --- /dev/null +++ b/espfs/heatshrink/Makefile @@ -0,0 +1,49 @@ +PROJECT = heatshrink +#OPTIMIZE = -O0 +#OPTIMIZE = -Os +OPTIMIZE = -O3 +WARN = -Wall -Wextra -pedantic #-Werror +CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE} +CFLAGS += -Wmissing-prototypes +CFLAGS += -Wstrict-prototypes +CFLAGS += -Wmissing-declarations + +# If libtheft is available, build additional property-based tests. +# Uncomment these to use it in test_heatshrink_dynamic. +#CFLAGS += -DHEATSHRINK_HAS_THEFT +#LDFLAGS += -ltheft + +all: + @echo "For tests, make test_heatshrink_dynamic (default) or change the" + @echo "config.h to disable static memory and build test_heatshrink_static." + @echo "For the standalone command-line tool, make heatshrink." + +${PROJECT}: heatshrink.c + +OBJS= heatshrink_encoder.o \ + heatshrink_decoder.o \ + +heatshrink: ${OBJS} +test_heatshrink_dynamic: ${OBJS} test_heatshrink_dynamic_theft.o +test_heatshrink_static: ${OBJS} + +*.o: Makefile heatshrink_config.h + +heatshrink_decoder.o: heatshrink_decoder.h heatshrink_common.h +heatshrink_encoder.o: heatshrink_encoder.h heatshrink_common.h + +tags: TAGS + +TAGS: + etags *.[ch] + +diagrams: dec_sm.png enc_sm.png + +dec_sm.png: dec_sm.dot + dot -o $@ -Tpng $< + +enc_sm.png: enc_sm.dot + dot -o $@ -Tpng $< + +clean: + rm -f ${PROJECT} test_heatshrink_{dynamic,static} *.o *.core {dec,enc}_sm.png TAGS diff --git a/espfs/heatshrink/README.md b/espfs/heatshrink/README.md new file mode 100644 index 0000000..ab150ee --- /dev/null +++ b/espfs/heatshrink/README.md @@ -0,0 +1,49 @@ +# heatshrink + +A data compression/decompression library for embedded/real-time systems. + +## Key Features: + +- **Low memory usage (as low as 50 bytes)** + It is useful for some cases with less than 50 bytes, and useful + for many general cases with < 300 bytes. +- **Incremental, bounded CPU use** + You can chew on input data in arbitrarily tiny bites. + This is a useful property in hard real-time environments. +- **Can use either static or dynamic memory allocation** + The library doesn't impose any constraints on memory management. +- **ISC license** + You can use it freely, even for commercial purposes. + +## Getting Started: + +There is a standalone command-line program, `heatshrink`, but the +encoder and decoder can also be used as libraries, independent of each +other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and +either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their +respective header) into your project. + +Dynamic allocation is used by default, but in an embedded context, you +probably want to statically allocate the encoder/decoder. Set +`HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`. + +## More Information and Benchmarks: + +heatshrink is based on [LZSS], since it's particularly suitable for +compression in small amounts of memory. It can use an optional, small +[index] to make compression significantly faster, but otherwise can run +in under 100 bytes of memory. The index currently adds 2^(window size+1) +bytes to memory usage for compression, and temporarily allocates 512 +bytes on the stack during index construction. + +For more information, see the [blog post] for an overview, and the +`heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API +documentation. + +[blog post]: http://spin.atomicobject.com/2013/03/14/heatshrink-embedded-data-compression/ +[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/ +[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski + +## Build Status + + [![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink) diff --git a/espfs/heatshrink/dec_sm.dot b/espfs/heatshrink/dec_sm.dot new file mode 100644 index 0000000..470012f --- /dev/null +++ b/espfs/heatshrink/dec_sm.dot @@ -0,0 +1,52 @@ +digraph { + graph [label="Decoder state machine", labelloc="t"] + Start [style="invis", shape="point"] + empty + input_available + yield_literal + backref_index_msb + backref_index_lsb + backref_count_msb + backref_count_lsb + yield_backref + check_for_more_input + done [peripheries=2] + + empty->input_available [label="sink()", color="blue", weight=10] + Start->empty + + input_available->yield_literal [label="pop 1-bit"] + input_available->backref_index_msb [label="pop 0-bit", weight=10] + input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10] + + yield_literal->yield_literal [label="sink()", color="blue"] + yield_literal->yield_literal [label="poll()", color="red"] + yield_literal->check_for_more_input [label="poll(), done", color="red"] + + backref_index_msb->backref_index_msb [label="sink()", color="blue"] + backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10] + backref_index_msb->done [label="finish()", color="blue"] + + backref_index_lsb->backref_index_lsb [label="sink()", color="blue"] + backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10] + backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10] + backref_index_lsb->done [label="finish()", color="blue"] + + backref_count_msb->backref_count_msb [label="sink()", color="blue"] + backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10] + backref_count_msb->done [label="finish()", color="blue"] + + backref_count_lsb->backref_count_lsb [label="sink()", color="blue"] + backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10] + backref_count_lsb->done [label="finish()", color="blue"] + + yield_backref->yield_backref [label="sink()", color="blue"] + yield_backref->yield_backref [label="poll()", color="red"] + yield_backref->check_for_more_input [label="poll(), done", + color="red", weight=10] + + check_for_more_input->empty [label="no"] + check_for_more_input->input_available [label="yes"] + + empty->done [label="finish()", color="blue"] +} diff --git a/espfs/heatshrink/enc_sm.dot b/espfs/heatshrink/enc_sm.dot new file mode 100644 index 0000000..6d3030f --- /dev/null +++ b/espfs/heatshrink/enc_sm.dot @@ -0,0 +1,51 @@ +digraph { + graph [label="Encoder state machine", labelloc="t"] + start [style="invis", shape="point"] + not_full + filled + search + yield_tag_bit + yield_literal + yield_br_length + yield_br_index + save_backlog + flush_bits + done [peripheries=2] + + start->not_full [label="start"] + + not_full->not_full [label="sink(), not full", color="blue"] + not_full->filled [label="sink(), buffer is full", color="blue"] + not_full->filled [label="finish(), set is_finished", color="blue"] + + filled->search [label="indexing (if any)"] + + search->search [label="step"] + search->yield_tag_bit [label="literal"] + search->yield_tag_bit [label="match found"] + search->save_backlog [label="input exhausted"] + + yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"] + yield_tag_bit->yield_literal [label="poll(), literal", color="red"] + yield_tag_bit->yield_br_index [label="poll(), no literal", color="red"] + yield_tag_bit->flush_bits [label="finishing, no literal"] + + yield_literal->yield_literal [label="poll(), full buf", color="red"] + yield_literal->search [label="poll(), no match", color="red"] + yield_literal->yield_tag_bit [label="poll(), match", color="red"] + yield_literal->flush_bits [label="poll(), final literal", color="red"] + + yield_br_index->yield_br_index [label="poll(), full buf", color="red"] + yield_br_index->yield_br_length [label="poll()", color="red"] + + yield_br_length->yield_br_length [label="poll(), full buf", color="red"] + yield_br_length->search [label="done"] + + save_backlog->flush_bits [label="finishing, no literal"] + save_backlog->yield_tag_bit [label="finishing, literal"] + save_backlog->not_full [label="expect more input"] + + flush_bits->flush_bits [label="poll(), full buf", color="red"] + flush_bits->done [label="poll(), flushed", color="red"] + flush_bits->done [label="no more output"] +} diff --git a/espfs/heatshrink/greatest.h b/espfs/heatshrink/greatest.h new file mode 100644 index 0000000..a92c642 --- /dev/null +++ b/espfs/heatshrink/greatest.h @@ -0,0 +1,591 @@ +/* + * Copyright (c) 2011 Scott Vokes + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef GREATEST_H +#define GREATEST_H + +#define GREATEST_VERSION_MAJOR 0 +#define GREATEST_VERSION_MINOR 9 +#define GREATEST_VERSION_PATCH 3 + +/* A unit testing system for C, contained in 1 file. + * It doesn't use dynamic allocation or depend on anything + * beyond ANSI C89. */ + + +/********************************************************************* + * Minimal test runner template + *********************************************************************/ +#if 0 + +#include "greatest.h" + +TEST foo_should_foo() { + PASS(); +} + +static void setup_cb(void *data) { + printf("setup callback for each test case\n"); +} + +static void teardown_cb(void *data) { + printf("teardown callback for each test case\n"); +} + +SUITE(suite) { + /* Optional setup/teardown callbacks which will be run before/after + * every test case in the suite. + * Cleared when the suite finishes. */ + SET_SETUP(setup_cb, voidp_to_callback_data); + SET_TEARDOWN(teardown_cb, voidp_to_callback_data); + + RUN_TEST(foo_should_foo); +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + RUN_SUITE(suite); + GREATEST_MAIN_END(); /* display results */ +} + +#endif +/*********************************************************************/ + + +#include +#include +#include +#include + + +/*********** + * Options * + ***********/ + +/* Default column width for non-verbose output. */ +#ifndef GREATEST_DEFAULT_WIDTH +#define GREATEST_DEFAULT_WIDTH 72 +#endif + +/* FILE *, for test logging. */ +#ifndef GREATEST_STDOUT +#define GREATEST_STDOUT stdout +#endif + +/* Remove GREATEST_ prefix from most commonly used symbols? */ +#ifndef GREATEST_USE_ABBREVS +#define GREATEST_USE_ABBREVS 1 +#endif + + +/********* + * Types * + *********/ + +/* Info for the current running suite. */ +typedef struct greatest_suite_info { + unsigned int tests_run; + unsigned int passed; + unsigned int failed; + unsigned int skipped; + + /* timers, pre/post running suite and individual tests */ + clock_t pre_suite; + clock_t post_suite; + clock_t pre_test; + clock_t post_test; +} greatest_suite_info; + +/* Type for a suite function. */ +typedef void (greatest_suite_cb)(void); + +/* Types for setup/teardown callbacks. If non-NULL, these will be run + * and passed the pointer to their additional data. */ +typedef void (greatest_setup_cb)(void *udata); +typedef void (greatest_teardown_cb)(void *udata); + +typedef enum { + GREATEST_FLAG_VERBOSE = 0x01, + GREATEST_FLAG_FIRST_FAIL = 0x02, + GREATEST_FLAG_LIST_ONLY = 0x04 +} GREATEST_FLAG; + +typedef struct greatest_run_info { + unsigned int flags; + unsigned int tests_run; /* total test count */ + + /* Overall pass/fail/skip counts. */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + + /* currently running test suite */ + greatest_suite_info suite; + + /* info to print about the most recent failure */ + const char *fail_file; + unsigned int fail_line; + const char *msg; + + /* current setup/teardown hooks and userdata */ + greatest_setup_cb *setup; + void *setup_udata; + greatest_teardown_cb *teardown; + void *teardown_udata; + + /* formatting info for ".....s...F"-style output */ + unsigned int col; + unsigned int width; + + /* only run a specific suite or test */ + char *suite_filter; + char *test_filter; + + /* overall timers */ + clock_t begin; + clock_t end; +} greatest_run_info; + +/* Global var for the current testing context. + * Initialized by GREATEST_MAIN_DEFS(). */ +extern greatest_run_info greatest_info; + + +/********************** + * Exported functions * + **********************/ + +void greatest_do_pass(const char *name); +void greatest_do_fail(const char *name); +void greatest_do_skip(const char *name); +int greatest_pre_test(const char *name); +void greatest_post_test(const char *name, int res); +void greatest_usage(const char *name); +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); + + +/********** + * Macros * + **********/ + +/* Define a suite. */ +#define GREATEST_SUITE(NAME) void NAME(void) + +/* Start defining a test function. + * The arguments are not included, to allow parametric testing. */ +#define GREATEST_TEST static int + +/* Run a suite. */ +#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) + +/* Run a test in the current suite. */ +#define GREATEST_RUN_TEST(TEST) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* Run a test in the current suite with one void* argument, + * which can be a pointer to a struct with multiple arguments. */ +#define GREATEST_RUN_TEST1(TEST, ENV) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(ENV); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* If __VA_ARGS__ (C99) is supported, allow parametric testing + * without needing to manually manage the argument struct. */ +#if __STDC_VERSION__ >= 19901L +#define GREATEST_RUN_TESTp(TEST, ...) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(__VA_ARGS__); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) +#endif + + +/* Check if the test runner is in verbose mode. */ +#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE) +#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) +#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) +#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) + +/* Message-less forms. */ +#define GREATEST_PASS() GREATEST_PASSm(NULL) +#define GREATEST_FAIL() GREATEST_FAILm(NULL) +#define GREATEST_SKIP() GREATEST_SKIPm(NULL) +#define GREATEST_ASSERT(COND) GREATEST_ASSERTm(#COND, COND) +#define GREATEST_ASSERT_FALSE(COND) GREATEST_ASSERT_FALSEm(#COND, COND) +#define GREATEST_ASSERT_EQ(EXP, GOT) GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_STR_EQ(EXP, GOT) GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) + +/* The following forms take an additional message argument first, + * to be displayed by the test runner. */ + +/* Fail if a condition is not true, with message. */ +#define GREATEST_ASSERTm(MSG, COND) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if (!(COND)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_FALSEm(MSG, COND) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if ((COND)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if ((EXP) != (GOT)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ + do { \ + const char *exp_s = (EXP); \ + const char *got_s = (GOT); \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if (0 != strcmp(exp_s, got_s)) { \ + fprintf(GREATEST_STDOUT, \ + "Expected:\n####\n%s\n####\n", exp_s); \ + fprintf(GREATEST_STDOUT, \ + "Got:\n####\n%s\n####\n", got_s); \ + return -1; \ + } \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_PASSm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return 0; \ + } while (0) + +#define GREATEST_FAILm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + return -1; \ + } while (0) + +#define GREATEST_SKIPm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return 1; \ + } while (0) + +#define GREATEST_SET_TIME(NAME) \ + NAME = clock(); \ + if (NAME == (clock_t) -1) { \ + fprintf(GREATEST_STDOUT, \ + "clock error: %s\n", #NAME); \ + exit(EXIT_FAILURE); \ + } + +#define GREATEST_CLOCK_DIFF(C1, C2) \ + fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ + (long unsigned int) (C2) - (C1), \ + (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) \ + +/* Include several function definitions in the main test file. */ +#define GREATEST_MAIN_DEFS() \ + \ +/* Is FILTER a subset of NAME? */ \ +static int greatest_name_match(const char *name, \ + const char *filter) { \ + size_t offset = 0; \ + size_t filter_len = strlen(filter); \ + while (name[offset] != '\0') { \ + if (name[offset] == filter[0]) { \ + if (0 == strncmp(&name[offset], filter, filter_len)) { \ + return 1; \ + } \ + } \ + offset++; \ + } \ + \ + return 0; \ +} \ + \ +int greatest_pre_test(const char *name) { \ + if (!GREATEST_LIST_ONLY() \ + && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ + && (greatest_info.test_filter == NULL || \ + greatest_name_match(name, greatest_info.test_filter))) { \ + GREATEST_SET_TIME(greatest_info.suite.pre_test); \ + if (greatest_info.setup) { \ + greatest_info.setup(greatest_info.setup_udata); \ + } \ + return 1; /* test should be run */ \ + } else { \ + return 0; /* skipped */ \ + } \ +} \ + \ +void greatest_post_test(const char *name, int res) { \ + GREATEST_SET_TIME(greatest_info.suite.post_test); \ + if (greatest_info.teardown) { \ + void *udata = greatest_info.teardown_udata; \ + greatest_info.teardown(udata); \ + } \ + \ + if (res < 0) { \ + greatest_do_fail(name); \ + } else if (res > 0) { \ + greatest_do_skip(name); \ + } else if (res == 0) { \ + greatest_do_pass(name); \ + } \ + greatest_info.suite.tests_run++; \ + greatest_info.col++; \ + if (GREATEST_IS_VERBOSE()) { \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ + greatest_info.suite.post_test); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else if (greatest_info.col % greatest_info.width == 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + if (GREATEST_STDOUT == stdout) fflush(stdout); \ +} \ + \ +static void greatest_run_suite(greatest_suite_cb *suite_cb, \ + const char *suite_name) { \ + if (greatest_info.suite_filter && \ + !greatest_name_match(suite_name, greatest_info.suite_filter)) \ + return; \ + if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) return; \ + greatest_info.suite.tests_run = 0; \ + greatest_info.suite.failed = 0; \ + greatest_info.suite.passed = 0; \ + greatest_info.suite.skipped = 0; \ + greatest_info.suite.pre_suite = 0; \ + greatest_info.suite.post_suite = 0; \ + greatest_info.suite.pre_test = 0; \ + greatest_info.suite.post_test = 0; \ + greatest_info.col = 0; \ + fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ + GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ + suite_cb(); \ + GREATEST_SET_TIME(greatest_info.suite.post_suite); \ + if (greatest_info.suite.tests_run > 0) { \ + fprintf(GREATEST_STDOUT, \ + "\n%u tests - %u pass, %u fail, %u skipped", \ + greatest_info.suite.tests_run, \ + greatest_info.suite.passed, \ + greatest_info.suite.failed, \ + greatest_info.suite.skipped); \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ + greatest_info.suite.post_suite); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } \ + greatest_info.setup = NULL; \ + greatest_info.setup_udata = NULL; \ + greatest_info.teardown = NULL; \ + greatest_info.teardown_udata = NULL; \ + greatest_info.passed += greatest_info.suite.passed; \ + greatest_info.failed += greatest_info.suite.failed; \ + greatest_info.skipped += greatest_info.suite.skipped; \ + greatest_info.tests_run += greatest_info.suite.tests_run; \ +} \ + \ +void greatest_do_pass(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "PASS %s: %s", \ + name, greatest_info.msg ? greatest_info.msg : ""); \ + } else { \ + fprintf(GREATEST_STDOUT, "."); \ + } \ + greatest_info.suite.passed++; \ +} \ + \ +void greatest_do_fail(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, \ + "FAIL %s: %s (%s:%u)", \ + name, greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } else { \ + fprintf(GREATEST_STDOUT, "F"); \ + /* add linebreak if in line of '.'s */ \ + if (greatest_info.col % greatest_info.width != 0) \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ + name, \ + greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } \ + greatest_info.suite.failed++; \ +} \ + \ +void greatest_do_skip(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ + name, \ + greatest_info.msg ? \ + greatest_info.msg : "" ); \ + } else { \ + fprintf(GREATEST_STDOUT, "s"); \ + } \ + greatest_info.suite.skipped++; \ +} \ + \ +void greatest_usage(const char *name) { \ + fprintf(GREATEST_STDOUT, \ + "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ + " -h print this Help\n" \ + " -l List suites and their tests, then exit\n" \ + " -f Stop runner after first failure\n" \ + " -v Verbose output\n" \ + " -s SUITE only run suite named SUITE\n" \ + " -t TEST only run test named TEST\n", \ + name); \ +} \ + \ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ + greatest_info.setup = cb; \ + greatest_info.setup_udata = udata; \ +} \ + \ +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ + void *udata) { \ + greatest_info.teardown = cb; \ + greatest_info.teardown_udata = udata; \ +} \ + \ +greatest_run_info greatest_info + +/* Handle command-line arguments, etc. */ +#define GREATEST_MAIN_BEGIN() \ + do { \ + int i = 0; \ + memset(&greatest_info, 0, sizeof(greatest_info)); \ + if (greatest_info.width == 0) { \ + greatest_info.width = GREATEST_DEFAULT_WIDTH; \ + } \ + for (i = 1; i < argc; i++) { \ + if (0 == strcmp("-t", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.test_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-s", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.suite_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-f", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ + } else if (0 == strcmp("-v", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_VERBOSE; \ + } else if (0 == strcmp("-l", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ + } else if (0 == strcmp("-h", argv[i])) { \ + greatest_usage(argv[0]); \ + exit(EXIT_SUCCESS); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "Unknown argument '%s'\n", argv[i]); \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + } \ + } while (0); \ + GREATEST_SET_TIME(greatest_info.begin) + +#define GREATEST_MAIN_END() \ + do { \ + if (!GREATEST_LIST_ONLY()) { \ + GREATEST_SET_TIME(greatest_info.end); \ + fprintf(GREATEST_STDOUT, \ + "\nTotal: %u tests", greatest_info.tests_run); \ + GREATEST_CLOCK_DIFF(greatest_info.begin, \ + greatest_info.end); \ + fprintf(GREATEST_STDOUT, "\n"); \ + fprintf(GREATEST_STDOUT, \ + "Pass: %u, fail: %u, skip: %u.\n", \ + greatest_info.passed, \ + greatest_info.failed, greatest_info.skipped); \ + } \ + return (greatest_info.failed > 0 \ + ? EXIT_FAILURE : EXIT_SUCCESS); \ + } while (0) + +/* Make abbreviations without the GREATEST_ prefix for the + * most commonly used symbols. */ +#if GREATEST_USE_ABBREVS +#define TEST GREATEST_TEST +#define SUITE GREATEST_SUITE +#define RUN_TEST GREATEST_RUN_TEST +#define RUN_TEST1 GREATEST_RUN_TEST1 +#define RUN_SUITE GREATEST_RUN_SUITE +#define ASSERT GREATEST_ASSERT +#define ASSERTm GREATEST_ASSERTm +#define ASSERT_FALSE GREATEST_ASSERT_FALSE +#define ASSERT_EQ GREATEST_ASSERT_EQ +#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ +#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm +#define ASSERT_EQm GREATEST_ASSERT_EQm +#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm +#define PASS GREATEST_PASS +#define FAIL GREATEST_FAIL +#define SKIP GREATEST_SKIP +#define PASSm GREATEST_PASSm +#define FAILm GREATEST_FAILm +#define SKIPm GREATEST_SKIPm +#define SET_SETUP GREATEST_SET_SETUP_CB +#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB + +#if __STDC_VERSION__ >= 19901L +#endif /* C99 */ +#define RUN_TESTp GREATEST_RUN_TESTp +#endif /* USE_ABBREVS */ + +#endif diff --git a/espfs/heatshrink/heatshrink.c b/espfs/heatshrink/heatshrink.c new file mode 100644 index 0000000..9de553f --- /dev/null +++ b/espfs/heatshrink/heatshrink.c @@ -0,0 +1,446 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" + +#define DEF_WINDOW_SZ2 11 +#define DEF_LOOKAHEAD_SZ2 4 +#define DEF_DECODER_INPUT_BUFFER_SIZE 256 +#define DEF_BUFFER_SIZE (64 * 1024) + +#if 0 +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#else +#define LOG(...) /* NO-OP */ +#endif + +static const int version_major = HEATSHRINK_VERSION_MAJOR; +static const int version_minor = HEATSHRINK_VERSION_MINOR; +static const int version_patch = HEATSHRINK_VERSION_PATCH; +static const char author[] = HEATSHRINK_AUTHOR; +static const char url[] = HEATSHRINK_URL; + +static void usage(void) { + fprintf(stderr, "heatshrink version %u.%u.%u by %s\n", + version_major, version_minor, version_patch, author); + fprintf(stderr, "Home page: %s\n\n", url); + fprintf(stderr, + "Usage:\n" + " heatshrink [-h] [-e|-d] [-v] [-w SIZE] [-l BITS] [IN_FILE] [OUT_FILE]\n" + "\n" + "heatshrink compresses or uncompresses byte streams using LZSS, and is\n" + "designed especially for embedded, low-memory, and/or hard real-time\n" + "systems.\n" + "\n" + " -h print help\n" + " -e encode (compress, default)\n" + " -d decode (uncompress)\n" + " -v verbose (print input & output sizes, compression ratio, etc.)\n" + "\n" + " -w SIZE Base-2 log of LZSS sliding window size\n" + "\n" + " A larger value allows searches a larger history of the data for repeated\n" + " patterns, potentially compressing more effectively, but will use\n" + " more memory and processing time.\n" + " Recommended default: -w 8 (embedded systems), -w 10 (elsewhere)\n" + " \n" + " -l BITS Number of bits used for back-reference lengths\n" + "\n" + " A larger value allows longer substitutions, but since all\n" + " back-references must use -w + -l bits, larger -w or -l can be\n" + " counterproductive if most patterns are small and/or local.\n" + " Recommended default: -l 4\n" + "\n" + " If IN_FILE or OUT_FILE are unspecified, they will default to\n" + " \"-\" for standard input and standard output, respectively.\n"); + exit(1); +} + +typedef enum { IO_READ, IO_WRITE, } IO_mode; +typedef enum { OP_ENC, OP_DEC, } Operation; + +typedef struct { + int fd; /* file descriptor */ + IO_mode mode; + size_t fill; /* fill index */ + size_t read; /* read index */ + size_t size; + size_t total; + uint8_t buf[]; +} io_handle; + +typedef struct { + uint8_t window_sz2; + uint8_t lookahead_sz2; + size_t decoder_input_buffer_size; + size_t buffer_size; + uint8_t verbose; + Operation cmd; + char *in_fname; + char *out_fname; + io_handle *in; + io_handle *out; +} config; + +static void die(char *msg) { + fprintf(stderr, "%s\n", msg); + exit(EXIT_FAILURE); +} + +static void report(config *cfg); + +/* Open an IO handle. Returns NULL on error. */ +static io_handle *handle_open(char *fname, IO_mode m, size_t buf_sz) { + io_handle *io = NULL; + io = malloc(sizeof(*io) + buf_sz); + if (io == NULL) { return NULL; } + memset(io, 0, sizeof(*io) + buf_sz); + io->fd = -1; + io->size = buf_sz; + io->mode = m; + + if (m == IO_READ) { + if (0 == strcmp("-", fname)) { + io->fd = STDIN_FILENO; + } else { + io->fd = open(fname, O_RDONLY); + } + } else if (m == IO_WRITE) { + if (0 == strcmp("-", fname)) { + io->fd = STDOUT_FILENO; + } else { + io->fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC /*| O_EXCL*/, 0644); + } + } + + if (io->fd == -1) { /* failed to open */ + free(io); + err(1, "open"); + return NULL; + } + + return io; +} + +/* Read SIZE bytes from an IO handle and return a pointer to the content. + * BUF contains at least size_t bytes. Returns 0 on EOF, -1 on error. */ +static ssize_t handle_read(io_handle *io, size_t size, uint8_t **buf) { + LOG("@ read %zd\n", size); + if (buf == NULL) { return -1; } + if (size > io->size) { + printf("size %zd, io->size %zd\n", size, io->size); + return -1; + } + if (io->mode != IO_READ) { return -1; } + + size_t rem = io->fill - io->read; + if (rem >= size) { + *buf = &io->buf[io->read]; + return size; + } else { /* read and replenish */ + if (io->fd == -1) { /* already closed, return what we've got */ + *buf = &io->buf[io->read]; + return rem; + } + + memmove(io->buf, &io->buf[io->read], rem); + io->fill -= io->read; + io->read = 0; + ssize_t read_sz = read(io->fd, &io->buf[io->fill], io->size - io->fill); + if (read_sz < 0) { err(1, "read"); } + io->total += read_sz; + if (read_sz == 0) { /* EOF */ + if (close(io->fd) < 0) { err(1, "close"); } + io->fd = -1; + } + io->fill += read_sz; + *buf = io->buf; + return io->fill > size ? size : io->fill; + } +} + +/* Drop the oldest SIZE bytes from the buffer. Returns <0 on error. */ +static int handle_drop(io_handle *io, size_t size) { + LOG("@ drop %zd\n", size); + if (io->read + size <= io->fill) { + io->read += size; + } else { + return -1; + } + if (io->read == io->fill) { + io->read = 0; + io->fill = 0; + } + return 0; +} + +/* Sink SIZE bytes from INPUT into the io handle. Returns the number of + * bytes written, or -1 on error. */ +static ssize_t handle_sink(io_handle *io, size_t size, uint8_t *input) { + LOG("@ sink %zd\n", size); + if (size > io->size) { return -1; } + if (io->mode != IO_WRITE) { return -1; } + + if (io->fill + size > io->size) { + ssize_t written = write(io->fd, io->buf, io->fill); + LOG("@ flushing %zd, wrote %zd\n", io->fill, written); + io->total += written; + if (written == -1) { err(1, "write"); } + memmove(io->buf, &io->buf[written], io->fill - written); + io->fill -= written; + } + memcpy(&io->buf[io->fill], input, size); + io->fill += size; + return size; +} + +static void handle_close(io_handle *io) { + if (io->fd != -1) { + if (io->mode == IO_WRITE) { + ssize_t written = write(io->fd, io->buf, io->fill); + io->total += written; + LOG("@ close: flushing %zd, wrote %zd\n", io->fill, written); + if (written == -1) { err(1, "write"); } + } + close(io->fd); + io->fd = -1; + } +} + +static void close_and_report(config *cfg) { + handle_close(cfg->in); + handle_close(cfg->out); + if (cfg->verbose) { report(cfg); } + free(cfg->in); + free(cfg->out); +} + +static int encoder_sink_read(config *cfg, heatshrink_encoder *hse, + uint8_t *data, size_t data_sz) { + size_t out_sz = 4096; + uint8_t out_buf[out_sz]; + memset(out_buf, 0, out_sz); + size_t sink_sz = 0; + size_t poll_sz = 0; + HSE_sink_res sres; + HSE_poll_res pres; + HSE_finish_res fres; + io_handle *out = cfg->out; + + size_t sunk = 0; + do { + if (data_sz > 0) { + sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz); + if (sres < 0) { die("sink"); } + sunk += sink_sz; + } + + do { + pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz); + if (pres < 0) { die("poll"); } + if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); + } while (pres == HSER_POLL_MORE); + + if (poll_sz == 0 && data_sz == 0) { + fres = heatshrink_encoder_finish(hse); + if (fres < 0) { die("finish"); } + if (fres == HSER_FINISH_DONE) { return 1; } + } + } while (sunk < data_sz); + return 0; +} + +static int encode(config *cfg) { + uint8_t window_sz2 = cfg->window_sz2; + size_t window_sz = 1 << window_sz2; + heatshrink_encoder *hse = heatshrink_encoder_alloc(window_sz2, cfg->lookahead_sz2); + if (hse == NULL) { die("failed to init encoder: bad settings"); } + ssize_t read_sz = 0; + io_handle *in = cfg->in; + + /* Process input until end of stream */ + while (1) { + uint8_t *input = NULL; + read_sz = handle_read(in, window_sz, &input); + if (input == NULL) { + printf("handle read failure\n"); + die("read"); + } + if (read_sz < 0) { die("read"); } + + /* Pass read to encoder and check if input is fully processed. */ + if (encoder_sink_read(cfg, hse, input, read_sz)) break; + + if (handle_drop(in, read_sz) < 0) { die("drop"); } + }; + + if (read_sz == -1) { err(1, "read"); } + + heatshrink_encoder_free(hse); + close_and_report(cfg); + return 0; +} + +static int decoder_sink_read(config *cfg, heatshrink_decoder *hsd, + uint8_t *data, size_t data_sz) { + io_handle *out = cfg->out; + size_t sink_sz = 0; + size_t poll_sz = 0; + size_t out_sz = 4096; + uint8_t out_buf[out_sz]; + memset(out_buf, 0, out_sz); + + HSD_sink_res sres; + HSD_poll_res pres; + HSD_finish_res fres; + + size_t sunk = 0; + do { + if (data_sz > 0) { + sres = heatshrink_decoder_sink(hsd, &data[sunk], data_sz - sunk, &sink_sz); + if (sres < 0) { die("sink"); } + sunk += sink_sz; + } + + do { + pres = heatshrink_decoder_poll(hsd, out_buf, out_sz, &poll_sz); + if (pres < 0) { die("poll"); } + if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); + } while (pres == HSDR_POLL_MORE); + + if (data_sz == 0 && poll_sz == 0) { + fres = heatshrink_decoder_finish(hsd); + if (fres < 0) { die("finish"); } + if (fres == HSDR_FINISH_DONE) { return 1; } + } + } while (sunk < data_sz); + + return 0; +} + +static int decode(config *cfg) { + uint8_t window_sz2 = cfg->window_sz2; + size_t window_sz = 1 << window_sz2; + size_t ibs = cfg->decoder_input_buffer_size; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(ibs, + window_sz2, cfg->lookahead_sz2); + if (hsd == NULL) { die("failed to init decoder"); } + + ssize_t read_sz = 0; + + io_handle *in = cfg->in; + + HSD_finish_res fres; + + /* Process input until end of stream */ + while (1) { + uint8_t *input = NULL; + read_sz = handle_read(in, window_sz, &input); + if (input == NULL) { + printf("handle read failure\n"); + die("read"); + } + if (read_sz == 0) { + fres = heatshrink_decoder_finish(hsd); + if (fres < 0) { die("finish"); } + if (fres == HSDR_FINISH_DONE) break; + } else if (read_sz < 0) { + die("read"); + } else { + if (decoder_sink_read(cfg, hsd, input, read_sz)) { break; } + if (handle_drop(in, read_sz) < 0) { die("drop"); } + } + } + if (read_sz == -1) { err(1, "read"); } + + heatshrink_decoder_free(hsd); + close_and_report(cfg); + return 0; +} + +static void report(config *cfg) { + size_t inb = cfg->in->total; + size_t outb = cfg->out->total; + fprintf(cfg->out->fd == STDOUT_FILENO ? stderr : stdout, + "%s %0.2f %%\t %zd -> %zd (-w %u -l %u)\n", + cfg->in_fname, 100.0 - (100.0 * outb) / inb, inb, outb, + cfg->window_sz2, cfg->lookahead_sz2); +} + +static void proc_args(config *cfg, int argc, char **argv) { + cfg->window_sz2 = DEF_WINDOW_SZ2; + cfg->lookahead_sz2 = DEF_LOOKAHEAD_SZ2; + cfg->buffer_size = DEF_BUFFER_SIZE; + cfg->decoder_input_buffer_size = DEF_DECODER_INPUT_BUFFER_SIZE; + cfg->cmd = OP_ENC; + cfg->verbose = 0; + cfg->in_fname = "-"; + cfg->out_fname = "-"; + + int a = 0; + while ((a = getopt(argc, argv, "hedi:w:l:v")) != -1) { + switch (a) { + case 'h': /* help */ + usage(); + case 'e': /* encode */ + cfg->cmd = OP_ENC; break; + case 'd': /* decode */ + cfg->cmd = OP_DEC; break; + case 'i': /* input buffer size */ + cfg->decoder_input_buffer_size = atoi(optarg); + break; + case 'w': /* window bits */ + cfg->window_sz2 = atoi(optarg); + break; + case 'l': /* lookahead bits */ + cfg->lookahead_sz2 = atoi(optarg); + break; + case 'v': /* verbosity++ */ + cfg->verbose++; + break; + case '?': /* unknown argument */ + default: + usage(); + } + } + argc -= optind; + argv += optind; + if (argc > 0) { + cfg->in_fname = argv[0]; + argc--; + argv++; + } + if (argc > 0) { cfg->out_fname = argv[0]; } +} + +int main(int argc, char **argv) { + config cfg; + memset(&cfg, 0, sizeof(cfg)); + proc_args(&cfg, argc, argv); + + if (0 == strcmp(cfg.in_fname, cfg.out_fname) + && (0 != strcmp("-", cfg.in_fname))) { + printf("Refusing to overwrite file '%s' with itself.\n", cfg.in_fname); + exit(1); + } + + cfg.in = handle_open(cfg.in_fname, IO_READ, cfg.buffer_size); + if (cfg.in == NULL) { die("Failed to open input file for read"); } + cfg.out = handle_open(cfg.out_fname, IO_WRITE, cfg.buffer_size); + if (cfg.out == NULL) { die("Failed to open output file for write"); } + + if (cfg.cmd == OP_ENC) { + return encode(&cfg); + } else if (cfg.cmd == OP_DEC) { + return decode(&cfg); + } else { + usage(); + } +} diff --git a/espfs/heatshrink/heatshrink_common.h b/espfs/heatshrink/heatshrink_common.h new file mode 100644 index 0000000..0d396b9 --- /dev/null +++ b/espfs/heatshrink/heatshrink_common.h @@ -0,0 +1,20 @@ +#ifndef HEATSHRINK_H +#define HEATSHRINK_H + +#define HEATSHRINK_AUTHOR "Scott Vokes " +#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" + +/* Version 0.3.1 */ +#define HEATSHRINK_VERSION_MAJOR 0 +#define HEATSHRINK_VERSION_MINOR 3 +#define HEATSHRINK_VERSION_PATCH 1 + +#define HEATSHRINK_MIN_WINDOW_BITS 4 +#define HEATSHRINK_MAX_WINDOW_BITS 15 + +#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2 + +#define HEATSHRINK_LITERAL_MARKER 0x01 +#define HEATSHRINK_BACKREF_MARKER 0x00 + +#endif diff --git a/espfs/heatshrink/heatshrink_config.h b/espfs/heatshrink/heatshrink_config.h new file mode 100644 index 0000000..51d4772 --- /dev/null +++ b/espfs/heatshrink/heatshrink_config.h @@ -0,0 +1,24 @@ +#ifndef HEATSHRINK_CONFIG_H +#define HEATSHRINK_CONFIG_H + +/* Should functionality assuming dynamic allocation be used? */ +#define HEATSHRINK_DYNAMIC_ALLOC 1 + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Optional replacement of malloc/free */ + #define HEATSHRINK_MALLOC(SZ) malloc(SZ) + #define HEATSHRINK_FREE(P, SZ) free(P) +#else + /* Required parameters for static configuration */ + #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 + #define HEATSHRINK_STATIC_WINDOW_BITS 8 + #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 +#endif + +/* Turn on logging for debugging. */ +#define HEATSHRINK_DEBUGGING_LOGS 0 + +/* Use indexing for faster compression. (This requires additional space.) */ +#define HEATSHRINK_USE_INDEX 1 + +#endif diff --git a/espfs/heatshrink/heatshrink_decoder.c b/espfs/heatshrink/heatshrink_decoder.c new file mode 100644 index 0000000..b92be13 --- /dev/null +++ b/espfs/heatshrink/heatshrink_decoder.c @@ -0,0 +1,382 @@ +#include +#include +#include "heatshrink_decoder.h" + +/* States for the polling state machine. */ +typedef enum { + HSDS_EMPTY, /* no input to process */ + HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */ + HSDS_YIELD_LITERAL, /* ready to yield literal byte */ + HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ + HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ + HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ + HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ + HSDS_YIELD_BACKREF, /* ready to yield back-reference */ + HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */ +} HSD_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "empty", + "input_available", + "yield_literal", + "backref_index", + "backref_count", + "yield_backref", + "check_for_more_input", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define NO_BITS ((uint32_t)-1) + +/* Forward references. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count); +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (input_buffer_size == 0) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + size_t buffers_sz = (1 << window_sz2) + input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); + if (hsd == NULL) { return NULL; } + hsd->input_buffer_size = input_buffer_size; + hsd->window_sz2 = window_sz2; + hsd->lookahead_sz2 = lookahead_sz2; + heatshrink_decoder_reset(hsd); + LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", + sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); + return hsd; +} + +void heatshrink_decoder_free(heatshrink_decoder *hsd) { + size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + HEATSHRINK_FREE(hsd, sz); + (void)sz; /* may not be used by free */ +} +#endif + +void heatshrink_decoder_reset(heatshrink_decoder *hsd) { + size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); + size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); + memset(hsd->buffers, 0, buf_sz + input_sz); + hsd->state = HSDS_EMPTY; + hsd->input_size = 0; + hsd->input_index = 0; + hsd->bit_index = 0x00; + hsd->current_byte = 0x00; + hsd->output_count = 0; + hsd->output_index = 0; + hsd->head_index = 0; + hsd->bit_accumulator = 0x00000000; +} + +/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSDR_SINK_ERROR_NULL; + } + + size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; + if (rem == 0) { + *input_size = 0; + return HSDR_SINK_FULL; + } + + size = rem < size ? rem : size; + LOG("-- sinking %zd bytes\n", size); + /* copy into input buffer (at head of buffers) */ + memcpy(&hsd->buffers[hsd->input_size], in_buf, size); + hsd->input_size += size; + if (hsd->state == HSDS_EMPTY) { + hsd->state = HSDS_INPUT_AVAILABLE; + hsd->input_index = 0; + } + *input_size = size; + return HSDR_SINK_OK; +} + + +/***************** + * Decompression * + *****************/ + +#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) +#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) + +// States +static HSD_state st_input_available(heatshrink_decoder *hsd); +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_check_for_input(heatshrink_decoder *hsd); + +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSDR_POLL_ERROR_NULL; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- poll, state is %d (%s), input_size %d\n", + hsd->state, state_names[hsd->state], hsd->input_size); + uint8_t in_state = hsd->state; + switch (in_state) { + case HSDS_EMPTY: + return HSDR_POLL_EMPTY; + case HSDS_INPUT_AVAILABLE: + hsd->state = st_input_available(hsd); + break; + case HSDS_YIELD_LITERAL: + hsd->state = st_yield_literal(hsd, &oi); + break; + case HSDS_BACKREF_INDEX_MSB: + hsd->state = st_backref_index_msb(hsd); + break; + case HSDS_BACKREF_INDEX_LSB: + hsd->state = st_backref_index_lsb(hsd); + break; + case HSDS_BACKREF_COUNT_MSB: + hsd->state = st_backref_count_msb(hsd); + break; + case HSDS_BACKREF_COUNT_LSB: + hsd->state = st_backref_count_lsb(hsd); + break; + case HSDS_YIELD_BACKREF: + hsd->state = st_yield_backref(hsd, &oi); + break; + case HSDS_CHECK_FOR_MORE_INPUT: + hsd->state = st_check_for_input(hsd); + break; + default: + return HSDR_POLL_ERROR_UNKNOWN; + } + + /* If the current state cannot advance, check if input or output + * buffer are exhausted. */ + if (hsd->state == in_state) { + if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } + return HSDR_POLL_EMPTY; + } + } +} + +static HSD_state st_input_available(heatshrink_decoder *hsd) { + uint32_t bits = get_bits(hsd, 1); // get tag bit + if (bits) { + return HSDS_YIELD_LITERAL; + } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { + return HSDS_BACKREF_INDEX_MSB; + } else { + hsd->output_index = 0; + return HSDS_BACKREF_INDEX_LSB; + } +} + +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi) { + /* Emit a repeated section from the window buffer, and add it (again) + * to the window buffer. (Note that the repetition can include + * itself.)*/ + if (*oi->output_size < oi->buf_size) { + uint32_t byte = get_bits(hsd, 8); + if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint8_t c = byte & 0xFF; + LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); + buf[hsd->head_index++ & mask] = c; + push_byte(hsd, oi, c); + return HSDS_CHECK_FOR_MORE_INPUT; + } else { + return HSDS_YIELD_LITERAL; + } +} + +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + ASSERT(bit_ct > 8); + uint32_t bits = get_bits(hsd, bit_ct - 8); + LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } + hsd->output_index = bits << 8; + return HSDS_BACKREF_INDEX_LSB; +} + +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); + LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } + hsd->output_index |= bits; + hsd->output_index++; + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + hsd->output_count = 0; + return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + ASSERT(br_bit_ct > 8); + uint32_t bits = get_bits(hsd, br_bit_ct - 8); + LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } + hsd->output_count = bits << 8; + return HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); + LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } + hsd->output_count |= bits; + hsd->output_count++; + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi) { + size_t count = oi->buf_size - *oi->output_size; + if (count > 0) { + if (hsd->output_count < count) count = hsd->output_count; + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint16_t neg_offset = hsd->output_index; + LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); + ASSERT(neg_offset < mask + 1); + ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd)); + + for (size_t i=0; ihead_index - neg_offset) & mask]; + push_byte(hsd, oi, c); + buf[hsd->head_index & mask] = c; + hsd->head_index++; + LOG(" -- ++ 0x%02x\n", c); + } + hsd->output_count -= count; + if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; } + } + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_check_for_input(heatshrink_decoder *hsd) { + return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE; +} + +/* Get the next COUNT bits from the input buffer, saving incremental progress. + * Returns NO_BITS on end of input, or if more than 31 bits are requested. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) { + if (count > 31) { return NO_BITS; } + LOG("-- popping %u bit(s)\n", count); + + /* If we aren't able to get COUNT bits, suspend immediately, because we + * don't track how many bits of COUNT we've accumulated before suspend. */ + if (hsd->input_size == 0) { + if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } + } + + for (int i = 0; i < count; i++) { + if (hsd->bit_index == 0x00) { + if (hsd->input_size == 0) { + LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", + hsd->bit_accumulator, hsd->bit_accumulator); + return NO_BITS; + } + hsd->current_byte = hsd->buffers[hsd->input_index++]; + LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); + if (hsd->input_index == hsd->input_size) { + hsd->input_index = 0; /* input is exhausted */ + hsd->input_size = 0; + } + hsd->bit_index = 0x80; + } + hsd->bit_accumulator <<= 1; + if (hsd->current_byte & hsd->bit_index) { + hsd->bit_accumulator |= 0x01; + if (0) { + LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } else { + if (0) { + LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } + hsd->bit_index >>= 1; + } + + uint32_t res = 0; + res = hsd->bit_accumulator; + hsd->bit_accumulator = 0x00000000; + if (count > 1) { LOG(" -- accumulated %08x\n", res); } + return res; +} + +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { + if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } + switch (hsd->state) { + case HSDS_EMPTY: + return HSDR_FINISH_DONE; + + /* If we want to finish with no input, but are in these states, it's + * because the 0-bit padding to the last byte looks like a backref + * marker bit followed by all 0s for index and count bits. */ + case HSDS_BACKREF_INDEX_LSB: + case HSDS_BACKREF_INDEX_MSB: + case HSDS_BACKREF_COUNT_LSB: + case HSDS_BACKREF_COUNT_MSB: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + /* If the output stream is padded with 0xFFs (possibly due to being in + * flash memory), also explicitly check the input size rather than + * uselessly returning MORE but yielding 0 bytes when polling. */ + case HSDS_YIELD_LITERAL: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + default: + return HSDR_FINISH_MORE; + } +} + +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { + LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); + oi->buf[(*oi->output_size)++] = byte; + (void)hsd; +} diff --git a/espfs/heatshrink/heatshrink_decoder.h b/espfs/heatshrink/heatshrink_decoder.h new file mode 100644 index 0000000..c1eb144 --- /dev/null +++ b/espfs/heatshrink/heatshrink_decoder.h @@ -0,0 +1,101 @@ +#ifndef HEATSHRINK_DECODER_H +#define HEATSHRINK_DECODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSDR_SINK_OK, /* data sunk, ready to poll */ + HSDR_SINK_FULL, /* out of space in internal buffer */ + HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ +} HSD_sink_res; + +typedef enum { + HSDR_POLL_EMPTY, /* input exhausted */ + HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ + HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ + HSDR_POLL_ERROR_UNKNOWN=-2, +} HSD_poll_res; + +typedef enum { + HSDR_FINISH_DONE, /* output is done */ + HSDR_FINISH_MORE, /* more output remains */ + HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ +} HSD_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ + ((BUF)->input_buffer_size) +#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ + ((BUF)->window_sz2) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + ((BUF)->lookahead_sz2) +#else +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ + HEATSHRINK_STATIC_INPUT_BUFFER_SIZE +#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t input_index; /* offset to next unprocessed input byte */ + uint16_t output_count; /* how many bytes to output */ + uint16_t output_index; /* index for bytes to output */ + uint16_t head_index; /* head of window buffer */ + uint16_t bit_accumulator; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of input */ + uint8_t bit_index; /* current bit index */ + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Fields that are only used if dynamically allocated. */ + uint8_t window_sz2; /* window buffer bits */ + uint8_t lookahead_sz2; /* lookahead bits */ + uint16_t input_buffer_size; /* input buffer size */ + + /* Input buffer, then expansion window buffer */ + uint8_t buffers[]; +#else + /* Input buffer, then expansion window buffer */ + uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) + + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; +#endif +} heatshrink_decoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, + * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead + * size of 2^lookahead_sz2. (The window buffer and lookahead sizes + * must match the settings used when the data was compressed.) + * Returns NULL on error. */ +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); + +/* Free a decoder. */ +void heatshrink_decoder_free(heatshrink_decoder *hsd); +#endif + +/* Reset a decoder. */ +void heatshrink_decoder_reset(heatshrink_decoder *hsd); + +/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to + * indicate how many bytes were actually sunk (in case a buffer was filled). */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the dencoder that the input stream is finished. + * If the return value is HSDR_FINISH_MORE, there is still more output, so + * call heatshrink_decoder_poll and repeat. */ +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); + +#endif diff --git a/espfs/heatshrink/heatshrink_encoder.c b/espfs/heatshrink/heatshrink_encoder.c new file mode 100644 index 0000000..ede5f60 --- /dev/null +++ b/espfs/heatshrink/heatshrink_encoder.c @@ -0,0 +1,650 @@ +#include +#include +#include +#include "heatshrink_encoder.h" + +typedef enum { + HSES_NOT_FULL, /* input buffer not full enough */ + HSES_FILLED, /* buffer is full */ + HSES_SEARCH, /* searching for patterns */ + HSES_YIELD_TAG_BIT, /* yield tag bit */ + HSES_YIELD_LITERAL, /* emit literal byte */ + HSES_YIELD_BR_INDEX, /* yielding backref index */ + HSES_YIELD_BR_LENGTH, /* yielding backref length */ + HSES_SAVE_BACKLOG, /* copying buffer to backlog */ + HSES_FLUSH_BITS, /* flush bit buffer */ + HSES_DONE, /* done */ +} HSE_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "not_full", + "filled", + "search", + "yield_tag_bit", + "yield_literal", + "yield_br_index", + "yield_br_length", + "save_backlog", + "flush_bits", + "done", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +// Encoder flags +enum { + FLAG_IS_FINISHING = 0x01, + FLAG_HAS_LITERAL = 0x02, + FLAG_ON_FINAL_LITERAL = 0x04, + FLAG_BACKLOG_IS_PARTIAL = 0x08, + FLAG_BACKLOG_IS_FILLED = 0x10, +}; + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define MATCH_NOT_FOUND ((uint16_t)-1) + +static uint16_t get_input_offset(heatshrink_encoder *hse); +static uint16_t get_input_buffer_size(heatshrink_encoder *hse); +static uint16_t get_lookahead_size(heatshrink_encoder *hse); +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); +static int can_take_byte(output_info *oi); +static int is_finishing(heatshrink_encoder *hse); +static int backlog_is_partial(heatshrink_encoder *hse); +static int backlog_is_filled(heatshrink_encoder *hse); +static int on_final_literal(heatshrink_encoder *hse); +static void save_backlog(heatshrink_encoder *hse); +static int has_literal(heatshrink_encoder *hse); + +/* Push COUNT (max 8) bits to the output buffer, which has room. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi); +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + + /* Note: 2 * the window size is used because the buffer needs to fit + * (1 << window_sz2) bytes for the current input, and an additional + * (1 << window_sz2) bytes for the previous buffer of input, which + * will be scanned for useful backreferences. */ + size_t buf_sz = (2 << window_sz2); + + heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); + if (hse == NULL) { return NULL; } + hse->window_sz2 = window_sz2; + hse->lookahead_sz2 = lookahead_sz2; + heatshrink_encoder_reset(hse); + +#if HEATSHRINK_USE_INDEX + size_t index_sz = buf_sz*sizeof(uint16_t); + hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); + if (hse->search_index == NULL) { + HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); + return NULL; + } + hse->search_index->size = index_sz; +#endif + + LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", + buf_sz, get_input_buffer_size(hse)); + return hse; +} + +void heatshrink_encoder_free(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); +#if HEATSHRINK_USE_INDEX + size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; + HEATSHRINK_FREE(hse->search_index, index_sz); + (void)index_sz; +#endif + HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); + (void)buf_sz; +} +#endif + +void heatshrink_encoder_reset(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + memset(hse->buffer, 0, buf_sz); + hse->input_size = 0; + hse->state = HSES_NOT_FULL; + hse->match_scan_index = 0; + hse->flags = 0; + hse->bit_index = 0x80; + hse->current_byte = 0x00; + hse->match_length = 0; + + hse->outgoing_bits = 0x0000; + hse->outgoing_bits_count = 0; + + #ifdef LOOP_DETECT + hse->loop_detect = (uint32_t)-1; + #endif +} + +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSER_SINK_ERROR_NULL; + } + + /* Sinking more content after saying the content is done, tsk tsk */ + if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } + + /* Sinking more content before processing is done */ + if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } + + uint16_t write_offset = get_input_offset(hse) + hse->input_size; + uint16_t ibs = get_input_buffer_size(hse); + uint16_t rem = ibs - hse->input_size; + uint16_t cp_sz = rem < size ? rem : size; + + memcpy(&hse->buffer[write_offset], in_buf, cp_sz); + *input_size = cp_sz; + hse->input_size += cp_sz; + + LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", + cp_sz, size, write_offset, hse->input_size); + if (cp_sz == rem) { + LOG("-- internal buffer is now full\n"); + hse->state = HSES_FILLED; + } + + return HSER_SINK_OK; +} + + +/*************** + * Compression * + ***************/ + +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length); +static void do_indexing(heatshrink_encoder *hse); + +static HSE_state st_step_search(heatshrink_encoder *hse); +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_save_backlog(heatshrink_encoder *hse); +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi); + +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSER_POLL_ERROR_NULL; + } + if (out_buf_size == 0) { + LOG("-- MISUSE: output buffer size is 0\n"); + return HSER_POLL_ERROR_MISUSE; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- polling, state %u (%s), flags 0x%02x\n", + hse->state, state_names[hse->state], hse->flags); + + uint8_t in_state = hse->state; + switch (in_state) { + case HSES_NOT_FULL: + return HSER_POLL_EMPTY; + case HSES_FILLED: + do_indexing(hse); + hse->state = HSES_SEARCH; + break; + case HSES_SEARCH: + hse->state = st_step_search(hse); + break; + case HSES_YIELD_TAG_BIT: + hse->state = st_yield_tag_bit(hse, &oi); + break; + case HSES_YIELD_LITERAL: + hse->state = st_yield_literal(hse, &oi); + break; + case HSES_YIELD_BR_INDEX: + hse->state = st_yield_br_index(hse, &oi); + break; + case HSES_YIELD_BR_LENGTH: + hse->state = st_yield_br_length(hse, &oi); + break; + case HSES_SAVE_BACKLOG: + hse->state = st_save_backlog(hse); + break; + case HSES_FLUSH_BITS: + hse->state = st_flush_bit_buffer(hse, &oi); + case HSES_DONE: + return HSER_POLL_EMPTY; + default: + LOG("-- bad state %s\n", state_names[hse->state]); + return HSER_POLL_ERROR_MISUSE; + } + + if (hse->state == in_state) { + /* Check if output buffer is exhausted. */ + if (*output_size == out_buf_size) return HSER_POLL_MORE; + } + } +} + +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { + if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } + LOG("-- setting is_finishing flag\n"); + hse->flags |= FLAG_IS_FINISHING; + if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } + return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; +} + +static HSE_state st_step_search(heatshrink_encoder *hse) { + uint16_t window_length = get_input_buffer_size(hse); + uint16_t lookahead_sz = get_lookahead_size(hse); + uint16_t msi = hse->match_scan_index; + LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", + msi, hse->input_size + msi, 2*window_length, hse->input_size); + + bool fin = is_finishing(hse); + if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) { + /* Current search buffer is exhausted, copy it into the + * backlog and await more input. */ + LOG("-- end of search @ %d, saving backlog\n", msi); + return HSES_SAVE_BACKLOG; + } + + uint16_t input_offset = get_input_offset(hse); + uint16_t end = input_offset + msi; + + uint16_t start = 0; + if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */ + start = end - window_length + 1; + } else if (backlog_is_partial(hse)) { /* clamp to available data */ + start = end - window_length + 1; + if (start < lookahead_sz) { start = lookahead_sz; } + } else { /* only scan available input */ + start = input_offset; + } + + uint16_t max_possible = lookahead_sz; + if (hse->input_size - msi < lookahead_sz) { + max_possible = hse->input_size - msi; + } + + uint16_t match_length = 0; + uint16_t match_pos = find_longest_match(hse, + start, end, max_possible, &match_length); + + if (match_pos == MATCH_NOT_FOUND) { + LOG("ss Match not found\n"); + hse->match_scan_index++; + hse->flags |= FLAG_HAS_LITERAL; + hse->match_length = 0; + return HSES_YIELD_TAG_BIT; + } else { + LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); + hse->match_pos = match_pos; + hse->match_length = match_length; + ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/); + + return HSES_YIELD_TAG_BIT; + } +} + +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + if (hse->match_length == 0) { + add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); + return HSES_YIELD_LITERAL; + } else { + add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); + hse->outgoing_bits = hse->match_pos - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); + return HSES_YIELD_BR_INDEX; + } + } else { + return HSES_YIELD_TAG_BIT; /* output is full, continue */ + } +} + +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + push_literal_byte(hse, oi); + hse->flags &= ~FLAG_HAS_LITERAL; + if (on_final_literal(hse)) { return HSES_FLUSH_BITS; } + return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH; + } else { + return HSES_YIELD_LITERAL; + } +} + +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref index %u\n", hse->match_pos); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_INDEX; /* continue */ + } else { + hse->outgoing_bits = hse->match_length - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); + return HSES_YIELD_BR_LENGTH; /* done */ + } + } else { + return HSES_YIELD_BR_INDEX; /* continue */ + } +} + +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref length %u\n", hse->match_length); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_LENGTH; + } else { + hse->match_scan_index += hse->match_length; + hse->match_length = 0; + return HSES_SEARCH; + } + } else { + return HSES_YIELD_BR_LENGTH; + } +} + +static HSE_state st_save_backlog(heatshrink_encoder *hse) { + if (is_finishing(hse)) { + /* copy remaining literal (if necessary) */ + if (has_literal(hse)) { + hse->flags |= FLAG_ON_FINAL_LITERAL; + return HSES_YIELD_TAG_BIT; + } else { + return HSES_FLUSH_BITS; + } + } else { + LOG("-- saving backlog\n"); + save_backlog(hse); + return HSES_NOT_FULL; + } +} + +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi) { + if (hse->bit_index == 0x80) { + LOG("-- done!\n"); + return HSES_DONE; + } else if (can_take_byte(oi)) { + LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); + oi->buf[(*oi->output_size)++] = hse->current_byte; + LOG("-- done!\n"); + return HSES_DONE; + } else { + return HSES_FLUSH_BITS; + } +} + +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { + LOG("-- adding tag bit: %d\n", tag); + push_bits(hse, 1, tag, oi); +} + +static uint16_t get_input_offset(heatshrink_encoder *hse) { + return get_input_buffer_size(hse); +} + +static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + (void)hse; +} + +static uint16_t get_lookahead_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); + (void)hse; +} + +static void do_indexing(heatshrink_encoder *hse) { +#if HEATSHRINK_USE_INDEX + /* Build an index array I that contains flattened linked lists + * for the previous instances of every byte in the buffer. + * + * For example, if buf[200] == 'x', then index[200] will either + * be an offset i such that buf[i] == 'x', or a negative offset + * to indicate end-of-list. This significantly speeds up matching, + * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. + * + * Future optimization options: + * 1. Since any negative value represents end-of-list, the other + * 15 bits could be used to improve the index dynamically. + * + * 2. Likewise, the last lookahead_sz bytes of the index will + * not be usable, so temporary data could be stored there to + * dynamically improve the index. + * */ + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + uint16_t last[256]; + memset(last, 0xFF, sizeof(last)); + + uint8_t * const data = hse->buffer; + int16_t * const index = hsi->index; + + const uint16_t input_offset = get_input_offset(hse); + const uint16_t end = input_offset + hse->input_size; + + for (uint16_t i=0; iflags & FLAG_IS_FINISHING; +} + +static int backlog_is_partial(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_PARTIAL; +} + +static int backlog_is_filled(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_FILLED; +} + +static int on_final_literal(heatshrink_encoder *hse) { + return hse->flags & FLAG_ON_FINAL_LITERAL; +} + +static int has_literal(heatshrink_encoder *hse) { + return (hse->flags & FLAG_HAS_LITERAL); +} + +static int can_take_byte(output_info *oi) { + return *oi->output_size < oi->buf_size; +} + +/* Return the longest match for the bytes at buf[end:end+maxlen] between + * buf[start] and buf[end-1]. If no match is found, return -1. */ +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length) { + LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", + end, end + maxlen, start, end + maxlen - 1, maxlen); + uint8_t *buf = hse->buffer; + + uint16_t match_maxlen = 0; + uint16_t match_index = MATCH_NOT_FOUND; + const uint16_t break_even_point = 3; + uint16_t len = 0; + uint8_t * const needlepoint = &buf[end]; +#if HEATSHRINK_USE_INDEX + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + int16_t pos = hsi->index[end]; + + while (pos >= start) { + uint8_t * const pospoint = &buf[pos]; + len = 0; + + /* Only check matches that will potentially beat the current maxlen. + * This is redundant with the index if match_maxlen is 0, but the + * added branch overhead to check if it == 0 seems to be worse. */ + if (pospoint[match_maxlen] != needlepoint[match_maxlen]) { + pos = hsi->index[pos]; + continue; + } + + for (len = 1; len < maxlen; len++) { + if (pospoint[len] != needlepoint[len]) break; + } + + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* won't find better */ + } + pos = hsi->index[pos]; + } +#else + for (int16_t pos=end - 1; pos >= start; pos--) { + uint8_t * const pospoint = &buf[pos]; + if ((pospoint[match_maxlen] == needlepoint[match_maxlen]) + && (*pospoint == *needlepoint)) { + for (len=1; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", + pos + len, pospoint[len], needlepoint[len], start); + } + if (pospoint[len] != needlepoint[len]) { break; } + } + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* don't keep searching */ + } + } + } +#endif + + if (match_maxlen >= break_even_point) { + LOG("-- best match: %u bytes at -%u\n", + match_maxlen, end - match_index); + *match_length = match_maxlen; + return end - match_index; + } + LOG("-- none found\n"); + return MATCH_NOT_FOUND; +} + +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { + uint8_t count = 0; + uint8_t bits = 0; + if (hse->outgoing_bits_count > 8) { + count = 8; + bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); + } else { + count = hse->outgoing_bits_count; + bits = hse->outgoing_bits; + } + + if (count > 0) { + LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); + push_bits(hse, count, bits, oi); + hse->outgoing_bits_count -= count; + } + return count; +} + +/* Push COUNT (max 8) bits to the output buffer, which has room. + * Bytes are set from the lowest bits, up. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi) { + ASSERT(count <= 8); + LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); + + /* If adding a whole byte and at the start of a new output byte, + * just push it through whole and skip the bit IO loop. */ + if (count == 8 && hse->bit_index == 0x80) { + oi->buf[(*oi->output_size)++] = bits; + } else { + for (int i=count - 1; i>=0; i--) { + bool bit = bits & (1 << i); + if (bit) { hse->current_byte |= hse->bit_index; } + if (0) { + LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", + bit ? 1 : 0, hse->bit_index, hse->current_byte); + } + hse->bit_index >>= 1; + if (hse->bit_index == 0x00) { + hse->bit_index = 0x80; + LOG(" > pushing byte 0x%02x\n", hse->current_byte); + oi->buf[(*oi->output_size)++] = hse->current_byte; + hse->current_byte = 0x00; + } + } + } +} + +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { + uint16_t processed_offset = hse->match_scan_index - 1; + uint16_t input_offset = get_input_offset(hse) + processed_offset; + uint8_t c = hse->buffer[input_offset]; + LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", + c, isprint(c) ? c : '.', input_offset); + push_bits(hse, 8, c, oi); +} + +static void save_backlog(heatshrink_encoder *hse) { + size_t input_buf_sz = get_input_buffer_size(hse); + + uint16_t msi = hse->match_scan_index; + + /* Copy processed data to beginning of buffer, so it can be + * used for future matches. Don't bother checking whether the + * input is less than the maximum size, because if it isn't, + * we're done anyway. */ + uint16_t rem = input_buf_sz - msi; // unprocessed bytes + uint16_t shift_sz = input_buf_sz + rem; + + memmove(&hse->buffer[0], + &hse->buffer[input_buf_sz - rem], + shift_sz); + + if (backlog_is_partial(hse)) { + /* The whole backlog is filled in now, so include it in scans. */ + hse->flags |= FLAG_BACKLOG_IS_FILLED; + } else { + /* Include backlog, except for the first lookahead_sz bytes, which + * are still undefined. */ + hse->flags |= FLAG_BACKLOG_IS_PARTIAL; + } + hse->match_scan_index = 0; + hse->input_size -= input_buf_sz - rem; +} diff --git a/espfs/heatshrink/heatshrink_encoder.h b/espfs/heatshrink/heatshrink_encoder.h new file mode 100644 index 0000000..18c1773 --- /dev/null +++ b/espfs/heatshrink/heatshrink_encoder.h @@ -0,0 +1,109 @@ +#ifndef HEATSHRINK_ENCODER_H +#define HEATSHRINK_ENCODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSER_SINK_OK, /* data sunk into input buffer */ + HSER_SINK_ERROR_NULL=-1, /* NULL argument */ + HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ +} HSE_sink_res; + +typedef enum { + HSER_POLL_EMPTY, /* input exhausted */ + HSER_POLL_MORE, /* poll again for more output */ + HSER_POLL_ERROR_NULL=-1, /* NULL argument */ + HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ +} HSE_poll_res; + +typedef enum { + HSER_FINISH_DONE, /* encoding is complete */ + HSER_FINISH_MORE, /* more output remaining; use poll */ + HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ +} HSE_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ + ((HSE)->window_sz2) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ + ((HSE)->lookahead_sz2) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + ((HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[]; +}; +#else +#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + (&(HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; +}; +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t match_scan_index; + uint16_t match_length; + uint16_t match_pos; + uint16_t outgoing_bits; /* enqueued outgoing bits */ + uint8_t outgoing_bits_count; + uint8_t flags; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of output */ + uint8_t bit_index; /* current bit index */ +#if HEATSHRINK_DYNAMIC_ALLOC + uint8_t window_sz2; /* 2^n size of window */ + uint8_t lookahead_sz2; /* 2^n size of lookahead */ +#if HEATSHRINK_USE_INDEX + struct hs_index *search_index; +#endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[]; +#else + #if HEATSHRINK_USE_INDEX + struct hs_index search_index; + #endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; +#endif +} heatshrink_encoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a new encoder struct and its buffers. + * Returns NULL on error. */ +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2); + +/* Free an encoder. */ +void heatshrink_encoder_free(heatshrink_encoder *hse); +#endif + +/* Reset an encoder. */ +void heatshrink_encoder_reset(heatshrink_encoder *hse); + +/* Sink up to SIZE bytes from IN_BUF into the encoder. + * INPUT_SIZE is set to the number of bytes actually sunk (in case a + * buffer was filled.). */ +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the encoder that the input stream is finished. + * If the return value is HSER_FINISH_MORE, there is still more output, so + * call heatshrink_encoder_poll and repeat. */ +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); + +#endif diff --git a/espfs/heatshrink/test_heatshrink_dynamic.c b/espfs/heatshrink/test_heatshrink_dynamic.c new file mode 100644 index 0000000..1e18a69 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_dynamic.c @@ -0,0 +1,999 @@ +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" + +#if !HEATSHRINK_DYNAMIC_ALLOC +#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for dynamic allocation test suite. +#endif + +SUITE(encoding); +SUITE(decoding); +SUITE(integration); + +#ifdef HEATSHRINK_HAS_THEFT +SUITE(properties); +#endif + +static void dump_buf(char *name, uint8_t *buf, uint16_t count) { + for (int i=0; iinput_size, 6); + ASSERT_EQ(hsd->input_index, 0); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_return_empty_if_empty(void) { + uint8_t output[256]; + size_t out_sz = 0; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_EMPTY, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_reject_null_hsd(void) { + uint8_t output[256]; + size_t out_sz = 0; + HSD_poll_res res = heatshrink_decoder_poll(NULL, output, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + PASS(); +} + +TEST decoder_poll_should_reject_null_output_buffer(void) { + size_t out_sz = 0; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, NULL, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_reject_null_output_size_pointer(void) { + uint8_t output[256]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, NULL); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_literal(void) { + uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0 }; //"foo" + uint8_t output[4]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 3); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + ASSERT_EQ(3, out_sz); + ASSERT_EQ('f', output[0]); + ASSERT_EQ('o', output[1]); + ASSERT_EQ('o', output[2]); + + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_literal_and_backref(void) { + uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo" + uint8_t output[6]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7); + memset(output, 0, sizeof(*output)); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + (void)heatshrink_decoder_poll(hsd, output, 6, &out_sz); + + if (0) dump_buf("output", output, out_sz); + ASSERT_EQ(6, out_sz); + ASSERT_EQ('f', output[0]); + ASSERT_EQ('o', output[1]); + ASSERT_EQ('o', output[2]); + ASSERT_EQ('f', output[3]); + ASSERT_EQ('o', output[4]); + ASSERT_EQ('o', output[5]); + + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_self_overlapping_backref(void) { + /* "aaaaa" == (literal, 1), ('a'), (backref, 1 back, 4 bytes) */ + uint8_t input[] = {0xb0, 0x80, 0x01, 0x80}; + uint8_t output[6]; + uint8_t expected[] = {'a', 'a', 'a', 'a', 'a'}; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 7); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + (void)heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); + + if (0) dump_buf("output", output, out_sz); + ASSERT_EQ(sizeof(expected), out_sz); + for (size_t i=0; iwindow_sz2, + cfg->lookahead_sz2); + heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size, + cfg->window_sz2, cfg->lookahead_sz2); + size_t comp_sz = input_size + (input_size/2) + 4; + size_t decomp_sz = input_size + (input_size/2) + 4; + uint8_t *comp = malloc(comp_sz); + uint8_t *decomp = malloc(decomp_sz); + if (comp == NULL) FAILm("malloc fail"); + if (decomp == NULL) FAILm("malloc fail"); + memset(comp, 0, comp_sz); + memset(decomp, 0, decomp_sz); + + size_t count = 0; + + if (cfg->log_lvl > 1) { + printf("\n^^ COMPRESSING\n"); + dump_buf("input", input, input_size); + } + + size_t sunk = 0; + size_t polled = 0; + while (sunk < input_size) { + ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0); + sunk += count; + if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + } + + HSE_poll_res pres; + do { /* "turn the crank" */ + pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSER_POLL_MORE); + ASSERT_EQ(HSER_POLL_EMPTY, pres); + if (polled >= comp_sz) FAILm("compression should never expand that much"); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); + } + } + if (cfg->log_lvl > 0) printf("in: %u compressed: %zu ", input_size, polled); + size_t compressed_size = polled; + sunk = 0; + polled = 0; + + if (cfg->log_lvl > 1) { + printf("\n^^ DECOMPRESSING\n"); + dump_buf("comp", comp, compressed_size); + } + while (sunk < compressed_size) { + ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); + sunk += count; + if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == compressed_size) { + ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd)); + } + + HSD_poll_res pres; + do { + pres = heatshrink_decoder_poll(hsd, &decomp[polled], + decomp_sz - polled, &count); + ASSERT(pres >= 0); + ASSERT(count > 0); + polled += count; + if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSDR_POLL_MORE); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + if (sunk == compressed_size) { + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + ASSERT_EQ(HSDR_FINISH_DONE, fres); + } + + if (polled > input_size) { + printf("\nExpected %zd, got %zu\n", (size_t)input_size, polled); + FAILm("Decompressed data is larger than original input"); + } + } + if (cfg->log_lvl > 0) printf("decompressed: %zu\n", polled); + if (polled != input_size) { + FAILm("Decompressed length does not match original input length"); + } + + if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled); + for (uint32_t i=0; i out[%d] == 0x%02x ('%c') %c\n", + j, input[j], isprint(input[j]) ? input[j] : '.', + j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.', + input[j] == decomp[j] ? ' ' : 'X'); + } + } + } + ASSERT_EQ(input[i], decomp[i]); + } + free(comp); + free(decomp); + heatshrink_encoder_free(hse); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST data_without_duplication_should_match(void) { + uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', + 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 256; + return compress_and_expand_and_check(input, sizeof(input), &cfg); +} + +TEST data_with_simple_repetition_should_compress_and_decompress_properly(void) { + uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', + 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h'}; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 256; + return compress_and_expand_and_check(input, sizeof(input), &cfg); +} + +TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) { + heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); + uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', + 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + uint8_t comp[60]; + uint8_t decomp[60]; + size_t count = 0; + int log = 0; + + if (log) dump_buf("input", input, sizeof(input)); + for (uint32_t i=0; i= 0); + } + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + + size_t packed_count = 0; + do { + ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); + packed_count += count; + } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); + + if (log) dump_buf("comp", comp, packed_count); + for (uint32_t i=0; i= 0); + } + + for (uint32_t i=0; i= 0); + } + + if (log) dump_buf("decomp", decomp, sizeof(input)); + for (uint32_t i=0; i= 0); + } + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + + size_t packed_count = 0; + do { + ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); + packed_count += count; + } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); + + if (log) dump_buf("comp", comp, packed_count); + for (uint32_t i=0; i= 0); + } + + for (uint32_t i=0; i= 0); + } + + if (log) dump_buf("decomp", decomp, sizeof(input)); + for (uint32_t i=0; ilog_lvl > 0) { + printf("\n-- size %u, seed %u, input buf %zu\n", + size, seed, cfg->decoder_input_buffer_size); + } + fill_with_pseudorandom_letters(input, size, seed); + return compress_and_expand_and_check(input, size, cfg); +} + +TEST small_input_buffer_should_not_impact_decoder_correctness(void) { + int size = 5; + uint8_t input[size]; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 5; + for (uint16_t i=0; i= 19901L + printf("\n\nFuzzing (single-byte sizes):\n"); + for (uint8_t lsize=3; lsize < 8; lsize++) { + for (uint32_t size=1; size < 128*1024L; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ + if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); + for (uint32_t seed=1; seed<=10; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = lsize; + cfg.decoder_input_buffer_size = ibs; + RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); + } + } + } + } + + printf("\nFuzzing (multi-byte sizes):\n"); + for (uint8_t lsize=6; lsize < 9; lsize++) { + for (uint32_t size=1; size < 128*1024L; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ + if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); + for (uint32_t seed=1; seed<=10; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 11; + cfg.lookahead_sz2 = lsize; + cfg.decoder_input_buffer_size = ibs; + RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); + } + } + } + } + +#endif +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + RUN_SUITE(encoding); + RUN_SUITE(decoding); + RUN_SUITE(integration); + #ifdef HEATSHRINK_HAS_THEFT + RUN_SUITE(properties); + #endif + GREATEST_MAIN_END(); /* display results */ +} diff --git a/espfs/heatshrink/test_heatshrink_dynamic_theft.c b/espfs/heatshrink/test_heatshrink_dynamic_theft.c new file mode 100644 index 0000000..2752886 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_dynamic_theft.c @@ -0,0 +1,521 @@ +#include "heatshrink_config.h" +#ifdef HEATSHRINK_HAS_THEFT + +#include +#include +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" +#include "theft.h" +#include "greatest_theft.h" + +#if !HEATSHRINK_DYNAMIC_ALLOC +#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for this test suite. +#endif + +SUITE(properties); + +typedef struct { + int limit; + int fails; + int dots; +} test_env; + +typedef struct { + size_t size; + uint8_t buf[]; +} rbuf; + +static void *rbuf_alloc_cb(struct theft *t, theft_hash seed, void *env) { + test_env *te = (test_env *)env; + //printf("seed is 0x%016llx\n", seed); + + size_t sz = (size_t)(seed % te->limit) + 1; + rbuf *r = malloc(sizeof(rbuf) + sz); + if (r == NULL) { return THEFT_ERROR; } + r->size = sz; + + for (size_t i = 0; i < sz; i += sizeof(theft_hash)) { + theft_hash s = theft_random(t); + for (uint8_t b = 0; b < sizeof(theft_hash); b++) { + if (i + b >= sz) { break; } + r->buf[i + b] = (uint8_t) (s >> (8*b)) & 0xff; + } + } + + return r; +} + +static void rbuf_free_cb(void *instance, void *env) { + free(instance); + (void)env; +} + +static uint64_t rbuf_hash_cb(void *instance, void *env) { + rbuf *r = (rbuf *)instance; + (void)env; + return theft_hash_onepass(r->buf, r->size); +} + +/* Make a copy of a buffer, keeping NEW_SZ bytes starting at OFFSET. */ +static void *copy_rbuf_subset(rbuf *cur, size_t new_sz, size_t byte_offset) { + if (new_sz == 0) { return THEFT_DEAD_END; } + rbuf *nr = malloc(sizeof(rbuf) + new_sz); + if (nr == NULL) { return THEFT_ERROR; } + nr->size = new_sz; + memcpy(nr->buf, &cur->buf[byte_offset], new_sz); + /* printf("%zu -> %zu\n", cur->size, new_sz); */ + return nr; +} + +/* Make a copy of a buffer, but only PORTION, starting OFFSET in + * (e.g. the third quarter is (0.25 at +0.75). Rounds to ints. */ +static void *copy_rbuf_percent(rbuf *cur, float portion, float offset) { + size_t new_sz = cur->size * portion; + size_t byte_offset = (size_t)(cur->size * offset); + return copy_rbuf_subset(cur, new_sz, byte_offset); +} + +/* How to shrink a random buffer to a simpler one. */ +static void *rbuf_shrink_cb(void *instance, uint32_t tactic, void *env) { + rbuf *cur = (rbuf *)instance; + + if (tactic == 0) { /* first half */ + return copy_rbuf_percent(cur, 0.5, 0); + } else if (tactic == 1) { /* second half */ + return copy_rbuf_percent(cur, 0.5, 0.5); + } else if (tactic <= 18) { /* drop 1-16 bytes at start */ + const int last_tactic = 1; + const size_t drop = tactic - last_tactic; + if (cur->size < drop) { return THEFT_DEAD_END; } + return copy_rbuf_subset(cur, cur->size - drop, drop); + } else if (tactic <= 34) { /* drop 1-16 bytes at end */ + const int last_tactic = 18; + const size_t drop = tactic - last_tactic; + if (cur->size < drop) { return THEFT_DEAD_END; } + return copy_rbuf_subset(cur, cur->size - drop, 0); + } else if (tactic == 35) { + /* Divide every byte by 2, saturating at 0 */ + rbuf *cp = copy_rbuf_percent(cur, 1, 0); + if (cp == NULL) { return THEFT_ERROR; } + for (size_t i = 0; i < cp->size; i++) { cp->buf[i] /= 2; } + return cp; + } else if (tactic == 36) { + /* subtract 1 from every byte, saturating at 0 */ + rbuf *cp = copy_rbuf_percent(cur, 1, 0); + if (cp == NULL) { return THEFT_ERROR; } + for (size_t i = 0; i < cp->size; i++) { + if (cp->buf[i] > 0) { cp->buf[i]--; } + } + return cp; + } else { + (void)env; + return THEFT_NO_MORE_TACTICS; + } + + return THEFT_NO_MORE_TACTICS; +} + +static void rbuf_print_cb(FILE *f, void *instance, void *env) { + rbuf *r = (rbuf *)instance; + (void)env; + fprintf(f, "buf[%zd]:\n ", r->size); + uint8_t bytes = 0; + for (size_t i = 0; i < r->size; i++) { + fprintf(f, "%02x", r->buf[i]); + bytes++; + if (bytes == 16) { + fprintf(f, "\n "); + bytes = 0; + } + } + fprintf(f, "\n"); +} + +static struct theft_type_info rbuf_info = { + .alloc = rbuf_alloc_cb, + .free = rbuf_free_cb, + .hash = rbuf_hash_cb, + .shrink = rbuf_shrink_cb, + .print = rbuf_print_cb, +}; + +static theft_progress_callback_res +progress_cb(struct theft_trial_info *info, void *env) { + test_env *te = (test_env *)env; + if ((info->trial & 0xff) == 0) { + printf("."); + fflush(stdout); + te->dots++; + if (te->dots == 64) { + printf("\n"); + te->dots = 0; + } + } + + if (info->status == THEFT_TRIAL_FAIL) { + te->fails++; + rbuf *cur = info->args[0]; + if (cur->size < 5) { return THEFT_PROGRESS_HALT; } + } + + if (te->fails > 10) { + return THEFT_PROGRESS_HALT; + } + return THEFT_PROGRESS_CONTINUE; +} + +/* For an arbitrary input buffer, it should never get stuck in a + * state where the data has been sunk but no data can be polled. */ +static theft_trial_res prop_should_not_get_stuck(void *input) { + /* Make a buffer large enough for the output: 4 KB of input with + * each 16 bits becoming up to 16 bytes will fit in a 64 KB buffer. + * (4 KB of input comes from `env.limit = 1 << 12;` below.) */ + uint8_t output[64 * 1024]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc((64 * 1024L) - 1, 12, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t count = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, r->buf, r->size, &count); + if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } + + size_t out_sz = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); + if (pres != HSDR_POLL_EMPTY) { return THEFT_TRIAL_FAIL; } + + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + heatshrink_decoder_free(hsd); + if (fres != HSDR_FINISH_DONE) { return THEFT_TRIAL_FAIL; } + + return THEFT_TRIAL_PASS; +} + +static bool get_time_seed(theft_seed *seed) +{ + struct timeval tv; + if (-1 == gettimeofday(&tv, NULL)) { return false; } + *seed = (theft_seed)((tv.tv_sec << 32) | tv.tv_usec); + /* printf("seed is 0x%016llx\n", *seed); */ + return true; +} + +TEST decoder_fuzzing_should_not_detect_stuck_state(void) { + // Get a random number seed based on the time + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + /* Pass the max buffer size for this property (4 KB) in a closure */ + test_env env = { .limit = 1 << 12 }; + + theft_seed always_seeds = { 0xe87bb1f61032a061 }; + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_should_not_get_stuck, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 100000, + .progress_cb = progress_cb, + .env = &env, + + .always_seeds = &always_seeds, + .always_seed_count = 1, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + GREATEST_ASSERT_EQm("should_not_get_stuck", THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res prop_encoded_and_decoded_data_should_match(void *input) { + uint8_t e_output[64 * 1024]; + uint8_t d_output[64 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + heatshrink_decoder *hsd = heatshrink_decoder_alloc(4096, 12, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t e_input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + r->buf, r->size, &e_input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + if (e_input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t e_output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + e_output, sizeof(e_output), &e_output_size); + if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t count = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, e_output, e_output_size, &count); + if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } + + size_t d_output_size = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, d_output, + sizeof(d_output), &d_output_size); + if (pres != HSDR_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + if (d_output_size != r->size) { + printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; + } + + if (0 != memcmp(d_output, r->buf, d_output_size)) { + return THEFT_TRIAL_FAIL; + } + + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + if (fres != HSDR_FINISH_DONE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + heatshrink_encoder_free(hse); + heatshrink_decoder_free(hsd); + return THEFT_TRIAL_PASS; +} + + +TEST encoded_and_decoded_data_should_match(void) { + test_env env = { .limit = 1 << 11 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoded_and_decoded_data_should_match, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 1000000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static size_t ceil_nine_eighths(size_t sz) { + return sz + sz/8 + (sz & 0x07 ? 1 : 0); +} + +static theft_trial_res +prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void *input) { + uint8_t output[32 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + r->buf, r->size, &input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + /* Assumes data fits in one sink, failure here means buffer must be larger. */ + if (input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + output, sizeof(output), &output_size); + if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t ceil_9_8s = ceil_nine_eighths(r->size); + if (output_size > ceil_9_8s) { + return THEFT_TRIAL_FAIL; + } + + heatshrink_encoder_free(hse); + return THEFT_TRIAL_PASS; +} + +TEST encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void) { + test_env env = { .limit = 1 << 11 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res +prop_encoder_should_always_make_progress(void *instance) { + uint8_t output[64 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)instance; + + size_t sunk = 0; + int no_progress = 0; + + while (1) { + if (sunk < r->size) { + size_t input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + &r->buf[sunk], r->size - sunk, &input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + sunk += input_size; + } else { + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres == HSER_FINISH_DONE) { + break; + } else if (efres != HSER_FINISH_MORE) { + printf("FAIL %d\n", __LINE__); + return THEFT_TRIAL_FAIL; + } + } + + size_t output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + output, sizeof(output), &output_size); + if (epres < 0) { return THEFT_TRIAL_ERROR; } + if (output_size == 0 && sunk == r->size) { + no_progress++; + if (no_progress > 2) { + return THEFT_TRIAL_FAIL; + } + } else { + no_progress = 0; + } + } + + heatshrink_encoder_free(hse); + return THEFT_TRIAL_PASS; +} + +TEST encoder_should_always_make_progress(void) { + test_env env = { .limit = 1 << 15 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoder_should_always_make_progress, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res +prop_decoder_should_always_make_progress(void *instance) { + uint8_t output[64 * 1024]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(512, 8, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)instance; + + size_t sunk = 0; + int no_progress = 0; + + while (1) { + if (sunk < r->size) { + size_t input_size = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, + &r->buf[sunk], r->size - sunk, &input_size); + if (sres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + sunk += input_size; + } else { + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + if (fres == HSDR_FINISH_DONE) { + break; + } else if (fres != HSDR_FINISH_MORE) { + printf("FAIL %d\n", __LINE__); + return THEFT_TRIAL_FAIL; + } + } + + size_t output_size = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, + output, sizeof(output), &output_size); + if (pres < 0) { return THEFT_TRIAL_ERROR; } + if (output_size == 0 && sunk == r->size) { + no_progress++; + if (no_progress > 2) { + return THEFT_TRIAL_FAIL; + } + } else { + no_progress = 0; + } + } + + heatshrink_decoder_free(hsd); + return THEFT_TRIAL_PASS; +} + +TEST decoder_should_always_make_progress(void) { + test_env env = { .limit = 1 << 15 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_decoder_should_always_make_progress, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +SUITE(properties) { + RUN_TEST(decoder_fuzzing_should_not_detect_stuck_state); + RUN_TEST(encoded_and_decoded_data_should_match); + RUN_TEST(encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst); + RUN_TEST(encoder_should_always_make_progress); + RUN_TEST(decoder_should_always_make_progress); +} +#else +struct because_iso_c_requires_at_least_one_declaration; +#endif diff --git a/espfs/heatshrink/test_heatshrink_static.c b/espfs/heatshrink/test_heatshrink_static.c new file mode 100644 index 0000000..e9c9754 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_static.c @@ -0,0 +1,167 @@ +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" + +#if HEATSHRINK_DYNAMIC_ALLOC +#error HEATSHRINK_DYNAMIC_ALLOC must be false for static allocation test suite. +#endif + +SUITE(integration); + +/* The majority of the tests are in test_heatshrink_dynamic, because that allows + * instantiating encoders/decoders with different settings at run-time. */ + +static heatshrink_encoder hse; +static heatshrink_decoder hsd; + +static void fill_with_pseudorandom_letters(uint8_t *buf, uint16_t size, uint32_t seed) { + uint64_t rn = 9223372036854775783; /* prime under 2^64 */ + for (int i=0; i 1) { + printf("\n^^ COMPRESSING\n"); + dump_buf("input", input, input_size); + } + + uint32_t sunk = 0; + uint32_t polled = 0; + while (sunk < input_size) { + ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); + sunk += count; + if (log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); + } + + HSE_poll_res pres; + do { /* "turn the crank" */ + pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSER_POLL_MORE); + ASSERT_EQ(HSER_POLL_EMPTY, pres); + if (polled >= comp_sz) FAILm("compression should never expand that much"); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); + } + } + if (log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled); + uint32_t compressed_size = polled; + sunk = 0; + polled = 0; + + if (log_lvl > 1) { + printf("\n^^ DECOMPRESSING\n"); + dump_buf("comp", comp, compressed_size); + } + while (sunk < compressed_size) { + ASSERT(heatshrink_decoder_sink(&hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); + sunk += count; + if (log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == compressed_size) { + ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(&hsd)); + } + + HSD_poll_res pres; + do { + pres = heatshrink_decoder_poll(&hsd, &decomp[polled], + decomp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSDR_POLL_MORE); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + if (sunk == compressed_size) { + HSD_finish_res fres = heatshrink_decoder_finish(&hsd); + ASSERT_EQ(HSDR_FINISH_DONE, fres); + } + + if (polled > input_size) { + FAILm("Decompressed data is larger than original input"); + } + } + if (log_lvl > 0) printf("decompressed: %u\n", polled); + if (polled != input_size) { + FAILm("Decompressed length does not match original input length"); + } + + if (log_lvl > 1) dump_buf("decomp", decomp, polled); + for (size_t i=0; i out[%zd] == 0x%02x ('%c')\n", + j, input[j], isprint(input[j]) ? input[j] : '.', + j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.'); + } + } + } + ASSERT_EQ(input[i], decomp[i]); + } + free(comp); + free(decomp); + PASS(); +} + +TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed) { + uint8_t input[size]; + fill_with_pseudorandom_letters(input, size, seed); + return compress_and_expand_and_check(input, size, 0); +} + +SUITE(integration) { +#if __STDC_VERSION__ >= 19901L + for (uint32_t size=1; size < 64*1024; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint32_t seed=1; seed<=100; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + RUN_TESTp(pseudorandom_data_should_match, size, seed); + } + } +#endif +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + printf("INPUT_BUFFER_SIZE: %u\n", HEATSHRINK_STATIC_INPUT_BUFFER_SIZE); + printf("WINDOW_BITS: %u\n", HEATSHRINK_STATIC_WINDOW_BITS); + printf("LOOKAHEAD_BITS: %u\n", HEATSHRINK_STATIC_LOOKAHEAD_BITS); + + printf("sizeof(heatshrink_encoder): %zd\n", sizeof(heatshrink_encoder)); + printf("sizeof(heatshrink_decoder): %zd\n", sizeof(heatshrink_decoder)); + RUN_SUITE(integration); + GREATEST_MAIN_END(); /* display results */ +} diff --git a/espfs/heatshrink_decoder.c b/espfs/heatshrink_decoder.c index 522b560..9a4a9f7 100644 --- a/espfs/heatshrink_decoder.c +++ b/espfs/heatshrink_decoder.c @@ -13,7 +13,7 @@ #endif #include "heatshrink_config_custom.h" -#include "../lib/heatshrink/heatshrink_decoder.c" +#include "heatshrink/heatshrink_decoder.c" #endif diff --git a/espfs/mkespfsimage/Makefile b/espfs/mkespfsimage/Makefile index 8a39d46..26484d4 100644 --- a/espfs/mkespfsimage/Makefile +++ b/espfs/mkespfsimage/Makefile @@ -1,24 +1,36 @@ GZIP_COMPRESSION ?= no USE_HEATSHRINK ?= yes -CFLAGS=-I../../lib/heatshrink -I.. -std=gnu99 +TARGET = mkespfsimage.exe + +CC = gcc +LD = $(CC) +CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 +LDFLAGS=-Lmman-win32 -lmman + ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP +CFLAGS += -DESPFS_GZIP +LDFLAGS += -lz endif ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK +CFLAGS += -DESPFS_HEATSHRINK endif -OBJS=main.o heatshrink_encoder.o -TARGET=mkespfsimage +OBJECTS = main.o heatshrink_encoder.o -$(TARGET): $(OBJS) -ifeq ("$(GZIP_COMPRESSION)","yes") - $(CC) -o $@ $^ -lz -else - $(CC) -o $@ $^ -endif +all: libmman $(TARGET) + +libmman: + $(Q) make -C mman-win32 + +$(TARGET): $(OBJECTS) + $(LD) -o $@ $^ $(LDFLAGS) + +%.o: %.c + $(CC) $(CFLAGS) -o $@ $^ clean: - rm -f $(TARGET) $(OBJS) \ No newline at end of file + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all clean diff --git a/espfs/mkespfsimage/Makefile.linux b/espfs/mkespfsimage/Makefile.linux new file mode 100644 index 0000000..f1ae1aa --- /dev/null +++ b/espfs/mkespfsimage/Makefile.linux @@ -0,0 +1,24 @@ +GZIP_COMPRESSION ?= no +USE_HEATSHRINK ?= yes + +CFLAGS=-I../../heatshrink -I../../include -I.. -std=gnu99 +ifeq ("$(GZIP_COMPRESSION)","yes") +CFLAGS += -DESPFS_GZIP +endif + +ifeq ("$(USE_HEATSHRINK)","yes") +CFLAGS += -DESPFS_HEATSHRINK +endif + +OBJS=main.o heatshrink_encoder.o +TARGET=mkespfsimage + +$(TARGET): $(OBJS) +ifeq ("$(GZIP_COMPRESSION)","yes") + $(CC) -o $@ $^ -lz +else + $(CC) -o $@ $^ +endif + +clean: + rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.windows b/espfs/mkespfsimage/Makefile.windows new file mode 100644 index 0000000..15aa8e3 --- /dev/null +++ b/espfs/mkespfsimage/Makefile.windows @@ -0,0 +1,33 @@ +GZIP_COMPRESSION ?= no +USE_HEATSHRINK ?= yes + +TARGET = mkespfsimage.exe + +CC = gcc +LD = $(CC) +CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 +LDFLAGS=-Lmman-win32 -lmman + +ifeq ("$(GZIP_COMPRESSION)","yes") +CFLAGS += -DESPFS_GZIP +LDFLAGS += -lz +endif + +ifeq ("$(USE_HEATSHRINK)","yes") +CFLAGS += -DESPFS_HEATSHRINK +endif + +OBJECTS = main.o heatshrink_encoder.o + +all: $(TARGET) + +$(TARGET): $(OBJECTS) + $(LD) -o $@ $^ $(LDFLAGS) + +%.o: %.c + $(CC) $(CFLAGS) -o $@ $^ + +clean: + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all clean diff --git a/espfs/mkespfsimage/build-linux.sh b/espfs/mkespfsimage/build-linux.sh new file mode 100644 index 0000000..69e3166 --- /dev/null +++ b/espfs/mkespfsimage/build-linux.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +make -f Makefile.linux clean +make -f Makefile.linux USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/build-win32.sh b/espfs/mkespfsimage/build-win32.sh new file mode 100644 index 0000000..f552ea9 --- /dev/null +++ b/espfs/mkespfsimage/build-win32.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd mman-win32 +./configure && make +cd .. +make -f Makefile.windows clean +make -f Makefile.windows USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/espfsformat.h b/espfs/mkespfsimage/espfsformat.h new file mode 100644 index 0000000..8ce5549 --- /dev/null +++ b/espfs/mkespfsimage/espfsformat.h @@ -0,0 +1,33 @@ +#ifndef ESPROFSFORMAT_H +#define ESPROFSFORMAT_H + +/* +Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module. +Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files, +headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself +when trying to do a <4byte or unaligned read. +*/ + +/* +The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data. +Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header +with the FLAG_LASTFILE flag set. +*/ + + +#define FLAG_LASTFILE (1<<0) +#define FLAG_GZIP (1<<1) +#define COMPRESS_NONE 0 +#define COMPRESS_HEATSHRINK 1 +#define ESPFS_MAGIC 0x73665345 + +typedef struct { + int32_t magic; + int8_t flags; + int8_t compression; + int16_t nameLen; + int32_t fileLenComp; + int32_t fileLenDecomp; +} __attribute__((packed)) EspFsHeader; + +#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/heatshrink_encoder.c b/espfs/mkespfsimage/heatshrink_encoder.c index b563970..fd82c9a 100644 --- a/espfs/mkespfsimage/heatshrink_encoder.c +++ b/espfs/mkespfsimage/heatshrink_encoder.c @@ -1,4 +1,4 @@ //Stupid wraparound include to make sure object file doesn't end up in heatshrink dir #ifdef ESPFS_HEATSHRINK -#include "../lib/heatshrink/heatshrink_encoder.c" +#include "../heatshrink/heatshrink_encoder.c" #endif \ No newline at end of file diff --git a/espfs/mkespfsimage/main.c b/espfs/mkespfsimage/main.c index 4d5da0d..20a2b00 100644 --- a/espfs/mkespfsimage/main.c +++ b/espfs/mkespfsimage/main.c @@ -5,22 +5,31 @@ #include #include #include +#include +#include "espfs.h" +#ifdef __MINGW32__ +#include +#else #include +#endif +#ifdef __WIN32__ +#include +#else #include +#endif #include -#include "espfs.h" #include "espfsformat.h" //Heatshrink #ifdef ESPFS_HEATSHRINK -#include "heatshrink_common.h" -#include "heatshrink_config.h" -#include "heatshrink_encoder.h" +#include "../heatshrink/heatshrink_common.h" +#include "../heatshrink/heatshrink_config.h" +#include "../heatshrink/heatshrink_encoder.h" #endif //Gzip #ifdef ESPFS_GZIP -// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" +// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" // to install missing package. #include #endif @@ -175,7 +184,7 @@ int parseGzipExtensions(char *input) { } #endif -int handleFile(int f, char *name, int compression, int level, char **compName, off_t *csizePtr) { +int handleFile(int f, char *name, int compression, int level, char **compName) { char *fdat, *cdat; off_t size, csize; EspFsHeader h; @@ -229,7 +238,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName, o h.nameLen=htoxs(h.nameLen); h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); - + write(1, &h, sizeof(EspFsHeader)); write(1, name, nameLen); while (nameLen&3) { @@ -257,7 +266,6 @@ int handleFile(int f, char *name, int compression, int level, char **compName, o *compName = "unknown"; } } - *csizePtr = csize; return (csize*100)/size; } @@ -310,7 +318,7 @@ int main(int argc, char **argv) { #ifdef ESPFS_GZIP if (gzipExtensions == NULL) { - parseGzipExtensions(strdup("html,css,js,ico")); + parseGzipExtensions(strdup("html,css,js")); } #endif @@ -334,6 +342,10 @@ int main(int argc, char **argv) { exit(0); } +#ifdef __WIN32__ + setmode(fileno(stdout), _O_BINARY); +#endif + while(fgets(fileName, sizeof(fileName), stdin)) { //Kill off '\n' at the end fileName[strlen(fileName)-1]=0; @@ -347,9 +359,8 @@ int main(int argc, char **argv) { f=open(fileName, O_RDONLY); if (f>0) { char *compName = "unknown"; - off_t csize; - rate=handleFile(f, realName, compType, compLvl, &compName, &csize); - fprintf(stderr, "%-16s (%3d%%, %s, %4u bytes)\n", realName, rate, compName, (uint32_t)csize); + rate=handleFile(f, realName, compType, compLvl, &compName); + fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); close(f); } else { perror(fileName); diff --git a/espfs/mkespfsimage/mman-win32/Makefile b/espfs/mkespfsimage/mman-win32/Makefile new file mode 100644 index 0000000..5624c1b --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/Makefile @@ -0,0 +1,48 @@ +# +# mman-win32 (mingw32) Makefile +# +include config.mak + +ifeq ($(BUILD_STATIC),yes) + TARGETS+=libmman.a + INSTALL+=static-install +endif +ifeq ($(BUILD_MSVC),yes) + SHFLAGS+=-Wl,--output-def,libmman.def + INSTALL+=lib-install +endif + +all: $(TARGETS) + +mman.o: mman.c mman.h + $(CC) -o mman.o -c mman.c -Wall -O3 -fomit-frame-pointer + +libmman.a: mman.o + $(AR) cru libmman.a mman.o + $(RANLIB) libmman.a + +static-install: + mkdir -p $(DESTDIR)$(libdir) + cp libmman.a $(DESTDIR)$(libdir) + mkdir -p $(DESTDIR)$(incdir) + cp mman.h $(DESTDIR)$(incdir) + +lib-install: + mkdir -p $(DESTDIR)$(libdir) + cp libmman.lib $(DESTDIR)$(libdir) + +install: $(INSTALL) + +test.exe: test.c mman.c mman.h + $(CC) -o test.exe test.c -L. -lmman + +test: $(TARGETS) test.exe + test.exe + +clean:: + rm -f mman.o libmman.a libmman.def libmman.lib test.exe *.dat + +distclean: clean + rm -f config.mak + +.PHONY: clean distclean install test diff --git a/espfs/mkespfsimage/mman-win32/config.mak b/espfs/mkespfsimage/mman-win32/config.mak new file mode 100644 index 0000000..2d2ae1b --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/config.mak @@ -0,0 +1,11 @@ +# Automatically generated by configure +PREFIX=/mingw +libdir=/mingw/lib +incdir=/mingw/include/sys +AR=ar +CC=gcc +RANLIB=ranlib +STRIP=strip +BUILD_STATIC=yes +BUILD_MSVC= +LIBCMD=echo ignoring lib diff --git a/espfs/mkespfsimage/mman-win32/configure b/espfs/mkespfsimage/mman-win32/configure new file mode 100644 index 0000000..c928f11 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/configure @@ -0,0 +1,157 @@ +#!/bin/sh +# mmap-win32 configure script +# +# Parts copied from FFmpeg's configure +# + +set_all(){ + value=$1 + shift + for var in $*; do + eval $var=$value + done +} + +enable(){ + set_all yes $* +} + +disable(){ + set_all no $* +} + +enabled(){ + eval test "x\$$1" = "xyes" +} + +disabled(){ + eval test "x\$$1" = "xno" +} + +show_help(){ + echo "Usage: configure [options]" + echo "Options: [defaults in brackets after descriptions]" + echo "All \"enable\" options have \"disable\" counterparts" + echo + echo " --help print this message" + echo " --prefix=PREFIX install in PREFIX [$PREFIX]" + echo " --libdir=DIR install libs in DIR [$PREFIX/lib]" + echo " --incdir=DIR install includes in DIR [$PREFIX/include]" + echo " --enable-static build static libraries [yes]" + echo " --enable-msvc create msvc-compatible import lib [auto]" + echo + echo " --cc=CC use C compiler CC [$cc_default]" + echo " --cross-prefix=PREFIX use PREFIX for compilation tools [$cross_prefix]" + exit 1 +} + +die_unknown(){ + echo "Unknown option \"$1\"." + echo "See $0 --help for available options." + exit 1 +} + +PREFIX="/mingw" +libdir="${PREFIX}/lib" +incdir="${PREFIX}/include/sys" +ar="ar" +cc_default="gcc" +ranlib="ranlib" +strip="strip" + +DEFAULT="msvc +" + +DEFAULT_YES="static + stripping +" + +CMDLINE_SELECT="$DEFAULT + $DEFAULT_NO + $DEFAULT_YES +" + +enable $DEFAULT_YES +disable $DEFAULT_NO + +for opt do + optval="${opt#*=}" + case "$opt" in + --help) + show_help + ;; + --prefix=*) + PREFIX="$optval" + ;; + --libdir=*) + libdir="$optval" + ;; + --incdir=*) + incdir="$optval" + ;; + --cc=*) + cc="$optval" + ;; + --cross-prefix=*) + cross_prefix="$optval" + ;; + --enable-?*|--disable-?*) + eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'` + echo "$CMDLINE_SELECT" | grep -q "^ *$option\$" || die_unknown $opt + $action $option + ;; + *) + die_unknown $opt + ;; + esac +done + +ar="${cross_prefix}${ar}" +cc_default="${cross_prefix}${cc_default}" +ranlib="${cross_prefix}${ranlib}" +strip="${cross_prefix}${strip}" + +if ! test -z $cc; then + cc_default="${cc}" +fi +cc="${cc_default}" + +disabled static && { + echo "At least one library type must be set."; + exit 1; +} + +if enabled msvc; then + lib /? > /dev/null 2>&1 /dev/null || { + echo "MSVC's lib command not found." + echo "Make sure MSVC is installed and its bin folder is in your \$PATH." + exit 1 + } +fi + +if ! enabled stripping; then + strip="echo ignoring strip" +fi + +enabled msvc && libcmd="lib" || libcmd="echo ignoring lib" + +echo "# Automatically generated by configure" > config.mak +echo "PREFIX=$PREFIX" >> config.mak +echo "libdir=$libdir" >> config.mak +echo "incdir=$incdir" >> config.mak +echo "AR=$ar" >> config.mak +echo "CC=$cc" >> config.mak +echo "RANLIB=$ranlib" >> config.mak +echo "STRIP=$strip" >> config.mak +echo "BUILD_STATIC=$static" >> config.mak +echo "BUILD_MSVC=$msvc" >> config.mak +echo "LIBCMD=$libcmd" >> config.mak + +echo "prefix: $PREFIX" +echo "libdir: $libdir" +echo "incdir: $incdir" +echo "ar: $ar" +echo "cc: $cc" +echo "ranlib: $ranlib" +echo "strip: $strip" +echo "static: $static" diff --git a/espfs/mkespfsimage/mman-win32/mman.c b/espfs/mkespfsimage/mman-win32/mman.c new file mode 100644 index 0000000..486ed94 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/mman.c @@ -0,0 +1,180 @@ + +#include +#include +#include + +#include "mman.h" + +#ifndef FILE_MAP_EXECUTE +#define FILE_MAP_EXECUTE 0x0020 +#endif /* FILE_MAP_EXECUTE */ + +static int __map_mman_error(const DWORD err, const int deferr) +{ + if (err == 0) + return 0; + //TODO: implement + return err; +} + +static DWORD __map_mmap_prot_page(const int prot) +{ + DWORD protect = 0; + + if (prot == PROT_NONE) + return protect; + + if ((prot & PROT_EXEC) != 0) + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; + } + else + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_READWRITE : PAGE_READONLY; + } + + return protect; +} + +static DWORD __map_mmap_prot_file(const int prot) +{ + DWORD desiredAccess = 0; + + if (prot == PROT_NONE) + return desiredAccess; + + if ((prot & PROT_READ) != 0) + desiredAccess |= FILE_MAP_READ; + if ((prot & PROT_WRITE) != 0) + desiredAccess |= FILE_MAP_WRITE; + if ((prot & PROT_EXEC) != 0) + desiredAccess |= FILE_MAP_EXECUTE; + + return desiredAccess; +} + +void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off) +{ + HANDLE fm, h; + + void * map = MAP_FAILED; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4293) +#endif + + const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)off : (DWORD)(off & 0xFFFFFFFFL); + const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL); + const DWORD protect = __map_mmap_prot_page(prot); + const DWORD desiredAccess = __map_mmap_prot_file(prot); + + const off_t maxSize = off + (off_t)len; + + const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL); + const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL); + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + errno = 0; + + if (len == 0 + /* Unsupported flag combinations */ + || (flags & MAP_FIXED) != 0 + /* Usupported protection combinations */ + || prot == PROT_EXEC) + { + errno = EINVAL; + return MAP_FAILED; + } + + h = ((flags & MAP_ANONYMOUS) == 0) ? + (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE; + + if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return MAP_FAILED; + } + + fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL); + + if (fm == NULL) + { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len); + + CloseHandle(fm); + + if (map == NULL) + { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + return map; +} + +int munmap(void *addr, size_t len) +{ + if (UnmapViewOfFile(addr)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int mprotect(void *addr, size_t len, int prot) +{ + DWORD newProtect = __map_mmap_prot_page(prot); + DWORD oldProtect = 0; + + if (VirtualProtect(addr, len, newProtect, &oldProtect)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int msync(void *addr, size_t len, int flags) +{ + if (FlushViewOfFile(addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int mlock(const void *addr, size_t len) +{ + if (VirtualLock((LPVOID)addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int munlock(const void *addr, size_t len) +{ + if (VirtualUnlock((LPVOID)addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} diff --git a/espfs/mkespfsimage/mman-win32/mman.h b/espfs/mkespfsimage/mman-win32/mman.h new file mode 100644 index 0000000..ffa3748 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/mman.h @@ -0,0 +1,55 @@ +/* + * sys/mman.h + * mman-win32 + */ + +#ifndef _SYS_MMAN_H_ +#define _SYS_MMAN_H_ + +#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later. +#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows. +#endif + +/* All the headers include this file. */ +#ifndef _MSC_VER +#include <_mingw.h> +#endif + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define PROT_NONE 0 +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + +#define MAP_FILE 0 +#define MAP_SHARED 1 +#define MAP_PRIVATE 2 +#define MAP_TYPE 0xf +#define MAP_FIXED 0x10 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS + +#define MAP_FAILED ((void *)-1) + +/* Flags for msync. */ +#define MS_ASYNC 1 +#define MS_SYNC 2 +#define MS_INVALIDATE 4 + +void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off); +int munmap(void *addr, size_t len); +int mprotect(void *addr, size_t len, int prot); +int msync(void *addr, size_t len, int flags); +int mlock(const void *addr, size_t len); +int munlock(const void *addr, size_t len); + +#ifdef __cplusplus +}; +#endif + +#endif /* _SYS_MMAN_H_ */ diff --git a/espfs/mkespfsimage/mman-win32/test.c b/espfs/mkespfsimage/mman-win32/test.c new file mode 100644 index 0000000..9374b9f --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/test.c @@ -0,0 +1,235 @@ + +#include "mman.h" + +#include +#include +#include + +#ifndef NULL +#define NULL (void*)0 +#endif + +const char* map_file_name = "map_file.dat"; + +int test_anon_map_readwrite() +{ + void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_readonly() +{ + void* map = mmap(NULL, 1024, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_writeonly() +{ + void* map = mmap(NULL, 1024, PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_readonly_nowrite() +{ + void* map = mmap(NULL, 1024, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + return -1; + } + + if (*((unsigned char*)map) != 0) + printf("test_anon_map_readonly_nowrite (MAP_ANONYMOUS, PROT_READ) returned unexpected value: %d\n", + (int)*((unsigned char*)map)); + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + + return result; +} + +int test_file_map_readwrite() +{ + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + + void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + + close(o); + + /*TODO: get file info and content and compare it with the sources conditions */ + unlink(map_file_name); + + return result; +} + +int test_file_map_mlock_munlock() +{ + const size_t map_size = 1024; + + int result = 0; + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + if (o == -1) + { + printf("unable to create file %s: %d\n", map_file_name, errno); + return -1; + } + + void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + result = -1; + goto done_close; + } + + if (mlock(map, map_size) != 0) + { + printf("mlock returned unexpected error: %d\n", errno); + result = -1; + goto done_munmap; + } + + *((unsigned char*)map) = 1; + + if (munlock(map, map_size) != 0) + { + printf("munlock returned unexpected error: %d\n", errno); + result = -1; + } + +done_munmap: + result = munmap(map, map_size); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + +done_close: + close(o); + + unlink(map_file_name); +done: + return result; +} + +int test_file_map_msync() +{ + const size_t map_size = 1024; + + int result = 0; + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + if (o == -1) + { + printf("unable to create file %s: %d\n", map_file_name, errno); + return -1; + } + + void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + result = -1; + goto done_close; + } + + *((unsigned char*)map) = 1; + + if (msync(map, map_size, MS_SYNC) != 0) + { + printf("msync returned unexpected error: %d\n", errno); + result = -1; + } + + result = munmap(map, map_size); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + +done_close: + close(o); + + unlink(map_file_name); +done: + return result; +} + +#define EXEC_TEST(name) \ + if (name() != 0) { result = -1; printf( #name ": fail\n"); } \ + else { printf(#name ": pass\n"); } + +int main() +{ + int result = 0; + + EXEC_TEST(test_anon_map_readwrite); + //NOTE: this test must cause an access violation exception + //EXEC_TEST(test_anon_map_readonly); + EXEC_TEST(test_anon_map_readonly_nowrite); + EXEC_TEST(test_anon_map_writeonly); + + EXEC_TEST(test_file_map_readwrite); + EXEC_TEST(test_file_map_mlock_munlock); + EXEC_TEST(test_file_map_msync); + //TODO: EXEC_TEST(test_file_map_mprotect); + + return result; +} diff --git a/espmake.cmd b/espmake.cmd new file mode 100644 index 0000000..42e25a6 --- /dev/null +++ b/espmake.cmd @@ -0,0 +1,7 @@ +@echo off + +REM remove automatic created obj folder +rd obj /S /Q >nul 2>&1 + +PATH=%PATH%;C:\Espressif\xtensa-lx106-elf\bin;C:\MinGW\bin;C:\MinGW\msys\1.0\bin;C:\espressif\git-bin;C:\espressif\java-bin;C:\Python27 +make -f Makefile %1 %2 %3 %4 %5 \ No newline at end of file From 3d2ac525e3b594a27dc256077f6b1d748694b081 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 05:51:46 -0500 Subject: [PATCH 15/23] Removed heatshrink and modified Makefiles --- Makefile | 17 +- esp-link.vcxproj | 62 +- espfs/espfs.c | 83 +- espfs/espfs.h | 4 - espfs/heatshrink/LICENSE | 14 - espfs/heatshrink/Makefile | 49 - espfs/heatshrink/README.md | 49 - espfs/heatshrink/dec_sm.dot | 52 - espfs/heatshrink/enc_sm.dot | 51 - espfs/heatshrink/greatest.h | 591 ----------- espfs/heatshrink/heatshrink.c | 446 -------- espfs/heatshrink/heatshrink_common.h | 20 - espfs/heatshrink/heatshrink_config.h | 24 - espfs/heatshrink/heatshrink_decoder.c | 382 ------- espfs/heatshrink/heatshrink_decoder.h | 101 -- espfs/heatshrink/heatshrink_encoder.c | 650 ------------ espfs/heatshrink/heatshrink_encoder.h | 109 -- espfs/heatshrink/test_heatshrink_dynamic.c | 999 ------------------ .../test_heatshrink_dynamic_theft.c | 521 --------- espfs/heatshrink/test_heatshrink_static.c | 167 --- espfs/heatshrink_config_custom.h | 30 - espfs/heatshrink_decoder.c | 19 - espfs/mkespfsimage/Makefile | 40 +- espfs/mkespfsimage/Makefile.linux | 24 - espfs/mkespfsimage/Makefile.windows | 33 - espfs/mkespfsimage/build-linux.sh | 4 - espfs/mkespfsimage/build-win32.sh | 7 - espfs/mkespfsimage/espfsformat.h | 33 - espfs/mkespfsimage/heatshrink_encoder.c | 4 - espfs/mkespfsimage/main.c | 87 +- include/esp8266.h | 5 +- include/espmissingincludes.h | 10 +- 32 files changed, 50 insertions(+), 4637 deletions(-) delete mode 100644 espfs/heatshrink/LICENSE delete mode 100644 espfs/heatshrink/Makefile delete mode 100644 espfs/heatshrink/README.md delete mode 100644 espfs/heatshrink/dec_sm.dot delete mode 100644 espfs/heatshrink/enc_sm.dot delete mode 100644 espfs/heatshrink/greatest.h delete mode 100644 espfs/heatshrink/heatshrink.c delete mode 100644 espfs/heatshrink/heatshrink_common.h delete mode 100644 espfs/heatshrink/heatshrink_config.h delete mode 100644 espfs/heatshrink/heatshrink_decoder.c delete mode 100644 espfs/heatshrink/heatshrink_decoder.h delete mode 100644 espfs/heatshrink/heatshrink_encoder.c delete mode 100644 espfs/heatshrink/heatshrink_encoder.h delete mode 100644 espfs/heatshrink/test_heatshrink_dynamic.c delete mode 100644 espfs/heatshrink/test_heatshrink_dynamic_theft.c delete mode 100644 espfs/heatshrink/test_heatshrink_static.c delete mode 100644 espfs/heatshrink_config_custom.h delete mode 100644 espfs/heatshrink_decoder.c delete mode 100644 espfs/mkespfsimage/Makefile.linux delete mode 100644 espfs/mkespfsimage/Makefile.windows delete mode 100644 espfs/mkespfsimage/build-linux.sh delete mode 100644 espfs/mkespfsimage/build-win32.sh delete mode 100644 espfs/mkespfsimage/espfsformat.h delete mode 100644 espfs/mkespfsimage/heatshrink_encoder.c diff --git a/Makefile b/Makefile index 297e4fe..af5fcbf 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,6 @@ CHANGE_TO_STA ?= yes # This could speed up the downloading of these files, but might break compatibility with older # web browsers not supporting gzip encoding because Accept-Encoding is simply ignored. # Enable this option if you have large static files to serve (for e.g. JQuery, Twitter bootstrap) -# By default only js, css and html files are compressed using heatshrink. # If you have text based static files with different extensions what you want to serve compressed # then you will need to add the extension to the following places: # - Add the extension to this Makefile at the webpages.espfs target to the find command @@ -127,10 +126,6 @@ GZIP_COMPRESSION ?= yes COMPRESS_W_YUI ?= yes YUI-COMPRESSOR ?= yuicompressor-2.4.8.jar -# If USE_HEATSHRINK is set to "yes" then the espfs files will be compressed with Heatshrink and -# decompressed on the fly while reading the file. -# Because the decompression is done in the esp8266, it does not require any support in the browser. -USE_HEATSHRINK ?= no # -------------- End of config options ------------- @@ -147,7 +142,7 @@ APPGEN_TOOL ?= gen_appbin.py # which modules (subdirectories) of the project to include in compiling MODULES = espfs httpd user serial -EXTRA_INCDIR = include . # lib/heatshrink/ +EXTRA_INCDIR = include . # libraries used in this project, mainly provided by the SDK LIBS = c gcc hal phy pp net80211 wpa main lwip @@ -216,10 +211,6 @@ ifeq ("$(GZIP_COMPRESSION)","yes") CFLAGS += -DGZIP_COMPRESSION endif -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - ifeq ("$(CHANGE_TO_STA)","yes") CFLAGS += -DCHANGE_TO_STA endif @@ -299,7 +290,11 @@ flash: all yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui + ifeq ($(OS),Windows_NT) cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + else + cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) + endif ifeq ("$(COMPRESS_W_YUI)","yes") $(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR) @@ -351,7 +346,7 @@ build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld endif espfs/mkespfsimage/mkespfsimage: espfs/mkespfsimage/ - $(Q) $(MAKE) -C espfs/mkespfsimage USE_HEATSHRINK="$(USE_HEATSHRINK)" GZIP_COMPRESSION="$(GZIP_COMPRESSION)" + $(Q) $(MAKE) -C espfs/mkespfsimage GZIP_COMPRESSION="$(GZIP_COMPRESSION)" release: all $(Q) rm -rf release; mkdir -p release/esp-link diff --git a/esp-link.vcxproj b/esp-link.vcxproj index 45dd2ae..d462b62 100644 --- a/esp-link.vcxproj +++ b/esp-link.vcxproj @@ -28,7 +28,7 @@ __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ - .\mqtt\include;.\serial;.\user;.\espfs;.\httpd;.\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include;c:\Espressif\sdk\include + .\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include @@ -54,17 +54,10 @@ - - - - - - - @@ -72,8 +65,6 @@ - - @@ -81,35 +72,13 @@ - - - - - - - - - - - - - - - - - - - - - - @@ -119,8 +88,6 @@ - - @@ -130,40 +97,13 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/espfs/espfs.c b/espfs/espfs.c index 1f503d5..128ca29 100644 --- a/espfs/espfs.c +++ b/espfs/espfs.c @@ -7,9 +7,9 @@ It's written for use with httpd, but doesn't need to be used as such. /* * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): - * Jeroen Domburg wrote this file. As long as you retain - * this notice you can do whatever you want with this stuff. If we meet some day, - * and you think this stuff is worth it, you can buy me a beer in return. + * Jeroen Domburg wrote this file. As long as you retain + * this notice you can do whatever you want with this stuff. If we meet some day, + * and you think this stuff is worth it, you can buy me a beer in return. * ---------------------------------------------------------------------------- */ @@ -40,14 +40,8 @@ It's written for use with httpd, but doesn't need to be used as such. #include "espfsformat.h" #include "espfs.h" -#ifdef ESPFS_HEATSHRINK -#include "heatshrink_config_custom.h" -#include "heatshrink/heatshrink_decoder.h" -#endif - static char* espFsData = NULL; - struct EspFsFile { EspFsHeader *header; char decompressor; @@ -147,19 +141,19 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { return NULL; } if (h.flags&FLAG_LASTFILE) { - os_printf("End of image.\n"); + //os_printf("End of image.\n"); return NULL; } //Grab the name of the file. - p+=sizeof(EspFsHeader); + p+=sizeof(EspFsHeader); os_memcpy(namebuf, p, sizeof(namebuf)); -// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", +// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", // namebuf, (unsigned int)h.nameLen, (unsigned int)h.fileLenComp, h.compression, h.flags); if (os_strcmp(namebuf, fileName)==0) { //Yay, this is the file we need! p+=h.nameLen; //Skip to content. r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem -// os_printf("Alloc %p\n", r); + //os_printf("Alloc %p[%d]\n", r, sizeof(EspFsFile)); if (r==NULL) return NULL; r->header=(EspFsHeader *)hpos; r->decompressor=h.compression; @@ -168,18 +162,6 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { r->posDecomp=0; if (h.compression==COMPRESS_NONE) { r->decompData=NULL; -#ifdef ESPFS_HEATSHRINK - } else if (h.compression==COMPRESS_HEATSHRINK) { - //File is compressed with Heatshrink. - char parm; - heatshrink_decoder *dec; - //Decoder params are stored in 1st byte. - memcpyAligned(&parm, r->posComp, 1); - r->posComp++; - os_printf("Heatshrink compressed file; decode parms = %x\n", parm); - dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf); - r->decompData=dec; -#endif } else { os_printf("Invalid compression: %d\n", h.compression); return NULL; @@ -210,48 +192,6 @@ int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) { fh->posComp+=len; // os_printf("Done reading %d bytes, pos=%x\n", len, fh->posComp); return len; -#ifdef ESPFS_HEATSHRINK - } else if (fh->decompressor==COMPRESS_HEATSHRINK) { - int decoded=0; - size_t elen, rlen; - char ebuff[16]; - heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData; -// os_printf("Alloc %p\n", dec); - if (fh->posDecomp == fdlen) { - return 0; - } - - // We must ensure that whole file is decompressed and written to output buffer. - // This means even when there is no input data (elen==0) try to poll decoder until - // posDecomp equals decompressed file length - - while(decodedposComp - fh->posStart); - if (elen>0) { - memcpyAligned(ebuff, fh->posComp, 16); - heatshrink_decoder_sink(dec, (uint8_t *)ebuff, (elen>16)?16:elen, &rlen); - fh->posComp+=rlen; - } - //Grab decompressed data and put into buff - heatshrink_decoder_poll(dec, (uint8_t *)buff, len-decoded, &rlen); - fh->posDecomp+=rlen; - buff+=rlen; - decoded+=rlen; - -// os_printf("Elen %d rlen %d d %d pd %ld fdl %d\n",elen,rlen,decoded, fh->posDecomp, fdlen); - - if (elen == 0) { - if (fh->posDecomp == fdlen) { -// os_printf("Decoder finish\n"); - heatshrink_decoder_finish(dec); - } - return decoded; - } - } - return len; -#endif } return 0; } @@ -259,14 +199,7 @@ int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) { //Close the file. void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) { if (fh==NULL) return; -#ifdef ESPFS_HEATSHRINK - if (fh->decompressor==COMPRESS_HEATSHRINK) { - heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData; - heatshrink_decoder_free(dec); -// os_printf("Freed %p\n", dec); - } -#endif -// os_printf("Freed %p\n", fh); + //os_printf("Freed %p\n", fh); os_free(fh); } diff --git a/espfs/espfs.h b/espfs/espfs.h index c41df0a..c8e13e7 100644 --- a/espfs/espfs.h +++ b/espfs/espfs.h @@ -1,10 +1,6 @@ #ifndef ESPFS_H #define ESPFS_H -// This define is done in Makefile. If you do not use default Makefile, uncomment -// to be able to use Heatshrink-compressed espfs images. -//#define ESPFS_HEATSHRINK - typedef enum { ESPFS_INIT_RESULT_OK, ESPFS_INIT_RESULT_NO_IMAGE, diff --git a/espfs/heatshrink/LICENSE b/espfs/heatshrink/LICENSE deleted file mode 100644 index 31ec3df..0000000 --- a/espfs/heatshrink/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (c) 2013, Scott Vokes -All rights reserved. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/espfs/heatshrink/Makefile b/espfs/heatshrink/Makefile deleted file mode 100644 index d8e6875..0000000 --- a/espfs/heatshrink/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -PROJECT = heatshrink -#OPTIMIZE = -O0 -#OPTIMIZE = -Os -OPTIMIZE = -O3 -WARN = -Wall -Wextra -pedantic #-Werror -CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE} -CFLAGS += -Wmissing-prototypes -CFLAGS += -Wstrict-prototypes -CFLAGS += -Wmissing-declarations - -# If libtheft is available, build additional property-based tests. -# Uncomment these to use it in test_heatshrink_dynamic. -#CFLAGS += -DHEATSHRINK_HAS_THEFT -#LDFLAGS += -ltheft - -all: - @echo "For tests, make test_heatshrink_dynamic (default) or change the" - @echo "config.h to disable static memory and build test_heatshrink_static." - @echo "For the standalone command-line tool, make heatshrink." - -${PROJECT}: heatshrink.c - -OBJS= heatshrink_encoder.o \ - heatshrink_decoder.o \ - -heatshrink: ${OBJS} -test_heatshrink_dynamic: ${OBJS} test_heatshrink_dynamic_theft.o -test_heatshrink_static: ${OBJS} - -*.o: Makefile heatshrink_config.h - -heatshrink_decoder.o: heatshrink_decoder.h heatshrink_common.h -heatshrink_encoder.o: heatshrink_encoder.h heatshrink_common.h - -tags: TAGS - -TAGS: - etags *.[ch] - -diagrams: dec_sm.png enc_sm.png - -dec_sm.png: dec_sm.dot - dot -o $@ -Tpng $< - -enc_sm.png: enc_sm.dot - dot -o $@ -Tpng $< - -clean: - rm -f ${PROJECT} test_heatshrink_{dynamic,static} *.o *.core {dec,enc}_sm.png TAGS diff --git a/espfs/heatshrink/README.md b/espfs/heatshrink/README.md deleted file mode 100644 index ab150ee..0000000 --- a/espfs/heatshrink/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# heatshrink - -A data compression/decompression library for embedded/real-time systems. - -## Key Features: - -- **Low memory usage (as low as 50 bytes)** - It is useful for some cases with less than 50 bytes, and useful - for many general cases with < 300 bytes. -- **Incremental, bounded CPU use** - You can chew on input data in arbitrarily tiny bites. - This is a useful property in hard real-time environments. -- **Can use either static or dynamic memory allocation** - The library doesn't impose any constraints on memory management. -- **ISC license** - You can use it freely, even for commercial purposes. - -## Getting Started: - -There is a standalone command-line program, `heatshrink`, but the -encoder and decoder can also be used as libraries, independent of each -other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and -either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their -respective header) into your project. - -Dynamic allocation is used by default, but in an embedded context, you -probably want to statically allocate the encoder/decoder. Set -`HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`. - -## More Information and Benchmarks: - -heatshrink is based on [LZSS], since it's particularly suitable for -compression in small amounts of memory. It can use an optional, small -[index] to make compression significantly faster, but otherwise can run -in under 100 bytes of memory. The index currently adds 2^(window size+1) -bytes to memory usage for compression, and temporarily allocates 512 -bytes on the stack during index construction. - -For more information, see the [blog post] for an overview, and the -`heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API -documentation. - -[blog post]: http://spin.atomicobject.com/2013/03/14/heatshrink-embedded-data-compression/ -[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/ -[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski - -## Build Status - - [![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink) diff --git a/espfs/heatshrink/dec_sm.dot b/espfs/heatshrink/dec_sm.dot deleted file mode 100644 index 470012f..0000000 --- a/espfs/heatshrink/dec_sm.dot +++ /dev/null @@ -1,52 +0,0 @@ -digraph { - graph [label="Decoder state machine", labelloc="t"] - Start [style="invis", shape="point"] - empty - input_available - yield_literal - backref_index_msb - backref_index_lsb - backref_count_msb - backref_count_lsb - yield_backref - check_for_more_input - done [peripheries=2] - - empty->input_available [label="sink()", color="blue", weight=10] - Start->empty - - input_available->yield_literal [label="pop 1-bit"] - input_available->backref_index_msb [label="pop 0-bit", weight=10] - input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10] - - yield_literal->yield_literal [label="sink()", color="blue"] - yield_literal->yield_literal [label="poll()", color="red"] - yield_literal->check_for_more_input [label="poll(), done", color="red"] - - backref_index_msb->backref_index_msb [label="sink()", color="blue"] - backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10] - backref_index_msb->done [label="finish()", color="blue"] - - backref_index_lsb->backref_index_lsb [label="sink()", color="blue"] - backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10] - backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10] - backref_index_lsb->done [label="finish()", color="blue"] - - backref_count_msb->backref_count_msb [label="sink()", color="blue"] - backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10] - backref_count_msb->done [label="finish()", color="blue"] - - backref_count_lsb->backref_count_lsb [label="sink()", color="blue"] - backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10] - backref_count_lsb->done [label="finish()", color="blue"] - - yield_backref->yield_backref [label="sink()", color="blue"] - yield_backref->yield_backref [label="poll()", color="red"] - yield_backref->check_for_more_input [label="poll(), done", - color="red", weight=10] - - check_for_more_input->empty [label="no"] - check_for_more_input->input_available [label="yes"] - - empty->done [label="finish()", color="blue"] -} diff --git a/espfs/heatshrink/enc_sm.dot b/espfs/heatshrink/enc_sm.dot deleted file mode 100644 index 6d3030f..0000000 --- a/espfs/heatshrink/enc_sm.dot +++ /dev/null @@ -1,51 +0,0 @@ -digraph { - graph [label="Encoder state machine", labelloc="t"] - start [style="invis", shape="point"] - not_full - filled - search - yield_tag_bit - yield_literal - yield_br_length - yield_br_index - save_backlog - flush_bits - done [peripheries=2] - - start->not_full [label="start"] - - not_full->not_full [label="sink(), not full", color="blue"] - not_full->filled [label="sink(), buffer is full", color="blue"] - not_full->filled [label="finish(), set is_finished", color="blue"] - - filled->search [label="indexing (if any)"] - - search->search [label="step"] - search->yield_tag_bit [label="literal"] - search->yield_tag_bit [label="match found"] - search->save_backlog [label="input exhausted"] - - yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"] - yield_tag_bit->yield_literal [label="poll(), literal", color="red"] - yield_tag_bit->yield_br_index [label="poll(), no literal", color="red"] - yield_tag_bit->flush_bits [label="finishing, no literal"] - - yield_literal->yield_literal [label="poll(), full buf", color="red"] - yield_literal->search [label="poll(), no match", color="red"] - yield_literal->yield_tag_bit [label="poll(), match", color="red"] - yield_literal->flush_bits [label="poll(), final literal", color="red"] - - yield_br_index->yield_br_index [label="poll(), full buf", color="red"] - yield_br_index->yield_br_length [label="poll()", color="red"] - - yield_br_length->yield_br_length [label="poll(), full buf", color="red"] - yield_br_length->search [label="done"] - - save_backlog->flush_bits [label="finishing, no literal"] - save_backlog->yield_tag_bit [label="finishing, literal"] - save_backlog->not_full [label="expect more input"] - - flush_bits->flush_bits [label="poll(), full buf", color="red"] - flush_bits->done [label="poll(), flushed", color="red"] - flush_bits->done [label="no more output"] -} diff --git a/espfs/heatshrink/greatest.h b/espfs/heatshrink/greatest.h deleted file mode 100644 index a92c642..0000000 --- a/espfs/heatshrink/greatest.h +++ /dev/null @@ -1,591 +0,0 @@ -/* - * Copyright (c) 2011 Scott Vokes - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef GREATEST_H -#define GREATEST_H - -#define GREATEST_VERSION_MAJOR 0 -#define GREATEST_VERSION_MINOR 9 -#define GREATEST_VERSION_PATCH 3 - -/* A unit testing system for C, contained in 1 file. - * It doesn't use dynamic allocation or depend on anything - * beyond ANSI C89. */ - - -/********************************************************************* - * Minimal test runner template - *********************************************************************/ -#if 0 - -#include "greatest.h" - -TEST foo_should_foo() { - PASS(); -} - -static void setup_cb(void *data) { - printf("setup callback for each test case\n"); -} - -static void teardown_cb(void *data) { - printf("teardown callback for each test case\n"); -} - -SUITE(suite) { - /* Optional setup/teardown callbacks which will be run before/after - * every test case in the suite. - * Cleared when the suite finishes. */ - SET_SETUP(setup_cb, voidp_to_callback_data); - SET_TEARDOWN(teardown_cb, voidp_to_callback_data); - - RUN_TEST(foo_should_foo); -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - RUN_SUITE(suite); - GREATEST_MAIN_END(); /* display results */ -} - -#endif -/*********************************************************************/ - - -#include -#include -#include -#include - - -/*********** - * Options * - ***********/ - -/* Default column width for non-verbose output. */ -#ifndef GREATEST_DEFAULT_WIDTH -#define GREATEST_DEFAULT_WIDTH 72 -#endif - -/* FILE *, for test logging. */ -#ifndef GREATEST_STDOUT -#define GREATEST_STDOUT stdout -#endif - -/* Remove GREATEST_ prefix from most commonly used symbols? */ -#ifndef GREATEST_USE_ABBREVS -#define GREATEST_USE_ABBREVS 1 -#endif - - -/********* - * Types * - *********/ - -/* Info for the current running suite. */ -typedef struct greatest_suite_info { - unsigned int tests_run; - unsigned int passed; - unsigned int failed; - unsigned int skipped; - - /* timers, pre/post running suite and individual tests */ - clock_t pre_suite; - clock_t post_suite; - clock_t pre_test; - clock_t post_test; -} greatest_suite_info; - -/* Type for a suite function. */ -typedef void (greatest_suite_cb)(void); - -/* Types for setup/teardown callbacks. If non-NULL, these will be run - * and passed the pointer to their additional data. */ -typedef void (greatest_setup_cb)(void *udata); -typedef void (greatest_teardown_cb)(void *udata); - -typedef enum { - GREATEST_FLAG_VERBOSE = 0x01, - GREATEST_FLAG_FIRST_FAIL = 0x02, - GREATEST_FLAG_LIST_ONLY = 0x04 -} GREATEST_FLAG; - -typedef struct greatest_run_info { - unsigned int flags; - unsigned int tests_run; /* total test count */ - - /* Overall pass/fail/skip counts. */ - unsigned int passed; - unsigned int failed; - unsigned int skipped; - - /* currently running test suite */ - greatest_suite_info suite; - - /* info to print about the most recent failure */ - const char *fail_file; - unsigned int fail_line; - const char *msg; - - /* current setup/teardown hooks and userdata */ - greatest_setup_cb *setup; - void *setup_udata; - greatest_teardown_cb *teardown; - void *teardown_udata; - - /* formatting info for ".....s...F"-style output */ - unsigned int col; - unsigned int width; - - /* only run a specific suite or test */ - char *suite_filter; - char *test_filter; - - /* overall timers */ - clock_t begin; - clock_t end; -} greatest_run_info; - -/* Global var for the current testing context. - * Initialized by GREATEST_MAIN_DEFS(). */ -extern greatest_run_info greatest_info; - - -/********************** - * Exported functions * - **********************/ - -void greatest_do_pass(const char *name); -void greatest_do_fail(const char *name); -void greatest_do_skip(const char *name); -int greatest_pre_test(const char *name); -void greatest_post_test(const char *name, int res); -void greatest_usage(const char *name); -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); - - -/********** - * Macros * - **********/ - -/* Define a suite. */ -#define GREATEST_SUITE(NAME) void NAME(void) - -/* Start defining a test function. - * The arguments are not included, to allow parametric testing. */ -#define GREATEST_TEST static int - -/* Run a suite. */ -#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) - -/* Run a test in the current suite. */ -#define GREATEST_RUN_TEST(TEST) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -/* Run a test in the current suite with one void* argument, - * which can be a pointer to a struct with multiple arguments. */ -#define GREATEST_RUN_TEST1(TEST, ENV) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(ENV); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -/* If __VA_ARGS__ (C99) is supported, allow parametric testing - * without needing to manually manage the argument struct. */ -#if __STDC_VERSION__ >= 19901L -#define GREATEST_RUN_TESTp(TEST, ...) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(__VA_ARGS__); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) -#endif - - -/* Check if the test runner is in verbose mode. */ -#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE) -#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) -#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) -#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) - -/* Message-less forms. */ -#define GREATEST_PASS() GREATEST_PASSm(NULL) -#define GREATEST_FAIL() GREATEST_FAILm(NULL) -#define GREATEST_SKIP() GREATEST_SKIPm(NULL) -#define GREATEST_ASSERT(COND) GREATEST_ASSERTm(#COND, COND) -#define GREATEST_ASSERT_FALSE(COND) GREATEST_ASSERT_FALSEm(#COND, COND) -#define GREATEST_ASSERT_EQ(EXP, GOT) GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) -#define GREATEST_ASSERT_STR_EQ(EXP, GOT) GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) - -/* The following forms take an additional message argument first, - * to be displayed by the test runner. */ - -/* Fail if a condition is not true, with message. */ -#define GREATEST_ASSERTm(MSG, COND) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if (!(COND)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_FALSEm(MSG, COND) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if ((COND)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if ((EXP) != (GOT)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ - do { \ - const char *exp_s = (EXP); \ - const char *got_s = (GOT); \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if (0 != strcmp(exp_s, got_s)) { \ - fprintf(GREATEST_STDOUT, \ - "Expected:\n####\n%s\n####\n", exp_s); \ - fprintf(GREATEST_STDOUT, \ - "Got:\n####\n%s\n####\n", got_s); \ - return -1; \ - } \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_PASSm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return 0; \ - } while (0) - -#define GREATEST_FAILm(MSG) \ - do { \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - greatest_info.msg = MSG; \ - return -1; \ - } while (0) - -#define GREATEST_SKIPm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return 1; \ - } while (0) - -#define GREATEST_SET_TIME(NAME) \ - NAME = clock(); \ - if (NAME == (clock_t) -1) { \ - fprintf(GREATEST_STDOUT, \ - "clock error: %s\n", #NAME); \ - exit(EXIT_FAILURE); \ - } - -#define GREATEST_CLOCK_DIFF(C1, C2) \ - fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ - (long unsigned int) (C2) - (C1), \ - (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) \ - -/* Include several function definitions in the main test file. */ -#define GREATEST_MAIN_DEFS() \ - \ -/* Is FILTER a subset of NAME? */ \ -static int greatest_name_match(const char *name, \ - const char *filter) { \ - size_t offset = 0; \ - size_t filter_len = strlen(filter); \ - while (name[offset] != '\0') { \ - if (name[offset] == filter[0]) { \ - if (0 == strncmp(&name[offset], filter, filter_len)) { \ - return 1; \ - } \ - } \ - offset++; \ - } \ - \ - return 0; \ -} \ - \ -int greatest_pre_test(const char *name) { \ - if (!GREATEST_LIST_ONLY() \ - && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ - && (greatest_info.test_filter == NULL || \ - greatest_name_match(name, greatest_info.test_filter))) { \ - GREATEST_SET_TIME(greatest_info.suite.pre_test); \ - if (greatest_info.setup) { \ - greatest_info.setup(greatest_info.setup_udata); \ - } \ - return 1; /* test should be run */ \ - } else { \ - return 0; /* skipped */ \ - } \ -} \ - \ -void greatest_post_test(const char *name, int res) { \ - GREATEST_SET_TIME(greatest_info.suite.post_test); \ - if (greatest_info.teardown) { \ - void *udata = greatest_info.teardown_udata; \ - greatest_info.teardown(udata); \ - } \ - \ - if (res < 0) { \ - greatest_do_fail(name); \ - } else if (res > 0) { \ - greatest_do_skip(name); \ - } else if (res == 0) { \ - greatest_do_pass(name); \ - } \ - greatest_info.suite.tests_run++; \ - greatest_info.col++; \ - if (GREATEST_IS_VERBOSE()) { \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ - greatest_info.suite.post_test); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } else if (greatest_info.col % greatest_info.width == 0) { \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - } \ - if (GREATEST_STDOUT == stdout) fflush(stdout); \ -} \ - \ -static void greatest_run_suite(greatest_suite_cb *suite_cb, \ - const char *suite_name) { \ - if (greatest_info.suite_filter && \ - !greatest_name_match(suite_name, greatest_info.suite_filter)) \ - return; \ - if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) return; \ - greatest_info.suite.tests_run = 0; \ - greatest_info.suite.failed = 0; \ - greatest_info.suite.passed = 0; \ - greatest_info.suite.skipped = 0; \ - greatest_info.suite.pre_suite = 0; \ - greatest_info.suite.post_suite = 0; \ - greatest_info.suite.pre_test = 0; \ - greatest_info.suite.post_test = 0; \ - greatest_info.col = 0; \ - fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ - GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ - suite_cb(); \ - GREATEST_SET_TIME(greatest_info.suite.post_suite); \ - if (greatest_info.suite.tests_run > 0) { \ - fprintf(GREATEST_STDOUT, \ - "\n%u tests - %u pass, %u fail, %u skipped", \ - greatest_info.suite.tests_run, \ - greatest_info.suite.passed, \ - greatest_info.suite.failed, \ - greatest_info.suite.skipped); \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ - greatest_info.suite.post_suite); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } \ - greatest_info.setup = NULL; \ - greatest_info.setup_udata = NULL; \ - greatest_info.teardown = NULL; \ - greatest_info.teardown_udata = NULL; \ - greatest_info.passed += greatest_info.suite.passed; \ - greatest_info.failed += greatest_info.suite.failed; \ - greatest_info.skipped += greatest_info.suite.skipped; \ - greatest_info.tests_run += greatest_info.suite.tests_run; \ -} \ - \ -void greatest_do_pass(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "PASS %s: %s", \ - name, greatest_info.msg ? greatest_info.msg : ""); \ - } else { \ - fprintf(GREATEST_STDOUT, "."); \ - } \ - greatest_info.suite.passed++; \ -} \ - \ -void greatest_do_fail(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, \ - "FAIL %s: %s (%s:%u)", \ - name, greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } else { \ - fprintf(GREATEST_STDOUT, "F"); \ - /* add linebreak if in line of '.'s */ \ - if (greatest_info.col % greatest_info.width != 0) \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ - name, \ - greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } \ - greatest_info.suite.failed++; \ -} \ - \ -void greatest_do_skip(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ - name, \ - greatest_info.msg ? \ - greatest_info.msg : "" ); \ - } else { \ - fprintf(GREATEST_STDOUT, "s"); \ - } \ - greatest_info.suite.skipped++; \ -} \ - \ -void greatest_usage(const char *name) { \ - fprintf(GREATEST_STDOUT, \ - "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ - " -h print this Help\n" \ - " -l List suites and their tests, then exit\n" \ - " -f Stop runner after first failure\n" \ - " -v Verbose output\n" \ - " -s SUITE only run suite named SUITE\n" \ - " -t TEST only run test named TEST\n", \ - name); \ -} \ - \ -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ - greatest_info.setup = cb; \ - greatest_info.setup_udata = udata; \ -} \ - \ -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ - void *udata) { \ - greatest_info.teardown = cb; \ - greatest_info.teardown_udata = udata; \ -} \ - \ -greatest_run_info greatest_info - -/* Handle command-line arguments, etc. */ -#define GREATEST_MAIN_BEGIN() \ - do { \ - int i = 0; \ - memset(&greatest_info, 0, sizeof(greatest_info)); \ - if (greatest_info.width == 0) { \ - greatest_info.width = GREATEST_DEFAULT_WIDTH; \ - } \ - for (i = 1; i < argc; i++) { \ - if (0 == strcmp("-t", argv[i])) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.test_filter = argv[i+1]; \ - i++; \ - } else if (0 == strcmp("-s", argv[i])) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.suite_filter = argv[i+1]; \ - i++; \ - } else if (0 == strcmp("-f", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ - } else if (0 == strcmp("-v", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_VERBOSE; \ - } else if (0 == strcmp("-l", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ - } else if (0 == strcmp("-h", argv[i])) { \ - greatest_usage(argv[0]); \ - exit(EXIT_SUCCESS); \ - } else { \ - fprintf(GREATEST_STDOUT, \ - "Unknown argument '%s'\n", argv[i]); \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - } \ - } while (0); \ - GREATEST_SET_TIME(greatest_info.begin) - -#define GREATEST_MAIN_END() \ - do { \ - if (!GREATEST_LIST_ONLY()) { \ - GREATEST_SET_TIME(greatest_info.end); \ - fprintf(GREATEST_STDOUT, \ - "\nTotal: %u tests", greatest_info.tests_run); \ - GREATEST_CLOCK_DIFF(greatest_info.begin, \ - greatest_info.end); \ - fprintf(GREATEST_STDOUT, "\n"); \ - fprintf(GREATEST_STDOUT, \ - "Pass: %u, fail: %u, skip: %u.\n", \ - greatest_info.passed, \ - greatest_info.failed, greatest_info.skipped); \ - } \ - return (greatest_info.failed > 0 \ - ? EXIT_FAILURE : EXIT_SUCCESS); \ - } while (0) - -/* Make abbreviations without the GREATEST_ prefix for the - * most commonly used symbols. */ -#if GREATEST_USE_ABBREVS -#define TEST GREATEST_TEST -#define SUITE GREATEST_SUITE -#define RUN_TEST GREATEST_RUN_TEST -#define RUN_TEST1 GREATEST_RUN_TEST1 -#define RUN_SUITE GREATEST_RUN_SUITE -#define ASSERT GREATEST_ASSERT -#define ASSERTm GREATEST_ASSERTm -#define ASSERT_FALSE GREATEST_ASSERT_FALSE -#define ASSERT_EQ GREATEST_ASSERT_EQ -#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ -#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm -#define ASSERT_EQm GREATEST_ASSERT_EQm -#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm -#define PASS GREATEST_PASS -#define FAIL GREATEST_FAIL -#define SKIP GREATEST_SKIP -#define PASSm GREATEST_PASSm -#define FAILm GREATEST_FAILm -#define SKIPm GREATEST_SKIPm -#define SET_SETUP GREATEST_SET_SETUP_CB -#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB - -#if __STDC_VERSION__ >= 19901L -#endif /* C99 */ -#define RUN_TESTp GREATEST_RUN_TESTp -#endif /* USE_ABBREVS */ - -#endif diff --git a/espfs/heatshrink/heatshrink.c b/espfs/heatshrink/heatshrink.c deleted file mode 100644 index 9de553f..0000000 --- a/espfs/heatshrink/heatshrink.c +++ /dev/null @@ -1,446 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" - -#define DEF_WINDOW_SZ2 11 -#define DEF_LOOKAHEAD_SZ2 4 -#define DEF_DECODER_INPUT_BUFFER_SIZE 256 -#define DEF_BUFFER_SIZE (64 * 1024) - -#if 0 -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#else -#define LOG(...) /* NO-OP */ -#endif - -static const int version_major = HEATSHRINK_VERSION_MAJOR; -static const int version_minor = HEATSHRINK_VERSION_MINOR; -static const int version_patch = HEATSHRINK_VERSION_PATCH; -static const char author[] = HEATSHRINK_AUTHOR; -static const char url[] = HEATSHRINK_URL; - -static void usage(void) { - fprintf(stderr, "heatshrink version %u.%u.%u by %s\n", - version_major, version_minor, version_patch, author); - fprintf(stderr, "Home page: %s\n\n", url); - fprintf(stderr, - "Usage:\n" - " heatshrink [-h] [-e|-d] [-v] [-w SIZE] [-l BITS] [IN_FILE] [OUT_FILE]\n" - "\n" - "heatshrink compresses or uncompresses byte streams using LZSS, and is\n" - "designed especially for embedded, low-memory, and/or hard real-time\n" - "systems.\n" - "\n" - " -h print help\n" - " -e encode (compress, default)\n" - " -d decode (uncompress)\n" - " -v verbose (print input & output sizes, compression ratio, etc.)\n" - "\n" - " -w SIZE Base-2 log of LZSS sliding window size\n" - "\n" - " A larger value allows searches a larger history of the data for repeated\n" - " patterns, potentially compressing more effectively, but will use\n" - " more memory and processing time.\n" - " Recommended default: -w 8 (embedded systems), -w 10 (elsewhere)\n" - " \n" - " -l BITS Number of bits used for back-reference lengths\n" - "\n" - " A larger value allows longer substitutions, but since all\n" - " back-references must use -w + -l bits, larger -w or -l can be\n" - " counterproductive if most patterns are small and/or local.\n" - " Recommended default: -l 4\n" - "\n" - " If IN_FILE or OUT_FILE are unspecified, they will default to\n" - " \"-\" for standard input and standard output, respectively.\n"); - exit(1); -} - -typedef enum { IO_READ, IO_WRITE, } IO_mode; -typedef enum { OP_ENC, OP_DEC, } Operation; - -typedef struct { - int fd; /* file descriptor */ - IO_mode mode; - size_t fill; /* fill index */ - size_t read; /* read index */ - size_t size; - size_t total; - uint8_t buf[]; -} io_handle; - -typedef struct { - uint8_t window_sz2; - uint8_t lookahead_sz2; - size_t decoder_input_buffer_size; - size_t buffer_size; - uint8_t verbose; - Operation cmd; - char *in_fname; - char *out_fname; - io_handle *in; - io_handle *out; -} config; - -static void die(char *msg) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); -} - -static void report(config *cfg); - -/* Open an IO handle. Returns NULL on error. */ -static io_handle *handle_open(char *fname, IO_mode m, size_t buf_sz) { - io_handle *io = NULL; - io = malloc(sizeof(*io) + buf_sz); - if (io == NULL) { return NULL; } - memset(io, 0, sizeof(*io) + buf_sz); - io->fd = -1; - io->size = buf_sz; - io->mode = m; - - if (m == IO_READ) { - if (0 == strcmp("-", fname)) { - io->fd = STDIN_FILENO; - } else { - io->fd = open(fname, O_RDONLY); - } - } else if (m == IO_WRITE) { - if (0 == strcmp("-", fname)) { - io->fd = STDOUT_FILENO; - } else { - io->fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC /*| O_EXCL*/, 0644); - } - } - - if (io->fd == -1) { /* failed to open */ - free(io); - err(1, "open"); - return NULL; - } - - return io; -} - -/* Read SIZE bytes from an IO handle and return a pointer to the content. - * BUF contains at least size_t bytes. Returns 0 on EOF, -1 on error. */ -static ssize_t handle_read(io_handle *io, size_t size, uint8_t **buf) { - LOG("@ read %zd\n", size); - if (buf == NULL) { return -1; } - if (size > io->size) { - printf("size %zd, io->size %zd\n", size, io->size); - return -1; - } - if (io->mode != IO_READ) { return -1; } - - size_t rem = io->fill - io->read; - if (rem >= size) { - *buf = &io->buf[io->read]; - return size; - } else { /* read and replenish */ - if (io->fd == -1) { /* already closed, return what we've got */ - *buf = &io->buf[io->read]; - return rem; - } - - memmove(io->buf, &io->buf[io->read], rem); - io->fill -= io->read; - io->read = 0; - ssize_t read_sz = read(io->fd, &io->buf[io->fill], io->size - io->fill); - if (read_sz < 0) { err(1, "read"); } - io->total += read_sz; - if (read_sz == 0) { /* EOF */ - if (close(io->fd) < 0) { err(1, "close"); } - io->fd = -1; - } - io->fill += read_sz; - *buf = io->buf; - return io->fill > size ? size : io->fill; - } -} - -/* Drop the oldest SIZE bytes from the buffer. Returns <0 on error. */ -static int handle_drop(io_handle *io, size_t size) { - LOG("@ drop %zd\n", size); - if (io->read + size <= io->fill) { - io->read += size; - } else { - return -1; - } - if (io->read == io->fill) { - io->read = 0; - io->fill = 0; - } - return 0; -} - -/* Sink SIZE bytes from INPUT into the io handle. Returns the number of - * bytes written, or -1 on error. */ -static ssize_t handle_sink(io_handle *io, size_t size, uint8_t *input) { - LOG("@ sink %zd\n", size); - if (size > io->size) { return -1; } - if (io->mode != IO_WRITE) { return -1; } - - if (io->fill + size > io->size) { - ssize_t written = write(io->fd, io->buf, io->fill); - LOG("@ flushing %zd, wrote %zd\n", io->fill, written); - io->total += written; - if (written == -1) { err(1, "write"); } - memmove(io->buf, &io->buf[written], io->fill - written); - io->fill -= written; - } - memcpy(&io->buf[io->fill], input, size); - io->fill += size; - return size; -} - -static void handle_close(io_handle *io) { - if (io->fd != -1) { - if (io->mode == IO_WRITE) { - ssize_t written = write(io->fd, io->buf, io->fill); - io->total += written; - LOG("@ close: flushing %zd, wrote %zd\n", io->fill, written); - if (written == -1) { err(1, "write"); } - } - close(io->fd); - io->fd = -1; - } -} - -static void close_and_report(config *cfg) { - handle_close(cfg->in); - handle_close(cfg->out); - if (cfg->verbose) { report(cfg); } - free(cfg->in); - free(cfg->out); -} - -static int encoder_sink_read(config *cfg, heatshrink_encoder *hse, - uint8_t *data, size_t data_sz) { - size_t out_sz = 4096; - uint8_t out_buf[out_sz]; - memset(out_buf, 0, out_sz); - size_t sink_sz = 0; - size_t poll_sz = 0; - HSE_sink_res sres; - HSE_poll_res pres; - HSE_finish_res fres; - io_handle *out = cfg->out; - - size_t sunk = 0; - do { - if (data_sz > 0) { - sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz); - if (sres < 0) { die("sink"); } - sunk += sink_sz; - } - - do { - pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz); - if (pres < 0) { die("poll"); } - if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); - } while (pres == HSER_POLL_MORE); - - if (poll_sz == 0 && data_sz == 0) { - fres = heatshrink_encoder_finish(hse); - if (fres < 0) { die("finish"); } - if (fres == HSER_FINISH_DONE) { return 1; } - } - } while (sunk < data_sz); - return 0; -} - -static int encode(config *cfg) { - uint8_t window_sz2 = cfg->window_sz2; - size_t window_sz = 1 << window_sz2; - heatshrink_encoder *hse = heatshrink_encoder_alloc(window_sz2, cfg->lookahead_sz2); - if (hse == NULL) { die("failed to init encoder: bad settings"); } - ssize_t read_sz = 0; - io_handle *in = cfg->in; - - /* Process input until end of stream */ - while (1) { - uint8_t *input = NULL; - read_sz = handle_read(in, window_sz, &input); - if (input == NULL) { - printf("handle read failure\n"); - die("read"); - } - if (read_sz < 0) { die("read"); } - - /* Pass read to encoder and check if input is fully processed. */ - if (encoder_sink_read(cfg, hse, input, read_sz)) break; - - if (handle_drop(in, read_sz) < 0) { die("drop"); } - }; - - if (read_sz == -1) { err(1, "read"); } - - heatshrink_encoder_free(hse); - close_and_report(cfg); - return 0; -} - -static int decoder_sink_read(config *cfg, heatshrink_decoder *hsd, - uint8_t *data, size_t data_sz) { - io_handle *out = cfg->out; - size_t sink_sz = 0; - size_t poll_sz = 0; - size_t out_sz = 4096; - uint8_t out_buf[out_sz]; - memset(out_buf, 0, out_sz); - - HSD_sink_res sres; - HSD_poll_res pres; - HSD_finish_res fres; - - size_t sunk = 0; - do { - if (data_sz > 0) { - sres = heatshrink_decoder_sink(hsd, &data[sunk], data_sz - sunk, &sink_sz); - if (sres < 0) { die("sink"); } - sunk += sink_sz; - } - - do { - pres = heatshrink_decoder_poll(hsd, out_buf, out_sz, &poll_sz); - if (pres < 0) { die("poll"); } - if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); - } while (pres == HSDR_POLL_MORE); - - if (data_sz == 0 && poll_sz == 0) { - fres = heatshrink_decoder_finish(hsd); - if (fres < 0) { die("finish"); } - if (fres == HSDR_FINISH_DONE) { return 1; } - } - } while (sunk < data_sz); - - return 0; -} - -static int decode(config *cfg) { - uint8_t window_sz2 = cfg->window_sz2; - size_t window_sz = 1 << window_sz2; - size_t ibs = cfg->decoder_input_buffer_size; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(ibs, - window_sz2, cfg->lookahead_sz2); - if (hsd == NULL) { die("failed to init decoder"); } - - ssize_t read_sz = 0; - - io_handle *in = cfg->in; - - HSD_finish_res fres; - - /* Process input until end of stream */ - while (1) { - uint8_t *input = NULL; - read_sz = handle_read(in, window_sz, &input); - if (input == NULL) { - printf("handle read failure\n"); - die("read"); - } - if (read_sz == 0) { - fres = heatshrink_decoder_finish(hsd); - if (fres < 0) { die("finish"); } - if (fres == HSDR_FINISH_DONE) break; - } else if (read_sz < 0) { - die("read"); - } else { - if (decoder_sink_read(cfg, hsd, input, read_sz)) { break; } - if (handle_drop(in, read_sz) < 0) { die("drop"); } - } - } - if (read_sz == -1) { err(1, "read"); } - - heatshrink_decoder_free(hsd); - close_and_report(cfg); - return 0; -} - -static void report(config *cfg) { - size_t inb = cfg->in->total; - size_t outb = cfg->out->total; - fprintf(cfg->out->fd == STDOUT_FILENO ? stderr : stdout, - "%s %0.2f %%\t %zd -> %zd (-w %u -l %u)\n", - cfg->in_fname, 100.0 - (100.0 * outb) / inb, inb, outb, - cfg->window_sz2, cfg->lookahead_sz2); -} - -static void proc_args(config *cfg, int argc, char **argv) { - cfg->window_sz2 = DEF_WINDOW_SZ2; - cfg->lookahead_sz2 = DEF_LOOKAHEAD_SZ2; - cfg->buffer_size = DEF_BUFFER_SIZE; - cfg->decoder_input_buffer_size = DEF_DECODER_INPUT_BUFFER_SIZE; - cfg->cmd = OP_ENC; - cfg->verbose = 0; - cfg->in_fname = "-"; - cfg->out_fname = "-"; - - int a = 0; - while ((a = getopt(argc, argv, "hedi:w:l:v")) != -1) { - switch (a) { - case 'h': /* help */ - usage(); - case 'e': /* encode */ - cfg->cmd = OP_ENC; break; - case 'd': /* decode */ - cfg->cmd = OP_DEC; break; - case 'i': /* input buffer size */ - cfg->decoder_input_buffer_size = atoi(optarg); - break; - case 'w': /* window bits */ - cfg->window_sz2 = atoi(optarg); - break; - case 'l': /* lookahead bits */ - cfg->lookahead_sz2 = atoi(optarg); - break; - case 'v': /* verbosity++ */ - cfg->verbose++; - break; - case '?': /* unknown argument */ - default: - usage(); - } - } - argc -= optind; - argv += optind; - if (argc > 0) { - cfg->in_fname = argv[0]; - argc--; - argv++; - } - if (argc > 0) { cfg->out_fname = argv[0]; } -} - -int main(int argc, char **argv) { - config cfg; - memset(&cfg, 0, sizeof(cfg)); - proc_args(&cfg, argc, argv); - - if (0 == strcmp(cfg.in_fname, cfg.out_fname) - && (0 != strcmp("-", cfg.in_fname))) { - printf("Refusing to overwrite file '%s' with itself.\n", cfg.in_fname); - exit(1); - } - - cfg.in = handle_open(cfg.in_fname, IO_READ, cfg.buffer_size); - if (cfg.in == NULL) { die("Failed to open input file for read"); } - cfg.out = handle_open(cfg.out_fname, IO_WRITE, cfg.buffer_size); - if (cfg.out == NULL) { die("Failed to open output file for write"); } - - if (cfg.cmd == OP_ENC) { - return encode(&cfg); - } else if (cfg.cmd == OP_DEC) { - return decode(&cfg); - } else { - usage(); - } -} diff --git a/espfs/heatshrink/heatshrink_common.h b/espfs/heatshrink/heatshrink_common.h deleted file mode 100644 index 0d396b9..0000000 --- a/espfs/heatshrink/heatshrink_common.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef HEATSHRINK_H -#define HEATSHRINK_H - -#define HEATSHRINK_AUTHOR "Scott Vokes " -#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" - -/* Version 0.3.1 */ -#define HEATSHRINK_VERSION_MAJOR 0 -#define HEATSHRINK_VERSION_MINOR 3 -#define HEATSHRINK_VERSION_PATCH 1 - -#define HEATSHRINK_MIN_WINDOW_BITS 4 -#define HEATSHRINK_MAX_WINDOW_BITS 15 - -#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2 - -#define HEATSHRINK_LITERAL_MARKER 0x01 -#define HEATSHRINK_BACKREF_MARKER 0x00 - -#endif diff --git a/espfs/heatshrink/heatshrink_config.h b/espfs/heatshrink/heatshrink_config.h deleted file mode 100644 index 51d4772..0000000 --- a/espfs/heatshrink/heatshrink_config.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef HEATSHRINK_CONFIG_H -#define HEATSHRINK_CONFIG_H - -/* Should functionality assuming dynamic allocation be used? */ -#define HEATSHRINK_DYNAMIC_ALLOC 1 - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Optional replacement of malloc/free */ - #define HEATSHRINK_MALLOC(SZ) malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) free(P) -#else - /* Required parameters for static configuration */ - #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 - #define HEATSHRINK_STATIC_WINDOW_BITS 8 - #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 -#endif - -/* Turn on logging for debugging. */ -#define HEATSHRINK_DEBUGGING_LOGS 0 - -/* Use indexing for faster compression. (This requires additional space.) */ -#define HEATSHRINK_USE_INDEX 1 - -#endif diff --git a/espfs/heatshrink/heatshrink_decoder.c b/espfs/heatshrink/heatshrink_decoder.c deleted file mode 100644 index b92be13..0000000 --- a/espfs/heatshrink/heatshrink_decoder.c +++ /dev/null @@ -1,382 +0,0 @@ -#include -#include -#include "heatshrink_decoder.h" - -/* States for the polling state machine. */ -typedef enum { - HSDS_EMPTY, /* no input to process */ - HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */ - HSDS_YIELD_LITERAL, /* ready to yield literal byte */ - HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ - HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ - HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ - HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ - HSDS_YIELD_BACKREF, /* ready to yield back-reference */ - HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */ -} HSD_state; - -#if HEATSHRINK_DEBUGGING_LOGS -#include -#include -#include -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#define ASSERT(X) assert(X) -static const char *state_names[] = { - "empty", - "input_available", - "yield_literal", - "backref_index", - "backref_count", - "yield_backref", - "check_for_more_input", -}; -#else -#define LOG(...) /* no-op */ -#define ASSERT(X) /* no-op */ -#endif - -typedef struct { - uint8_t *buf; /* output buffer */ - size_t buf_size; /* buffer size */ - size_t *output_size; /* bytes pushed to buffer, so far */ -} output_info; - -#define NO_BITS ((uint32_t)-1) - -/* Forward references. */ -static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count); -static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); - -#if HEATSHRINK_DYNAMIC_ALLOC -heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, - uint8_t window_sz2, - uint8_t lookahead_sz2) { - if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || - (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || - (input_buffer_size == 0) || - (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || - (lookahead_sz2 > window_sz2)) { - return NULL; - } - size_t buffers_sz = (1 << window_sz2) + input_buffer_size; - size_t sz = sizeof(heatshrink_decoder) + buffers_sz; - heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); - if (hsd == NULL) { return NULL; } - hsd->input_buffer_size = input_buffer_size; - hsd->window_sz2 = window_sz2; - hsd->lookahead_sz2 = lookahead_sz2; - heatshrink_decoder_reset(hsd); - LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", - sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); - return hsd; -} - -void heatshrink_decoder_free(heatshrink_decoder *hsd) { - size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; - size_t sz = sizeof(heatshrink_decoder) + buffers_sz; - HEATSHRINK_FREE(hsd, sz); - (void)sz; /* may not be used by free */ -} -#endif - -void heatshrink_decoder_reset(heatshrink_decoder *hsd) { - size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); - size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); - memset(hsd->buffers, 0, buf_sz + input_sz); - hsd->state = HSDS_EMPTY; - hsd->input_size = 0; - hsd->input_index = 0; - hsd->bit_index = 0x00; - hsd->current_byte = 0x00; - hsd->output_count = 0; - hsd->output_index = 0; - hsd->head_index = 0; - hsd->bit_accumulator = 0x00000000; -} - -/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ -HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, - uint8_t *in_buf, size_t size, size_t *input_size) { - if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { - return HSDR_SINK_ERROR_NULL; - } - - size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; - if (rem == 0) { - *input_size = 0; - return HSDR_SINK_FULL; - } - - size = rem < size ? rem : size; - LOG("-- sinking %zd bytes\n", size); - /* copy into input buffer (at head of buffers) */ - memcpy(&hsd->buffers[hsd->input_size], in_buf, size); - hsd->input_size += size; - if (hsd->state == HSDS_EMPTY) { - hsd->state = HSDS_INPUT_AVAILABLE; - hsd->input_index = 0; - } - *input_size = size; - return HSDR_SINK_OK; -} - - -/***************** - * Decompression * - *****************/ - -#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) -#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) - -// States -static HSD_state st_input_available(heatshrink_decoder *hsd); -static HSD_state st_yield_literal(heatshrink_decoder *hsd, - output_info *oi); -static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); -static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); -static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); -static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); -static HSD_state st_yield_backref(heatshrink_decoder *hsd, - output_info *oi); -static HSD_state st_check_for_input(heatshrink_decoder *hsd); - -HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { - if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { - return HSDR_POLL_ERROR_NULL; - } - *output_size = 0; - - output_info oi; - oi.buf = out_buf; - oi.buf_size = out_buf_size; - oi.output_size = output_size; - - while (1) { - LOG("-- poll, state is %d (%s), input_size %d\n", - hsd->state, state_names[hsd->state], hsd->input_size); - uint8_t in_state = hsd->state; - switch (in_state) { - case HSDS_EMPTY: - return HSDR_POLL_EMPTY; - case HSDS_INPUT_AVAILABLE: - hsd->state = st_input_available(hsd); - break; - case HSDS_YIELD_LITERAL: - hsd->state = st_yield_literal(hsd, &oi); - break; - case HSDS_BACKREF_INDEX_MSB: - hsd->state = st_backref_index_msb(hsd); - break; - case HSDS_BACKREF_INDEX_LSB: - hsd->state = st_backref_index_lsb(hsd); - break; - case HSDS_BACKREF_COUNT_MSB: - hsd->state = st_backref_count_msb(hsd); - break; - case HSDS_BACKREF_COUNT_LSB: - hsd->state = st_backref_count_lsb(hsd); - break; - case HSDS_YIELD_BACKREF: - hsd->state = st_yield_backref(hsd, &oi); - break; - case HSDS_CHECK_FOR_MORE_INPUT: - hsd->state = st_check_for_input(hsd); - break; - default: - return HSDR_POLL_ERROR_UNKNOWN; - } - - /* If the current state cannot advance, check if input or output - * buffer are exhausted. */ - if (hsd->state == in_state) { - if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } - return HSDR_POLL_EMPTY; - } - } -} - -static HSD_state st_input_available(heatshrink_decoder *hsd) { - uint32_t bits = get_bits(hsd, 1); // get tag bit - if (bits) { - return HSDS_YIELD_LITERAL; - } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { - return HSDS_BACKREF_INDEX_MSB; - } else { - hsd->output_index = 0; - return HSDS_BACKREF_INDEX_LSB; - } -} - -static HSD_state st_yield_literal(heatshrink_decoder *hsd, - output_info *oi) { - /* Emit a repeated section from the window buffer, and add it (again) - * to the window buffer. (Note that the repetition can include - * itself.)*/ - if (*oi->output_size < oi->buf_size) { - uint32_t byte = get_bits(hsd, 8); - if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ - uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; - uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; - uint8_t c = byte & 0xFF; - LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); - buf[hsd->head_index++ & mask] = c; - push_byte(hsd, oi, c); - return HSDS_CHECK_FOR_MORE_INPUT; - } else { - return HSDS_YIELD_LITERAL; - } -} - -static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { - uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); - ASSERT(bit_ct > 8); - uint32_t bits = get_bits(hsd, bit_ct - 8); - LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } - hsd->output_index = bits << 8; - return HSDS_BACKREF_INDEX_LSB; -} - -static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { - uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); - uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); - LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } - hsd->output_index |= bits; - hsd->output_index++; - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - hsd->output_count = 0; - return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; -} - -static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - ASSERT(br_bit_ct > 8); - uint32_t bits = get_bits(hsd, br_bit_ct - 8); - LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } - hsd->output_count = bits << 8; - return HSDS_BACKREF_COUNT_LSB; -} - -static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); - LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } - hsd->output_count |= bits; - hsd->output_count++; - return HSDS_YIELD_BACKREF; -} - -static HSD_state st_yield_backref(heatshrink_decoder *hsd, - output_info *oi) { - size_t count = oi->buf_size - *oi->output_size; - if (count > 0) { - if (hsd->output_count < count) count = hsd->output_count; - uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; - uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; - uint16_t neg_offset = hsd->output_index; - LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); - ASSERT(neg_offset < mask + 1); - ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd)); - - for (size_t i=0; ihead_index - neg_offset) & mask]; - push_byte(hsd, oi, c); - buf[hsd->head_index & mask] = c; - hsd->head_index++; - LOG(" -- ++ 0x%02x\n", c); - } - hsd->output_count -= count; - if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; } - } - return HSDS_YIELD_BACKREF; -} - -static HSD_state st_check_for_input(heatshrink_decoder *hsd) { - return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE; -} - -/* Get the next COUNT bits from the input buffer, saving incremental progress. - * Returns NO_BITS on end of input, or if more than 31 bits are requested. */ -static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) { - if (count > 31) { return NO_BITS; } - LOG("-- popping %u bit(s)\n", count); - - /* If we aren't able to get COUNT bits, suspend immediately, because we - * don't track how many bits of COUNT we've accumulated before suspend. */ - if (hsd->input_size == 0) { - if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } - } - - for (int i = 0; i < count; i++) { - if (hsd->bit_index == 0x00) { - if (hsd->input_size == 0) { - LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", - hsd->bit_accumulator, hsd->bit_accumulator); - return NO_BITS; - } - hsd->current_byte = hsd->buffers[hsd->input_index++]; - LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); - if (hsd->input_index == hsd->input_size) { - hsd->input_index = 0; /* input is exhausted */ - hsd->input_size = 0; - } - hsd->bit_index = 0x80; - } - hsd->bit_accumulator <<= 1; - if (hsd->current_byte & hsd->bit_index) { - hsd->bit_accumulator |= 0x01; - if (0) { - LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", - hsd->bit_accumulator, hsd->bit_index); - } - } else { - if (0) { - LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", - hsd->bit_accumulator, hsd->bit_index); - } - } - hsd->bit_index >>= 1; - } - - uint32_t res = 0; - res = hsd->bit_accumulator; - hsd->bit_accumulator = 0x00000000; - if (count > 1) { LOG(" -- accumulated %08x\n", res); } - return res; -} - -HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { - if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } - switch (hsd->state) { - case HSDS_EMPTY: - return HSDR_FINISH_DONE; - - /* If we want to finish with no input, but are in these states, it's - * because the 0-bit padding to the last byte looks like a backref - * marker bit followed by all 0s for index and count bits. */ - case HSDS_BACKREF_INDEX_LSB: - case HSDS_BACKREF_INDEX_MSB: - case HSDS_BACKREF_COUNT_LSB: - case HSDS_BACKREF_COUNT_MSB: - return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; - - /* If the output stream is padded with 0xFFs (possibly due to being in - * flash memory), also explicitly check the input size rather than - * uselessly returning MORE but yielding 0 bytes when polling. */ - case HSDS_YIELD_LITERAL: - return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; - - default: - return HSDR_FINISH_MORE; - } -} - -static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { - LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); - oi->buf[(*oi->output_size)++] = byte; - (void)hsd; -} diff --git a/espfs/heatshrink/heatshrink_decoder.h b/espfs/heatshrink/heatshrink_decoder.h deleted file mode 100644 index c1eb144..0000000 --- a/espfs/heatshrink/heatshrink_decoder.h +++ /dev/null @@ -1,101 +0,0 @@ -#ifndef HEATSHRINK_DECODER_H -#define HEATSHRINK_DECODER_H - -#include -#include -#include "heatshrink_common.h" -#include "heatshrink_config.h" - -typedef enum { - HSDR_SINK_OK, /* data sunk, ready to poll */ - HSDR_SINK_FULL, /* out of space in internal buffer */ - HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ -} HSD_sink_res; - -typedef enum { - HSDR_POLL_EMPTY, /* input exhausted */ - HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ - HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ - HSDR_POLL_ERROR_UNKNOWN=-2, -} HSD_poll_res; - -typedef enum { - HSDR_FINISH_DONE, /* output is done */ - HSDR_FINISH_MORE, /* more output remains */ - HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ -} HSD_finish_res; - -#if HEATSHRINK_DYNAMIC_ALLOC -#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ - ((BUF)->input_buffer_size) -#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ - ((BUF)->window_sz2) -#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ - ((BUF)->lookahead_sz2) -#else -#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ - HEATSHRINK_STATIC_INPUT_BUFFER_SIZE -#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ - (HEATSHRINK_STATIC_WINDOW_BITS) -#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ - (HEATSHRINK_STATIC_LOOKAHEAD_BITS) -#endif - -typedef struct { - uint16_t input_size; /* bytes in input buffer */ - uint16_t input_index; /* offset to next unprocessed input byte */ - uint16_t output_count; /* how many bytes to output */ - uint16_t output_index; /* index for bytes to output */ - uint16_t head_index; /* head of window buffer */ - uint16_t bit_accumulator; - uint8_t state; /* current state machine node */ - uint8_t current_byte; /* current byte of input */ - uint8_t bit_index; /* current bit index */ - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Fields that are only used if dynamically allocated. */ - uint8_t window_sz2; /* window buffer bits */ - uint8_t lookahead_sz2; /* lookahead bits */ - uint16_t input_buffer_size; /* input buffer size */ - - /* Input buffer, then expansion window buffer */ - uint8_t buffers[]; -#else - /* Input buffer, then expansion window buffer */ - uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) - + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; -#endif -} heatshrink_decoder; - -#if HEATSHRINK_DYNAMIC_ALLOC -/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, - * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead - * size of 2^lookahead_sz2. (The window buffer and lookahead sizes - * must match the settings used when the data was compressed.) - * Returns NULL on error. */ -heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, - uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); - -/* Free a decoder. */ -void heatshrink_decoder_free(heatshrink_decoder *hsd); -#endif - -/* Reset a decoder. */ -void heatshrink_decoder_reset(heatshrink_decoder *hsd); - -/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to - * indicate how many bytes were actually sunk (in case a buffer was filled). */ -HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, - uint8_t *in_buf, size_t size, size_t *input_size); - -/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into - * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ -HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size); - -/* Notify the dencoder that the input stream is finished. - * If the return value is HSDR_FINISH_MORE, there is still more output, so - * call heatshrink_decoder_poll and repeat. */ -HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); - -#endif diff --git a/espfs/heatshrink/heatshrink_encoder.c b/espfs/heatshrink/heatshrink_encoder.c deleted file mode 100644 index ede5f60..0000000 --- a/espfs/heatshrink/heatshrink_encoder.c +++ /dev/null @@ -1,650 +0,0 @@ -#include -#include -#include -#include "heatshrink_encoder.h" - -typedef enum { - HSES_NOT_FULL, /* input buffer not full enough */ - HSES_FILLED, /* buffer is full */ - HSES_SEARCH, /* searching for patterns */ - HSES_YIELD_TAG_BIT, /* yield tag bit */ - HSES_YIELD_LITERAL, /* emit literal byte */ - HSES_YIELD_BR_INDEX, /* yielding backref index */ - HSES_YIELD_BR_LENGTH, /* yielding backref length */ - HSES_SAVE_BACKLOG, /* copying buffer to backlog */ - HSES_FLUSH_BITS, /* flush bit buffer */ - HSES_DONE, /* done */ -} HSE_state; - -#if HEATSHRINK_DEBUGGING_LOGS -#include -#include -#include -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#define ASSERT(X) assert(X) -static const char *state_names[] = { - "not_full", - "filled", - "search", - "yield_tag_bit", - "yield_literal", - "yield_br_index", - "yield_br_length", - "save_backlog", - "flush_bits", - "done", -}; -#else -#define LOG(...) /* no-op */ -#define ASSERT(X) /* no-op */ -#endif - -// Encoder flags -enum { - FLAG_IS_FINISHING = 0x01, - FLAG_HAS_LITERAL = 0x02, - FLAG_ON_FINAL_LITERAL = 0x04, - FLAG_BACKLOG_IS_PARTIAL = 0x08, - FLAG_BACKLOG_IS_FILLED = 0x10, -}; - -typedef struct { - uint8_t *buf; /* output buffer */ - size_t buf_size; /* buffer size */ - size_t *output_size; /* bytes pushed to buffer, so far */ -} output_info; - -#define MATCH_NOT_FOUND ((uint16_t)-1) - -static uint16_t get_input_offset(heatshrink_encoder *hse); -static uint16_t get_input_buffer_size(heatshrink_encoder *hse); -static uint16_t get_lookahead_size(heatshrink_encoder *hse); -static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); -static int can_take_byte(output_info *oi); -static int is_finishing(heatshrink_encoder *hse); -static int backlog_is_partial(heatshrink_encoder *hse); -static int backlog_is_filled(heatshrink_encoder *hse); -static int on_final_literal(heatshrink_encoder *hse); -static void save_backlog(heatshrink_encoder *hse); -static int has_literal(heatshrink_encoder *hse); - -/* Push COUNT (max 8) bits to the output buffer, which has room. */ -static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, - output_info *oi); -static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); -static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); - -#if HEATSHRINK_DYNAMIC_ALLOC -heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, - uint8_t lookahead_sz2) { - if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || - (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || - (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || - (lookahead_sz2 > window_sz2)) { - return NULL; - } - - /* Note: 2 * the window size is used because the buffer needs to fit - * (1 << window_sz2) bytes for the current input, and an additional - * (1 << window_sz2) bytes for the previous buffer of input, which - * will be scanned for useful backreferences. */ - size_t buf_sz = (2 << window_sz2); - - heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); - if (hse == NULL) { return NULL; } - hse->window_sz2 = window_sz2; - hse->lookahead_sz2 = lookahead_sz2; - heatshrink_encoder_reset(hse); - -#if HEATSHRINK_USE_INDEX - size_t index_sz = buf_sz*sizeof(uint16_t); - hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); - if (hse->search_index == NULL) { - HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); - return NULL; - } - hse->search_index->size = index_sz; -#endif - - LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", - buf_sz, get_input_buffer_size(hse)); - return hse; -} - -void heatshrink_encoder_free(heatshrink_encoder *hse) { - size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); -#if HEATSHRINK_USE_INDEX - size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; - HEATSHRINK_FREE(hse->search_index, index_sz); - (void)index_sz; -#endif - HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); - (void)buf_sz; -} -#endif - -void heatshrink_encoder_reset(heatshrink_encoder *hse) { - size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); - memset(hse->buffer, 0, buf_sz); - hse->input_size = 0; - hse->state = HSES_NOT_FULL; - hse->match_scan_index = 0; - hse->flags = 0; - hse->bit_index = 0x80; - hse->current_byte = 0x00; - hse->match_length = 0; - - hse->outgoing_bits = 0x0000; - hse->outgoing_bits_count = 0; - - #ifdef LOOP_DETECT - hse->loop_detect = (uint32_t)-1; - #endif -} - -HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, - uint8_t *in_buf, size_t size, size_t *input_size) { - if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { - return HSER_SINK_ERROR_NULL; - } - - /* Sinking more content after saying the content is done, tsk tsk */ - if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } - - /* Sinking more content before processing is done */ - if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } - - uint16_t write_offset = get_input_offset(hse) + hse->input_size; - uint16_t ibs = get_input_buffer_size(hse); - uint16_t rem = ibs - hse->input_size; - uint16_t cp_sz = rem < size ? rem : size; - - memcpy(&hse->buffer[write_offset], in_buf, cp_sz); - *input_size = cp_sz; - hse->input_size += cp_sz; - - LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", - cp_sz, size, write_offset, hse->input_size); - if (cp_sz == rem) { - LOG("-- internal buffer is now full\n"); - hse->state = HSES_FILLED; - } - - return HSER_SINK_OK; -} - - -/*************** - * Compression * - ***************/ - -static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, - uint16_t end, const uint16_t maxlen, uint16_t *match_length); -static void do_indexing(heatshrink_encoder *hse); - -static HSE_state st_step_search(heatshrink_encoder *hse); -static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_literal(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_br_index(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_br_length(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_save_backlog(heatshrink_encoder *hse); -static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, - output_info *oi); - -HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { - if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { - return HSER_POLL_ERROR_NULL; - } - if (out_buf_size == 0) { - LOG("-- MISUSE: output buffer size is 0\n"); - return HSER_POLL_ERROR_MISUSE; - } - *output_size = 0; - - output_info oi; - oi.buf = out_buf; - oi.buf_size = out_buf_size; - oi.output_size = output_size; - - while (1) { - LOG("-- polling, state %u (%s), flags 0x%02x\n", - hse->state, state_names[hse->state], hse->flags); - - uint8_t in_state = hse->state; - switch (in_state) { - case HSES_NOT_FULL: - return HSER_POLL_EMPTY; - case HSES_FILLED: - do_indexing(hse); - hse->state = HSES_SEARCH; - break; - case HSES_SEARCH: - hse->state = st_step_search(hse); - break; - case HSES_YIELD_TAG_BIT: - hse->state = st_yield_tag_bit(hse, &oi); - break; - case HSES_YIELD_LITERAL: - hse->state = st_yield_literal(hse, &oi); - break; - case HSES_YIELD_BR_INDEX: - hse->state = st_yield_br_index(hse, &oi); - break; - case HSES_YIELD_BR_LENGTH: - hse->state = st_yield_br_length(hse, &oi); - break; - case HSES_SAVE_BACKLOG: - hse->state = st_save_backlog(hse); - break; - case HSES_FLUSH_BITS: - hse->state = st_flush_bit_buffer(hse, &oi); - case HSES_DONE: - return HSER_POLL_EMPTY; - default: - LOG("-- bad state %s\n", state_names[hse->state]); - return HSER_POLL_ERROR_MISUSE; - } - - if (hse->state == in_state) { - /* Check if output buffer is exhausted. */ - if (*output_size == out_buf_size) return HSER_POLL_MORE; - } - } -} - -HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { - if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } - LOG("-- setting is_finishing flag\n"); - hse->flags |= FLAG_IS_FINISHING; - if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } - return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; -} - -static HSE_state st_step_search(heatshrink_encoder *hse) { - uint16_t window_length = get_input_buffer_size(hse); - uint16_t lookahead_sz = get_lookahead_size(hse); - uint16_t msi = hse->match_scan_index; - LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", - msi, hse->input_size + msi, 2*window_length, hse->input_size); - - bool fin = is_finishing(hse); - if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) { - /* Current search buffer is exhausted, copy it into the - * backlog and await more input. */ - LOG("-- end of search @ %d, saving backlog\n", msi); - return HSES_SAVE_BACKLOG; - } - - uint16_t input_offset = get_input_offset(hse); - uint16_t end = input_offset + msi; - - uint16_t start = 0; - if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */ - start = end - window_length + 1; - } else if (backlog_is_partial(hse)) { /* clamp to available data */ - start = end - window_length + 1; - if (start < lookahead_sz) { start = lookahead_sz; } - } else { /* only scan available input */ - start = input_offset; - } - - uint16_t max_possible = lookahead_sz; - if (hse->input_size - msi < lookahead_sz) { - max_possible = hse->input_size - msi; - } - - uint16_t match_length = 0; - uint16_t match_pos = find_longest_match(hse, - start, end, max_possible, &match_length); - - if (match_pos == MATCH_NOT_FOUND) { - LOG("ss Match not found\n"); - hse->match_scan_index++; - hse->flags |= FLAG_HAS_LITERAL; - hse->match_length = 0; - return HSES_YIELD_TAG_BIT; - } else { - LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); - hse->match_pos = match_pos; - hse->match_length = match_length; - ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/); - - return HSES_YIELD_TAG_BIT; - } -} - -static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - if (hse->match_length == 0) { - add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); - return HSES_YIELD_LITERAL; - } else { - add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); - hse->outgoing_bits = hse->match_pos - 1; - hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); - return HSES_YIELD_BR_INDEX; - } - } else { - return HSES_YIELD_TAG_BIT; /* output is full, continue */ - } -} - -static HSE_state st_yield_literal(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - push_literal_byte(hse, oi); - hse->flags &= ~FLAG_HAS_LITERAL; - if (on_final_literal(hse)) { return HSES_FLUSH_BITS; } - return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH; - } else { - return HSES_YIELD_LITERAL; - } -} - -static HSE_state st_yield_br_index(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - LOG("-- yielding backref index %u\n", hse->match_pos); - if (push_outgoing_bits(hse, oi) > 0) { - return HSES_YIELD_BR_INDEX; /* continue */ - } else { - hse->outgoing_bits = hse->match_length - 1; - hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); - return HSES_YIELD_BR_LENGTH; /* done */ - } - } else { - return HSES_YIELD_BR_INDEX; /* continue */ - } -} - -static HSE_state st_yield_br_length(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - LOG("-- yielding backref length %u\n", hse->match_length); - if (push_outgoing_bits(hse, oi) > 0) { - return HSES_YIELD_BR_LENGTH; - } else { - hse->match_scan_index += hse->match_length; - hse->match_length = 0; - return HSES_SEARCH; - } - } else { - return HSES_YIELD_BR_LENGTH; - } -} - -static HSE_state st_save_backlog(heatshrink_encoder *hse) { - if (is_finishing(hse)) { - /* copy remaining literal (if necessary) */ - if (has_literal(hse)) { - hse->flags |= FLAG_ON_FINAL_LITERAL; - return HSES_YIELD_TAG_BIT; - } else { - return HSES_FLUSH_BITS; - } - } else { - LOG("-- saving backlog\n"); - save_backlog(hse); - return HSES_NOT_FULL; - } -} - -static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, - output_info *oi) { - if (hse->bit_index == 0x80) { - LOG("-- done!\n"); - return HSES_DONE; - } else if (can_take_byte(oi)) { - LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); - oi->buf[(*oi->output_size)++] = hse->current_byte; - LOG("-- done!\n"); - return HSES_DONE; - } else { - return HSES_FLUSH_BITS; - } -} - -static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { - LOG("-- adding tag bit: %d\n", tag); - push_bits(hse, 1, tag, oi); -} - -static uint16_t get_input_offset(heatshrink_encoder *hse) { - return get_input_buffer_size(hse); -} - -static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { - return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); - (void)hse; -} - -static uint16_t get_lookahead_size(heatshrink_encoder *hse) { - return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); - (void)hse; -} - -static void do_indexing(heatshrink_encoder *hse) { -#if HEATSHRINK_USE_INDEX - /* Build an index array I that contains flattened linked lists - * for the previous instances of every byte in the buffer. - * - * For example, if buf[200] == 'x', then index[200] will either - * be an offset i such that buf[i] == 'x', or a negative offset - * to indicate end-of-list. This significantly speeds up matching, - * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. - * - * Future optimization options: - * 1. Since any negative value represents end-of-list, the other - * 15 bits could be used to improve the index dynamically. - * - * 2. Likewise, the last lookahead_sz bytes of the index will - * not be usable, so temporary data could be stored there to - * dynamically improve the index. - * */ - struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); - uint16_t last[256]; - memset(last, 0xFF, sizeof(last)); - - uint8_t * const data = hse->buffer; - int16_t * const index = hsi->index; - - const uint16_t input_offset = get_input_offset(hse); - const uint16_t end = input_offset + hse->input_size; - - for (uint16_t i=0; iflags & FLAG_IS_FINISHING; -} - -static int backlog_is_partial(heatshrink_encoder *hse) { - return hse->flags & FLAG_BACKLOG_IS_PARTIAL; -} - -static int backlog_is_filled(heatshrink_encoder *hse) { - return hse->flags & FLAG_BACKLOG_IS_FILLED; -} - -static int on_final_literal(heatshrink_encoder *hse) { - return hse->flags & FLAG_ON_FINAL_LITERAL; -} - -static int has_literal(heatshrink_encoder *hse) { - return (hse->flags & FLAG_HAS_LITERAL); -} - -static int can_take_byte(output_info *oi) { - return *oi->output_size < oi->buf_size; -} - -/* Return the longest match for the bytes at buf[end:end+maxlen] between - * buf[start] and buf[end-1]. If no match is found, return -1. */ -static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, - uint16_t end, const uint16_t maxlen, uint16_t *match_length) { - LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", - end, end + maxlen, start, end + maxlen - 1, maxlen); - uint8_t *buf = hse->buffer; - - uint16_t match_maxlen = 0; - uint16_t match_index = MATCH_NOT_FOUND; - const uint16_t break_even_point = 3; - uint16_t len = 0; - uint8_t * const needlepoint = &buf[end]; -#if HEATSHRINK_USE_INDEX - struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); - int16_t pos = hsi->index[end]; - - while (pos >= start) { - uint8_t * const pospoint = &buf[pos]; - len = 0; - - /* Only check matches that will potentially beat the current maxlen. - * This is redundant with the index if match_maxlen is 0, but the - * added branch overhead to check if it == 0 seems to be worse. */ - if (pospoint[match_maxlen] != needlepoint[match_maxlen]) { - pos = hsi->index[pos]; - continue; - } - - for (len = 1; len < maxlen; len++) { - if (pospoint[len] != needlepoint[len]) break; - } - - if (len > match_maxlen) { - match_maxlen = len; - match_index = pos; - if (len == maxlen) { break; } /* won't find better */ - } - pos = hsi->index[pos]; - } -#else - for (int16_t pos=end - 1; pos >= start; pos--) { - uint8_t * const pospoint = &buf[pos]; - if ((pospoint[match_maxlen] == needlepoint[match_maxlen]) - && (*pospoint == *needlepoint)) { - for (len=1; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", - pos + len, pospoint[len], needlepoint[len], start); - } - if (pospoint[len] != needlepoint[len]) { break; } - } - if (len > match_maxlen) { - match_maxlen = len; - match_index = pos; - if (len == maxlen) { break; } /* don't keep searching */ - } - } - } -#endif - - if (match_maxlen >= break_even_point) { - LOG("-- best match: %u bytes at -%u\n", - match_maxlen, end - match_index); - *match_length = match_maxlen; - return end - match_index; - } - LOG("-- none found\n"); - return MATCH_NOT_FOUND; -} - -static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { - uint8_t count = 0; - uint8_t bits = 0; - if (hse->outgoing_bits_count > 8) { - count = 8; - bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); - } else { - count = hse->outgoing_bits_count; - bits = hse->outgoing_bits; - } - - if (count > 0) { - LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); - push_bits(hse, count, bits, oi); - hse->outgoing_bits_count -= count; - } - return count; -} - -/* Push COUNT (max 8) bits to the output buffer, which has room. - * Bytes are set from the lowest bits, up. */ -static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, - output_info *oi) { - ASSERT(count <= 8); - LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); - - /* If adding a whole byte and at the start of a new output byte, - * just push it through whole and skip the bit IO loop. */ - if (count == 8 && hse->bit_index == 0x80) { - oi->buf[(*oi->output_size)++] = bits; - } else { - for (int i=count - 1; i>=0; i--) { - bool bit = bits & (1 << i); - if (bit) { hse->current_byte |= hse->bit_index; } - if (0) { - LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", - bit ? 1 : 0, hse->bit_index, hse->current_byte); - } - hse->bit_index >>= 1; - if (hse->bit_index == 0x00) { - hse->bit_index = 0x80; - LOG(" > pushing byte 0x%02x\n", hse->current_byte); - oi->buf[(*oi->output_size)++] = hse->current_byte; - hse->current_byte = 0x00; - } - } - } -} - -static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { - uint16_t processed_offset = hse->match_scan_index - 1; - uint16_t input_offset = get_input_offset(hse) + processed_offset; - uint8_t c = hse->buffer[input_offset]; - LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", - c, isprint(c) ? c : '.', input_offset); - push_bits(hse, 8, c, oi); -} - -static void save_backlog(heatshrink_encoder *hse) { - size_t input_buf_sz = get_input_buffer_size(hse); - - uint16_t msi = hse->match_scan_index; - - /* Copy processed data to beginning of buffer, so it can be - * used for future matches. Don't bother checking whether the - * input is less than the maximum size, because if it isn't, - * we're done anyway. */ - uint16_t rem = input_buf_sz - msi; // unprocessed bytes - uint16_t shift_sz = input_buf_sz + rem; - - memmove(&hse->buffer[0], - &hse->buffer[input_buf_sz - rem], - shift_sz); - - if (backlog_is_partial(hse)) { - /* The whole backlog is filled in now, so include it in scans. */ - hse->flags |= FLAG_BACKLOG_IS_FILLED; - } else { - /* Include backlog, except for the first lookahead_sz bytes, which - * are still undefined. */ - hse->flags |= FLAG_BACKLOG_IS_PARTIAL; - } - hse->match_scan_index = 0; - hse->input_size -= input_buf_sz - rem; -} diff --git a/espfs/heatshrink/heatshrink_encoder.h b/espfs/heatshrink/heatshrink_encoder.h deleted file mode 100644 index 18c1773..0000000 --- a/espfs/heatshrink/heatshrink_encoder.h +++ /dev/null @@ -1,109 +0,0 @@ -#ifndef HEATSHRINK_ENCODER_H -#define HEATSHRINK_ENCODER_H - -#include -#include -#include "heatshrink_common.h" -#include "heatshrink_config.h" - -typedef enum { - HSER_SINK_OK, /* data sunk into input buffer */ - HSER_SINK_ERROR_NULL=-1, /* NULL argument */ - HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ -} HSE_sink_res; - -typedef enum { - HSER_POLL_EMPTY, /* input exhausted */ - HSER_POLL_MORE, /* poll again for more output */ - HSER_POLL_ERROR_NULL=-1, /* NULL argument */ - HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ -} HSE_poll_res; - -typedef enum { - HSER_FINISH_DONE, /* encoding is complete */ - HSER_FINISH_MORE, /* more output remaining; use poll */ - HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ -} HSE_finish_res; - -#if HEATSHRINK_DYNAMIC_ALLOC -#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ - ((HSE)->window_sz2) -#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ - ((HSE)->lookahead_sz2) -#define HEATSHRINK_ENCODER_INDEX(HSE) \ - ((HSE)->search_index) -struct hs_index { - uint16_t size; - int16_t index[]; -}; -#else -#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ - (HEATSHRINK_STATIC_WINDOW_BITS) -#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ - (HEATSHRINK_STATIC_LOOKAHEAD_BITS) -#define HEATSHRINK_ENCODER_INDEX(HSE) \ - (&(HSE)->search_index) -struct hs_index { - uint16_t size; - int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; -}; -#endif - -typedef struct { - uint16_t input_size; /* bytes in input buffer */ - uint16_t match_scan_index; - uint16_t match_length; - uint16_t match_pos; - uint16_t outgoing_bits; /* enqueued outgoing bits */ - uint8_t outgoing_bits_count; - uint8_t flags; - uint8_t state; /* current state machine node */ - uint8_t current_byte; /* current byte of output */ - uint8_t bit_index; /* current bit index */ -#if HEATSHRINK_DYNAMIC_ALLOC - uint8_t window_sz2; /* 2^n size of window */ - uint8_t lookahead_sz2; /* 2^n size of lookahead */ -#if HEATSHRINK_USE_INDEX - struct hs_index *search_index; -#endif - /* input buffer and / sliding window for expansion */ - uint8_t buffer[]; -#else - #if HEATSHRINK_USE_INDEX - struct hs_index search_index; - #endif - /* input buffer and / sliding window for expansion */ - uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; -#endif -} heatshrink_encoder; - -#if HEATSHRINK_DYNAMIC_ALLOC -/* Allocate a new encoder struct and its buffers. - * Returns NULL on error. */ -heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, - uint8_t lookahead_sz2); - -/* Free an encoder. */ -void heatshrink_encoder_free(heatshrink_encoder *hse); -#endif - -/* Reset an encoder. */ -void heatshrink_encoder_reset(heatshrink_encoder *hse); - -/* Sink up to SIZE bytes from IN_BUF into the encoder. - * INPUT_SIZE is set to the number of bytes actually sunk (in case a - * buffer was filled.). */ -HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, - uint8_t *in_buf, size_t size, size_t *input_size); - -/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into - * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ -HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size); - -/* Notify the encoder that the input stream is finished. - * If the return value is HSER_FINISH_MORE, there is still more output, so - * call heatshrink_encoder_poll and repeat. */ -HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); - -#endif diff --git a/espfs/heatshrink/test_heatshrink_dynamic.c b/espfs/heatshrink/test_heatshrink_dynamic.c deleted file mode 100644 index 1e18a69..0000000 --- a/espfs/heatshrink/test_heatshrink_dynamic.c +++ /dev/null @@ -1,999 +0,0 @@ -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" - -#if !HEATSHRINK_DYNAMIC_ALLOC -#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for dynamic allocation test suite. -#endif - -SUITE(encoding); -SUITE(decoding); -SUITE(integration); - -#ifdef HEATSHRINK_HAS_THEFT -SUITE(properties); -#endif - -static void dump_buf(char *name, uint8_t *buf, uint16_t count) { - for (int i=0; iinput_size, 6); - ASSERT_EQ(hsd->input_index, 0); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_return_empty_if_empty(void) { - uint8_t output[256]; - size_t out_sz = 0; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_EMPTY, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_reject_null_hsd(void) { - uint8_t output[256]; - size_t out_sz = 0; - HSD_poll_res res = heatshrink_decoder_poll(NULL, output, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - PASS(); -} - -TEST decoder_poll_should_reject_null_output_buffer(void) { - size_t out_sz = 0; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, NULL, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_reject_null_output_size_pointer(void) { - uint8_t output[256]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, NULL); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_literal(void) { - uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0 }; //"foo" - uint8_t output[4]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 3); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - ASSERT_EQ(3, out_sz); - ASSERT_EQ('f', output[0]); - ASSERT_EQ('o', output[1]); - ASSERT_EQ('o', output[2]); - - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_literal_and_backref(void) { - uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo" - uint8_t output[6]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7); - memset(output, 0, sizeof(*output)); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - (void)heatshrink_decoder_poll(hsd, output, 6, &out_sz); - - if (0) dump_buf("output", output, out_sz); - ASSERT_EQ(6, out_sz); - ASSERT_EQ('f', output[0]); - ASSERT_EQ('o', output[1]); - ASSERT_EQ('o', output[2]); - ASSERT_EQ('f', output[3]); - ASSERT_EQ('o', output[4]); - ASSERT_EQ('o', output[5]); - - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_self_overlapping_backref(void) { - /* "aaaaa" == (literal, 1), ('a'), (backref, 1 back, 4 bytes) */ - uint8_t input[] = {0xb0, 0x80, 0x01, 0x80}; - uint8_t output[6]; - uint8_t expected[] = {'a', 'a', 'a', 'a', 'a'}; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 7); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - (void)heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); - - if (0) dump_buf("output", output, out_sz); - ASSERT_EQ(sizeof(expected), out_sz); - for (size_t i=0; iwindow_sz2, - cfg->lookahead_sz2); - heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size, - cfg->window_sz2, cfg->lookahead_sz2); - size_t comp_sz = input_size + (input_size/2) + 4; - size_t decomp_sz = input_size + (input_size/2) + 4; - uint8_t *comp = malloc(comp_sz); - uint8_t *decomp = malloc(decomp_sz); - if (comp == NULL) FAILm("malloc fail"); - if (decomp == NULL) FAILm("malloc fail"); - memset(comp, 0, comp_sz); - memset(decomp, 0, decomp_sz); - - size_t count = 0; - - if (cfg->log_lvl > 1) { - printf("\n^^ COMPRESSING\n"); - dump_buf("input", input, input_size); - } - - size_t sunk = 0; - size_t polled = 0; - while (sunk < input_size) { - ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0); - sunk += count; - if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - } - - HSE_poll_res pres; - do { /* "turn the crank" */ - pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSER_POLL_MORE); - ASSERT_EQ(HSER_POLL_EMPTY, pres); - if (polled >= comp_sz) FAILm("compression should never expand that much"); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); - } - } - if (cfg->log_lvl > 0) printf("in: %u compressed: %zu ", input_size, polled); - size_t compressed_size = polled; - sunk = 0; - polled = 0; - - if (cfg->log_lvl > 1) { - printf("\n^^ DECOMPRESSING\n"); - dump_buf("comp", comp, compressed_size); - } - while (sunk < compressed_size) { - ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); - sunk += count; - if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == compressed_size) { - ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd)); - } - - HSD_poll_res pres; - do { - pres = heatshrink_decoder_poll(hsd, &decomp[polled], - decomp_sz - polled, &count); - ASSERT(pres >= 0); - ASSERT(count > 0); - polled += count; - if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSDR_POLL_MORE); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - if (sunk == compressed_size) { - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - ASSERT_EQ(HSDR_FINISH_DONE, fres); - } - - if (polled > input_size) { - printf("\nExpected %zd, got %zu\n", (size_t)input_size, polled); - FAILm("Decompressed data is larger than original input"); - } - } - if (cfg->log_lvl > 0) printf("decompressed: %zu\n", polled); - if (polled != input_size) { - FAILm("Decompressed length does not match original input length"); - } - - if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled); - for (uint32_t i=0; i out[%d] == 0x%02x ('%c') %c\n", - j, input[j], isprint(input[j]) ? input[j] : '.', - j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.', - input[j] == decomp[j] ? ' ' : 'X'); - } - } - } - ASSERT_EQ(input[i], decomp[i]); - } - free(comp); - free(decomp); - heatshrink_encoder_free(hse); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST data_without_duplication_should_match(void) { - uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', - 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 256; - return compress_and_expand_and_check(input, sizeof(input), &cfg); -} - -TEST data_with_simple_repetition_should_compress_and_decompress_properly(void) { - uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', - 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', - 'c', 'd', 'e', 'f', 'g', 'h'}; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 256; - return compress_and_expand_and_check(input, sizeof(input), &cfg); -} - -TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) { - heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); - uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', - 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - uint8_t comp[60]; - uint8_t decomp[60]; - size_t count = 0; - int log = 0; - - if (log) dump_buf("input", input, sizeof(input)); - for (uint32_t i=0; i= 0); - } - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - - size_t packed_count = 0; - do { - ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); - packed_count += count; - } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); - - if (log) dump_buf("comp", comp, packed_count); - for (uint32_t i=0; i= 0); - } - - for (uint32_t i=0; i= 0); - } - - if (log) dump_buf("decomp", decomp, sizeof(input)); - for (uint32_t i=0; i= 0); - } - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - - size_t packed_count = 0; - do { - ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); - packed_count += count; - } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); - - if (log) dump_buf("comp", comp, packed_count); - for (uint32_t i=0; i= 0); - } - - for (uint32_t i=0; i= 0); - } - - if (log) dump_buf("decomp", decomp, sizeof(input)); - for (uint32_t i=0; ilog_lvl > 0) { - printf("\n-- size %u, seed %u, input buf %zu\n", - size, seed, cfg->decoder_input_buffer_size); - } - fill_with_pseudorandom_letters(input, size, seed); - return compress_and_expand_and_check(input, size, cfg); -} - -TEST small_input_buffer_should_not_impact_decoder_correctness(void) { - int size = 5; - uint8_t input[size]; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 5; - for (uint16_t i=0; i= 19901L - printf("\n\nFuzzing (single-byte sizes):\n"); - for (uint8_t lsize=3; lsize < 8; lsize++) { - for (uint32_t size=1; size < 128*1024L; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ - if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); - for (uint32_t seed=1; seed<=10; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = lsize; - cfg.decoder_input_buffer_size = ibs; - RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); - } - } - } - } - - printf("\nFuzzing (multi-byte sizes):\n"); - for (uint8_t lsize=6; lsize < 9; lsize++) { - for (uint32_t size=1; size < 128*1024L; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ - if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); - for (uint32_t seed=1; seed<=10; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 11; - cfg.lookahead_sz2 = lsize; - cfg.decoder_input_buffer_size = ibs; - RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); - } - } - } - } - -#endif -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - RUN_SUITE(encoding); - RUN_SUITE(decoding); - RUN_SUITE(integration); - #ifdef HEATSHRINK_HAS_THEFT - RUN_SUITE(properties); - #endif - GREATEST_MAIN_END(); /* display results */ -} diff --git a/espfs/heatshrink/test_heatshrink_dynamic_theft.c b/espfs/heatshrink/test_heatshrink_dynamic_theft.c deleted file mode 100644 index 2752886..0000000 --- a/espfs/heatshrink/test_heatshrink_dynamic_theft.c +++ /dev/null @@ -1,521 +0,0 @@ -#include "heatshrink_config.h" -#ifdef HEATSHRINK_HAS_THEFT - -#include -#include -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" -#include "theft.h" -#include "greatest_theft.h" - -#if !HEATSHRINK_DYNAMIC_ALLOC -#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for this test suite. -#endif - -SUITE(properties); - -typedef struct { - int limit; - int fails; - int dots; -} test_env; - -typedef struct { - size_t size; - uint8_t buf[]; -} rbuf; - -static void *rbuf_alloc_cb(struct theft *t, theft_hash seed, void *env) { - test_env *te = (test_env *)env; - //printf("seed is 0x%016llx\n", seed); - - size_t sz = (size_t)(seed % te->limit) + 1; - rbuf *r = malloc(sizeof(rbuf) + sz); - if (r == NULL) { return THEFT_ERROR; } - r->size = sz; - - for (size_t i = 0; i < sz; i += sizeof(theft_hash)) { - theft_hash s = theft_random(t); - for (uint8_t b = 0; b < sizeof(theft_hash); b++) { - if (i + b >= sz) { break; } - r->buf[i + b] = (uint8_t) (s >> (8*b)) & 0xff; - } - } - - return r; -} - -static void rbuf_free_cb(void *instance, void *env) { - free(instance); - (void)env; -} - -static uint64_t rbuf_hash_cb(void *instance, void *env) { - rbuf *r = (rbuf *)instance; - (void)env; - return theft_hash_onepass(r->buf, r->size); -} - -/* Make a copy of a buffer, keeping NEW_SZ bytes starting at OFFSET. */ -static void *copy_rbuf_subset(rbuf *cur, size_t new_sz, size_t byte_offset) { - if (new_sz == 0) { return THEFT_DEAD_END; } - rbuf *nr = malloc(sizeof(rbuf) + new_sz); - if (nr == NULL) { return THEFT_ERROR; } - nr->size = new_sz; - memcpy(nr->buf, &cur->buf[byte_offset], new_sz); - /* printf("%zu -> %zu\n", cur->size, new_sz); */ - return nr; -} - -/* Make a copy of a buffer, but only PORTION, starting OFFSET in - * (e.g. the third quarter is (0.25 at +0.75). Rounds to ints. */ -static void *copy_rbuf_percent(rbuf *cur, float portion, float offset) { - size_t new_sz = cur->size * portion; - size_t byte_offset = (size_t)(cur->size * offset); - return copy_rbuf_subset(cur, new_sz, byte_offset); -} - -/* How to shrink a random buffer to a simpler one. */ -static void *rbuf_shrink_cb(void *instance, uint32_t tactic, void *env) { - rbuf *cur = (rbuf *)instance; - - if (tactic == 0) { /* first half */ - return copy_rbuf_percent(cur, 0.5, 0); - } else if (tactic == 1) { /* second half */ - return copy_rbuf_percent(cur, 0.5, 0.5); - } else if (tactic <= 18) { /* drop 1-16 bytes at start */ - const int last_tactic = 1; - const size_t drop = tactic - last_tactic; - if (cur->size < drop) { return THEFT_DEAD_END; } - return copy_rbuf_subset(cur, cur->size - drop, drop); - } else if (tactic <= 34) { /* drop 1-16 bytes at end */ - const int last_tactic = 18; - const size_t drop = tactic - last_tactic; - if (cur->size < drop) { return THEFT_DEAD_END; } - return copy_rbuf_subset(cur, cur->size - drop, 0); - } else if (tactic == 35) { - /* Divide every byte by 2, saturating at 0 */ - rbuf *cp = copy_rbuf_percent(cur, 1, 0); - if (cp == NULL) { return THEFT_ERROR; } - for (size_t i = 0; i < cp->size; i++) { cp->buf[i] /= 2; } - return cp; - } else if (tactic == 36) { - /* subtract 1 from every byte, saturating at 0 */ - rbuf *cp = copy_rbuf_percent(cur, 1, 0); - if (cp == NULL) { return THEFT_ERROR; } - for (size_t i = 0; i < cp->size; i++) { - if (cp->buf[i] > 0) { cp->buf[i]--; } - } - return cp; - } else { - (void)env; - return THEFT_NO_MORE_TACTICS; - } - - return THEFT_NO_MORE_TACTICS; -} - -static void rbuf_print_cb(FILE *f, void *instance, void *env) { - rbuf *r = (rbuf *)instance; - (void)env; - fprintf(f, "buf[%zd]:\n ", r->size); - uint8_t bytes = 0; - for (size_t i = 0; i < r->size; i++) { - fprintf(f, "%02x", r->buf[i]); - bytes++; - if (bytes == 16) { - fprintf(f, "\n "); - bytes = 0; - } - } - fprintf(f, "\n"); -} - -static struct theft_type_info rbuf_info = { - .alloc = rbuf_alloc_cb, - .free = rbuf_free_cb, - .hash = rbuf_hash_cb, - .shrink = rbuf_shrink_cb, - .print = rbuf_print_cb, -}; - -static theft_progress_callback_res -progress_cb(struct theft_trial_info *info, void *env) { - test_env *te = (test_env *)env; - if ((info->trial & 0xff) == 0) { - printf("."); - fflush(stdout); - te->dots++; - if (te->dots == 64) { - printf("\n"); - te->dots = 0; - } - } - - if (info->status == THEFT_TRIAL_FAIL) { - te->fails++; - rbuf *cur = info->args[0]; - if (cur->size < 5) { return THEFT_PROGRESS_HALT; } - } - - if (te->fails > 10) { - return THEFT_PROGRESS_HALT; - } - return THEFT_PROGRESS_CONTINUE; -} - -/* For an arbitrary input buffer, it should never get stuck in a - * state where the data has been sunk but no data can be polled. */ -static theft_trial_res prop_should_not_get_stuck(void *input) { - /* Make a buffer large enough for the output: 4 KB of input with - * each 16 bits becoming up to 16 bytes will fit in a 64 KB buffer. - * (4 KB of input comes from `env.limit = 1 << 12;` below.) */ - uint8_t output[64 * 1024]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc((64 * 1024L) - 1, 12, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t count = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, r->buf, r->size, &count); - if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } - - size_t out_sz = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); - if (pres != HSDR_POLL_EMPTY) { return THEFT_TRIAL_FAIL; } - - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - heatshrink_decoder_free(hsd); - if (fres != HSDR_FINISH_DONE) { return THEFT_TRIAL_FAIL; } - - return THEFT_TRIAL_PASS; -} - -static bool get_time_seed(theft_seed *seed) -{ - struct timeval tv; - if (-1 == gettimeofday(&tv, NULL)) { return false; } - *seed = (theft_seed)((tv.tv_sec << 32) | tv.tv_usec); - /* printf("seed is 0x%016llx\n", *seed); */ - return true; -} - -TEST decoder_fuzzing_should_not_detect_stuck_state(void) { - // Get a random number seed based on the time - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - /* Pass the max buffer size for this property (4 KB) in a closure */ - test_env env = { .limit = 1 << 12 }; - - theft_seed always_seeds = { 0xe87bb1f61032a061 }; - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_should_not_get_stuck, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 100000, - .progress_cb = progress_cb, - .env = &env, - - .always_seeds = &always_seeds, - .always_seed_count = 1, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - GREATEST_ASSERT_EQm("should_not_get_stuck", THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res prop_encoded_and_decoded_data_should_match(void *input) { - uint8_t e_output[64 * 1024]; - uint8_t d_output[64 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - heatshrink_decoder *hsd = heatshrink_decoder_alloc(4096, 12, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t e_input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - r->buf, r->size, &e_input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - if (e_input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t e_output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - e_output, sizeof(e_output), &e_output_size); - if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t count = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, e_output, e_output_size, &count); - if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } - - size_t d_output_size = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, d_output, - sizeof(d_output), &d_output_size); - if (pres != HSDR_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - if (d_output_size != r->size) { - printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; - } - - if (0 != memcmp(d_output, r->buf, d_output_size)) { - return THEFT_TRIAL_FAIL; - } - - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - if (fres != HSDR_FINISH_DONE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - heatshrink_encoder_free(hse); - heatshrink_decoder_free(hsd); - return THEFT_TRIAL_PASS; -} - - -TEST encoded_and_decoded_data_should_match(void) { - test_env env = { .limit = 1 << 11 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoded_and_decoded_data_should_match, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 1000000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static size_t ceil_nine_eighths(size_t sz) { - return sz + sz/8 + (sz & 0x07 ? 1 : 0); -} - -static theft_trial_res -prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void *input) { - uint8_t output[32 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - r->buf, r->size, &input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - /* Assumes data fits in one sink, failure here means buffer must be larger. */ - if (input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - output, sizeof(output), &output_size); - if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t ceil_9_8s = ceil_nine_eighths(r->size); - if (output_size > ceil_9_8s) { - return THEFT_TRIAL_FAIL; - } - - heatshrink_encoder_free(hse); - return THEFT_TRIAL_PASS; -} - -TEST encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void) { - test_env env = { .limit = 1 << 11 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res -prop_encoder_should_always_make_progress(void *instance) { - uint8_t output[64 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)instance; - - size_t sunk = 0; - int no_progress = 0; - - while (1) { - if (sunk < r->size) { - size_t input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - &r->buf[sunk], r->size - sunk, &input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - sunk += input_size; - } else { - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres == HSER_FINISH_DONE) { - break; - } else if (efres != HSER_FINISH_MORE) { - printf("FAIL %d\n", __LINE__); - return THEFT_TRIAL_FAIL; - } - } - - size_t output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - output, sizeof(output), &output_size); - if (epres < 0) { return THEFT_TRIAL_ERROR; } - if (output_size == 0 && sunk == r->size) { - no_progress++; - if (no_progress > 2) { - return THEFT_TRIAL_FAIL; - } - } else { - no_progress = 0; - } - } - - heatshrink_encoder_free(hse); - return THEFT_TRIAL_PASS; -} - -TEST encoder_should_always_make_progress(void) { - test_env env = { .limit = 1 << 15 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoder_should_always_make_progress, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res -prop_decoder_should_always_make_progress(void *instance) { - uint8_t output[64 * 1024]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(512, 8, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)instance; - - size_t sunk = 0; - int no_progress = 0; - - while (1) { - if (sunk < r->size) { - size_t input_size = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, - &r->buf[sunk], r->size - sunk, &input_size); - if (sres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - sunk += input_size; - } else { - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - if (fres == HSDR_FINISH_DONE) { - break; - } else if (fres != HSDR_FINISH_MORE) { - printf("FAIL %d\n", __LINE__); - return THEFT_TRIAL_FAIL; - } - } - - size_t output_size = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, - output, sizeof(output), &output_size); - if (pres < 0) { return THEFT_TRIAL_ERROR; } - if (output_size == 0 && sunk == r->size) { - no_progress++; - if (no_progress > 2) { - return THEFT_TRIAL_FAIL; - } - } else { - no_progress = 0; - } - } - - heatshrink_decoder_free(hsd); - return THEFT_TRIAL_PASS; -} - -TEST decoder_should_always_make_progress(void) { - test_env env = { .limit = 1 << 15 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_decoder_should_always_make_progress, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -SUITE(properties) { - RUN_TEST(decoder_fuzzing_should_not_detect_stuck_state); - RUN_TEST(encoded_and_decoded_data_should_match); - RUN_TEST(encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst); - RUN_TEST(encoder_should_always_make_progress); - RUN_TEST(decoder_should_always_make_progress); -} -#else -struct because_iso_c_requires_at_least_one_declaration; -#endif diff --git a/espfs/heatshrink/test_heatshrink_static.c b/espfs/heatshrink/test_heatshrink_static.c deleted file mode 100644 index e9c9754..0000000 --- a/espfs/heatshrink/test_heatshrink_static.c +++ /dev/null @@ -1,167 +0,0 @@ -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" - -#if HEATSHRINK_DYNAMIC_ALLOC -#error HEATSHRINK_DYNAMIC_ALLOC must be false for static allocation test suite. -#endif - -SUITE(integration); - -/* The majority of the tests are in test_heatshrink_dynamic, because that allows - * instantiating encoders/decoders with different settings at run-time. */ - -static heatshrink_encoder hse; -static heatshrink_decoder hsd; - -static void fill_with_pseudorandom_letters(uint8_t *buf, uint16_t size, uint32_t seed) { - uint64_t rn = 9223372036854775783; /* prime under 2^64 */ - for (int i=0; i 1) { - printf("\n^^ COMPRESSING\n"); - dump_buf("input", input, input_size); - } - - uint32_t sunk = 0; - uint32_t polled = 0; - while (sunk < input_size) { - ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); - sunk += count; - if (log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); - } - - HSE_poll_res pres; - do { /* "turn the crank" */ - pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSER_POLL_MORE); - ASSERT_EQ(HSER_POLL_EMPTY, pres); - if (polled >= comp_sz) FAILm("compression should never expand that much"); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); - } - } - if (log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled); - uint32_t compressed_size = polled; - sunk = 0; - polled = 0; - - if (log_lvl > 1) { - printf("\n^^ DECOMPRESSING\n"); - dump_buf("comp", comp, compressed_size); - } - while (sunk < compressed_size) { - ASSERT(heatshrink_decoder_sink(&hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); - sunk += count; - if (log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == compressed_size) { - ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(&hsd)); - } - - HSD_poll_res pres; - do { - pres = heatshrink_decoder_poll(&hsd, &decomp[polled], - decomp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSDR_POLL_MORE); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - if (sunk == compressed_size) { - HSD_finish_res fres = heatshrink_decoder_finish(&hsd); - ASSERT_EQ(HSDR_FINISH_DONE, fres); - } - - if (polled > input_size) { - FAILm("Decompressed data is larger than original input"); - } - } - if (log_lvl > 0) printf("decompressed: %u\n", polled); - if (polled != input_size) { - FAILm("Decompressed length does not match original input length"); - } - - if (log_lvl > 1) dump_buf("decomp", decomp, polled); - for (size_t i=0; i out[%zd] == 0x%02x ('%c')\n", - j, input[j], isprint(input[j]) ? input[j] : '.', - j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.'); - } - } - } - ASSERT_EQ(input[i], decomp[i]); - } - free(comp); - free(decomp); - PASS(); -} - -TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed) { - uint8_t input[size]; - fill_with_pseudorandom_letters(input, size, seed); - return compress_and_expand_and_check(input, size, 0); -} - -SUITE(integration) { -#if __STDC_VERSION__ >= 19901L - for (uint32_t size=1; size < 64*1024; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint32_t seed=1; seed<=100; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - RUN_TESTp(pseudorandom_data_should_match, size, seed); - } - } -#endif -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - printf("INPUT_BUFFER_SIZE: %u\n", HEATSHRINK_STATIC_INPUT_BUFFER_SIZE); - printf("WINDOW_BITS: %u\n", HEATSHRINK_STATIC_WINDOW_BITS); - printf("LOOKAHEAD_BITS: %u\n", HEATSHRINK_STATIC_LOOKAHEAD_BITS); - - printf("sizeof(heatshrink_encoder): %zd\n", sizeof(heatshrink_encoder)); - printf("sizeof(heatshrink_decoder): %zd\n", sizeof(heatshrink_decoder)); - RUN_SUITE(integration); - GREATEST_MAIN_END(); /* display results */ -} diff --git a/espfs/heatshrink_config_custom.h b/espfs/heatshrink_config_custom.h deleted file mode 100644 index f885f87..0000000 --- a/espfs/heatshrink_config_custom.h +++ /dev/null @@ -1,30 +0,0 @@ -//Heatshrink config for the decompressor. -#ifndef HEATSHRINK_CONFIG_H -#define HEATSHRINK_CONFIG_H - -/* Should functionality assuming dynamic allocation be used? */ -#define HEATSHRINK_DYNAMIC_ALLOC 1 - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Optional replacement of malloc/free */ - #ifdef __ets__ - #define HEATSHRINK_MALLOC(SZ) os_malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) os_free(P) - #else - #define HEATSHRINK_MALLOC(SZ) malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) free(P) - #endif -#else - /* Required parameters for static configuration */ - #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 - #define HEATSHRINK_STATIC_WINDOW_BITS 8 - #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 -#endif - -/* Turn on logging for debugging. */ -#define HEATSHRINK_DEBUGGING_LOGS 0 - -/* Use indexing for faster compression. (This requires additional space.) */ -#define HEATSHRINK_USE_INDEX 1 - -#endif diff --git a/espfs/heatshrink_decoder.c b/espfs/heatshrink_decoder.c deleted file mode 100644 index 9a4a9f7..0000000 --- a/espfs/heatshrink_decoder.c +++ /dev/null @@ -1,19 +0,0 @@ -#include "espfs.h" -#ifdef ESPFS_HEATSHRINK -//Stupid wrapper so we don't have to move c-files around -//Also loads httpd-specific config. - -#ifdef __ets__ -//esp build - -#include - -#define memset(x,y,z) os_memset(x,y,z) -#define memcpy(x,y,z) os_memcpy(x,y,z) -#endif - -#include "heatshrink_config_custom.h" -#include "heatshrink/heatshrink_decoder.c" - - -#endif diff --git a/espfs/mkespfsimage/Makefile b/espfs/mkespfsimage/Makefile index 26484d4..b05d5b0 100644 --- a/espfs/mkespfsimage/Makefile +++ b/espfs/mkespfsimage/Makefile @@ -1,36 +1,24 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes +GZIP_COMPRESSION?=no -TARGET = mkespfsimage.exe - -CC = gcc -LD = $(CC) -CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 -LDFLAGS=-Lmman-win32 -lmman +CFLAGS=-I.. -std=gnu99 ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -LDFLAGS += -lz +LDFLAGS+=-lz +CFLAGS+=-DESPFS_GZIP endif -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK +ifeq ($(OS),Windows_NT) +CFLAGS+=-Imman-win32 +LDFLAGS+=-Lmman-win32 -lmman +TARGET =mkespfsimage.exe +else +TARGET =mkespfsimage endif -OBJECTS = main.o heatshrink_encoder.o - -all: libmman $(TARGET) - -libmman: - $(Q) make -C mman-win32 +OBJS=main.o -$(TARGET): $(OBJECTS) - $(LD) -o $@ $^ $(LDFLAGS) - -%.o: %.c - $(CC) $(CFLAGS) -o $@ $^ +$(TARGET): $(OBJS) + $(CC) -o $@ $^ $(LDFLAGS) clean: - rm -f $(OBJECTS) $(TARGET) - -.PHONY: all clean + rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.linux b/espfs/mkespfsimage/Makefile.linux deleted file mode 100644 index f1ae1aa..0000000 --- a/espfs/mkespfsimage/Makefile.linux +++ /dev/null @@ -1,24 +0,0 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes - -CFLAGS=-I../../heatshrink -I../../include -I.. -std=gnu99 -ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -endif - -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - -OBJS=main.o heatshrink_encoder.o -TARGET=mkespfsimage - -$(TARGET): $(OBJS) -ifeq ("$(GZIP_COMPRESSION)","yes") - $(CC) -o $@ $^ -lz -else - $(CC) -o $@ $^ -endif - -clean: - rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.windows b/espfs/mkespfsimage/Makefile.windows deleted file mode 100644 index 15aa8e3..0000000 --- a/espfs/mkespfsimage/Makefile.windows +++ /dev/null @@ -1,33 +0,0 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes - -TARGET = mkespfsimage.exe - -CC = gcc -LD = $(CC) -CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 -LDFLAGS=-Lmman-win32 -lmman - -ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -LDFLAGS += -lz -endif - -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - -OBJECTS = main.o heatshrink_encoder.o - -all: $(TARGET) - -$(TARGET): $(OBJECTS) - $(LD) -o $@ $^ $(LDFLAGS) - -%.o: %.c - $(CC) $(CFLAGS) -o $@ $^ - -clean: - rm -f $(OBJECTS) $(TARGET) - -.PHONY: all clean diff --git a/espfs/mkespfsimage/build-linux.sh b/espfs/mkespfsimage/build-linux.sh deleted file mode 100644 index 69e3166..0000000 --- a/espfs/mkespfsimage/build-linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -make -f Makefile.linux clean -make -f Makefile.linux USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/build-win32.sh b/espfs/mkespfsimage/build-win32.sh deleted file mode 100644 index f552ea9..0000000 --- a/espfs/mkespfsimage/build-win32.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -cd mman-win32 -./configure && make -cd .. -make -f Makefile.windows clean -make -f Makefile.windows USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/espfsformat.h b/espfs/mkespfsimage/espfsformat.h deleted file mode 100644 index 8ce5549..0000000 --- a/espfs/mkespfsimage/espfsformat.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef ESPROFSFORMAT_H -#define ESPROFSFORMAT_H - -/* -Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module. -Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files, -headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself -when trying to do a <4byte or unaligned read. -*/ - -/* -The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data. -Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header -with the FLAG_LASTFILE flag set. -*/ - - -#define FLAG_LASTFILE (1<<0) -#define FLAG_GZIP (1<<1) -#define COMPRESS_NONE 0 -#define COMPRESS_HEATSHRINK 1 -#define ESPFS_MAGIC 0x73665345 - -typedef struct { - int32_t magic; - int8_t flags; - int8_t compression; - int16_t nameLen; - int32_t fileLenComp; - int32_t fileLenDecomp; -} __attribute__((packed)) EspFsHeader; - -#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/heatshrink_encoder.c b/espfs/mkespfsimage/heatshrink_encoder.c deleted file mode 100644 index fd82c9a..0000000 --- a/espfs/mkespfsimage/heatshrink_encoder.c +++ /dev/null @@ -1,4 +0,0 @@ -//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir -#ifdef ESPFS_HEATSHRINK -#include "../heatshrink/heatshrink_encoder.c" -#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/main.c b/espfs/mkespfsimage/main.c index 20a2b00..be3aeb7 100644 --- a/espfs/mkespfsimage/main.c +++ b/espfs/mkespfsimage/main.c @@ -8,7 +8,7 @@ #include #include "espfs.h" #ifdef __MINGW32__ -#include +#include "mman-win32/mman.h" #else #include #endif @@ -17,16 +17,8 @@ #else #include #endif -#include #include "espfsformat.h" -//Heatshrink -#ifdef ESPFS_HEATSHRINK -#include "../heatshrink/heatshrink_common.h" -#include "../heatshrink/heatshrink_config.h" -#include "../heatshrink/heatshrink_encoder.h" -#endif - //Gzip #ifdef ESPFS_GZIP // If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" @@ -35,6 +27,7 @@ #endif + //Routines to convert host format to the endianness used in the xtensa short htoxs(short in) { char r[2]; @@ -52,53 +45,6 @@ int htoxl(int in) { return *((int *)r); } -#ifdef ESPFS_HEATSHRINK -size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) { - char *inp=in; - char *outp=out; - size_t len; - int ws[]={5, 6, 8, 11, 13}; - int ls[]={3, 3, 4, 4, 4}; - HSE_poll_res pres; - HSE_sink_res sres; - size_t r; - if (level==-1) level=8; - level=(level-1)/2; //level is now 0, 1, 2, 3, 4 - heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]); - if (enc==NULL) { - perror("allocating mem for heatshrink"); - exit(1); - } - //Save encoder parms as first byte - *outp=(ws[level]<<4)|ls[level]; - outp++; outsize--; - - r=1; - do { - if (insize>0) { - sres=heatshrink_encoder_sink(enc, inp, insize, &len); - if (sres!=HSER_SINK_OK) break; - inp+=len; insize-=len; - if (insize==0) heatshrink_encoder_finish(enc); - } - do { - pres=heatshrink_encoder_poll(enc, outp, outsize, &len); - if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break; - outp+=len; outsize-=len; - r+=len; - } while (pres==HSER_POLL_MORE); - } while (insize!=0); - - if (insize!=0) { - fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres); - exit(1); - } - - heatshrink_encoder_free(enc); - return r; -} -#endif - #ifdef ESPFS_GZIP size_t compressGzip(char *in, int insize, char *out, int outsize, int level) { z_stream stream; @@ -184,7 +130,7 @@ int parseGzipExtensions(char *input) { } #endif -int handleFile(int f, char *name, int compression, int level, char **compName) { +int handleFile(int f, char *name, int compression, int level, char **compName, off_t *csizePtr) { char *fdat, *cdat; off_t size, csize; EspFsHeader h; @@ -211,11 +157,6 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { if (compression==COMPRESS_NONE) { csize=size; cdat=fdat; -#ifdef ESPFS_HEATSHRINK - } else if (compression==COMPRESS_HEATSHRINK) { - cdat=malloc(size*2); - csize=compressHeatshrink(fdat, size, cdat, size*2, level); -#endif } else { fprintf(stderr, "Unknown compression - %d\n", compression); exit(1); @@ -238,7 +179,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { h.nameLen=htoxs(h.nameLen); h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); - + write(1, &h, sizeof(EspFsHeader)); write(1, name, nameLen); while (nameLen&3) { @@ -254,9 +195,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { munmap(fdat, size); if (compName != NULL) { - if (h.compression==COMPRESS_HEATSHRINK) { - *compName = "heatshrink"; - } else if (h.compression==COMPRESS_NONE) { + if (h.compression==COMPRESS_NONE) { if (h.flags & FLAG_GZIP) { *compName = "gzip"; } else { @@ -266,6 +205,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { *compName = "unknown"; } } + *csizePtr = csize; return (csize*100)/size; } @@ -292,11 +232,7 @@ int main(int argc, char **argv) { int compType; //default compression type - heatshrink int compLvl=-1; -#ifdef ESPFS_HEATSHRINK - compType = COMPRESS_HEATSHRINK; -#else compType = COMPRESS_NONE; -#endif for (x=1; x=x-2) { @@ -318,7 +254,7 @@ int main(int argc, char **argv) { #ifdef ESPFS_GZIP if (gzipExtensions == NULL) { - parseGzipExtensions(strdup("html,css,js")); + parseGzipExtensions(strdup("html,css,js,ico")); } #endif @@ -330,11 +266,7 @@ int main(int argc, char **argv) { #endif fprintf(stderr, "> out.espfs\n"); fprintf(stderr, "Compressors:\n"); -#ifdef ESPFS_HEATSHRINK - fprintf(stderr, "0 - None\n1 - Heatshrink(default)\n"); -#else fprintf(stderr, "0 - None(default)\n"); -#endif fprintf(stderr, "\nCompression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n"); #ifdef ESPFS_GZIP fprintf(stderr, "\nGzipped extensions: list of comma separated, case sensitive file extensions \nthat will be gzipped. Defaults to 'html,css,js'\n"); @@ -359,8 +291,9 @@ int main(int argc, char **argv) { f=open(fileName, O_RDONLY); if (f>0) { char *compName = "unknown"; - rate=handleFile(f, realName, compType, compLvl, &compName); - fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); + off_t csize; + rate=handleFile(f, realName, compType, compLvl, &compName, &csize); + fprintf(stderr, "%-16s (%3d%%, %s, %4u bytes)\n", realName, rate, compName, (uint32_t)csize); close(f); } else { perror(fileName); diff --git a/include/esp8266.h b/include/esp8266.h index ce92455..96a7364 100644 --- a/include/esp8266.h +++ b/include/esp8266.h @@ -13,7 +13,10 @@ #include #include #include -#include #include "espmissingincludes.h" #include "uart_hw.h" + +#ifdef __WIN32__ +#include <_mingw.h> +#endif diff --git a/include/espmissingincludes.h b/include/espmissingincludes.h index c69b9fc..e5a20a0 100644 --- a/include/espmissingincludes.h +++ b/include/espmissingincludes.h @@ -1,9 +1,8 @@ #ifndef ESPMISSINGINCLUDES_H #define ESPMISSINGINCLUDES_H -#include -#include -#include +#include +#include //Missing function prototypes in include folders. Gcc will warn on these if we don't define 'em anywhere. //MOST OF THESE ARE GUESSED! but they seem to work and shut up the compiler. @@ -54,6 +53,11 @@ int rand(void); void ets_bzero(void *s, size_t n); void ets_delay_us(int ms); +// Shortcuts for memory functions +#define os_malloc pvPortMalloc +#define os_free vPortFree +#define os_zalloc pvPortZalloc + // disappeared in SDK 1.1.0: #define os_timer_done ets_timer_done #define os_timer_handler_isr ets_timer_handler_isr From 80f3788564165858042fd7c2fb70677fb83123f6 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 06:08:09 -0500 Subject: [PATCH 16/23] Added Makefile comment --- Makefile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index af5fcbf..9047faa 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.3.0) # Esptool.py path and port, only used for 1-time serial flashing # Typically you'll use https://github.com/themadinventor/esptool +# Windows users use the com port i.e: ESPPORT ?= com3 ESPTOOL ?= $(abspath ../esp-open-sdk/esptool/esptool.py) ESPPORT ?= /dev/ttyUSB0 ESPBAUD ?= 460800 @@ -132,10 +133,10 @@ YUI-COMPRESSOR ?= yuicompressor-2.4.8.jar # Output directors to store intermediate compiled files # relative to the project directory BUILD_BASE = build -FW_BASE = firmware +FW_BASE = firmware # name for the target project -TARGET = httpd +TARGET = httpd # espressif tool to concatenate sections for OTA upload using bootloader v1.2+ APPGEN_TOOL ?= gen_appbin.py @@ -145,10 +146,10 @@ MODULES = espfs httpd user serial EXTRA_INCDIR = include . # libraries used in this project, mainly provided by the SDK -LIBS = c gcc hal phy pp net80211 wpa main lwip +LIBS = c gcc hal phy pp net80211 wpa main lwip # compiler flags using during compilation of source files -CFLAGS = -Os -ggdb -std=c99 -Werror -Wpointer-arith -Wundef -Wall -Wl,-EL -fno-inline-functions \ +CFLAGS = -Os -ggdb -std=c99 -Werror -Wpointer-arith -Wundef -Wall -Wl,-EL -fno-inline-functions \ -nostdlib -mlongcalls -mtext-section-literals -ffunction-sections -fdata-sections \ -D__ets__ -DICACHE_FLASH -D_STDINT_H -Wno-address -DFIRMWARE_SIZE=$(ESP_FLASH_MAX) \ -DMCU_RESET_PIN=$(MCU_RESET_PIN) -DMCU_ISP_PIN=$(MCU_ISP_PIN) \ @@ -208,11 +209,11 @@ vecho := @echo endif ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DGZIP_COMPRESSION +CFLAGS += -DGZIP_COMPRESSION endif ifeq ("$(CHANGE_TO_STA)","yes") -CFLAGS += -DCHANGE_TO_STA +CFLAGS += -DCHANGE_TO_STA endif vpath %.c $(SRC_DIR) @@ -289,9 +290,9 @@ flash: all $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): - $(Q) mkdir -p yui + $(Q) mkdir -p yui ifeq ($(OS),Windows_NT) - cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) else cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) endif From 706ae4844e80f269910054711cdd20471df6e839 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 06:42:57 -0500 Subject: [PATCH 17/23] Weird line endings or something throwing make error, fixed now. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9047faa..7a74e19 100644 --- a/Makefile +++ b/Makefile @@ -290,9 +290,9 @@ flash: all $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): - $(Q) mkdir -p yui + $(Q) mkdir -p yui ifeq ($(OS),Windows_NT) - cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) else cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) endif From 6cbad0d16000e5b8bf70c5baacfcf8e431a5566d Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Mon, 24 Aug 2015 13:47:36 -0500 Subject: [PATCH 18/23] Fixed merge misses --- serial/console.c | 51 ---------------------------------------------- serial/serbridge.c | 26 ----------------------- 2 files changed, 77 deletions(-) diff --git a/serial/console.c b/serial/console.c index 00a59d3..d0d0dd7 100644 --- a/serial/console.c +++ b/serial/console.c @@ -45,13 +45,8 @@ console_prev(void) { void ICACHE_FLASH_ATTR console_write_char(char c) { -<<<<<<< HEAD - //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good - console_write(c); -======= //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good console_write(c); ->>>>>>> master } int ICACHE_FLASH_ATTR @@ -88,51 +83,6 @@ ajaxConsoleBaud(HttpdConnData *connData) { int ICACHE_FLASH_ATTR ajaxConsole(HttpdConnData *connData) { -<<<<<<< HEAD - if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. - char buff[2048]; - int len; // length of text in buff - int console_len = (console_wr+BUF_MAX-console_rd) % BUF_MAX; // num chars in console_buf - int start = 0; // offset onto console_wr to start sending out chars - - jsonHeader(connData, 200); - - // figure out where to start in buffer based on URI param - len = httpdFindArg(connData->getArgs, "start", buff, sizeof(buff)); - if (len > 0) { - start = atoi(buff); - if (start < console_pos) { - start = 0; - } else if (start >= console_pos+console_len) { - start = console_len; - } else { - start = start - console_pos; - } - } - - // start outputting - len = os_sprintf(buff, "{\"len\":%d, \"start\":%d, \"text\": \"", - console_len-start, console_pos+start); - - int rd = (console_rd+start) % BUF_MAX; - while (len < 2040 && rd != console_wr) { - uint8_t c = console_buf[rd]; - if (c == '\\' || c == '"') { - buff[len++] = '\\'; - buff[len++] = c; - } else if (c == '\r') { - // this is crummy, but browsers display a newline for \r\n sequences - } else if (c < ' ') { - len += os_sprintf(buff+len, "\\u%04x", c); - } else { - buff[len++] = c; - } - rd = (rd + 1) % BUF_MAX; - } - os_strcpy(buff+len, "\"}"); len+=2; - httpdSend(connData, buff, len); - return HTTPD_CGI_DONE; -======= if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. char buff[2048]; int len; // length of text in buff @@ -176,7 +126,6 @@ ajaxConsole(HttpdConnData *connData) { os_strcpy(buff+len, "\"}"); len+=2; httpdSend(connData, buff, len); return HTTPD_CGI_DONE; ->>>>>>> master } void ICACHE_FLASH_ATTR consoleInit() { diff --git a/serial/serbridge.c b/serial/serbridge.c index f583ffe..415772b 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -334,31 +334,6 @@ static void ICACHE_FLASH_ATTR serbridgeConnectCb(void *arg) { //===== Initialization void ICACHE_FLASH_ATTR serbridgeInitPins() { -<<<<<<< HEAD - mcu_reset_pin = flashConfig.reset_pin; - mcu_isp_pin = flashConfig.isp_pin; - os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", - mcu_reset_pin, mcu_isp_pin, flashConfig.swap_uart); - - if (flashConfig.swap_uart) { - PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); - PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); - PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTCK_U); - PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTDO_U); - system_uart_swap(); - } else { - PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); - PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, 0); - system_uart_de_swap(); - } - - // set both pins to 1 before turning them on so we don't cause a reset - if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); - if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 1); - // switch pin mux to make these pins GPIO pins - if (mcu_reset_pin >= 0) makeGpio(mcu_reset_pin); - if (mcu_isp_pin >= 0) makeGpio(mcu_isp_pin); -======= mcu_reset_pin = flashConfig.reset_pin; mcu_isp_pin = flashConfig.isp_pin; os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", @@ -382,7 +357,6 @@ void ICACHE_FLASH_ATTR serbridgeInitPins() { // switch pin mux to make these pins GPIO pins if (mcu_reset_pin >= 0) makeGpio(mcu_reset_pin); if (mcu_isp_pin >= 0) makeGpio(mcu_isp_pin); ->>>>>>> master } // Start transparent serial bridge TCP server on specified port (typ. 23) From f3d18243aeae0ba38fc6f25fde51a3d5225346e9 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Mon, 24 Aug 2015 14:02:06 -0500 Subject: [PATCH 19/23] Reverting some things that didn't merge right --- Makefile | 12 ++++++------ esp-link.vcxproj | 12 ++++++++++++ user/config.c | 2 ++ user/config.h | 2 ++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index a60880d..61cd321 100644 --- a/Makefile +++ b/Makefile @@ -31,14 +31,14 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- # Pick your flash size: "512KB", "1MB", or "4MB" -FLASH_SIZE ?= 512KB +FLASH_SIZE ?= 4MB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 ESP_SPI_SIZE ?= 0 # 0->512KB (256KB+256KB) -ESP_FLASH_MODE ?= 0 # 0->QIO -ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz -ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +ESP_FLASH_MODE ?= 0 # 0->QIO +ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz +ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB ET_FS ?= 4m # 4Mbit flash size in esptool flash command ET_FF ?= 40m # 40Mhz flash speed in esptool flash command ET_BLANK ?= 0x7E000 # where to flash blank.bin to erase wireless settings @@ -330,11 +330,11 @@ ifeq ("$(FLASH_SIZE)","512KB") build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ else build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app1.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ diff --git a/esp-link.vcxproj b/esp-link.vcxproj index d462b62..558f09a 100644 --- a/esp-link.vcxproj +++ b/esp-link.vcxproj @@ -63,6 +63,10 @@ + + + + @@ -73,12 +77,15 @@ + + + @@ -86,6 +93,9 @@ + + + @@ -98,12 +108,14 @@ + + diff --git a/user/config.c b/user/config.c index b8122be..b44b329 100644 --- a/user/config.c +++ b/user/config.c @@ -18,6 +18,8 @@ FlashConfig flashDefault = { 0, 0x00ffffff, 0, // static ip, netmask, gateway 0, // log mode 0, // swap uart (don't by default) + 1, 0, // tcp_enable, rssi_enable + "\0", // api_key }; typedef union { diff --git a/user/config.h b/user/config.h index d92e7f2..18284a5 100644 --- a/user/config.h +++ b/user/config.h @@ -14,6 +14,8 @@ typedef struct { uint32_t staticip, netmask, gateway; // using DHCP if staticip==0 uint8_t log_mode; // UART log debug mode uint8_t swap_uart; // swap uart0 to gpio 13&15 + uint8_t tcp_enable, rssi_enable; // TCP client settings + char api_key[48]; // RSSI submission API key (Grovestreams for now) } FlashConfig; extern FlashConfig flashConfig; From c68a281580972a31c3c65058b3349c183aacf5c3 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Mon, 24 Aug 2015 14:30:24 -0500 Subject: [PATCH 20/23] Fixed VS includes and replaced missing cmd module --- Makefile | 2 +- esp-link.vcxproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 61cd321..08f0dcd 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ TARGET = httpd APPGEN_TOOL ?= gen_appbin.py # which modules (subdirectories) of the project to include in compiling -MODULES = espfs httpd user serial +MODULES = espfs httpd user serial cmd EXTRA_INCDIR = include . # libraries used in this project, mainly provided by the SDK diff --git a/esp-link.vcxproj b/esp-link.vcxproj index 558f09a..1fc9e5c 100644 --- a/esp-link.vcxproj +++ b/esp-link.vcxproj @@ -28,7 +28,7 @@ __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ - .\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include + .\cmd;.\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;C:\tools\mingw64\include From c586f38765e04fd5caa2e9936a0c7dbfd7ee8c2f Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Mon, 24 Aug 2015 15:48:12 -0500 Subject: [PATCH 21/23] swapped out for offsite web files --- Makefile | 6 +- cmd/cmd.h | 1 + esp-link.vcxproj | 2 +- html-old/console.html | 46 +++++++++++++++ {html => html-old}/console.js | 0 {html => html-old}/favicon.ico | Bin html-old/head- | 10 ++++ html-old/home.html | 87 ++++++++++++++++++++++++++++ {html => html-old}/jl-400x110.png- | Bin html-old/log.html | 48 ++++++++++++++++ {html => html-old}/pure.css | 0 {html => html-old}/style.css | 0 {html => html-old}/ui.js | 0 {html => html-old}/wifi/icons.png | Bin html-old/wifi/wifi.html | 84 +++++++++++++++++++++++++++ {html => html-old}/wifi/wifi.js | 0 html/console.html | 47 +-------------- html/head- | 10 ---- html/home.html | 88 +---------------------------- html/log.html | 49 +--------------- html/wifi/wifi.html | 85 +--------------------------- user/user_main.c | 59 ++++++++++--------- 22 files changed, 317 insertions(+), 305 deletions(-) create mode 100644 html-old/console.html rename {html => html-old}/console.js (100%) rename {html => html-old}/favicon.ico (100%) mode change 100755 => 100644 create mode 100644 html-old/head- create mode 100644 html-old/home.html rename {html => html-old}/jl-400x110.png- (100%) mode change 100755 => 100644 create mode 100644 html-old/log.html rename {html => html-old}/pure.css (100%) rename {html => html-old}/style.css (100%) rename {html => html-old}/ui.js (100%) rename {html => html-old}/wifi/icons.png (100%) create mode 100644 html-old/wifi/wifi.html rename {html => html-old}/wifi/wifi.js (100%) diff --git a/Makefile b/Makefile index 08f0dcd..a3df665 100644 --- a/Makefile +++ b/Makefile @@ -25,13 +25,13 @@ SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.3.0) # Typically you'll use https://github.com/themadinventor/esptool # Windows users use the com port i.e: ESPPORT ?= com3 ESPTOOL ?= $(abspath ../esp-open-sdk/esptool/esptool.py) -ESPPORT ?= /dev/ttyUSB0 -ESPBAUD ?= 460800 +ESPPORT ?= com10 +ESPBAUD ?= 230400 # --------------- chipset configuration --------------- # Pick your flash size: "512KB", "1MB", or "4MB" -FLASH_SIZE ?= 4MB +FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 diff --git a/cmd/cmd.h b/cmd/cmd.h index 5a66bc7..23bd584 100644 --- a/cmd/cmd.h +++ b/cmd/cmd.h @@ -4,6 +4,7 @@ #ifndef CMD_H #define CMD_H +#include // Escape chars used by tuanpmt, dunno why he didn't use std ones... #define SLIP_START 0x7E diff --git a/esp-link.vcxproj b/esp-link.vcxproj index 1fc9e5c..2a774e9 100644 --- a/esp-link.vcxproj +++ b/esp-link.vcxproj @@ -28,7 +28,7 @@ __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ - .\cmd;.\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;C:\tools\mingw64\include + .\cmd;.\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;..\xtensa-lx106-elf\xtensa-lx106-elf\include;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include diff --git a/html-old/console.html b/html-old/console.html new file mode 100644 index 0000000..db4c0a6 --- /dev/null +++ b/html-old/console.html @@ -0,0 +1,46 @@ +
+
+

Microcontroller Console

+
+ +
+

The Microcontroller console shows the last 1024 characters + received from UART0, to which a microcontroller is typically attached. + The UART is configured for 8 bits, no parity, 1 stop bit (8N1).

+

+ Reset µC +  Baud: + +

+ 

+    

+  

+

+
+
+
+
+
diff --git a/html/console.js b/html-old/console.js
similarity index 100%
rename from html/console.js
rename to html-old/console.js
diff --git a/html/favicon.ico b/html-old/favicon.ico
old mode 100755
new mode 100644
similarity index 100%
rename from html/favicon.ico
rename to html-old/favicon.ico
diff --git a/html-old/head- b/html-old/head-
new file mode 100644
index 0000000..514ff53
--- /dev/null
+++ b/html-old/head-
@@ -0,0 +1,10 @@
+
+
+  esp-link
+  
+  
+  
+  
+
+
+

diff --git a/html-old/home.html b/html-old/home.html
new file mode 100644
index 0000000..46a984d
--- /dev/null
+++ b/html-old/home.html
@@ -0,0 +1,87 @@
+  

+    

+			
JEELABS

+      
esp-link

+      

+    

+
+		

+			

+				

+					
The JeeLabs esp-link firmware bridges the ESP8266 serial port to Wifi and can
+					program microcontrollers over the serial port, in particular Arduinos, AVRs, and
+					NXP's LPC800 and other ARM processors.

+					
Program an Arduino/AVR using avrdude using a command
+					line similar to:

+					
/home/arduino-1.0.5/hardware/tools/avrdude \

+					  -DV -patmega328p -Pnet:esp-link.local:23 -carduino -b115200 -U \

+					  -C /home/arduino-1.0.5/hardware/tools/avrdude.conf flash:w:my_sketch.hex:i
+					

+					
where -Pnet:esp-link.local:23 tells avrdude to connect to port 23 of esp-link.
+					You can substitute the IP address of your esp-link for esp-link.local if necessary.

+					
Please refer to
+					the online README
+					for up-to-date help and to the forthcoming
+					JeeLabs blog for an intro to the codebase.

+				

+			

+			

+				

+					

+						
Wifi summary

+						
+						
+						
+						
+						
+						
+						
+						
+						
+					

+					

+						
TCP client

+						

+							TCP client support in esp-link
+							

+								
+								
+							

+							

+							Grovestreams data push
+							

+								
+								
+							

+							

+								
+								
+							

+							
+						

+					

+				

+				

+					
Pin assignment

+					Select one of the following signal/pin assignments to match your hardware
+					

+						
+					

+				

+			

+			

+			

+    

+  

+

+
+
+
diff --git a/html/jl-400x110.png- b/html-old/jl-400x110.png-
old mode 100755
new mode 100644
similarity index 100%
rename from html/jl-400x110.png-
rename to html-old/jl-400x110.png-
diff --git a/html-old/log.html b/html-old/log.html
new file mode 100644
index 0000000..46ec785
--- /dev/null
+++ b/html-old/log.html
@@ -0,0 +1,48 @@
+  

+    

+      
Debug Log

+    

+
+    

+      
The debug log shows the most recent characters printed by the esp-link software itself to
+      its own debug log.

+      

+        

+          Refresh
+        

+        

+          UART debug log:
+          auto
+          off
+          on
+        

+      

+      

+    

+  

+
+
+
+
+
+
diff --git a/html/pure.css b/html-old/pure.css
similarity index 100%
rename from html/pure.css
rename to html-old/pure.css
diff --git a/html/style.css b/html-old/style.css
similarity index 100%
rename from html/style.css
rename to html-old/style.css
diff --git a/html/ui.js b/html-old/ui.js
similarity index 100%
rename from html/ui.js
rename to html-old/ui.js
diff --git a/html/wifi/icons.png b/html-old/wifi/icons.png
similarity index 100%
rename from html/wifi/icons.png
rename to html-old/wifi/icons.png
diff --git a/html-old/wifi/wifi.html b/html-old/wifi/wifi.html
new file mode 100644
index 0000000..b6c142b
--- /dev/null
+++ b/html-old/wifi/wifi.html
@@ -0,0 +1,84 @@
+  

+    

+      
Wifi Configuration

+    

+
+    

+      

+        

+          
Wifi State

+          
+          
+          
+					
+          
+          
+          
+          
+          
+          
+          
+          
+        

+        

+          
Wifi Association

+          
+          

+            To connect to a WiFi network, please select one of the detected networks,
+               enter the password, and hit the connect button...
+            
+            
Scanning... 

+            
+            
+            
+          

+        

+      

+      

+        

+          
Special Settings

+          

+            Special settings, use with care!
+            

+              
+              
+            

+            

+              
+              
+            

+            

+              
+              
+              
+              
+              
+              
+            

+            
+          

+        

+      

+    

+  

+
+
+
+
+
+
diff --git a/html/wifi/wifi.js b/html-old/wifi/wifi.js
similarity index 100%
rename from html/wifi/wifi.js
rename to html-old/wifi/wifi.js
diff --git a/html/console.html b/html/console.html
index db4c0a6..549ae82 100644
--- a/html/console.html
+++ b/html/console.html
@@ -1,46 +1 @@
-  

-    

-      
Microcontroller Console

-    

-
-    

-      
The Microcontroller console shows the last 1024 characters
-         received from UART0, to which a microcontroller is typically attached.
-         The UART is configured for 8 bits, no parity, 1 stop bit (8N1).

-      

-        Reset µC
-         Baud:
-        
-      

-      

-    

-  

-
-
-
-
-
-
+
diff --git a/html/head- b/html/head-
index 514ff53..e69de29 100644
--- a/html/head-
+++ b/html/head-
@@ -1,10 +0,0 @@
-
-
-  esp-link
-  
-  
-  
-  
-
-
-

diff --git a/html/home.html b/html/home.html
index 46a984d..2677264 100644
--- a/html/home.html
+++ b/html/home.html
@@ -1,87 +1 @@
-  

-    

-			
JEELABS

-      
esp-link

-      

-    

-
-		

-			

-				

-					
The JeeLabs esp-link firmware bridges the ESP8266 serial port to Wifi and can
-					program microcontrollers over the serial port, in particular Arduinos, AVRs, and
-					NXP's LPC800 and other ARM processors.

-					
Program an Arduino/AVR using avrdude using a command
-					line similar to:

-					
/home/arduino-1.0.5/hardware/tools/avrdude \

-					  -DV -patmega328p -Pnet:esp-link.local:23 -carduino -b115200 -U \

-					  -C /home/arduino-1.0.5/hardware/tools/avrdude.conf flash:w:my_sketch.hex:i
-					

-					
where -Pnet:esp-link.local:23 tells avrdude to connect to port 23 of esp-link.
-					You can substitute the IP address of your esp-link for esp-link.local if necessary.

-					
Please refer to
-					the online README
-					for up-to-date help and to the forthcoming
-					JeeLabs blog for an intro to the codebase.

-				

-			

-			

-				

-					

-						
Wifi summary

-						
-						
-						
-						
-						
-						
-						
-						
-						
-					

-					

-						
TCP client

-						

-							TCP client support in esp-link
-							

-								
-								
-							

-							

-							Grovestreams data push
-							

-								
-								
-							

-							

-								
-								
-							

-							
-						

-					

-				

-				

-					
Pin assignment

-					Select one of the following signal/pin assignments to match your hardware
-					

-						
-					

-				

-			

-			

-			

-    

-  

-

-
-
-
+
diff --git a/html/log.html b/html/log.html
index 46ec785..7bec267 100644
--- a/html/log.html
+++ b/html/log.html
@@ -1,48 +1 @@
-  

-    

-      
Debug Log

-    

-
-    

-      
The debug log shows the most recent characters printed by the esp-link software itself to
-      its own debug log.

-      

-        

-          Refresh
-        

-        

-          UART debug log:
-          auto
-          off
-          on
-        

-      

-      

-    

-  

-
-
-
-
-
-
+
diff --git a/html/wifi/wifi.html b/html/wifi/wifi.html
index b6c142b..a6dd616 100644
--- a/html/wifi/wifi.html
+++ b/html/wifi/wifi.html
@@ -1,84 +1 @@
-  

-    

-      
Wifi Configuration

-    

-
-    

-      

-        

-          
Wifi State

-          
-          
-          
-					
-          
-          
-          
-          
-          
-          
-          
-          
-        

-        

-          
Wifi Association

-          
-          

-            To connect to a WiFi network, please select one of the detected networks,
-               enter the password, and hit the connect button...
-            
-            
Scanning... 

-            
-            
-            
-          

-        

-      

-      

-        

-          
Special Settings

-          

-            Special settings, use with care!
-            

-              
-              
-            

-            

-              
-              
-            

-            

-              
-              
-              
-              
-              
-              
-            

-            
-          

-        

-      

-    

-  

-
-
-
-
-
-
+
diff --git a/user/user_main.c b/user/user_main.c
index 54ca59e..cf6d266 100644
--- a/user/user_main.c
+++ b/user/user_main.c
@@ -48,7 +48,6 @@ int myPassFn(HttpdConnData *connData, int no, char *user, int userLen, char *pas
 	return 0;
 }
 
-
 /*
 This is the main url->function dispatching data struct.
 In short, it's a struct with various URLs plus their handlers. The handlers can
@@ -95,7 +94,6 @@ HttpdBuiltInUrl builtInUrls[]={
 	{NULL, NULL, NULL}
 };
 
-
 //#define SHOW_HEAP_USE
 
 #ifdef SHOW_HEAP_USE
@@ -106,7 +104,36 @@ static void ICACHE_FLASH_ATTR prHeapTimerCb(void *arg) {
 }
 #endif
 
+# define VERS_STR_STR(V) #V
+# define VERS_STR(V) VERS_STR_STR(V)
+char *esp_link_version = VERS_STR(VERSION);
+
 void user_rf_pre_init(void) {
+  // get the flash config so we know how to init things
+  //configWipe(); // uncomment to reset the config for testing purposes  
+  bool restoreOk = configRestore();
+
+  // init gpio pin registers
+  gpio_init();
+
+  // init UART
+  uart_init(flashConfig.baud_rate, 115200);
+  logInit(); // must come after init of uart
+
+  // say hello (leave some time to cause break in TX after boot loader's msg
+  os_delay_us(10000L);
+  os_printf("\n\n** %s\n", esp_link_version);
+  os_printf("Flash config restore %s\n", restoreOk ? "ok" : "*FAILED*");
+
+  // Status LEDs
+  statusInit();
+  serledInit();
+
+#ifdef SHOW_HEAP_USE
+  os_timer_disarm(&prHeapTimer);
+  os_timer_setfn(&prHeapTimer, prHeapTimerCb, NULL);
+  os_timer_arm(&prHeapTimer, 10000, 1);
+#endif
 }
 
 // address of espfs binary blob
@@ -116,43 +143,23 @@ static char *rst_codes[] = {
 	"normal", "wdt reset", "exception", "soft wdt", "restart", "deep sleep", "external",
 };
 
-# define VERS_STR_STR(V) #V
-# define VERS_STR(V) VERS_STR_STR(V)
-char *esp_link_version = VERS_STR(VERSION);
-
 //Main routine. Initialize stdout, the I/O, filesystem and the webserver and we're done.
 void user_init(void) {
-	// get the flash config so we know how to init things
-	//configWipe(); // uncomment to reset the config for testing purposes
-	bool restoreOk = configRestore();
-	// init gpio pin registers
-	gpio_init();
-	// init UART
-	uart_init(flashConfig.baud_rate, 115200);
-	logInit(); // must come after init of uart
-	// say hello (leave some time to cause break in TX after boot loader's msg
-	os_delay_us(10000L);
-	os_printf("\n\n** %s\n", esp_link_version);
-	os_printf("Flash config restore %s\n", restoreOk ? "ok" : "*FAILED*");
-	// Status LEDs
-	statusInit();
-	serledInit();
 	// Wifi
 	wifiInit();
+
 	// init the flash filesystem with the html stuff
 	espFsInit(&_binary_espfs_img_start);
+
 	//EspFsInitResult res = espFsInit(&_binary_espfs_img_start);
 	//os_printf("espFsInit %s\n", res?"ERR":"ok");
+
 	// mount the http handlers
 	httpdInit(builtInUrls, 80);
+
 	// init the wifi-serial transparent bridge (port 23)
 	serbridgeInit(23);
 	uart_add_recv_cb(&serbridgeUartCb);
-#ifdef SHOW_HEAP_USE
-	os_timer_disarm(&prHeapTimer);
-	os_timer_setfn(&prHeapTimer, prHeapTimerCb, NULL);
-	os_timer_arm(&prHeapTimer, 10000, 1);
-#endif
 
 	struct rst_info *rst_info = system_get_rst_info();
 	os_printf("Reset cause: %d=%s\n", rst_info->reason, rst_codes[rst_info->reason]);

From 38922383883cd06a6b2cfd408de476dffbf8095c Mon Sep 17 00:00:00 2001
From: Benjamin Runnels 
Date: Fri, 28 Aug 2015 15:00:59 -0500
Subject: [PATCH 22/23] dynamic arduino callback support

---
 cmd/cmd.h                                    |  24 +++--
 cmd/handlers.c                               | 102 +++++++++++++++++--
 {html-old => httpd/html-old}/console.html    |   0
 {html-old => httpd/html-old}/console.js      |   0
 {html-old => httpd/html-old}/favicon.ico     | Bin
 {html-old => httpd/html-old}/head-           |   0
 {html-old => httpd/html-old}/home.html       |   0
 {html-old => httpd/html-old}/jl-400x110.png- | Bin
 {html-old => httpd/html-old}/log.html        |   0
 {html-old => httpd/html-old}/pure.css        |   0
 {html-old => httpd/html-old}/style.css       |   0
 {html-old => httpd/html-old}/ui.js           |   0
 {html-old => httpd/html-old}/wifi/icons.png  | Bin
 {html-old => httpd/html-old}/wifi/wifi.html  |   0
 {html-old => httpd/html-old}/wifi/wifi.js    |   0
 15 files changed, 110 insertions(+), 16 deletions(-)
 rename {html-old => httpd/html-old}/console.html (100%)
 rename {html-old => httpd/html-old}/console.js (100%)
 rename {html-old => httpd/html-old}/favicon.ico (100%)
 rename {html-old => httpd/html-old}/head- (100%)
 rename {html-old => httpd/html-old}/home.html (100%)
 rename {html-old => httpd/html-old}/jl-400x110.png- (100%)
 rename {html-old => httpd/html-old}/log.html (100%)
 rename {html-old => httpd/html-old}/pure.css (100%)
 rename {html-old => httpd/html-old}/style.css (100%)
 rename {html-old => httpd/html-old}/ui.js (100%)
 rename {html-old => httpd/html-old}/wifi/icons.png (100%)
 rename {html-old => httpd/html-old}/wifi/wifi.html (100%)
 rename {html-old => httpd/html-old}/wifi/wifi.js (100%)

diff --git a/cmd/cmd.h b/cmd/cmd.h
index 23bd584..b489108 100644
--- a/cmd/cmd.h
+++ b/cmd/cmd.h
@@ -20,12 +20,12 @@
 #define SLIP_ESC_ESC  0335    // ESC ESC_ESC means ESC data byte
 #endif
 
-typedef struct __attribute((__packed__)) {
+typedef struct __attribute__((__packed__)) {
   uint16_t  len;      // length of data
   uint8_t   data[0];  // really data[len]
 } CmdArg;
 
-typedef struct __attribute((__packed__)) {
+typedef struct __attribute__((__packed__)) {
   uint16_t  cmd;      // command to perform, from CmdName enum
   uint32_t  callback; // callback pointer to embed in response
   uint32_t  _return;  // return value to embed in response (?)
@@ -54,7 +54,9 @@ typedef enum {
   CMD_REST_SETUP,     // (11)
   CMD_REST_REQUEST,
   CMD_REST_SETHEADER,
-  CMD_REST_EVENTS
+  CMD_REST_EVENTS,
+  CMD_ADD_SENSOR, // 15
+  CMD_SENSOR_EVENTS
 } CmdName;
 
 typedef uint32_t (*cmdfunc_t)(CmdPacket *cmd);
@@ -64,18 +66,24 @@ typedef struct {
   cmdfunc_t sc_function;
 } CmdList;
 
-void CMD_parse_packet(uint8_t *buf, short len);
+typedef struct {
+  char name[16];
+  uint32_t callback;
+} cmdCallback;
+
+void ICACHE_FLASH_ATTR CMD_parse_packet(uint8_t *buf, short len);
+cmdCallback* ICACHE_FLASH_ATTR CMD_GetCbByName(char* name);
 
 // Responses
 
 // Start a response, returns the partial CRC
-uint16_t CMD_ResponseStart(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc);
+uint16_t ICACHE_FLASH_ATTR CMD_ResponseStart(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc);
 // Adds data to a response, returns the partial CRC
-uint16_t CMD_ResponseBody(uint16_t crc_in, uint8_t* data, short len);
+uint16_t ICACHE_FLASH_ATTR CMD_ResponseBody(uint16_t crc_in, uint8_t* data, short len);
 // Ends a response
-void CMD_ResponseEnd(uint16_t crc);
+void ICACHE_FLASH_ATTR CMD_ResponseEnd(uint16_t crc);
 
-void CMD_Response(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc, CmdArg* args[]);
+//void ICACHE_FLASH_ATTR CMD_Response(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc, CmdArg* args[]);
 
 // Requests
 
diff --git a/cmd/handlers.c b/cmd/handlers.c
index 73370dd..8e6deb0 100644
--- a/cmd/handlers.c
+++ b/cmd/handlers.c
@@ -9,10 +9,14 @@
 #include "serbridge.h"
 #include "uart.h"
 #include "cgiwifi.h"
+#include 
 
 static uint32_t ICACHE_FLASH_ATTR CMD_Null(CmdPacket *cmd);
 static uint32_t ICACHE_FLASH_ATTR CMD_IsReady(CmdPacket *cmd);
 static uint32_t ICACHE_FLASH_ATTR CMD_WifiConnect(CmdPacket *cmd);
+static uint32_t ICACHE_FLASH_ATTR CMD_AddSensor(CmdPacket *cmd);
+
+static uint8_t lastWifiStatus = wifiIsDisconnected;
 
 // Command dispatch table for serial -> ESP commands
 const CmdList commands[] = {
@@ -33,34 +37,83 @@ const CmdList commands[] = {
   {CMD_REST_SETUP,      REST_Setup},
   {CMD_REST_REQUEST,    REST_Request},
   {CMD_REST_SETHEADER,  REST_SetHeader},
+
+  {CMD_ADD_SENSOR,      CMD_AddSensor },
+
   {CMD_NULL,            NULL}
 };
 
+// WifiCb plus 10 for sensors
+cmdCallback callbacks[12] = {
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 },
+  { { '\0' }, -1 }
+};
+
 // Command handler for IsReady (healthcheck) command
 static uint32_t ICACHE_FLASH_ATTR
 CMD_IsReady(CmdPacket *cmd) {
-  os_printf("CMD: Check ready\n");
+  os_printf("CMD_IsReady: Check ready\n");
   return 1;
 }
 
 // Command handler for Null command
 static uint32_t ICACHE_FLASH_ATTR
 CMD_Null(CmdPacket *cmd) {
-  os_printf("CMD: NULL/unsupported command\n");
+  os_printf("CMD_Null: NULL/unsupported command\n");
   return 1;
 }
 
-static uint8_t lastWifiStatus;
-static uint32_t wifiCallback;
+static void ICACHE_FLASH_ATTR 
+CMD_AddCb(char* name, uint32_t cb) {
+  char checkname[16];
+  os_strncpy(checkname, name, sizeof(checkname));
+  for (uint8_t i = 0; i < sizeof(commands); i++) {    
+    os_printf("CMD_AddCb: index %d name=%s cb=%p\n", i, callbacks[i].name, (void *)callbacks[i].callback);
+    // find existing callback or add to the end
+    if (os_strcmp(callbacks[i].name, checkname) == 0 || callbacks[i].name[0] == '\0') {
+      os_strncpy(callbacks[i].name, checkname, sizeof(checkname));
+      callbacks[i].callback = cb;
+      os_printf("CMD_AddCb: cb %s added at index %d\n", callbacks[i].name, i);
+      break;
+    }
+  }
+}
+
+cmdCallback* ICACHE_FLASH_ATTR
+CMD_GetCbByName(char* name) {
+  char checkname[16];
+  os_strncpy(checkname, name, sizeof(checkname));
+  for (uint8_t i = 0; i < sizeof(commands); i++) {
+    os_printf("CMD_GetCbByName: index %d name=%s cb=%p\n", i, callbacks[i].name, (void *)callbacks[i].callback);
+    // if callback doesn't exist or it's null
+    if (os_strcmp(callbacks[i].name, checkname) == 0) {
+      os_printf("CMD_GetCbByName: cb %s found at index %d\n", callbacks[i].name, i);
+      return &callbacks[i];
+    }
+  }
+  os_printf("CMD_GetCbByName: cb %s not found\n", name);
+  return 0;
+}
 
 // Callback from wifi subsystem to notify us of status changes
 static void ICACHE_FLASH_ATTR
 CMD_WifiCb(uint8_t wifiStatus) {
   if (wifiStatus != lastWifiStatus){
+    os_printf("CMD_WifiCb: wifiStatus=%d\n", wifiStatus);
     lastWifiStatus = wifiStatus;
-    if (wifiCallback) {
+    cmdCallback *wifiCb = CMD_GetCbByName("wifiCb");
+    if ((uint32_t)wifiCb->callback != -1) {
       uint8_t status = wifiStatus == wifiGotIP ? 5 : 1;
-      uint16_t crc = CMD_ResponseStart(CMD_WIFI_CONNECT, wifiCallback, 0, 1);
+      uint16_t crc = CMD_ResponseStart(CMD_WIFI_CONNECT, (uint32_t)wifiCb->callback, 0, 1);
       crc = CMD_ResponseBody(crc, (uint8_t*)&status, 1);
       CMD_ResponseEnd(crc);
     }
@@ -70,10 +123,43 @@ CMD_WifiCb(uint8_t wifiStatus) {
 // Command handler for Wifi connect command
 static uint32_t ICACHE_FLASH_ATTR
 CMD_WifiConnect(CmdPacket *cmd) {
+  CmdRequest req;
+  CMD_Request(&req, cmd);
+  os_printf("CMD_WifiConnect: setup argc=%ld\n", CMD_GetArgc(&req));
 	if(cmd->argc != 2 || cmd->callback == 0)
-		return 0xFFFFFFFF;
+		return 0;
 
   wifiStatusCb = CMD_WifiCb;    // register our callback with wifi subsystem
-	wifiCallback = cmd->callback; // save the MCU's callback
+  CMD_AddCb("wifiCb", (uint32_t)cmd->callback); // save the MCU's callback
+  lastWifiStatus = wifiIsDisconnected;
+  if (statusWifiState() != wifiIsDisconnected) {
+    CMD_WifiCb(statusWifiState());
+  }
+
+  return 1;
+}
+
+// Command handler for Wifi connect command
+static uint32_t ICACHE_FLASH_ATTR
+CMD_AddSensor(CmdPacket *cmd) {
+  CmdRequest req;
+  CMD_Request(&req, cmd);
+  os_printf("CMD_AddSensor: setup argc=%ld\n", CMD_GetArgc(&req));
+  if (cmd->argc != 1 || cmd->callback == 0)
+    return 0;
+
+  uint8_t* name;
+  uint16_t len;
+
+  // get the sensor name
+  len = CMD_ArgLen(&req);
+  os_printf("CMD_AddSensor: name len=%d\n", len);
+  if (len > 15) return 0; // max size of name is 15 characters
+  name = (uint8_t*)os_zalloc(len + 1);
+  if (CMD_PopArg(&req, name, len)) return 0;
+  name[len] = 0;
+  os_printf("CMD_AddSensor: name=%s\n", name);
+
+  CMD_AddCb((char*)name, (uint32_t)cmd->callback); // save the sensor callback 
   return 1;
 }
diff --git a/html-old/console.html b/httpd/html-old/console.html
similarity index 100%
rename from html-old/console.html
rename to httpd/html-old/console.html
diff --git a/html-old/console.js b/httpd/html-old/console.js
similarity index 100%
rename from html-old/console.js
rename to httpd/html-old/console.js
diff --git a/html-old/favicon.ico b/httpd/html-old/favicon.ico
similarity index 100%
rename from html-old/favicon.ico
rename to httpd/html-old/favicon.ico
diff --git a/html-old/head- b/httpd/html-old/head-
similarity index 100%
rename from html-old/head-
rename to httpd/html-old/head-
diff --git a/html-old/home.html b/httpd/html-old/home.html
similarity index 100%
rename from html-old/home.html
rename to httpd/html-old/home.html
diff --git a/html-old/jl-400x110.png- b/httpd/html-old/jl-400x110.png-
similarity index 100%
rename from html-old/jl-400x110.png-
rename to httpd/html-old/jl-400x110.png-
diff --git a/html-old/log.html b/httpd/html-old/log.html
similarity index 100%
rename from html-old/log.html
rename to httpd/html-old/log.html
diff --git a/html-old/pure.css b/httpd/html-old/pure.css
similarity index 100%
rename from html-old/pure.css
rename to httpd/html-old/pure.css
diff --git a/html-old/style.css b/httpd/html-old/style.css
similarity index 100%
rename from html-old/style.css
rename to httpd/html-old/style.css
diff --git a/html-old/ui.js b/httpd/html-old/ui.js
similarity index 100%
rename from html-old/ui.js
rename to httpd/html-old/ui.js
diff --git a/html-old/wifi/icons.png b/httpd/html-old/wifi/icons.png
similarity index 100%
rename from html-old/wifi/icons.png
rename to httpd/html-old/wifi/icons.png
diff --git a/html-old/wifi/wifi.html b/httpd/html-old/wifi/wifi.html
similarity index 100%
rename from html-old/wifi/wifi.html
rename to httpd/html-old/wifi/wifi.html
diff --git a/html-old/wifi/wifi.js b/httpd/html-old/wifi/wifi.js
similarity index 100%
rename from html-old/wifi/wifi.js
rename to httpd/html-old/wifi/wifi.js

From 8e1516aacb3752f86a0bde967cc52cb387f22f07 Mon Sep 17 00:00:00 2001
From: Benjamin Runnels 
Date: Fri, 28 Aug 2015 15:23:34 -0500
Subject: [PATCH 23/23] merge regressions

---
 cmd/handlers.c   | 4 +---
 esp-link.vcxproj | 2 +-
 user/status.c    | 1 -
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/cmd/handlers.c b/cmd/handlers.c
index 3033e5c..28382dd 100644
--- a/cmd/handlers.c
+++ b/cmd/handlers.c
@@ -131,9 +131,7 @@ CMD_WifiConnect(CmdPacket *cmd) {
   wifiStatusCb = CMD_WifiCb;    // register our callback with wifi subsystem
   CMD_AddCb("wifiCb", (uint32_t)cmd->callback); // save the MCU's callback
   lastWifiStatus = wifiIsDisconnected;
-  if (statusWifiState() != wifiIsDisconnected) {
-    CMD_WifiCb(statusWifiState());
-  }
+  CMD_WifiCb(wifiState);
 
   return 1;
 }
diff --git a/esp-link.vcxproj b/esp-link.vcxproj
index 2a774e9..f8db9b3 100644
--- a/esp-link.vcxproj
+++ b/esp-link.vcxproj
@@ -44,7 +44,7 @@
     espmake clean
   
   
-    espmake baseflash
+    espmake all wiflash
     espmake clean all
     espmake clean
   
diff --git a/user/status.c b/user/status.c
index 50afb85..a6080fe 100644
--- a/user/status.c
+++ b/user/status.c
@@ -23,7 +23,6 @@ static void ICACHE_FLASH_ATTR setLed(int on) {
 }
 
 static uint8_t ledState = 0;
-static uint8_t wifiState = 0;
 
 // Timer callback to update the LED
 static void ICACHE_FLASH_ATTR ledTimerCb(void *v) {