From 9ee1182d04570c23a620a0751fbcf6c48a3087f5 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 13 Jul 2015 10:25:29 -0700 Subject: [PATCH 01/24] reduce size of icons --- html/favicon.ico | Bin 622 -> 874 bytes html/home.html | 2 +- html/jl-200x55.png | Bin 2751 -> 0 bytes html/style.css | 7 + html/ui.js | 358 --------------------------------------------- 5 files changed, 8 insertions(+), 359 deletions(-) delete mode 100755 html/jl-200x55.png delete mode 100644 html/ui.js diff --git a/html/favicon.ico b/html/favicon.ico index ea4eca11c5ea71459cf3dcecea6e9c4c6bf90fa3..fe24a7b535b3d0acfd1db7f56d08b424ef11f880 100755 GIT binary patch literal 874 zcmV-w1C{(oNk%w1VL$*t0P6q%|NsC0{{UlaYpoa|=HA|SZf>NktgRs>r5G6Aq^!kV zV*l^&R#sN6EiJ7W7^MspA^8LW00031EC2ui06+jh000F4Fvv-(wIb^+uK!?I6(nbp zg{ZEh>Aoo|Tg+?U-OMrt?w>kU4OcK2kA|Kv__-AcfT(l|AS7?h%FVE#dc6w+Hf!U4 z&tI`Crc-hX==j;aVYBEOs64Kp64o*=U3+K*3||Cz87wD+az$v0D~*nE0)1bSA$V|= z01B83nn0Vsd5Sf0|x^NqKAZkbsj?+ot_Q50}oP3m1-Lo zCZ16V2)wn zS4jY|&=Txy^$gv*0*c4URS~k3zN^KqjE4Lefdvb$f5En_*)gyK;$ocs4mq#|tPmh@ z;H7t1;Nl2OmA7<)OSgW#s5a`g@h;QUxWfkj9i4D@04Ai#)J#D84ImF56h!oi1NkVB zKmrO}lM+Y3*+kNXAchzsCbVQ_*8jg_yR@t#1B&+Nf3(Q9PEVR)wlH3m^RQsz9+IH(L1rY!MJGW+g AmH+?% literal 622 zcmZ?wbhEHbRA5kGSj)ih|NsAg|Nf<=r>~MzDkv=69~QZC)vC>#H$Qs(_~y-W_-g;e`3n{tO-Zk;s@iI0ckI})UmriddGqGh z+)qmoN^8pc?NdZa4UcGWFSz5y!S41)Ps2yzJm~o!NdHTtYwwY~5 zE9EXQh_$6j|5P)1Q6iAa$eUg5=EKrlEF_mEB~Z+f9qv=#puj9H#I9Jv>)XM^){w@> zEW;qiE!dY{$>iEu)F8;rEX^q|xm`4!dx~?xQby73LLw5|m<5YB1Tl3N?_u62#>l!| za*~g;H_H*}?Sc{_LVLNdr^os;_;4MOVdi6$=M>&9E3qV<>+U-y9T^Fa4+lDb0$Sv;b6n2XlXCSAL=Z98!MNyiEub191P`SJ=B)_ z?TkjFP-MR4gbNK!TcZmqnHEQtPHEvxG|yRlVL|ha{mG}DgbQAZTO_49Tr}0rm4C!A zS&aW>a75!4|F%hcBu%%7$l2x{nZP(Dt+&BBa6`_+M^C#0wbsT;M_yVo_s#9FCI$v; E07DSuRsaA1 diff --git a/html/home.html b/html/home.html index 64d80e1..59e9ab2 100644 --- a/html/home.html +++ b/html/home.html @@ -1,6 +1,6 @@
-
+
JEELABS

esp-link

diff --git a/html/jl-200x55.png b/html/jl-200x55.png deleted file mode 100755 index 3bc4a42cdfe781b8e1b0a9e2dc48e6b6e8577a4c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2751 zcmV;w3PAOVP) zPly}W9ml^;?Ze8jU9tsd-LX> z+}Q5}Vl^{QZ+t06Na=I)FySFdAvfA(2F7Pm-M# zbv+N@1MDh7SULbsR}7<-luQyy91>BGqONaZU@4O1YQ->4C)}q*5{g&~Qq=W3h;A)J zX;lp4aLVV9NT@_okfN?H0(c`?Q36@27{=n5&nbDt39TSSUH@97ou+9x@xlv2eg5_1 zk71goNYh@$Fb)WHDv>0Bu!0nIJqzFtuT2=GU{SBvajjktYIdMv7`<30l1Ky)QV=IQ zb&GX;?(A6{IdUX1p#Y%SY@%AL@tPc#*)B=i1S$wmIdZuiUcY!TaiQESudHBeYm0SQ zDv2ajzQsGpQ;t%pgrC0nVq$_UT)c>HPflWXcGk21=+UG2ZZ3yrvq}5d{12agx_0yC zP3dk)QsJ?H<5cBXU0ucO%uJ$!^!t6Lo0iMvu)4a+`Y1Opl1SpfR1j9S$|#qSnV3ja zh-S0tvm7Lui3u3xGWD2fW=$dq3spgix=y`abME4S{QdRcFppJhHCz;oWmqyewHb*d zQCJq_0=c79Dn+&&t)*pj-g_S#Pdtf@C!VA#i)i1Oi3yZSCEB5J7cM1_06qnAhL3p2 z<#MrNj(`3B9pvp@I?fcu?xcY;`OgD_={Q2{~eMMbo z7BnT2u#grcOE#dnGGX}glTSU{ZtJmLE_F=XNI{Ca zK1UvOU%XN1efGI;U-lP2AHCn&+VaF7_qVq_Z42-IG4LET8H>7pDvX>G2@AIc@kP>c z$5%C8dDXMM-xcV$Zr{fA^z>-E^WOWu&mVm5*}!u!WMpRrB$5y?3x)2BH!5VneeB&o zjNY%WuX|z$GUk~2{`Z8>lgVVrT~opzmPj}-6-3jtSRK<+yZeb$F)wv2;$98W&YTe! zOd<)vUVt8dFalsUEr--%bCL*5*6E+Lg0f ztKr0n6H!k2vnU^ugu_(S0%iA(PSu#&icjq(wG3|990?)VuI1NrBPUH}uoDQcOqq(B)oiPi@aa47Dq`ql3=LXhfvO<=en0XMliS_@ zdbXdr{3afE`dTlSJ+%*VoR&Kc+-x>6Jv|+JsEZ^u$UOeg#c9BMN56vvm$)?*brppB ztl1W81Zh0`QN5lDhoRm3=6$uesGhH~%wyv6r$~t@jzy;UwHO;ns@!l|t)esbh z3$GeZPOB9uB*v`f%hgWOgJnI2G>GlAGFm+ZB$%9=hsd>X!AbLa;;Ny7{E79cukPK$ zm!EvN|p_Gq`;NvyNQT~tdF4O3BlvrS2h7$Tz-mni^uOhx5RanNk* zu%jp<8Nq^+1OTw>VpTyz{Oxudxm*st&p!91V$b~N-@aaHwGVeS9N8& z!yWh!sTf98xPyd9!PTKw+#fK$%#ffvbPFM35#D#{q?g@y`n;*Ai)4n1(&rimvB$8V zIZ)EZaHXgp0|44Gp612|0I?CZfIt3lTLf}i?cX?)sG=~GRgx~w@R~TEFC-y{W3kqV z-Jys5Xtq_=Z0i6+87}~w!HS&K4DpvA9UHk?Uwrrxbz&25TJ0Afe&lKEE+j^QCo2t8 zQE!DX`Pais^w1J_VGLH)JqiVpMy8?;hmZ)hchLj9S*!&R=5uR%8?!Sr*grGlnap&5 zdz)JUcW16>(`rdtOUu*&i!9tO9?iW1$JcZfnQobi+5vD*v#o}ysHZS^I+lkZvrY6} z5N#`%$(jn$XGbJcB-$S-AI*A*xRjKPRzc){=g64_-7V@+`k2FRLSfU%Sb5UB&bTX| z;yUiwu7XqyV~t!7Yk7GYM!AfaZd?zX{_4g(Fp)Q{HVJ42!ir&#<`)3fSI%Phi=)vA zDNdL>q=-L09Dypr5`aa4cB1`}@{!iVH}aK12i~dNo0Pt3gta!F3b=FO(0KI~ z;TzJ$^2H)U3vlY6!D}Jxo1F9o@IljRiJ#jJmX45E9QZ5;|N)I*4=sJ;u$8KMyg z?K`B^QDSm{Ecj`*)pLIDM##Q8Zkb61@jdG7x0fyf*uD6lf4l6fR~&rqS-f=Px=)Ej zrq#w_C>sIyqISNtLttBn4uC^p=QBLy%n{q7)?-MA*iK?fbQOc&X%DX&uG!Xc=Qlr1 zk;6?314Z*(KfXIEZJpbfPoBid2xGJK`FWH|rNkGy2AB`pkx!tKBnmBy<-5SH)$8c* z>_j?28p_e%*B`xh2OrzH#^r`0A2LzaeoESn^0m_aM=@dqLV z%_EPBK2I+)Rt#gT1NJ132D}I8P~jzSkA+^V7)H3@NFoV==wqOnN&*a?`jqrz z+4?&>L5TsW9Fo)#4s4yp<(LJYOeTZUrAx7wEbVr?SYBB{zu)IIJNF=lvr8U5LS=r6 zx;_QqZcw{?J`e588DVAVcDpb)HqdGbO#(b1V-Crlh;1mX{th3^k76vP-T&);}T&71Nkx0Ht{14hL^?zhM5i|e*002ovPDHLk FV1fk~Lhb+n diff --git a/html/style.css b/html/style.css index 4fa393a..29d15a3 100644 --- a/html/style.css +++ b/html/style.css @@ -114,6 +114,13 @@ div.tt { .header h1 .esp { font-size: 1.25em; } +.jl { + font: normal 800 1.5em sans-serif; + position: relative; + bottom: 19px; + color: #9d1414; + margin-left: 3px; +} .content-subhead { margin: 50px 0 20px 0; diff --git a/html/ui.js b/html/ui.js deleted file mode 100644 index b67f077..0000000 --- a/html/ui.js +++ /dev/null @@ -1,358 +0,0 @@ -//===== Collection of small utilities - -/* - * Bind/Unbind events - * - * Usage: - * var el = document.getElementyById('#container'); - * bnd(el, 'click', function() { - * console.log('clicked'); - * }); - */ - -var bnd = function( - d, // a DOM element - e, // an event name such as "click" - f // a handler function -){ - d.addEventListener(e, f, false); -} - -/* - * Create DOM element - * - * Usage: - * var el = m('

Hello

'); - * document.body.appendChild(el); - * - * Copyright (C) 2011 Jed Schmidt - WTFPL - * More: https://gist.github.com/966233 - */ - -var m = function( - a, // an HTML string - b, // placeholder - c // placeholder -){ - b = document; // get the document, - c = b.createElement("p"); // create a container element, - c.innerHTML = a; // write the HTML to it, and - a = b.createDocumentFragment(); // create a fragment. - - while ( // while - b = c.firstChild // the container element has a first child - ) a.appendChild(b); // append the child to the fragment, - - return a // and then return the fragment. -} - -/* - * DOM selector - * - * Usage: - * $('div'); - * $('#name'); - * $('.name'); - * - * Copyright (C) 2011 Jed Schmidt - WTFPL - * More: https://gist.github.com/991057 - */ - -var $ = function( - a, // take a simple selector like "name", "#name", or ".name", and - b // an optional context, and -){ - a = a.match(/^(\W)?(.*)/); // split the selector into name and symbol. - return( // return an element or list, from within the scope of - b // the passed context - || document // or document, - )[ - "getElement" + ( // obtained by the appropriate method calculated by - a[1] - ? a[1] == "#" - ? "ById" // the node by ID, - : "sByClassName" // the nodes by class name, or - : "sByTagName" // the nodes by tag name, - ) - ]( - a[2] // called with the name. - ) -} - -/* - * Get cross browser xhr object - * - * Copyright (C) 2011 Jed Schmidt - * More: https://gist.github.com/993585 - */ - -var j = function( - a // cursor placeholder -){ - for( // for all a - a=0; // from 0 - a<4; // to 4, - a++ // incrementing - ) try { // try - return a // returning - ? new ActiveXObject( // a new ActiveXObject - [ // reflecting - , // (elided) - "Msxml2", // the various - "Msxml3", // working - "Microsoft" // options - ][a] + // for Microsoft implementations, and - ".XMLHTTP" // the appropriate suffix, - ) // but make sure to - : new XMLHttpRequest // try the w3c standard first, and - } - - catch(e){} // ignore when it fails. -} - -// createElement short-hand - -e = function(a) { return document.createElement(a); } - -// chain onload handlers - -function onLoad(f) { - var old = window.onload; - if (typeof old != 'function') { - window.onload = f; - } else { - window.onload = function() { - old(); - f(); - } - } -} - -//===== helpers to add/remove/toggle HTML element classes - -function addClass(el, cl) { - el.className += ' ' + cl; -} -function removeClass(el, cl) { - var cls = el.className.split(/\s+/), - l = cls.length; - for (var i=0; i= 200 && xhr.status < 300) { - console.log("XHR done:", method, url, "->", xhr.status); - ok_cb(xhr.responseText); - } else { - console.log("XHR ERR :", method, url, "->", xhr.status, xhr.responseText, xhr); - err_cb(xhr.status, xhr.responseText); - } - } - console.log("XHR send:", method, url); - try { - xhr.send(); - } catch(err) { - console.log("XHR EXC :", method, url, "->", err); - err_cb(599, err); - } -} - -function dispatchJson(resp, ok_cb, err_cb) { - var j; - try { j = JSON.parse(resp); } - catch(err) { - console.log("JSON parse error: " + err + ". In: " + resp); - err_cb(500, "JSON parse error: " + err); - return; - } - ok_cb(j); -} - -function ajaxJson(method, url, ok_cb, err_cb) { - ajaxReq(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); -} - -function ajaxSpin(method, url, ok_cb, err_cb) { - $("#spinner").removeAttribute('hidden'); - ajaxReq(method, url, function(resp) { - $("#spinner").setAttribute('hidden', ''); - ok_cb(resp); - }, function(status, statusText) { - $("#spinner").setAttribute('hidden', ''); - //showWarning("Error: " + statusText); - err_cb(status, statusText); - }); -} - -function ajaxJsonSpin(method, url, ok_cb, err_cb) { - ajaxSpin(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); -} - -//===== main menu, header spinner and notification boxes - -onLoad(function() { - var l = $("#layout"); - var o = l.childNodes[0]; - // spinner - l.insertBefore(m(''), o); - // notification boxes - l.insertBefore(m( - '
'), o); - // menu hamburger button - l.insertBefore(m(''), o); - // menu left-pane - var mm = m( - '\ - '); - l.insertBefore(mm, o); - - // make hamburger button pull out menu - var ml = $('#menuLink'), mm = $('#menu'); - bnd(ml, 'click', function (e) { - console.log("hamburger time"); - var active = 'active'; - e.preventDefault(); - toggleClass(l, active); - toggleClass(mm, active); - toggleClass(ml, active); - }); - - // populate menu via ajax call - var getMenu = function() { - ajaxJson("GET", "/menu", function(data) { - var html = "", path = window.location.pathname; - for (var i=0; i" + - "" + - data.menu[i] + ""); - } - $("#menu-list").innerHTML = html; - - v = $("#version"); - if (v != null) { v.innerHTML = data.version; } - }, function() { setTimeout(getMenu, 1000); }); - }; - getMenu(); -}); - -//===== Wifi info - -function showWifiInfo(data) { - Object.keys(data).forEach(function(v) { - el = $("#wifi-" + v); - if (el != null) { - if (el.nodeName === "INPUT") el.value = data[v]; - else el.innerHTML = data[v]; - } - }); - var dhcp = $('#dhcp-r'+data.dhcp); - if (dhcp) dhcp.click(); - $("#wifi-spinner").setAttribute("hidden", ""); - $("#wifi-table").removeAttribute("hidden"); - currAp = data.ssid; -} - -function getWifiInfo() { - ajaxJson('GET', "/wifi/info", showWifiInfo, - function(s, st) { window.setTimeout(getWifiInfo, 1000); }); -} - -//===== Notifications - -function showWarning(text) { - var el = $("#warning"); - el.innerHTML = text; - el.removeAttribute('hidden'); -} -function hideWarning() { - el = $("#warning").setAttribute('hidden', ''); -} -var notifTimeout = null; -function showNotification(text) { - var el = $("#notification"); - el.innerHTML = text; - el.removeAttribute('hidden'); - if (notifTimeout != null) clearTimeout(notifTimeout); - notifTimout = setTimeout(function() { - el.setAttribute('hidden', ''); - notifTimout = null; - }, 4000); -} - -//===== GPIO Pin mux card - -var currPin; -// pin={reset:12, isp:13, LED_conn:0, LED_ser:2} -function createInputForPin(pin) { - var input = document.createElement("input"); - input.type = "radio"; - input.name = "pins"; - input.data = pin.name; - input.className = "pin-input"; - input.value= pin.value; - input.id = "opt-" + pin.value; - if (currPin == pin.name) input.checked = "1"; - - var descr = m('"); - var div = document.createElement("div"); - div.appendChild(input); - div.appendChild(descr); - return div; -} - -function displayPins(resp) { - var po = $("#pin-mux"); - po.innerHTML = ""; - currPin = resp.curr; - resp.map.forEach(function(v) { - po.appendChild(createInputForPin(v)); - }); - var i, inputs = $(".pin-input"); - for (i=0; i Date: Tue, 14 Jul 2015 00:09:21 -0700 Subject: [PATCH 02/24] oops, readd mistakenly deleted file --- html/ui.js | 358 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 html/ui.js diff --git a/html/ui.js b/html/ui.js new file mode 100644 index 0000000..5de2f4c --- /dev/null +++ b/html/ui.js @@ -0,0 +1,358 @@ +//===== Collection of small utilities + +/* + * Bind/Unbind events + * + * Usage: + * var el = document.getElementyById('#container'); + * bnd(el, 'click', function() { + * console.log('clicked'); + * }); + */ + +var bnd = function( + d, // a DOM element + e, // an event name such as "click" + f // a handler function +){ + d.addEventListener(e, f, false); +} + +/* + * Create DOM element + * + * Usage: + * var el = m('

Hello

'); + * document.body.appendChild(el); + * + * Copyright (C) 2011 Jed Schmidt - WTFPL + * More: https://gist.github.com/966233 + */ + +var m = function( + a, // an HTML string + b, // placeholder + c // placeholder +){ + b = document; // get the document, + c = b.createElement("p"); // create a container element, + c.innerHTML = a; // write the HTML to it, and + a = b.createDocumentFragment(); // create a fragment. + + while ( // while + b = c.firstChild // the container element has a first child + ) a.appendChild(b); // append the child to the fragment, + + return a // and then return the fragment. +} + +/* + * DOM selector + * + * Usage: + * $('div'); + * $('#name'); + * $('.name'); + * + * Copyright (C) 2011 Jed Schmidt - WTFPL + * More: https://gist.github.com/991057 + */ + +var $ = function( + a, // take a simple selector like "name", "#name", or ".name", and + b // an optional context, and +){ + a = a.match(/^(\W)?(.*)/); // split the selector into name and symbol. + return( // return an element or list, from within the scope of + b // the passed context + || document // or document, + )[ + "getElement" + ( // obtained by the appropriate method calculated by + a[1] + ? a[1] == "#" + ? "ById" // the node by ID, + : "sByClassName" // the nodes by class name, or + : "sByTagName" // the nodes by tag name, + ) + ]( + a[2] // called with the name. + ) +} + +/* + * Get cross browser xhr object + * + * Copyright (C) 2011 Jed Schmidt + * More: https://gist.github.com/993585 + */ + +var j = function( + a // cursor placeholder +){ + for( // for all a + a=0; // from 0 + a<4; // to 4, + a++ // incrementing + ) try { // try + return a // returning + ? new ActiveXObject( // a new ActiveXObject + [ // reflecting + , // (elided) + "Msxml2", // the various + "Msxml3", // working + "Microsoft" // options + ][a] + // for Microsoft implementations, and + ".XMLHTTP" // the appropriate suffix, + ) // but make sure to + : new XMLHttpRequest // try the w3c standard first, and + } + + catch(e){} // ignore when it fails. +} + +// createElement short-hand + +e = function(a) { return document.createElement(a); } + +// chain onload handlers + +function onLoad(f) { + var old = window.onload; + if (typeof old != 'function') { + window.onload = f; + } else { + window.onload = function() { + old(); + f(); + } + } +} + +//===== helpers to add/remove/toggle HTML element classes + +function addClass(el, cl) { + el.className += ' ' + cl; +} +function removeClass(el, cl) { + var cls = el.className.split(/\s+/), + l = cls.length; + for (var i=0; i= 200 && xhr.status < 300) { + console.log("XHR done:", method, url, "->", xhr.status); + ok_cb(xhr.responseText); + } else { + console.log("XHR ERR :", method, url, "->", xhr.status, xhr.responseText, xhr); + err_cb(xhr.status, xhr.responseText); + } + } + console.log("XHR send:", method, url); + try { + xhr.send(); + } catch(err) { + console.log("XHR EXC :", method, url, "->", err); + err_cb(599, err); + } +} + +function dispatchJson(resp, ok_cb, err_cb) { + var j; + try { j = JSON.parse(resp); } + catch(err) { + console.log("JSON parse error: " + err + ". In: " + resp); + err_cb(500, "JSON parse error: " + err); + return; + } + ok_cb(j); +} + +function ajaxJson(method, url, ok_cb, err_cb) { + ajaxReq(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); +} + +function ajaxSpin(method, url, ok_cb, err_cb) { + $("#spinner").removeAttribute('hidden'); + ajaxReq(method, url, function(resp) { + $("#spinner").setAttribute('hidden', ''); + ok_cb(resp); + }, function(status, statusText) { + $("#spinner").setAttribute('hidden', ''); + //showWarning("Error: " + statusText); + err_cb(status, statusText); + }); +} + +function ajaxJsonSpin(method, url, ok_cb, err_cb) { + ajaxSpin(method, url, function(resp) { dispatchJson(resp, ok_cb, err_cb); }, err_cb); +} + +//===== main menu, header spinner and notification boxes + +onLoad(function() { + var l = $("#layout"); + var o = l.childNodes[0]; + // spinner + l.insertBefore(m(''), o); + // notification boxes + l.insertBefore(m( + '
'), o); + // menu hamburger button + l.insertBefore(m(''), o); + // menu left-pane + var mm = m( + '\ + '); + l.insertBefore(mm, o); + + // make hamburger button pull out menu + var ml = $('#menuLink'), mm = $('#menu'); + bnd(ml, 'click', function (e) { + console.log("hamburger time"); + var active = 'active'; + e.preventDefault(); + toggleClass(l, active); + toggleClass(mm, active); + toggleClass(ml, active); + }); + + // populate menu via ajax call + var getMenu = function() { + ajaxJson("GET", "/menu", function(data) { + var html = "", path = window.location.pathname; + for (var i=0; i" + + "" + + data.menu[i] + ""); + } + $("#menu-list").innerHTML = html; + + v = $("#version"); + if (v != null) { v.innerHTML = data.version; } + }, function() { setTimeout(getMenu, 1000); }); + }; + getMenu(); +}); + +//===== Wifi info + +function showWifiInfo(data) { + Object.keys(data).forEach(function(v) { + el = $("#wifi-" + v); + if (el != null) { + if (el.nodeName === "INPUT") el.value = data[v]; + else el.innerHTML = data[v]; + } + }); + var dhcp = $('#dhcp-r'+data.dhcp); + if (dhcp) dhcp.click(); + $("#wifi-spinner").setAttribute("hidden", ""); + $("#wifi-table").removeAttribute("hidden"); + currAp = data.ssid; +} + +function getWifiInfo() { + ajaxJson('GET', "/wifi/info", showWifiInfo, + function(s, st) { window.setTimeout(getWifiInfo, 1000); }); +} + +//===== Notifications + +function showWarning(text) { + var el = $("#warning"); + el.innerHTML = text; + el.removeAttribute('hidden'); +} +function hideWarning() { + el = $("#warning").setAttribute('hidden', ''); +} +var notifTimeout = null; +function showNotification(text) { + var el = $("#notification"); + el.innerHTML = text; + el.removeAttribute('hidden'); + if (notifTimeout != null) clearTimeout(notifTimeout); + notifTimout = setTimeout(function() { + el.setAttribute('hidden', ''); + notifTimout = null; + }, 4000); +} + +//===== GPIO Pin mux card + +var currPin; +// pin={reset:12, isp:13, LED_conn:0, LED_ser:2} +function createInputForPin(pin) { + var input = document.createElement("input"); + input.type = "radio"; + input.name = "pins"; + input.data = pin.name; + input.className = "pin-input"; + input.value= pin.value; + input.id = "opt-" + pin.value; + if (currPin == pin.name) input.checked = "1"; + + var descr = m('"); + var div = document.createElement("div"); + div.appendChild(input); + div.appendChild(descr); + return div; +} + +function displayPins(resp) { + var po = $("#pin-mux"); + po.innerHTML = ""; + currPin = resp.curr; + resp.map.forEach(function(v) { + po.appendChild(createInputForPin(v)); + }); + var i, inputs = $(".pin-input"); + for (i=0; i Date: Tue, 14 Jul 2015 23:38:18 -0700 Subject: [PATCH 03/24] make esp-12 with 4MB flash work --- Makefile | 55 +++++++++++++++++++++++++++++++----------------- user/cgiflash.c | 4 +++- user/user_main.c | 1 + 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index cad8624..99d5617 100644 --- a/Makefile +++ b/Makefile @@ -29,11 +29,28 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- +# Pick your flash size: "512KB" or "4MB" +FLASH_SIZE ?= 4MB + +ifeq ("$(FLASH_SIZE)","512KB") +# Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 ESP_SPI_SIZE ?= 0 # 0->512KB ESP_FLASH_MODE ?= 0 # 0->QIO ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +else +# Winbond 25Q32 4MB flash, typ for esp-12 +# Here we're using two partitions of approx 0.5MB because that's what's easily available in terms +# of linker scripts in the SDK. Ideally we'd use two partitions of approx 1MB, the remaining 2MB +# cannot be used for code. +ESP_SPI_SIZE ?= 4 # 6->4MB (1MB+1MB) or 4->4MB (512KB+512KB) +ESP_FLASH_MODE ?= 0 # 0->QIO, 2->DIO +ESP_FLASH_FREQ_DIV ?= 15 # 15->80Mhz +ESP_FLASH_MAX ?= 503808 # max bin file for 512KB flash partition: 492KB +#ESP_FLASH_MAX ?= 1028096 # max bin file for 1MB flash partition: 1004KB +endif + # hostname or IP address for wifi flashing ESP_HOSTNAME ?= esp-link @@ -163,7 +180,6 @@ SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.c)) OBJ := $(patsubst %.c,$(BUILD_BASE)/%.o,$(SRC)) $(BUILD_BASE)/espfs_img.o LIBS := $(addprefix -l,$(LIBS)) APP_AR := $(addprefix $(BUILD_BASE)/,$(TARGET)_app.a) -TARGET_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).out) USER1_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).user1.out) USER2_OUT := $(addprefix $(BUILD_BASE)/,$(TARGET).user2.out) @@ -202,16 +218,11 @@ endef .PHONY: all checkdirs clean webpages.espfs wiflash -all: echo_version checkdirs $(FW_BASE) firmware/user1.bin firmware/user2.bin +all: echo_version checkdirs $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin echo_version: @echo VERSION: $(VERSION) -$(TARGET_OUT): $(APP_AR) $(LD_SCRIPT) - $(vecho) "LD $@" - $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ -# $(OBJDP) -x $(TARGET_OUT) | egrep '(espfs_img)' - $(USER1_OUT): $(APP_AR) $(LD_SCRIPT1) $(vecho) "LD $@" $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT1) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ @@ -224,12 +235,11 @@ $(USER2_OUT): $(APP_AR) $(LD_SCRIPT2) $(Q) $(LD) -L$(SDK_LIBDIR) -T$(LD_SCRIPT2) $(LDFLAGS) -Wl,--start-group $(LIBS) $(APP_AR) -Wl,--end-group -o $@ # $(Q) $(OBJDP) -x $(TARGET_OUT) | egrep espfs_img -$(FW_BASE): $(TARGET_OUT) +$(FW_BASE): $(vecho) "FW $@" $(Q) mkdir -p $@ - $(Q) $(ESPTOOL) elf2image $(TARGET_OUT) --output $@/ -firmware/user1.bin: $(USER1_OUT) +$(FW_BASE)/user1.bin: $(USER1_OUT) $(FW_BASE) $(Q) $(OBJCP) --only-section .text -O binary $(USER1_OUT) eagle.app.v6.text.bin $(Q) $(OBJCP) --only-section .data -O binary $(USER1_OUT) eagle.app.v6.data.bin $(Q) $(OBJCP) --only-section .rodata -O binary $(USER1_OUT) eagle.app.v6.rodata.bin @@ -241,7 +251,7 @@ firmware/user1.bin: $(USER1_OUT) @echo "** user1.bin uses $$(stat -c '%s' $@) bytes of" $(ESP_FLASH_MAX) "available" $(Q) if [ $$(stat -c '%s' $@) -gt $$(( $(ESP_FLASH_MAX) )) ]; then echo "$@ too big!"; false; fi -firmware/user2.bin: $(USER2_OUT) +$(FW_BASE)/user2.bin: $(USER2_OUT) $(FW_BASE) $(Q) $(OBJCP) --only-section .text -O binary $(USER2_OUT) eagle.app.v6.text.bin $(Q) $(OBJCP) --only-section .data -O binary $(USER2_OUT) eagle.app.v6.data.bin $(Q) $(OBJCP) --only-section .rodata -O binary $(USER2_OUT) eagle.app.v6.rodata.bin @@ -251,7 +261,6 @@ firmware/user2.bin: $(USER2_OUT) $(Q) mv eagle.app.flash.bin $@ $(Q) if [ $$(stat -c '%s' $@) -gt $$(( $(ESP_FLASH_MAX) )) ]; then echo "$@ too big!"; false; fi - $(APP_AR): $(OBJ) $(vecho) "AR $@" $(Q) $(AR) cru $@ $^ @@ -262,7 +271,7 @@ $(BUILD_DIR): $(Q) mkdir -p $@ wiflash: all - ./wiflash $(ESP_HOSTNAME) firmware/user1.bin firmware/user2.bin + ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin flash: all $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash \ @@ -302,24 +311,32 @@ endif # we also adjust the sizes of the segments 'cause we need more irom0 # in the end the only thing that matters wrt size is that the whole shebang fits into the # 236KB available (in a 512KB flash) -build/eagle.esphttpd.v6.ld: $(SDK_LDDIR)/eagle.app.v6.ld - $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ - $(SDK_LDDIR)/eagle.app.v6.ld >$@ +ifeq ("$(FLASH_SIZE)","512KB") build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app1.ld >$@ build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ -e '/^ irom0_0_seg/ s/2B000/38000/' \ - $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ + $(SDK_LDDIR)/eagle.app.v6.new.512.app2.ld >$@ +else +build/eagle.esphttpd1.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app1.ld + $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ + -e '/^ irom0_0_seg/ s/6B000/7C000/' \ + $(SDK_LDDIR)/eagle.app.v6.new.1024.app1.ld >$@ +build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld + $(Q) sed -e '/\.irom\.text/{' -e 'a . = ALIGN (4);' -e 'a *(.espfs)' -e '}' \ + -e '/^ irom0_0_seg/ s/6B000/7C000/' \ + $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld >$@ +endif espfs/mkespfsimage/mkespfsimage: espfs/mkespfsimage/ $(Q) $(MAKE) -C espfs/mkespfsimage USE_HEATSHRINK="$(USE_HEATSHRINK)" GZIP_COMPRESSION="$(GZIP_COMPRESSION)" release: all $(Q) rm -rf release; mkdir -p release/esp-link - $(Q) cp firmware/user1.bin firmware/user2.bin $(SDK_BASE)/bin/blank.bin \ + $(Q) cp $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin $(SDK_BASE)/bin/blank.bin \ "$(SDK_BASE)/bin/boot_v1.4(b1).bin" wiflash release/esp-link $(Q) tar zcf esp-link.tgz -C release esp-link $(Q) rm -rf release diff --git a/user/cgiflash.c b/user/cgiflash.c index 65bfd21..70d4b1c 100644 --- a/user/cgiflash.c +++ b/user/cgiflash.c @@ -22,8 +22,10 @@ Some flash handling cgi routines. Used for reading the existing flash and updati // Check that the header of the firmware blob looks like actual firmware... static char* ICACHE_FLASH_ATTR check_header(void *buf) { uint8_t *cd = (uint8_t *)buf; + uint32_t *buf32 = buf; + os_printf("%p: %08lX %08lX %08lX %08lX\n", buf, buf32[0], buf32[1], buf32[2], buf32[3]); if (cd[0] != 0xEA) return "IROM magic missing"; - if (cd[1] != 4 || cd[2] > 3 || cd[3] > 0x40) return "bad flash header"; + if (cd[1] != 4 || cd[2] > 3 || (cd[3]>>4) > 6) return "bad flash header"; if (((uint16_t *)buf)[3] != 0x4010) return "Invalid entry addr"; if (((uint32_t *)buf)[2] != 0) return "Invalid start offset"; return NULL; diff --git a/user/user_main.c b/user/user_main.c index 57443ae..645d215 100644 --- a/user/user_main.c +++ b/user/user_main.c @@ -156,6 +156,7 @@ void user_init(void) { os_printf("exccause=%d epc1=0x%x epc2=0x%x epc3=0x%x excvaddr=0x%x depc=0x%x\n", rst_info->exccause, rst_info->epc1, rst_info->epc2, rst_info->epc3, rst_info->excvaddr, rst_info->depc); + os_printf("Flash map %d, chip %08X\n", system_get_flash_size_map(), spi_flash_get_id()); os_printf("** esp-link ready\n"); } From 886ce62df0bd1d3df8f932ed7409bb026c6a66d6 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Tue, 28 Jul 2015 11:29:20 -0700 Subject: [PATCH 04/24] initial support for TCP client connections from uC --- Makefile | 2 +- html/home.html | 22 ++++ serial/serbridge.c | 261 ++++++++++++++++++++++++++++--------------- serial/serbridge.h | 8 +- serial/tcpclient.c | 272 +++++++++++++++++++++++++++++++++++++++++++++ serial/tcpclient.h | 8 ++ user/cgiflash.c | 1 + user/status.c | 119 +++++++++++++++++++- 8 files changed, 597 insertions(+), 96 deletions(-) create mode 100644 serial/tcpclient.c create mode 100644 serial/tcpclient.h diff --git a/Makefile b/Makefile index 99d5617..32cce5c 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- # Pick your flash size: "512KB" or "4MB" -FLASH_SIZE ?= 4MB +FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 diff --git a/html/home.html b/html/home.html index 59e9ab2..35c8436 100644 --- a/html/home.html +++ b/html/home.html @@ -44,6 +44,28 @@
+
+
+

HTTP service

+
+ External web service configuration +
+ + + + +
+
+ + + + +
+ +
+
+
diff --git a/serial/serbridge.c b/serial/serbridge.c index 6d45610..528c486 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -13,20 +13,21 @@ #include "serled.h" #include "config.h" #include "console.h" +#include "tcpclient.h" static struct espconn serbridgeConn; static esp_tcp serbridgeTcp; static int8_t mcu_reset_pin, mcu_isp_pin; -sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len); +static sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len); // Connection pool serbridgeConnData connData[MAX_CONN]; -// Transmit buffers for the connection pool -static char txbuffer[MAX_CONN][MAX_TXBUFFER]; - // Given a pointer to an espconn struct find the connection that correcponds to it static serbridgeConnData ICACHE_FLASH_ATTR *serbridgeFindConnData(void *arg) { + struct espconn *conn = arg; + return (serbridgeConnData *)conn->reverse; +#if 0 for (int i=0; itxbuffer -// returns result from espconn_sent if data in buffer or ESPCONN_OK (0) -// Use only internally from espbuffsend and serbridgeSentCb -static sint8 ICACHE_FLASH_ATTR sendtxbuffer(serbridgeConnData *conn) { - sint8 result = ESPCONN_OK; - if (conn->txbufferlen != 0) { - //os_printf("%d TX %d\n", system_get_time(), conn->txbufferlen); - conn->readytosend = false; - result = espconn_sent(conn->conn, (uint8_t*)conn->txbuffer, conn->txbufferlen); - conn->txbufferlen = 0; - if (result != ESPCONN_OK) { - os_printf("sendtxbuffer: espconn_sent error %d on conn %p\n", result, conn); - } - } - return result; -} - -// espbuffsend adds data to the send buffer. If the previous send was completed it calls -// sendtxbuffer and espconn_sent. -// Returns ESPCONN_OK (0) for success, -128 if buffer is full or error from espconn_sent -// Use espbuffsend instead of espconn_sent as it solves the problem that espconn_sent must -// only be called *after* receiving an espconn_sent_callback for the previous packet. -sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len) { - if (conn->txbufferlen + len > MAX_TXBUFFER) { - os_printf("espbuffsend: txbuffer full on conn %p\n", conn); - return -128; - } - os_memcpy(conn->txbuffer + conn->txbufferlen, data, len); - conn->txbufferlen += len; - if (conn->readytosend) { - return sendtxbuffer(conn); - } else { - //os_printf("%d QU %d\n", system_get_time(), conn->txbufferlen); - } - return ESPCONN_OK; -} - -//callback after the data are sent -static void ICACHE_FLASH_ATTR serbridgeSentCb(void *arg) { - serbridgeConnData *conn = serbridgeFindConnData(arg); - //os_printf("Sent callback on conn %p\n", conn); - if (conn == NULL) return; - //os_printf("%d ST\n", system_get_time()); - conn->readytosend = true; - sendtxbuffer(conn); // send possible new data in txbuffer -} +//===== TCP -> UART // Telnet protocol characters #define IAC 255 // escape @@ -189,7 +146,6 @@ void ICACHE_FLASH_ATTR serbridgeReset() { } else os_printf("MCU reset: no pin\n"); } - // Receive callback static void ICACHE_FLASH_ATTR serbridgeRecvCb(void *arg, char *data, unsigned short len) { serbridgeConnData *conn = serbridgeFindConnData(arg); @@ -233,6 +189,9 @@ static void ICACHE_FLASH_ATTR serbridgeRecvCb(void *arg, char *data, unsigned sh } else { conn->conn_mode = cmTransparent; } + + // Process return data on TCP client connections + } else if (conn->conn_mode == cmTcpClient) { } // write the buffer to the uart @@ -245,41 +204,181 @@ static void ICACHE_FLASH_ATTR serbridgeRecvCb(void *arg, char *data, unsigned sh serledFlash(50); // short blink on serial LED } +//===== UART -> TCP + +// Transmit buffers for the connection pool +static char txbuffer[MAX_CONN][MAX_TXBUFFER]; + +// Send all data in conn->txbuffer +// returns result from espconn_sent if data in buffer or ESPCONN_OK (0) +// Use only internally from espbuffsend and serbridgeSentCb +static sint8 ICACHE_FLASH_ATTR sendtxbuffer(serbridgeConnData *conn) { + sint8 result = ESPCONN_OK; + if (conn->txbufferlen != 0) { + //os_printf("%d TX %d\n", system_get_time(), conn->txbufferlen); + conn->readytosend = false; + result = espconn_sent(conn->conn, (uint8_t*)conn->txbuffer, conn->txbufferlen); + conn->txbufferlen = 0; + if (result != ESPCONN_OK) { + os_printf("sendtxbuffer: espconn_sent error %d on conn %p\n", result, conn); + } + } + return result; +} + +// espbuffsend adds data to the send buffer. If the previous send was completed it calls +// sendtxbuffer and espconn_sent. +// Returns ESPCONN_OK (0) for success, -128 if buffer is full or error from espconn_sent +// Use espbuffsend instead of espconn_sent as it solves the problem that espconn_sent must +// only be called *after* receiving an espconn_sent_callback for the previous packet. +static sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len) { + if (conn->txbufferlen + len > MAX_TXBUFFER) { + os_printf("espbuffsend: txbuffer full on conn %p\n", conn); + return -128; + } + os_memcpy(conn->txbuffer + conn->txbufferlen, data, len); + conn->txbufferlen += len; + if (conn->readytosend) { + return sendtxbuffer(conn); + } else { + //os_printf("%d QU %d\n", system_get_time(), conn->txbufferlen); + } + return ESPCONN_OK; +} + +//callback after the data are sent +static void ICACHE_FLASH_ATTR +serbridgeSentCb(void *arg) { + serbridgeConnData *conn = serbridgeFindConnData(arg); + //os_printf("Sent callback on conn %p\n", conn); + if (conn == NULL) return; + //os_printf("%d ST\n", system_get_time()); + conn->readytosend = true; + sendtxbuffer(conn); // send possible new data in txbuffer +} + +// TCP client connection state machine +// This processes commands from the attached uC to open outboud TCP connections +enum { + TC_idle, // in-between commands + TC_newline, // newline seen + TC_start, // start character (~) seen + TC_cmd, // command character (0) seen + TC_cmdLine, // accumulating command + TC_tdata, // data character (1-9) seen, and in text data mode +}; +static uint8_t tcState = TC_newline; +static uint8_t tcChan; // channel for current command (index into tcConn) + +#define CMD_MAX 256 +static char tcCmdBuf[CMD_MAX]; +static short tcCmdBufLen = 0; +static char tcCmdChar; + +// scan a buffer for tcp client commands +static int ICACHE_FLASH_ATTR +tcpClientProcess(char *buf, int len) +{ + char *in=buf, *out=buf; + for (short i=0; i= '1' && c <= '9') { + tcChan = c-'1'; + tcState = TC_tdata; + continue; + } + *out++ = '~'; // make up for '~' we skipped + break; + case TC_cmd: // saw channel number 0 (command), expect command char + tcCmdChar = c; // save command character + tcCmdBufLen = 0; // empty the command buffer + tcState = TC_cmdLine; + continue; + case TC_cmdLine: // accumulating command in buffer + if (c != '\n') { + if (tcCmdBufLen < CMD_MAX) tcCmdBuf[tcCmdBufLen++] = c; + } else { + tcpClientCommand(tcCmdChar, tcCmdBuf); + tcState = TC_newline; + } + continue; + case TC_tdata: // saw channel number, getting data characters + if (c != 0) { + tcpClientSendChar(tcChan, c); + } else { + tcpClientSendPush(tcChan); + tcState = TC_idle; + } + continue; + } + *out++ = c; + } + if (tcState != TC_idle) os_printf("tcState=%d\n", tcState); + return out-buf; +} + +// callback with a buffer of characters that have arrived on the uart +void ICACHE_FLASH_ATTR +serbridgeUartCb(char *buf, int length) { + // push the buffer into the microcontroller console + for (int i=0; i 0) { + for (int i = 0; i < MAX_CONN; ++i) { + if (connData[i].conn && connData[i].conn_mode != cmTcpClient) { + espbuffsend(&connData[i], buf, length); + } + } + } + serledFlash(50); // short blink on serial LED +} + +//===== Connect / disconnect + // Error callback (it's really poorly named, it's not a "connection reconnected" callback, // it's really a "connection broken, please reconnect" callback) static void ICACHE_FLASH_ATTR serbridgeReconCb(void *arg, sint8 err) { - serbridgeConnData *conn=serbridgeFindConnData(arg); - if (conn == NULL) return; + serbridgeConnData *sbConn = serbridgeFindConnData(arg); + if (sbConn == NULL) return; // Close the connection - espconn_disconnect(conn->conn); - conn->conn = NULL; + espconn_disconnect(sbConn->conn); + // free connection slot + sbConn->conn = NULL; } // Disconnection callback static void ICACHE_FLASH_ATTR serbridgeDisconCb(void *arg) { - // Iterate through all the connections and deallocate the ones that are in a state that - // indicates that they're closed - for (int i=0; istate == ESPCONN_NONE || connData[i].conn->state == ESPCONN_CLOSE)) - { - if (connData[i].conn_mode == cmAVR) { - // send reset to arduino/ARM - if (mcu_reset_pin >= 0) { - GPIO_OUTPUT_SET(mcu_reset_pin, 0); - os_delay_us(100L); - GPIO_OUTPUT_SET(mcu_reset_pin, 1); - } - } - connData[i].conn = NULL; - } + serbridgeConnData *sbConn = serbridgeFindConnData(arg); + if (sbConn == NULL) return; + // send reset to arduino/ARM + if (sbConn->conn_mode == cmAVR && mcu_reset_pin >= 0) { + GPIO_OUTPUT_SET(mcu_reset_pin, 0); + os_delay_us(100L); + GPIO_OUTPUT_SET(mcu_reset_pin, 1); } + // free connection slot + sbConn->conn = NULL; } // New connection callback, use one of the connection descriptors, if we have one left. static void ICACHE_FLASH_ATTR serbridgeConnectCb(void *arg) { struct espconn *conn = arg; - //Find empty conndata in pool + // Find empty conndata in pool int i; for (i=0; ireverse = connData+i; + connData[i].conn = conn; connData[i].txbufferlen = 0; connData[i].readytosend = true; connData[i].telnet_state = 0; @@ -304,20 +404,7 @@ static void ICACHE_FLASH_ATTR serbridgeConnectCb(void *arg) { espconn_set_opt(conn, ESPCONN_REUSEADDR|ESPCONN_NODELAY); } -// callback with a buffer of characters that have arrived on the uart -void ICACHE_FLASH_ATTR -serbridgeUartCb(char *buf, int length) { - // push the buffer into the microcontroller console - for (int i=0; i +#include "config.h" +#include "uart.h" +#include "serled.h" +#include "tcpclient.h" + +// max number of channels the client can open +#define MAX_CHAN 8 +// size of tx buffer +#define MAX_TXBUF 1024 + +enum TcpState { + TCP_idle, // unused connection + TCP_dns, // doing gethostbyname + TCP_conn, // connecting to remote server + TCP_data, // connected +}; + +// Connections +typedef struct { + struct espconn *conn; // esp connection structure + esp_tcp *tcp; // esp TCP parameters + char *txBuf; // buffer to accumulate into + char *txBufSent; // buffer held by espconn + uint8_t txBufLen; // number of chars in txbuf + enum TcpState state; +} TcpConn; + +#define MAX_CONN (8) +static TcpConn tcpConn[MAX_CONN]; + +// Free a connection dynamically. +static void ICACHE_FLASH_ATTR +tcpConnFree(TcpConn* tci) { + if (tci->conn != NULL) os_free(tci->conn); + if (tci->tcp != NULL) os_free(tci->tcp); + if (tci->txBuf != NULL) os_free(tci->txBuf); + if (tci->txBufSent != NULL) os_free(tci->txBufSent); + memset(tci, 0, sizeof(TcpConn)); +} + +// DNS name resolution callback +static void ICACHE_FLASH_ATTR +tcpClientHostnameCb(const char *name, ip_addr_t *ipaddr, void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP dns CB (%p %p)\n", arg, tci); + if (ipaddr == NULL) { + os_printf("TCP %s not found\n", name); + } else { + os_printf("TCP %s -> %d.%d.%d.%d\n", name, IP2STR(ipaddr)); + tci->tcp->remote_ip[0] = ip4_addr1(ipaddr); + tci->tcp->remote_ip[1] = ip4_addr2(ipaddr); + tci->tcp->remote_ip[2] = ip4_addr3(ipaddr); + tci->tcp->remote_ip[3] = ip4_addr4(ipaddr); + os_printf("TCP connect %d.%d.%d.%d (%p)\n", IP2STR(tci->tcp->remote_ip), tci); + if (espconn_connect(tci->conn) == ESPCONN_OK) { + tci->state = TCP_conn; + return; + } + os_printf("TCP connect failure\n"); + } + // oops + tcpConnFree(tci); +} + +// Send the next buffer (assumes that the connection is in a state that allows it) +static void tcpDoSend(TcpConn *tci) { + sint8 err = espconn_sent(tci->conn, (uint8*)tci->txBuf, tci->txBufLen); + if (err == ESPCONN_OK) { + // send successful + os_printf("TCP sent (%p %p)\n", tci->conn, tci); + tci->txBufSent = tci->txBuf; + tci->txBuf = NULL; + tci->txBufLen = 0; + } else { + // send error, leave as-is and try again later... + os_printf("TCP send err (%p %p) %d\n", tci->conn, tci, err); + } +} + +// Connected callback +static void ICACHE_FLASH_ATTR tcpConnectCb(void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP connect CB (%p %p)\n", arg, tci); + tci->state = TCP_data; + // send any buffered data + if (tci->txBuf != NULL && tci->txBufLen > 0) tcpDoSend(tci); +} + +// Sent callback +static void ICACHE_FLASH_ATTR tcpSentCb(void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP sent CB (%p %p)\n", arg, tci); + if (tci->txBufSent != NULL) os_free(tci->txBufSent); + tci->txBufSent = NULL; + + if (tci->txBuf != NULL && tci->txBufLen == MAX_TXBUF) { + // next buffer is full, send it now + tcpDoSend(tci); + } +} + +// Recv callback +static void ICACHE_FLASH_ATTR tcpRecvCb(void *arg, char *data, uint16_t len) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP recv CB (%p %p)\n", arg, tci); + if (tci->state == TCP_data) { + uart0_tx_buffer(data, len); + } + serledFlash(50); // short blink on serial LED +} + +// Disconnect callback +static void ICACHE_FLASH_ATTR tcpDisconCb(void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP disconnect CB (%p %p)\n", arg, tci); + tcpConnFree(tci); +} + +// Connection reset callback +static void ICACHE_FLASH_ATTR tcpResetCb(void *arg, sint8 err) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP reset CB (%p %p) err=%d\n", arg, tci, err); + tcpConnFree(tci); +} + +// Allocate a new connection dynamically and return it. Returns NULL if no +// connection could be allocated. +static TcpConn* ICACHE_FLASH_ATTR +tcpConnAlloc(void) { + int i; + for (i=0; iconn = os_malloc(sizeof(struct espconn)); + if (tci->conn == NULL) goto fail; + memset(tci->conn, 0, sizeof(struct espconn)); + // malloc esp_tcp struct + tci->tcp = os_malloc(sizeof(esp_tcp)); + if (tci->tcp == NULL) goto fail; + memset(tci->tcp, 0, sizeof(esp_tcp)); + + // common init + tci->state = TCP_dns; + tci->conn->type = ESPCONN_TCP; + tci->conn->state = ESPCONN_NONE; + tci->conn->proto.tcp = tci->tcp; + tci->tcp->remote_port = 80; + tci->tcp->remote_ip[0] = 173; + espconn_regist_connectcb(tci->conn, tcpConnectCb); + espconn_regist_reconcb(tci->conn, tcpResetCb); + espconn_regist_sentcb(tci->conn, tcpSentCb); + espconn_regist_recvcb(tci->conn, tcpRecvCb); + espconn_regist_disconcb(tci->conn, tcpDisconCb); + tci->conn->reverse = tci; + + return tci; + +fail: + tcpConnFree(tci); + return NULL; +} + +static TcpConn *tcConn[MAX_CHAN]; + +// Perform a TCP command: parse the command and do the right thing. +// Returns true on success. +bool ICACHE_FLASH_ATTR +tcpClientCommand(char cmd, char *cmdBuf) { + uint8_t tcChan; + TcpConn *tci; + + switch (cmd) { + //== TCP Connect command + case 'T': + if (*cmdBuf < '1' || *cmdBuf > ('0'+MAX_CHAN)) break; + tcChan = *cmdBuf++ - '1'; + char *hostname = cmdBuf; + char *port = hostname; + while (*port != 0 && *port != ':') port++; + if (*port != ':') break; + *port = 0; + port++; + int portInt = atoi(port); + if (portInt < 1 || portInt > 65535) break; + + // allocate a connection + tci = tcpConnAlloc(); + if (tci == NULL) break; + tci->state = TCP_dns; + tci->tcp->remote_port = portInt; + + // start the DNS resolution + os_printf("TCP %p resolving %s (conn=%p)\n", tci, hostname, tci->conn); + ip_addr_t ip; + err_t err = espconn_gethostbyname(tci->conn, hostname, &ip, tcpClientHostnameCb); + if (err == ESPCONN_OK) { + // dns cache hit, got the IP address, fake the callback (sigh) + os_printf("TCP DNS hit\n"); + tcpClientHostnameCb(hostname, &ip, tci->conn); + } else if (err != ESPCONN_INPROGRESS) { + tcpConnFree(tci); + break; + } + + tcConn[tcChan] = tci; + return true; + + //== TCP Close/disconnect command + case 'C': + if (*cmdBuf < '1' || *cmdBuf > ('0'+MAX_CHAN)) break; + tcChan = *cmdBuf++ - '1'; + tci = tcConn[tcChan]; + if (tci->state > TCP_idle) { + tci->state = TCP_idle; // hackish... + espconn_disconnect(tci->conn); + } + break; + + } + return false; +} + +void ICACHE_FLASH_ATTR +tcpClientSendChar(uint8_t chan, char c) { + TcpConn *tci = tcConn[chan]; + if (tci->state == TCP_idle) return; + + if (tci->txBuf != NULL) { + // we have a buffer + if (tci->txBufLen < MAX_TXBUF) { + // buffer has space, add char and return + tci->txBuf[tci->txBufLen++] = c; + return; + } else if (tci->txBufSent == NULL) { + // we don't have a send pending, send full buffer off + tcpDoSend(tci); + if (tci->txBuf != NULL) return; // something went wrong + } else { + // buffers all backed-up, drop char + return; + } + } + // we do not have a buffer (either didn't have one or sent it off) + // allocate one + tci->txBuf = os_malloc(MAX_TXBUF); + tci->txBufLen = 0; + if (tci->txBuf != NULL) { + tci->txBuf[tci->txBufLen++] = c; + } +} + +void ICACHE_FLASH_ATTR +tcpClientSendPush(uint8_t chan) { + TcpConn *tci = tcConn[chan]; + if (tci->state == TCP_idle) return; // no active connection on this channel + if (tci->txBuf == NULL || tci->txBufLen == 0) return; // no chars accumulated to send + if (tci->txBufSent != NULL) return; // already got a send in progress + tcpDoSend(tci); +} diff --git a/serial/tcpclient.h b/serial/tcpclient.h new file mode 100644 index 0000000..c80d2d3 --- /dev/null +++ b/serial/tcpclient.h @@ -0,0 +1,8 @@ +#ifndef __TCP_CLIENT_H__ +#define __TCP_CLIENT_H__ + +bool tcpClientCommand(char cmd, char *cmdBuf); +void tcpClientSendChar(uint8_t chan, char c); +void tcpClientSendPush(uint8_t chan); + +#endif /* __TCP_CLIENT_H__ */ diff --git a/user/cgiflash.c b/user/cgiflash.c index 70d4b1c..847b5ab 100644 --- a/user/cgiflash.c +++ b/user/cgiflash.c @@ -160,6 +160,7 @@ int ICACHE_FLASH_ATTR cgiRebootFirmware(HttpdConnData *connData) { int address = id == 1 ? 4*1024 // either start after 4KB boot partition : 4*1024 + FIRMWARE_SIZE + 16*1024 + 4*1024; // 4KB boot, fw1, 16KB user param, 4KB reserved uint32 buf[8]; + os_printf("Checking %p\n", (void *)address); spi_flash_read(address, buf, sizeof(buf)); char *err = check_header(buf); if (err != NULL) { diff --git a/user/status.c b/user/status.c index 23f693f..7b46d72 100644 --- a/user/status.c +++ b/user/status.c @@ -5,8 +5,11 @@ #include "serled.h" #include "cgiwifi.h" +//===== "CONN" LED status indication + static ETSTimer ledTimer; +// Set the LED on or off, respecting the defined polarity static void ICACHE_FLASH_ATTR setLed(int on) { int8_t pin = flashConfig.conn_led_pin; if (pin < 0) return; // disabled @@ -21,11 +24,12 @@ static void ICACHE_FLASH_ATTR setLed(int on) { static uint8_t ledState = 0; static uint8_t wifiState = 0; +// Timer callback to update the LED static void ICACHE_FLASH_ATTR ledTimerCb(void *v) { int time = 1000; if (wifiState == wifiGotIP) { - // connected, all is good, solid light + // connected, all is good, solid light with a short dark blip every 3 seconds ledState = 1-ledState; time = ledState ? 2900 : 100; } else if (wifiState == wifiIsConnected) { @@ -33,7 +37,7 @@ static void ICACHE_FLASH_ATTR ledTimerCb(void *v) { ledState = 1 - ledState; time = 1000; } else { - // idle + // not connected switch (wifi_get_opmode()) { case 1: // STA ledState = 0; @@ -53,7 +57,7 @@ static void ICACHE_FLASH_ATTR ledTimerCb(void *v) { os_timer_arm(&ledTimer, time, 0); } -// change the wifi state +// change the wifi state indication void ICACHE_FLASH_ATTR statusWifiUpdate(uint8_t state) { wifiState = state; // schedule an update (don't want to run into concurrency issues) @@ -62,6 +66,111 @@ void ICACHE_FLASH_ATTR statusWifiUpdate(uint8_t state) { os_timer_arm(&ledTimer, 500, 0); } +//===== RSSI Status update sent to GroveStreams + +#define RSSI_INTERVAL (60*1000) + +static ETSTimer rssiTimer; + +static uint8_t rssiSendState = 0; +static sint8 rssiLast = 0; //last RSSI value + +static esp_tcp rssiTcp; +static struct espconn rssiConn; + +#define GS_API_KEY "2eb868a8-224f-3faa-939d-c79bd605912a" +#define GS_COMP_ID "esp-link" +#define GS_STREAM "rssi" + +// Connected callback +static void ICACHE_FLASH_ATTR rssiConnectCb(void *arg) { + struct espconn *conn = (struct espconn *)arg; + os_printf("RSSI connect CB (%p %p)\n", arg, conn->reverse); + + char buf[2000]; + + // http header + int hdrLen = os_sprintf(buf, + "PUT /api/feed?api_key=%s HTTP/1.0\r\n" + "Content-Type: application/json\r\n" + "Content-Length: XXXXX\r\n\r\n", + GS_API_KEY); + + // http body + int dataLen = os_sprintf(buf+hdrLen, + "[{\"compId\":\"%s\", \"streamId\":\"%s\", \"data\":%d}]", + GS_COMP_ID, GS_STREAM, rssiLast); + + // hackish way to fill in the content-length + os_sprintf(buf+hdrLen-9, "%5d", dataLen); + buf[hdrLen-4] = '\r'; + + // send it off + if (espconn_sent(conn, (uint8*)buf, hdrLen+dataLen) == ESPCONN_OK) { + os_printf("RSSI sent rssi=%d\n", rssiLast); + os_printf("RSSI sent <<%s>>\n", buf); + } +} + +// Sent callback +static void ICACHE_FLASH_ATTR rssiSentCb(void *arg) { + struct espconn *conn = (struct espconn *)arg; + os_printf("RSSI sent CB (%p %p)\n", arg, conn->reverse); +} + +// Recv callback +static void ICACHE_FLASH_ATTR rssiRecvCb(void *arg, char *data, uint16_t len) { + struct espconn *conn = (struct espconn *)arg; + os_printf("RSSI recv CB (%p %p)\n", arg, conn->reverse); + data[len] = 0; // hack!!! + os_printf("GOT %d: <<%s>>\n", len, data); + + espconn_disconnect(conn); +} + +// Disconnect callback +static void ICACHE_FLASH_ATTR rssiDisconCb(void *arg) { + struct espconn *conn = (struct espconn *)arg; + os_printf("RSSI disconnect CB (%p %p)\n", arg, conn->reverse); + rssiSendState = 0; +} + +// Connection reset callback +static void ICACHE_FLASH_ATTR rssiResetCb(void *arg, sint8 err) { + struct espconn *conn = (struct espconn *)arg; + os_printf("RSSI reset CB (%p %p) err=%d\n", arg, conn->reverse, err); + rssiSendState = 0; +} + +// Timer callback to send an RSSI update to a monitoring system +static void ICACHE_FLASH_ATTR rssiTimerCb(void *v) { + sint8 rssi = wifi_station_get_rssi(); + if (rssi >= 0) return; // not connected or other error + rssiLast = rssi; + if (rssiSendState > 0) return; // not done with previous rssi report + + rssiConn.type = ESPCONN_TCP; + rssiConn.state = ESPCONN_NONE; + rssiConn.proto.tcp = &rssiTcp; + rssiTcp.remote_port = 80; + rssiTcp.remote_ip[0] = 173; + rssiTcp.remote_ip[1] = 236; + rssiTcp.remote_ip[2] = 12; + rssiTcp.remote_ip[3] = 163; + espconn_regist_connectcb(&rssiConn, rssiConnectCb); + espconn_regist_reconcb(&rssiConn, rssiResetCb); + espconn_regist_sentcb(&rssiConn, rssiSentCb); + espconn_regist_recvcb(&rssiConn, rssiRecvCb); + espconn_regist_disconcb(&rssiConn, rssiDisconCb); + rssiConn.reverse = (void *)0xdeadf00d; + os_printf("RSSI connect (%p)\n", &rssiConn); + if (espconn_connect(&rssiConn) == ESPCONN_OK) { + rssiSendState++; + } +} + +//===== Init status stuff + void ICACHE_FLASH_ATTR statusInit(void) { if (flashConfig.conn_led_pin >= 0) { makeGpio(flashConfig.conn_led_pin); @@ -72,6 +181,10 @@ void ICACHE_FLASH_ATTR statusInit(void) { os_timer_disarm(&ledTimer); os_timer_setfn(&ledTimer, ledTimerCb, NULL); os_timer_arm(&ledTimer, 2000, 0); + + os_timer_disarm(&rssiTimer); + os_timer_setfn(&rssiTimer, rssiTimerCb, NULL); + os_timer_arm(&rssiTimer, RSSI_INTERVAL, 1); // recurring timer } From 2fbd9953d592be54fc8e54652f2612a9ae1787f6 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Sat, 1 Aug 2015 13:56:13 -0700 Subject: [PATCH 05/24] TCP client support; Grovestreams RSSI submission support --- html/home.html | 66 ++++++----- html/ui.js | 39 +++++++ httpd/httpd.c | 6 +- serial/serbridge.c | 66 +++++++++-- serial/tcpclient.c | 272 ++++++++++++++++++++++++++------------------- serial/tcpclient.h | 10 +- user/cgipins.c | 3 +- user/cgitcp.c | 74 ++++++++++++ user/cgitcp.h | 8 ++ user/config.c | 13 ++- user/config.h | 3 + user/status.c | 102 ++++------------- user/user_main.c | 7 +- 13 files changed, 422 insertions(+), 247 deletions(-) create mode 100644 user/cgitcp.c create mode 100644 user/cgitcp.h diff --git a/html/home.html b/html/home.html index 35c8436..ddc8e4a 100644 --- a/html/home.html +++ b/html/home.html @@ -26,16 +26,40 @@
-
-

Wifi summary

-
- - - - - - -
+
+
+

Wifi summary

+
+ + + + + + +
+
+

TCP client

+
+ TCP client support in esp-link +
+ + +
+
+ Grovestreams data push +
+ + +
+
+ + +
+ +
+
+

Pin assignment

Select one of the following signal/pin assignments to match your hardware @@ -45,26 +69,6 @@
-
-

HTTP service

-
- External web service configuration -
- - - - -
-
- - - - -
- -
-
@@ -74,6 +78,8 @@ onLoad(function() { fetchPins(); getWifiInfo(); + fetchTcpClient(); + bnd($("#tcpform"), "submit", changeTcpClient); }); diff --git a/html/ui.js b/html/ui.js index 5de2f4c..56cc256 100644 --- a/html/ui.js +++ b/html/ui.js @@ -356,3 +356,42 @@ function setPins(v, name) { }); } +//===== TCP client card + +function tcpEn(){return document.querySelector('input[name="tcp_enable"]')} +function rssiEn(){return document.querySelector('input[name="rssi_enable"]')} +function apiKey(){return document.querySelector('input[name="api_key"]')} + +function changeTcpClient(e) { + e.preventDefault(); + var url = "tcpclient"; + url += "?tcp_enable=" + tcpEn().checked; + url += "&rssi_enable=" + rssiEn().checked; + url += "&api_key=" + encodeURIComponent(apiKey().value); + + hideWarning(); + var cb = $("#tcp-button"); + addClass(cb, 'pure-button-disabled'); + ajaxSpin("POST", url, function(resp) { + removeClass(cb, 'pure-button-disabled'); + getWifiInfo(); + }, function(s, st) { + showWarning("Error: "+st); + removeClass(cb, 'pure-button-disabled'); + getWifiInfo(); + }); +} + +function displayTcpClient(resp) { + tcpEn().checked = resp.tcp_enable > 0; + rssiEn().checked = resp.rssi_enable > 0; + apiKey().value = resp.api_key; +} + +function fetchTcpClient() { + ajaxJson("GET", "/tcpclient", displayTcpClient, function() { + window.setTimeout(fetchTcpClient, 1000); + }); +} + + diff --git a/httpd/httpd.c b/httpd/httpd.c index 3249889..fe22047 100644 --- a/httpd/httpd.c +++ b/httpd/httpd.c @@ -314,13 +314,13 @@ static void ICACHE_FLASH_ATTR httpdSentCb(void *arg) { conn->cgi=NULL; //mark for destruction. } if (r==HTTPD_CGI_NOTFOUND || r==HTTPD_CGI_AUTHENTICATED) { - os_printf("%s ERROR! CGI fn returns code %d after sending data! Bad CGI!\n", connStr, r); + os_printf("%s ERROR! Bad CGI code %d\n", connStr, r); conn->cgi=NULL; //mark for destruction. } xmitSendBuff(conn); } -static const char *httpNotFoundHeader="HTTP/1.0 404 Not Found\r\nServer: esp8266-httpd/"HTTPDVER"\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nNot Found.\r\n"; +static const char *httpNotFoundHeader="HTTP/1.0 404 Not Found\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nNot Found.\r\n"; //This is called when the headers have been received and the connection is ready to send //the result headers and data. @@ -419,7 +419,7 @@ static void ICACHE_FLASH_ATTR httpdParseHeader(char *h, HttpdConnData *conn) { if (conn->getArgs!=0) { *conn->getArgs=0; conn->getArgs++; - os_printf("%s GET args = %s\n", connStr, conn->getArgs); + os_printf("%s args = %s\n", connStr, conn->getArgs); } else { conn->getArgs=NULL; } diff --git a/serial/serbridge.c b/serial/serbridge.c index 528c486..ca6e0af 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -263,9 +263,13 @@ enum { TC_idle, // in-between commands TC_newline, // newline seen TC_start, // start character (~) seen - TC_cmd, // command character (0) seen + TC_cmd, // command start (@) seen + TC_cmdChar, // command character seen TC_cmdLine, // accumulating command - TC_tdata, // data character (1-9) seen, and in text data mode + TC_tdchan, // saw data channel character + TC_tdlen1, // saw first data length character + TC_tdata0, // accumulate data, zero-terminated + TC_tdataN, // accumulate data, length-terminated }; static uint8_t tcState = TC_newline; static uint8_t tcChan; // channel for current command (index into tcConn) @@ -274,6 +278,7 @@ static uint8_t tcChan; // channel for current command (index into tcConn) static char tcCmdBuf[CMD_MAX]; static short tcCmdBufLen = 0; static char tcCmdChar; +static short tcLen; // scan a buffer for tcp client commands static int ICACHE_FLASH_ATTR @@ -291,17 +296,27 @@ tcpClientProcess(char *buf, int len) if (c == '~') tcState = TC_start; continue; // gobble up the ~ case TC_start: // saw ~, expect channel number - if (c == '0') { + if (c == '@') { tcState = TC_cmd; continue; - } else if (c >= '1' && c <= '9') { - tcChan = c-'1'; - tcState = TC_tdata; + } else if (c >= '0' && c <= '9') { + tcChan = c-'0'; + tcState = TC_tdchan; continue; } *out++ = '~'; // make up for '~' we skipped break; - case TC_cmd: // saw channel number 0 (command), expect command char + case TC_cmd: // saw control char (@), expect channel char + if (c >= '0' && c <= '9') { + tcChan = c-'0'; + tcState = TC_cmdChar; + continue; + } else { + *out++ = '~'; // make up for '~' we skipped + *out++ = '@'; // make up for '@' we skipped + break; + } + case TC_cmdChar: // saw channel number, expect command char tcCmdChar = c; // save command character tcCmdBufLen = 0; // empty the command buffer tcState = TC_cmdLine; @@ -310,11 +325,36 @@ tcpClientProcess(char *buf, int len) if (c != '\n') { if (tcCmdBufLen < CMD_MAX) tcCmdBuf[tcCmdBufLen++] = c; } else { - tcpClientCommand(tcCmdChar, tcCmdBuf); + tcpClientCommand(tcChan, tcCmdChar, tcCmdBuf); tcState = TC_newline; } continue; - case TC_tdata: // saw channel number, getting data characters + case TC_tdchan: // saw channel number, getting first length char + if (c >= '0' && c <= '9') { + tcLen = c-'0'; + } else if (c >= 'A' && c <= 'F') { + tcLen = c-'A'+10; + } else { + *out++ = '~'; // make up for '~' we skipped + *out++ = '0'+tcChan; + break; + } + tcState = TC_tdlen1; + continue; + case TC_tdlen1: // saw first length char, get second + tcLen *= 16; + if (c >= '0' && c <= '9') { + tcLen += c-'0'; + } else if (c >= 'A' && c <= 'F') { + tcLen += c-'A'+10; + } else { + *out++ = '~'; // make up for '~' we skipped + *out++ = '0'+tcChan; + break; + } + tcState = tcLen == 0 ? TC_tdata0 : TC_tdataN; + continue; + case TC_tdata0: // saw data length, getting data characters zero-terminated if (c != 0) { tcpClientSendChar(tcChan, c); } else { @@ -322,6 +362,14 @@ tcpClientProcess(char *buf, int len) tcState = TC_idle; } continue; + case TC_tdataN: // saw data length, getting data characters length-terminated + tcpClientSendChar(tcChan, c); + tcLen--; + if (tcLen == 0) { + tcpClientSendPush(tcChan); + tcState = TC_idle; + } + continue; } *out++ = c; } diff --git a/serial/tcpclient.c b/serial/tcpclient.c index 9580b18..3515459 100644 --- a/serial/tcpclient.c +++ b/serial/tcpclient.c @@ -1,4 +1,8 @@ // Copyright 2015 by Thorsten von Eicken, see LICENSE.txt +// +// TCP client library allowing uControllers attached to the serial port to send commands +// to open/close TCP connections and send/recv data. +// The serial protocol is described in https://gist.github.com/tve/a46c44bf1f6b42bc572e #include #include "config.h" @@ -7,7 +11,7 @@ #include "tcpclient.h" // max number of channels the client can open -#define MAX_CHAN 8 +#define MAX_CHAN MAX_TCP_CHAN // size of tx buffer #define MAX_TXBUF 1024 @@ -28,8 +32,54 @@ typedef struct { enum TcpState state; } TcpConn; -#define MAX_CONN (8) -static TcpConn tcpConn[MAX_CONN]; +static TcpConn tcpConn[MAX_CHAN]; + +// forward declarations +static void tcpConnFree(TcpConn* tci); +static TcpConn* tcpConnAlloc(uint8_t chan); +static void tcpDoSend(TcpConn *tci); +static void tcpConnectCb(void *arg); +static void tcpDisconCb(void *arg); +static void tcpResetCb(void *arg, sint8 err); +static void tcpSentCb(void *arg); +static void tcpRecvCb(void *arg, char *data, uint16_t len); + +//===== allocate / free connections + +// Allocate a new connection dynamically and return it. Returns NULL if buf alloc failed +static TcpConn* ICACHE_FLASH_ATTR +tcpConnAlloc(uint8_t chan) { + TcpConn *tci = tcpConn+chan; + if (tci->state != TCP_idle && tci->conn != NULL) return tci; + + // malloc and return espconn struct + tci->conn = os_malloc(sizeof(struct espconn)); + if (tci->conn == NULL) goto fail; + memset(tci->conn, 0, sizeof(struct espconn)); + // malloc esp_tcp struct + tci->tcp = os_malloc(sizeof(esp_tcp)); + if (tci->tcp == NULL) goto fail; + memset(tci->tcp, 0, sizeof(esp_tcp)); + + // common init + tci->state = TCP_dns; + tci->conn->type = ESPCONN_TCP; + tci->conn->state = ESPCONN_NONE; + tci->conn->proto.tcp = tci->tcp; + tci->tcp->remote_port = 80; + espconn_regist_connectcb(tci->conn, tcpConnectCb); + espconn_regist_reconcb(tci->conn, tcpResetCb); + espconn_regist_sentcb(tci->conn, tcpSentCb); + espconn_regist_recvcb(tci->conn, tcpRecvCb); + espconn_regist_disconcb(tci->conn, tcpDisconCb); + tci->conn->reverse = tci; + + return tci; + +fail: + tcpConnFree(tci); + return NULL; +} // Free a connection dynamically. static void ICACHE_FLASH_ATTR @@ -41,6 +91,8 @@ tcpConnFree(TcpConn* tci) { memset(tci, 0, sizeof(TcpConn)); } +//===== DNS + // DNS name resolution callback static void ICACHE_FLASH_ATTR tcpClientHostnameCb(const char *name, ip_addr_t *ipaddr, void *arg) { @@ -66,12 +118,59 @@ tcpClientHostnameCb(const char *name, ip_addr_t *ipaddr, void *arg) { tcpConnFree(tci); } +//===== Connect / disconnect + +// Connected callback +static void ICACHE_FLASH_ATTR +tcpConnectCb(void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP connect CB (%p %p)\n", arg, tci); + tci->state = TCP_data; + // send any buffered data + if (tci->txBuf != NULL && tci->txBufLen > 0) tcpDoSend(tci); + // reply to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dC\n", tci-tcpConn); + uart0_tx_buffer(buf, l); +} + +// Disconnect callback +static void ICACHE_FLASH_ATTR tcpDisconCb(void *arg) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP disconnect CB (%p %p)\n", arg, tci); + // notify to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); + uart0_tx_buffer(buf, l); + // free + tcpConnFree(tci); +} + +// Connection reset callback +static void ICACHE_FLASH_ATTR tcpResetCb(void *arg, sint8 err) { + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP reset CB (%p %p) err=%d\n", arg, tci, err); + // notify to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); + uart0_tx_buffer(buf, l); + // free + tcpConnFree(tci); +} + +//===== Sending and receiving + // Send the next buffer (assumes that the connection is in a state that allows it) -static void tcpDoSend(TcpConn *tci) { +static void ICACHE_FLASH_ATTR +tcpDoSend(TcpConn *tci) { sint8 err = espconn_sent(tci->conn, (uint8*)tci->txBuf, tci->txBufLen); if (err == ESPCONN_OK) { // send successful os_printf("TCP sent (%p %p)\n", tci->conn, tci); + tci->txBuf[tci->txBufLen] = 0; os_printf("TCP data: %s\n", tci->txBuf); tci->txBufSent = tci->txBuf; tci->txBuf = NULL; tci->txBufLen = 0; @@ -81,18 +180,9 @@ static void tcpDoSend(TcpConn *tci) { } } -// Connected callback -static void ICACHE_FLASH_ATTR tcpConnectCb(void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP connect CB (%p %p)\n", arg, tci); - tci->state = TCP_data; - // send any buffered data - if (tci->txBuf != NULL && tci->txBufLen > 0) tcpDoSend(tci); -} - // Sent callback -static void ICACHE_FLASH_ATTR tcpSentCb(void *arg) { +static void ICACHE_FLASH_ATTR +tcpSentCb(void *arg) { struct espconn *conn = arg; TcpConn *tci = conn->reverse; os_printf("TCP sent CB (%p %p)\n", arg, tci); @@ -111,84 +201,71 @@ static void ICACHE_FLASH_ATTR tcpRecvCb(void *arg, char *data, uint16_t len) { TcpConn *tci = conn->reverse; os_printf("TCP recv CB (%p %p)\n", arg, tci); if (tci->state == TCP_data) { + uint8_t chan; + for (chan=0; chan= MAX_CHAN) return; // oops!? + char buf[6]; + short l = os_sprintf(buf, "\n~%d", chan); + uart0_tx_buffer(buf, l); uart0_tx_buffer(data, len); + uart0_tx_buffer("\0\n", 2); } serledFlash(50); // short blink on serial LED } -// Disconnect callback -static void ICACHE_FLASH_ATTR tcpDisconCb(void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP disconnect CB (%p %p)\n", arg, tci); - tcpConnFree(tci); -} - -// Connection reset callback -static void ICACHE_FLASH_ATTR tcpResetCb(void *arg, sint8 err) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP reset CB (%p %p) err=%d\n", arg, tci, err); - tcpConnFree(tci); -} +void ICACHE_FLASH_ATTR +tcpClientSendChar(uint8_t chan, char c) { + TcpConn *tci = tcpConn+chan; + if (tci->state == TCP_idle) return; -// Allocate a new connection dynamically and return it. Returns NULL if no -// connection could be allocated. -static TcpConn* ICACHE_FLASH_ATTR -tcpConnAlloc(void) { - int i; - for (i=0; itxBuf != NULL) { + // we have a buffer + if (tci->txBufLen < MAX_TXBUF) { + // buffer has space, add char and return + tci->txBuf[tci->txBufLen++] = c; + return; + } else if (tci->txBufSent == NULL) { + // we don't have a send pending, send full buffer off + if (tci->state == TCP_data) tcpDoSend(tci); + if (tci->txBuf != NULL) return; // something went wrong + } else { + // buffers all backed-up, drop char + return; + } } - if (i == MAX_CONN) return NULL; - - // found an empty slot, malloc and return espconn struct - TcpConn *tci = tcpConn+i; - tci->conn = os_malloc(sizeof(struct espconn)); - if (tci->conn == NULL) goto fail; - memset(tci->conn, 0, sizeof(struct espconn)); - // malloc esp_tcp struct - tci->tcp = os_malloc(sizeof(esp_tcp)); - if (tci->tcp == NULL) goto fail; - memset(tci->tcp, 0, sizeof(esp_tcp)); - - // common init - tci->state = TCP_dns; - tci->conn->type = ESPCONN_TCP; - tci->conn->state = ESPCONN_NONE; - tci->conn->proto.tcp = tci->tcp; - tci->tcp->remote_port = 80; - tci->tcp->remote_ip[0] = 173; - espconn_regist_connectcb(tci->conn, tcpConnectCb); - espconn_regist_reconcb(tci->conn, tcpResetCb); - espconn_regist_sentcb(tci->conn, tcpSentCb); - espconn_regist_recvcb(tci->conn, tcpRecvCb); - espconn_regist_disconcb(tci->conn, tcpDisconCb); - tci->conn->reverse = tci; - - return tci; + // we do not have a buffer (either didn't have one or sent it off) + // allocate one + tci->txBuf = os_malloc(MAX_TXBUF); + tci->txBufLen = 0; + if (tci->txBuf != NULL) { + tci->txBuf[tci->txBufLen++] = c; + } +} -fail: - tcpConnFree(tci); - return NULL; +void ICACHE_FLASH_ATTR +tcpClientSendPush(uint8_t chan) { + TcpConn *tci = tcpConn+chan; + if (tci->state != TCP_data) return; // no active connection on this channel + if (tci->txBuf == NULL || tci->txBufLen == 0) return; // no chars accumulated to send + if (tci->txBufSent != NULL) return; // already got a send in progress + tcpDoSend(tci); } -static TcpConn *tcConn[MAX_CHAN]; +//===== Command parsing // Perform a TCP command: parse the command and do the right thing. // Returns true on success. bool ICACHE_FLASH_ATTR -tcpClientCommand(char cmd, char *cmdBuf) { - uint8_t tcChan; +tcpClientCommand(uint8_t chan, char cmd, char *cmdBuf) { TcpConn *tci; + char *hostname; + char *port; switch (cmd) { //== TCP Connect command case 'T': - if (*cmdBuf < '1' || *cmdBuf > ('0'+MAX_CHAN)) break; - tcChan = *cmdBuf++ - '1'; - char *hostname = cmdBuf; - char *port = hostname; + hostname = cmdBuf; + port = hostname; while (*port != 0 && *port != ':') port++; if (*port != ':') break; *port = 0; @@ -197,13 +274,13 @@ tcpClientCommand(char cmd, char *cmdBuf) { if (portInt < 1 || portInt > 65535) break; // allocate a connection - tci = tcpConnAlloc(); + tci = tcpConnAlloc(chan); if (tci == NULL) break; tci->state = TCP_dns; tci->tcp->remote_port = portInt; // start the DNS resolution - os_printf("TCP %p resolving %s (conn=%p)\n", tci, hostname, tci->conn); + os_printf("TCP %p resolving %s for chan %d (conn=%p)\n", tci, hostname, chan ,tci->conn); ip_addr_t ip; err_t err = espconn_gethostbyname(tci->conn, hostname, &ip, tcpClientHostnameCb); if (err == ESPCONN_OK) { @@ -215,14 +292,12 @@ tcpClientCommand(char cmd, char *cmdBuf) { break; } - tcConn[tcChan] = tci; return true; //== TCP Close/disconnect command case 'C': - if (*cmdBuf < '1' || *cmdBuf > ('0'+MAX_CHAN)) break; - tcChan = *cmdBuf++ - '1'; - tci = tcConn[tcChan]; + os_printf("TCP closing chan %d\n", chan); + tci = tcpConn+chan; if (tci->state > TCP_idle) { tci->state = TCP_idle; // hackish... espconn_disconnect(tci->conn); @@ -233,40 +308,3 @@ tcpClientCommand(char cmd, char *cmdBuf) { return false; } -void ICACHE_FLASH_ATTR -tcpClientSendChar(uint8_t chan, char c) { - TcpConn *tci = tcConn[chan]; - if (tci->state == TCP_idle) return; - - if (tci->txBuf != NULL) { - // we have a buffer - if (tci->txBufLen < MAX_TXBUF) { - // buffer has space, add char and return - tci->txBuf[tci->txBufLen++] = c; - return; - } else if (tci->txBufSent == NULL) { - // we don't have a send pending, send full buffer off - tcpDoSend(tci); - if (tci->txBuf != NULL) return; // something went wrong - } else { - // buffers all backed-up, drop char - return; - } - } - // we do not have a buffer (either didn't have one or sent it off) - // allocate one - tci->txBuf = os_malloc(MAX_TXBUF); - tci->txBufLen = 0; - if (tci->txBuf != NULL) { - tci->txBuf[tci->txBufLen++] = c; - } -} - -void ICACHE_FLASH_ATTR -tcpClientSendPush(uint8_t chan) { - TcpConn *tci = tcConn[chan]; - if (tci->state == TCP_idle) return; // no active connection on this channel - if (tci->txBuf == NULL || tci->txBufLen == 0) return; // no chars accumulated to send - if (tci->txBufSent != NULL) return; // already got a send in progress - tcpDoSend(tci); -} diff --git a/serial/tcpclient.h b/serial/tcpclient.h index c80d2d3..9bfeb01 100644 --- a/serial/tcpclient.h +++ b/serial/tcpclient.h @@ -1,8 +1,16 @@ #ifndef __TCP_CLIENT_H__ #define __TCP_CLIENT_H__ -bool tcpClientCommand(char cmd, char *cmdBuf); +// max number of channels the client can open +#define MAX_TCP_CHAN 8 + +// Parse and perform the commandm cmdBuf must be null-terminated +bool tcpClientCommand(uint8_t chan, char cmd, char *cmdBuf); + +// Append a character to the specified channel void tcpClientSendChar(uint8_t chan, char c); + +// Enqueue the buffered characters for transmission on the specified channel void tcpClientSendPush(uint8_t chan); #endif /* __TCP_CLIENT_H__ */ diff --git a/user/cgipins.c b/user/cgipins.c index cfee33f..459f03e 100644 --- a/user/cgipins.c +++ b/user/cgipins.c @@ -1,3 +1,4 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt #include #include "cgi.h" @@ -93,11 +94,9 @@ int ICACHE_FLASH_ATTR cgiPinsSet(HttpdConnData *connData) { statusInit(); if (configSave()) { - os_printf("New config saved\n"); httpdStartResponse(connData, 200); httpdEndHeaders(connData); } else { - os_printf("*** Failed to save config ***\n"); httpdStartResponse(connData, 500); httpdEndHeaders(connData); httpdSend(connData, "Failed to save config", -1); diff --git a/user/cgitcp.c b/user/cgitcp.c new file mode 100644 index 0000000..f8d9036 --- /dev/null +++ b/user/cgitcp.c @@ -0,0 +1,74 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt +// // TCP Client settings + +#include +#include "cgi.h" +#include "config.h" +#include "cgitcp.h" + +// Cgi to return TCP client settings +int ICACHE_FLASH_ATTR cgiTcpGet(HttpdConnData *connData) { + char buff[1024]; + int len; + + if (connData->conn==NULL) return HTTPD_CGI_DONE; + + len = os_sprintf(buff, "{ \"tcp_enable\":%d, \"rssi_enable\": %d, \"api_key\":\"%s\" }", + flashConfig.tcp_enable, flashConfig.rssi_enable, flashConfig.api_key); + + jsonHeader(connData, 200); + httpdSend(connData, buff, len); + return HTTPD_CGI_DONE; +} + +// Cgi to change choice of pin assignments +int ICACHE_FLASH_ATTR cgiTcpSet(HttpdConnData *connData) { + if (connData->conn==NULL) return HTTPD_CGI_DONE; + + // Handle tcp_enable flag + char buff[128]; + int len = httpdFindArg(connData->getArgs, "tcp_enable", buff, sizeof(buff)); + if (len <= 0) { + jsonHeader(connData, 400); + return HTTPD_CGI_DONE; + } + flashConfig.tcp_enable = os_strcmp(buff, "true") == 0; + + // Handle rssi_enable flag + len = httpdFindArg(connData->getArgs, "rssi_enable", buff, sizeof(buff)); + if (len <= 0) { + jsonHeader(connData, 400); + return HTTPD_CGI_DONE; + } + flashConfig.rssi_enable = os_strcmp(buff, "true") == 0; + + // Handle api_key flag + len = httpdFindArg(connData->getArgs, "api_key", buff, sizeof(buff)); + if (len < 0) { + jsonHeader(connData, 400); + return HTTPD_CGI_DONE; + } + buff[sizeof(flashConfig.api_key)-1] = 0; // ensure we don't get an overrun + os_strcpy(flashConfig.api_key, buff); + + if (configSave()) { + httpdStartResponse(connData, 200); + httpdEndHeaders(connData); + } else { + httpdStartResponse(connData, 500); + httpdEndHeaders(connData); + httpdSend(connData, "Failed to save config", -1); + } + return HTTPD_CGI_DONE; +} + +int ICACHE_FLASH_ATTR cgiTcp(HttpdConnData *connData) { + if (connData->requestType == HTTPD_METHOD_GET) { + return cgiTcpGet(connData); + } else if (connData->requestType == HTTPD_METHOD_POST) { + return cgiTcpSet(connData); + } else { + jsonHeader(connData, 404); + return HTTPD_CGI_DONE; + } +} diff --git a/user/cgitcp.h b/user/cgitcp.h new file mode 100644 index 0000000..2b7b2a9 --- /dev/null +++ b/user/cgitcp.h @@ -0,0 +1,8 @@ +#ifndef CGITCP_H +#define CGITCP_H + +#include "httpd.h" + +int cgiTcp(HttpdConnData *connData); + +#endif diff --git a/user/config.c b/user/config.c index b840c8f..6023f5d 100644 --- a/user/config.c +++ b/user/config.c @@ -17,6 +17,9 @@ FlashConfig flashDefault = { "esp-link\0 ", // hostname 0, 0x00ffffff, 0, // static ip, netmask, gateway 0, // log mode + 0, // swap_uart + 1, 0, // tcp_enable, rssi_enable + "\0", // api_key }; typedef union { @@ -40,7 +43,6 @@ static void memDump(void *addr, int len) { #endif bool ICACHE_FLASH_ATTR configSave(void) { - FlashFull ff; memset(&ff, 0, sizeof(ff)); memcpy(&ff, &flashConfig, sizeof(FlashConfig)); @@ -48,7 +50,7 @@ bool ICACHE_FLASH_ATTR configSave(void) { // erase secondary uint32_t addr = FLASH_ADDR + (1-flash_pri)*FLASH_SECT; if (spi_flash_erase_sector(addr>>12) != SPI_FLASH_RESULT_OK) - return false; // no harm done, give up + goto fail; // no harm done, give up // calculate CRC ff.fc.seq = seq; ff.fc.magic = FLASH_MAGIC; @@ -60,11 +62,11 @@ bool ICACHE_FLASH_ATTR configSave(void) { // write primary with incorrect seq ff.fc.seq = 0xffffffff; if (spi_flash_write(addr, (void *)&ff, sizeof(ff)) != SPI_FLASH_RESULT_OK) - return false; // no harm done, give up + goto fail; // no harm done, give up // fill in correct seq ff.fc.seq = seq; if (spi_flash_write(addr, (void *)&ff, sizeof(uint32_t)) != SPI_FLASH_RESULT_OK) - return false; // most likely failed, but no harm if successful + goto fail; // most likely failed, but no harm if successful // now that we have safely written the new version, erase old primary addr = FLASH_ADDR + flash_pri*FLASH_SECT; flash_pri = 1-flash_pri; @@ -77,6 +79,9 @@ bool ICACHE_FLASH_ATTR configSave(void) { ff.fc.seq = seq; spi_flash_write(addr, (void *)&ff, sizeof(uint32_t)); return true; +fail: + os_printf("*** Failed to save config ***\n"); + return false; } void ICACHE_FLASH_ATTR configWipe(void) { diff --git a/user/config.h b/user/config.h index c326a37..51b0995 100644 --- a/user/config.h +++ b/user/config.h @@ -9,6 +9,9 @@ typedef struct { char hostname[32]; // if using DHCP uint32_t staticip, netmask, gateway; // using DHCP if staticip==0 uint8_t log_mode; // UART log debug mode + uint8_t swap_uart; // swap uart0 to gpio 13&15 + uint8_t tcp_enable, rssi_enable; // TCP client settings + char api_key[48]; // RSSI submission API key (Grovestreams for now) } FlashConfig; extern FlashConfig flashConfig; diff --git a/user/status.c b/user/status.c index 7b46d72..6d7747e 100644 --- a/user/status.c +++ b/user/status.c @@ -4,6 +4,7 @@ #include "config.h" #include "serled.h" #include "cgiwifi.h" +#include "tcpclient.h" //===== "CONN" LED status indication @@ -72,101 +73,44 @@ void ICACHE_FLASH_ATTR statusWifiUpdate(uint8_t state) { static ETSTimer rssiTimer; -static uint8_t rssiSendState = 0; -static sint8 rssiLast = 0; //last RSSI value - -static esp_tcp rssiTcp; -static struct espconn rssiConn; - -#define GS_API_KEY "2eb868a8-224f-3faa-939d-c79bd605912a" -#define GS_COMP_ID "esp-link" #define GS_STREAM "rssi" -// Connected callback -static void ICACHE_FLASH_ATTR rssiConnectCb(void *arg) { - struct espconn *conn = (struct espconn *)arg; - os_printf("RSSI connect CB (%p %p)\n", arg, conn->reverse); +// Timer callback to send an RSSI update to a monitoring system +static void ICACHE_FLASH_ATTR rssiTimerCb(void *v) { + if (!flashConfig.rssi_enable || !flashConfig.tcp_enable || flashConfig.api_key[0]==0) + return; + + sint8 rssi = wifi_station_get_rssi(); + if (rssi >= 0) return; // not connected or other error - char buf[2000]; + // compose TCP command + uint8_t chan = MAX_TCP_CHAN-1; + tcpClientCommand(chan, 'T', "grovestreams.com:80"); - // http header + // compose http header + char buf[1024]; int hdrLen = os_sprintf(buf, "PUT /api/feed?api_key=%s HTTP/1.0\r\n" "Content-Type: application/json\r\n" "Content-Length: XXXXX\r\n\r\n", - GS_API_KEY); + flashConfig.api_key); // http body int dataLen = os_sprintf(buf+hdrLen, - "[{\"compId\":\"%s\", \"streamId\":\"%s\", \"data\":%d}]", - GS_COMP_ID, GS_STREAM, rssiLast); + "[{\"compId\":\"%s\", \"streamId\":\"%s\", \"data\":%d}]\r", + flashConfig.hostname, GS_STREAM, rssi); + buf[hdrLen+dataLen++] = 0; + buf[hdrLen+dataLen++] = '\r'; // hackish way to fill in the content-length os_sprintf(buf+hdrLen-9, "%5d", dataLen); - buf[hdrLen-4] = '\r'; - - // send it off - if (espconn_sent(conn, (uint8*)buf, hdrLen+dataLen) == ESPCONN_OK) { - os_printf("RSSI sent rssi=%d\n", rssiLast); - os_printf("RSSI sent <<%s>>\n", buf); - } -} - -// Sent callback -static void ICACHE_FLASH_ATTR rssiSentCb(void *arg) { - struct espconn *conn = (struct espconn *)arg; - os_printf("RSSI sent CB (%p %p)\n", arg, conn->reverse); -} + buf[hdrLen-4] = '\r'; // fix-up the \0 inserted by sprintf (hack!) -// Recv callback -static void ICACHE_FLASH_ATTR rssiRecvCb(void *arg, char *data, uint16_t len) { - struct espconn *conn = (struct espconn *)arg; - os_printf("RSSI recv CB (%p %p)\n", arg, conn->reverse); - data[len] = 0; // hack!!! - os_printf("GOT %d: <<%s>>\n", len, data); - - espconn_disconnect(conn); -} - -// Disconnect callback -static void ICACHE_FLASH_ATTR rssiDisconCb(void *arg) { - struct espconn *conn = (struct espconn *)arg; - os_printf("RSSI disconnect CB (%p %p)\n", arg, conn->reverse); - rssiSendState = 0; -} - -// Connection reset callback -static void ICACHE_FLASH_ATTR rssiResetCb(void *arg, sint8 err) { - struct espconn *conn = (struct espconn *)arg; - os_printf("RSSI reset CB (%p %p) err=%d\n", arg, conn->reverse, err); - rssiSendState = 0; -} - -// Timer callback to send an RSSI update to a monitoring system -static void ICACHE_FLASH_ATTR rssiTimerCb(void *v) { - sint8 rssi = wifi_station_get_rssi(); - if (rssi >= 0) return; // not connected or other error - rssiLast = rssi; - if (rssiSendState > 0) return; // not done with previous rssi report - - rssiConn.type = ESPCONN_TCP; - rssiConn.state = ESPCONN_NONE; - rssiConn.proto.tcp = &rssiTcp; - rssiTcp.remote_port = 80; - rssiTcp.remote_ip[0] = 173; - rssiTcp.remote_ip[1] = 236; - rssiTcp.remote_ip[2] = 12; - rssiTcp.remote_ip[3] = 163; - espconn_regist_connectcb(&rssiConn, rssiConnectCb); - espconn_regist_reconcb(&rssiConn, rssiResetCb); - espconn_regist_sentcb(&rssiConn, rssiSentCb); - espconn_regist_recvcb(&rssiConn, rssiRecvCb); - espconn_regist_disconcb(&rssiConn, rssiDisconCb); - rssiConn.reverse = (void *)0xdeadf00d; - os_printf("RSSI connect (%p)\n", &rssiConn); - if (espconn_connect(&rssiConn) == ESPCONN_OK) { - rssiSendState++; + // send the request off and forget about it... + for (short i=0; i Date: Sat, 1 Aug 2015 21:55:32 -0700 Subject: [PATCH 06/24] updated readme --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 78d0027..e7898ae 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ ESP-LINK ======== -This firmware implements a transparent bridge between Wifi and serial using an ESP8266 module. -It also provides support for flash-programming Arduino/AVR microcontrollers as well as -LPC800-series and other ARM microcontrollers via the ESP8266. +This firmware connects an attached micro-controller to the internet using a ESP8266 Wifi module. +It implements a number of features: +- transparent bridge between Wifi and serial, useful for debugging or inputting into a uC +- flash-programming attached Arduino/AVR microcontrollers as well as LPC800-series and other + ARM microcontrollers via Wifi +- outbound TCP (and thus HTTP) connections from the attached micro-controller to the internet The firmware includes a tiny HTTP server based on [esphttpd](http://www.esp8266.com/viewforum.php?f=34) @@ -12,6 +15,8 @@ with a simple web interface, many thanks to Jeroen Domburg for making it availab [![Chat at https://gitter.im/jeelabs/esp-link](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/jeelabs/esp-link?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ###[Latest release](https://github.com/jeelabs/esp-link/releases) +Note that the [stable V1.0 release](https://github.com/jeelabs/esp-link/releases/tag/v1.0.0) is +recommended if you do not need the outbound TCP connections and have a 512KB flash chip. Eye Candy --------- @@ -215,6 +220,12 @@ modes are supported that can be set in the web UI (and the mode is saved in flas Note that even if the UART log is always off the bootloader prints to uart0 whenever the esp8266 comes out of reset. This cannot be disabled. +Outbound TCP connections +------------------------ +The attached micro-controller can open outbound TCP connections using a simple +[serial protocol](https://gist.github.com/tve/a46c44bf1f6b42bc572e). +More info and sample code forthcoming... + Contact ------- If you find problems with esp-link, please create a github issue. If you have a question, please From 5fc65bd7ee3ad1f767c3c7907770d656635c2416 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Sat, 1 Aug 2015 23:43:01 -0700 Subject: [PATCH 07/24] add support for wifi-link-12 with swapped uart pins --- Makefile | 2 +- serial/serbridge.c | 13 +++++++++++-- user/cgipins.c | 20 ++++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 32cce5c..99d5617 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- # Pick your flash size: "512KB" or "4MB" -FLASH_SIZE ?= 512KB +FLASH_SIZE ?= 4MB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 diff --git a/serial/serbridge.c b/serial/serbridge.c index 449afbd..c4306f7 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -457,9 +457,18 @@ static void ICACHE_FLASH_ATTR serbridgeConnectCb(void *arg) { void ICACHE_FLASH_ATTR serbridgeInitPins() { mcu_reset_pin = flashConfig.reset_pin; mcu_isp_pin = flashConfig.isp_pin; - os_printf("Serbridge pins: reset=%d isp=%d\n", mcu_reset_pin, mcu_isp_pin); + os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", + mcu_reset_pin, mcu_isp_pin, flashConfig.swap_uart); - if (flashConfig.swap_uart) system_uart_swap(); else system_uart_de_swap(); + if (flashConfig.swap_uart) { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); + system_uart_swap(); + } else { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, 0); + system_uart_de_swap(); + } // set both pins to 1 before turning them on so we don't cause a reset if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); diff --git a/user/cgipins.c b/user/cgipins.c index fd8b554..2d71849 100644 --- a/user/cgipins.c +++ b/user/cgipins.c @@ -24,15 +24,13 @@ static const int num_map_func = sizeof(map_func)/sizeof(char*); // Cgi to return choice of pin assignments int ICACHE_FLASH_ATTR cgiPinsGet(HttpdConnData *connData) { - char buff[1024]; - int len; + if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted - if (connData->conn==NULL) { - return HTTPD_CGI_DONE; // Connection aborted - } + char buff[2048]; + int len; // figure out current mapping - int curr = 99; + int curr = 0; for (int i=0; i= 0) len += os_sprintf(buff+len, " %s:gpio%d", map_func[f], p); - else len += os_sprintf(buff+len, " %s:n/a", map_func[f]); + if (f == 4) + len += os_sprintf(buff+len, " %s:%s", map_func[f], p?"yes":"no"); + else if (p >= 0) + len += os_sprintf(buff+len, " %s:gpio%d", map_func[f], p); + else + len += os_sprintf(buff+len, " %s:n/a", map_func[f]); } len += os_sprintf(buff+len, "\" }"); } @@ -89,6 +92,7 @@ int ICACHE_FLASH_ATTR cgiPinsSet(HttpdConnData *connData) { flashConfig.isp_pin = map[1]; flashConfig.conn_led_pin = map[2]; flashConfig.ser_led_pin = map[3]; + flashConfig.swap_uart = map[4]; serbridgeInitPins(); serledInit(); From 71bc355038d08733f4454645849c95011a213770 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 3 Aug 2015 20:56:39 -0700 Subject: [PATCH 08/24] clear bssid before reassociating --- user/cgiwifi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/user/cgiwifi.c b/user/cgiwifi.c index 7292aa4..0d96e96 100644 --- a/user/cgiwifi.c +++ b/user/cgiwifi.c @@ -261,6 +261,7 @@ static ETSTimer reassTimer; static void ICACHE_FLASH_ATTR reassTimerCb(void *arg) { os_printf("Wifi changing association\n"); wifi_station_disconnect(); + stconf.bssid_set = 0; wifi_station_set_config(&stconf); wifi_station_connect(); // Schedule check From 29100391b1c84d970b2ee4d9709fd56c87d97248 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 3 Aug 2015 22:20:36 -0700 Subject: [PATCH 09/24] fix TCP client request buffer parsing --- serial/tcpclient.c | 7 ++++++- user/status.c | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/serial/tcpclient.c b/serial/tcpclient.c index 3515459..166f533 100644 --- a/serial/tcpclient.c +++ b/serial/tcpclient.c @@ -261,10 +261,15 @@ tcpClientCommand(uint8_t chan, char cmd, char *cmdBuf) { char *hostname; char *port; + // copy the command so we can modify it + char buf[128]; + os_strncpy(buf, cmdBuf, 128); + buf[127] = 0; + switch (cmd) { //== TCP Connect command case 'T': - hostname = cmdBuf; + hostname = buf; port = hostname; while (*port != 0 && *port != ':') port++; if (*port != ':') break; diff --git a/user/status.c b/user/status.c index 6d7747e..50afb85 100644 --- a/user/status.c +++ b/user/status.c @@ -81,6 +81,7 @@ static void ICACHE_FLASH_ATTR rssiTimerCb(void *v) { return; sint8 rssi = wifi_station_get_rssi(); + os_printf("timer rssi=%d\n", rssi); if (rssi >= 0) return; // not connected or other error // compose TCP command @@ -100,7 +101,7 @@ static void ICACHE_FLASH_ATTR rssiTimerCb(void *v) { "[{\"compId\":\"%s\", \"streamId\":\"%s\", \"data\":%d}]\r", flashConfig.hostname, GS_STREAM, rssi); buf[hdrLen+dataLen++] = 0; - buf[hdrLen+dataLen++] = '\r'; + buf[hdrLen+dataLen++] = '\n'; // hackish way to fill in the content-length os_sprintf(buf+hdrLen-9, "%5d", dataLen); From a74177383a2ebb2458ba3a341daba015ed5d6ba3 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Sat, 8 Aug 2015 00:53:55 -0700 Subject: [PATCH 10/24] fix serial bridge problem --- serial/serbridge.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/serial/serbridge.c b/serial/serbridge.c index c4306f7..5c00175 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -26,16 +26,7 @@ serbridgeConnData connData[MAX_CONN]; // Given a pointer to an espconn struct find the connection that correcponds to it static serbridgeConnData ICACHE_FLASH_ATTR *serbridgeFindConnData(void *arg) { struct espconn *conn = arg; - return (serbridgeConnData *)conn->reverse; -#if 0 - for (int i=0; ireverse; } //===== TCP -> UART @@ -293,8 +284,12 @@ tcpClientProcess(char *buf, int len) if (c == '\n') tcState = TC_newline; break; case TC_newline: // saw newline, expect ~ - if (c == '~') tcState = TC_start; - continue; // gobble up the ~ + if (c == '~') { + tcState = TC_start; + continue; // gobble up the ~ + } else { + break; + } case TC_start: // saw ~, expect channel number if (c == '@') { tcState = TC_cmd; @@ -373,7 +368,7 @@ tcpClientProcess(char *buf, int len) } *out++ = c; } - if (tcState != TC_idle) os_printf("tcState=%d\n", tcState); + //if (tcState != TC_idle) os_printf("tcState=%d\n", tcState); return out-buf; } @@ -387,7 +382,7 @@ serbridgeUartCb(char *buf, int length) { length = tcpClientProcess(buf, length); // push the buffer into each open connection if (length > 0) { - for (int i = 0; i < MAX_CONN; ++i) { + for (int i=0; i Date: Wed, 19 Aug 2015 13:39:01 -0700 Subject: [PATCH 11/24] tweak \n handling on console page --- serial/console.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/serial/console.c b/serial/console.c index d38e73a..6fa4374 100644 --- a/serial/console.c +++ b/serial/console.c @@ -34,16 +34,18 @@ console_write(char c) { } } +#if 0 // return previous character in console, 0 if at start static char ICACHE_FLASH_ATTR console_prev(void) { if (console_wr == console_rd) return 0; return console_buf[(console_wr-1+BUF_MAX)%BUF_MAX]; } +#endif void ICACHE_FLASH_ATTR console_write_char(char c) { - if (c == '\n' && console_prev() != '\r') console_write('\r'); + //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good console_write(c); } @@ -112,6 +114,8 @@ ajaxConsole(HttpdConnData *connData) { if (c == '\\' || c == '"') { buff[len++] = '\\'; buff[len++] = c; + } else if (c == '\r') { + // this is crummy, but browsers display a newline for \r\n sequences } else if (c < ' ') { len += os_sprintf(buff+len, "\\u%04x", c); } else { From dd7f9a16a906e0beea00608b919b2969ae327a87 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:13:30 -0700 Subject: [PATCH 12/24] better support for 4MB flash, add 1MB flash --- Makefile | 30 +++++++++++++++++++++++------- user/config.c | 8 +++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 99d5617..df0bd6d 100644 --- a/Makefile +++ b/Makefile @@ -34,21 +34,37 @@ FLASH_SIZE ?= 4MB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 -ESP_SPI_SIZE ?= 0 # 0->512KB -ESP_FLASH_MODE ?= 0 # 0->QIO -ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz -ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +ESP_SPI_SIZE ?= 0 # 0->512KB (256KB+256KB) +ESP_FLASH_MODE ?= 0 # 0->QIO +ESP_FLASH_FREQ_DIV ?= 0 # 0->40Mhz +ESP_FLASH_MAX ?= 241664 # max bin file for 512KB flash: 236KB +ET_FS ?= 4m # 4Mbit flash size in esptool flash command +ET_FF ?= 40m # 40Mhz flash speed in esptool flash command +ET_BLANK ?= 0x7E000 # where to flash blank.bin to erase wireless settings + +else ifeq ("$(FLASH_SIZE)","1MB") +# ESP-01E +ESP_SPI_SIZE ?= 2 # 2->1MB (512KB+512KB) +ESP_FLASH_MODE ?= 0 # 0->QIO +ESP_FLASH_FREQ_DIV ?= 15 # 15->80MHz +ESP_FLASH_MAX ?= 503808 # max bin file for 1MB flash: 492KB +ET_FS ?= 8m # 8Mbit flash size in esptool flash command +ET_FF ?= 80m # 80Mhz flash speed in esptool flash command +ET_BLANK ?= 0xFE000 # where to flash blank.bin to erase wireless settings else # Winbond 25Q32 4MB flash, typ for esp-12 # Here we're using two partitions of approx 0.5MB because that's what's easily available in terms # of linker scripts in the SDK. Ideally we'd use two partitions of approx 1MB, the remaining 2MB -# cannot be used for code. +# cannot be used for code (esp8266 limitation). ESP_SPI_SIZE ?= 4 # 6->4MB (1MB+1MB) or 4->4MB (512KB+512KB) ESP_FLASH_MODE ?= 0 # 0->QIO, 2->DIO ESP_FLASH_FREQ_DIV ?= 15 # 15->80Mhz ESP_FLASH_MAX ?= 503808 # max bin file for 512KB flash partition: 492KB #ESP_FLASH_MAX ?= 1028096 # max bin file for 1MB flash partition: 1004KB +ET_FS ?= 32m # 32Mbit flash size in esptool flash command +ET_FF ?= 80m # 80Mhz flash speed in esptool flash command +ET_BLANK ?= 0x3FE000 # where to flash blank.bin to erase wireless settings endif # hostname or IP address for wifi flashing @@ -274,9 +290,9 @@ wiflash: all ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin flash: all - $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash \ + $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \ 0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \ - 0x7E000 $(SDK_BASE)/bin/blank.bin + $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui diff --git a/user/config.c b/user/config.c index 6023f5d..c44a188 100644 --- a/user/config.c +++ b/user/config.c @@ -27,10 +27,16 @@ typedef union { uint8_t block[128]; } FlashFull; +// magic number to recognize thet these are our flash settings as opposed to some random stuff #define FLASH_MAGIC (0xaa55) -#define FLASH_ADDR (0x3E000) +// size of the setting sector #define FLASH_SECT (4096) + +// address where to flash the settings: there are 16KB of reserved space at the end of the first +// flash partition, we use the upper 8KB (2 sectors) +#define FLASH_ADDR (FLASH_SECT + FIRMWARE_SIZE + 2*FLASH_SECT) + static int flash_pri; // primary flash sector (0 or 1, or -1 for error) #if 0 From a58904f1be34c6b1f76a8b53e23666d93fb9da81 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:13:55 -0700 Subject: [PATCH 13/24] cgiflash cleanup --- user/cgiflash.c | 24 ------------------------ user/config.c | 4 ++-- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/user/cgiflash.c b/user/cgiflash.c index 597b73a..e07443b 100644 --- a/user/cgiflash.c +++ b/user/cgiflash.c @@ -31,30 +31,6 @@ static char* ICACHE_FLASH_ATTR check_header(void *buf) { return NULL; } -#if 0 -//===== Cgi that reads the SPI flash. Assumes 512KByte flash. -int ICACHE_FLASH_ATTR cgiReadFlash(HttpdConnData *connData) { - int *pos=(int *)&connData->cgiData; - if (connData->conn==NULL) { - //Connection aborted. Clean up. - return HTTPD_CGI_DONE; - } - - if (*pos==0) { - os_printf("Start flash download.\n"); - httpdStartResponse(connData, 200); - httpdHeader(connData, "Content-Type", "application/bin"); - httpdEndHeaders(connData); - *pos=0x40200000; - return HTTPD_CGI_MORE; - } - //Send 1K of flash per call. We will get called again if we haven't sent 512K yet. - espconn_sent(connData->conn, (uint8 *)(*pos), 1024); - *pos+=1024; - if (*pos>=0x40200000+(512*1024)) return HTTPD_CGI_DONE; else return HTTPD_CGI_MORE; -} -#endif - //===== Cgi to query which firmware needs to be uploaded next int ICACHE_FLASH_ATTR cgiGetFirmwareNext(HttpdConnData *connData) { if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. diff --git a/user/config.c b/user/config.c index c44a188..c15f2ef 100644 --- a/user/config.c +++ b/user/config.c @@ -95,7 +95,7 @@ void ICACHE_FLASH_ATTR configWipe(void) { spi_flash_erase_sector((FLASH_ADDR+FLASH_SECT)>>12); } -static uint32_t ICACHE_FLASH_ATTR selectPrimary(FlashFull *fc0, FlashFull *fc1); +static int ICACHE_FLASH_ATTR selectPrimary(FlashFull *fc0, FlashFull *fc1); bool ICACHE_FLASH_ATTR configRestore(void) { FlashFull ff0, ff1; @@ -117,7 +117,7 @@ bool ICACHE_FLASH_ATTR configRestore(void) { return true; } -static uint32_t ICACHE_FLASH_ATTR selectPrimary(FlashFull *ff0, FlashFull *ff1) { +static int ICACHE_FLASH_ATTR selectPrimary(FlashFull *ff0, FlashFull *ff1) { // check CRC of ff0 uint16_t crc = ff0->fc.crc; ff0->fc.crc = 0; From bc063ef71dfbd314db2f2c4e661985e2463d24f4 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:14:10 -0700 Subject: [PATCH 14/24] correct reset codes --- user/user_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user/user_main.c b/user/user_main.c index 4b8bdab..54ca59e 100644 --- a/user/user_main.c +++ b/user/user_main.c @@ -113,7 +113,7 @@ void user_rf_pre_init(void) { extern uint32_t _binary_espfs_img_start; static char *rst_codes[] = { - "normal", "wdt reset", "exception", "soft wdt", "restart", "deep sleep", "???", + "normal", "wdt reset", "exception", "soft wdt", "restart", "deep sleep", "external", }; # define VERS_STR_STR(V) #V From aa7eadb382340cef5886aad69d53122e7e966983 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 19:30:59 -0700 Subject: [PATCH 15/24] small flash config improvement plus more comments Conflicts: user/config.c --- user/config.c | 2 +- user/config.h | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/user/config.c b/user/config.c index c15f2ef..835e74f 100644 --- a/user/config.c +++ b/user/config.c @@ -17,7 +17,7 @@ FlashConfig flashDefault = { "esp-link\0 ", // hostname 0, 0x00ffffff, 0, // static ip, netmask, gateway 0, // log mode - 0, // swap_uart + 0, // swap_uart (don't by default) 1, 0, // tcp_enable, rssi_enable "\0", // api_key }; diff --git a/user/config.h b/user/config.h index 51b0995..18284a5 100644 --- a/user/config.h +++ b/user/config.h @@ -1,6 +1,10 @@ #ifndef CONFIG_H #define CONFIG_H +// Flash configuration settings. When adding new items always add them at the end and formulate +// them such that a value of zero is an appropriate default or backwards compatible. Existing +// modules that are upgraded will have zero in the new fields. This ensures that an upgrade does +// not wipe out the old settings. typedef struct { uint32_t seq; // flash write sequence number uint16_t magic, crc; From a2990767e129bfffa008551663f0cc7ff247a570 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Wed, 19 Aug 2015 21:43:08 -0700 Subject: [PATCH 16/24] makefile tweak Conflicts: Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index df0bd6d..8629ddb 100644 --- a/Makefile +++ b/Makefile @@ -29,8 +29,8 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- -# Pick your flash size: "512KB" or "4MB" -FLASH_SIZE ?= 4MB +# Pick your flash size: "512KB", "1MB", or "4MB" +FLASH_SIZE ?= 512KB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 From 5cec16c3fdebed65480954e42426d5e64018f9fb Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Fri, 21 Aug 2015 17:52:40 -0500 Subject: [PATCH 17/24] Added windows build support --- .gitignore | 5 + Makefile | 5 +- esp-link.sln | 22 + esp-link.vcxproj | 175 +++ espfs/espfs.c | 21 +- espfs/heatshrink/LICENSE | 14 + espfs/heatshrink/Makefile | 49 + espfs/heatshrink/README.md | 49 + espfs/heatshrink/dec_sm.dot | 52 + espfs/heatshrink/enc_sm.dot | 51 + espfs/heatshrink/greatest.h | 591 +++++++++++ espfs/heatshrink/heatshrink.c | 446 ++++++++ espfs/heatshrink/heatshrink_common.h | 20 + espfs/heatshrink/heatshrink_config.h | 24 + espfs/heatshrink/heatshrink_decoder.c | 382 +++++++ espfs/heatshrink/heatshrink_decoder.h | 101 ++ espfs/heatshrink/heatshrink_encoder.c | 650 ++++++++++++ espfs/heatshrink/heatshrink_encoder.h | 109 ++ espfs/heatshrink/test_heatshrink_dynamic.c | 999 ++++++++++++++++++ .../test_heatshrink_dynamic_theft.c | 521 +++++++++ espfs/heatshrink/test_heatshrink_static.c | 167 +++ espfs/heatshrink_decoder.c | 2 +- espfs/mkespfsimage/Makefile | 36 +- espfs/mkespfsimage/Makefile.linux | 24 + espfs/mkespfsimage/Makefile.windows | 33 + espfs/mkespfsimage/build-linux.sh | 4 + espfs/mkespfsimage/build-win32.sh | 7 + espfs/mkespfsimage/espfsformat.h | 33 + espfs/mkespfsimage/heatshrink_encoder.c | 2 +- espfs/mkespfsimage/main.c | 35 +- espfs/mkespfsimage/mman-win32/Makefile | 48 + espfs/mkespfsimage/mman-win32/config.mak | 11 + espfs/mkespfsimage/mman-win32/configure | 157 +++ espfs/mkespfsimage/mman-win32/mman.c | 180 ++++ espfs/mkespfsimage/mman-win32/mman.h | 55 + espfs/mkespfsimage/mman-win32/test.c | 235 ++++ espmake.cmd | 7 + 37 files changed, 5285 insertions(+), 37 deletions(-) create mode 100644 esp-link.sln create mode 100644 esp-link.vcxproj create mode 100644 espfs/heatshrink/LICENSE create mode 100644 espfs/heatshrink/Makefile create mode 100644 espfs/heatshrink/README.md create mode 100644 espfs/heatshrink/dec_sm.dot create mode 100644 espfs/heatshrink/enc_sm.dot create mode 100644 espfs/heatshrink/greatest.h create mode 100644 espfs/heatshrink/heatshrink.c create mode 100644 espfs/heatshrink/heatshrink_common.h create mode 100644 espfs/heatshrink/heatshrink_config.h create mode 100644 espfs/heatshrink/heatshrink_decoder.c create mode 100644 espfs/heatshrink/heatshrink_decoder.h create mode 100644 espfs/heatshrink/heatshrink_encoder.c create mode 100644 espfs/heatshrink/heatshrink_encoder.h create mode 100644 espfs/heatshrink/test_heatshrink_dynamic.c create mode 100644 espfs/heatshrink/test_heatshrink_dynamic_theft.c create mode 100644 espfs/heatshrink/test_heatshrink_static.c create mode 100644 espfs/mkespfsimage/Makefile.linux create mode 100644 espfs/mkespfsimage/Makefile.windows create mode 100644 espfs/mkespfsimage/build-linux.sh create mode 100644 espfs/mkespfsimage/build-win32.sh create mode 100644 espfs/mkespfsimage/espfsformat.h create mode 100644 espfs/mkespfsimage/mman-win32/Makefile create mode 100644 espfs/mkespfsimage/mman-win32/config.mak create mode 100644 espfs/mkespfsimage/mman-win32/configure create mode 100644 espfs/mkespfsimage/mman-win32/mman.c create mode 100644 espfs/mkespfsimage/mman-win32/mman.h create mode 100644 espfs/mkespfsimage/mman-win32/test.c create mode 100644 espmake.cmd diff --git a/.gitignore b/.gitignore index 7b7ee90..899b7c3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ html_compressed/ esp-link.tgz tve-patch/ yui +espfs/mkespfsimage/mman-win32/mman.o +esp-link.opensdf +esp-link.sdf +espfs/mkespfsimage/mman-win32/libmman.a +.localhistory/ diff --git a/Makefile b/Makefile index 709a0a1..297e4fe 100644 --- a/Makefile +++ b/Makefile @@ -289,6 +289,9 @@ $(BUILD_DIR): wiflash: all ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin +baseflash: all + $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash 0x01000 $(FW_BASE)/user1.bin + flash: all $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \ 0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \ @@ -296,7 +299,7 @@ flash: all yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui - cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) ifeq ("$(COMPRESS_W_YUI)","yes") $(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR) diff --git a/esp-link.sln b/esp-link.sln new file mode 100644 index 0000000..fb7b468 --- /dev/null +++ b/esp-link.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 2013 +VisualStudioVersion = 12.0.31101.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "esp-link", "esp-link.vcxproj", "{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|ARM = Debug|ARM + Release|ARM = Release|ARM + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.ActiveCfg = Debug|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.Build.0 = Debug|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.ActiveCfg = Release|ARM + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.Build.0 = Release|ARM + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/esp-link.vcxproj b/esp-link.vcxproj new file mode 100644 index 0000000..45dd2ae --- /dev/null +++ b/esp-link.vcxproj @@ -0,0 +1,175 @@ + + + + + Debug + ARM + + + Release + ARM + + + + {A92F0CAA-F89B-4F78-AD2A-A042429BD87F} + MakeFileProj + + + + Makefile + + + + + + + + + + + __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ + .\mqtt\include;.\serial;.\user;.\espfs;.\httpd;.\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include;c:\Espressif\sdk\include + + + + + + + bin + build + + + espmake flash + espmake clean all + espmake clean + + + espmake baseflash + espmake clean all + espmake clean + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/espfs/espfs.c b/espfs/espfs.c index 3da6692..1f503d5 100644 --- a/espfs/espfs.c +++ b/espfs/espfs.c @@ -7,9 +7,9 @@ It's written for use with httpd, but doesn't need to be used as such. /* * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): - * Jeroen Domburg wrote this file. As long as you retain - * this notice you can do whatever you want with this stuff. If we meet some day, - * and you think this stuff is worth it, you can buy me a beer in return. + * Jeroen Domburg wrote this file. As long as you retain + * this notice you can do whatever you want with this stuff. If we meet some day, + * and you think this stuff is worth it, you can buy me a beer in return. * ---------------------------------------------------------------------------- */ @@ -42,11 +42,12 @@ It's written for use with httpd, but doesn't need to be used as such. #ifdef ESPFS_HEATSHRINK #include "heatshrink_config_custom.h" -#include "heatshrink_decoder.h" +#include "heatshrink/heatshrink_decoder.h" #endif static char* espFsData = NULL; + struct EspFsFile { EspFsHeader *header; char decompressor; @@ -146,19 +147,19 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { return NULL; } if (h.flags&FLAG_LASTFILE) { - //os_printf("End of image.\n"); + os_printf("End of image.\n"); return NULL; } //Grab the name of the file. - p+=sizeof(EspFsHeader); + p+=sizeof(EspFsHeader); os_memcpy(namebuf, p, sizeof(namebuf)); -// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", +// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", // namebuf, (unsigned int)h.nameLen, (unsigned int)h.fileLenComp, h.compression, h.flags); if (os_strcmp(namebuf, fileName)==0) { //Yay, this is the file we need! p+=h.nameLen; //Skip to content. r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem - //os_printf("Alloc %p[%d]\n", r, sizeof(EspFsFile)); +// os_printf("Alloc %p\n", r); if (r==NULL) return NULL; r->header=(EspFsHeader *)hpos; r->decompressor=h.compression; @@ -175,7 +176,7 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { //Decoder params are stored in 1st byte. memcpyAligned(&parm, r->posComp, 1); r->posComp++; - //os_printf("Heatshrink compressed file; decode parms = %x\n", parm); + os_printf("Heatshrink compressed file; decode parms = %x\n", parm); dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf); r->decompData=dec; #endif @@ -265,7 +266,7 @@ void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) { // os_printf("Freed %p\n", dec); } #endif - //os_printf("Freed %p\n", fh); +// os_printf("Freed %p\n", fh); os_free(fh); } diff --git a/espfs/heatshrink/LICENSE b/espfs/heatshrink/LICENSE new file mode 100644 index 0000000..31ec3df --- /dev/null +++ b/espfs/heatshrink/LICENSE @@ -0,0 +1,14 @@ +Copyright (c) 2013, Scott Vokes +All rights reserved. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/espfs/heatshrink/Makefile b/espfs/heatshrink/Makefile new file mode 100644 index 0000000..d8e6875 --- /dev/null +++ b/espfs/heatshrink/Makefile @@ -0,0 +1,49 @@ +PROJECT = heatshrink +#OPTIMIZE = -O0 +#OPTIMIZE = -Os +OPTIMIZE = -O3 +WARN = -Wall -Wextra -pedantic #-Werror +CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE} +CFLAGS += -Wmissing-prototypes +CFLAGS += -Wstrict-prototypes +CFLAGS += -Wmissing-declarations + +# If libtheft is available, build additional property-based tests. +# Uncomment these to use it in test_heatshrink_dynamic. +#CFLAGS += -DHEATSHRINK_HAS_THEFT +#LDFLAGS += -ltheft + +all: + @echo "For tests, make test_heatshrink_dynamic (default) or change the" + @echo "config.h to disable static memory and build test_heatshrink_static." + @echo "For the standalone command-line tool, make heatshrink." + +${PROJECT}: heatshrink.c + +OBJS= heatshrink_encoder.o \ + heatshrink_decoder.o \ + +heatshrink: ${OBJS} +test_heatshrink_dynamic: ${OBJS} test_heatshrink_dynamic_theft.o +test_heatshrink_static: ${OBJS} + +*.o: Makefile heatshrink_config.h + +heatshrink_decoder.o: heatshrink_decoder.h heatshrink_common.h +heatshrink_encoder.o: heatshrink_encoder.h heatshrink_common.h + +tags: TAGS + +TAGS: + etags *.[ch] + +diagrams: dec_sm.png enc_sm.png + +dec_sm.png: dec_sm.dot + dot -o $@ -Tpng $< + +enc_sm.png: enc_sm.dot + dot -o $@ -Tpng $< + +clean: + rm -f ${PROJECT} test_heatshrink_{dynamic,static} *.o *.core {dec,enc}_sm.png TAGS diff --git a/espfs/heatshrink/README.md b/espfs/heatshrink/README.md new file mode 100644 index 0000000..ab150ee --- /dev/null +++ b/espfs/heatshrink/README.md @@ -0,0 +1,49 @@ +# heatshrink + +A data compression/decompression library for embedded/real-time systems. + +## Key Features: + +- **Low memory usage (as low as 50 bytes)** + It is useful for some cases with less than 50 bytes, and useful + for many general cases with < 300 bytes. +- **Incremental, bounded CPU use** + You can chew on input data in arbitrarily tiny bites. + This is a useful property in hard real-time environments. +- **Can use either static or dynamic memory allocation** + The library doesn't impose any constraints on memory management. +- **ISC license** + You can use it freely, even for commercial purposes. + +## Getting Started: + +There is a standalone command-line program, `heatshrink`, but the +encoder and decoder can also be used as libraries, independent of each +other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and +either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their +respective header) into your project. + +Dynamic allocation is used by default, but in an embedded context, you +probably want to statically allocate the encoder/decoder. Set +`HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`. + +## More Information and Benchmarks: + +heatshrink is based on [LZSS], since it's particularly suitable for +compression in small amounts of memory. It can use an optional, small +[index] to make compression significantly faster, but otherwise can run +in under 100 bytes of memory. The index currently adds 2^(window size+1) +bytes to memory usage for compression, and temporarily allocates 512 +bytes on the stack during index construction. + +For more information, see the [blog post] for an overview, and the +`heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API +documentation. + +[blog post]: http://spin.atomicobject.com/2013/03/14/heatshrink-embedded-data-compression/ +[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/ +[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski + +## Build Status + + [![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink) diff --git a/espfs/heatshrink/dec_sm.dot b/espfs/heatshrink/dec_sm.dot new file mode 100644 index 0000000..470012f --- /dev/null +++ b/espfs/heatshrink/dec_sm.dot @@ -0,0 +1,52 @@ +digraph { + graph [label="Decoder state machine", labelloc="t"] + Start [style="invis", shape="point"] + empty + input_available + yield_literal + backref_index_msb + backref_index_lsb + backref_count_msb + backref_count_lsb + yield_backref + check_for_more_input + done [peripheries=2] + + empty->input_available [label="sink()", color="blue", weight=10] + Start->empty + + input_available->yield_literal [label="pop 1-bit"] + input_available->backref_index_msb [label="pop 0-bit", weight=10] + input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10] + + yield_literal->yield_literal [label="sink()", color="blue"] + yield_literal->yield_literal [label="poll()", color="red"] + yield_literal->check_for_more_input [label="poll(), done", color="red"] + + backref_index_msb->backref_index_msb [label="sink()", color="blue"] + backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10] + backref_index_msb->done [label="finish()", color="blue"] + + backref_index_lsb->backref_index_lsb [label="sink()", color="blue"] + backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10] + backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10] + backref_index_lsb->done [label="finish()", color="blue"] + + backref_count_msb->backref_count_msb [label="sink()", color="blue"] + backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10] + backref_count_msb->done [label="finish()", color="blue"] + + backref_count_lsb->backref_count_lsb [label="sink()", color="blue"] + backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10] + backref_count_lsb->done [label="finish()", color="blue"] + + yield_backref->yield_backref [label="sink()", color="blue"] + yield_backref->yield_backref [label="poll()", color="red"] + yield_backref->check_for_more_input [label="poll(), done", + color="red", weight=10] + + check_for_more_input->empty [label="no"] + check_for_more_input->input_available [label="yes"] + + empty->done [label="finish()", color="blue"] +} diff --git a/espfs/heatshrink/enc_sm.dot b/espfs/heatshrink/enc_sm.dot new file mode 100644 index 0000000..6d3030f --- /dev/null +++ b/espfs/heatshrink/enc_sm.dot @@ -0,0 +1,51 @@ +digraph { + graph [label="Encoder state machine", labelloc="t"] + start [style="invis", shape="point"] + not_full + filled + search + yield_tag_bit + yield_literal + yield_br_length + yield_br_index + save_backlog + flush_bits + done [peripheries=2] + + start->not_full [label="start"] + + not_full->not_full [label="sink(), not full", color="blue"] + not_full->filled [label="sink(), buffer is full", color="blue"] + not_full->filled [label="finish(), set is_finished", color="blue"] + + filled->search [label="indexing (if any)"] + + search->search [label="step"] + search->yield_tag_bit [label="literal"] + search->yield_tag_bit [label="match found"] + search->save_backlog [label="input exhausted"] + + yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"] + yield_tag_bit->yield_literal [label="poll(), literal", color="red"] + yield_tag_bit->yield_br_index [label="poll(), no literal", color="red"] + yield_tag_bit->flush_bits [label="finishing, no literal"] + + yield_literal->yield_literal [label="poll(), full buf", color="red"] + yield_literal->search [label="poll(), no match", color="red"] + yield_literal->yield_tag_bit [label="poll(), match", color="red"] + yield_literal->flush_bits [label="poll(), final literal", color="red"] + + yield_br_index->yield_br_index [label="poll(), full buf", color="red"] + yield_br_index->yield_br_length [label="poll()", color="red"] + + yield_br_length->yield_br_length [label="poll(), full buf", color="red"] + yield_br_length->search [label="done"] + + save_backlog->flush_bits [label="finishing, no literal"] + save_backlog->yield_tag_bit [label="finishing, literal"] + save_backlog->not_full [label="expect more input"] + + flush_bits->flush_bits [label="poll(), full buf", color="red"] + flush_bits->done [label="poll(), flushed", color="red"] + flush_bits->done [label="no more output"] +} diff --git a/espfs/heatshrink/greatest.h b/espfs/heatshrink/greatest.h new file mode 100644 index 0000000..a92c642 --- /dev/null +++ b/espfs/heatshrink/greatest.h @@ -0,0 +1,591 @@ +/* + * Copyright (c) 2011 Scott Vokes + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef GREATEST_H +#define GREATEST_H + +#define GREATEST_VERSION_MAJOR 0 +#define GREATEST_VERSION_MINOR 9 +#define GREATEST_VERSION_PATCH 3 + +/* A unit testing system for C, contained in 1 file. + * It doesn't use dynamic allocation or depend on anything + * beyond ANSI C89. */ + + +/********************************************************************* + * Minimal test runner template + *********************************************************************/ +#if 0 + +#include "greatest.h" + +TEST foo_should_foo() { + PASS(); +} + +static void setup_cb(void *data) { + printf("setup callback for each test case\n"); +} + +static void teardown_cb(void *data) { + printf("teardown callback for each test case\n"); +} + +SUITE(suite) { + /* Optional setup/teardown callbacks which will be run before/after + * every test case in the suite. + * Cleared when the suite finishes. */ + SET_SETUP(setup_cb, voidp_to_callback_data); + SET_TEARDOWN(teardown_cb, voidp_to_callback_data); + + RUN_TEST(foo_should_foo); +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + RUN_SUITE(suite); + GREATEST_MAIN_END(); /* display results */ +} + +#endif +/*********************************************************************/ + + +#include +#include +#include +#include + + +/*********** + * Options * + ***********/ + +/* Default column width for non-verbose output. */ +#ifndef GREATEST_DEFAULT_WIDTH +#define GREATEST_DEFAULT_WIDTH 72 +#endif + +/* FILE *, for test logging. */ +#ifndef GREATEST_STDOUT +#define GREATEST_STDOUT stdout +#endif + +/* Remove GREATEST_ prefix from most commonly used symbols? */ +#ifndef GREATEST_USE_ABBREVS +#define GREATEST_USE_ABBREVS 1 +#endif + + +/********* + * Types * + *********/ + +/* Info for the current running suite. */ +typedef struct greatest_suite_info { + unsigned int tests_run; + unsigned int passed; + unsigned int failed; + unsigned int skipped; + + /* timers, pre/post running suite and individual tests */ + clock_t pre_suite; + clock_t post_suite; + clock_t pre_test; + clock_t post_test; +} greatest_suite_info; + +/* Type for a suite function. */ +typedef void (greatest_suite_cb)(void); + +/* Types for setup/teardown callbacks. If non-NULL, these will be run + * and passed the pointer to their additional data. */ +typedef void (greatest_setup_cb)(void *udata); +typedef void (greatest_teardown_cb)(void *udata); + +typedef enum { + GREATEST_FLAG_VERBOSE = 0x01, + GREATEST_FLAG_FIRST_FAIL = 0x02, + GREATEST_FLAG_LIST_ONLY = 0x04 +} GREATEST_FLAG; + +typedef struct greatest_run_info { + unsigned int flags; + unsigned int tests_run; /* total test count */ + + /* Overall pass/fail/skip counts. */ + unsigned int passed; + unsigned int failed; + unsigned int skipped; + + /* currently running test suite */ + greatest_suite_info suite; + + /* info to print about the most recent failure */ + const char *fail_file; + unsigned int fail_line; + const char *msg; + + /* current setup/teardown hooks and userdata */ + greatest_setup_cb *setup; + void *setup_udata; + greatest_teardown_cb *teardown; + void *teardown_udata; + + /* formatting info for ".....s...F"-style output */ + unsigned int col; + unsigned int width; + + /* only run a specific suite or test */ + char *suite_filter; + char *test_filter; + + /* overall timers */ + clock_t begin; + clock_t end; +} greatest_run_info; + +/* Global var for the current testing context. + * Initialized by GREATEST_MAIN_DEFS(). */ +extern greatest_run_info greatest_info; + + +/********************** + * Exported functions * + **********************/ + +void greatest_do_pass(const char *name); +void greatest_do_fail(const char *name); +void greatest_do_skip(const char *name); +int greatest_pre_test(const char *name); +void greatest_post_test(const char *name, int res); +void greatest_usage(const char *name); +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); + + +/********** + * Macros * + **********/ + +/* Define a suite. */ +#define GREATEST_SUITE(NAME) void NAME(void) + +/* Start defining a test function. + * The arguments are not included, to allow parametric testing. */ +#define GREATEST_TEST static int + +/* Run a suite. */ +#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) + +/* Run a test in the current suite. */ +#define GREATEST_RUN_TEST(TEST) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* Run a test in the current suite with one void* argument, + * which can be a pointer to a struct with multiple arguments. */ +#define GREATEST_RUN_TEST1(TEST, ENV) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(ENV); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) + +/* If __VA_ARGS__ (C99) is supported, allow parametric testing + * without needing to manually manage the argument struct. */ +#if __STDC_VERSION__ >= 19901L +#define GREATEST_RUN_TESTp(TEST, ...) \ + do { \ + if (greatest_pre_test(#TEST) == 1) { \ + int res = TEST(__VA_ARGS__); \ + greatest_post_test(#TEST, res); \ + } else if (GREATEST_LIST_ONLY()) { \ + fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ + } \ + } while (0) +#endif + + +/* Check if the test runner is in verbose mode. */ +#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE) +#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) +#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) +#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) + +/* Message-less forms. */ +#define GREATEST_PASS() GREATEST_PASSm(NULL) +#define GREATEST_FAIL() GREATEST_FAILm(NULL) +#define GREATEST_SKIP() GREATEST_SKIPm(NULL) +#define GREATEST_ASSERT(COND) GREATEST_ASSERTm(#COND, COND) +#define GREATEST_ASSERT_FALSE(COND) GREATEST_ASSERT_FALSEm(#COND, COND) +#define GREATEST_ASSERT_EQ(EXP, GOT) GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) +#define GREATEST_ASSERT_STR_EQ(EXP, GOT) GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) + +/* The following forms take an additional message argument first, + * to be displayed by the test runner. */ + +/* Fail if a condition is not true, with message. */ +#define GREATEST_ASSERTm(MSG, COND) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if (!(COND)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_FALSEm(MSG, COND) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if ((COND)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ + do { \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if ((EXP) != (GOT)) return -1; \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ + do { \ + const char *exp_s = (EXP); \ + const char *got_s = (GOT); \ + greatest_info.msg = MSG; \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + if (0 != strcmp(exp_s, got_s)) { \ + fprintf(GREATEST_STDOUT, \ + "Expected:\n####\n%s\n####\n", exp_s); \ + fprintf(GREATEST_STDOUT, \ + "Got:\n####\n%s\n####\n", got_s); \ + return -1; \ + } \ + greatest_info.msg = NULL; \ + } while (0) + +#define GREATEST_PASSm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return 0; \ + } while (0) + +#define GREATEST_FAILm(MSG) \ + do { \ + greatest_info.fail_file = __FILE__; \ + greatest_info.fail_line = __LINE__; \ + greatest_info.msg = MSG; \ + return -1; \ + } while (0) + +#define GREATEST_SKIPm(MSG) \ + do { \ + greatest_info.msg = MSG; \ + return 1; \ + } while (0) + +#define GREATEST_SET_TIME(NAME) \ + NAME = clock(); \ + if (NAME == (clock_t) -1) { \ + fprintf(GREATEST_STDOUT, \ + "clock error: %s\n", #NAME); \ + exit(EXIT_FAILURE); \ + } + +#define GREATEST_CLOCK_DIFF(C1, C2) \ + fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ + (long unsigned int) (C2) - (C1), \ + (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) \ + +/* Include several function definitions in the main test file. */ +#define GREATEST_MAIN_DEFS() \ + \ +/* Is FILTER a subset of NAME? */ \ +static int greatest_name_match(const char *name, \ + const char *filter) { \ + size_t offset = 0; \ + size_t filter_len = strlen(filter); \ + while (name[offset] != '\0') { \ + if (name[offset] == filter[0]) { \ + if (0 == strncmp(&name[offset], filter, filter_len)) { \ + return 1; \ + } \ + } \ + offset++; \ + } \ + \ + return 0; \ +} \ + \ +int greatest_pre_test(const char *name) { \ + if (!GREATEST_LIST_ONLY() \ + && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ + && (greatest_info.test_filter == NULL || \ + greatest_name_match(name, greatest_info.test_filter))) { \ + GREATEST_SET_TIME(greatest_info.suite.pre_test); \ + if (greatest_info.setup) { \ + greatest_info.setup(greatest_info.setup_udata); \ + } \ + return 1; /* test should be run */ \ + } else { \ + return 0; /* skipped */ \ + } \ +} \ + \ +void greatest_post_test(const char *name, int res) { \ + GREATEST_SET_TIME(greatest_info.suite.post_test); \ + if (greatest_info.teardown) { \ + void *udata = greatest_info.teardown_udata; \ + greatest_info.teardown(udata); \ + } \ + \ + if (res < 0) { \ + greatest_do_fail(name); \ + } else if (res > 0) { \ + greatest_do_skip(name); \ + } else if (res == 0) { \ + greatest_do_pass(name); \ + } \ + greatest_info.suite.tests_run++; \ + greatest_info.col++; \ + if (GREATEST_IS_VERBOSE()) { \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ + greatest_info.suite.post_test); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } else if (greatest_info.col % greatest_info.width == 0) { \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + } \ + if (GREATEST_STDOUT == stdout) fflush(stdout); \ +} \ + \ +static void greatest_run_suite(greatest_suite_cb *suite_cb, \ + const char *suite_name) { \ + if (greatest_info.suite_filter && \ + !greatest_name_match(suite_name, greatest_info.suite_filter)) \ + return; \ + if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) return; \ + greatest_info.suite.tests_run = 0; \ + greatest_info.suite.failed = 0; \ + greatest_info.suite.passed = 0; \ + greatest_info.suite.skipped = 0; \ + greatest_info.suite.pre_suite = 0; \ + greatest_info.suite.post_suite = 0; \ + greatest_info.suite.pre_test = 0; \ + greatest_info.suite.post_test = 0; \ + greatest_info.col = 0; \ + fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ + GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ + suite_cb(); \ + GREATEST_SET_TIME(greatest_info.suite.post_suite); \ + if (greatest_info.suite.tests_run > 0) { \ + fprintf(GREATEST_STDOUT, \ + "\n%u tests - %u pass, %u fail, %u skipped", \ + greatest_info.suite.tests_run, \ + greatest_info.suite.passed, \ + greatest_info.suite.failed, \ + greatest_info.suite.skipped); \ + GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ + greatest_info.suite.post_suite); \ + fprintf(GREATEST_STDOUT, "\n"); \ + } \ + greatest_info.setup = NULL; \ + greatest_info.setup_udata = NULL; \ + greatest_info.teardown = NULL; \ + greatest_info.teardown_udata = NULL; \ + greatest_info.passed += greatest_info.suite.passed; \ + greatest_info.failed += greatest_info.suite.failed; \ + greatest_info.skipped += greatest_info.suite.skipped; \ + greatest_info.tests_run += greatest_info.suite.tests_run; \ +} \ + \ +void greatest_do_pass(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "PASS %s: %s", \ + name, greatest_info.msg ? greatest_info.msg : ""); \ + } else { \ + fprintf(GREATEST_STDOUT, "."); \ + } \ + greatest_info.suite.passed++; \ +} \ + \ +void greatest_do_fail(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, \ + "FAIL %s: %s (%s:%u)", \ + name, greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } else { \ + fprintf(GREATEST_STDOUT, "F"); \ + /* add linebreak if in line of '.'s */ \ + if (greatest_info.col % greatest_info.width != 0) \ + fprintf(GREATEST_STDOUT, "\n"); \ + greatest_info.col = 0; \ + fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ + name, \ + greatest_info.msg ? greatest_info.msg : "", \ + greatest_info.fail_file, greatest_info.fail_line); \ + } \ + greatest_info.suite.failed++; \ +} \ + \ +void greatest_do_skip(const char *name) { \ + if (GREATEST_IS_VERBOSE()) { \ + fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ + name, \ + greatest_info.msg ? \ + greatest_info.msg : "" ); \ + } else { \ + fprintf(GREATEST_STDOUT, "s"); \ + } \ + greatest_info.suite.skipped++; \ +} \ + \ +void greatest_usage(const char *name) { \ + fprintf(GREATEST_STDOUT, \ + "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ + " -h print this Help\n" \ + " -l List suites and their tests, then exit\n" \ + " -f Stop runner after first failure\n" \ + " -v Verbose output\n" \ + " -s SUITE only run suite named SUITE\n" \ + " -t TEST only run test named TEST\n", \ + name); \ +} \ + \ +void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ + greatest_info.setup = cb; \ + greatest_info.setup_udata = udata; \ +} \ + \ +void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ + void *udata) { \ + greatest_info.teardown = cb; \ + greatest_info.teardown_udata = udata; \ +} \ + \ +greatest_run_info greatest_info + +/* Handle command-line arguments, etc. */ +#define GREATEST_MAIN_BEGIN() \ + do { \ + int i = 0; \ + memset(&greatest_info, 0, sizeof(greatest_info)); \ + if (greatest_info.width == 0) { \ + greatest_info.width = GREATEST_DEFAULT_WIDTH; \ + } \ + for (i = 1; i < argc; i++) { \ + if (0 == strcmp("-t", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.test_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-s", argv[i])) { \ + if (argc <= i + 1) { \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + greatest_info.suite_filter = argv[i+1]; \ + i++; \ + } else if (0 == strcmp("-f", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ + } else if (0 == strcmp("-v", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_VERBOSE; \ + } else if (0 == strcmp("-l", argv[i])) { \ + greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ + } else if (0 == strcmp("-h", argv[i])) { \ + greatest_usage(argv[0]); \ + exit(EXIT_SUCCESS); \ + } else { \ + fprintf(GREATEST_STDOUT, \ + "Unknown argument '%s'\n", argv[i]); \ + greatest_usage(argv[0]); \ + exit(EXIT_FAILURE); \ + } \ + } \ + } while (0); \ + GREATEST_SET_TIME(greatest_info.begin) + +#define GREATEST_MAIN_END() \ + do { \ + if (!GREATEST_LIST_ONLY()) { \ + GREATEST_SET_TIME(greatest_info.end); \ + fprintf(GREATEST_STDOUT, \ + "\nTotal: %u tests", greatest_info.tests_run); \ + GREATEST_CLOCK_DIFF(greatest_info.begin, \ + greatest_info.end); \ + fprintf(GREATEST_STDOUT, "\n"); \ + fprintf(GREATEST_STDOUT, \ + "Pass: %u, fail: %u, skip: %u.\n", \ + greatest_info.passed, \ + greatest_info.failed, greatest_info.skipped); \ + } \ + return (greatest_info.failed > 0 \ + ? EXIT_FAILURE : EXIT_SUCCESS); \ + } while (0) + +/* Make abbreviations without the GREATEST_ prefix for the + * most commonly used symbols. */ +#if GREATEST_USE_ABBREVS +#define TEST GREATEST_TEST +#define SUITE GREATEST_SUITE +#define RUN_TEST GREATEST_RUN_TEST +#define RUN_TEST1 GREATEST_RUN_TEST1 +#define RUN_SUITE GREATEST_RUN_SUITE +#define ASSERT GREATEST_ASSERT +#define ASSERTm GREATEST_ASSERTm +#define ASSERT_FALSE GREATEST_ASSERT_FALSE +#define ASSERT_EQ GREATEST_ASSERT_EQ +#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ +#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm +#define ASSERT_EQm GREATEST_ASSERT_EQm +#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm +#define PASS GREATEST_PASS +#define FAIL GREATEST_FAIL +#define SKIP GREATEST_SKIP +#define PASSm GREATEST_PASSm +#define FAILm GREATEST_FAILm +#define SKIPm GREATEST_SKIPm +#define SET_SETUP GREATEST_SET_SETUP_CB +#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB + +#if __STDC_VERSION__ >= 19901L +#endif /* C99 */ +#define RUN_TESTp GREATEST_RUN_TESTp +#endif /* USE_ABBREVS */ + +#endif diff --git a/espfs/heatshrink/heatshrink.c b/espfs/heatshrink/heatshrink.c new file mode 100644 index 0000000..9de553f --- /dev/null +++ b/espfs/heatshrink/heatshrink.c @@ -0,0 +1,446 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" + +#define DEF_WINDOW_SZ2 11 +#define DEF_LOOKAHEAD_SZ2 4 +#define DEF_DECODER_INPUT_BUFFER_SIZE 256 +#define DEF_BUFFER_SIZE (64 * 1024) + +#if 0 +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#else +#define LOG(...) /* NO-OP */ +#endif + +static const int version_major = HEATSHRINK_VERSION_MAJOR; +static const int version_minor = HEATSHRINK_VERSION_MINOR; +static const int version_patch = HEATSHRINK_VERSION_PATCH; +static const char author[] = HEATSHRINK_AUTHOR; +static const char url[] = HEATSHRINK_URL; + +static void usage(void) { + fprintf(stderr, "heatshrink version %u.%u.%u by %s\n", + version_major, version_minor, version_patch, author); + fprintf(stderr, "Home page: %s\n\n", url); + fprintf(stderr, + "Usage:\n" + " heatshrink [-h] [-e|-d] [-v] [-w SIZE] [-l BITS] [IN_FILE] [OUT_FILE]\n" + "\n" + "heatshrink compresses or uncompresses byte streams using LZSS, and is\n" + "designed especially for embedded, low-memory, and/or hard real-time\n" + "systems.\n" + "\n" + " -h print help\n" + " -e encode (compress, default)\n" + " -d decode (uncompress)\n" + " -v verbose (print input & output sizes, compression ratio, etc.)\n" + "\n" + " -w SIZE Base-2 log of LZSS sliding window size\n" + "\n" + " A larger value allows searches a larger history of the data for repeated\n" + " patterns, potentially compressing more effectively, but will use\n" + " more memory and processing time.\n" + " Recommended default: -w 8 (embedded systems), -w 10 (elsewhere)\n" + " \n" + " -l BITS Number of bits used for back-reference lengths\n" + "\n" + " A larger value allows longer substitutions, but since all\n" + " back-references must use -w + -l bits, larger -w or -l can be\n" + " counterproductive if most patterns are small and/or local.\n" + " Recommended default: -l 4\n" + "\n" + " If IN_FILE or OUT_FILE are unspecified, they will default to\n" + " \"-\" for standard input and standard output, respectively.\n"); + exit(1); +} + +typedef enum { IO_READ, IO_WRITE, } IO_mode; +typedef enum { OP_ENC, OP_DEC, } Operation; + +typedef struct { + int fd; /* file descriptor */ + IO_mode mode; + size_t fill; /* fill index */ + size_t read; /* read index */ + size_t size; + size_t total; + uint8_t buf[]; +} io_handle; + +typedef struct { + uint8_t window_sz2; + uint8_t lookahead_sz2; + size_t decoder_input_buffer_size; + size_t buffer_size; + uint8_t verbose; + Operation cmd; + char *in_fname; + char *out_fname; + io_handle *in; + io_handle *out; +} config; + +static void die(char *msg) { + fprintf(stderr, "%s\n", msg); + exit(EXIT_FAILURE); +} + +static void report(config *cfg); + +/* Open an IO handle. Returns NULL on error. */ +static io_handle *handle_open(char *fname, IO_mode m, size_t buf_sz) { + io_handle *io = NULL; + io = malloc(sizeof(*io) + buf_sz); + if (io == NULL) { return NULL; } + memset(io, 0, sizeof(*io) + buf_sz); + io->fd = -1; + io->size = buf_sz; + io->mode = m; + + if (m == IO_READ) { + if (0 == strcmp("-", fname)) { + io->fd = STDIN_FILENO; + } else { + io->fd = open(fname, O_RDONLY); + } + } else if (m == IO_WRITE) { + if (0 == strcmp("-", fname)) { + io->fd = STDOUT_FILENO; + } else { + io->fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC /*| O_EXCL*/, 0644); + } + } + + if (io->fd == -1) { /* failed to open */ + free(io); + err(1, "open"); + return NULL; + } + + return io; +} + +/* Read SIZE bytes from an IO handle and return a pointer to the content. + * BUF contains at least size_t bytes. Returns 0 on EOF, -1 on error. */ +static ssize_t handle_read(io_handle *io, size_t size, uint8_t **buf) { + LOG("@ read %zd\n", size); + if (buf == NULL) { return -1; } + if (size > io->size) { + printf("size %zd, io->size %zd\n", size, io->size); + return -1; + } + if (io->mode != IO_READ) { return -1; } + + size_t rem = io->fill - io->read; + if (rem >= size) { + *buf = &io->buf[io->read]; + return size; + } else { /* read and replenish */ + if (io->fd == -1) { /* already closed, return what we've got */ + *buf = &io->buf[io->read]; + return rem; + } + + memmove(io->buf, &io->buf[io->read], rem); + io->fill -= io->read; + io->read = 0; + ssize_t read_sz = read(io->fd, &io->buf[io->fill], io->size - io->fill); + if (read_sz < 0) { err(1, "read"); } + io->total += read_sz; + if (read_sz == 0) { /* EOF */ + if (close(io->fd) < 0) { err(1, "close"); } + io->fd = -1; + } + io->fill += read_sz; + *buf = io->buf; + return io->fill > size ? size : io->fill; + } +} + +/* Drop the oldest SIZE bytes from the buffer. Returns <0 on error. */ +static int handle_drop(io_handle *io, size_t size) { + LOG("@ drop %zd\n", size); + if (io->read + size <= io->fill) { + io->read += size; + } else { + return -1; + } + if (io->read == io->fill) { + io->read = 0; + io->fill = 0; + } + return 0; +} + +/* Sink SIZE bytes from INPUT into the io handle. Returns the number of + * bytes written, or -1 on error. */ +static ssize_t handle_sink(io_handle *io, size_t size, uint8_t *input) { + LOG("@ sink %zd\n", size); + if (size > io->size) { return -1; } + if (io->mode != IO_WRITE) { return -1; } + + if (io->fill + size > io->size) { + ssize_t written = write(io->fd, io->buf, io->fill); + LOG("@ flushing %zd, wrote %zd\n", io->fill, written); + io->total += written; + if (written == -1) { err(1, "write"); } + memmove(io->buf, &io->buf[written], io->fill - written); + io->fill -= written; + } + memcpy(&io->buf[io->fill], input, size); + io->fill += size; + return size; +} + +static void handle_close(io_handle *io) { + if (io->fd != -1) { + if (io->mode == IO_WRITE) { + ssize_t written = write(io->fd, io->buf, io->fill); + io->total += written; + LOG("@ close: flushing %zd, wrote %zd\n", io->fill, written); + if (written == -1) { err(1, "write"); } + } + close(io->fd); + io->fd = -1; + } +} + +static void close_and_report(config *cfg) { + handle_close(cfg->in); + handle_close(cfg->out); + if (cfg->verbose) { report(cfg); } + free(cfg->in); + free(cfg->out); +} + +static int encoder_sink_read(config *cfg, heatshrink_encoder *hse, + uint8_t *data, size_t data_sz) { + size_t out_sz = 4096; + uint8_t out_buf[out_sz]; + memset(out_buf, 0, out_sz); + size_t sink_sz = 0; + size_t poll_sz = 0; + HSE_sink_res sres; + HSE_poll_res pres; + HSE_finish_res fres; + io_handle *out = cfg->out; + + size_t sunk = 0; + do { + if (data_sz > 0) { + sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz); + if (sres < 0) { die("sink"); } + sunk += sink_sz; + } + + do { + pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz); + if (pres < 0) { die("poll"); } + if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); + } while (pres == HSER_POLL_MORE); + + if (poll_sz == 0 && data_sz == 0) { + fres = heatshrink_encoder_finish(hse); + if (fres < 0) { die("finish"); } + if (fres == HSER_FINISH_DONE) { return 1; } + } + } while (sunk < data_sz); + return 0; +} + +static int encode(config *cfg) { + uint8_t window_sz2 = cfg->window_sz2; + size_t window_sz = 1 << window_sz2; + heatshrink_encoder *hse = heatshrink_encoder_alloc(window_sz2, cfg->lookahead_sz2); + if (hse == NULL) { die("failed to init encoder: bad settings"); } + ssize_t read_sz = 0; + io_handle *in = cfg->in; + + /* Process input until end of stream */ + while (1) { + uint8_t *input = NULL; + read_sz = handle_read(in, window_sz, &input); + if (input == NULL) { + printf("handle read failure\n"); + die("read"); + } + if (read_sz < 0) { die("read"); } + + /* Pass read to encoder and check if input is fully processed. */ + if (encoder_sink_read(cfg, hse, input, read_sz)) break; + + if (handle_drop(in, read_sz) < 0) { die("drop"); } + }; + + if (read_sz == -1) { err(1, "read"); } + + heatshrink_encoder_free(hse); + close_and_report(cfg); + return 0; +} + +static int decoder_sink_read(config *cfg, heatshrink_decoder *hsd, + uint8_t *data, size_t data_sz) { + io_handle *out = cfg->out; + size_t sink_sz = 0; + size_t poll_sz = 0; + size_t out_sz = 4096; + uint8_t out_buf[out_sz]; + memset(out_buf, 0, out_sz); + + HSD_sink_res sres; + HSD_poll_res pres; + HSD_finish_res fres; + + size_t sunk = 0; + do { + if (data_sz > 0) { + sres = heatshrink_decoder_sink(hsd, &data[sunk], data_sz - sunk, &sink_sz); + if (sres < 0) { die("sink"); } + sunk += sink_sz; + } + + do { + pres = heatshrink_decoder_poll(hsd, out_buf, out_sz, &poll_sz); + if (pres < 0) { die("poll"); } + if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); + } while (pres == HSDR_POLL_MORE); + + if (data_sz == 0 && poll_sz == 0) { + fres = heatshrink_decoder_finish(hsd); + if (fres < 0) { die("finish"); } + if (fres == HSDR_FINISH_DONE) { return 1; } + } + } while (sunk < data_sz); + + return 0; +} + +static int decode(config *cfg) { + uint8_t window_sz2 = cfg->window_sz2; + size_t window_sz = 1 << window_sz2; + size_t ibs = cfg->decoder_input_buffer_size; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(ibs, + window_sz2, cfg->lookahead_sz2); + if (hsd == NULL) { die("failed to init decoder"); } + + ssize_t read_sz = 0; + + io_handle *in = cfg->in; + + HSD_finish_res fres; + + /* Process input until end of stream */ + while (1) { + uint8_t *input = NULL; + read_sz = handle_read(in, window_sz, &input); + if (input == NULL) { + printf("handle read failure\n"); + die("read"); + } + if (read_sz == 0) { + fres = heatshrink_decoder_finish(hsd); + if (fres < 0) { die("finish"); } + if (fres == HSDR_FINISH_DONE) break; + } else if (read_sz < 0) { + die("read"); + } else { + if (decoder_sink_read(cfg, hsd, input, read_sz)) { break; } + if (handle_drop(in, read_sz) < 0) { die("drop"); } + } + } + if (read_sz == -1) { err(1, "read"); } + + heatshrink_decoder_free(hsd); + close_and_report(cfg); + return 0; +} + +static void report(config *cfg) { + size_t inb = cfg->in->total; + size_t outb = cfg->out->total; + fprintf(cfg->out->fd == STDOUT_FILENO ? stderr : stdout, + "%s %0.2f %%\t %zd -> %zd (-w %u -l %u)\n", + cfg->in_fname, 100.0 - (100.0 * outb) / inb, inb, outb, + cfg->window_sz2, cfg->lookahead_sz2); +} + +static void proc_args(config *cfg, int argc, char **argv) { + cfg->window_sz2 = DEF_WINDOW_SZ2; + cfg->lookahead_sz2 = DEF_LOOKAHEAD_SZ2; + cfg->buffer_size = DEF_BUFFER_SIZE; + cfg->decoder_input_buffer_size = DEF_DECODER_INPUT_BUFFER_SIZE; + cfg->cmd = OP_ENC; + cfg->verbose = 0; + cfg->in_fname = "-"; + cfg->out_fname = "-"; + + int a = 0; + while ((a = getopt(argc, argv, "hedi:w:l:v")) != -1) { + switch (a) { + case 'h': /* help */ + usage(); + case 'e': /* encode */ + cfg->cmd = OP_ENC; break; + case 'd': /* decode */ + cfg->cmd = OP_DEC; break; + case 'i': /* input buffer size */ + cfg->decoder_input_buffer_size = atoi(optarg); + break; + case 'w': /* window bits */ + cfg->window_sz2 = atoi(optarg); + break; + case 'l': /* lookahead bits */ + cfg->lookahead_sz2 = atoi(optarg); + break; + case 'v': /* verbosity++ */ + cfg->verbose++; + break; + case '?': /* unknown argument */ + default: + usage(); + } + } + argc -= optind; + argv += optind; + if (argc > 0) { + cfg->in_fname = argv[0]; + argc--; + argv++; + } + if (argc > 0) { cfg->out_fname = argv[0]; } +} + +int main(int argc, char **argv) { + config cfg; + memset(&cfg, 0, sizeof(cfg)); + proc_args(&cfg, argc, argv); + + if (0 == strcmp(cfg.in_fname, cfg.out_fname) + && (0 != strcmp("-", cfg.in_fname))) { + printf("Refusing to overwrite file '%s' with itself.\n", cfg.in_fname); + exit(1); + } + + cfg.in = handle_open(cfg.in_fname, IO_READ, cfg.buffer_size); + if (cfg.in == NULL) { die("Failed to open input file for read"); } + cfg.out = handle_open(cfg.out_fname, IO_WRITE, cfg.buffer_size); + if (cfg.out == NULL) { die("Failed to open output file for write"); } + + if (cfg.cmd == OP_ENC) { + return encode(&cfg); + } else if (cfg.cmd == OP_DEC) { + return decode(&cfg); + } else { + usage(); + } +} diff --git a/espfs/heatshrink/heatshrink_common.h b/espfs/heatshrink/heatshrink_common.h new file mode 100644 index 0000000..0d396b9 --- /dev/null +++ b/espfs/heatshrink/heatshrink_common.h @@ -0,0 +1,20 @@ +#ifndef HEATSHRINK_H +#define HEATSHRINK_H + +#define HEATSHRINK_AUTHOR "Scott Vokes " +#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" + +/* Version 0.3.1 */ +#define HEATSHRINK_VERSION_MAJOR 0 +#define HEATSHRINK_VERSION_MINOR 3 +#define HEATSHRINK_VERSION_PATCH 1 + +#define HEATSHRINK_MIN_WINDOW_BITS 4 +#define HEATSHRINK_MAX_WINDOW_BITS 15 + +#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2 + +#define HEATSHRINK_LITERAL_MARKER 0x01 +#define HEATSHRINK_BACKREF_MARKER 0x00 + +#endif diff --git a/espfs/heatshrink/heatshrink_config.h b/espfs/heatshrink/heatshrink_config.h new file mode 100644 index 0000000..51d4772 --- /dev/null +++ b/espfs/heatshrink/heatshrink_config.h @@ -0,0 +1,24 @@ +#ifndef HEATSHRINK_CONFIG_H +#define HEATSHRINK_CONFIG_H + +/* Should functionality assuming dynamic allocation be used? */ +#define HEATSHRINK_DYNAMIC_ALLOC 1 + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Optional replacement of malloc/free */ + #define HEATSHRINK_MALLOC(SZ) malloc(SZ) + #define HEATSHRINK_FREE(P, SZ) free(P) +#else + /* Required parameters for static configuration */ + #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 + #define HEATSHRINK_STATIC_WINDOW_BITS 8 + #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 +#endif + +/* Turn on logging for debugging. */ +#define HEATSHRINK_DEBUGGING_LOGS 0 + +/* Use indexing for faster compression. (This requires additional space.) */ +#define HEATSHRINK_USE_INDEX 1 + +#endif diff --git a/espfs/heatshrink/heatshrink_decoder.c b/espfs/heatshrink/heatshrink_decoder.c new file mode 100644 index 0000000..b92be13 --- /dev/null +++ b/espfs/heatshrink/heatshrink_decoder.c @@ -0,0 +1,382 @@ +#include +#include +#include "heatshrink_decoder.h" + +/* States for the polling state machine. */ +typedef enum { + HSDS_EMPTY, /* no input to process */ + HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */ + HSDS_YIELD_LITERAL, /* ready to yield literal byte */ + HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ + HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ + HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ + HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ + HSDS_YIELD_BACKREF, /* ready to yield back-reference */ + HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */ +} HSD_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "empty", + "input_available", + "yield_literal", + "backref_index", + "backref_count", + "yield_backref", + "check_for_more_input", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define NO_BITS ((uint32_t)-1) + +/* Forward references. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count); +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (input_buffer_size == 0) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + size_t buffers_sz = (1 << window_sz2) + input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); + if (hsd == NULL) { return NULL; } + hsd->input_buffer_size = input_buffer_size; + hsd->window_sz2 = window_sz2; + hsd->lookahead_sz2 = lookahead_sz2; + heatshrink_decoder_reset(hsd); + LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", + sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); + return hsd; +} + +void heatshrink_decoder_free(heatshrink_decoder *hsd) { + size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; + size_t sz = sizeof(heatshrink_decoder) + buffers_sz; + HEATSHRINK_FREE(hsd, sz); + (void)sz; /* may not be used by free */ +} +#endif + +void heatshrink_decoder_reset(heatshrink_decoder *hsd) { + size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); + size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); + memset(hsd->buffers, 0, buf_sz + input_sz); + hsd->state = HSDS_EMPTY; + hsd->input_size = 0; + hsd->input_index = 0; + hsd->bit_index = 0x00; + hsd->current_byte = 0x00; + hsd->output_count = 0; + hsd->output_index = 0; + hsd->head_index = 0; + hsd->bit_accumulator = 0x00000000; +} + +/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSDR_SINK_ERROR_NULL; + } + + size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; + if (rem == 0) { + *input_size = 0; + return HSDR_SINK_FULL; + } + + size = rem < size ? rem : size; + LOG("-- sinking %zd bytes\n", size); + /* copy into input buffer (at head of buffers) */ + memcpy(&hsd->buffers[hsd->input_size], in_buf, size); + hsd->input_size += size; + if (hsd->state == HSDS_EMPTY) { + hsd->state = HSDS_INPUT_AVAILABLE; + hsd->input_index = 0; + } + *input_size = size; + return HSDR_SINK_OK; +} + + +/***************** + * Decompression * + *****************/ + +#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) +#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) + +// States +static HSD_state st_input_available(heatshrink_decoder *hsd); +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi); +static HSD_state st_check_for_input(heatshrink_decoder *hsd); + +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSDR_POLL_ERROR_NULL; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- poll, state is %d (%s), input_size %d\n", + hsd->state, state_names[hsd->state], hsd->input_size); + uint8_t in_state = hsd->state; + switch (in_state) { + case HSDS_EMPTY: + return HSDR_POLL_EMPTY; + case HSDS_INPUT_AVAILABLE: + hsd->state = st_input_available(hsd); + break; + case HSDS_YIELD_LITERAL: + hsd->state = st_yield_literal(hsd, &oi); + break; + case HSDS_BACKREF_INDEX_MSB: + hsd->state = st_backref_index_msb(hsd); + break; + case HSDS_BACKREF_INDEX_LSB: + hsd->state = st_backref_index_lsb(hsd); + break; + case HSDS_BACKREF_COUNT_MSB: + hsd->state = st_backref_count_msb(hsd); + break; + case HSDS_BACKREF_COUNT_LSB: + hsd->state = st_backref_count_lsb(hsd); + break; + case HSDS_YIELD_BACKREF: + hsd->state = st_yield_backref(hsd, &oi); + break; + case HSDS_CHECK_FOR_MORE_INPUT: + hsd->state = st_check_for_input(hsd); + break; + default: + return HSDR_POLL_ERROR_UNKNOWN; + } + + /* If the current state cannot advance, check if input or output + * buffer are exhausted. */ + if (hsd->state == in_state) { + if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } + return HSDR_POLL_EMPTY; + } + } +} + +static HSD_state st_input_available(heatshrink_decoder *hsd) { + uint32_t bits = get_bits(hsd, 1); // get tag bit + if (bits) { + return HSDS_YIELD_LITERAL; + } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { + return HSDS_BACKREF_INDEX_MSB; + } else { + hsd->output_index = 0; + return HSDS_BACKREF_INDEX_LSB; + } +} + +static HSD_state st_yield_literal(heatshrink_decoder *hsd, + output_info *oi) { + /* Emit a repeated section from the window buffer, and add it (again) + * to the window buffer. (Note that the repetition can include + * itself.)*/ + if (*oi->output_size < oi->buf_size) { + uint32_t byte = get_bits(hsd, 8); + if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint8_t c = byte & 0xFF; + LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); + buf[hsd->head_index++ & mask] = c; + push_byte(hsd, oi, c); + return HSDS_CHECK_FOR_MORE_INPUT; + } else { + return HSDS_YIELD_LITERAL; + } +} + +static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + ASSERT(bit_ct > 8); + uint32_t bits = get_bits(hsd, bit_ct - 8); + LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } + hsd->output_index = bits << 8; + return HSDS_BACKREF_INDEX_LSB; +} + +static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { + uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); + uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); + LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } + hsd->output_index |= bits; + hsd->output_index++; + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + hsd->output_count = 0; + return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + ASSERT(br_bit_ct > 8); + uint32_t bits = get_bits(hsd, br_bit_ct - 8); + LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } + hsd->output_count = bits << 8; + return HSDS_BACKREF_COUNT_LSB; +} + +static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { + uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); + uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); + LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); + if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } + hsd->output_count |= bits; + hsd->output_count++; + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_yield_backref(heatshrink_decoder *hsd, + output_info *oi) { + size_t count = oi->buf_size - *oi->output_size; + if (count > 0) { + if (hsd->output_count < count) count = hsd->output_count; + uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; + uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; + uint16_t neg_offset = hsd->output_index; + LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); + ASSERT(neg_offset < mask + 1); + ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd)); + + for (size_t i=0; ihead_index - neg_offset) & mask]; + push_byte(hsd, oi, c); + buf[hsd->head_index & mask] = c; + hsd->head_index++; + LOG(" -- ++ 0x%02x\n", c); + } + hsd->output_count -= count; + if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; } + } + return HSDS_YIELD_BACKREF; +} + +static HSD_state st_check_for_input(heatshrink_decoder *hsd) { + return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE; +} + +/* Get the next COUNT bits from the input buffer, saving incremental progress. + * Returns NO_BITS on end of input, or if more than 31 bits are requested. */ +static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) { + if (count > 31) { return NO_BITS; } + LOG("-- popping %u bit(s)\n", count); + + /* If we aren't able to get COUNT bits, suspend immediately, because we + * don't track how many bits of COUNT we've accumulated before suspend. */ + if (hsd->input_size == 0) { + if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } + } + + for (int i = 0; i < count; i++) { + if (hsd->bit_index == 0x00) { + if (hsd->input_size == 0) { + LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", + hsd->bit_accumulator, hsd->bit_accumulator); + return NO_BITS; + } + hsd->current_byte = hsd->buffers[hsd->input_index++]; + LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); + if (hsd->input_index == hsd->input_size) { + hsd->input_index = 0; /* input is exhausted */ + hsd->input_size = 0; + } + hsd->bit_index = 0x80; + } + hsd->bit_accumulator <<= 1; + if (hsd->current_byte & hsd->bit_index) { + hsd->bit_accumulator |= 0x01; + if (0) { + LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } else { + if (0) { + LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", + hsd->bit_accumulator, hsd->bit_index); + } + } + hsd->bit_index >>= 1; + } + + uint32_t res = 0; + res = hsd->bit_accumulator; + hsd->bit_accumulator = 0x00000000; + if (count > 1) { LOG(" -- accumulated %08x\n", res); } + return res; +} + +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { + if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } + switch (hsd->state) { + case HSDS_EMPTY: + return HSDR_FINISH_DONE; + + /* If we want to finish with no input, but are in these states, it's + * because the 0-bit padding to the last byte looks like a backref + * marker bit followed by all 0s for index and count bits. */ + case HSDS_BACKREF_INDEX_LSB: + case HSDS_BACKREF_INDEX_MSB: + case HSDS_BACKREF_COUNT_LSB: + case HSDS_BACKREF_COUNT_MSB: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + /* If the output stream is padded with 0xFFs (possibly due to being in + * flash memory), also explicitly check the input size rather than + * uselessly returning MORE but yielding 0 bytes when polling. */ + case HSDS_YIELD_LITERAL: + return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; + + default: + return HSDR_FINISH_MORE; + } +} + +static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { + LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); + oi->buf[(*oi->output_size)++] = byte; + (void)hsd; +} diff --git a/espfs/heatshrink/heatshrink_decoder.h b/espfs/heatshrink/heatshrink_decoder.h new file mode 100644 index 0000000..c1eb144 --- /dev/null +++ b/espfs/heatshrink/heatshrink_decoder.h @@ -0,0 +1,101 @@ +#ifndef HEATSHRINK_DECODER_H +#define HEATSHRINK_DECODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSDR_SINK_OK, /* data sunk, ready to poll */ + HSDR_SINK_FULL, /* out of space in internal buffer */ + HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ +} HSD_sink_res; + +typedef enum { + HSDR_POLL_EMPTY, /* input exhausted */ + HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ + HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ + HSDR_POLL_ERROR_UNKNOWN=-2, +} HSD_poll_res; + +typedef enum { + HSDR_FINISH_DONE, /* output is done */ + HSDR_FINISH_MORE, /* more output remains */ + HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ +} HSD_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ + ((BUF)->input_buffer_size) +#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ + ((BUF)->window_sz2) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + ((BUF)->lookahead_sz2) +#else +#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ + HEATSHRINK_STATIC_INPUT_BUFFER_SIZE +#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t input_index; /* offset to next unprocessed input byte */ + uint16_t output_count; /* how many bytes to output */ + uint16_t output_index; /* index for bytes to output */ + uint16_t head_index; /* head of window buffer */ + uint16_t bit_accumulator; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of input */ + uint8_t bit_index; /* current bit index */ + +#if HEATSHRINK_DYNAMIC_ALLOC + /* Fields that are only used if dynamically allocated. */ + uint8_t window_sz2; /* window buffer bits */ + uint8_t lookahead_sz2; /* lookahead bits */ + uint16_t input_buffer_size; /* input buffer size */ + + /* Input buffer, then expansion window buffer */ + uint8_t buffers[]; +#else + /* Input buffer, then expansion window buffer */ + uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) + + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; +#endif +} heatshrink_decoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, + * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead + * size of 2^lookahead_sz2. (The window buffer and lookahead sizes + * must match the settings used when the data was compressed.) + * Returns NULL on error. */ +heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, + uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); + +/* Free a decoder. */ +void heatshrink_decoder_free(heatshrink_decoder *hsd); +#endif + +/* Reset a decoder. */ +void heatshrink_decoder_reset(heatshrink_decoder *hsd); + +/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to + * indicate how many bytes were actually sunk (in case a buffer was filled). */ +HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the dencoder that the input stream is finished. + * If the return value is HSDR_FINISH_MORE, there is still more output, so + * call heatshrink_decoder_poll and repeat. */ +HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); + +#endif diff --git a/espfs/heatshrink/heatshrink_encoder.c b/espfs/heatshrink/heatshrink_encoder.c new file mode 100644 index 0000000..ede5f60 --- /dev/null +++ b/espfs/heatshrink/heatshrink_encoder.c @@ -0,0 +1,650 @@ +#include +#include +#include +#include "heatshrink_encoder.h" + +typedef enum { + HSES_NOT_FULL, /* input buffer not full enough */ + HSES_FILLED, /* buffer is full */ + HSES_SEARCH, /* searching for patterns */ + HSES_YIELD_TAG_BIT, /* yield tag bit */ + HSES_YIELD_LITERAL, /* emit literal byte */ + HSES_YIELD_BR_INDEX, /* yielding backref index */ + HSES_YIELD_BR_LENGTH, /* yielding backref length */ + HSES_SAVE_BACKLOG, /* copying buffer to backlog */ + HSES_FLUSH_BITS, /* flush bit buffer */ + HSES_DONE, /* done */ +} HSE_state; + +#if HEATSHRINK_DEBUGGING_LOGS +#include +#include +#include +#define LOG(...) fprintf(stderr, __VA_ARGS__) +#define ASSERT(X) assert(X) +static const char *state_names[] = { + "not_full", + "filled", + "search", + "yield_tag_bit", + "yield_literal", + "yield_br_index", + "yield_br_length", + "save_backlog", + "flush_bits", + "done", +}; +#else +#define LOG(...) /* no-op */ +#define ASSERT(X) /* no-op */ +#endif + +// Encoder flags +enum { + FLAG_IS_FINISHING = 0x01, + FLAG_HAS_LITERAL = 0x02, + FLAG_ON_FINAL_LITERAL = 0x04, + FLAG_BACKLOG_IS_PARTIAL = 0x08, + FLAG_BACKLOG_IS_FILLED = 0x10, +}; + +typedef struct { + uint8_t *buf; /* output buffer */ + size_t buf_size; /* buffer size */ + size_t *output_size; /* bytes pushed to buffer, so far */ +} output_info; + +#define MATCH_NOT_FOUND ((uint16_t)-1) + +static uint16_t get_input_offset(heatshrink_encoder *hse); +static uint16_t get_input_buffer_size(heatshrink_encoder *hse); +static uint16_t get_lookahead_size(heatshrink_encoder *hse); +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); +static int can_take_byte(output_info *oi); +static int is_finishing(heatshrink_encoder *hse); +static int backlog_is_partial(heatshrink_encoder *hse); +static int backlog_is_filled(heatshrink_encoder *hse); +static int on_final_literal(heatshrink_encoder *hse); +static void save_backlog(heatshrink_encoder *hse); +static int has_literal(heatshrink_encoder *hse); + +/* Push COUNT (max 8) bits to the output buffer, which has room. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi); +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); + +#if HEATSHRINK_DYNAMIC_ALLOC +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2) { + if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || + (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || + (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || + (lookahead_sz2 > window_sz2)) { + return NULL; + } + + /* Note: 2 * the window size is used because the buffer needs to fit + * (1 << window_sz2) bytes for the current input, and an additional + * (1 << window_sz2) bytes for the previous buffer of input, which + * will be scanned for useful backreferences. */ + size_t buf_sz = (2 << window_sz2); + + heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); + if (hse == NULL) { return NULL; } + hse->window_sz2 = window_sz2; + hse->lookahead_sz2 = lookahead_sz2; + heatshrink_encoder_reset(hse); + +#if HEATSHRINK_USE_INDEX + size_t index_sz = buf_sz*sizeof(uint16_t); + hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); + if (hse->search_index == NULL) { + HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); + return NULL; + } + hse->search_index->size = index_sz; +#endif + + LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", + buf_sz, get_input_buffer_size(hse)); + return hse; +} + +void heatshrink_encoder_free(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); +#if HEATSHRINK_USE_INDEX + size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; + HEATSHRINK_FREE(hse->search_index, index_sz); + (void)index_sz; +#endif + HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); + (void)buf_sz; +} +#endif + +void heatshrink_encoder_reset(heatshrink_encoder *hse) { + size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + memset(hse->buffer, 0, buf_sz); + hse->input_size = 0; + hse->state = HSES_NOT_FULL; + hse->match_scan_index = 0; + hse->flags = 0; + hse->bit_index = 0x80; + hse->current_byte = 0x00; + hse->match_length = 0; + + hse->outgoing_bits = 0x0000; + hse->outgoing_bits_count = 0; + + #ifdef LOOP_DETECT + hse->loop_detect = (uint32_t)-1; + #endif +} + +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size) { + if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { + return HSER_SINK_ERROR_NULL; + } + + /* Sinking more content after saying the content is done, tsk tsk */ + if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } + + /* Sinking more content before processing is done */ + if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } + + uint16_t write_offset = get_input_offset(hse) + hse->input_size; + uint16_t ibs = get_input_buffer_size(hse); + uint16_t rem = ibs - hse->input_size; + uint16_t cp_sz = rem < size ? rem : size; + + memcpy(&hse->buffer[write_offset], in_buf, cp_sz); + *input_size = cp_sz; + hse->input_size += cp_sz; + + LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", + cp_sz, size, write_offset, hse->input_size); + if (cp_sz == rem) { + LOG("-- internal buffer is now full\n"); + hse->state = HSES_FILLED; + } + + return HSER_SINK_OK; +} + + +/*************** + * Compression * + ***************/ + +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length); +static void do_indexing(heatshrink_encoder *hse); + +static HSE_state st_step_search(heatshrink_encoder *hse); +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi); +static HSE_state st_save_backlog(heatshrink_encoder *hse); +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi); + +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { + if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { + return HSER_POLL_ERROR_NULL; + } + if (out_buf_size == 0) { + LOG("-- MISUSE: output buffer size is 0\n"); + return HSER_POLL_ERROR_MISUSE; + } + *output_size = 0; + + output_info oi; + oi.buf = out_buf; + oi.buf_size = out_buf_size; + oi.output_size = output_size; + + while (1) { + LOG("-- polling, state %u (%s), flags 0x%02x\n", + hse->state, state_names[hse->state], hse->flags); + + uint8_t in_state = hse->state; + switch (in_state) { + case HSES_NOT_FULL: + return HSER_POLL_EMPTY; + case HSES_FILLED: + do_indexing(hse); + hse->state = HSES_SEARCH; + break; + case HSES_SEARCH: + hse->state = st_step_search(hse); + break; + case HSES_YIELD_TAG_BIT: + hse->state = st_yield_tag_bit(hse, &oi); + break; + case HSES_YIELD_LITERAL: + hse->state = st_yield_literal(hse, &oi); + break; + case HSES_YIELD_BR_INDEX: + hse->state = st_yield_br_index(hse, &oi); + break; + case HSES_YIELD_BR_LENGTH: + hse->state = st_yield_br_length(hse, &oi); + break; + case HSES_SAVE_BACKLOG: + hse->state = st_save_backlog(hse); + break; + case HSES_FLUSH_BITS: + hse->state = st_flush_bit_buffer(hse, &oi); + case HSES_DONE: + return HSER_POLL_EMPTY; + default: + LOG("-- bad state %s\n", state_names[hse->state]); + return HSER_POLL_ERROR_MISUSE; + } + + if (hse->state == in_state) { + /* Check if output buffer is exhausted. */ + if (*output_size == out_buf_size) return HSER_POLL_MORE; + } + } +} + +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { + if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } + LOG("-- setting is_finishing flag\n"); + hse->flags |= FLAG_IS_FINISHING; + if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } + return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; +} + +static HSE_state st_step_search(heatshrink_encoder *hse) { + uint16_t window_length = get_input_buffer_size(hse); + uint16_t lookahead_sz = get_lookahead_size(hse); + uint16_t msi = hse->match_scan_index; + LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", + msi, hse->input_size + msi, 2*window_length, hse->input_size); + + bool fin = is_finishing(hse); + if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) { + /* Current search buffer is exhausted, copy it into the + * backlog and await more input. */ + LOG("-- end of search @ %d, saving backlog\n", msi); + return HSES_SAVE_BACKLOG; + } + + uint16_t input_offset = get_input_offset(hse); + uint16_t end = input_offset + msi; + + uint16_t start = 0; + if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */ + start = end - window_length + 1; + } else if (backlog_is_partial(hse)) { /* clamp to available data */ + start = end - window_length + 1; + if (start < lookahead_sz) { start = lookahead_sz; } + } else { /* only scan available input */ + start = input_offset; + } + + uint16_t max_possible = lookahead_sz; + if (hse->input_size - msi < lookahead_sz) { + max_possible = hse->input_size - msi; + } + + uint16_t match_length = 0; + uint16_t match_pos = find_longest_match(hse, + start, end, max_possible, &match_length); + + if (match_pos == MATCH_NOT_FOUND) { + LOG("ss Match not found\n"); + hse->match_scan_index++; + hse->flags |= FLAG_HAS_LITERAL; + hse->match_length = 0; + return HSES_YIELD_TAG_BIT; + } else { + LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); + hse->match_pos = match_pos; + hse->match_length = match_length; + ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/); + + return HSES_YIELD_TAG_BIT; + } +} + +static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + if (hse->match_length == 0) { + add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); + return HSES_YIELD_LITERAL; + } else { + add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); + hse->outgoing_bits = hse->match_pos - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); + return HSES_YIELD_BR_INDEX; + } + } else { + return HSES_YIELD_TAG_BIT; /* output is full, continue */ + } +} + +static HSE_state st_yield_literal(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + push_literal_byte(hse, oi); + hse->flags &= ~FLAG_HAS_LITERAL; + if (on_final_literal(hse)) { return HSES_FLUSH_BITS; } + return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH; + } else { + return HSES_YIELD_LITERAL; + } +} + +static HSE_state st_yield_br_index(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref index %u\n", hse->match_pos); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_INDEX; /* continue */ + } else { + hse->outgoing_bits = hse->match_length - 1; + hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); + return HSES_YIELD_BR_LENGTH; /* done */ + } + } else { + return HSES_YIELD_BR_INDEX; /* continue */ + } +} + +static HSE_state st_yield_br_length(heatshrink_encoder *hse, + output_info *oi) { + if (can_take_byte(oi)) { + LOG("-- yielding backref length %u\n", hse->match_length); + if (push_outgoing_bits(hse, oi) > 0) { + return HSES_YIELD_BR_LENGTH; + } else { + hse->match_scan_index += hse->match_length; + hse->match_length = 0; + return HSES_SEARCH; + } + } else { + return HSES_YIELD_BR_LENGTH; + } +} + +static HSE_state st_save_backlog(heatshrink_encoder *hse) { + if (is_finishing(hse)) { + /* copy remaining literal (if necessary) */ + if (has_literal(hse)) { + hse->flags |= FLAG_ON_FINAL_LITERAL; + return HSES_YIELD_TAG_BIT; + } else { + return HSES_FLUSH_BITS; + } + } else { + LOG("-- saving backlog\n"); + save_backlog(hse); + return HSES_NOT_FULL; + } +} + +static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, + output_info *oi) { + if (hse->bit_index == 0x80) { + LOG("-- done!\n"); + return HSES_DONE; + } else if (can_take_byte(oi)) { + LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); + oi->buf[(*oi->output_size)++] = hse->current_byte; + LOG("-- done!\n"); + return HSES_DONE; + } else { + return HSES_FLUSH_BITS; + } +} + +static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { + LOG("-- adding tag bit: %d\n", tag); + push_bits(hse, 1, tag, oi); +} + +static uint16_t get_input_offset(heatshrink_encoder *hse) { + return get_input_buffer_size(hse); +} + +static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); + (void)hse; +} + +static uint16_t get_lookahead_size(heatshrink_encoder *hse) { + return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); + (void)hse; +} + +static void do_indexing(heatshrink_encoder *hse) { +#if HEATSHRINK_USE_INDEX + /* Build an index array I that contains flattened linked lists + * for the previous instances of every byte in the buffer. + * + * For example, if buf[200] == 'x', then index[200] will either + * be an offset i such that buf[i] == 'x', or a negative offset + * to indicate end-of-list. This significantly speeds up matching, + * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. + * + * Future optimization options: + * 1. Since any negative value represents end-of-list, the other + * 15 bits could be used to improve the index dynamically. + * + * 2. Likewise, the last lookahead_sz bytes of the index will + * not be usable, so temporary data could be stored there to + * dynamically improve the index. + * */ + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + uint16_t last[256]; + memset(last, 0xFF, sizeof(last)); + + uint8_t * const data = hse->buffer; + int16_t * const index = hsi->index; + + const uint16_t input_offset = get_input_offset(hse); + const uint16_t end = input_offset + hse->input_size; + + for (uint16_t i=0; iflags & FLAG_IS_FINISHING; +} + +static int backlog_is_partial(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_PARTIAL; +} + +static int backlog_is_filled(heatshrink_encoder *hse) { + return hse->flags & FLAG_BACKLOG_IS_FILLED; +} + +static int on_final_literal(heatshrink_encoder *hse) { + return hse->flags & FLAG_ON_FINAL_LITERAL; +} + +static int has_literal(heatshrink_encoder *hse) { + return (hse->flags & FLAG_HAS_LITERAL); +} + +static int can_take_byte(output_info *oi) { + return *oi->output_size < oi->buf_size; +} + +/* Return the longest match for the bytes at buf[end:end+maxlen] between + * buf[start] and buf[end-1]. If no match is found, return -1. */ +static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, + uint16_t end, const uint16_t maxlen, uint16_t *match_length) { + LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", + end, end + maxlen, start, end + maxlen - 1, maxlen); + uint8_t *buf = hse->buffer; + + uint16_t match_maxlen = 0; + uint16_t match_index = MATCH_NOT_FOUND; + const uint16_t break_even_point = 3; + uint16_t len = 0; + uint8_t * const needlepoint = &buf[end]; +#if HEATSHRINK_USE_INDEX + struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); + int16_t pos = hsi->index[end]; + + while (pos >= start) { + uint8_t * const pospoint = &buf[pos]; + len = 0; + + /* Only check matches that will potentially beat the current maxlen. + * This is redundant with the index if match_maxlen is 0, but the + * added branch overhead to check if it == 0 seems to be worse. */ + if (pospoint[match_maxlen] != needlepoint[match_maxlen]) { + pos = hsi->index[pos]; + continue; + } + + for (len = 1; len < maxlen; len++) { + if (pospoint[len] != needlepoint[len]) break; + } + + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* won't find better */ + } + pos = hsi->index[pos]; + } +#else + for (int16_t pos=end - 1; pos >= start; pos--) { + uint8_t * const pospoint = &buf[pos]; + if ((pospoint[match_maxlen] == needlepoint[match_maxlen]) + && (*pospoint == *needlepoint)) { + for (len=1; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", + pos + len, pospoint[len], needlepoint[len], start); + } + if (pospoint[len] != needlepoint[len]) { break; } + } + if (len > match_maxlen) { + match_maxlen = len; + match_index = pos; + if (len == maxlen) { break; } /* don't keep searching */ + } + } + } +#endif + + if (match_maxlen >= break_even_point) { + LOG("-- best match: %u bytes at -%u\n", + match_maxlen, end - match_index); + *match_length = match_maxlen; + return end - match_index; + } + LOG("-- none found\n"); + return MATCH_NOT_FOUND; +} + +static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { + uint8_t count = 0; + uint8_t bits = 0; + if (hse->outgoing_bits_count > 8) { + count = 8; + bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); + } else { + count = hse->outgoing_bits_count; + bits = hse->outgoing_bits; + } + + if (count > 0) { + LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); + push_bits(hse, count, bits, oi); + hse->outgoing_bits_count -= count; + } + return count; +} + +/* Push COUNT (max 8) bits to the output buffer, which has room. + * Bytes are set from the lowest bits, up. */ +static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, + output_info *oi) { + ASSERT(count <= 8); + LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); + + /* If adding a whole byte and at the start of a new output byte, + * just push it through whole and skip the bit IO loop. */ + if (count == 8 && hse->bit_index == 0x80) { + oi->buf[(*oi->output_size)++] = bits; + } else { + for (int i=count - 1; i>=0; i--) { + bool bit = bits & (1 << i); + if (bit) { hse->current_byte |= hse->bit_index; } + if (0) { + LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", + bit ? 1 : 0, hse->bit_index, hse->current_byte); + } + hse->bit_index >>= 1; + if (hse->bit_index == 0x00) { + hse->bit_index = 0x80; + LOG(" > pushing byte 0x%02x\n", hse->current_byte); + oi->buf[(*oi->output_size)++] = hse->current_byte; + hse->current_byte = 0x00; + } + } + } +} + +static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { + uint16_t processed_offset = hse->match_scan_index - 1; + uint16_t input_offset = get_input_offset(hse) + processed_offset; + uint8_t c = hse->buffer[input_offset]; + LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", + c, isprint(c) ? c : '.', input_offset); + push_bits(hse, 8, c, oi); +} + +static void save_backlog(heatshrink_encoder *hse) { + size_t input_buf_sz = get_input_buffer_size(hse); + + uint16_t msi = hse->match_scan_index; + + /* Copy processed data to beginning of buffer, so it can be + * used for future matches. Don't bother checking whether the + * input is less than the maximum size, because if it isn't, + * we're done anyway. */ + uint16_t rem = input_buf_sz - msi; // unprocessed bytes + uint16_t shift_sz = input_buf_sz + rem; + + memmove(&hse->buffer[0], + &hse->buffer[input_buf_sz - rem], + shift_sz); + + if (backlog_is_partial(hse)) { + /* The whole backlog is filled in now, so include it in scans. */ + hse->flags |= FLAG_BACKLOG_IS_FILLED; + } else { + /* Include backlog, except for the first lookahead_sz bytes, which + * are still undefined. */ + hse->flags |= FLAG_BACKLOG_IS_PARTIAL; + } + hse->match_scan_index = 0; + hse->input_size -= input_buf_sz - rem; +} diff --git a/espfs/heatshrink/heatshrink_encoder.h b/espfs/heatshrink/heatshrink_encoder.h new file mode 100644 index 0000000..18c1773 --- /dev/null +++ b/espfs/heatshrink/heatshrink_encoder.h @@ -0,0 +1,109 @@ +#ifndef HEATSHRINK_ENCODER_H +#define HEATSHRINK_ENCODER_H + +#include +#include +#include "heatshrink_common.h" +#include "heatshrink_config.h" + +typedef enum { + HSER_SINK_OK, /* data sunk into input buffer */ + HSER_SINK_ERROR_NULL=-1, /* NULL argument */ + HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ +} HSE_sink_res; + +typedef enum { + HSER_POLL_EMPTY, /* input exhausted */ + HSER_POLL_MORE, /* poll again for more output */ + HSER_POLL_ERROR_NULL=-1, /* NULL argument */ + HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ +} HSE_poll_res; + +typedef enum { + HSER_FINISH_DONE, /* encoding is complete */ + HSER_FINISH_MORE, /* more output remaining; use poll */ + HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ +} HSE_finish_res; + +#if HEATSHRINK_DYNAMIC_ALLOC +#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ + ((HSE)->window_sz2) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ + ((HSE)->lookahead_sz2) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + ((HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[]; +}; +#else +#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ + (HEATSHRINK_STATIC_WINDOW_BITS) +#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ + (HEATSHRINK_STATIC_LOOKAHEAD_BITS) +#define HEATSHRINK_ENCODER_INDEX(HSE) \ + (&(HSE)->search_index) +struct hs_index { + uint16_t size; + int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; +}; +#endif + +typedef struct { + uint16_t input_size; /* bytes in input buffer */ + uint16_t match_scan_index; + uint16_t match_length; + uint16_t match_pos; + uint16_t outgoing_bits; /* enqueued outgoing bits */ + uint8_t outgoing_bits_count; + uint8_t flags; + uint8_t state; /* current state machine node */ + uint8_t current_byte; /* current byte of output */ + uint8_t bit_index; /* current bit index */ +#if HEATSHRINK_DYNAMIC_ALLOC + uint8_t window_sz2; /* 2^n size of window */ + uint8_t lookahead_sz2; /* 2^n size of lookahead */ +#if HEATSHRINK_USE_INDEX + struct hs_index *search_index; +#endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[]; +#else + #if HEATSHRINK_USE_INDEX + struct hs_index search_index; + #endif + /* input buffer and / sliding window for expansion */ + uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; +#endif +} heatshrink_encoder; + +#if HEATSHRINK_DYNAMIC_ALLOC +/* Allocate a new encoder struct and its buffers. + * Returns NULL on error. */ +heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, + uint8_t lookahead_sz2); + +/* Free an encoder. */ +void heatshrink_encoder_free(heatshrink_encoder *hse); +#endif + +/* Reset an encoder. */ +void heatshrink_encoder_reset(heatshrink_encoder *hse); + +/* Sink up to SIZE bytes from IN_BUF into the encoder. + * INPUT_SIZE is set to the number of bytes actually sunk (in case a + * buffer was filled.). */ +HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, + uint8_t *in_buf, size_t size, size_t *input_size); + +/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into + * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ +HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, + uint8_t *out_buf, size_t out_buf_size, size_t *output_size); + +/* Notify the encoder that the input stream is finished. + * If the return value is HSER_FINISH_MORE, there is still more output, so + * call heatshrink_encoder_poll and repeat. */ +HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); + +#endif diff --git a/espfs/heatshrink/test_heatshrink_dynamic.c b/espfs/heatshrink/test_heatshrink_dynamic.c new file mode 100644 index 0000000..1e18a69 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_dynamic.c @@ -0,0 +1,999 @@ +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" + +#if !HEATSHRINK_DYNAMIC_ALLOC +#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for dynamic allocation test suite. +#endif + +SUITE(encoding); +SUITE(decoding); +SUITE(integration); + +#ifdef HEATSHRINK_HAS_THEFT +SUITE(properties); +#endif + +static void dump_buf(char *name, uint8_t *buf, uint16_t count) { + for (int i=0; iinput_size, 6); + ASSERT_EQ(hsd->input_index, 0); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_return_empty_if_empty(void) { + uint8_t output[256]; + size_t out_sz = 0; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_EMPTY, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_reject_null_hsd(void) { + uint8_t output[256]; + size_t out_sz = 0; + HSD_poll_res res = heatshrink_decoder_poll(NULL, output, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + PASS(); +} + +TEST decoder_poll_should_reject_null_output_buffer(void) { + size_t out_sz = 0; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, NULL, 256, &out_sz); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_reject_null_output_size_pointer(void) { + uint8_t output[256]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, + HEATSHRINK_MIN_WINDOW_BITS, 4); + HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, NULL); + ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_literal(void) { + uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0 }; //"foo" + uint8_t output[4]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 3); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + ASSERT_EQ(3, out_sz); + ASSERT_EQ('f', output[0]); + ASSERT_EQ('o', output[1]); + ASSERT_EQ('o', output[2]); + + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_literal_and_backref(void) { + uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo" + uint8_t output[6]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7); + memset(output, 0, sizeof(*output)); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + (void)heatshrink_decoder_poll(hsd, output, 6, &out_sz); + + if (0) dump_buf("output", output, out_sz); + ASSERT_EQ(6, out_sz); + ASSERT_EQ('f', output[0]); + ASSERT_EQ('o', output[1]); + ASSERT_EQ('o', output[2]); + ASSERT_EQ('f', output[3]); + ASSERT_EQ('o', output[4]); + ASSERT_EQ('o', output[5]); + + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST decoder_poll_should_expand_short_self_overlapping_backref(void) { + /* "aaaaa" == (literal, 1), ('a'), (backref, 1 back, 4 bytes) */ + uint8_t input[] = {0xb0, 0x80, 0x01, 0x80}; + uint8_t output[6]; + uint8_t expected[] = {'a', 'a', 'a', 'a', 'a'}; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 7); + size_t count = 0; + + HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); + ASSERT_EQ(HSDR_SINK_OK, sres); + + size_t out_sz = 0; + (void)heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); + + if (0) dump_buf("output", output, out_sz); + ASSERT_EQ(sizeof(expected), out_sz); + for (size_t i=0; iwindow_sz2, + cfg->lookahead_sz2); + heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size, + cfg->window_sz2, cfg->lookahead_sz2); + size_t comp_sz = input_size + (input_size/2) + 4; + size_t decomp_sz = input_size + (input_size/2) + 4; + uint8_t *comp = malloc(comp_sz); + uint8_t *decomp = malloc(decomp_sz); + if (comp == NULL) FAILm("malloc fail"); + if (decomp == NULL) FAILm("malloc fail"); + memset(comp, 0, comp_sz); + memset(decomp, 0, decomp_sz); + + size_t count = 0; + + if (cfg->log_lvl > 1) { + printf("\n^^ COMPRESSING\n"); + dump_buf("input", input, input_size); + } + + size_t sunk = 0; + size_t polled = 0; + while (sunk < input_size) { + ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0); + sunk += count; + if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + } + + HSE_poll_res pres; + do { /* "turn the crank" */ + pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSER_POLL_MORE); + ASSERT_EQ(HSER_POLL_EMPTY, pres); + if (polled >= comp_sz) FAILm("compression should never expand that much"); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); + } + } + if (cfg->log_lvl > 0) printf("in: %u compressed: %zu ", input_size, polled); + size_t compressed_size = polled; + sunk = 0; + polled = 0; + + if (cfg->log_lvl > 1) { + printf("\n^^ DECOMPRESSING\n"); + dump_buf("comp", comp, compressed_size); + } + while (sunk < compressed_size) { + ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); + sunk += count; + if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == compressed_size) { + ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd)); + } + + HSD_poll_res pres; + do { + pres = heatshrink_decoder_poll(hsd, &decomp[polled], + decomp_sz - polled, &count); + ASSERT(pres >= 0); + ASSERT(count > 0); + polled += count; + if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSDR_POLL_MORE); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + if (sunk == compressed_size) { + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + ASSERT_EQ(HSDR_FINISH_DONE, fres); + } + + if (polled > input_size) { + printf("\nExpected %zd, got %zu\n", (size_t)input_size, polled); + FAILm("Decompressed data is larger than original input"); + } + } + if (cfg->log_lvl > 0) printf("decompressed: %zu\n", polled); + if (polled != input_size) { + FAILm("Decompressed length does not match original input length"); + } + + if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled); + for (uint32_t i=0; i out[%d] == 0x%02x ('%c') %c\n", + j, input[j], isprint(input[j]) ? input[j] : '.', + j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.', + input[j] == decomp[j] ? ' ' : 'X'); + } + } + } + ASSERT_EQ(input[i], decomp[i]); + } + free(comp); + free(decomp); + heatshrink_encoder_free(hse); + heatshrink_decoder_free(hsd); + PASS(); +} + +TEST data_without_duplication_should_match(void) { + uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', + 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 256; + return compress_and_expand_and_check(input, sizeof(input), &cfg); +} + +TEST data_with_simple_repetition_should_compress_and_decompress_properly(void) { + uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', + 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h'}; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 256; + return compress_and_expand_and_check(input, sizeof(input), &cfg); +} + +TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) { + heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); + heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); + uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', + 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', + 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; + uint8_t comp[60]; + uint8_t decomp[60]; + size_t count = 0; + int log = 0; + + if (log) dump_buf("input", input, sizeof(input)); + for (uint32_t i=0; i= 0); + } + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + + size_t packed_count = 0; + do { + ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); + packed_count += count; + } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); + + if (log) dump_buf("comp", comp, packed_count); + for (uint32_t i=0; i= 0); + } + + for (uint32_t i=0; i= 0); + } + + if (log) dump_buf("decomp", decomp, sizeof(input)); + for (uint32_t i=0; i= 0); + } + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); + + size_t packed_count = 0; + do { + ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); + packed_count += count; + } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); + + if (log) dump_buf("comp", comp, packed_count); + for (uint32_t i=0; i= 0); + } + + for (uint32_t i=0; i= 0); + } + + if (log) dump_buf("decomp", decomp, sizeof(input)); + for (uint32_t i=0; ilog_lvl > 0) { + printf("\n-- size %u, seed %u, input buf %zu\n", + size, seed, cfg->decoder_input_buffer_size); + } + fill_with_pseudorandom_letters(input, size, seed); + return compress_and_expand_and_check(input, size, cfg); +} + +TEST small_input_buffer_should_not_impact_decoder_correctness(void) { + int size = 5; + uint8_t input[size]; + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = 3; + cfg.decoder_input_buffer_size = 5; + for (uint16_t i=0; i= 19901L + printf("\n\nFuzzing (single-byte sizes):\n"); + for (uint8_t lsize=3; lsize < 8; lsize++) { + for (uint32_t size=1; size < 128*1024L; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ + if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); + for (uint32_t seed=1; seed<=10; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 8; + cfg.lookahead_sz2 = lsize; + cfg.decoder_input_buffer_size = ibs; + RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); + } + } + } + } + + printf("\nFuzzing (multi-byte sizes):\n"); + for (uint8_t lsize=6; lsize < 9; lsize++) { + for (uint32_t size=1; size < 128*1024L; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ + if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); + for (uint32_t seed=1; seed<=10; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + cfg_info cfg; + cfg.log_lvl = 0; + cfg.window_sz2 = 11; + cfg.lookahead_sz2 = lsize; + cfg.decoder_input_buffer_size = ibs; + RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); + } + } + } + } + +#endif +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + RUN_SUITE(encoding); + RUN_SUITE(decoding); + RUN_SUITE(integration); + #ifdef HEATSHRINK_HAS_THEFT + RUN_SUITE(properties); + #endif + GREATEST_MAIN_END(); /* display results */ +} diff --git a/espfs/heatshrink/test_heatshrink_dynamic_theft.c b/espfs/heatshrink/test_heatshrink_dynamic_theft.c new file mode 100644 index 0000000..2752886 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_dynamic_theft.c @@ -0,0 +1,521 @@ +#include "heatshrink_config.h" +#ifdef HEATSHRINK_HAS_THEFT + +#include +#include +#include +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" +#include "theft.h" +#include "greatest_theft.h" + +#if !HEATSHRINK_DYNAMIC_ALLOC +#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for this test suite. +#endif + +SUITE(properties); + +typedef struct { + int limit; + int fails; + int dots; +} test_env; + +typedef struct { + size_t size; + uint8_t buf[]; +} rbuf; + +static void *rbuf_alloc_cb(struct theft *t, theft_hash seed, void *env) { + test_env *te = (test_env *)env; + //printf("seed is 0x%016llx\n", seed); + + size_t sz = (size_t)(seed % te->limit) + 1; + rbuf *r = malloc(sizeof(rbuf) + sz); + if (r == NULL) { return THEFT_ERROR; } + r->size = sz; + + for (size_t i = 0; i < sz; i += sizeof(theft_hash)) { + theft_hash s = theft_random(t); + for (uint8_t b = 0; b < sizeof(theft_hash); b++) { + if (i + b >= sz) { break; } + r->buf[i + b] = (uint8_t) (s >> (8*b)) & 0xff; + } + } + + return r; +} + +static void rbuf_free_cb(void *instance, void *env) { + free(instance); + (void)env; +} + +static uint64_t rbuf_hash_cb(void *instance, void *env) { + rbuf *r = (rbuf *)instance; + (void)env; + return theft_hash_onepass(r->buf, r->size); +} + +/* Make a copy of a buffer, keeping NEW_SZ bytes starting at OFFSET. */ +static void *copy_rbuf_subset(rbuf *cur, size_t new_sz, size_t byte_offset) { + if (new_sz == 0) { return THEFT_DEAD_END; } + rbuf *nr = malloc(sizeof(rbuf) + new_sz); + if (nr == NULL) { return THEFT_ERROR; } + nr->size = new_sz; + memcpy(nr->buf, &cur->buf[byte_offset], new_sz); + /* printf("%zu -> %zu\n", cur->size, new_sz); */ + return nr; +} + +/* Make a copy of a buffer, but only PORTION, starting OFFSET in + * (e.g. the third quarter is (0.25 at +0.75). Rounds to ints. */ +static void *copy_rbuf_percent(rbuf *cur, float portion, float offset) { + size_t new_sz = cur->size * portion; + size_t byte_offset = (size_t)(cur->size * offset); + return copy_rbuf_subset(cur, new_sz, byte_offset); +} + +/* How to shrink a random buffer to a simpler one. */ +static void *rbuf_shrink_cb(void *instance, uint32_t tactic, void *env) { + rbuf *cur = (rbuf *)instance; + + if (tactic == 0) { /* first half */ + return copy_rbuf_percent(cur, 0.5, 0); + } else if (tactic == 1) { /* second half */ + return copy_rbuf_percent(cur, 0.5, 0.5); + } else if (tactic <= 18) { /* drop 1-16 bytes at start */ + const int last_tactic = 1; + const size_t drop = tactic - last_tactic; + if (cur->size < drop) { return THEFT_DEAD_END; } + return copy_rbuf_subset(cur, cur->size - drop, drop); + } else if (tactic <= 34) { /* drop 1-16 bytes at end */ + const int last_tactic = 18; + const size_t drop = tactic - last_tactic; + if (cur->size < drop) { return THEFT_DEAD_END; } + return copy_rbuf_subset(cur, cur->size - drop, 0); + } else if (tactic == 35) { + /* Divide every byte by 2, saturating at 0 */ + rbuf *cp = copy_rbuf_percent(cur, 1, 0); + if (cp == NULL) { return THEFT_ERROR; } + for (size_t i = 0; i < cp->size; i++) { cp->buf[i] /= 2; } + return cp; + } else if (tactic == 36) { + /* subtract 1 from every byte, saturating at 0 */ + rbuf *cp = copy_rbuf_percent(cur, 1, 0); + if (cp == NULL) { return THEFT_ERROR; } + for (size_t i = 0; i < cp->size; i++) { + if (cp->buf[i] > 0) { cp->buf[i]--; } + } + return cp; + } else { + (void)env; + return THEFT_NO_MORE_TACTICS; + } + + return THEFT_NO_MORE_TACTICS; +} + +static void rbuf_print_cb(FILE *f, void *instance, void *env) { + rbuf *r = (rbuf *)instance; + (void)env; + fprintf(f, "buf[%zd]:\n ", r->size); + uint8_t bytes = 0; + for (size_t i = 0; i < r->size; i++) { + fprintf(f, "%02x", r->buf[i]); + bytes++; + if (bytes == 16) { + fprintf(f, "\n "); + bytes = 0; + } + } + fprintf(f, "\n"); +} + +static struct theft_type_info rbuf_info = { + .alloc = rbuf_alloc_cb, + .free = rbuf_free_cb, + .hash = rbuf_hash_cb, + .shrink = rbuf_shrink_cb, + .print = rbuf_print_cb, +}; + +static theft_progress_callback_res +progress_cb(struct theft_trial_info *info, void *env) { + test_env *te = (test_env *)env; + if ((info->trial & 0xff) == 0) { + printf("."); + fflush(stdout); + te->dots++; + if (te->dots == 64) { + printf("\n"); + te->dots = 0; + } + } + + if (info->status == THEFT_TRIAL_FAIL) { + te->fails++; + rbuf *cur = info->args[0]; + if (cur->size < 5) { return THEFT_PROGRESS_HALT; } + } + + if (te->fails > 10) { + return THEFT_PROGRESS_HALT; + } + return THEFT_PROGRESS_CONTINUE; +} + +/* For an arbitrary input buffer, it should never get stuck in a + * state where the data has been sunk but no data can be polled. */ +static theft_trial_res prop_should_not_get_stuck(void *input) { + /* Make a buffer large enough for the output: 4 KB of input with + * each 16 bits becoming up to 16 bytes will fit in a 64 KB buffer. + * (4 KB of input comes from `env.limit = 1 << 12;` below.) */ + uint8_t output[64 * 1024]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc((64 * 1024L) - 1, 12, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t count = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, r->buf, r->size, &count); + if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } + + size_t out_sz = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); + if (pres != HSDR_POLL_EMPTY) { return THEFT_TRIAL_FAIL; } + + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + heatshrink_decoder_free(hsd); + if (fres != HSDR_FINISH_DONE) { return THEFT_TRIAL_FAIL; } + + return THEFT_TRIAL_PASS; +} + +static bool get_time_seed(theft_seed *seed) +{ + struct timeval tv; + if (-1 == gettimeofday(&tv, NULL)) { return false; } + *seed = (theft_seed)((tv.tv_sec << 32) | tv.tv_usec); + /* printf("seed is 0x%016llx\n", *seed); */ + return true; +} + +TEST decoder_fuzzing_should_not_detect_stuck_state(void) { + // Get a random number seed based on the time + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + /* Pass the max buffer size for this property (4 KB) in a closure */ + test_env env = { .limit = 1 << 12 }; + + theft_seed always_seeds = { 0xe87bb1f61032a061 }; + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_should_not_get_stuck, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 100000, + .progress_cb = progress_cb, + .env = &env, + + .always_seeds = &always_seeds, + .always_seed_count = 1, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + GREATEST_ASSERT_EQm("should_not_get_stuck", THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res prop_encoded_and_decoded_data_should_match(void *input) { + uint8_t e_output[64 * 1024]; + uint8_t d_output[64 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + heatshrink_decoder *hsd = heatshrink_decoder_alloc(4096, 12, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t e_input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + r->buf, r->size, &e_input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + if (e_input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t e_output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + e_output, sizeof(e_output), &e_output_size); + if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t count = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, e_output, e_output_size, &count); + if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } + + size_t d_output_size = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, d_output, + sizeof(d_output), &d_output_size); + if (pres != HSDR_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + if (d_output_size != r->size) { + printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; + } + + if (0 != memcmp(d_output, r->buf, d_output_size)) { + return THEFT_TRIAL_FAIL; + } + + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + if (fres != HSDR_FINISH_DONE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + heatshrink_encoder_free(hse); + heatshrink_decoder_free(hsd); + return THEFT_TRIAL_PASS; +} + + +TEST encoded_and_decoded_data_should_match(void) { + test_env env = { .limit = 1 << 11 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoded_and_decoded_data_should_match, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 1000000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static size_t ceil_nine_eighths(size_t sz) { + return sz + sz/8 + (sz & 0x07 ? 1 : 0); +} + +static theft_trial_res +prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void *input) { + uint8_t output[32 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)input; + + size_t input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + r->buf, r->size, &input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + /* Assumes data fits in one sink, failure here means buffer must be larger. */ + if (input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + output, sizeof(output), &output_size); + if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } + + size_t ceil_9_8s = ceil_nine_eighths(r->size); + if (output_size > ceil_9_8s) { + return THEFT_TRIAL_FAIL; + } + + heatshrink_encoder_free(hse); + return THEFT_TRIAL_PASS; +} + +TEST encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void) { + test_env env = { .limit = 1 << 11 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res +prop_encoder_should_always_make_progress(void *instance) { + uint8_t output[64 * 1024]; + heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); + if (hse == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)instance; + + size_t sunk = 0; + int no_progress = 0; + + while (1) { + if (sunk < r->size) { + size_t input_size = 0; + HSE_sink_res esres = heatshrink_encoder_sink(hse, + &r->buf[sunk], r->size - sunk, &input_size); + if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + sunk += input_size; + } else { + HSE_finish_res efres = heatshrink_encoder_finish(hse); + if (efres == HSER_FINISH_DONE) { + break; + } else if (efres != HSER_FINISH_MORE) { + printf("FAIL %d\n", __LINE__); + return THEFT_TRIAL_FAIL; + } + } + + size_t output_size = 0; + HSE_poll_res epres = heatshrink_encoder_poll(hse, + output, sizeof(output), &output_size); + if (epres < 0) { return THEFT_TRIAL_ERROR; } + if (output_size == 0 && sunk == r->size) { + no_progress++; + if (no_progress > 2) { + return THEFT_TRIAL_FAIL; + } + } else { + no_progress = 0; + } + } + + heatshrink_encoder_free(hse); + return THEFT_TRIAL_PASS; +} + +TEST encoder_should_always_make_progress(void) { + test_env env = { .limit = 1 << 15 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_encoder_should_always_make_progress, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +static theft_trial_res +prop_decoder_should_always_make_progress(void *instance) { + uint8_t output[64 * 1024]; + heatshrink_decoder *hsd = heatshrink_decoder_alloc(512, 8, 4); + if (hsd == NULL) { return THEFT_TRIAL_ERROR; } + + rbuf *r = (rbuf *)instance; + + size_t sunk = 0; + int no_progress = 0; + + while (1) { + if (sunk < r->size) { + size_t input_size = 0; + HSD_sink_res sres = heatshrink_decoder_sink(hsd, + &r->buf[sunk], r->size - sunk, &input_size); + if (sres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } + sunk += input_size; + } else { + HSD_finish_res fres = heatshrink_decoder_finish(hsd); + if (fres == HSDR_FINISH_DONE) { + break; + } else if (fres != HSDR_FINISH_MORE) { + printf("FAIL %d\n", __LINE__); + return THEFT_TRIAL_FAIL; + } + } + + size_t output_size = 0; + HSD_poll_res pres = heatshrink_decoder_poll(hsd, + output, sizeof(output), &output_size); + if (pres < 0) { return THEFT_TRIAL_ERROR; } + if (output_size == 0 && sunk == r->size) { + no_progress++; + if (no_progress > 2) { + return THEFT_TRIAL_FAIL; + } + } else { + no_progress = 0; + } + } + + heatshrink_decoder_free(hsd); + return THEFT_TRIAL_PASS; +} + +TEST decoder_should_always_make_progress(void) { + test_env env = { .limit = 1 << 15 }; + + theft_seed seed; + if (!get_time_seed(&seed)) { FAIL(); } + + struct theft *t = theft_init(0); + struct theft_cfg cfg = { + .name = __func__, + .fun = prop_decoder_should_always_make_progress, + .type_info = { &rbuf_info }, + .seed = seed, + .trials = 10000, + .env = &env, + .progress_cb = progress_cb, + }; + + theft_run_res res = theft_run(t, &cfg); + theft_free(t); + printf("\n"); + ASSERT_EQ(THEFT_RUN_PASS, res); + PASS(); +} + +SUITE(properties) { + RUN_TEST(decoder_fuzzing_should_not_detect_stuck_state); + RUN_TEST(encoded_and_decoded_data_should_match); + RUN_TEST(encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst); + RUN_TEST(encoder_should_always_make_progress); + RUN_TEST(decoder_should_always_make_progress); +} +#else +struct because_iso_c_requires_at_least_one_declaration; +#endif diff --git a/espfs/heatshrink/test_heatshrink_static.c b/espfs/heatshrink/test_heatshrink_static.c new file mode 100644 index 0000000..e9c9754 --- /dev/null +++ b/espfs/heatshrink/test_heatshrink_static.c @@ -0,0 +1,167 @@ +#include +#include + +#include "heatshrink_encoder.h" +#include "heatshrink_decoder.h" +#include "greatest.h" + +#if HEATSHRINK_DYNAMIC_ALLOC +#error HEATSHRINK_DYNAMIC_ALLOC must be false for static allocation test suite. +#endif + +SUITE(integration); + +/* The majority of the tests are in test_heatshrink_dynamic, because that allows + * instantiating encoders/decoders with different settings at run-time. */ + +static heatshrink_encoder hse; +static heatshrink_decoder hsd; + +static void fill_with_pseudorandom_letters(uint8_t *buf, uint16_t size, uint32_t seed) { + uint64_t rn = 9223372036854775783; /* prime under 2^64 */ + for (int i=0; i 1) { + printf("\n^^ COMPRESSING\n"); + dump_buf("input", input, input_size); + } + + uint32_t sunk = 0; + uint32_t polled = 0; + while (sunk < input_size) { + ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); + sunk += count; + if (log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); + } + + HSE_poll_res pres; + do { /* "turn the crank" */ + pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSER_POLL_MORE); + ASSERT_EQ(HSER_POLL_EMPTY, pres); + if (polled >= comp_sz) FAILm("compression should never expand that much"); + if (sunk == input_size) { + ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); + } + } + if (log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled); + uint32_t compressed_size = polled; + sunk = 0; + polled = 0; + + if (log_lvl > 1) { + printf("\n^^ DECOMPRESSING\n"); + dump_buf("comp", comp, compressed_size); + } + while (sunk < compressed_size) { + ASSERT(heatshrink_decoder_sink(&hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); + sunk += count; + if (log_lvl > 1) printf("^^ sunk %zd\n", count); + if (sunk == compressed_size) { + ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(&hsd)); + } + + HSD_poll_res pres; + do { + pres = heatshrink_decoder_poll(&hsd, &decomp[polled], + decomp_sz - polled, &count); + ASSERT(pres >= 0); + polled += count; + if (log_lvl > 1) printf("^^ polled %zd\n", count); + } while (pres == HSDR_POLL_MORE); + ASSERT_EQ(HSDR_POLL_EMPTY, pres); + if (sunk == compressed_size) { + HSD_finish_res fres = heatshrink_decoder_finish(&hsd); + ASSERT_EQ(HSDR_FINISH_DONE, fres); + } + + if (polled > input_size) { + FAILm("Decompressed data is larger than original input"); + } + } + if (log_lvl > 0) printf("decompressed: %u\n", polled); + if (polled != input_size) { + FAILm("Decompressed length does not match original input length"); + } + + if (log_lvl > 1) dump_buf("decomp", decomp, polled); + for (size_t i=0; i out[%zd] == 0x%02x ('%c')\n", + j, input[j], isprint(input[j]) ? input[j] : '.', + j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.'); + } + } + } + ASSERT_EQ(input[i], decomp[i]); + } + free(comp); + free(decomp); + PASS(); +} + +TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed) { + uint8_t input[size]; + fill_with_pseudorandom_letters(input, size, seed); + return compress_and_expand_and_check(input, size, 0); +} + +SUITE(integration) { +#if __STDC_VERSION__ >= 19901L + for (uint32_t size=1; size < 64*1024; size <<= 1) { + if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); + for (uint32_t seed=1; seed<=100; seed++) { + if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); + RUN_TESTp(pseudorandom_data_should_match, size, seed); + } + } +#endif +} + +/* Add all the definitions that need to be in the test runner's main file. */ +GREATEST_MAIN_DEFS(); + +int main(int argc, char **argv) { + GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ + printf("INPUT_BUFFER_SIZE: %u\n", HEATSHRINK_STATIC_INPUT_BUFFER_SIZE); + printf("WINDOW_BITS: %u\n", HEATSHRINK_STATIC_WINDOW_BITS); + printf("LOOKAHEAD_BITS: %u\n", HEATSHRINK_STATIC_LOOKAHEAD_BITS); + + printf("sizeof(heatshrink_encoder): %zd\n", sizeof(heatshrink_encoder)); + printf("sizeof(heatshrink_decoder): %zd\n", sizeof(heatshrink_decoder)); + RUN_SUITE(integration); + GREATEST_MAIN_END(); /* display results */ +} diff --git a/espfs/heatshrink_decoder.c b/espfs/heatshrink_decoder.c index 522b560..9a4a9f7 100644 --- a/espfs/heatshrink_decoder.c +++ b/espfs/heatshrink_decoder.c @@ -13,7 +13,7 @@ #endif #include "heatshrink_config_custom.h" -#include "../lib/heatshrink/heatshrink_decoder.c" +#include "heatshrink/heatshrink_decoder.c" #endif diff --git a/espfs/mkespfsimage/Makefile b/espfs/mkespfsimage/Makefile index 8a39d46..26484d4 100644 --- a/espfs/mkespfsimage/Makefile +++ b/espfs/mkespfsimage/Makefile @@ -1,24 +1,36 @@ GZIP_COMPRESSION ?= no USE_HEATSHRINK ?= yes -CFLAGS=-I../../lib/heatshrink -I.. -std=gnu99 +TARGET = mkespfsimage.exe + +CC = gcc +LD = $(CC) +CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 +LDFLAGS=-Lmman-win32 -lmman + ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP +CFLAGS += -DESPFS_GZIP +LDFLAGS += -lz endif ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK +CFLAGS += -DESPFS_HEATSHRINK endif -OBJS=main.o heatshrink_encoder.o -TARGET=mkespfsimage +OBJECTS = main.o heatshrink_encoder.o -$(TARGET): $(OBJS) -ifeq ("$(GZIP_COMPRESSION)","yes") - $(CC) -o $@ $^ -lz -else - $(CC) -o $@ $^ -endif +all: libmman $(TARGET) + +libmman: + $(Q) make -C mman-win32 + +$(TARGET): $(OBJECTS) + $(LD) -o $@ $^ $(LDFLAGS) + +%.o: %.c + $(CC) $(CFLAGS) -o $@ $^ clean: - rm -f $(TARGET) $(OBJS) \ No newline at end of file + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all clean diff --git a/espfs/mkespfsimage/Makefile.linux b/espfs/mkespfsimage/Makefile.linux new file mode 100644 index 0000000..f1ae1aa --- /dev/null +++ b/espfs/mkespfsimage/Makefile.linux @@ -0,0 +1,24 @@ +GZIP_COMPRESSION ?= no +USE_HEATSHRINK ?= yes + +CFLAGS=-I../../heatshrink -I../../include -I.. -std=gnu99 +ifeq ("$(GZIP_COMPRESSION)","yes") +CFLAGS += -DESPFS_GZIP +endif + +ifeq ("$(USE_HEATSHRINK)","yes") +CFLAGS += -DESPFS_HEATSHRINK +endif + +OBJS=main.o heatshrink_encoder.o +TARGET=mkespfsimage + +$(TARGET): $(OBJS) +ifeq ("$(GZIP_COMPRESSION)","yes") + $(CC) -o $@ $^ -lz +else + $(CC) -o $@ $^ +endif + +clean: + rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.windows b/espfs/mkespfsimage/Makefile.windows new file mode 100644 index 0000000..15aa8e3 --- /dev/null +++ b/espfs/mkespfsimage/Makefile.windows @@ -0,0 +1,33 @@ +GZIP_COMPRESSION ?= no +USE_HEATSHRINK ?= yes + +TARGET = mkespfsimage.exe + +CC = gcc +LD = $(CC) +CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 +LDFLAGS=-Lmman-win32 -lmman + +ifeq ("$(GZIP_COMPRESSION)","yes") +CFLAGS += -DESPFS_GZIP +LDFLAGS += -lz +endif + +ifeq ("$(USE_HEATSHRINK)","yes") +CFLAGS += -DESPFS_HEATSHRINK +endif + +OBJECTS = main.o heatshrink_encoder.o + +all: $(TARGET) + +$(TARGET): $(OBJECTS) + $(LD) -o $@ $^ $(LDFLAGS) + +%.o: %.c + $(CC) $(CFLAGS) -o $@ $^ + +clean: + rm -f $(OBJECTS) $(TARGET) + +.PHONY: all clean diff --git a/espfs/mkespfsimage/build-linux.sh b/espfs/mkespfsimage/build-linux.sh new file mode 100644 index 0000000..69e3166 --- /dev/null +++ b/espfs/mkespfsimage/build-linux.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +make -f Makefile.linux clean +make -f Makefile.linux USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/build-win32.sh b/espfs/mkespfsimage/build-win32.sh new file mode 100644 index 0000000..f552ea9 --- /dev/null +++ b/espfs/mkespfsimage/build-win32.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd mman-win32 +./configure && make +cd .. +make -f Makefile.windows clean +make -f Makefile.windows USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/espfsformat.h b/espfs/mkespfsimage/espfsformat.h new file mode 100644 index 0000000..8ce5549 --- /dev/null +++ b/espfs/mkespfsimage/espfsformat.h @@ -0,0 +1,33 @@ +#ifndef ESPROFSFORMAT_H +#define ESPROFSFORMAT_H + +/* +Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module. +Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files, +headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself +when trying to do a <4byte or unaligned read. +*/ + +/* +The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data. +Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header +with the FLAG_LASTFILE flag set. +*/ + + +#define FLAG_LASTFILE (1<<0) +#define FLAG_GZIP (1<<1) +#define COMPRESS_NONE 0 +#define COMPRESS_HEATSHRINK 1 +#define ESPFS_MAGIC 0x73665345 + +typedef struct { + int32_t magic; + int8_t flags; + int8_t compression; + int16_t nameLen; + int32_t fileLenComp; + int32_t fileLenDecomp; +} __attribute__((packed)) EspFsHeader; + +#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/heatshrink_encoder.c b/espfs/mkespfsimage/heatshrink_encoder.c index b563970..fd82c9a 100644 --- a/espfs/mkespfsimage/heatshrink_encoder.c +++ b/espfs/mkespfsimage/heatshrink_encoder.c @@ -1,4 +1,4 @@ //Stupid wraparound include to make sure object file doesn't end up in heatshrink dir #ifdef ESPFS_HEATSHRINK -#include "../lib/heatshrink/heatshrink_encoder.c" +#include "../heatshrink/heatshrink_encoder.c" #endif \ No newline at end of file diff --git a/espfs/mkespfsimage/main.c b/espfs/mkespfsimage/main.c index 4d5da0d..20a2b00 100644 --- a/espfs/mkespfsimage/main.c +++ b/espfs/mkespfsimage/main.c @@ -5,22 +5,31 @@ #include #include #include +#include +#include "espfs.h" +#ifdef __MINGW32__ +#include +#else #include +#endif +#ifdef __WIN32__ +#include +#else #include +#endif #include -#include "espfs.h" #include "espfsformat.h" //Heatshrink #ifdef ESPFS_HEATSHRINK -#include "heatshrink_common.h" -#include "heatshrink_config.h" -#include "heatshrink_encoder.h" +#include "../heatshrink/heatshrink_common.h" +#include "../heatshrink/heatshrink_config.h" +#include "../heatshrink/heatshrink_encoder.h" #endif //Gzip #ifdef ESPFS_GZIP -// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" +// If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" // to install missing package. #include #endif @@ -175,7 +184,7 @@ int parseGzipExtensions(char *input) { } #endif -int handleFile(int f, char *name, int compression, int level, char **compName, off_t *csizePtr) { +int handleFile(int f, char *name, int compression, int level, char **compName) { char *fdat, *cdat; off_t size, csize; EspFsHeader h; @@ -229,7 +238,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName, o h.nameLen=htoxs(h.nameLen); h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); - + write(1, &h, sizeof(EspFsHeader)); write(1, name, nameLen); while (nameLen&3) { @@ -257,7 +266,6 @@ int handleFile(int f, char *name, int compression, int level, char **compName, o *compName = "unknown"; } } - *csizePtr = csize; return (csize*100)/size; } @@ -310,7 +318,7 @@ int main(int argc, char **argv) { #ifdef ESPFS_GZIP if (gzipExtensions == NULL) { - parseGzipExtensions(strdup("html,css,js,ico")); + parseGzipExtensions(strdup("html,css,js")); } #endif @@ -334,6 +342,10 @@ int main(int argc, char **argv) { exit(0); } +#ifdef __WIN32__ + setmode(fileno(stdout), _O_BINARY); +#endif + while(fgets(fileName, sizeof(fileName), stdin)) { //Kill off '\n' at the end fileName[strlen(fileName)-1]=0; @@ -347,9 +359,8 @@ int main(int argc, char **argv) { f=open(fileName, O_RDONLY); if (f>0) { char *compName = "unknown"; - off_t csize; - rate=handleFile(f, realName, compType, compLvl, &compName, &csize); - fprintf(stderr, "%-16s (%3d%%, %s, %4u bytes)\n", realName, rate, compName, (uint32_t)csize); + rate=handleFile(f, realName, compType, compLvl, &compName); + fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); close(f); } else { perror(fileName); diff --git a/espfs/mkespfsimage/mman-win32/Makefile b/espfs/mkespfsimage/mman-win32/Makefile new file mode 100644 index 0000000..5624c1b --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/Makefile @@ -0,0 +1,48 @@ +# +# mman-win32 (mingw32) Makefile +# +include config.mak + +ifeq ($(BUILD_STATIC),yes) + TARGETS+=libmman.a + INSTALL+=static-install +endif +ifeq ($(BUILD_MSVC),yes) + SHFLAGS+=-Wl,--output-def,libmman.def + INSTALL+=lib-install +endif + +all: $(TARGETS) + +mman.o: mman.c mman.h + $(CC) -o mman.o -c mman.c -Wall -O3 -fomit-frame-pointer + +libmman.a: mman.o + $(AR) cru libmman.a mman.o + $(RANLIB) libmman.a + +static-install: + mkdir -p $(DESTDIR)$(libdir) + cp libmman.a $(DESTDIR)$(libdir) + mkdir -p $(DESTDIR)$(incdir) + cp mman.h $(DESTDIR)$(incdir) + +lib-install: + mkdir -p $(DESTDIR)$(libdir) + cp libmman.lib $(DESTDIR)$(libdir) + +install: $(INSTALL) + +test.exe: test.c mman.c mman.h + $(CC) -o test.exe test.c -L. -lmman + +test: $(TARGETS) test.exe + test.exe + +clean:: + rm -f mman.o libmman.a libmman.def libmman.lib test.exe *.dat + +distclean: clean + rm -f config.mak + +.PHONY: clean distclean install test diff --git a/espfs/mkespfsimage/mman-win32/config.mak b/espfs/mkespfsimage/mman-win32/config.mak new file mode 100644 index 0000000..2d2ae1b --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/config.mak @@ -0,0 +1,11 @@ +# Automatically generated by configure +PREFIX=/mingw +libdir=/mingw/lib +incdir=/mingw/include/sys +AR=ar +CC=gcc +RANLIB=ranlib +STRIP=strip +BUILD_STATIC=yes +BUILD_MSVC= +LIBCMD=echo ignoring lib diff --git a/espfs/mkespfsimage/mman-win32/configure b/espfs/mkespfsimage/mman-win32/configure new file mode 100644 index 0000000..c928f11 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/configure @@ -0,0 +1,157 @@ +#!/bin/sh +# mmap-win32 configure script +# +# Parts copied from FFmpeg's configure +# + +set_all(){ + value=$1 + shift + for var in $*; do + eval $var=$value + done +} + +enable(){ + set_all yes $* +} + +disable(){ + set_all no $* +} + +enabled(){ + eval test "x\$$1" = "xyes" +} + +disabled(){ + eval test "x\$$1" = "xno" +} + +show_help(){ + echo "Usage: configure [options]" + echo "Options: [defaults in brackets after descriptions]" + echo "All \"enable\" options have \"disable\" counterparts" + echo + echo " --help print this message" + echo " --prefix=PREFIX install in PREFIX [$PREFIX]" + echo " --libdir=DIR install libs in DIR [$PREFIX/lib]" + echo " --incdir=DIR install includes in DIR [$PREFIX/include]" + echo " --enable-static build static libraries [yes]" + echo " --enable-msvc create msvc-compatible import lib [auto]" + echo + echo " --cc=CC use C compiler CC [$cc_default]" + echo " --cross-prefix=PREFIX use PREFIX for compilation tools [$cross_prefix]" + exit 1 +} + +die_unknown(){ + echo "Unknown option \"$1\"." + echo "See $0 --help for available options." + exit 1 +} + +PREFIX="/mingw" +libdir="${PREFIX}/lib" +incdir="${PREFIX}/include/sys" +ar="ar" +cc_default="gcc" +ranlib="ranlib" +strip="strip" + +DEFAULT="msvc +" + +DEFAULT_YES="static + stripping +" + +CMDLINE_SELECT="$DEFAULT + $DEFAULT_NO + $DEFAULT_YES +" + +enable $DEFAULT_YES +disable $DEFAULT_NO + +for opt do + optval="${opt#*=}" + case "$opt" in + --help) + show_help + ;; + --prefix=*) + PREFIX="$optval" + ;; + --libdir=*) + libdir="$optval" + ;; + --incdir=*) + incdir="$optval" + ;; + --cc=*) + cc="$optval" + ;; + --cross-prefix=*) + cross_prefix="$optval" + ;; + --enable-?*|--disable-?*) + eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'` + echo "$CMDLINE_SELECT" | grep -q "^ *$option\$" || die_unknown $opt + $action $option + ;; + *) + die_unknown $opt + ;; + esac +done + +ar="${cross_prefix}${ar}" +cc_default="${cross_prefix}${cc_default}" +ranlib="${cross_prefix}${ranlib}" +strip="${cross_prefix}${strip}" + +if ! test -z $cc; then + cc_default="${cc}" +fi +cc="${cc_default}" + +disabled static && { + echo "At least one library type must be set."; + exit 1; +} + +if enabled msvc; then + lib /? > /dev/null 2>&1 /dev/null || { + echo "MSVC's lib command not found." + echo "Make sure MSVC is installed and its bin folder is in your \$PATH." + exit 1 + } +fi + +if ! enabled stripping; then + strip="echo ignoring strip" +fi + +enabled msvc && libcmd="lib" || libcmd="echo ignoring lib" + +echo "# Automatically generated by configure" > config.mak +echo "PREFIX=$PREFIX" >> config.mak +echo "libdir=$libdir" >> config.mak +echo "incdir=$incdir" >> config.mak +echo "AR=$ar" >> config.mak +echo "CC=$cc" >> config.mak +echo "RANLIB=$ranlib" >> config.mak +echo "STRIP=$strip" >> config.mak +echo "BUILD_STATIC=$static" >> config.mak +echo "BUILD_MSVC=$msvc" >> config.mak +echo "LIBCMD=$libcmd" >> config.mak + +echo "prefix: $PREFIX" +echo "libdir: $libdir" +echo "incdir: $incdir" +echo "ar: $ar" +echo "cc: $cc" +echo "ranlib: $ranlib" +echo "strip: $strip" +echo "static: $static" diff --git a/espfs/mkespfsimage/mman-win32/mman.c b/espfs/mkespfsimage/mman-win32/mman.c new file mode 100644 index 0000000..486ed94 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/mman.c @@ -0,0 +1,180 @@ + +#include +#include +#include + +#include "mman.h" + +#ifndef FILE_MAP_EXECUTE +#define FILE_MAP_EXECUTE 0x0020 +#endif /* FILE_MAP_EXECUTE */ + +static int __map_mman_error(const DWORD err, const int deferr) +{ + if (err == 0) + return 0; + //TODO: implement + return err; +} + +static DWORD __map_mmap_prot_page(const int prot) +{ + DWORD protect = 0; + + if (prot == PROT_NONE) + return protect; + + if ((prot & PROT_EXEC) != 0) + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; + } + else + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_READWRITE : PAGE_READONLY; + } + + return protect; +} + +static DWORD __map_mmap_prot_file(const int prot) +{ + DWORD desiredAccess = 0; + + if (prot == PROT_NONE) + return desiredAccess; + + if ((prot & PROT_READ) != 0) + desiredAccess |= FILE_MAP_READ; + if ((prot & PROT_WRITE) != 0) + desiredAccess |= FILE_MAP_WRITE; + if ((prot & PROT_EXEC) != 0) + desiredAccess |= FILE_MAP_EXECUTE; + + return desiredAccess; +} + +void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off) +{ + HANDLE fm, h; + + void * map = MAP_FAILED; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable: 4293) +#endif + + const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)off : (DWORD)(off & 0xFFFFFFFFL); + const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL); + const DWORD protect = __map_mmap_prot_page(prot); + const DWORD desiredAccess = __map_mmap_prot_file(prot); + + const off_t maxSize = off + (off_t)len; + + const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL); + const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL); + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + errno = 0; + + if (len == 0 + /* Unsupported flag combinations */ + || (flags & MAP_FIXED) != 0 + /* Usupported protection combinations */ + || prot == PROT_EXEC) + { + errno = EINVAL; + return MAP_FAILED; + } + + h = ((flags & MAP_ANONYMOUS) == 0) ? + (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE; + + if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return MAP_FAILED; + } + + fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL); + + if (fm == NULL) + { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len); + + CloseHandle(fm); + + if (map == NULL) + { + errno = __map_mman_error(GetLastError(), EPERM); + return MAP_FAILED; + } + + return map; +} + +int munmap(void *addr, size_t len) +{ + if (UnmapViewOfFile(addr)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int mprotect(void *addr, size_t len, int prot) +{ + DWORD newProtect = __map_mmap_prot_page(prot); + DWORD oldProtect = 0; + + if (VirtualProtect(addr, len, newProtect, &oldProtect)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int msync(void *addr, size_t len, int flags) +{ + if (FlushViewOfFile(addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int mlock(const void *addr, size_t len) +{ + if (VirtualLock((LPVOID)addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} + +int munlock(const void *addr, size_t len) +{ + if (VirtualUnlock((LPVOID)addr, len)) + return 0; + + errno = __map_mman_error(GetLastError(), EPERM); + + return -1; +} diff --git a/espfs/mkespfsimage/mman-win32/mman.h b/espfs/mkespfsimage/mman-win32/mman.h new file mode 100644 index 0000000..ffa3748 --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/mman.h @@ -0,0 +1,55 @@ +/* + * sys/mman.h + * mman-win32 + */ + +#ifndef _SYS_MMAN_H_ +#define _SYS_MMAN_H_ + +#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later. +#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows. +#endif + +/* All the headers include this file. */ +#ifndef _MSC_VER +#include <_mingw.h> +#endif + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define PROT_NONE 0 +#define PROT_READ 1 +#define PROT_WRITE 2 +#define PROT_EXEC 4 + +#define MAP_FILE 0 +#define MAP_SHARED 1 +#define MAP_PRIVATE 2 +#define MAP_TYPE 0xf +#define MAP_FIXED 0x10 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS + +#define MAP_FAILED ((void *)-1) + +/* Flags for msync. */ +#define MS_ASYNC 1 +#define MS_SYNC 2 +#define MS_INVALIDATE 4 + +void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off); +int munmap(void *addr, size_t len); +int mprotect(void *addr, size_t len, int prot); +int msync(void *addr, size_t len, int flags); +int mlock(const void *addr, size_t len); +int munlock(const void *addr, size_t len); + +#ifdef __cplusplus +}; +#endif + +#endif /* _SYS_MMAN_H_ */ diff --git a/espfs/mkespfsimage/mman-win32/test.c b/espfs/mkespfsimage/mman-win32/test.c new file mode 100644 index 0000000..9374b9f --- /dev/null +++ b/espfs/mkespfsimage/mman-win32/test.c @@ -0,0 +1,235 @@ + +#include "mman.h" + +#include +#include +#include + +#ifndef NULL +#define NULL (void*)0 +#endif + +const char* map_file_name = "map_file.dat"; + +int test_anon_map_readwrite() +{ + void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_readonly() +{ + void* map = mmap(NULL, 1024, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_writeonly() +{ + void* map = mmap(NULL, 1024, PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno); + + return result; +} + +int test_anon_map_readonly_nowrite() +{ + void* map = mmap(NULL, 1024, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) + { + printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + return -1; + } + + if (*((unsigned char*)map) != 0) + printf("test_anon_map_readonly_nowrite (MAP_ANONYMOUS, PROT_READ) returned unexpected value: %d\n", + (int)*((unsigned char*)map)); + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno); + + return result; +} + +int test_file_map_readwrite() +{ + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + + void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + return -1; + } + + *((unsigned char*)map) = 1; + + int result = munmap(map, 1024); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + + close(o); + + /*TODO: get file info and content and compare it with the sources conditions */ + unlink(map_file_name); + + return result; +} + +int test_file_map_mlock_munlock() +{ + const size_t map_size = 1024; + + int result = 0; + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + if (o == -1) + { + printf("unable to create file %s: %d\n", map_file_name, errno); + return -1; + } + + void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + result = -1; + goto done_close; + } + + if (mlock(map, map_size) != 0) + { + printf("mlock returned unexpected error: %d\n", errno); + result = -1; + goto done_munmap; + } + + *((unsigned char*)map) = 1; + + if (munlock(map, map_size) != 0) + { + printf("munlock returned unexpected error: %d\n", errno); + result = -1; + } + +done_munmap: + result = munmap(map, map_size); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + +done_close: + close(o); + + unlink(map_file_name); +done: + return result; +} + +int test_file_map_msync() +{ + const size_t map_size = 1024; + + int result = 0; + mode_t mode = S_IRUSR | S_IWUSR; + int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode); + if (o == -1) + { + printf("unable to create file %s: %d\n", map_file_name, errno); + return -1; + } + + void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0); + if (map == MAP_FAILED) + { + printf("mmap returned unexpected error: %d\n", errno); + result = -1; + goto done_close; + } + + *((unsigned char*)map) = 1; + + if (msync(map, map_size, MS_SYNC) != 0) + { + printf("msync returned unexpected error: %d\n", errno); + result = -1; + } + + result = munmap(map, map_size); + + if (result != 0) + printf("munmap returned unexpected error: %d\n", errno); + +done_close: + close(o); + + unlink(map_file_name); +done: + return result; +} + +#define EXEC_TEST(name) \ + if (name() != 0) { result = -1; printf( #name ": fail\n"); } \ + else { printf(#name ": pass\n"); } + +int main() +{ + int result = 0; + + EXEC_TEST(test_anon_map_readwrite); + //NOTE: this test must cause an access violation exception + //EXEC_TEST(test_anon_map_readonly); + EXEC_TEST(test_anon_map_readonly_nowrite); + EXEC_TEST(test_anon_map_writeonly); + + EXEC_TEST(test_file_map_readwrite); + EXEC_TEST(test_file_map_mlock_munlock); + EXEC_TEST(test_file_map_msync); + //TODO: EXEC_TEST(test_file_map_mprotect); + + return result; +} diff --git a/espmake.cmd b/espmake.cmd new file mode 100644 index 0000000..42e25a6 --- /dev/null +++ b/espmake.cmd @@ -0,0 +1,7 @@ +@echo off + +REM remove automatic created obj folder +rd obj /S /Q >nul 2>&1 + +PATH=%PATH%;C:\Espressif\xtensa-lx106-elf\bin;C:\MinGW\bin;C:\MinGW\msys\1.0\bin;C:\espressif\git-bin;C:\espressif\java-bin;C:\Python27 +make -f Makefile %1 %2 %3 %4 %5 \ No newline at end of file From 6867989b477da5fc5fdd652e9aea131b95f5d3b8 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Fri, 21 Aug 2015 23:10:41 -0700 Subject: [PATCH 18/24] remove heatshrink submodule; switch to SDK 1.3 --- .gitmodules | 3 --- Makefile | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 .gitmodules diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 772d424..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "lib/heatshrink"] - path = lib/heatshrink - url = https://github.com/atomicobject/heatshrink.git diff --git a/Makefile b/Makefile index 8629ddb..5b694ae 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ XTENSA_TOOLS_ROOT ?= $(abspath ../esp-open-sdk/xtensa-lx106-elf/bin)/ # Base directory of the ESP8266 SDK package, absolute # Typically you'll download from Espressif's BBS, http://bbs.espressif.com/viewforum.php?f=5 -SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.2.0) +SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.3.0) # Esptool.py path and port, only used for 1-time serial flashing # Typically you'll use https://github.com/themadinventor/esptool @@ -30,7 +30,7 @@ ESPBAUD ?= 460800 # --------------- chipset configuration --------------- # Pick your flash size: "512KB", "1MB", or "4MB" -FLASH_SIZE ?= 512KB +FLASH_SIZE ?= 4MB ifeq ("$(FLASH_SIZE)","512KB") # Winbond 25Q40 512KB flash, typ for esp-01 thru esp-11 From 3d2ac525e3b594a27dc256077f6b1d748694b081 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 05:51:46 -0500 Subject: [PATCH 19/24] Removed heatshrink and modified Makefiles --- Makefile | 17 +- esp-link.vcxproj | 62 +- espfs/espfs.c | 83 +- espfs/espfs.h | 4 - espfs/heatshrink/LICENSE | 14 - espfs/heatshrink/Makefile | 49 - espfs/heatshrink/README.md | 49 - espfs/heatshrink/dec_sm.dot | 52 - espfs/heatshrink/enc_sm.dot | 51 - espfs/heatshrink/greatest.h | 591 ----------- espfs/heatshrink/heatshrink.c | 446 -------- espfs/heatshrink/heatshrink_common.h | 20 - espfs/heatshrink/heatshrink_config.h | 24 - espfs/heatshrink/heatshrink_decoder.c | 382 ------- espfs/heatshrink/heatshrink_decoder.h | 101 -- espfs/heatshrink/heatshrink_encoder.c | 650 ------------ espfs/heatshrink/heatshrink_encoder.h | 109 -- espfs/heatshrink/test_heatshrink_dynamic.c | 999 ------------------ .../test_heatshrink_dynamic_theft.c | 521 --------- espfs/heatshrink/test_heatshrink_static.c | 167 --- espfs/heatshrink_config_custom.h | 30 - espfs/heatshrink_decoder.c | 19 - espfs/mkespfsimage/Makefile | 40 +- espfs/mkespfsimage/Makefile.linux | 24 - espfs/mkespfsimage/Makefile.windows | 33 - espfs/mkespfsimage/build-linux.sh | 4 - espfs/mkespfsimage/build-win32.sh | 7 - espfs/mkespfsimage/espfsformat.h | 33 - espfs/mkespfsimage/heatshrink_encoder.c | 4 - espfs/mkespfsimage/main.c | 87 +- include/esp8266.h | 5 +- include/espmissingincludes.h | 10 +- 32 files changed, 50 insertions(+), 4637 deletions(-) delete mode 100644 espfs/heatshrink/LICENSE delete mode 100644 espfs/heatshrink/Makefile delete mode 100644 espfs/heatshrink/README.md delete mode 100644 espfs/heatshrink/dec_sm.dot delete mode 100644 espfs/heatshrink/enc_sm.dot delete mode 100644 espfs/heatshrink/greatest.h delete mode 100644 espfs/heatshrink/heatshrink.c delete mode 100644 espfs/heatshrink/heatshrink_common.h delete mode 100644 espfs/heatshrink/heatshrink_config.h delete mode 100644 espfs/heatshrink/heatshrink_decoder.c delete mode 100644 espfs/heatshrink/heatshrink_decoder.h delete mode 100644 espfs/heatshrink/heatshrink_encoder.c delete mode 100644 espfs/heatshrink/heatshrink_encoder.h delete mode 100644 espfs/heatshrink/test_heatshrink_dynamic.c delete mode 100644 espfs/heatshrink/test_heatshrink_dynamic_theft.c delete mode 100644 espfs/heatshrink/test_heatshrink_static.c delete mode 100644 espfs/heatshrink_config_custom.h delete mode 100644 espfs/heatshrink_decoder.c delete mode 100644 espfs/mkespfsimage/Makefile.linux delete mode 100644 espfs/mkespfsimage/Makefile.windows delete mode 100644 espfs/mkespfsimage/build-linux.sh delete mode 100644 espfs/mkespfsimage/build-win32.sh delete mode 100644 espfs/mkespfsimage/espfsformat.h delete mode 100644 espfs/mkespfsimage/heatshrink_encoder.c diff --git a/Makefile b/Makefile index 297e4fe..af5fcbf 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,6 @@ CHANGE_TO_STA ?= yes # This could speed up the downloading of these files, but might break compatibility with older # web browsers not supporting gzip encoding because Accept-Encoding is simply ignored. # Enable this option if you have large static files to serve (for e.g. JQuery, Twitter bootstrap) -# By default only js, css and html files are compressed using heatshrink. # If you have text based static files with different extensions what you want to serve compressed # then you will need to add the extension to the following places: # - Add the extension to this Makefile at the webpages.espfs target to the find command @@ -127,10 +126,6 @@ GZIP_COMPRESSION ?= yes COMPRESS_W_YUI ?= yes YUI-COMPRESSOR ?= yuicompressor-2.4.8.jar -# If USE_HEATSHRINK is set to "yes" then the espfs files will be compressed with Heatshrink and -# decompressed on the fly while reading the file. -# Because the decompression is done in the esp8266, it does not require any support in the browser. -USE_HEATSHRINK ?= no # -------------- End of config options ------------- @@ -147,7 +142,7 @@ APPGEN_TOOL ?= gen_appbin.py # which modules (subdirectories) of the project to include in compiling MODULES = espfs httpd user serial -EXTRA_INCDIR = include . # lib/heatshrink/ +EXTRA_INCDIR = include . # libraries used in this project, mainly provided by the SDK LIBS = c gcc hal phy pp net80211 wpa main lwip @@ -216,10 +211,6 @@ ifeq ("$(GZIP_COMPRESSION)","yes") CFLAGS += -DGZIP_COMPRESSION endif -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - ifeq ("$(CHANGE_TO_STA)","yes") CFLAGS += -DCHANGE_TO_STA endif @@ -299,7 +290,11 @@ flash: all yui/$(YUI-COMPRESSOR): $(Q) mkdir -p yui + ifeq ($(OS),Windows_NT) cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + else + cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) + endif ifeq ("$(COMPRESS_W_YUI)","yes") $(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR) @@ -351,7 +346,7 @@ build/eagle.esphttpd2.v6.ld: $(SDK_LDDIR)/eagle.app.v6.new.1024.app2.ld endif espfs/mkespfsimage/mkespfsimage: espfs/mkespfsimage/ - $(Q) $(MAKE) -C espfs/mkespfsimage USE_HEATSHRINK="$(USE_HEATSHRINK)" GZIP_COMPRESSION="$(GZIP_COMPRESSION)" + $(Q) $(MAKE) -C espfs/mkespfsimage GZIP_COMPRESSION="$(GZIP_COMPRESSION)" release: all $(Q) rm -rf release; mkdir -p release/esp-link diff --git a/esp-link.vcxproj b/esp-link.vcxproj index 45dd2ae..d462b62 100644 --- a/esp-link.vcxproj +++ b/esp-link.vcxproj @@ -28,7 +28,7 @@ __ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__ - .\mqtt\include;.\serial;.\user;.\espfs;.\httpd;.\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include;c:\Espressif\sdk\include + .\serial;.\user;.\espfs;.\httpd;.\include;..\esp_iot_sdk_v1.3.0\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include @@ -54,17 +54,10 @@ - - - - - - - @@ -72,8 +65,6 @@ - - @@ -81,35 +72,13 @@ - - - - - - - - - - - - - - - - - - - - - - @@ -119,8 +88,6 @@ - - @@ -130,40 +97,13 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/espfs/espfs.c b/espfs/espfs.c index 1f503d5..128ca29 100644 --- a/espfs/espfs.c +++ b/espfs/espfs.c @@ -7,9 +7,9 @@ It's written for use with httpd, but doesn't need to be used as such. /* * ---------------------------------------------------------------------------- * "THE BEER-WARE LICENSE" (Revision 42): - * Jeroen Domburg wrote this file. As long as you retain - * this notice you can do whatever you want with this stuff. If we meet some day, - * and you think this stuff is worth it, you can buy me a beer in return. + * Jeroen Domburg wrote this file. As long as you retain + * this notice you can do whatever you want with this stuff. If we meet some day, + * and you think this stuff is worth it, you can buy me a beer in return. * ---------------------------------------------------------------------------- */ @@ -40,14 +40,8 @@ It's written for use with httpd, but doesn't need to be used as such. #include "espfsformat.h" #include "espfs.h" -#ifdef ESPFS_HEATSHRINK -#include "heatshrink_config_custom.h" -#include "heatshrink/heatshrink_decoder.h" -#endif - static char* espFsData = NULL; - struct EspFsFile { EspFsHeader *header; char decompressor; @@ -147,19 +141,19 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { return NULL; } if (h.flags&FLAG_LASTFILE) { - os_printf("End of image.\n"); + //os_printf("End of image.\n"); return NULL; } //Grab the name of the file. - p+=sizeof(EspFsHeader); + p+=sizeof(EspFsHeader); os_memcpy(namebuf, p, sizeof(namebuf)); -// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", +// os_printf("Found file '%s'. Namelen=%x fileLenComp=%x, compr=%d flags=%d\n", // namebuf, (unsigned int)h.nameLen, (unsigned int)h.fileLenComp, h.compression, h.flags); if (os_strcmp(namebuf, fileName)==0) { //Yay, this is the file we need! p+=h.nameLen; //Skip to content. r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem -// os_printf("Alloc %p\n", r); + //os_printf("Alloc %p[%d]\n", r, sizeof(EspFsFile)); if (r==NULL) return NULL; r->header=(EspFsHeader *)hpos; r->decompressor=h.compression; @@ -168,18 +162,6 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) { r->posDecomp=0; if (h.compression==COMPRESS_NONE) { r->decompData=NULL; -#ifdef ESPFS_HEATSHRINK - } else if (h.compression==COMPRESS_HEATSHRINK) { - //File is compressed with Heatshrink. - char parm; - heatshrink_decoder *dec; - //Decoder params are stored in 1st byte. - memcpyAligned(&parm, r->posComp, 1); - r->posComp++; - os_printf("Heatshrink compressed file; decode parms = %x\n", parm); - dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf); - r->decompData=dec; -#endif } else { os_printf("Invalid compression: %d\n", h.compression); return NULL; @@ -210,48 +192,6 @@ int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) { fh->posComp+=len; // os_printf("Done reading %d bytes, pos=%x\n", len, fh->posComp); return len; -#ifdef ESPFS_HEATSHRINK - } else if (fh->decompressor==COMPRESS_HEATSHRINK) { - int decoded=0; - size_t elen, rlen; - char ebuff[16]; - heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData; -// os_printf("Alloc %p\n", dec); - if (fh->posDecomp == fdlen) { - return 0; - } - - // We must ensure that whole file is decompressed and written to output buffer. - // This means even when there is no input data (elen==0) try to poll decoder until - // posDecomp equals decompressed file length - - while(decodedposComp - fh->posStart); - if (elen>0) { - memcpyAligned(ebuff, fh->posComp, 16); - heatshrink_decoder_sink(dec, (uint8_t *)ebuff, (elen>16)?16:elen, &rlen); - fh->posComp+=rlen; - } - //Grab decompressed data and put into buff - heatshrink_decoder_poll(dec, (uint8_t *)buff, len-decoded, &rlen); - fh->posDecomp+=rlen; - buff+=rlen; - decoded+=rlen; - -// os_printf("Elen %d rlen %d d %d pd %ld fdl %d\n",elen,rlen,decoded, fh->posDecomp, fdlen); - - if (elen == 0) { - if (fh->posDecomp == fdlen) { -// os_printf("Decoder finish\n"); - heatshrink_decoder_finish(dec); - } - return decoded; - } - } - return len; -#endif } return 0; } @@ -259,14 +199,7 @@ int ICACHE_FLASH_ATTR espFsRead(EspFsFile *fh, char *buff, int len) { //Close the file. void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) { if (fh==NULL) return; -#ifdef ESPFS_HEATSHRINK - if (fh->decompressor==COMPRESS_HEATSHRINK) { - heatshrink_decoder *dec=(heatshrink_decoder *)fh->decompData; - heatshrink_decoder_free(dec); -// os_printf("Freed %p\n", dec); - } -#endif -// os_printf("Freed %p\n", fh); + //os_printf("Freed %p\n", fh); os_free(fh); } diff --git a/espfs/espfs.h b/espfs/espfs.h index c41df0a..c8e13e7 100644 --- a/espfs/espfs.h +++ b/espfs/espfs.h @@ -1,10 +1,6 @@ #ifndef ESPFS_H #define ESPFS_H -// This define is done in Makefile. If you do not use default Makefile, uncomment -// to be able to use Heatshrink-compressed espfs images. -//#define ESPFS_HEATSHRINK - typedef enum { ESPFS_INIT_RESULT_OK, ESPFS_INIT_RESULT_NO_IMAGE, diff --git a/espfs/heatshrink/LICENSE b/espfs/heatshrink/LICENSE deleted file mode 100644 index 31ec3df..0000000 --- a/espfs/heatshrink/LICENSE +++ /dev/null @@ -1,14 +0,0 @@ -Copyright (c) 2013, Scott Vokes -All rights reserved. - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/espfs/heatshrink/Makefile b/espfs/heatshrink/Makefile deleted file mode 100644 index d8e6875..0000000 --- a/espfs/heatshrink/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -PROJECT = heatshrink -#OPTIMIZE = -O0 -#OPTIMIZE = -Os -OPTIMIZE = -O3 -WARN = -Wall -Wextra -pedantic #-Werror -CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE} -CFLAGS += -Wmissing-prototypes -CFLAGS += -Wstrict-prototypes -CFLAGS += -Wmissing-declarations - -# If libtheft is available, build additional property-based tests. -# Uncomment these to use it in test_heatshrink_dynamic. -#CFLAGS += -DHEATSHRINK_HAS_THEFT -#LDFLAGS += -ltheft - -all: - @echo "For tests, make test_heatshrink_dynamic (default) or change the" - @echo "config.h to disable static memory and build test_heatshrink_static." - @echo "For the standalone command-line tool, make heatshrink." - -${PROJECT}: heatshrink.c - -OBJS= heatshrink_encoder.o \ - heatshrink_decoder.o \ - -heatshrink: ${OBJS} -test_heatshrink_dynamic: ${OBJS} test_heatshrink_dynamic_theft.o -test_heatshrink_static: ${OBJS} - -*.o: Makefile heatshrink_config.h - -heatshrink_decoder.o: heatshrink_decoder.h heatshrink_common.h -heatshrink_encoder.o: heatshrink_encoder.h heatshrink_common.h - -tags: TAGS - -TAGS: - etags *.[ch] - -diagrams: dec_sm.png enc_sm.png - -dec_sm.png: dec_sm.dot - dot -o $@ -Tpng $< - -enc_sm.png: enc_sm.dot - dot -o $@ -Tpng $< - -clean: - rm -f ${PROJECT} test_heatshrink_{dynamic,static} *.o *.core {dec,enc}_sm.png TAGS diff --git a/espfs/heatshrink/README.md b/espfs/heatshrink/README.md deleted file mode 100644 index ab150ee..0000000 --- a/espfs/heatshrink/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# heatshrink - -A data compression/decompression library for embedded/real-time systems. - -## Key Features: - -- **Low memory usage (as low as 50 bytes)** - It is useful for some cases with less than 50 bytes, and useful - for many general cases with < 300 bytes. -- **Incremental, bounded CPU use** - You can chew on input data in arbitrarily tiny bites. - This is a useful property in hard real-time environments. -- **Can use either static or dynamic memory allocation** - The library doesn't impose any constraints on memory management. -- **ISC license** - You can use it freely, even for commercial purposes. - -## Getting Started: - -There is a standalone command-line program, `heatshrink`, but the -encoder and decoder can also be used as libraries, independent of each -other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and -either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their -respective header) into your project. - -Dynamic allocation is used by default, but in an embedded context, you -probably want to statically allocate the encoder/decoder. Set -`HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`. - -## More Information and Benchmarks: - -heatshrink is based on [LZSS], since it's particularly suitable for -compression in small amounts of memory. It can use an optional, small -[index] to make compression significantly faster, but otherwise can run -in under 100 bytes of memory. The index currently adds 2^(window size+1) -bytes to memory usage for compression, and temporarily allocates 512 -bytes on the stack during index construction. - -For more information, see the [blog post] for an overview, and the -`heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API -documentation. - -[blog post]: http://spin.atomicobject.com/2013/03/14/heatshrink-embedded-data-compression/ -[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/ -[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski - -## Build Status - - [![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink) diff --git a/espfs/heatshrink/dec_sm.dot b/espfs/heatshrink/dec_sm.dot deleted file mode 100644 index 470012f..0000000 --- a/espfs/heatshrink/dec_sm.dot +++ /dev/null @@ -1,52 +0,0 @@ -digraph { - graph [label="Decoder state machine", labelloc="t"] - Start [style="invis", shape="point"] - empty - input_available - yield_literal - backref_index_msb - backref_index_lsb - backref_count_msb - backref_count_lsb - yield_backref - check_for_more_input - done [peripheries=2] - - empty->input_available [label="sink()", color="blue", weight=10] - Start->empty - - input_available->yield_literal [label="pop 1-bit"] - input_available->backref_index_msb [label="pop 0-bit", weight=10] - input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10] - - yield_literal->yield_literal [label="sink()", color="blue"] - yield_literal->yield_literal [label="poll()", color="red"] - yield_literal->check_for_more_input [label="poll(), done", color="red"] - - backref_index_msb->backref_index_msb [label="sink()", color="blue"] - backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10] - backref_index_msb->done [label="finish()", color="blue"] - - backref_index_lsb->backref_index_lsb [label="sink()", color="blue"] - backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10] - backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10] - backref_index_lsb->done [label="finish()", color="blue"] - - backref_count_msb->backref_count_msb [label="sink()", color="blue"] - backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10] - backref_count_msb->done [label="finish()", color="blue"] - - backref_count_lsb->backref_count_lsb [label="sink()", color="blue"] - backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10] - backref_count_lsb->done [label="finish()", color="blue"] - - yield_backref->yield_backref [label="sink()", color="blue"] - yield_backref->yield_backref [label="poll()", color="red"] - yield_backref->check_for_more_input [label="poll(), done", - color="red", weight=10] - - check_for_more_input->empty [label="no"] - check_for_more_input->input_available [label="yes"] - - empty->done [label="finish()", color="blue"] -} diff --git a/espfs/heatshrink/enc_sm.dot b/espfs/heatshrink/enc_sm.dot deleted file mode 100644 index 6d3030f..0000000 --- a/espfs/heatshrink/enc_sm.dot +++ /dev/null @@ -1,51 +0,0 @@ -digraph { - graph [label="Encoder state machine", labelloc="t"] - start [style="invis", shape="point"] - not_full - filled - search - yield_tag_bit - yield_literal - yield_br_length - yield_br_index - save_backlog - flush_bits - done [peripheries=2] - - start->not_full [label="start"] - - not_full->not_full [label="sink(), not full", color="blue"] - not_full->filled [label="sink(), buffer is full", color="blue"] - not_full->filled [label="finish(), set is_finished", color="blue"] - - filled->search [label="indexing (if any)"] - - search->search [label="step"] - search->yield_tag_bit [label="literal"] - search->yield_tag_bit [label="match found"] - search->save_backlog [label="input exhausted"] - - yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"] - yield_tag_bit->yield_literal [label="poll(), literal", color="red"] - yield_tag_bit->yield_br_index [label="poll(), no literal", color="red"] - yield_tag_bit->flush_bits [label="finishing, no literal"] - - yield_literal->yield_literal [label="poll(), full buf", color="red"] - yield_literal->search [label="poll(), no match", color="red"] - yield_literal->yield_tag_bit [label="poll(), match", color="red"] - yield_literal->flush_bits [label="poll(), final literal", color="red"] - - yield_br_index->yield_br_index [label="poll(), full buf", color="red"] - yield_br_index->yield_br_length [label="poll()", color="red"] - - yield_br_length->yield_br_length [label="poll(), full buf", color="red"] - yield_br_length->search [label="done"] - - save_backlog->flush_bits [label="finishing, no literal"] - save_backlog->yield_tag_bit [label="finishing, literal"] - save_backlog->not_full [label="expect more input"] - - flush_bits->flush_bits [label="poll(), full buf", color="red"] - flush_bits->done [label="poll(), flushed", color="red"] - flush_bits->done [label="no more output"] -} diff --git a/espfs/heatshrink/greatest.h b/espfs/heatshrink/greatest.h deleted file mode 100644 index a92c642..0000000 --- a/espfs/heatshrink/greatest.h +++ /dev/null @@ -1,591 +0,0 @@ -/* - * Copyright (c) 2011 Scott Vokes - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef GREATEST_H -#define GREATEST_H - -#define GREATEST_VERSION_MAJOR 0 -#define GREATEST_VERSION_MINOR 9 -#define GREATEST_VERSION_PATCH 3 - -/* A unit testing system for C, contained in 1 file. - * It doesn't use dynamic allocation or depend on anything - * beyond ANSI C89. */ - - -/********************************************************************* - * Minimal test runner template - *********************************************************************/ -#if 0 - -#include "greatest.h" - -TEST foo_should_foo() { - PASS(); -} - -static void setup_cb(void *data) { - printf("setup callback for each test case\n"); -} - -static void teardown_cb(void *data) { - printf("teardown callback for each test case\n"); -} - -SUITE(suite) { - /* Optional setup/teardown callbacks which will be run before/after - * every test case in the suite. - * Cleared when the suite finishes. */ - SET_SETUP(setup_cb, voidp_to_callback_data); - SET_TEARDOWN(teardown_cb, voidp_to_callback_data); - - RUN_TEST(foo_should_foo); -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - RUN_SUITE(suite); - GREATEST_MAIN_END(); /* display results */ -} - -#endif -/*********************************************************************/ - - -#include -#include -#include -#include - - -/*********** - * Options * - ***********/ - -/* Default column width for non-verbose output. */ -#ifndef GREATEST_DEFAULT_WIDTH -#define GREATEST_DEFAULT_WIDTH 72 -#endif - -/* FILE *, for test logging. */ -#ifndef GREATEST_STDOUT -#define GREATEST_STDOUT stdout -#endif - -/* Remove GREATEST_ prefix from most commonly used symbols? */ -#ifndef GREATEST_USE_ABBREVS -#define GREATEST_USE_ABBREVS 1 -#endif - - -/********* - * Types * - *********/ - -/* Info for the current running suite. */ -typedef struct greatest_suite_info { - unsigned int tests_run; - unsigned int passed; - unsigned int failed; - unsigned int skipped; - - /* timers, pre/post running suite and individual tests */ - clock_t pre_suite; - clock_t post_suite; - clock_t pre_test; - clock_t post_test; -} greatest_suite_info; - -/* Type for a suite function. */ -typedef void (greatest_suite_cb)(void); - -/* Types for setup/teardown callbacks. If non-NULL, these will be run - * and passed the pointer to their additional data. */ -typedef void (greatest_setup_cb)(void *udata); -typedef void (greatest_teardown_cb)(void *udata); - -typedef enum { - GREATEST_FLAG_VERBOSE = 0x01, - GREATEST_FLAG_FIRST_FAIL = 0x02, - GREATEST_FLAG_LIST_ONLY = 0x04 -} GREATEST_FLAG; - -typedef struct greatest_run_info { - unsigned int flags; - unsigned int tests_run; /* total test count */ - - /* Overall pass/fail/skip counts. */ - unsigned int passed; - unsigned int failed; - unsigned int skipped; - - /* currently running test suite */ - greatest_suite_info suite; - - /* info to print about the most recent failure */ - const char *fail_file; - unsigned int fail_line; - const char *msg; - - /* current setup/teardown hooks and userdata */ - greatest_setup_cb *setup; - void *setup_udata; - greatest_teardown_cb *teardown; - void *teardown_udata; - - /* formatting info for ".....s...F"-style output */ - unsigned int col; - unsigned int width; - - /* only run a specific suite or test */ - char *suite_filter; - char *test_filter; - - /* overall timers */ - clock_t begin; - clock_t end; -} greatest_run_info; - -/* Global var for the current testing context. - * Initialized by GREATEST_MAIN_DEFS(). */ -extern greatest_run_info greatest_info; - - -/********************** - * Exported functions * - **********************/ - -void greatest_do_pass(const char *name); -void greatest_do_fail(const char *name); -void greatest_do_skip(const char *name); -int greatest_pre_test(const char *name); -void greatest_post_test(const char *name, int res); -void greatest_usage(const char *name); -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata); -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata); - - -/********** - * Macros * - **********/ - -/* Define a suite. */ -#define GREATEST_SUITE(NAME) void NAME(void) - -/* Start defining a test function. - * The arguments are not included, to allow parametric testing. */ -#define GREATEST_TEST static int - -/* Run a suite. */ -#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME) - -/* Run a test in the current suite. */ -#define GREATEST_RUN_TEST(TEST) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -/* Run a test in the current suite with one void* argument, - * which can be a pointer to a struct with multiple arguments. */ -#define GREATEST_RUN_TEST1(TEST, ENV) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(ENV); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) - -/* If __VA_ARGS__ (C99) is supported, allow parametric testing - * without needing to manually manage the argument struct. */ -#if __STDC_VERSION__ >= 19901L -#define GREATEST_RUN_TESTp(TEST, ...) \ - do { \ - if (greatest_pre_test(#TEST) == 1) { \ - int res = TEST(__VA_ARGS__); \ - greatest_post_test(#TEST, res); \ - } else if (GREATEST_LIST_ONLY()) { \ - fprintf(GREATEST_STDOUT, " %s\n", #TEST); \ - } \ - } while (0) -#endif - - -/* Check if the test runner is in verbose mode. */ -#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE) -#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY) -#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL) -#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL()) - -/* Message-less forms. */ -#define GREATEST_PASS() GREATEST_PASSm(NULL) -#define GREATEST_FAIL() GREATEST_FAILm(NULL) -#define GREATEST_SKIP() GREATEST_SKIPm(NULL) -#define GREATEST_ASSERT(COND) GREATEST_ASSERTm(#COND, COND) -#define GREATEST_ASSERT_FALSE(COND) GREATEST_ASSERT_FALSEm(#COND, COND) -#define GREATEST_ASSERT_EQ(EXP, GOT) GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT) -#define GREATEST_ASSERT_STR_EQ(EXP, GOT) GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT) - -/* The following forms take an additional message argument first, - * to be displayed by the test runner. */ - -/* Fail if a condition is not true, with message. */ -#define GREATEST_ASSERTm(MSG, COND) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if (!(COND)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_FALSEm(MSG, COND) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if ((COND)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \ - do { \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if ((EXP) != (GOT)) return -1; \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \ - do { \ - const char *exp_s = (EXP); \ - const char *got_s = (GOT); \ - greatest_info.msg = MSG; \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - if (0 != strcmp(exp_s, got_s)) { \ - fprintf(GREATEST_STDOUT, \ - "Expected:\n####\n%s\n####\n", exp_s); \ - fprintf(GREATEST_STDOUT, \ - "Got:\n####\n%s\n####\n", got_s); \ - return -1; \ - } \ - greatest_info.msg = NULL; \ - } while (0) - -#define GREATEST_PASSm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return 0; \ - } while (0) - -#define GREATEST_FAILm(MSG) \ - do { \ - greatest_info.fail_file = __FILE__; \ - greatest_info.fail_line = __LINE__; \ - greatest_info.msg = MSG; \ - return -1; \ - } while (0) - -#define GREATEST_SKIPm(MSG) \ - do { \ - greatest_info.msg = MSG; \ - return 1; \ - } while (0) - -#define GREATEST_SET_TIME(NAME) \ - NAME = clock(); \ - if (NAME == (clock_t) -1) { \ - fprintf(GREATEST_STDOUT, \ - "clock error: %s\n", #NAME); \ - exit(EXIT_FAILURE); \ - } - -#define GREATEST_CLOCK_DIFF(C1, C2) \ - fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \ - (long unsigned int) (C2) - (C1), \ - (double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) \ - -/* Include several function definitions in the main test file. */ -#define GREATEST_MAIN_DEFS() \ - \ -/* Is FILTER a subset of NAME? */ \ -static int greatest_name_match(const char *name, \ - const char *filter) { \ - size_t offset = 0; \ - size_t filter_len = strlen(filter); \ - while (name[offset] != '\0') { \ - if (name[offset] == filter[0]) { \ - if (0 == strncmp(&name[offset], filter, filter_len)) { \ - return 1; \ - } \ - } \ - offset++; \ - } \ - \ - return 0; \ -} \ - \ -int greatest_pre_test(const char *name) { \ - if (!GREATEST_LIST_ONLY() \ - && (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \ - && (greatest_info.test_filter == NULL || \ - greatest_name_match(name, greatest_info.test_filter))) { \ - GREATEST_SET_TIME(greatest_info.suite.pre_test); \ - if (greatest_info.setup) { \ - greatest_info.setup(greatest_info.setup_udata); \ - } \ - return 1; /* test should be run */ \ - } else { \ - return 0; /* skipped */ \ - } \ -} \ - \ -void greatest_post_test(const char *name, int res) { \ - GREATEST_SET_TIME(greatest_info.suite.post_test); \ - if (greatest_info.teardown) { \ - void *udata = greatest_info.teardown_udata; \ - greatest_info.teardown(udata); \ - } \ - \ - if (res < 0) { \ - greatest_do_fail(name); \ - } else if (res > 0) { \ - greatest_do_skip(name); \ - } else if (res == 0) { \ - greatest_do_pass(name); \ - } \ - greatest_info.suite.tests_run++; \ - greatest_info.col++; \ - if (GREATEST_IS_VERBOSE()) { \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \ - greatest_info.suite.post_test); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } else if (greatest_info.col % greatest_info.width == 0) { \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - } \ - if (GREATEST_STDOUT == stdout) fflush(stdout); \ -} \ - \ -static void greatest_run_suite(greatest_suite_cb *suite_cb, \ - const char *suite_name) { \ - if (greatest_info.suite_filter && \ - !greatest_name_match(suite_name, greatest_info.suite_filter)) \ - return; \ - if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) return; \ - greatest_info.suite.tests_run = 0; \ - greatest_info.suite.failed = 0; \ - greatest_info.suite.passed = 0; \ - greatest_info.suite.skipped = 0; \ - greatest_info.suite.pre_suite = 0; \ - greatest_info.suite.post_suite = 0; \ - greatest_info.suite.pre_test = 0; \ - greatest_info.suite.post_test = 0; \ - greatest_info.col = 0; \ - fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \ - GREATEST_SET_TIME(greatest_info.suite.pre_suite); \ - suite_cb(); \ - GREATEST_SET_TIME(greatest_info.suite.post_suite); \ - if (greatest_info.suite.tests_run > 0) { \ - fprintf(GREATEST_STDOUT, \ - "\n%u tests - %u pass, %u fail, %u skipped", \ - greatest_info.suite.tests_run, \ - greatest_info.suite.passed, \ - greatest_info.suite.failed, \ - greatest_info.suite.skipped); \ - GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \ - greatest_info.suite.post_suite); \ - fprintf(GREATEST_STDOUT, "\n"); \ - } \ - greatest_info.setup = NULL; \ - greatest_info.setup_udata = NULL; \ - greatest_info.teardown = NULL; \ - greatest_info.teardown_udata = NULL; \ - greatest_info.passed += greatest_info.suite.passed; \ - greatest_info.failed += greatest_info.suite.failed; \ - greatest_info.skipped += greatest_info.suite.skipped; \ - greatest_info.tests_run += greatest_info.suite.tests_run; \ -} \ - \ -void greatest_do_pass(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "PASS %s: %s", \ - name, greatest_info.msg ? greatest_info.msg : ""); \ - } else { \ - fprintf(GREATEST_STDOUT, "."); \ - } \ - greatest_info.suite.passed++; \ -} \ - \ -void greatest_do_fail(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, \ - "FAIL %s: %s (%s:%u)", \ - name, greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } else { \ - fprintf(GREATEST_STDOUT, "F"); \ - /* add linebreak if in line of '.'s */ \ - if (greatest_info.col % greatest_info.width != 0) \ - fprintf(GREATEST_STDOUT, "\n"); \ - greatest_info.col = 0; \ - fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \ - name, \ - greatest_info.msg ? greatest_info.msg : "", \ - greatest_info.fail_file, greatest_info.fail_line); \ - } \ - greatest_info.suite.failed++; \ -} \ - \ -void greatest_do_skip(const char *name) { \ - if (GREATEST_IS_VERBOSE()) { \ - fprintf(GREATEST_STDOUT, "SKIP %s: %s", \ - name, \ - greatest_info.msg ? \ - greatest_info.msg : "" ); \ - } else { \ - fprintf(GREATEST_STDOUT, "s"); \ - } \ - greatest_info.suite.skipped++; \ -} \ - \ -void greatest_usage(const char *name) { \ - fprintf(GREATEST_STDOUT, \ - "Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \ - " -h print this Help\n" \ - " -l List suites and their tests, then exit\n" \ - " -f Stop runner after first failure\n" \ - " -v Verbose output\n" \ - " -s SUITE only run suite named SUITE\n" \ - " -t TEST only run test named TEST\n", \ - name); \ -} \ - \ -void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \ - greatest_info.setup = cb; \ - greatest_info.setup_udata = udata; \ -} \ - \ -void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \ - void *udata) { \ - greatest_info.teardown = cb; \ - greatest_info.teardown_udata = udata; \ -} \ - \ -greatest_run_info greatest_info - -/* Handle command-line arguments, etc. */ -#define GREATEST_MAIN_BEGIN() \ - do { \ - int i = 0; \ - memset(&greatest_info, 0, sizeof(greatest_info)); \ - if (greatest_info.width == 0) { \ - greatest_info.width = GREATEST_DEFAULT_WIDTH; \ - } \ - for (i = 1; i < argc; i++) { \ - if (0 == strcmp("-t", argv[i])) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.test_filter = argv[i+1]; \ - i++; \ - } else if (0 == strcmp("-s", argv[i])) { \ - if (argc <= i + 1) { \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - greatest_info.suite_filter = argv[i+1]; \ - i++; \ - } else if (0 == strcmp("-f", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \ - } else if (0 == strcmp("-v", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_VERBOSE; \ - } else if (0 == strcmp("-l", argv[i])) { \ - greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \ - } else if (0 == strcmp("-h", argv[i])) { \ - greatest_usage(argv[0]); \ - exit(EXIT_SUCCESS); \ - } else { \ - fprintf(GREATEST_STDOUT, \ - "Unknown argument '%s'\n", argv[i]); \ - greatest_usage(argv[0]); \ - exit(EXIT_FAILURE); \ - } \ - } \ - } while (0); \ - GREATEST_SET_TIME(greatest_info.begin) - -#define GREATEST_MAIN_END() \ - do { \ - if (!GREATEST_LIST_ONLY()) { \ - GREATEST_SET_TIME(greatest_info.end); \ - fprintf(GREATEST_STDOUT, \ - "\nTotal: %u tests", greatest_info.tests_run); \ - GREATEST_CLOCK_DIFF(greatest_info.begin, \ - greatest_info.end); \ - fprintf(GREATEST_STDOUT, "\n"); \ - fprintf(GREATEST_STDOUT, \ - "Pass: %u, fail: %u, skip: %u.\n", \ - greatest_info.passed, \ - greatest_info.failed, greatest_info.skipped); \ - } \ - return (greatest_info.failed > 0 \ - ? EXIT_FAILURE : EXIT_SUCCESS); \ - } while (0) - -/* Make abbreviations without the GREATEST_ prefix for the - * most commonly used symbols. */ -#if GREATEST_USE_ABBREVS -#define TEST GREATEST_TEST -#define SUITE GREATEST_SUITE -#define RUN_TEST GREATEST_RUN_TEST -#define RUN_TEST1 GREATEST_RUN_TEST1 -#define RUN_SUITE GREATEST_RUN_SUITE -#define ASSERT GREATEST_ASSERT -#define ASSERTm GREATEST_ASSERTm -#define ASSERT_FALSE GREATEST_ASSERT_FALSE -#define ASSERT_EQ GREATEST_ASSERT_EQ -#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ -#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm -#define ASSERT_EQm GREATEST_ASSERT_EQm -#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm -#define PASS GREATEST_PASS -#define FAIL GREATEST_FAIL -#define SKIP GREATEST_SKIP -#define PASSm GREATEST_PASSm -#define FAILm GREATEST_FAILm -#define SKIPm GREATEST_SKIPm -#define SET_SETUP GREATEST_SET_SETUP_CB -#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB - -#if __STDC_VERSION__ >= 19901L -#endif /* C99 */ -#define RUN_TESTp GREATEST_RUN_TESTp -#endif /* USE_ABBREVS */ - -#endif diff --git a/espfs/heatshrink/heatshrink.c b/espfs/heatshrink/heatshrink.c deleted file mode 100644 index 9de553f..0000000 --- a/espfs/heatshrink/heatshrink.c +++ /dev/null @@ -1,446 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" - -#define DEF_WINDOW_SZ2 11 -#define DEF_LOOKAHEAD_SZ2 4 -#define DEF_DECODER_INPUT_BUFFER_SIZE 256 -#define DEF_BUFFER_SIZE (64 * 1024) - -#if 0 -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#else -#define LOG(...) /* NO-OP */ -#endif - -static const int version_major = HEATSHRINK_VERSION_MAJOR; -static const int version_minor = HEATSHRINK_VERSION_MINOR; -static const int version_patch = HEATSHRINK_VERSION_PATCH; -static const char author[] = HEATSHRINK_AUTHOR; -static const char url[] = HEATSHRINK_URL; - -static void usage(void) { - fprintf(stderr, "heatshrink version %u.%u.%u by %s\n", - version_major, version_minor, version_patch, author); - fprintf(stderr, "Home page: %s\n\n", url); - fprintf(stderr, - "Usage:\n" - " heatshrink [-h] [-e|-d] [-v] [-w SIZE] [-l BITS] [IN_FILE] [OUT_FILE]\n" - "\n" - "heatshrink compresses or uncompresses byte streams using LZSS, and is\n" - "designed especially for embedded, low-memory, and/or hard real-time\n" - "systems.\n" - "\n" - " -h print help\n" - " -e encode (compress, default)\n" - " -d decode (uncompress)\n" - " -v verbose (print input & output sizes, compression ratio, etc.)\n" - "\n" - " -w SIZE Base-2 log of LZSS sliding window size\n" - "\n" - " A larger value allows searches a larger history of the data for repeated\n" - " patterns, potentially compressing more effectively, but will use\n" - " more memory and processing time.\n" - " Recommended default: -w 8 (embedded systems), -w 10 (elsewhere)\n" - " \n" - " -l BITS Number of bits used for back-reference lengths\n" - "\n" - " A larger value allows longer substitutions, but since all\n" - " back-references must use -w + -l bits, larger -w or -l can be\n" - " counterproductive if most patterns are small and/or local.\n" - " Recommended default: -l 4\n" - "\n" - " If IN_FILE or OUT_FILE are unspecified, they will default to\n" - " \"-\" for standard input and standard output, respectively.\n"); - exit(1); -} - -typedef enum { IO_READ, IO_WRITE, } IO_mode; -typedef enum { OP_ENC, OP_DEC, } Operation; - -typedef struct { - int fd; /* file descriptor */ - IO_mode mode; - size_t fill; /* fill index */ - size_t read; /* read index */ - size_t size; - size_t total; - uint8_t buf[]; -} io_handle; - -typedef struct { - uint8_t window_sz2; - uint8_t lookahead_sz2; - size_t decoder_input_buffer_size; - size_t buffer_size; - uint8_t verbose; - Operation cmd; - char *in_fname; - char *out_fname; - io_handle *in; - io_handle *out; -} config; - -static void die(char *msg) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); -} - -static void report(config *cfg); - -/* Open an IO handle. Returns NULL on error. */ -static io_handle *handle_open(char *fname, IO_mode m, size_t buf_sz) { - io_handle *io = NULL; - io = malloc(sizeof(*io) + buf_sz); - if (io == NULL) { return NULL; } - memset(io, 0, sizeof(*io) + buf_sz); - io->fd = -1; - io->size = buf_sz; - io->mode = m; - - if (m == IO_READ) { - if (0 == strcmp("-", fname)) { - io->fd = STDIN_FILENO; - } else { - io->fd = open(fname, O_RDONLY); - } - } else if (m == IO_WRITE) { - if (0 == strcmp("-", fname)) { - io->fd = STDOUT_FILENO; - } else { - io->fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC /*| O_EXCL*/, 0644); - } - } - - if (io->fd == -1) { /* failed to open */ - free(io); - err(1, "open"); - return NULL; - } - - return io; -} - -/* Read SIZE bytes from an IO handle and return a pointer to the content. - * BUF contains at least size_t bytes. Returns 0 on EOF, -1 on error. */ -static ssize_t handle_read(io_handle *io, size_t size, uint8_t **buf) { - LOG("@ read %zd\n", size); - if (buf == NULL) { return -1; } - if (size > io->size) { - printf("size %zd, io->size %zd\n", size, io->size); - return -1; - } - if (io->mode != IO_READ) { return -1; } - - size_t rem = io->fill - io->read; - if (rem >= size) { - *buf = &io->buf[io->read]; - return size; - } else { /* read and replenish */ - if (io->fd == -1) { /* already closed, return what we've got */ - *buf = &io->buf[io->read]; - return rem; - } - - memmove(io->buf, &io->buf[io->read], rem); - io->fill -= io->read; - io->read = 0; - ssize_t read_sz = read(io->fd, &io->buf[io->fill], io->size - io->fill); - if (read_sz < 0) { err(1, "read"); } - io->total += read_sz; - if (read_sz == 0) { /* EOF */ - if (close(io->fd) < 0) { err(1, "close"); } - io->fd = -1; - } - io->fill += read_sz; - *buf = io->buf; - return io->fill > size ? size : io->fill; - } -} - -/* Drop the oldest SIZE bytes from the buffer. Returns <0 on error. */ -static int handle_drop(io_handle *io, size_t size) { - LOG("@ drop %zd\n", size); - if (io->read + size <= io->fill) { - io->read += size; - } else { - return -1; - } - if (io->read == io->fill) { - io->read = 0; - io->fill = 0; - } - return 0; -} - -/* Sink SIZE bytes from INPUT into the io handle. Returns the number of - * bytes written, or -1 on error. */ -static ssize_t handle_sink(io_handle *io, size_t size, uint8_t *input) { - LOG("@ sink %zd\n", size); - if (size > io->size) { return -1; } - if (io->mode != IO_WRITE) { return -1; } - - if (io->fill + size > io->size) { - ssize_t written = write(io->fd, io->buf, io->fill); - LOG("@ flushing %zd, wrote %zd\n", io->fill, written); - io->total += written; - if (written == -1) { err(1, "write"); } - memmove(io->buf, &io->buf[written], io->fill - written); - io->fill -= written; - } - memcpy(&io->buf[io->fill], input, size); - io->fill += size; - return size; -} - -static void handle_close(io_handle *io) { - if (io->fd != -1) { - if (io->mode == IO_WRITE) { - ssize_t written = write(io->fd, io->buf, io->fill); - io->total += written; - LOG("@ close: flushing %zd, wrote %zd\n", io->fill, written); - if (written == -1) { err(1, "write"); } - } - close(io->fd); - io->fd = -1; - } -} - -static void close_and_report(config *cfg) { - handle_close(cfg->in); - handle_close(cfg->out); - if (cfg->verbose) { report(cfg); } - free(cfg->in); - free(cfg->out); -} - -static int encoder_sink_read(config *cfg, heatshrink_encoder *hse, - uint8_t *data, size_t data_sz) { - size_t out_sz = 4096; - uint8_t out_buf[out_sz]; - memset(out_buf, 0, out_sz); - size_t sink_sz = 0; - size_t poll_sz = 0; - HSE_sink_res sres; - HSE_poll_res pres; - HSE_finish_res fres; - io_handle *out = cfg->out; - - size_t sunk = 0; - do { - if (data_sz > 0) { - sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz); - if (sres < 0) { die("sink"); } - sunk += sink_sz; - } - - do { - pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz); - if (pres < 0) { die("poll"); } - if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); - } while (pres == HSER_POLL_MORE); - - if (poll_sz == 0 && data_sz == 0) { - fres = heatshrink_encoder_finish(hse); - if (fres < 0) { die("finish"); } - if (fres == HSER_FINISH_DONE) { return 1; } - } - } while (sunk < data_sz); - return 0; -} - -static int encode(config *cfg) { - uint8_t window_sz2 = cfg->window_sz2; - size_t window_sz = 1 << window_sz2; - heatshrink_encoder *hse = heatshrink_encoder_alloc(window_sz2, cfg->lookahead_sz2); - if (hse == NULL) { die("failed to init encoder: bad settings"); } - ssize_t read_sz = 0; - io_handle *in = cfg->in; - - /* Process input until end of stream */ - while (1) { - uint8_t *input = NULL; - read_sz = handle_read(in, window_sz, &input); - if (input == NULL) { - printf("handle read failure\n"); - die("read"); - } - if (read_sz < 0) { die("read"); } - - /* Pass read to encoder and check if input is fully processed. */ - if (encoder_sink_read(cfg, hse, input, read_sz)) break; - - if (handle_drop(in, read_sz) < 0) { die("drop"); } - }; - - if (read_sz == -1) { err(1, "read"); } - - heatshrink_encoder_free(hse); - close_and_report(cfg); - return 0; -} - -static int decoder_sink_read(config *cfg, heatshrink_decoder *hsd, - uint8_t *data, size_t data_sz) { - io_handle *out = cfg->out; - size_t sink_sz = 0; - size_t poll_sz = 0; - size_t out_sz = 4096; - uint8_t out_buf[out_sz]; - memset(out_buf, 0, out_sz); - - HSD_sink_res sres; - HSD_poll_res pres; - HSD_finish_res fres; - - size_t sunk = 0; - do { - if (data_sz > 0) { - sres = heatshrink_decoder_sink(hsd, &data[sunk], data_sz - sunk, &sink_sz); - if (sres < 0) { die("sink"); } - sunk += sink_sz; - } - - do { - pres = heatshrink_decoder_poll(hsd, out_buf, out_sz, &poll_sz); - if (pres < 0) { die("poll"); } - if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink"); - } while (pres == HSDR_POLL_MORE); - - if (data_sz == 0 && poll_sz == 0) { - fres = heatshrink_decoder_finish(hsd); - if (fres < 0) { die("finish"); } - if (fres == HSDR_FINISH_DONE) { return 1; } - } - } while (sunk < data_sz); - - return 0; -} - -static int decode(config *cfg) { - uint8_t window_sz2 = cfg->window_sz2; - size_t window_sz = 1 << window_sz2; - size_t ibs = cfg->decoder_input_buffer_size; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(ibs, - window_sz2, cfg->lookahead_sz2); - if (hsd == NULL) { die("failed to init decoder"); } - - ssize_t read_sz = 0; - - io_handle *in = cfg->in; - - HSD_finish_res fres; - - /* Process input until end of stream */ - while (1) { - uint8_t *input = NULL; - read_sz = handle_read(in, window_sz, &input); - if (input == NULL) { - printf("handle read failure\n"); - die("read"); - } - if (read_sz == 0) { - fres = heatshrink_decoder_finish(hsd); - if (fres < 0) { die("finish"); } - if (fres == HSDR_FINISH_DONE) break; - } else if (read_sz < 0) { - die("read"); - } else { - if (decoder_sink_read(cfg, hsd, input, read_sz)) { break; } - if (handle_drop(in, read_sz) < 0) { die("drop"); } - } - } - if (read_sz == -1) { err(1, "read"); } - - heatshrink_decoder_free(hsd); - close_and_report(cfg); - return 0; -} - -static void report(config *cfg) { - size_t inb = cfg->in->total; - size_t outb = cfg->out->total; - fprintf(cfg->out->fd == STDOUT_FILENO ? stderr : stdout, - "%s %0.2f %%\t %zd -> %zd (-w %u -l %u)\n", - cfg->in_fname, 100.0 - (100.0 * outb) / inb, inb, outb, - cfg->window_sz2, cfg->lookahead_sz2); -} - -static void proc_args(config *cfg, int argc, char **argv) { - cfg->window_sz2 = DEF_WINDOW_SZ2; - cfg->lookahead_sz2 = DEF_LOOKAHEAD_SZ2; - cfg->buffer_size = DEF_BUFFER_SIZE; - cfg->decoder_input_buffer_size = DEF_DECODER_INPUT_BUFFER_SIZE; - cfg->cmd = OP_ENC; - cfg->verbose = 0; - cfg->in_fname = "-"; - cfg->out_fname = "-"; - - int a = 0; - while ((a = getopt(argc, argv, "hedi:w:l:v")) != -1) { - switch (a) { - case 'h': /* help */ - usage(); - case 'e': /* encode */ - cfg->cmd = OP_ENC; break; - case 'd': /* decode */ - cfg->cmd = OP_DEC; break; - case 'i': /* input buffer size */ - cfg->decoder_input_buffer_size = atoi(optarg); - break; - case 'w': /* window bits */ - cfg->window_sz2 = atoi(optarg); - break; - case 'l': /* lookahead bits */ - cfg->lookahead_sz2 = atoi(optarg); - break; - case 'v': /* verbosity++ */ - cfg->verbose++; - break; - case '?': /* unknown argument */ - default: - usage(); - } - } - argc -= optind; - argv += optind; - if (argc > 0) { - cfg->in_fname = argv[0]; - argc--; - argv++; - } - if (argc > 0) { cfg->out_fname = argv[0]; } -} - -int main(int argc, char **argv) { - config cfg; - memset(&cfg, 0, sizeof(cfg)); - proc_args(&cfg, argc, argv); - - if (0 == strcmp(cfg.in_fname, cfg.out_fname) - && (0 != strcmp("-", cfg.in_fname))) { - printf("Refusing to overwrite file '%s' with itself.\n", cfg.in_fname); - exit(1); - } - - cfg.in = handle_open(cfg.in_fname, IO_READ, cfg.buffer_size); - if (cfg.in == NULL) { die("Failed to open input file for read"); } - cfg.out = handle_open(cfg.out_fname, IO_WRITE, cfg.buffer_size); - if (cfg.out == NULL) { die("Failed to open output file for write"); } - - if (cfg.cmd == OP_ENC) { - return encode(&cfg); - } else if (cfg.cmd == OP_DEC) { - return decode(&cfg); - } else { - usage(); - } -} diff --git a/espfs/heatshrink/heatshrink_common.h b/espfs/heatshrink/heatshrink_common.h deleted file mode 100644 index 0d396b9..0000000 --- a/espfs/heatshrink/heatshrink_common.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef HEATSHRINK_H -#define HEATSHRINK_H - -#define HEATSHRINK_AUTHOR "Scott Vokes " -#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink" - -/* Version 0.3.1 */ -#define HEATSHRINK_VERSION_MAJOR 0 -#define HEATSHRINK_VERSION_MINOR 3 -#define HEATSHRINK_VERSION_PATCH 1 - -#define HEATSHRINK_MIN_WINDOW_BITS 4 -#define HEATSHRINK_MAX_WINDOW_BITS 15 - -#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2 - -#define HEATSHRINK_LITERAL_MARKER 0x01 -#define HEATSHRINK_BACKREF_MARKER 0x00 - -#endif diff --git a/espfs/heatshrink/heatshrink_config.h b/espfs/heatshrink/heatshrink_config.h deleted file mode 100644 index 51d4772..0000000 --- a/espfs/heatshrink/heatshrink_config.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef HEATSHRINK_CONFIG_H -#define HEATSHRINK_CONFIG_H - -/* Should functionality assuming dynamic allocation be used? */ -#define HEATSHRINK_DYNAMIC_ALLOC 1 - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Optional replacement of malloc/free */ - #define HEATSHRINK_MALLOC(SZ) malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) free(P) -#else - /* Required parameters for static configuration */ - #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 - #define HEATSHRINK_STATIC_WINDOW_BITS 8 - #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 -#endif - -/* Turn on logging for debugging. */ -#define HEATSHRINK_DEBUGGING_LOGS 0 - -/* Use indexing for faster compression. (This requires additional space.) */ -#define HEATSHRINK_USE_INDEX 1 - -#endif diff --git a/espfs/heatshrink/heatshrink_decoder.c b/espfs/heatshrink/heatshrink_decoder.c deleted file mode 100644 index b92be13..0000000 --- a/espfs/heatshrink/heatshrink_decoder.c +++ /dev/null @@ -1,382 +0,0 @@ -#include -#include -#include "heatshrink_decoder.h" - -/* States for the polling state machine. */ -typedef enum { - HSDS_EMPTY, /* no input to process */ - HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */ - HSDS_YIELD_LITERAL, /* ready to yield literal byte */ - HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */ - HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */ - HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */ - HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */ - HSDS_YIELD_BACKREF, /* ready to yield back-reference */ - HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */ -} HSD_state; - -#if HEATSHRINK_DEBUGGING_LOGS -#include -#include -#include -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#define ASSERT(X) assert(X) -static const char *state_names[] = { - "empty", - "input_available", - "yield_literal", - "backref_index", - "backref_count", - "yield_backref", - "check_for_more_input", -}; -#else -#define LOG(...) /* no-op */ -#define ASSERT(X) /* no-op */ -#endif - -typedef struct { - uint8_t *buf; /* output buffer */ - size_t buf_size; /* buffer size */ - size_t *output_size; /* bytes pushed to buffer, so far */ -} output_info; - -#define NO_BITS ((uint32_t)-1) - -/* Forward references. */ -static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count); -static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte); - -#if HEATSHRINK_DYNAMIC_ALLOC -heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, - uint8_t window_sz2, - uint8_t lookahead_sz2) { - if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || - (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || - (input_buffer_size == 0) || - (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || - (lookahead_sz2 > window_sz2)) { - return NULL; - } - size_t buffers_sz = (1 << window_sz2) + input_buffer_size; - size_t sz = sizeof(heatshrink_decoder) + buffers_sz; - heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz); - if (hsd == NULL) { return NULL; } - hsd->input_buffer_size = input_buffer_size; - hsd->window_sz2 = window_sz2; - hsd->lookahead_sz2 = lookahead_sz2; - heatshrink_decoder_reset(hsd); - LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n", - sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size); - return hsd; -} - -void heatshrink_decoder_free(heatshrink_decoder *hsd) { - size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size; - size_t sz = sizeof(heatshrink_decoder) + buffers_sz; - HEATSHRINK_FREE(hsd, sz); - (void)sz; /* may not be used by free */ -} -#endif - -void heatshrink_decoder_reset(heatshrink_decoder *hsd) { - size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd); - size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd); - memset(hsd->buffers, 0, buf_sz + input_sz); - hsd->state = HSDS_EMPTY; - hsd->input_size = 0; - hsd->input_index = 0; - hsd->bit_index = 0x00; - hsd->current_byte = 0x00; - hsd->output_count = 0; - hsd->output_index = 0; - hsd->head_index = 0; - hsd->bit_accumulator = 0x00000000; -} - -/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */ -HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, - uint8_t *in_buf, size_t size, size_t *input_size) { - if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) { - return HSDR_SINK_ERROR_NULL; - } - - size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size; - if (rem == 0) { - *input_size = 0; - return HSDR_SINK_FULL; - } - - size = rem < size ? rem : size; - LOG("-- sinking %zd bytes\n", size); - /* copy into input buffer (at head of buffers) */ - memcpy(&hsd->buffers[hsd->input_size], in_buf, size); - hsd->input_size += size; - if (hsd->state == HSDS_EMPTY) { - hsd->state = HSDS_INPUT_AVAILABLE; - hsd->input_index = 0; - } - *input_size = size; - return HSDR_SINK_OK; -} - - -/***************** - * Decompression * - *****************/ - -#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD)) -#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD)) - -// States -static HSD_state st_input_available(heatshrink_decoder *hsd); -static HSD_state st_yield_literal(heatshrink_decoder *hsd, - output_info *oi); -static HSD_state st_backref_index_msb(heatshrink_decoder *hsd); -static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd); -static HSD_state st_backref_count_msb(heatshrink_decoder *hsd); -static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd); -static HSD_state st_yield_backref(heatshrink_decoder *hsd, - output_info *oi); -static HSD_state st_check_for_input(heatshrink_decoder *hsd); - -HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { - if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) { - return HSDR_POLL_ERROR_NULL; - } - *output_size = 0; - - output_info oi; - oi.buf = out_buf; - oi.buf_size = out_buf_size; - oi.output_size = output_size; - - while (1) { - LOG("-- poll, state is %d (%s), input_size %d\n", - hsd->state, state_names[hsd->state], hsd->input_size); - uint8_t in_state = hsd->state; - switch (in_state) { - case HSDS_EMPTY: - return HSDR_POLL_EMPTY; - case HSDS_INPUT_AVAILABLE: - hsd->state = st_input_available(hsd); - break; - case HSDS_YIELD_LITERAL: - hsd->state = st_yield_literal(hsd, &oi); - break; - case HSDS_BACKREF_INDEX_MSB: - hsd->state = st_backref_index_msb(hsd); - break; - case HSDS_BACKREF_INDEX_LSB: - hsd->state = st_backref_index_lsb(hsd); - break; - case HSDS_BACKREF_COUNT_MSB: - hsd->state = st_backref_count_msb(hsd); - break; - case HSDS_BACKREF_COUNT_LSB: - hsd->state = st_backref_count_lsb(hsd); - break; - case HSDS_YIELD_BACKREF: - hsd->state = st_yield_backref(hsd, &oi); - break; - case HSDS_CHECK_FOR_MORE_INPUT: - hsd->state = st_check_for_input(hsd); - break; - default: - return HSDR_POLL_ERROR_UNKNOWN; - } - - /* If the current state cannot advance, check if input or output - * buffer are exhausted. */ - if (hsd->state == in_state) { - if (*output_size == out_buf_size) { return HSDR_POLL_MORE; } - return HSDR_POLL_EMPTY; - } - } -} - -static HSD_state st_input_available(heatshrink_decoder *hsd) { - uint32_t bits = get_bits(hsd, 1); // get tag bit - if (bits) { - return HSDS_YIELD_LITERAL; - } else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) { - return HSDS_BACKREF_INDEX_MSB; - } else { - hsd->output_index = 0; - return HSDS_BACKREF_INDEX_LSB; - } -} - -static HSD_state st_yield_literal(heatshrink_decoder *hsd, - output_info *oi) { - /* Emit a repeated section from the window buffer, and add it (again) - * to the window buffer. (Note that the repetition can include - * itself.)*/ - if (*oi->output_size < oi->buf_size) { - uint32_t byte = get_bits(hsd, 8); - if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */ - uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; - uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; - uint8_t c = byte & 0xFF; - LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.'); - buf[hsd->head_index++ & mask] = c; - push_byte(hsd, oi, c); - return HSDS_CHECK_FOR_MORE_INPUT; - } else { - return HSDS_YIELD_LITERAL; - } -} - -static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) { - uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); - ASSERT(bit_ct > 8); - uint32_t bits = get_bits(hsd, bit_ct - 8); - LOG("-- backref index (msb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; } - hsd->output_index = bits << 8; - return HSDS_BACKREF_INDEX_LSB; -} - -static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) { - uint8_t bit_ct = BACKREF_INDEX_BITS(hsd); - uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8); - LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; } - hsd->output_index |= bits; - hsd->output_index++; - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - hsd->output_count = 0; - return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB; -} - -static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) { - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - ASSERT(br_bit_ct > 8); - uint32_t bits = get_bits(hsd, br_bit_ct - 8); - LOG("-- backref count (msb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; } - hsd->output_count = bits << 8; - return HSDS_BACKREF_COUNT_LSB; -} - -static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) { - uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd); - uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8); - LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits); - if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; } - hsd->output_count |= bits; - hsd->output_count++; - return HSDS_YIELD_BACKREF; -} - -static HSD_state st_yield_backref(heatshrink_decoder *hsd, - output_info *oi) { - size_t count = oi->buf_size - *oi->output_size; - if (count > 0) { - if (hsd->output_count < count) count = hsd->output_count; - uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)]; - uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1; - uint16_t neg_offset = hsd->output_index; - LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset); - ASSERT(neg_offset < mask + 1); - ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd)); - - for (size_t i=0; ihead_index - neg_offset) & mask]; - push_byte(hsd, oi, c); - buf[hsd->head_index & mask] = c; - hsd->head_index++; - LOG(" -- ++ 0x%02x\n", c); - } - hsd->output_count -= count; - if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; } - } - return HSDS_YIELD_BACKREF; -} - -static HSD_state st_check_for_input(heatshrink_decoder *hsd) { - return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE; -} - -/* Get the next COUNT bits from the input buffer, saving incremental progress. - * Returns NO_BITS on end of input, or if more than 31 bits are requested. */ -static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) { - if (count > 31) { return NO_BITS; } - LOG("-- popping %u bit(s)\n", count); - - /* If we aren't able to get COUNT bits, suspend immediately, because we - * don't track how many bits of COUNT we've accumulated before suspend. */ - if (hsd->input_size == 0) { - if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; } - } - - for (int i = 0; i < count; i++) { - if (hsd->bit_index == 0x00) { - if (hsd->input_size == 0) { - LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n", - hsd->bit_accumulator, hsd->bit_accumulator); - return NO_BITS; - } - hsd->current_byte = hsd->buffers[hsd->input_index++]; - LOG(" -- pulled byte 0x%02x\n", hsd->current_byte); - if (hsd->input_index == hsd->input_size) { - hsd->input_index = 0; /* input is exhausted */ - hsd->input_size = 0; - } - hsd->bit_index = 0x80; - } - hsd->bit_accumulator <<= 1; - if (hsd->current_byte & hsd->bit_index) { - hsd->bit_accumulator |= 0x01; - if (0) { - LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n", - hsd->bit_accumulator, hsd->bit_index); - } - } else { - if (0) { - LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n", - hsd->bit_accumulator, hsd->bit_index); - } - } - hsd->bit_index >>= 1; - } - - uint32_t res = 0; - res = hsd->bit_accumulator; - hsd->bit_accumulator = 0x00000000; - if (count > 1) { LOG(" -- accumulated %08x\n", res); } - return res; -} - -HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) { - if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; } - switch (hsd->state) { - case HSDS_EMPTY: - return HSDR_FINISH_DONE; - - /* If we want to finish with no input, but are in these states, it's - * because the 0-bit padding to the last byte looks like a backref - * marker bit followed by all 0s for index and count bits. */ - case HSDS_BACKREF_INDEX_LSB: - case HSDS_BACKREF_INDEX_MSB: - case HSDS_BACKREF_COUNT_LSB: - case HSDS_BACKREF_COUNT_MSB: - return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; - - /* If the output stream is padded with 0xFFs (possibly due to being in - * flash memory), also explicitly check the input size rather than - * uselessly returning MORE but yielding 0 bytes when polling. */ - case HSDS_YIELD_LITERAL: - return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE; - - default: - return HSDR_FINISH_MORE; - } -} - -static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) { - LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.'); - oi->buf[(*oi->output_size)++] = byte; - (void)hsd; -} diff --git a/espfs/heatshrink/heatshrink_decoder.h b/espfs/heatshrink/heatshrink_decoder.h deleted file mode 100644 index c1eb144..0000000 --- a/espfs/heatshrink/heatshrink_decoder.h +++ /dev/null @@ -1,101 +0,0 @@ -#ifndef HEATSHRINK_DECODER_H -#define HEATSHRINK_DECODER_H - -#include -#include -#include "heatshrink_common.h" -#include "heatshrink_config.h" - -typedef enum { - HSDR_SINK_OK, /* data sunk, ready to poll */ - HSDR_SINK_FULL, /* out of space in internal buffer */ - HSDR_SINK_ERROR_NULL=-1, /* NULL argument */ -} HSD_sink_res; - -typedef enum { - HSDR_POLL_EMPTY, /* input exhausted */ - HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */ - HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */ - HSDR_POLL_ERROR_UNKNOWN=-2, -} HSD_poll_res; - -typedef enum { - HSDR_FINISH_DONE, /* output is done */ - HSDR_FINISH_MORE, /* more output remains */ - HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */ -} HSD_finish_res; - -#if HEATSHRINK_DYNAMIC_ALLOC -#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \ - ((BUF)->input_buffer_size) -#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \ - ((BUF)->window_sz2) -#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ - ((BUF)->lookahead_sz2) -#else -#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \ - HEATSHRINK_STATIC_INPUT_BUFFER_SIZE -#define HEATSHRINK_DECODER_WINDOW_BITS(_) \ - (HEATSHRINK_STATIC_WINDOW_BITS) -#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \ - (HEATSHRINK_STATIC_LOOKAHEAD_BITS) -#endif - -typedef struct { - uint16_t input_size; /* bytes in input buffer */ - uint16_t input_index; /* offset to next unprocessed input byte */ - uint16_t output_count; /* how many bytes to output */ - uint16_t output_index; /* index for bytes to output */ - uint16_t head_index; /* head of window buffer */ - uint16_t bit_accumulator; - uint8_t state; /* current state machine node */ - uint8_t current_byte; /* current byte of input */ - uint8_t bit_index; /* current bit index */ - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Fields that are only used if dynamically allocated. */ - uint8_t window_sz2; /* window buffer bits */ - uint8_t lookahead_sz2; /* lookahead bits */ - uint16_t input_buffer_size; /* input buffer size */ - - /* Input buffer, then expansion window buffer */ - uint8_t buffers[]; -#else - /* Input buffer, then expansion window buffer */ - uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_)) - + HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)]; -#endif -} heatshrink_decoder; - -#if HEATSHRINK_DYNAMIC_ALLOC -/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes, - * an expansion buffer size of 2^WINDOW_SZ2, and a lookahead - * size of 2^lookahead_sz2. (The window buffer and lookahead sizes - * must match the settings used when the data was compressed.) - * Returns NULL on error. */ -heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size, - uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2); - -/* Free a decoder. */ -void heatshrink_decoder_free(heatshrink_decoder *hsd); -#endif - -/* Reset a decoder. */ -void heatshrink_decoder_reset(heatshrink_decoder *hsd); - -/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to - * indicate how many bytes were actually sunk (in case a buffer was filled). */ -HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd, - uint8_t *in_buf, size_t size, size_t *input_size); - -/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into - * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ -HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size); - -/* Notify the dencoder that the input stream is finished. - * If the return value is HSDR_FINISH_MORE, there is still more output, so - * call heatshrink_decoder_poll and repeat. */ -HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd); - -#endif diff --git a/espfs/heatshrink/heatshrink_encoder.c b/espfs/heatshrink/heatshrink_encoder.c deleted file mode 100644 index ede5f60..0000000 --- a/espfs/heatshrink/heatshrink_encoder.c +++ /dev/null @@ -1,650 +0,0 @@ -#include -#include -#include -#include "heatshrink_encoder.h" - -typedef enum { - HSES_NOT_FULL, /* input buffer not full enough */ - HSES_FILLED, /* buffer is full */ - HSES_SEARCH, /* searching for patterns */ - HSES_YIELD_TAG_BIT, /* yield tag bit */ - HSES_YIELD_LITERAL, /* emit literal byte */ - HSES_YIELD_BR_INDEX, /* yielding backref index */ - HSES_YIELD_BR_LENGTH, /* yielding backref length */ - HSES_SAVE_BACKLOG, /* copying buffer to backlog */ - HSES_FLUSH_BITS, /* flush bit buffer */ - HSES_DONE, /* done */ -} HSE_state; - -#if HEATSHRINK_DEBUGGING_LOGS -#include -#include -#include -#define LOG(...) fprintf(stderr, __VA_ARGS__) -#define ASSERT(X) assert(X) -static const char *state_names[] = { - "not_full", - "filled", - "search", - "yield_tag_bit", - "yield_literal", - "yield_br_index", - "yield_br_length", - "save_backlog", - "flush_bits", - "done", -}; -#else -#define LOG(...) /* no-op */ -#define ASSERT(X) /* no-op */ -#endif - -// Encoder flags -enum { - FLAG_IS_FINISHING = 0x01, - FLAG_HAS_LITERAL = 0x02, - FLAG_ON_FINAL_LITERAL = 0x04, - FLAG_BACKLOG_IS_PARTIAL = 0x08, - FLAG_BACKLOG_IS_FILLED = 0x10, -}; - -typedef struct { - uint8_t *buf; /* output buffer */ - size_t buf_size; /* buffer size */ - size_t *output_size; /* bytes pushed to buffer, so far */ -} output_info; - -#define MATCH_NOT_FOUND ((uint16_t)-1) - -static uint16_t get_input_offset(heatshrink_encoder *hse); -static uint16_t get_input_buffer_size(heatshrink_encoder *hse); -static uint16_t get_lookahead_size(heatshrink_encoder *hse); -static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag); -static int can_take_byte(output_info *oi); -static int is_finishing(heatshrink_encoder *hse); -static int backlog_is_partial(heatshrink_encoder *hse); -static int backlog_is_filled(heatshrink_encoder *hse); -static int on_final_literal(heatshrink_encoder *hse); -static void save_backlog(heatshrink_encoder *hse); -static int has_literal(heatshrink_encoder *hse); - -/* Push COUNT (max 8) bits to the output buffer, which has room. */ -static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, - output_info *oi); -static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi); -static void push_literal_byte(heatshrink_encoder *hse, output_info *oi); - -#if HEATSHRINK_DYNAMIC_ALLOC -heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, - uint8_t lookahead_sz2) { - if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) || - (window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) || - (lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) || - (lookahead_sz2 > window_sz2)) { - return NULL; - } - - /* Note: 2 * the window size is used because the buffer needs to fit - * (1 << window_sz2) bytes for the current input, and an additional - * (1 << window_sz2) bytes for the previous buffer of input, which - * will be scanned for useful backreferences. */ - size_t buf_sz = (2 << window_sz2); - - heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz); - if (hse == NULL) { return NULL; } - hse->window_sz2 = window_sz2; - hse->lookahead_sz2 = lookahead_sz2; - heatshrink_encoder_reset(hse); - -#if HEATSHRINK_USE_INDEX - size_t index_sz = buf_sz*sizeof(uint16_t); - hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index)); - if (hse->search_index == NULL) { - HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz); - return NULL; - } - hse->search_index->size = index_sz; -#endif - - LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n", - buf_sz, get_input_buffer_size(hse)); - return hse; -} - -void heatshrink_encoder_free(heatshrink_encoder *hse) { - size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); -#if HEATSHRINK_USE_INDEX - size_t index_sz = sizeof(struct hs_index) + hse->search_index->size; - HEATSHRINK_FREE(hse->search_index, index_sz); - (void)index_sz; -#endif - HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz); - (void)buf_sz; -} -#endif - -void heatshrink_encoder_reset(heatshrink_encoder *hse) { - size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); - memset(hse->buffer, 0, buf_sz); - hse->input_size = 0; - hse->state = HSES_NOT_FULL; - hse->match_scan_index = 0; - hse->flags = 0; - hse->bit_index = 0x80; - hse->current_byte = 0x00; - hse->match_length = 0; - - hse->outgoing_bits = 0x0000; - hse->outgoing_bits_count = 0; - - #ifdef LOOP_DETECT - hse->loop_detect = (uint32_t)-1; - #endif -} - -HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, - uint8_t *in_buf, size_t size, size_t *input_size) { - if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) { - return HSER_SINK_ERROR_NULL; - } - - /* Sinking more content after saying the content is done, tsk tsk */ - if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; } - - /* Sinking more content before processing is done */ - if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; } - - uint16_t write_offset = get_input_offset(hse) + hse->input_size; - uint16_t ibs = get_input_buffer_size(hse); - uint16_t rem = ibs - hse->input_size; - uint16_t cp_sz = rem < size ? rem : size; - - memcpy(&hse->buffer[write_offset], in_buf, cp_sz); - *input_size = cp_sz; - hse->input_size += cp_sz; - - LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n", - cp_sz, size, write_offset, hse->input_size); - if (cp_sz == rem) { - LOG("-- internal buffer is now full\n"); - hse->state = HSES_FILLED; - } - - return HSER_SINK_OK; -} - - -/*************** - * Compression * - ***************/ - -static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, - uint16_t end, const uint16_t maxlen, uint16_t *match_length); -static void do_indexing(heatshrink_encoder *hse); - -static HSE_state st_step_search(heatshrink_encoder *hse); -static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_literal(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_br_index(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_yield_br_length(heatshrink_encoder *hse, - output_info *oi); -static HSE_state st_save_backlog(heatshrink_encoder *hse); -static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, - output_info *oi); - -HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size) { - if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) { - return HSER_POLL_ERROR_NULL; - } - if (out_buf_size == 0) { - LOG("-- MISUSE: output buffer size is 0\n"); - return HSER_POLL_ERROR_MISUSE; - } - *output_size = 0; - - output_info oi; - oi.buf = out_buf; - oi.buf_size = out_buf_size; - oi.output_size = output_size; - - while (1) { - LOG("-- polling, state %u (%s), flags 0x%02x\n", - hse->state, state_names[hse->state], hse->flags); - - uint8_t in_state = hse->state; - switch (in_state) { - case HSES_NOT_FULL: - return HSER_POLL_EMPTY; - case HSES_FILLED: - do_indexing(hse); - hse->state = HSES_SEARCH; - break; - case HSES_SEARCH: - hse->state = st_step_search(hse); - break; - case HSES_YIELD_TAG_BIT: - hse->state = st_yield_tag_bit(hse, &oi); - break; - case HSES_YIELD_LITERAL: - hse->state = st_yield_literal(hse, &oi); - break; - case HSES_YIELD_BR_INDEX: - hse->state = st_yield_br_index(hse, &oi); - break; - case HSES_YIELD_BR_LENGTH: - hse->state = st_yield_br_length(hse, &oi); - break; - case HSES_SAVE_BACKLOG: - hse->state = st_save_backlog(hse); - break; - case HSES_FLUSH_BITS: - hse->state = st_flush_bit_buffer(hse, &oi); - case HSES_DONE: - return HSER_POLL_EMPTY; - default: - LOG("-- bad state %s\n", state_names[hse->state]); - return HSER_POLL_ERROR_MISUSE; - } - - if (hse->state == in_state) { - /* Check if output buffer is exhausted. */ - if (*output_size == out_buf_size) return HSER_POLL_MORE; - } - } -} - -HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) { - if (hse == NULL) { return HSER_FINISH_ERROR_NULL; } - LOG("-- setting is_finishing flag\n"); - hse->flags |= FLAG_IS_FINISHING; - if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; } - return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE; -} - -static HSE_state st_step_search(heatshrink_encoder *hse) { - uint16_t window_length = get_input_buffer_size(hse); - uint16_t lookahead_sz = get_lookahead_size(hse); - uint16_t msi = hse->match_scan_index; - LOG("## step_search, scan @ +%d (%d/%d), input size %d\n", - msi, hse->input_size + msi, 2*window_length, hse->input_size); - - bool fin = is_finishing(hse); - if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) { - /* Current search buffer is exhausted, copy it into the - * backlog and await more input. */ - LOG("-- end of search @ %d, saving backlog\n", msi); - return HSES_SAVE_BACKLOG; - } - - uint16_t input_offset = get_input_offset(hse); - uint16_t end = input_offset + msi; - - uint16_t start = 0; - if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */ - start = end - window_length + 1; - } else if (backlog_is_partial(hse)) { /* clamp to available data */ - start = end - window_length + 1; - if (start < lookahead_sz) { start = lookahead_sz; } - } else { /* only scan available input */ - start = input_offset; - } - - uint16_t max_possible = lookahead_sz; - if (hse->input_size - msi < lookahead_sz) { - max_possible = hse->input_size - msi; - } - - uint16_t match_length = 0; - uint16_t match_pos = find_longest_match(hse, - start, end, max_possible, &match_length); - - if (match_pos == MATCH_NOT_FOUND) { - LOG("ss Match not found\n"); - hse->match_scan_index++; - hse->flags |= FLAG_HAS_LITERAL; - hse->match_length = 0; - return HSES_YIELD_TAG_BIT; - } else { - LOG("ss Found match of %d bytes at %d\n", match_length, match_pos); - hse->match_pos = match_pos; - hse->match_length = match_length; - ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/); - - return HSES_YIELD_TAG_BIT; - } -} - -static HSE_state st_yield_tag_bit(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - if (hse->match_length == 0) { - add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER); - return HSES_YIELD_LITERAL; - } else { - add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER); - hse->outgoing_bits = hse->match_pos - 1; - hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse); - return HSES_YIELD_BR_INDEX; - } - } else { - return HSES_YIELD_TAG_BIT; /* output is full, continue */ - } -} - -static HSE_state st_yield_literal(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - push_literal_byte(hse, oi); - hse->flags &= ~FLAG_HAS_LITERAL; - if (on_final_literal(hse)) { return HSES_FLUSH_BITS; } - return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH; - } else { - return HSES_YIELD_LITERAL; - } -} - -static HSE_state st_yield_br_index(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - LOG("-- yielding backref index %u\n", hse->match_pos); - if (push_outgoing_bits(hse, oi) > 0) { - return HSES_YIELD_BR_INDEX; /* continue */ - } else { - hse->outgoing_bits = hse->match_length - 1; - hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse); - return HSES_YIELD_BR_LENGTH; /* done */ - } - } else { - return HSES_YIELD_BR_INDEX; /* continue */ - } -} - -static HSE_state st_yield_br_length(heatshrink_encoder *hse, - output_info *oi) { - if (can_take_byte(oi)) { - LOG("-- yielding backref length %u\n", hse->match_length); - if (push_outgoing_bits(hse, oi) > 0) { - return HSES_YIELD_BR_LENGTH; - } else { - hse->match_scan_index += hse->match_length; - hse->match_length = 0; - return HSES_SEARCH; - } - } else { - return HSES_YIELD_BR_LENGTH; - } -} - -static HSE_state st_save_backlog(heatshrink_encoder *hse) { - if (is_finishing(hse)) { - /* copy remaining literal (if necessary) */ - if (has_literal(hse)) { - hse->flags |= FLAG_ON_FINAL_LITERAL; - return HSES_YIELD_TAG_BIT; - } else { - return HSES_FLUSH_BITS; - } - } else { - LOG("-- saving backlog\n"); - save_backlog(hse); - return HSES_NOT_FULL; - } -} - -static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse, - output_info *oi) { - if (hse->bit_index == 0x80) { - LOG("-- done!\n"); - return HSES_DONE; - } else if (can_take_byte(oi)) { - LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index); - oi->buf[(*oi->output_size)++] = hse->current_byte; - LOG("-- done!\n"); - return HSES_DONE; - } else { - return HSES_FLUSH_BITS; - } -} - -static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) { - LOG("-- adding tag bit: %d\n", tag); - push_bits(hse, 1, tag, oi); -} - -static uint16_t get_input_offset(heatshrink_encoder *hse) { - return get_input_buffer_size(hse); -} - -static uint16_t get_input_buffer_size(heatshrink_encoder *hse) { - return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse)); - (void)hse; -} - -static uint16_t get_lookahead_size(heatshrink_encoder *hse) { - return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse)); - (void)hse; -} - -static void do_indexing(heatshrink_encoder *hse) { -#if HEATSHRINK_USE_INDEX - /* Build an index array I that contains flattened linked lists - * for the previous instances of every byte in the buffer. - * - * For example, if buf[200] == 'x', then index[200] will either - * be an offset i such that buf[i] == 'x', or a negative offset - * to indicate end-of-list. This significantly speeds up matching, - * while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM. - * - * Future optimization options: - * 1. Since any negative value represents end-of-list, the other - * 15 bits could be used to improve the index dynamically. - * - * 2. Likewise, the last lookahead_sz bytes of the index will - * not be usable, so temporary data could be stored there to - * dynamically improve the index. - * */ - struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); - uint16_t last[256]; - memset(last, 0xFF, sizeof(last)); - - uint8_t * const data = hse->buffer; - int16_t * const index = hsi->index; - - const uint16_t input_offset = get_input_offset(hse); - const uint16_t end = input_offset + hse->input_size; - - for (uint16_t i=0; iflags & FLAG_IS_FINISHING; -} - -static int backlog_is_partial(heatshrink_encoder *hse) { - return hse->flags & FLAG_BACKLOG_IS_PARTIAL; -} - -static int backlog_is_filled(heatshrink_encoder *hse) { - return hse->flags & FLAG_BACKLOG_IS_FILLED; -} - -static int on_final_literal(heatshrink_encoder *hse) { - return hse->flags & FLAG_ON_FINAL_LITERAL; -} - -static int has_literal(heatshrink_encoder *hse) { - return (hse->flags & FLAG_HAS_LITERAL); -} - -static int can_take_byte(output_info *oi) { - return *oi->output_size < oi->buf_size; -} - -/* Return the longest match for the bytes at buf[end:end+maxlen] between - * buf[start] and buf[end-1]. If no match is found, return -1. */ -static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start, - uint16_t end, const uint16_t maxlen, uint16_t *match_length) { - LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n", - end, end + maxlen, start, end + maxlen - 1, maxlen); - uint8_t *buf = hse->buffer; - - uint16_t match_maxlen = 0; - uint16_t match_index = MATCH_NOT_FOUND; - const uint16_t break_even_point = 3; - uint16_t len = 0; - uint8_t * const needlepoint = &buf[end]; -#if HEATSHRINK_USE_INDEX - struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse); - int16_t pos = hsi->index[end]; - - while (pos >= start) { - uint8_t * const pospoint = &buf[pos]; - len = 0; - - /* Only check matches that will potentially beat the current maxlen. - * This is redundant with the index if match_maxlen is 0, but the - * added branch overhead to check if it == 0 seems to be worse. */ - if (pospoint[match_maxlen] != needlepoint[match_maxlen]) { - pos = hsi->index[pos]; - continue; - } - - for (len = 1; len < maxlen; len++) { - if (pospoint[len] != needlepoint[len]) break; - } - - if (len > match_maxlen) { - match_maxlen = len; - match_index = pos; - if (len == maxlen) { break; } /* won't find better */ - } - pos = hsi->index[pos]; - } -#else - for (int16_t pos=end - 1; pos >= start; pos--) { - uint8_t * const pospoint = &buf[pos]; - if ((pospoint[match_maxlen] == needlepoint[match_maxlen]) - && (*pospoint == *needlepoint)) { - for (len=1; len cmp buf[%d] == 0x%02x against %02x (start %u)\n", - pos + len, pospoint[len], needlepoint[len], start); - } - if (pospoint[len] != needlepoint[len]) { break; } - } - if (len > match_maxlen) { - match_maxlen = len; - match_index = pos; - if (len == maxlen) { break; } /* don't keep searching */ - } - } - } -#endif - - if (match_maxlen >= break_even_point) { - LOG("-- best match: %u bytes at -%u\n", - match_maxlen, end - match_index); - *match_length = match_maxlen; - return end - match_index; - } - LOG("-- none found\n"); - return MATCH_NOT_FOUND; -} - -static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) { - uint8_t count = 0; - uint8_t bits = 0; - if (hse->outgoing_bits_count > 8) { - count = 8; - bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8); - } else { - count = hse->outgoing_bits_count; - bits = hse->outgoing_bits; - } - - if (count > 0) { - LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits); - push_bits(hse, count, bits, oi); - hse->outgoing_bits_count -= count; - } - return count; -} - -/* Push COUNT (max 8) bits to the output buffer, which has room. - * Bytes are set from the lowest bits, up. */ -static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits, - output_info *oi) { - ASSERT(count <= 8); - LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits); - - /* If adding a whole byte and at the start of a new output byte, - * just push it through whole and skip the bit IO loop. */ - if (count == 8 && hse->bit_index == 0x80) { - oi->buf[(*oi->output_size)++] = bits; - } else { - for (int i=count - 1; i>=0; i--) { - bool bit = bits & (1 << i); - if (bit) { hse->current_byte |= hse->bit_index; } - if (0) { - LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n", - bit ? 1 : 0, hse->bit_index, hse->current_byte); - } - hse->bit_index >>= 1; - if (hse->bit_index == 0x00) { - hse->bit_index = 0x80; - LOG(" > pushing byte 0x%02x\n", hse->current_byte); - oi->buf[(*oi->output_size)++] = hse->current_byte; - hse->current_byte = 0x00; - } - } - } -} - -static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) { - uint16_t processed_offset = hse->match_scan_index - 1; - uint16_t input_offset = get_input_offset(hse) + processed_offset; - uint8_t c = hse->buffer[input_offset]; - LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n", - c, isprint(c) ? c : '.', input_offset); - push_bits(hse, 8, c, oi); -} - -static void save_backlog(heatshrink_encoder *hse) { - size_t input_buf_sz = get_input_buffer_size(hse); - - uint16_t msi = hse->match_scan_index; - - /* Copy processed data to beginning of buffer, so it can be - * used for future matches. Don't bother checking whether the - * input is less than the maximum size, because if it isn't, - * we're done anyway. */ - uint16_t rem = input_buf_sz - msi; // unprocessed bytes - uint16_t shift_sz = input_buf_sz + rem; - - memmove(&hse->buffer[0], - &hse->buffer[input_buf_sz - rem], - shift_sz); - - if (backlog_is_partial(hse)) { - /* The whole backlog is filled in now, so include it in scans. */ - hse->flags |= FLAG_BACKLOG_IS_FILLED; - } else { - /* Include backlog, except for the first lookahead_sz bytes, which - * are still undefined. */ - hse->flags |= FLAG_BACKLOG_IS_PARTIAL; - } - hse->match_scan_index = 0; - hse->input_size -= input_buf_sz - rem; -} diff --git a/espfs/heatshrink/heatshrink_encoder.h b/espfs/heatshrink/heatshrink_encoder.h deleted file mode 100644 index 18c1773..0000000 --- a/espfs/heatshrink/heatshrink_encoder.h +++ /dev/null @@ -1,109 +0,0 @@ -#ifndef HEATSHRINK_ENCODER_H -#define HEATSHRINK_ENCODER_H - -#include -#include -#include "heatshrink_common.h" -#include "heatshrink_config.h" - -typedef enum { - HSER_SINK_OK, /* data sunk into input buffer */ - HSER_SINK_ERROR_NULL=-1, /* NULL argument */ - HSER_SINK_ERROR_MISUSE=-2, /* API misuse */ -} HSE_sink_res; - -typedef enum { - HSER_POLL_EMPTY, /* input exhausted */ - HSER_POLL_MORE, /* poll again for more output */ - HSER_POLL_ERROR_NULL=-1, /* NULL argument */ - HSER_POLL_ERROR_MISUSE=-2, /* API misuse */ -} HSE_poll_res; - -typedef enum { - HSER_FINISH_DONE, /* encoding is complete */ - HSER_FINISH_MORE, /* more output remaining; use poll */ - HSER_FINISH_ERROR_NULL=-1, /* NULL argument */ -} HSE_finish_res; - -#if HEATSHRINK_DYNAMIC_ALLOC -#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \ - ((HSE)->window_sz2) -#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \ - ((HSE)->lookahead_sz2) -#define HEATSHRINK_ENCODER_INDEX(HSE) \ - ((HSE)->search_index) -struct hs_index { - uint16_t size; - int16_t index[]; -}; -#else -#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \ - (HEATSHRINK_STATIC_WINDOW_BITS) -#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \ - (HEATSHRINK_STATIC_LOOKAHEAD_BITS) -#define HEATSHRINK_ENCODER_INDEX(HSE) \ - (&(HSE)->search_index) -struct hs_index { - uint16_t size; - int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS]; -}; -#endif - -typedef struct { - uint16_t input_size; /* bytes in input buffer */ - uint16_t match_scan_index; - uint16_t match_length; - uint16_t match_pos; - uint16_t outgoing_bits; /* enqueued outgoing bits */ - uint8_t outgoing_bits_count; - uint8_t flags; - uint8_t state; /* current state machine node */ - uint8_t current_byte; /* current byte of output */ - uint8_t bit_index; /* current bit index */ -#if HEATSHRINK_DYNAMIC_ALLOC - uint8_t window_sz2; /* 2^n size of window */ - uint8_t lookahead_sz2; /* 2^n size of lookahead */ -#if HEATSHRINK_USE_INDEX - struct hs_index *search_index; -#endif - /* input buffer and / sliding window for expansion */ - uint8_t buffer[]; -#else - #if HEATSHRINK_USE_INDEX - struct hs_index search_index; - #endif - /* input buffer and / sliding window for expansion */ - uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)]; -#endif -} heatshrink_encoder; - -#if HEATSHRINK_DYNAMIC_ALLOC -/* Allocate a new encoder struct and its buffers. - * Returns NULL on error. */ -heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2, - uint8_t lookahead_sz2); - -/* Free an encoder. */ -void heatshrink_encoder_free(heatshrink_encoder *hse); -#endif - -/* Reset an encoder. */ -void heatshrink_encoder_reset(heatshrink_encoder *hse); - -/* Sink up to SIZE bytes from IN_BUF into the encoder. - * INPUT_SIZE is set to the number of bytes actually sunk (in case a - * buffer was filled.). */ -HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse, - uint8_t *in_buf, size_t size, size_t *input_size); - -/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into - * OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */ -HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse, - uint8_t *out_buf, size_t out_buf_size, size_t *output_size); - -/* Notify the encoder that the input stream is finished. - * If the return value is HSER_FINISH_MORE, there is still more output, so - * call heatshrink_encoder_poll and repeat. */ -HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse); - -#endif diff --git a/espfs/heatshrink/test_heatshrink_dynamic.c b/espfs/heatshrink/test_heatshrink_dynamic.c deleted file mode 100644 index 1e18a69..0000000 --- a/espfs/heatshrink/test_heatshrink_dynamic.c +++ /dev/null @@ -1,999 +0,0 @@ -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" - -#if !HEATSHRINK_DYNAMIC_ALLOC -#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for dynamic allocation test suite. -#endif - -SUITE(encoding); -SUITE(decoding); -SUITE(integration); - -#ifdef HEATSHRINK_HAS_THEFT -SUITE(properties); -#endif - -static void dump_buf(char *name, uint8_t *buf, uint16_t count) { - for (int i=0; iinput_size, 6); - ASSERT_EQ(hsd->input_index, 0); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_return_empty_if_empty(void) { - uint8_t output[256]; - size_t out_sz = 0; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_EMPTY, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_reject_null_hsd(void) { - uint8_t output[256]; - size_t out_sz = 0; - HSD_poll_res res = heatshrink_decoder_poll(NULL, output, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - PASS(); -} - -TEST decoder_poll_should_reject_null_output_buffer(void) { - size_t out_sz = 0; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, NULL, 256, &out_sz); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_reject_null_output_size_pointer(void) { - uint8_t output[256]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, - HEATSHRINK_MIN_WINDOW_BITS, 4); - HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, NULL); - ASSERT_EQ(HSDR_POLL_ERROR_NULL, res); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_literal(void) { - uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0 }; //"foo" - uint8_t output[4]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 3); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - ASSERT_EQ(3, out_sz); - ASSERT_EQ('f', output[0]); - ASSERT_EQ('o', output[1]); - ASSERT_EQ('o', output[2]); - - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_literal_and_backref(void) { - uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo" - uint8_t output[6]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7); - memset(output, 0, sizeof(*output)); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - (void)heatshrink_decoder_poll(hsd, output, 6, &out_sz); - - if (0) dump_buf("output", output, out_sz); - ASSERT_EQ(6, out_sz); - ASSERT_EQ('f', output[0]); - ASSERT_EQ('o', output[1]); - ASSERT_EQ('o', output[2]); - ASSERT_EQ('f', output[3]); - ASSERT_EQ('o', output[4]); - ASSERT_EQ('o', output[5]); - - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST decoder_poll_should_expand_short_self_overlapping_backref(void) { - /* "aaaaa" == (literal, 1), ('a'), (backref, 1 back, 4 bytes) */ - uint8_t input[] = {0xb0, 0x80, 0x01, 0x80}; - uint8_t output[6]; - uint8_t expected[] = {'a', 'a', 'a', 'a', 'a'}; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 7); - size_t count = 0; - - HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count); - ASSERT_EQ(HSDR_SINK_OK, sres); - - size_t out_sz = 0; - (void)heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); - - if (0) dump_buf("output", output, out_sz); - ASSERT_EQ(sizeof(expected), out_sz); - for (size_t i=0; iwindow_sz2, - cfg->lookahead_sz2); - heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size, - cfg->window_sz2, cfg->lookahead_sz2); - size_t comp_sz = input_size + (input_size/2) + 4; - size_t decomp_sz = input_size + (input_size/2) + 4; - uint8_t *comp = malloc(comp_sz); - uint8_t *decomp = malloc(decomp_sz); - if (comp == NULL) FAILm("malloc fail"); - if (decomp == NULL) FAILm("malloc fail"); - memset(comp, 0, comp_sz); - memset(decomp, 0, decomp_sz); - - size_t count = 0; - - if (cfg->log_lvl > 1) { - printf("\n^^ COMPRESSING\n"); - dump_buf("input", input, input_size); - } - - size_t sunk = 0; - size_t polled = 0; - while (sunk < input_size) { - ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0); - sunk += count; - if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - } - - HSE_poll_res pres; - do { /* "turn the crank" */ - pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSER_POLL_MORE); - ASSERT_EQ(HSER_POLL_EMPTY, pres); - if (polled >= comp_sz) FAILm("compression should never expand that much"); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse)); - } - } - if (cfg->log_lvl > 0) printf("in: %u compressed: %zu ", input_size, polled); - size_t compressed_size = polled; - sunk = 0; - polled = 0; - - if (cfg->log_lvl > 1) { - printf("\n^^ DECOMPRESSING\n"); - dump_buf("comp", comp, compressed_size); - } - while (sunk < compressed_size) { - ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); - sunk += count; - if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == compressed_size) { - ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd)); - } - - HSD_poll_res pres; - do { - pres = heatshrink_decoder_poll(hsd, &decomp[polled], - decomp_sz - polled, &count); - ASSERT(pres >= 0); - ASSERT(count > 0); - polled += count; - if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSDR_POLL_MORE); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - if (sunk == compressed_size) { - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - ASSERT_EQ(HSDR_FINISH_DONE, fres); - } - - if (polled > input_size) { - printf("\nExpected %zd, got %zu\n", (size_t)input_size, polled); - FAILm("Decompressed data is larger than original input"); - } - } - if (cfg->log_lvl > 0) printf("decompressed: %zu\n", polled); - if (polled != input_size) { - FAILm("Decompressed length does not match original input length"); - } - - if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled); - for (uint32_t i=0; i out[%d] == 0x%02x ('%c') %c\n", - j, input[j], isprint(input[j]) ? input[j] : '.', - j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.', - input[j] == decomp[j] ? ' ' : 'X'); - } - } - } - ASSERT_EQ(input[i], decomp[i]); - } - free(comp); - free(decomp); - heatshrink_encoder_free(hse); - heatshrink_decoder_free(hsd); - PASS(); -} - -TEST data_without_duplication_should_match(void) { - uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', - 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 256; - return compress_and_expand_and_check(input, sizeof(input), &cfg); -} - -TEST data_with_simple_repetition_should_compress_and_decompress_properly(void) { - uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b', - 'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f', - 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b', - 'c', 'd', 'e', 'f', 'g', 'h'}; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 256; - return compress_and_expand_and_check(input, sizeof(input), &cfg); -} - -TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) { - heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3); - heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3); - uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', - 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', - 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'}; - uint8_t comp[60]; - uint8_t decomp[60]; - size_t count = 0; - int log = 0; - - if (log) dump_buf("input", input, sizeof(input)); - for (uint32_t i=0; i= 0); - } - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - - size_t packed_count = 0; - do { - ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); - packed_count += count; - } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); - - if (log) dump_buf("comp", comp, packed_count); - for (uint32_t i=0; i= 0); - } - - for (uint32_t i=0; i= 0); - } - - if (log) dump_buf("decomp", decomp, sizeof(input)); - for (uint32_t i=0; i= 0); - } - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse)); - - size_t packed_count = 0; - do { - ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0); - packed_count += count; - } while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE); - - if (log) dump_buf("comp", comp, packed_count); - for (uint32_t i=0; i= 0); - } - - for (uint32_t i=0; i= 0); - } - - if (log) dump_buf("decomp", decomp, sizeof(input)); - for (uint32_t i=0; ilog_lvl > 0) { - printf("\n-- size %u, seed %u, input buf %zu\n", - size, seed, cfg->decoder_input_buffer_size); - } - fill_with_pseudorandom_letters(input, size, seed); - return compress_and_expand_and_check(input, size, cfg); -} - -TEST small_input_buffer_should_not_impact_decoder_correctness(void) { - int size = 5; - uint8_t input[size]; - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = 3; - cfg.decoder_input_buffer_size = 5; - for (uint16_t i=0; i= 19901L - printf("\n\nFuzzing (single-byte sizes):\n"); - for (uint8_t lsize=3; lsize < 8; lsize++) { - for (uint32_t size=1; size < 128*1024L; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ - if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); - for (uint32_t seed=1; seed<=10; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 8; - cfg.lookahead_sz2 = lsize; - cfg.decoder_input_buffer_size = ibs; - RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); - } - } - } - } - - printf("\nFuzzing (multi-byte sizes):\n"); - for (uint8_t lsize=6; lsize < 9; lsize++) { - for (uint32_t size=1; size < 128*1024L; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */ - if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs); - for (uint32_t seed=1; seed<=10; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - cfg_info cfg; - cfg.log_lvl = 0; - cfg.window_sz2 = 11; - cfg.lookahead_sz2 = lsize; - cfg.decoder_input_buffer_size = ibs; - RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg); - } - } - } - } - -#endif -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - RUN_SUITE(encoding); - RUN_SUITE(decoding); - RUN_SUITE(integration); - #ifdef HEATSHRINK_HAS_THEFT - RUN_SUITE(properties); - #endif - GREATEST_MAIN_END(); /* display results */ -} diff --git a/espfs/heatshrink/test_heatshrink_dynamic_theft.c b/espfs/heatshrink/test_heatshrink_dynamic_theft.c deleted file mode 100644 index 2752886..0000000 --- a/espfs/heatshrink/test_heatshrink_dynamic_theft.c +++ /dev/null @@ -1,521 +0,0 @@ -#include "heatshrink_config.h" -#ifdef HEATSHRINK_HAS_THEFT - -#include -#include -#include -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" -#include "theft.h" -#include "greatest_theft.h" - -#if !HEATSHRINK_DYNAMIC_ALLOC -#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for this test suite. -#endif - -SUITE(properties); - -typedef struct { - int limit; - int fails; - int dots; -} test_env; - -typedef struct { - size_t size; - uint8_t buf[]; -} rbuf; - -static void *rbuf_alloc_cb(struct theft *t, theft_hash seed, void *env) { - test_env *te = (test_env *)env; - //printf("seed is 0x%016llx\n", seed); - - size_t sz = (size_t)(seed % te->limit) + 1; - rbuf *r = malloc(sizeof(rbuf) + sz); - if (r == NULL) { return THEFT_ERROR; } - r->size = sz; - - for (size_t i = 0; i < sz; i += sizeof(theft_hash)) { - theft_hash s = theft_random(t); - for (uint8_t b = 0; b < sizeof(theft_hash); b++) { - if (i + b >= sz) { break; } - r->buf[i + b] = (uint8_t) (s >> (8*b)) & 0xff; - } - } - - return r; -} - -static void rbuf_free_cb(void *instance, void *env) { - free(instance); - (void)env; -} - -static uint64_t rbuf_hash_cb(void *instance, void *env) { - rbuf *r = (rbuf *)instance; - (void)env; - return theft_hash_onepass(r->buf, r->size); -} - -/* Make a copy of a buffer, keeping NEW_SZ bytes starting at OFFSET. */ -static void *copy_rbuf_subset(rbuf *cur, size_t new_sz, size_t byte_offset) { - if (new_sz == 0) { return THEFT_DEAD_END; } - rbuf *nr = malloc(sizeof(rbuf) + new_sz); - if (nr == NULL) { return THEFT_ERROR; } - nr->size = new_sz; - memcpy(nr->buf, &cur->buf[byte_offset], new_sz); - /* printf("%zu -> %zu\n", cur->size, new_sz); */ - return nr; -} - -/* Make a copy of a buffer, but only PORTION, starting OFFSET in - * (e.g. the third quarter is (0.25 at +0.75). Rounds to ints. */ -static void *copy_rbuf_percent(rbuf *cur, float portion, float offset) { - size_t new_sz = cur->size * portion; - size_t byte_offset = (size_t)(cur->size * offset); - return copy_rbuf_subset(cur, new_sz, byte_offset); -} - -/* How to shrink a random buffer to a simpler one. */ -static void *rbuf_shrink_cb(void *instance, uint32_t tactic, void *env) { - rbuf *cur = (rbuf *)instance; - - if (tactic == 0) { /* first half */ - return copy_rbuf_percent(cur, 0.5, 0); - } else if (tactic == 1) { /* second half */ - return copy_rbuf_percent(cur, 0.5, 0.5); - } else if (tactic <= 18) { /* drop 1-16 bytes at start */ - const int last_tactic = 1; - const size_t drop = tactic - last_tactic; - if (cur->size < drop) { return THEFT_DEAD_END; } - return copy_rbuf_subset(cur, cur->size - drop, drop); - } else if (tactic <= 34) { /* drop 1-16 bytes at end */ - const int last_tactic = 18; - const size_t drop = tactic - last_tactic; - if (cur->size < drop) { return THEFT_DEAD_END; } - return copy_rbuf_subset(cur, cur->size - drop, 0); - } else if (tactic == 35) { - /* Divide every byte by 2, saturating at 0 */ - rbuf *cp = copy_rbuf_percent(cur, 1, 0); - if (cp == NULL) { return THEFT_ERROR; } - for (size_t i = 0; i < cp->size; i++) { cp->buf[i] /= 2; } - return cp; - } else if (tactic == 36) { - /* subtract 1 from every byte, saturating at 0 */ - rbuf *cp = copy_rbuf_percent(cur, 1, 0); - if (cp == NULL) { return THEFT_ERROR; } - for (size_t i = 0; i < cp->size; i++) { - if (cp->buf[i] > 0) { cp->buf[i]--; } - } - return cp; - } else { - (void)env; - return THEFT_NO_MORE_TACTICS; - } - - return THEFT_NO_MORE_TACTICS; -} - -static void rbuf_print_cb(FILE *f, void *instance, void *env) { - rbuf *r = (rbuf *)instance; - (void)env; - fprintf(f, "buf[%zd]:\n ", r->size); - uint8_t bytes = 0; - for (size_t i = 0; i < r->size; i++) { - fprintf(f, "%02x", r->buf[i]); - bytes++; - if (bytes == 16) { - fprintf(f, "\n "); - bytes = 0; - } - } - fprintf(f, "\n"); -} - -static struct theft_type_info rbuf_info = { - .alloc = rbuf_alloc_cb, - .free = rbuf_free_cb, - .hash = rbuf_hash_cb, - .shrink = rbuf_shrink_cb, - .print = rbuf_print_cb, -}; - -static theft_progress_callback_res -progress_cb(struct theft_trial_info *info, void *env) { - test_env *te = (test_env *)env; - if ((info->trial & 0xff) == 0) { - printf("."); - fflush(stdout); - te->dots++; - if (te->dots == 64) { - printf("\n"); - te->dots = 0; - } - } - - if (info->status == THEFT_TRIAL_FAIL) { - te->fails++; - rbuf *cur = info->args[0]; - if (cur->size < 5) { return THEFT_PROGRESS_HALT; } - } - - if (te->fails > 10) { - return THEFT_PROGRESS_HALT; - } - return THEFT_PROGRESS_CONTINUE; -} - -/* For an arbitrary input buffer, it should never get stuck in a - * state where the data has been sunk but no data can be polled. */ -static theft_trial_res prop_should_not_get_stuck(void *input) { - /* Make a buffer large enough for the output: 4 KB of input with - * each 16 bits becoming up to 16 bytes will fit in a 64 KB buffer. - * (4 KB of input comes from `env.limit = 1 << 12;` below.) */ - uint8_t output[64 * 1024]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc((64 * 1024L) - 1, 12, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t count = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, r->buf, r->size, &count); - if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } - - size_t out_sz = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz); - if (pres != HSDR_POLL_EMPTY) { return THEFT_TRIAL_FAIL; } - - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - heatshrink_decoder_free(hsd); - if (fres != HSDR_FINISH_DONE) { return THEFT_TRIAL_FAIL; } - - return THEFT_TRIAL_PASS; -} - -static bool get_time_seed(theft_seed *seed) -{ - struct timeval tv; - if (-1 == gettimeofday(&tv, NULL)) { return false; } - *seed = (theft_seed)((tv.tv_sec << 32) | tv.tv_usec); - /* printf("seed is 0x%016llx\n", *seed); */ - return true; -} - -TEST decoder_fuzzing_should_not_detect_stuck_state(void) { - // Get a random number seed based on the time - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - /* Pass the max buffer size for this property (4 KB) in a closure */ - test_env env = { .limit = 1 << 12 }; - - theft_seed always_seeds = { 0xe87bb1f61032a061 }; - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_should_not_get_stuck, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 100000, - .progress_cb = progress_cb, - .env = &env, - - .always_seeds = &always_seeds, - .always_seed_count = 1, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - GREATEST_ASSERT_EQm("should_not_get_stuck", THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res prop_encoded_and_decoded_data_should_match(void *input) { - uint8_t e_output[64 * 1024]; - uint8_t d_output[64 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - heatshrink_decoder *hsd = heatshrink_decoder_alloc(4096, 12, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t e_input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - r->buf, r->size, &e_input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - if (e_input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t e_output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - e_output, sizeof(e_output), &e_output_size); - if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t count = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, e_output, e_output_size, &count); - if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; } - - size_t d_output_size = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, d_output, - sizeof(d_output), &d_output_size); - if (pres != HSDR_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - if (d_output_size != r->size) { - printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; - } - - if (0 != memcmp(d_output, r->buf, d_output_size)) { - return THEFT_TRIAL_FAIL; - } - - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - if (fres != HSDR_FINISH_DONE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - heatshrink_encoder_free(hse); - heatshrink_decoder_free(hsd); - return THEFT_TRIAL_PASS; -} - - -TEST encoded_and_decoded_data_should_match(void) { - test_env env = { .limit = 1 << 11 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoded_and_decoded_data_should_match, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 1000000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static size_t ceil_nine_eighths(size_t sz) { - return sz + sz/8 + (sz & 0x07 ? 1 : 0); -} - -static theft_trial_res -prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void *input) { - uint8_t output[32 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)input; - - size_t input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - r->buf, r->size, &input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - /* Assumes data fits in one sink, failure here means buffer must be larger. */ - if (input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - output, sizeof(output), &output_size); - if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; } - - size_t ceil_9_8s = ceil_nine_eighths(r->size); - if (output_size > ceil_9_8s) { - return THEFT_TRIAL_FAIL; - } - - heatshrink_encoder_free(hse); - return THEFT_TRIAL_PASS; -} - -TEST encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void) { - test_env env = { .limit = 1 << 11 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res -prop_encoder_should_always_make_progress(void *instance) { - uint8_t output[64 * 1024]; - heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4); - if (hse == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)instance; - - size_t sunk = 0; - int no_progress = 0; - - while (1) { - if (sunk < r->size) { - size_t input_size = 0; - HSE_sink_res esres = heatshrink_encoder_sink(hse, - &r->buf[sunk], r->size - sunk, &input_size); - if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - sunk += input_size; - } else { - HSE_finish_res efres = heatshrink_encoder_finish(hse); - if (efres == HSER_FINISH_DONE) { - break; - } else if (efres != HSER_FINISH_MORE) { - printf("FAIL %d\n", __LINE__); - return THEFT_TRIAL_FAIL; - } - } - - size_t output_size = 0; - HSE_poll_res epres = heatshrink_encoder_poll(hse, - output, sizeof(output), &output_size); - if (epres < 0) { return THEFT_TRIAL_ERROR; } - if (output_size == 0 && sunk == r->size) { - no_progress++; - if (no_progress > 2) { - return THEFT_TRIAL_FAIL; - } - } else { - no_progress = 0; - } - } - - heatshrink_encoder_free(hse); - return THEFT_TRIAL_PASS; -} - -TEST encoder_should_always_make_progress(void) { - test_env env = { .limit = 1 << 15 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_encoder_should_always_make_progress, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -static theft_trial_res -prop_decoder_should_always_make_progress(void *instance) { - uint8_t output[64 * 1024]; - heatshrink_decoder *hsd = heatshrink_decoder_alloc(512, 8, 4); - if (hsd == NULL) { return THEFT_TRIAL_ERROR; } - - rbuf *r = (rbuf *)instance; - - size_t sunk = 0; - int no_progress = 0; - - while (1) { - if (sunk < r->size) { - size_t input_size = 0; - HSD_sink_res sres = heatshrink_decoder_sink(hsd, - &r->buf[sunk], r->size - sunk, &input_size); - if (sres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; } - sunk += input_size; - } else { - HSD_finish_res fres = heatshrink_decoder_finish(hsd); - if (fres == HSDR_FINISH_DONE) { - break; - } else if (fres != HSDR_FINISH_MORE) { - printf("FAIL %d\n", __LINE__); - return THEFT_TRIAL_FAIL; - } - } - - size_t output_size = 0; - HSD_poll_res pres = heatshrink_decoder_poll(hsd, - output, sizeof(output), &output_size); - if (pres < 0) { return THEFT_TRIAL_ERROR; } - if (output_size == 0 && sunk == r->size) { - no_progress++; - if (no_progress > 2) { - return THEFT_TRIAL_FAIL; - } - } else { - no_progress = 0; - } - } - - heatshrink_decoder_free(hsd); - return THEFT_TRIAL_PASS; -} - -TEST decoder_should_always_make_progress(void) { - test_env env = { .limit = 1 << 15 }; - - theft_seed seed; - if (!get_time_seed(&seed)) { FAIL(); } - - struct theft *t = theft_init(0); - struct theft_cfg cfg = { - .name = __func__, - .fun = prop_decoder_should_always_make_progress, - .type_info = { &rbuf_info }, - .seed = seed, - .trials = 10000, - .env = &env, - .progress_cb = progress_cb, - }; - - theft_run_res res = theft_run(t, &cfg); - theft_free(t); - printf("\n"); - ASSERT_EQ(THEFT_RUN_PASS, res); - PASS(); -} - -SUITE(properties) { - RUN_TEST(decoder_fuzzing_should_not_detect_stuck_state); - RUN_TEST(encoded_and_decoded_data_should_match); - RUN_TEST(encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst); - RUN_TEST(encoder_should_always_make_progress); - RUN_TEST(decoder_should_always_make_progress); -} -#else -struct because_iso_c_requires_at_least_one_declaration; -#endif diff --git a/espfs/heatshrink/test_heatshrink_static.c b/espfs/heatshrink/test_heatshrink_static.c deleted file mode 100644 index e9c9754..0000000 --- a/espfs/heatshrink/test_heatshrink_static.c +++ /dev/null @@ -1,167 +0,0 @@ -#include -#include - -#include "heatshrink_encoder.h" -#include "heatshrink_decoder.h" -#include "greatest.h" - -#if HEATSHRINK_DYNAMIC_ALLOC -#error HEATSHRINK_DYNAMIC_ALLOC must be false for static allocation test suite. -#endif - -SUITE(integration); - -/* The majority of the tests are in test_heatshrink_dynamic, because that allows - * instantiating encoders/decoders with different settings at run-time. */ - -static heatshrink_encoder hse; -static heatshrink_decoder hsd; - -static void fill_with_pseudorandom_letters(uint8_t *buf, uint16_t size, uint32_t seed) { - uint64_t rn = 9223372036854775783; /* prime under 2^64 */ - for (int i=0; i 1) { - printf("\n^^ COMPRESSING\n"); - dump_buf("input", input, input_size); - } - - uint32_t sunk = 0; - uint32_t polled = 0; - while (sunk < input_size) { - ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0); - sunk += count; - if (log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse)); - } - - HSE_poll_res pres; - do { /* "turn the crank" */ - pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSER_POLL_MORE); - ASSERT_EQ(HSER_POLL_EMPTY, pres); - if (polled >= comp_sz) FAILm("compression should never expand that much"); - if (sunk == input_size) { - ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse)); - } - } - if (log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled); - uint32_t compressed_size = polled; - sunk = 0; - polled = 0; - - if (log_lvl > 1) { - printf("\n^^ DECOMPRESSING\n"); - dump_buf("comp", comp, compressed_size); - } - while (sunk < compressed_size) { - ASSERT(heatshrink_decoder_sink(&hsd, &comp[sunk], compressed_size - sunk, &count) >= 0); - sunk += count; - if (log_lvl > 1) printf("^^ sunk %zd\n", count); - if (sunk == compressed_size) { - ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(&hsd)); - } - - HSD_poll_res pres; - do { - pres = heatshrink_decoder_poll(&hsd, &decomp[polled], - decomp_sz - polled, &count); - ASSERT(pres >= 0); - polled += count; - if (log_lvl > 1) printf("^^ polled %zd\n", count); - } while (pres == HSDR_POLL_MORE); - ASSERT_EQ(HSDR_POLL_EMPTY, pres); - if (sunk == compressed_size) { - HSD_finish_res fres = heatshrink_decoder_finish(&hsd); - ASSERT_EQ(HSDR_FINISH_DONE, fres); - } - - if (polled > input_size) { - FAILm("Decompressed data is larger than original input"); - } - } - if (log_lvl > 0) printf("decompressed: %u\n", polled); - if (polled != input_size) { - FAILm("Decompressed length does not match original input length"); - } - - if (log_lvl > 1) dump_buf("decomp", decomp, polled); - for (size_t i=0; i out[%zd] == 0x%02x ('%c')\n", - j, input[j], isprint(input[j]) ? input[j] : '.', - j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.'); - } - } - } - ASSERT_EQ(input[i], decomp[i]); - } - free(comp); - free(decomp); - PASS(); -} - -TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed) { - uint8_t input[size]; - fill_with_pseudorandom_letters(input, size, seed); - return compress_and_expand_and_check(input, size, 0); -} - -SUITE(integration) { -#if __STDC_VERSION__ >= 19901L - for (uint32_t size=1; size < 64*1024; size <<= 1) { - if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size); - for (uint32_t seed=1; seed<=100; seed++) { - if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed); - RUN_TESTp(pseudorandom_data_should_match, size, seed); - } - } -#endif -} - -/* Add all the definitions that need to be in the test runner's main file. */ -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */ - printf("INPUT_BUFFER_SIZE: %u\n", HEATSHRINK_STATIC_INPUT_BUFFER_SIZE); - printf("WINDOW_BITS: %u\n", HEATSHRINK_STATIC_WINDOW_BITS); - printf("LOOKAHEAD_BITS: %u\n", HEATSHRINK_STATIC_LOOKAHEAD_BITS); - - printf("sizeof(heatshrink_encoder): %zd\n", sizeof(heatshrink_encoder)); - printf("sizeof(heatshrink_decoder): %zd\n", sizeof(heatshrink_decoder)); - RUN_SUITE(integration); - GREATEST_MAIN_END(); /* display results */ -} diff --git a/espfs/heatshrink_config_custom.h b/espfs/heatshrink_config_custom.h deleted file mode 100644 index f885f87..0000000 --- a/espfs/heatshrink_config_custom.h +++ /dev/null @@ -1,30 +0,0 @@ -//Heatshrink config for the decompressor. -#ifndef HEATSHRINK_CONFIG_H -#define HEATSHRINK_CONFIG_H - -/* Should functionality assuming dynamic allocation be used? */ -#define HEATSHRINK_DYNAMIC_ALLOC 1 - -#if HEATSHRINK_DYNAMIC_ALLOC - /* Optional replacement of malloc/free */ - #ifdef __ets__ - #define HEATSHRINK_MALLOC(SZ) os_malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) os_free(P) - #else - #define HEATSHRINK_MALLOC(SZ) malloc(SZ) - #define HEATSHRINK_FREE(P, SZ) free(P) - #endif -#else - /* Required parameters for static configuration */ - #define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32 - #define HEATSHRINK_STATIC_WINDOW_BITS 8 - #define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4 -#endif - -/* Turn on logging for debugging. */ -#define HEATSHRINK_DEBUGGING_LOGS 0 - -/* Use indexing for faster compression. (This requires additional space.) */ -#define HEATSHRINK_USE_INDEX 1 - -#endif diff --git a/espfs/heatshrink_decoder.c b/espfs/heatshrink_decoder.c deleted file mode 100644 index 9a4a9f7..0000000 --- a/espfs/heatshrink_decoder.c +++ /dev/null @@ -1,19 +0,0 @@ -#include "espfs.h" -#ifdef ESPFS_HEATSHRINK -//Stupid wrapper so we don't have to move c-files around -//Also loads httpd-specific config. - -#ifdef __ets__ -//esp build - -#include - -#define memset(x,y,z) os_memset(x,y,z) -#define memcpy(x,y,z) os_memcpy(x,y,z) -#endif - -#include "heatshrink_config_custom.h" -#include "heatshrink/heatshrink_decoder.c" - - -#endif diff --git a/espfs/mkespfsimage/Makefile b/espfs/mkespfsimage/Makefile index 26484d4..b05d5b0 100644 --- a/espfs/mkespfsimage/Makefile +++ b/espfs/mkespfsimage/Makefile @@ -1,36 +1,24 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes +GZIP_COMPRESSION?=no -TARGET = mkespfsimage.exe - -CC = gcc -LD = $(CC) -CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 -LDFLAGS=-Lmman-win32 -lmman +CFLAGS=-I.. -std=gnu99 ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -LDFLAGS += -lz +LDFLAGS+=-lz +CFLAGS+=-DESPFS_GZIP endif -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK +ifeq ($(OS),Windows_NT) +CFLAGS+=-Imman-win32 +LDFLAGS+=-Lmman-win32 -lmman +TARGET =mkespfsimage.exe +else +TARGET =mkespfsimage endif -OBJECTS = main.o heatshrink_encoder.o - -all: libmman $(TARGET) - -libmman: - $(Q) make -C mman-win32 +OBJS=main.o -$(TARGET): $(OBJECTS) - $(LD) -o $@ $^ $(LDFLAGS) - -%.o: %.c - $(CC) $(CFLAGS) -o $@ $^ +$(TARGET): $(OBJS) + $(CC) -o $@ $^ $(LDFLAGS) clean: - rm -f $(OBJECTS) $(TARGET) - -.PHONY: all clean + rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.linux b/espfs/mkespfsimage/Makefile.linux deleted file mode 100644 index f1ae1aa..0000000 --- a/espfs/mkespfsimage/Makefile.linux +++ /dev/null @@ -1,24 +0,0 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes - -CFLAGS=-I../../heatshrink -I../../include -I.. -std=gnu99 -ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -endif - -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - -OBJS=main.o heatshrink_encoder.o -TARGET=mkespfsimage - -$(TARGET): $(OBJS) -ifeq ("$(GZIP_COMPRESSION)","yes") - $(CC) -o $@ $^ -lz -else - $(CC) -o $@ $^ -endif - -clean: - rm -f $(TARGET) $(OBJS) \ No newline at end of file diff --git a/espfs/mkespfsimage/Makefile.windows b/espfs/mkespfsimage/Makefile.windows deleted file mode 100644 index 15aa8e3..0000000 --- a/espfs/mkespfsimage/Makefile.windows +++ /dev/null @@ -1,33 +0,0 @@ -GZIP_COMPRESSION ?= no -USE_HEATSHRINK ?= yes - -TARGET = mkespfsimage.exe - -CC = gcc -LD = $(CC) -CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99 -LDFLAGS=-Lmman-win32 -lmman - -ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DESPFS_GZIP -LDFLAGS += -lz -endif - -ifeq ("$(USE_HEATSHRINK)","yes") -CFLAGS += -DESPFS_HEATSHRINK -endif - -OBJECTS = main.o heatshrink_encoder.o - -all: $(TARGET) - -$(TARGET): $(OBJECTS) - $(LD) -o $@ $^ $(LDFLAGS) - -%.o: %.c - $(CC) $(CFLAGS) -o $@ $^ - -clean: - rm -f $(OBJECTS) $(TARGET) - -.PHONY: all clean diff --git a/espfs/mkespfsimage/build-linux.sh b/espfs/mkespfsimage/build-linux.sh deleted file mode 100644 index 69e3166..0000000 --- a/espfs/mkespfsimage/build-linux.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -make -f Makefile.linux clean -make -f Makefile.linux USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/build-win32.sh b/espfs/mkespfsimage/build-win32.sh deleted file mode 100644 index f552ea9..0000000 --- a/espfs/mkespfsimage/build-win32.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -cd mman-win32 -./configure && make -cd .. -make -f Makefile.windows clean -make -f Makefile.windows USE_HEATSHRINK="yes" GZIP_COMPRESSION="no" diff --git a/espfs/mkespfsimage/espfsformat.h b/espfs/mkespfsimage/espfsformat.h deleted file mode 100644 index 8ce5549..0000000 --- a/espfs/mkespfsimage/espfsformat.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef ESPROFSFORMAT_H -#define ESPROFSFORMAT_H - -/* -Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module. -Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files, -headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself -when trying to do a <4byte or unaligned read. -*/ - -/* -The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data. -Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header -with the FLAG_LASTFILE flag set. -*/ - - -#define FLAG_LASTFILE (1<<0) -#define FLAG_GZIP (1<<1) -#define COMPRESS_NONE 0 -#define COMPRESS_HEATSHRINK 1 -#define ESPFS_MAGIC 0x73665345 - -typedef struct { - int32_t magic; - int8_t flags; - int8_t compression; - int16_t nameLen; - int32_t fileLenComp; - int32_t fileLenDecomp; -} __attribute__((packed)) EspFsHeader; - -#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/heatshrink_encoder.c b/espfs/mkespfsimage/heatshrink_encoder.c deleted file mode 100644 index fd82c9a..0000000 --- a/espfs/mkespfsimage/heatshrink_encoder.c +++ /dev/null @@ -1,4 +0,0 @@ -//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir -#ifdef ESPFS_HEATSHRINK -#include "../heatshrink/heatshrink_encoder.c" -#endif \ No newline at end of file diff --git a/espfs/mkespfsimage/main.c b/espfs/mkespfsimage/main.c index 20a2b00..be3aeb7 100644 --- a/espfs/mkespfsimage/main.c +++ b/espfs/mkespfsimage/main.c @@ -8,7 +8,7 @@ #include #include "espfs.h" #ifdef __MINGW32__ -#include +#include "mman-win32/mman.h" #else #include #endif @@ -17,16 +17,8 @@ #else #include #endif -#include #include "espfsformat.h" -//Heatshrink -#ifdef ESPFS_HEATSHRINK -#include "../heatshrink/heatshrink_common.h" -#include "../heatshrink/heatshrink_config.h" -#include "../heatshrink/heatshrink_encoder.h" -#endif - //Gzip #ifdef ESPFS_GZIP // If compiler complains about missing header, try running "sudo apt-get install zlib1g-dev" @@ -35,6 +27,7 @@ #endif + //Routines to convert host format to the endianness used in the xtensa short htoxs(short in) { char r[2]; @@ -52,53 +45,6 @@ int htoxl(int in) { return *((int *)r); } -#ifdef ESPFS_HEATSHRINK -size_t compressHeatshrink(char *in, int insize, char *out, int outsize, int level) { - char *inp=in; - char *outp=out; - size_t len; - int ws[]={5, 6, 8, 11, 13}; - int ls[]={3, 3, 4, 4, 4}; - HSE_poll_res pres; - HSE_sink_res sres; - size_t r; - if (level==-1) level=8; - level=(level-1)/2; //level is now 0, 1, 2, 3, 4 - heatshrink_encoder *enc=heatshrink_encoder_alloc(ws[level], ls[level]); - if (enc==NULL) { - perror("allocating mem for heatshrink"); - exit(1); - } - //Save encoder parms as first byte - *outp=(ws[level]<<4)|ls[level]; - outp++; outsize--; - - r=1; - do { - if (insize>0) { - sres=heatshrink_encoder_sink(enc, inp, insize, &len); - if (sres!=HSER_SINK_OK) break; - inp+=len; insize-=len; - if (insize==0) heatshrink_encoder_finish(enc); - } - do { - pres=heatshrink_encoder_poll(enc, outp, outsize, &len); - if (pres!=HSER_POLL_MORE && pres!=HSER_POLL_EMPTY) break; - outp+=len; outsize-=len; - r+=len; - } while (pres==HSER_POLL_MORE); - } while (insize!=0); - - if (insize!=0) { - fprintf(stderr, "Heatshrink: Bug? insize is still %d. sres=%d pres=%d\n", insize, sres, pres); - exit(1); - } - - heatshrink_encoder_free(enc); - return r; -} -#endif - #ifdef ESPFS_GZIP size_t compressGzip(char *in, int insize, char *out, int outsize, int level) { z_stream stream; @@ -184,7 +130,7 @@ int parseGzipExtensions(char *input) { } #endif -int handleFile(int f, char *name, int compression, int level, char **compName) { +int handleFile(int f, char *name, int compression, int level, char **compName, off_t *csizePtr) { char *fdat, *cdat; off_t size, csize; EspFsHeader h; @@ -211,11 +157,6 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { if (compression==COMPRESS_NONE) { csize=size; cdat=fdat; -#ifdef ESPFS_HEATSHRINK - } else if (compression==COMPRESS_HEATSHRINK) { - cdat=malloc(size*2); - csize=compressHeatshrink(fdat, size, cdat, size*2, level); -#endif } else { fprintf(stderr, "Unknown compression - %d\n", compression); exit(1); @@ -238,7 +179,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { h.nameLen=htoxs(h.nameLen); h.fileLenComp=htoxl(csize); h.fileLenDecomp=htoxl(size); - + write(1, &h, sizeof(EspFsHeader)); write(1, name, nameLen); while (nameLen&3) { @@ -254,9 +195,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { munmap(fdat, size); if (compName != NULL) { - if (h.compression==COMPRESS_HEATSHRINK) { - *compName = "heatshrink"; - } else if (h.compression==COMPRESS_NONE) { + if (h.compression==COMPRESS_NONE) { if (h.flags & FLAG_GZIP) { *compName = "gzip"; } else { @@ -266,6 +205,7 @@ int handleFile(int f, char *name, int compression, int level, char **compName) { *compName = "unknown"; } } + *csizePtr = csize; return (csize*100)/size; } @@ -292,11 +232,7 @@ int main(int argc, char **argv) { int compType; //default compression type - heatshrink int compLvl=-1; -#ifdef ESPFS_HEATSHRINK - compType = COMPRESS_HEATSHRINK; -#else compType = COMPRESS_NONE; -#endif for (x=1; x=x-2) { @@ -318,7 +254,7 @@ int main(int argc, char **argv) { #ifdef ESPFS_GZIP if (gzipExtensions == NULL) { - parseGzipExtensions(strdup("html,css,js")); + parseGzipExtensions(strdup("html,css,js,ico")); } #endif @@ -330,11 +266,7 @@ int main(int argc, char **argv) { #endif fprintf(stderr, "> out.espfs\n"); fprintf(stderr, "Compressors:\n"); -#ifdef ESPFS_HEATSHRINK - fprintf(stderr, "0 - None\n1 - Heatshrink(default)\n"); -#else fprintf(stderr, "0 - None(default)\n"); -#endif fprintf(stderr, "\nCompression level: 1 is worst but low RAM usage, higher is better compression \nbut uses more ram on decompression. -1 = compressors default.\n"); #ifdef ESPFS_GZIP fprintf(stderr, "\nGzipped extensions: list of comma separated, case sensitive file extensions \nthat will be gzipped. Defaults to 'html,css,js'\n"); @@ -359,8 +291,9 @@ int main(int argc, char **argv) { f=open(fileName, O_RDONLY); if (f>0) { char *compName = "unknown"; - rate=handleFile(f, realName, compType, compLvl, &compName); - fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName); + off_t csize; + rate=handleFile(f, realName, compType, compLvl, &compName, &csize); + fprintf(stderr, "%-16s (%3d%%, %s, %4u bytes)\n", realName, rate, compName, (uint32_t)csize); close(f); } else { perror(fileName); diff --git a/include/esp8266.h b/include/esp8266.h index ce92455..96a7364 100644 --- a/include/esp8266.h +++ b/include/esp8266.h @@ -13,7 +13,10 @@ #include #include #include -#include #include "espmissingincludes.h" #include "uart_hw.h" + +#ifdef __WIN32__ +#include <_mingw.h> +#endif diff --git a/include/espmissingincludes.h b/include/espmissingincludes.h index c69b9fc..e5a20a0 100644 --- a/include/espmissingincludes.h +++ b/include/espmissingincludes.h @@ -1,9 +1,8 @@ #ifndef ESPMISSINGINCLUDES_H #define ESPMISSINGINCLUDES_H -#include -#include -#include +#include +#include //Missing function prototypes in include folders. Gcc will warn on these if we don't define 'em anywhere. //MOST OF THESE ARE GUESSED! but they seem to work and shut up the compiler. @@ -54,6 +53,11 @@ int rand(void); void ets_bzero(void *s, size_t n); void ets_delay_us(int ms); +// Shortcuts for memory functions +#define os_malloc pvPortMalloc +#define os_free vPortFree +#define os_zalloc pvPortZalloc + // disappeared in SDK 1.1.0: #define os_timer_done ets_timer_done #define os_timer_handler_isr ets_timer_handler_isr From 80f3788564165858042fd7c2fb70677fb83123f6 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 06:08:09 -0500 Subject: [PATCH 20/24] Added Makefile comment --- Makefile | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index af5fcbf..9047faa 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ SDK_BASE ?= $(abspath ../esp_iot_sdk_v1.3.0) # Esptool.py path and port, only used for 1-time serial flashing # Typically you'll use https://github.com/themadinventor/esptool +# Windows users use the com port i.e: ESPPORT ?= com3 ESPTOOL ?= $(abspath ../esp-open-sdk/esptool/esptool.py) ESPPORT ?= /dev/ttyUSB0 ESPBAUD ?= 460800 @@ -132,10 +133,10 @@ YUI-COMPRESSOR ?= yuicompressor-2.4.8.jar # Output directors to store intermediate compiled files # relative to the project directory BUILD_BASE = build -FW_BASE = firmware +FW_BASE = firmware # name for the target project -TARGET = httpd +TARGET = httpd # espressif tool to concatenate sections for OTA upload using bootloader v1.2+ APPGEN_TOOL ?= gen_appbin.py @@ -145,10 +146,10 @@ MODULES = espfs httpd user serial EXTRA_INCDIR = include . # libraries used in this project, mainly provided by the SDK -LIBS = c gcc hal phy pp net80211 wpa main lwip +LIBS = c gcc hal phy pp net80211 wpa main lwip # compiler flags using during compilation of source files -CFLAGS = -Os -ggdb -std=c99 -Werror -Wpointer-arith -Wundef -Wall -Wl,-EL -fno-inline-functions \ +CFLAGS = -Os -ggdb -std=c99 -Werror -Wpointer-arith -Wundef -Wall -Wl,-EL -fno-inline-functions \ -nostdlib -mlongcalls -mtext-section-literals -ffunction-sections -fdata-sections \ -D__ets__ -DICACHE_FLASH -D_STDINT_H -Wno-address -DFIRMWARE_SIZE=$(ESP_FLASH_MAX) \ -DMCU_RESET_PIN=$(MCU_RESET_PIN) -DMCU_ISP_PIN=$(MCU_ISP_PIN) \ @@ -208,11 +209,11 @@ vecho := @echo endif ifeq ("$(GZIP_COMPRESSION)","yes") -CFLAGS += -DGZIP_COMPRESSION +CFLAGS += -DGZIP_COMPRESSION endif ifeq ("$(CHANGE_TO_STA)","yes") -CFLAGS += -DCHANGE_TO_STA +CFLAGS += -DCHANGE_TO_STA endif vpath %.c $(SRC_DIR) @@ -289,9 +290,9 @@ flash: all $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): - $(Q) mkdir -p yui + $(Q) mkdir -p yui ifeq ($(OS),Windows_NT) - cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) else cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) endif From 706ae4844e80f269910054711cdd20471df6e839 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Sat, 22 Aug 2015 06:42:57 -0500 Subject: [PATCH 21/24] Weird line endings or something throwing make error, fixed now. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9047faa..7a74e19 100644 --- a/Makefile +++ b/Makefile @@ -290,9 +290,9 @@ flash: all $(ET_BLANK) $(SDK_BASE)/bin/blank.bin yui/$(YUI-COMPRESSOR): - $(Q) mkdir -p yui + $(Q) mkdir -p yui ifeq ($(OS),Windows_NT) - cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) + cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR) else cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) endif From 7490e45aab45854e675830b1da22635343e773e4 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Sat, 22 Aug 2015 10:29:51 -0700 Subject: [PATCH 22/24] fix indentation to use spaces --- serial/console.c | 160 ++++++++--------- serial/serbridge.h | 20 +-- serial/tcpclient.c | 422 ++++++++++++++++++++++----------------------- 3 files changed, 301 insertions(+), 301 deletions(-) diff --git a/serial/console.c b/serial/console.c index 6fa4374..d0d0dd7 100644 --- a/serial/console.c +++ b/serial/console.c @@ -25,112 +25,112 @@ static int console_pos; // offset since reset of buffer static void ICACHE_FLASH_ATTR console_write(char c) { - console_buf[console_wr] = c; - console_wr = (console_wr+1) % BUF_MAX; - if (console_wr == console_rd) { - // full, we write anyway and loose the oldest char - console_rd = (console_rd+1) % BUF_MAX; // full, eat first char - console_pos++; - } + console_buf[console_wr] = c; + console_wr = (console_wr+1) % BUF_MAX; + if (console_wr == console_rd) { + // full, we write anyway and loose the oldest char + console_rd = (console_rd+1) % BUF_MAX; // full, eat first char + console_pos++; + } } #if 0 // return previous character in console, 0 if at start static char ICACHE_FLASH_ATTR console_prev(void) { - if (console_wr == console_rd) return 0; - return console_buf[(console_wr-1+BUF_MAX)%BUF_MAX]; + if (console_wr == console_rd) return 0; + return console_buf[(console_wr-1+BUF_MAX)%BUF_MAX]; } #endif void ICACHE_FLASH_ATTR console_write_char(char c) { - //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good - console_write(c); + //if (c == '\n' && console_prev() != '\r') console_write('\r'); // does more harm than good + console_write(c); } int ICACHE_FLASH_ATTR ajaxConsoleReset(HttpdConnData *connData) { - if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. - jsonHeader(connData, 200); - console_rd = console_wr = console_pos = 0; - serbridgeReset(); - return HTTPD_CGI_DONE; + if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. + jsonHeader(connData, 200); + console_rd = console_wr = console_pos = 0; + serbridgeReset(); + return HTTPD_CGI_DONE; } int ICACHE_FLASH_ATTR ajaxConsoleBaud(HttpdConnData *connData) { - if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. - char buff[512]; - int len, status = 400; - len = httpdFindArg(connData->getArgs, "rate", buff, sizeof(buff)); - if (len > 0) { - int rate = atoi(buff); - if (rate >= 9600 && rate <= 1000000) { - uart0_baud(rate); - flashConfig.baud_rate = rate; - status = configSave() ? 200 : 400; - } - } else if (connData->requestType == HTTPD_METHOD_GET) { - status = 200; - } - - jsonHeader(connData, status); - os_sprintf(buff, "{\"rate\": %ld}", flashConfig.baud_rate); - httpdSend(connData, buff, -1); - return HTTPD_CGI_DONE; + if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. + char buff[512]; + int len, status = 400; + len = httpdFindArg(connData->getArgs, "rate", buff, sizeof(buff)); + if (len > 0) { + int rate = atoi(buff); + if (rate >= 9600 && rate <= 1000000) { + uart0_baud(rate); + flashConfig.baud_rate = rate; + status = configSave() ? 200 : 400; + } + } else if (connData->requestType == HTTPD_METHOD_GET) { + status = 200; + } + + jsonHeader(connData, status); + os_sprintf(buff, "{\"rate\": %ld}", flashConfig.baud_rate); + httpdSend(connData, buff, -1); + return HTTPD_CGI_DONE; } int ICACHE_FLASH_ATTR ajaxConsole(HttpdConnData *connData) { - if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. - char buff[2048]; - int len; // length of text in buff - int console_len = (console_wr+BUF_MAX-console_rd) % BUF_MAX; // num chars in console_buf - int start = 0; // offset onto console_wr to start sending out chars - - jsonHeader(connData, 200); - - // figure out where to start in buffer based on URI param - len = httpdFindArg(connData->getArgs, "start", buff, sizeof(buff)); - if (len > 0) { - start = atoi(buff); - if (start < console_pos) { - start = 0; - } else if (start >= console_pos+console_len) { - start = console_len; - } else { - start = start - console_pos; - } - } - - // start outputting - len = os_sprintf(buff, "{\"len\":%d, \"start\":%d, \"text\": \"", - console_len-start, console_pos+start); - - int rd = (console_rd+start) % BUF_MAX; - while (len < 2040 && rd != console_wr) { - uint8_t c = console_buf[rd]; - if (c == '\\' || c == '"') { - buff[len++] = '\\'; - buff[len++] = c; - } else if (c == '\r') { - // this is crummy, but browsers display a newline for \r\n sequences - } else if (c < ' ') { - len += os_sprintf(buff+len, "\\u%04x", c); - } else { - buff[len++] = c; - } - rd = (rd + 1) % BUF_MAX; - } - os_strcpy(buff+len, "\"}"); len+=2; - httpdSend(connData, buff, len); - return HTTPD_CGI_DONE; + if (connData->conn==NULL) return HTTPD_CGI_DONE; // Connection aborted. Clean up. + char buff[2048]; + int len; // length of text in buff + int console_len = (console_wr+BUF_MAX-console_rd) % BUF_MAX; // num chars in console_buf + int start = 0; // offset onto console_wr to start sending out chars + + jsonHeader(connData, 200); + + // figure out where to start in buffer based on URI param + len = httpdFindArg(connData->getArgs, "start", buff, sizeof(buff)); + if (len > 0) { + start = atoi(buff); + if (start < console_pos) { + start = 0; + } else if (start >= console_pos+console_len) { + start = console_len; + } else { + start = start - console_pos; + } + } + + // start outputting + len = os_sprintf(buff, "{\"len\":%d, \"start\":%d, \"text\": \"", + console_len-start, console_pos+start); + + int rd = (console_rd+start) % BUF_MAX; + while (len < 2040 && rd != console_wr) { + uint8_t c = console_buf[rd]; + if (c == '\\' || c == '"') { + buff[len++] = '\\'; + buff[len++] = c; + } else if (c == '\r') { + // this is crummy, but browsers display a newline for \r\n sequences + } else if (c < ' ') { + len += os_sprintf(buff+len, "\\u%04x", c); + } else { + buff[len++] = c; + } + rd = (rd + 1) % BUF_MAX; + } + os_strcpy(buff+len, "\"}"); len+=2; + httpdSend(connData, buff, len); + return HTTPD_CGI_DONE; } void ICACHE_FLASH_ATTR consoleInit() { - console_wr = 0; - console_rd = 0; + console_wr = 0; + console_rd = 0; } diff --git a/serial/serbridge.h b/serial/serbridge.h index fea8602..33c96a2 100644 --- a/serial/serbridge.h +++ b/serial/serbridge.h @@ -12,21 +12,21 @@ #define MAX_TXBUFFER 1024 enum connModes { - cmInit = 0, // initialization mode: nothing received yet - cmTransparent, // transparent mode - cmAVR, // Arduino/AVR programming mode - cmARM, // ARM (LPC8xx) programming - cmEcho, // simply echo characters (used for debugging latency) + cmInit = 0, // initialization mode: nothing received yet + cmTransparent, // transparent mode + cmAVR, // Arduino/AVR programming mode + cmARM, // ARM (LPC8xx) programming + cmEcho, // simply echo characters (used for debugging latency) cmTelnet, // use telnet escape sequences for programming mode cmTcpClient, // client connection (initiated via serial) }; typedef struct serbridgeConnData { - struct espconn *conn; - enum connModes conn_mode; // connection mode - char *txbuffer; // buffer for the data to send - uint16 txbufferlen; // length of data in txbuffer - bool readytosend; // true, if txbuffer can send by espconn_sent + struct espconn *conn; + enum connModes conn_mode; // connection mode + char *txbuffer; // buffer for the data to send + uint16 txbufferlen; // length of data in txbuffer + bool readytosend; // true, if txbuffer can send by espconn_sent uint8_t telnet_state; } serbridgeConnData; diff --git a/serial/tcpclient.c b/serial/tcpclient.c index 166f533..fa1b458 100644 --- a/serial/tcpclient.c +++ b/serial/tcpclient.c @@ -16,20 +16,20 @@ #define MAX_TXBUF 1024 enum TcpState { - TCP_idle, // unused connection - TCP_dns, // doing gethostbyname - TCP_conn, // connecting to remote server - TCP_data, // connected + TCP_idle, // unused connection + TCP_dns, // doing gethostbyname + TCP_conn, // connecting to remote server + TCP_data, // connected }; // Connections typedef struct { - struct espconn *conn; // esp connection structure - esp_tcp *tcp; // esp TCP parameters - char *txBuf; // buffer to accumulate into - char *txBufSent; // buffer held by espconn - uint8_t txBufLen; // number of chars in txbuf - enum TcpState state; + struct espconn *conn; // esp connection structure + esp_tcp *tcp; // esp TCP parameters + char *txBuf; // buffer to accumulate into + char *txBufSent; // buffer held by espconn + uint8_t txBufLen; // number of chars in txbuf + enum TcpState state; } TcpConn; static TcpConn tcpConn[MAX_CHAN]; @@ -49,46 +49,46 @@ static void tcpRecvCb(void *arg, char *data, uint16_t len); // Allocate a new connection dynamically and return it. Returns NULL if buf alloc failed static TcpConn* ICACHE_FLASH_ATTR tcpConnAlloc(uint8_t chan) { - TcpConn *tci = tcpConn+chan; - if (tci->state != TCP_idle && tci->conn != NULL) return tci; - - // malloc and return espconn struct - tci->conn = os_malloc(sizeof(struct espconn)); - if (tci->conn == NULL) goto fail; - memset(tci->conn, 0, sizeof(struct espconn)); - // malloc esp_tcp struct - tci->tcp = os_malloc(sizeof(esp_tcp)); - if (tci->tcp == NULL) goto fail; - memset(tci->tcp, 0, sizeof(esp_tcp)); - - // common init - tci->state = TCP_dns; - tci->conn->type = ESPCONN_TCP; - tci->conn->state = ESPCONN_NONE; - tci->conn->proto.tcp = tci->tcp; - tci->tcp->remote_port = 80; - espconn_regist_connectcb(tci->conn, tcpConnectCb); - espconn_regist_reconcb(tci->conn, tcpResetCb); - espconn_regist_sentcb(tci->conn, tcpSentCb); - espconn_regist_recvcb(tci->conn, tcpRecvCb); - espconn_regist_disconcb(tci->conn, tcpDisconCb); - tci->conn->reverse = tci; - - return tci; + TcpConn *tci = tcpConn+chan; + if (tci->state != TCP_idle && tci->conn != NULL) return tci; + + // malloc and return espconn struct + tci->conn = os_malloc(sizeof(struct espconn)); + if (tci->conn == NULL) goto fail; + memset(tci->conn, 0, sizeof(struct espconn)); + // malloc esp_tcp struct + tci->tcp = os_malloc(sizeof(esp_tcp)); + if (tci->tcp == NULL) goto fail; + memset(tci->tcp, 0, sizeof(esp_tcp)); + + // common init + tci->state = TCP_dns; + tci->conn->type = ESPCONN_TCP; + tci->conn->state = ESPCONN_NONE; + tci->conn->proto.tcp = tci->tcp; + tci->tcp->remote_port = 80; + espconn_regist_connectcb(tci->conn, tcpConnectCb); + espconn_regist_reconcb(tci->conn, tcpResetCb); + espconn_regist_sentcb(tci->conn, tcpSentCb); + espconn_regist_recvcb(tci->conn, tcpRecvCb); + espconn_regist_disconcb(tci->conn, tcpDisconCb); + tci->conn->reverse = tci; + + return tci; fail: - tcpConnFree(tci); - return NULL; + tcpConnFree(tci); + return NULL; } // Free a connection dynamically. static void ICACHE_FLASH_ATTR tcpConnFree(TcpConn* tci) { - if (tci->conn != NULL) os_free(tci->conn); - if (tci->tcp != NULL) os_free(tci->tcp); - if (tci->txBuf != NULL) os_free(tci->txBuf); - if (tci->txBufSent != NULL) os_free(tci->txBufSent); - memset(tci, 0, sizeof(TcpConn)); + if (tci->conn != NULL) os_free(tci->conn); + if (tci->tcp != NULL) os_free(tci->tcp); + if (tci->txBuf != NULL) os_free(tci->txBuf); + if (tci->txBufSent != NULL) os_free(tci->txBufSent); + memset(tci, 0, sizeof(TcpConn)); } //===== DNS @@ -96,26 +96,26 @@ tcpConnFree(TcpConn* tci) { // DNS name resolution callback static void ICACHE_FLASH_ATTR tcpClientHostnameCb(const char *name, ip_addr_t *ipaddr, void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP dns CB (%p %p)\n", arg, tci); - if (ipaddr == NULL) { - os_printf("TCP %s not found\n", name); - } else { - os_printf("TCP %s -> %d.%d.%d.%d\n", name, IP2STR(ipaddr)); - tci->tcp->remote_ip[0] = ip4_addr1(ipaddr); - tci->tcp->remote_ip[1] = ip4_addr2(ipaddr); - tci->tcp->remote_ip[2] = ip4_addr3(ipaddr); - tci->tcp->remote_ip[3] = ip4_addr4(ipaddr); - os_printf("TCP connect %d.%d.%d.%d (%p)\n", IP2STR(tci->tcp->remote_ip), tci); - if (espconn_connect(tci->conn) == ESPCONN_OK) { - tci->state = TCP_conn; - return; - } - os_printf("TCP connect failure\n"); - } - // oops - tcpConnFree(tci); + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP dns CB (%p %p)\n", arg, tci); + if (ipaddr == NULL) { + os_printf("TCP %s not found\n", name); + } else { + os_printf("TCP %s -> %d.%d.%d.%d\n", name, IP2STR(ipaddr)); + tci->tcp->remote_ip[0] = ip4_addr1(ipaddr); + tci->tcp->remote_ip[1] = ip4_addr2(ipaddr); + tci->tcp->remote_ip[2] = ip4_addr3(ipaddr); + tci->tcp->remote_ip[3] = ip4_addr4(ipaddr); + os_printf("TCP connect %d.%d.%d.%d (%p)\n", IP2STR(tci->tcp->remote_ip), tci); + if (espconn_connect(tci->conn) == ESPCONN_OK) { + tci->state = TCP_conn; + return; + } + os_printf("TCP connect failure\n"); + } + // oops + tcpConnFree(tci); } //===== Connect / disconnect @@ -123,42 +123,42 @@ tcpClientHostnameCb(const char *name, ip_addr_t *ipaddr, void *arg) { // Connected callback static void ICACHE_FLASH_ATTR tcpConnectCb(void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP connect CB (%p %p)\n", arg, tci); - tci->state = TCP_data; - // send any buffered data - if (tci->txBuf != NULL && tci->txBufLen > 0) tcpDoSend(tci); - // reply to serial - char buf[6]; - short l = os_sprintf(buf, "\n~@%dC\n", tci-tcpConn); - uart0_tx_buffer(buf, l); + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP connect CB (%p %p)\n", arg, tci); + tci->state = TCP_data; + // send any buffered data + if (tci->txBuf != NULL && tci->txBufLen > 0) tcpDoSend(tci); + // reply to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dC\n", tci-tcpConn); + uart0_tx_buffer(buf, l); } // Disconnect callback static void ICACHE_FLASH_ATTR tcpDisconCb(void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP disconnect CB (%p %p)\n", arg, tci); - // notify to serial - char buf[6]; - short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); - uart0_tx_buffer(buf, l); - // free - tcpConnFree(tci); + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP disconnect CB (%p %p)\n", arg, tci); + // notify to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); + uart0_tx_buffer(buf, l); + // free + tcpConnFree(tci); } // Connection reset callback static void ICACHE_FLASH_ATTR tcpResetCb(void *arg, sint8 err) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP reset CB (%p %p) err=%d\n", arg, tci, err); - // notify to serial - char buf[6]; - short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); - uart0_tx_buffer(buf, l); - // free - tcpConnFree(tci); + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP reset CB (%p %p) err=%d\n", arg, tci, err); + // notify to serial + char buf[6]; + short l = os_sprintf(buf, "\n~@%dZ\n", tci-tcpConn); + uart0_tx_buffer(buf, l); + // free + tcpConnFree(tci); } //===== Sending and receiving @@ -166,89 +166,89 @@ static void ICACHE_FLASH_ATTR tcpResetCb(void *arg, sint8 err) { // Send the next buffer (assumes that the connection is in a state that allows it) static void ICACHE_FLASH_ATTR tcpDoSend(TcpConn *tci) { - sint8 err = espconn_sent(tci->conn, (uint8*)tci->txBuf, tci->txBufLen); - if (err == ESPCONN_OK) { - // send successful - os_printf("TCP sent (%p %p)\n", tci->conn, tci); - tci->txBuf[tci->txBufLen] = 0; os_printf("TCP data: %s\n", tci->txBuf); - tci->txBufSent = tci->txBuf; - tci->txBuf = NULL; - tci->txBufLen = 0; - } else { - // send error, leave as-is and try again later... - os_printf("TCP send err (%p %p) %d\n", tci->conn, tci, err); - } + sint8 err = espconn_sent(tci->conn, (uint8*)tci->txBuf, tci->txBufLen); + if (err == ESPCONN_OK) { + // send successful + os_printf("TCP sent (%p %p)\n", tci->conn, tci); + tci->txBuf[tci->txBufLen] = 0; os_printf("TCP data: %s\n", tci->txBuf); + tci->txBufSent = tci->txBuf; + tci->txBuf = NULL; + tci->txBufLen = 0; + } else { + // send error, leave as-is and try again later... + os_printf("TCP send err (%p %p) %d\n", tci->conn, tci, err); + } } // Sent callback static void ICACHE_FLASH_ATTR tcpSentCb(void *arg) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP sent CB (%p %p)\n", arg, tci); - if (tci->txBufSent != NULL) os_free(tci->txBufSent); - tci->txBufSent = NULL; - - if (tci->txBuf != NULL && tci->txBufLen == MAX_TXBUF) { - // next buffer is full, send it now - tcpDoSend(tci); - } + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP sent CB (%p %p)\n", arg, tci); + if (tci->txBufSent != NULL) os_free(tci->txBufSent); + tci->txBufSent = NULL; + + if (tci->txBuf != NULL && tci->txBufLen == MAX_TXBUF) { + // next buffer is full, send it now + tcpDoSend(tci); + } } // Recv callback static void ICACHE_FLASH_ATTR tcpRecvCb(void *arg, char *data, uint16_t len) { - struct espconn *conn = arg; - TcpConn *tci = conn->reverse; - os_printf("TCP recv CB (%p %p)\n", arg, tci); - if (tci->state == TCP_data) { - uint8_t chan; - for (chan=0; chan= MAX_CHAN) return; // oops!? - char buf[6]; - short l = os_sprintf(buf, "\n~%d", chan); - uart0_tx_buffer(buf, l); - uart0_tx_buffer(data, len); - uart0_tx_buffer("\0\n", 2); - } - serledFlash(50); // short blink on serial LED + struct espconn *conn = arg; + TcpConn *tci = conn->reverse; + os_printf("TCP recv CB (%p %p)\n", arg, tci); + if (tci->state == TCP_data) { + uint8_t chan; + for (chan=0; chan= MAX_CHAN) return; // oops!? + char buf[6]; + short l = os_sprintf(buf, "\n~%d", chan); + uart0_tx_buffer(buf, l); + uart0_tx_buffer(data, len); + uart0_tx_buffer("\0\n", 2); + } + serledFlash(50); // short blink on serial LED } void ICACHE_FLASH_ATTR tcpClientSendChar(uint8_t chan, char c) { - TcpConn *tci = tcpConn+chan; - if (tci->state == TCP_idle) return; - - if (tci->txBuf != NULL) { - // we have a buffer - if (tci->txBufLen < MAX_TXBUF) { - // buffer has space, add char and return - tci->txBuf[tci->txBufLen++] = c; - return; - } else if (tci->txBufSent == NULL) { - // we don't have a send pending, send full buffer off - if (tci->state == TCP_data) tcpDoSend(tci); - if (tci->txBuf != NULL) return; // something went wrong - } else { - // buffers all backed-up, drop char - return; - } - } - // we do not have a buffer (either didn't have one or sent it off) - // allocate one - tci->txBuf = os_malloc(MAX_TXBUF); - tci->txBufLen = 0; - if (tci->txBuf != NULL) { - tci->txBuf[tci->txBufLen++] = c; - } + TcpConn *tci = tcpConn+chan; + if (tci->state == TCP_idle) return; + + if (tci->txBuf != NULL) { + // we have a buffer + if (tci->txBufLen < MAX_TXBUF) { + // buffer has space, add char and return + tci->txBuf[tci->txBufLen++] = c; + return; + } else if (tci->txBufSent == NULL) { + // we don't have a send pending, send full buffer off + if (tci->state == TCP_data) tcpDoSend(tci); + if (tci->txBuf != NULL) return; // something went wrong + } else { + // buffers all backed-up, drop char + return; + } + } + // we do not have a buffer (either didn't have one or sent it off) + // allocate one + tci->txBuf = os_malloc(MAX_TXBUF); + tci->txBufLen = 0; + if (tci->txBuf != NULL) { + tci->txBuf[tci->txBufLen++] = c; + } } void ICACHE_FLASH_ATTR tcpClientSendPush(uint8_t chan) { - TcpConn *tci = tcpConn+chan; - if (tci->state != TCP_data) return; // no active connection on this channel - if (tci->txBuf == NULL || tci->txBufLen == 0) return; // no chars accumulated to send - if (tci->txBufSent != NULL) return; // already got a send in progress - tcpDoSend(tci); + TcpConn *tci = tcpConn+chan; + if (tci->state != TCP_data) return; // no active connection on this channel + if (tci->txBuf == NULL || tci->txBufLen == 0) return; // no chars accumulated to send + if (tci->txBufSent != NULL) return; // already got a send in progress + tcpDoSend(tci); } //===== Command parsing @@ -257,59 +257,59 @@ tcpClientSendPush(uint8_t chan) { // Returns true on success. bool ICACHE_FLASH_ATTR tcpClientCommand(uint8_t chan, char cmd, char *cmdBuf) { - TcpConn *tci; - char *hostname; - char *port; - - // copy the command so we can modify it - char buf[128]; - os_strncpy(buf, cmdBuf, 128); - buf[127] = 0; - - switch (cmd) { - //== TCP Connect command - case 'T': - hostname = buf; - port = hostname; - while (*port != 0 && *port != ':') port++; - if (*port != ':') break; - *port = 0; - port++; - int portInt = atoi(port); - if (portInt < 1 || portInt > 65535) break; - - // allocate a connection - tci = tcpConnAlloc(chan); - if (tci == NULL) break; - tci->state = TCP_dns; - tci->tcp->remote_port = portInt; - - // start the DNS resolution - os_printf("TCP %p resolving %s for chan %d (conn=%p)\n", tci, hostname, chan ,tci->conn); - ip_addr_t ip; - err_t err = espconn_gethostbyname(tci->conn, hostname, &ip, tcpClientHostnameCb); - if (err == ESPCONN_OK) { - // dns cache hit, got the IP address, fake the callback (sigh) - os_printf("TCP DNS hit\n"); - tcpClientHostnameCb(hostname, &ip, tci->conn); - } else if (err != ESPCONN_INPROGRESS) { - tcpConnFree(tci); - break; - } - - return true; - - //== TCP Close/disconnect command - case 'C': - os_printf("TCP closing chan %d\n", chan); - tci = tcpConn+chan; - if (tci->state > TCP_idle) { - tci->state = TCP_idle; // hackish... - espconn_disconnect(tci->conn); - } - break; - - } - return false; + TcpConn *tci; + char *hostname; + char *port; + + // copy the command so we can modify it + char buf[128]; + os_strncpy(buf, cmdBuf, 128); + buf[127] = 0; + + switch (cmd) { + //== TCP Connect command + case 'T': + hostname = buf; + port = hostname; + while (*port != 0 && *port != ':') port++; + if (*port != ':') break; + *port = 0; + port++; + int portInt = atoi(port); + if (portInt < 1 || portInt > 65535) break; + + // allocate a connection + tci = tcpConnAlloc(chan); + if (tci == NULL) break; + tci->state = TCP_dns; + tci->tcp->remote_port = portInt; + + // start the DNS resolution + os_printf("TCP %p resolving %s for chan %d (conn=%p)\n", tci, hostname, chan ,tci->conn); + ip_addr_t ip; + err_t err = espconn_gethostbyname(tci->conn, hostname, &ip, tcpClientHostnameCb); + if (err == ESPCONN_OK) { + // dns cache hit, got the IP address, fake the callback (sigh) + os_printf("TCP DNS hit\n"); + tcpClientHostnameCb(hostname, &ip, tci->conn); + } else if (err != ESPCONN_INPROGRESS) { + tcpConnFree(tci); + break; + } + + return true; + + //== TCP Close/disconnect command + case 'C': + os_printf("TCP closing chan %d\n", chan); + tci = tcpConn+chan; + if (tci->state > TCP_idle) { + tci->state = TCP_idle; // hackish... + espconn_disconnect(tci->conn); + } + break; + + } + return false; } From ddd2af1a65b02d2a9352a664635f6f2825f0f7d4 Mon Sep 17 00:00:00 2001 From: Thorsten von Eicken Date: Mon, 24 Aug 2015 10:25:24 -0700 Subject: [PATCH 23/24] incorporated tuanpmt's slip protocol and rest library --- Makefile | 2 +- cmd/cmd.c | 179 ++++++++++ cmd/cmd.h | 90 +++++ cmd/handlers.c | 79 +++++ cmd/rest.c | 394 +++++++++++++++++++++ cmd/rest.h | 39 +++ {serial => cmd}/tcpclient.c | 0 {serial => cmd}/tcpclient.h | 2 +- serial/crc16.c | 78 +++++ serial/crc16.h | 100 ++++++ serial/serbridge.c | 670 +++++++++++++++--------------------- serial/serbridge.h | 2 +- serial/slip.c | 136 ++++++++ serial/uart.h | 2 +- user/cgiwifi.c | 5 +- user/cgiwifi.h | 2 + 16 files changed, 1382 insertions(+), 398 deletions(-) create mode 100644 cmd/cmd.c create mode 100644 cmd/cmd.h create mode 100644 cmd/handlers.c create mode 100644 cmd/rest.c create mode 100644 cmd/rest.h rename {serial => cmd}/tcpclient.c (100%) rename {serial => cmd}/tcpclient.h (86%) create mode 100644 serial/crc16.c create mode 100644 serial/crc16.h create mode 100644 serial/slip.c diff --git a/Makefile b/Makefile index 5b694ae..647ba20 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ TARGET = httpd APPGEN_TOOL ?= gen_appbin.py # which modules (subdirectories) of the project to include in compiling -MODULES = espfs httpd user serial +MODULES = espfs httpd user serial cmd EXTRA_INCDIR = include . # lib/heatshrink/ # libraries used in this project, mainly provided by the SDK diff --git a/cmd/cmd.c b/cmd/cmd.c new file mode 100644 index 0000000..06430ca --- /dev/null +++ b/cmd/cmd.c @@ -0,0 +1,179 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt +// +// Adapted from: github.com/tuanpmt/esp_bridge, Created on: Jan 9, 2015, Author: Minh + +#include "esp8266.h" +#include "cmd.h" +#include "crc16.h" +#include "serbridge.h" +#include "uart.h" + +extern const CmdList commands[]; + +//===== ESP -> Serial responses + +static void ICACHE_FLASH_ATTR +CMD_ProtoWrite(uint8_t data) { + switch(data){ + case SLIP_START: + case SLIP_END: + case SLIP_REPL: + uart0_write_char(SLIP_REPL); + uart0_write_char(SLIP_ESC(data)); + break; + default: + uart0_write_char(data); + } +} + +static void ICACHE_FLASH_ATTR +CMD_ProtoWriteBuf(uint8_t *data, short len) { + while (len--) CMD_ProtoWrite(*data++); +} + +// Start a response, returns the partial CRC +uint16_t ICACHE_FLASH_ATTR +CMD_ResponseStart(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc) { + uint16_t crc = 0; + + uart0_write_char(SLIP_START); + CMD_ProtoWriteBuf((uint8_t*)&cmd, 2); + crc = crc16_data((uint8_t*)&cmd, 2, crc); + CMD_ProtoWriteBuf((uint8_t*)&callback, 4); + crc = crc16_data((uint8_t*)&callback, 4, crc); + CMD_ProtoWriteBuf((uint8_t*)&_return, 4); + crc = crc16_data((uint8_t*)&_return, 4, crc); + CMD_ProtoWriteBuf((uint8_t*)&argc, 2); + crc = crc16_data((uint8_t*)&argc, 2, crc); + return crc; +} + +// Adds data to a response, returns the partial CRC +uint16_t ICACHE_FLASH_ATTR +CMD_ResponseBody(uint16_t crc_in, uint8_t* data, short len) { + short pad_len = len+3 - (len+3)%4; // round up to multiple of 4 + CMD_ProtoWriteBuf((uint8_t*)&pad_len, 2); + crc_in = crc16_data((uint8_t*)&pad_len, 2, crc_in); + + CMD_ProtoWriteBuf(data, len); + crc_in = crc16_data(data, len, crc_in); + + if (pad_len > len) { + uint32_t temp = 0; + CMD_ProtoWriteBuf((uint8_t*)&temp, pad_len-len); + crc_in = crc16_data((uint8_t*)&temp, pad_len-len, crc_in); + } + + return crc_in; +} + +// Ends a response +void ICACHE_FLASH_ATTR +CMD_ResponseEnd(uint16_t crc) { + CMD_ProtoWriteBuf((uint8_t*)&crc, 2); + uart0_write_char(SLIP_END); +} + +//===== serial -> ESP commands + +// Execute a parsed command +static uint32_t ICACHE_FLASH_ATTR +CMD_Exec(const CmdList *scp, CmdPacket *packet) { + uint16_t crc = 0; + // Iterate through the command table and call the appropriate function + while (scp->sc_function != NULL) { + if(scp->sc_name == packet->cmd) { + //os_printf("CMD: Dispatching cmd=%d\n", packet->cmd); + // call command function + uint32_t ret = scp->sc_function(packet); + // if requestor asked for a response, send it + if (packet->_return){ + os_printf("CMD: Response: %lu, cmd: %d\r\n", ret, packet->cmd); + crc = CMD_ResponseStart(packet->cmd, 0, ret, 0); + CMD_ResponseEnd(crc); + } else { + //os_printf("CMD: no response (%lu)\n", packet->_return); + } + return ret; + } + scp++; + } + os_printf("CMD: cmd=%d not found\n", packet->cmd); + return 0; +} + +// Parse a packet and print info about it +void ICACHE_FLASH_ATTR +CMD_parse_packet(uint8_t *buf, short len) { + // minimum command length + if (len < 12) return; + + // init pointers into buffer + CmdPacket *packet = (CmdPacket*)buf; + uint8_t *data_ptr = (uint8_t*)&packet->args; + uint8_t *data_limit = data_ptr+len; + uint16_t argc = packet->argc; + uint16_t argn = 0; + + os_printf("CMD: cmd=%d argc=%d cb=%p ret=%lu\n", + packet->cmd, packet->argc, (void *)packet->callback, packet->_return); + + // print out arguments + while (data_ptr+2 < data_limit && argc--) { + short l = *(uint16_t*)data_ptr; + os_printf("CMD: arg[%d] len=%d:", argn++, l); + data_ptr += 2; + while (data_ptr < data_limit && l--) { + os_printf(" %02X", *data_ptr++); + } + os_printf("\n"); + } + + if (data_ptr <= data_limit) { + CMD_Exec(commands, packet); + } else { + os_printf("CMD: packet length overrun, parsing arg %d\n", argn-1); + } +} + +//===== Helpers to parse a command packet + +// Fill out a CmdRequest struct given a CmdPacket +void ICACHE_FLASH_ATTR +CMD_Request(CmdRequest *req, CmdPacket* cmd) { + req->cmd = cmd; + req->arg_num = 0; + req->arg_ptr = (uint8_t*)&cmd->args; +} + +// Return the number of arguments given a command struct +uint32_t ICACHE_FLASH_ATTR +CMD_GetArgc(CmdRequest *req) { + return req->cmd->argc; +} + +// Copy the next argument from a command structure into the data pointer, returns 0 on success +// -1 on error +int32_t ICACHE_FLASH_ATTR +CMD_PopArg(CmdRequest *req, void *data, uint16_t len) { + uint16_t length; + + if (req->arg_num >= req->cmd->argc) + return -1; + + length = *(uint16_t*)req->arg_ptr; + if (length != len) return -1; // safety check + + req->arg_ptr += 2; + os_memcpy(data, req->arg_ptr, length); + req->arg_ptr += length; + + req->arg_num ++; + return 0; +} + +// Return the length of the next argument +uint16_t ICACHE_FLASH_ATTR +CMD_ArgLen(CmdRequest *req) { + return *(uint16_t*)req->arg_ptr; +} diff --git a/cmd/cmd.h b/cmd/cmd.h new file mode 100644 index 0000000..5a66bc7 --- /dev/null +++ b/cmd/cmd.h @@ -0,0 +1,90 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt +// +// Adapted from: github.com/tuanpmt/esp_bridge, Created on: Jan 9, 2015, Author: Minh + +#ifndef CMD_H +#define CMD_H + +// Escape chars used by tuanpmt, dunno why he didn't use std ones... +#define SLIP_START 0x7E +#define SLIP_END 0x7F +#define SLIP_REPL 0x7D +#define SLIP_ESC(x) (x ^ 0x20) + +#if 0 +// Proper SLIP escape chars from RFC +#define SLIP_END 0300 // indicates end of packet +#define SLIP_ESC 0333 // indicates byte stuffing +#define SLIP_ESC_END 0334 // ESC ESC_END means END data byte +#define SLIP_ESC_ESC 0335 // ESC ESC_ESC means ESC data byte +#endif + +typedef struct __attribute((__packed__)) { + uint16_t len; // length of data + uint8_t data[0]; // really data[len] +} CmdArg; + +typedef struct __attribute((__packed__)) { + uint16_t cmd; // command to perform, from CmdName enum + uint32_t callback; // callback pointer to embed in response + uint32_t _return; // return value to embed in response (?) + uint16_t argc; // number of arguments to command + CmdArg args[0]; // really args[argc] +} CmdPacket; + +typedef struct { + CmdPacket *cmd; // command packet header + uint32_t arg_num; // number of args parsed + uint8_t *arg_ptr; // pointer to ?? +} CmdRequest; + +typedef enum { + CMD_NULL = 0, + CMD_RESET, // reset esp (not honored in this implementation) + CMD_IS_READY, // health-check + CMD_WIFI_CONNECT, // (3) connect to AP (not honored in this implementation) + CMD_MQTT_SETUP, + CMD_MQTT_CONNECT, + CMD_MQTT_DISCONNECT, + CMD_MQTT_PUBLISH, + CMD_MQTT_SUBSCRIBE, + CMD_MQTT_LWT, + CMD_MQTT_EVENTS, + CMD_REST_SETUP, // (11) + CMD_REST_REQUEST, + CMD_REST_SETHEADER, + CMD_REST_EVENTS +} CmdName; + +typedef uint32_t (*cmdfunc_t)(CmdPacket *cmd); + +typedef struct { + CmdName sc_name; + cmdfunc_t sc_function; +} CmdList; + +void CMD_parse_packet(uint8_t *buf, short len); + +// Responses + +// Start a response, returns the partial CRC +uint16_t CMD_ResponseStart(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc); +// Adds data to a response, returns the partial CRC +uint16_t CMD_ResponseBody(uint16_t crc_in, uint8_t* data, short len); +// Ends a response +void CMD_ResponseEnd(uint16_t crc); + +void CMD_Response(uint16_t cmd, uint32_t callback, uint32_t _return, uint16_t argc, CmdArg* args[]); + +// Requests + +// Fill out a CmdRequest struct given a CmdPacket +void CMD_Request(CmdRequest *req, CmdPacket* cmd); +// Return the number of arguments given a request +uint32_t CMD_GetArgc(CmdRequest *req); +// Return the length of the next argument +uint16_t CMD_ArgLen(CmdRequest *req); +// Copy next arg from request into the data pointer, returns 0 on success, -1 on error +int32_t CMD_PopArg(CmdRequest *req, void *data, uint16_t len); + +#endif diff --git a/cmd/handlers.c b/cmd/handlers.c new file mode 100644 index 0000000..73370dd --- /dev/null +++ b/cmd/handlers.c @@ -0,0 +1,79 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt +// +// Adapted from: github.com/tuanpmt/esp_bridge, Created on: Jan 9, 2015, Author: Minh + +#include "esp8266.h" +#include "cmd.h" +#include "rest.h" +#include "crc16.h" +#include "serbridge.h" +#include "uart.h" +#include "cgiwifi.h" + +static uint32_t ICACHE_FLASH_ATTR CMD_Null(CmdPacket *cmd); +static uint32_t ICACHE_FLASH_ATTR CMD_IsReady(CmdPacket *cmd); +static uint32_t ICACHE_FLASH_ATTR CMD_WifiConnect(CmdPacket *cmd); + +// Command dispatch table for serial -> ESP commands +const CmdList commands[] = { + {CMD_NULL, CMD_Null}, + {CMD_RESET, CMD_Null}, + {CMD_IS_READY, CMD_IsReady}, + {CMD_WIFI_CONNECT, CMD_WifiConnect}, + +/* + {CMD_MQTT_SETUP, MQTTAPP_Setup}, + {CMD_MQTT_CONNECT, MQTTAPP_Connect}, + {CMD_MQTT_DISCONNECT, MQTTAPP_Disconnect}, + {CMD_MQTT_PUBLISH, MQTTAPP_Publish}, + {CMD_MQTT_SUBSCRIBE , MQTTAPP_Subscribe}, + {CMD_MQTT_LWT, MQTTAPP_Lwt}, + */ + + {CMD_REST_SETUP, REST_Setup}, + {CMD_REST_REQUEST, REST_Request}, + {CMD_REST_SETHEADER, REST_SetHeader}, + {CMD_NULL, NULL} +}; + +// Command handler for IsReady (healthcheck) command +static uint32_t ICACHE_FLASH_ATTR +CMD_IsReady(CmdPacket *cmd) { + os_printf("CMD: Check ready\n"); + return 1; +} + +// Command handler for Null command +static uint32_t ICACHE_FLASH_ATTR +CMD_Null(CmdPacket *cmd) { + os_printf("CMD: NULL/unsupported command\n"); + return 1; +} + +static uint8_t lastWifiStatus; +static uint32_t wifiCallback; + +// Callback from wifi subsystem to notify us of status changes +static void ICACHE_FLASH_ATTR +CMD_WifiCb(uint8_t wifiStatus) { + if (wifiStatus != lastWifiStatus){ + lastWifiStatus = wifiStatus; + if (wifiCallback) { + uint8_t status = wifiStatus == wifiGotIP ? 5 : 1; + uint16_t crc = CMD_ResponseStart(CMD_WIFI_CONNECT, wifiCallback, 0, 1); + crc = CMD_ResponseBody(crc, (uint8_t*)&status, 1); + CMD_ResponseEnd(crc); + } + } +} + +// Command handler for Wifi connect command +static uint32_t ICACHE_FLASH_ATTR +CMD_WifiConnect(CmdPacket *cmd) { + if(cmd->argc != 2 || cmd->callback == 0) + return 0xFFFFFFFF; + + wifiStatusCb = CMD_WifiCb; // register our callback with wifi subsystem + wifiCallback = cmd->callback; // save the MCU's callback + return 1; +} diff --git a/cmd/rest.c b/cmd/rest.c new file mode 100644 index 0000000..133bf2f --- /dev/null +++ b/cmd/rest.c @@ -0,0 +1,394 @@ +/* + * api.c + * + * Created on: Mar 4, 2015 + * Author: Minh + */ +#include "esp8266.h" +#include "rest.h" +#include "cmd.h" + +extern uint8_t ICACHE_FLASH_ATTR UTILS_StrToIP(const char* str, void *ip); + +static void ICACHE_FLASH_ATTR +tcpclient_discon_cb(void *arg) { + //struct espconn *pespconn = (struct espconn *)arg; + //RestClient* client = (RestClient *)pespconn->reverse; +} + +static void ICACHE_FLASH_ATTR +tcpclient_recv(void *arg, char *pdata, unsigned short len) { + uint8_t currentLineIsBlank = 0; + uint8_t httpBody = 0; + uint8_t inStatus = 0; + char statusCode[4]; + int i = 0, j; + uint32_t code = 0; + uint16_t crc; + + struct espconn *pCon = (struct espconn*)arg; + RestClient *client = (RestClient *)pCon->reverse; + + for(j=0 ;jresp_cb, code, 0); + } else { + crc = CMD_ResponseStart(CMD_REST_EVENTS, client->resp_cb, code, 1); + crc = CMD_ResponseBody(crc, (uint8_t*)(pdata+j), body_len); + } + CMD_ResponseEnd(crc); + break; + } else { + if (c == '\n' && currentLineIsBlank) { + httpBody = true; + } + if (c == '\n') { + // you're starting a new line + currentLineIsBlank = true; + } else if (c != '\r') { + // you've gotten a character on the current line + currentLineIsBlank = false; + } + } + } + //if(client->security) + // espconn_secure_disconnect(client->pCon); + //else + espconn_disconnect(client->pCon); + +} + +static void ICACHE_FLASH_ATTR +tcpclient_sent_cb(void *arg) { + //struct espconn *pCon = (struct espconn *)arg; + //RestClient* client = (RestClient *)pCon->reverse; + os_printf("REST: Sent\n"); +} + +static void ICACHE_FLASH_ATTR +tcpclient_connect_cb(void *arg) { + struct espconn *pCon = (struct espconn *)arg; + RestClient* client = (RestClient *)pCon->reverse; + + espconn_regist_disconcb(client->pCon, tcpclient_discon_cb); + espconn_regist_recvcb(client->pCon, tcpclient_recv);//////// + espconn_regist_sentcb(client->pCon, tcpclient_sent_cb);/////// + + //if(client->security){ + // espconn_secure_sent(client->pCon, client->data, client->data_len); + //} + //else{ + espconn_sent(client->pCon, client->data, client->data_len); + //} +} + +static void ICACHE_FLASH_ATTR +tcpclient_recon_cb(void *arg, sint8 errType) { + //struct espconn *pCon = (struct espconn *)arg; + //RestClient* client = (RestClient *)pCon->reverse; +} + +static void ICACHE_FLASH_ATTR +rest_dns_found(const char *name, ip_addr_t *ipaddr, void *arg) { + struct espconn *pConn = (struct espconn *)arg; + RestClient* client = (RestClient *)pConn->reverse; + + if(ipaddr == NULL) { + os_printf("REST DNS: Got no ip, try to reconnect\n"); + return; + } + + os_printf("REST DNS: found ip %d.%d.%d.%d\n", + *((uint8 *) &ipaddr->addr), + *((uint8 *) &ipaddr->addr + 1), + *((uint8 *) &ipaddr->addr + 2), + *((uint8 *) &ipaddr->addr + 3)); + + if(client->ip.addr == 0 && ipaddr->addr != 0) { + os_memcpy(client->pCon->proto.tcp->remote_ip, &ipaddr->addr, 4); + //if(client->security){ + // espconn_secure_connect(client->pCon); + //} + //else { + espconn_connect(client->pCon); + //} + os_printf("REST: connecting...\n"); + } +} + +uint32_t ICACHE_FLASH_ATTR +REST_Setup(CmdPacket *cmd) { + CmdRequest req; + RestClient *client; + uint8_t *rest_host; + uint16_t len; + uint32_t port, security; + + + // start parsing the command + CMD_Request(&req, cmd); + os_printf("REST: setup argc=%ld\n", CMD_GetArgc(&req)); + if(CMD_GetArgc(&req) != 3) + return 0; + + // get the hostname + len = CMD_ArgLen(&req); + os_printf("REST: len=%d\n", len); + if (len > 128) return 0; // safety check + rest_host = (uint8_t*)os_zalloc(len + 1); + if (CMD_PopArg(&req, rest_host, len)) return 0; + rest_host[len] = 0; + os_printf("REST: setup host=%s", rest_host); + + // get the port + if (CMD_PopArg(&req, (uint8_t*)&port, 4)) { + os_free(rest_host); + return 0; + } + os_printf(" port=%ld", port); + + // get the security mode + if (CMD_PopArg(&req, (uint8_t*)&security, 4)) { + os_free(rest_host); + return 0; + } + os_printf(" security=%ld\n", security); + + // allocate a connection structure + client = (RestClient*)os_zalloc(sizeof(RestClient)); + os_memset(client, 0, sizeof(RestClient)); + if(client == NULL) + return 0; + + os_printf("REST: setup host=%s port=%ld security=%ld\n", rest_host, port, security); + + client->resp_cb = cmd->callback; + + client->host = rest_host; + client->port = port; + client->security = security; + client->ip.addr = 0; + + client->data = (uint8_t*)os_zalloc(1024); + + client->header = (uint8_t*)os_zalloc(4); + client->header[0] = 0; + + client->content_type = (uint8_t*)os_zalloc(22); + os_sprintf((char *)client->content_type, "x-www-form-urlencoded"); + client->content_type[21] = 0; + + client->user_agent = (uint8_t*)os_zalloc(9); + os_sprintf((char *)client->user_agent, "esp-link"); + + client->pCon = (struct espconn *)os_zalloc(sizeof(struct espconn)); + client->pCon->proto.tcp = (esp_tcp *)os_zalloc(sizeof(esp_tcp)); + + client->pCon->type = ESPCONN_TCP; + client->pCon->state = ESPCONN_NONE; + client->pCon->proto.tcp->local_port = espconn_port(); + client->pCon->proto.tcp->remote_port = client->port; + + client->pCon->reverse = client; + + return (uint32_t)client; +} + +uint32_t ICACHE_FLASH_ATTR +REST_SetHeader(CmdPacket *cmd) { + CmdRequest req; + RestClient *client; + uint16_t len; + uint32_t header_index, client_ptr = 0; + + CMD_Request(&req, cmd); + + if(CMD_GetArgc(&req) != 3) + return 0; + + // Get client -- TODO: Whoa, this is totally unsafe! + if (CMD_PopArg(&req, (uint8_t*)&client_ptr, 4)) return 0; + client = (RestClient*)client_ptr; + + // Get header selector + if (CMD_PopArg(&req, (uint8_t*)&header_index, 4)) return 0; + + // Get header value + len = CMD_ArgLen(&req); + if (len > 256) return 0; //safety check + switch(header_index) { + case HEADER_GENERIC: + if(client->header) + os_free(client->header); + client->header = (uint8_t*)os_zalloc(len + 1); + CMD_PopArg(&req, (uint8_t*)client->header, len); + client->header[len] = 0; + os_printf("REST: Set header: %s\r\n", client->header); + break; + case HEADER_CONTENT_TYPE: + if(client->content_type) + os_free(client->content_type); + client->content_type = (uint8_t*)os_zalloc(len + 1); + CMD_PopArg(&req, (uint8_t*)client->content_type, len); + client->content_type[len] = 0; + os_printf("REST: Set content_type: %s\r\n", client->content_type); + break; + case HEADER_USER_AGENT: + if(client->user_agent) + os_free(client->user_agent); + client->user_agent = (uint8_t*)os_zalloc(len + 1); + CMD_PopArg(&req, (uint8_t*)client->user_agent, len); + client->user_agent[len] = 0; + os_printf("REST: Set user_agent: %s\r\n", client->user_agent); + break; + } + return 1; +} + +uint32_t ICACHE_FLASH_ATTR +REST_Request(CmdPacket *cmd) { + CmdRequest req; + RestClient *client; + uint16_t len, realLen = 0; + uint32_t client_ptr; + uint8_t *body = NULL; + uint8_t method[16]; + uint8_t path[1024]; + + CMD_Request(&req, cmd); + + if(CMD_GetArgc(&req) <3) + return 0; + + // Get client -- TODO: Whoa, this is totally unsafe! + if(CMD_PopArg(&req, (uint8_t*)&client_ptr, 4)) return 0; + client = (RestClient*)client_ptr; + + // Get HTTP method + len = CMD_ArgLen(&req); + if (len > 15) return 0; + CMD_PopArg(&req, method, len); + method[len] = 0; + + // Get HTTP path + len = CMD_ArgLen(&req); + if (len > 1023) return 0; + CMD_PopArg(&req, path, len); + path[len] = 0; + + // Get HTTP body + if (CMD_GetArgc(&req) == 3){ + realLen = 0; + len = 0; + } else { + CMD_PopArg(&req, (uint8_t*)&realLen, 4); + + len = CMD_ArgLen(&req); + if (len > 2048 || realLen > len) return 0; + body = (uint8_t*)os_zalloc(len + 1); + CMD_PopArg(&req, body, len); + body[len] = 0; + } + + client->pCon->state = ESPCONN_NONE; + + os_printf("REST: method: %s, path: %s\n", method, path); + + client->data_len = os_sprintf((char*)client->data, "%s %s HTTP/1.1\r\n" + "Host: %s\r\n" + "%s" + "Content-Length: %d\r\n" + "Connection: close\r\n" + "Content-Type: %s\r\n" + "User-Agent: %s\r\n\r\n", + method, path, + client->host, + client->header, + realLen, + client->content_type, + client->user_agent); + + if (realLen > 0){ + os_memcpy(client->data + client->data_len, body, realLen); + client->data_len += realLen; + //os_sprintf(client->data + client->data_len, "\r\n\r\n"); + //client->data_len += 4; + } + + client->pCon->state = ESPCONN_NONE; + espconn_regist_connectcb(client->pCon, tcpclient_connect_cb); + espconn_regist_reconcb(client->pCon, tcpclient_recon_cb); + + if(UTILS_StrToIP((char *)client->host, &client->pCon->proto.tcp->remote_ip)) { + os_printf("REST: Connect to ip %s:%ld\n",client->host, client->port); + //if(client->security){ + // espconn_secure_connect(client->pCon); + //} + //else { + espconn_connect(client->pCon); + //} + } else { + os_printf("REST: Connect to host %s:%ld\n", client->host, client->port); + espconn_gethostbyname(client->pCon, (char *)client->host, &client->ip, rest_dns_found); + } + + if(body) os_free(body); + return 1; +} + +uint8_t ICACHE_FLASH_ATTR +UTILS_StrToIP(const char* str, void *ip) +{ + /* The count of the number of bytes processed. */ + int i; + /* A pointer to the next digit to process. */ + const char * start; + + start = str; + for (i = 0; i < 4; i++) { + /* The digit being processed. */ + char c; + /* The value of this byte. */ + int n = 0; + while (1) { + c = * start; + start++; + if (c >= '0' && c <= '9') { + n *= 10; + n += c - '0'; + } + /* We insist on stopping at "." if we are still parsing + the first, second, or third numbers. If we have reached + the end of the numbers, we will allow any character. */ + else if ((i < 3 && c == '.') || i == 3) { + break; + } + else { + return 0; + } + } + if (n >= 256) { + return 0; + } + ((uint8_t*)ip)[i] = n; + } + return 1; +} diff --git a/cmd/rest.h b/cmd/rest.h new file mode 100644 index 0000000..13159ea --- /dev/null +++ b/cmd/rest.h @@ -0,0 +1,39 @@ +/* + * api.h + * + * Created on: Mar 4, 2015 + * Author: Minh + */ + +#ifndef MODULES_API_H_ +#define MODULES_API_H_ + +#include "c_types.h" +#include "ip_addr.h" +#include "cmd.h" + +typedef enum { + HEADER_GENERIC = 0, + HEADER_CONTENT_TYPE, + HEADER_USER_AGENT +} HEADER_TYPE; + +typedef struct { + uint8_t *host; + uint32_t port; + uint32_t security; + ip_addr_t ip; + struct espconn *pCon; + uint8_t *header; + uint8_t *data; + uint32_t data_len; + uint8_t *content_type; + uint8_t *user_agent; + uint32_t resp_cb; +} RestClient; + +uint32_t REST_Setup(CmdPacket *cmd); +uint32_t REST_Request(CmdPacket *cmd); +uint32_t REST_SetHeader(CmdPacket *cmd); + +#endif /* MODULES_INCLUDE_API_H_ */ diff --git a/serial/tcpclient.c b/cmd/tcpclient.c similarity index 100% rename from serial/tcpclient.c rename to cmd/tcpclient.c diff --git a/serial/tcpclient.h b/cmd/tcpclient.h similarity index 86% rename from serial/tcpclient.h rename to cmd/tcpclient.h index 9bfeb01..ff0ff9d 100644 --- a/serial/tcpclient.h +++ b/cmd/tcpclient.h @@ -4,7 +4,7 @@ // max number of channels the client can open #define MAX_TCP_CHAN 8 -// Parse and perform the commandm cmdBuf must be null-terminated +// Parse and perform the command, cmdBuf must be null-terminated bool tcpClientCommand(uint8_t chan, char cmd, char *cmdBuf); // Append a character to the specified channel diff --git a/serial/crc16.c b/serial/crc16.c new file mode 100644 index 0000000..22a6651 --- /dev/null +++ b/serial/crc16.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2005, Swedish Institute of Computer Science + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file is part of the Contiki operating system. + * + */ + +/** \addtogroup crc16 + * @{ */ + +/** + * \file + * Implementation of the CRC16 calculcation + * \author + * Adam Dunkels + * + */ + +/* CITT CRC16 polynomial ^16 + ^12 + ^5 + 1 */ +/*---------------------------------------------------------------------------*/ +unsigned short +crc16_add(unsigned char b, unsigned short acc) +{ + /* + acc = (unsigned char)(acc >> 8) | (acc << 8); + acc ^= b; + acc ^= (unsigned char)(acc & 0xff) >> 4; + acc ^= (acc << 8) << 4; + acc ^= ((acc & 0xff) << 4) << 1; + */ + + acc ^= b; + acc = (acc >> 8) | (acc << 8); + acc ^= (acc & 0xff00) << 4; + acc ^= (acc >> 8) >> 4; + acc ^= (acc & 0xff00) >> 5; + return acc; +} +/*---------------------------------------------------------------------------*/ +unsigned short +crc16_data(const unsigned char *data, int len, unsigned short acc) +{ + int i; + + for(i = 0; i < len; ++i) { + acc = crc16_add(*data, acc); + ++data; + } + return acc; +} +/*---------------------------------------------------------------------------*/ + +/** @} */ diff --git a/serial/crc16.h b/serial/crc16.h new file mode 100644 index 0000000..bd4c52e --- /dev/null +++ b/serial/crc16.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2005, Swedish Institute of Computer Science + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the Institute nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE INSTITUTE AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE INSTITUTE OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This file is part of the Contiki operating system. + * + */ + +/** + * \file + * Header file for the CRC16 calculcation + * \author + * Adam Dunkels + * + */ + +/** \addtogroup lib + * @{ */ + +/** + * \defgroup crc16 Cyclic Redundancy Check 16 (CRC16) calculation + * + * The Cyclic Redundancy Check 16 is a hash function that produces a + * checksum that is used to detect errors in transmissions. The CRC16 + * calculation module is an iterative CRC calculator that can be used + * to cumulatively update a CRC checksum for every incoming byte. + * + * @{ + */ + +#ifndef CRC16_H_ +#define CRC16_H_ +#ifdef __cplusplus +extern "C" { +#endif +/** + * \brief Update an accumulated CRC16 checksum with one byte. + * \param b The byte to be added to the checksum + * \param crc The accumulated CRC that is to be updated. + * \return The updated CRC checksum. + * + * This function updates an accumulated CRC16 checksum + * with one byte. It can be used as a running checksum, or + * to checksum an entire data block. + * + * \note The algorithm used in this implementation is + * tailored for a running checksum and does not perform as + * well as a table-driven algorithm when checksumming an + * entire data block. + * + */ +unsigned short crc16_add(unsigned char b, unsigned short crc); + +/** + * \brief Calculate the CRC16 over a data area + * \param data Pointer to the data + * \param datalen The length of the data + * \param acc The accumulated CRC that is to be updated (or zero). + * \return The CRC16 checksum. + * + * This function calculates the CRC16 checksum of a data area. + * + * \note The algorithm used in this implementation is + * tailored for a running checksum and does not perform as + * well as a table-driven algorithm when checksumming an + * entire data block. + */ +unsigned short crc16_data(const unsigned char *data, int datalen, + unsigned short acc); +#ifdef __cplusplus +} +#endif +#endif /* CRC16_H_ */ + +/** @} */ +/** @} */ diff --git a/serial/serbridge.c b/serial/serbridge.c index 5c00175..415772b 100644 --- a/serial/serbridge.c +++ b/serial/serbridge.c @@ -9,24 +9,27 @@ #include "gpio.h" #include "uart.h" +#include "crc16.h" #include "serbridge.h" #include "serled.h" #include "config.h" #include "console.h" -#include "tcpclient.h" +#include "cmd.h" static struct espconn serbridgeConn; static esp_tcp serbridgeTcp; static int8_t mcu_reset_pin, mcu_isp_pin; +extern uint8_t slip_disabled; // disable slip to allow flashing of attached MCU + static sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len); // Connection pool serbridgeConnData connData[MAX_CONN]; // Given a pointer to an espconn struct find the connection that correcponds to it static serbridgeConnData ICACHE_FLASH_ATTR *serbridgeFindConnData(void *arg) { - struct espconn *conn = arg; - return arg == NULL ? NULL : (serbridgeConnData *)conn->reverse; + struct espconn *conn = arg; + return arg == NULL ? NULL : (serbridgeConnData *)conn->reverse; } //===== TCP -> UART @@ -50,149 +53,152 @@ enum { TN_normal, TN_iac, TN_will, TN_start, TN_end, TN_comPort, TN_setControl } static uint8_t ICACHE_FLASH_ATTR telnetUnwrap(uint8_t *inBuf, int len, uint8_t state) { - for (int i=0; i write one to outbuf and go normal again - state = TN_normal; - uart0_write_char(c); - break; - case WILL: // negotiation - state = TN_will; - break; - case SB: // command sequence begin - state = TN_start; - break; - case SE: // command sequence end - state = TN_normal; - break; - default: // not sure... let's ignore - uart0_write_char(IAC); - uart0_write_char(c); - } - break; - case TN_will: - state = TN_normal; // yes, we do COM port options, let's go back to normal - break; - case TN_start: // in command seq, now comes the type of cmd - if (c == ComPortOpt) state = TN_comPort; - else state = TN_end; // an option we don't know, skip 'til the end seq - break; - case TN_end: // wait for end seq - if (c == IAC) state = TN_iac; // simple wait to accept end or next escape seq - break; - case TN_comPort: - if (c == SetControl) state = TN_setControl; - else state = TN_end; - break; - case TN_setControl: // switch control line and delay a tad - switch (c) { - case DTR_ON: - if (mcu_reset_pin >= 0) { - os_printf("MCU reset gpio%d\n", mcu_reset_pin); - GPIO_OUTPUT_SET(mcu_reset_pin, 0); - os_delay_us(100L); - } else os_printf("MCU reset: no pin\n"); - break; - case DTR_OFF: - if (mcu_reset_pin >= 0) { - GPIO_OUTPUT_SET(mcu_reset_pin, 1); - os_delay_us(100L); - } - break; - case RTS_ON: - if (mcu_isp_pin >= 0) { - os_printf("MCU ISP gpio%d\n", mcu_isp_pin); - GPIO_OUTPUT_SET(mcu_isp_pin, 0); - os_delay_us(100L); - } else os_printf("MCU isp: no pin\n"); - break; - case RTS_OFF: - if (mcu_isp_pin >= 0) { - GPIO_OUTPUT_SET(mcu_isp_pin, 1); - os_delay_us(100L); - } - break; - } - state = TN_end; - break; - } - } - return state; + for (int i=0; i write one to outbuf and go normal again + state = TN_normal; + uart0_write_char(c); + break; + case WILL: // negotiation + state = TN_will; + break; + case SB: // command sequence begin + state = TN_start; + break; + case SE: // command sequence end + state = TN_normal; + break; + default: // not sure... let's ignore + uart0_write_char(IAC); + uart0_write_char(c); + } + break; + case TN_will: + state = TN_normal; // yes, we do COM port options, let's go back to normal + break; + case TN_start: // in command seq, now comes the type of cmd + if (c == ComPortOpt) state = TN_comPort; + else state = TN_end; // an option we don't know, skip 'til the end seq + break; + case TN_end: // wait for end seq + if (c == IAC) state = TN_iac; // simple wait to accept end or next escape seq + break; + case TN_comPort: + if (c == SetControl) state = TN_setControl; + else state = TN_end; + break; + case TN_setControl: // switch control line and delay a tad + switch (c) { + case DTR_ON: + if (mcu_reset_pin >= 0) { + os_printf("MCU reset gpio%d\n", mcu_reset_pin); + GPIO_OUTPUT_SET(mcu_reset_pin, 0); + os_delay_us(100L); + } else os_printf("MCU reset: no pin\n"); + break; + case DTR_OFF: + if (mcu_reset_pin >= 0) { + GPIO_OUTPUT_SET(mcu_reset_pin, 1); + os_delay_us(100L); + } + break; + case RTS_ON: + if (mcu_isp_pin >= 0) { + os_printf("MCU ISP gpio%d\n", mcu_isp_pin); + GPIO_OUTPUT_SET(mcu_isp_pin, 0); + os_delay_us(100L); + } else os_printf("MCU isp: no pin\n"); + slip_disabled++; + break; + case RTS_OFF: + if (mcu_isp_pin >= 0) { + GPIO_OUTPUT_SET(mcu_isp_pin, 1); + os_delay_us(100L); + } + if (slip_disabled > 0) slip_disabled--; + break; + } + state = TN_end; + break; + } + } + return state; } +// Generate a reset pulse for the attached microcontroller void ICACHE_FLASH_ATTR serbridgeReset() { - if (mcu_reset_pin >= 0) { - os_printf("MCU reset gpio%d\n", mcu_reset_pin); - GPIO_OUTPUT_SET(mcu_reset_pin, 0); - os_delay_us(100L); - GPIO_OUTPUT_SET(mcu_reset_pin, 1); - } else os_printf("MCU reset: no pin\n"); + if (mcu_reset_pin >= 0) { + os_printf("MCU reset gpio%d\n", mcu_reset_pin); + GPIO_OUTPUT_SET(mcu_reset_pin, 0); + os_delay_us(100L); + GPIO_OUTPUT_SET(mcu_reset_pin, 1); + } else os_printf("MCU reset: no pin\n"); } // Receive callback static void ICACHE_FLASH_ATTR serbridgeRecvCb(void *arg, char *data, unsigned short len) { - serbridgeConnData *conn = serbridgeFindConnData(arg); - //os_printf("Receive callback on conn %p\n", conn); - if (conn == NULL) return; - - // at the start of a connection we're in cmInit mode and we wait for the first few characters - // to arrive in order to decide what type of connection this is.. The following if statements - // do this dispatch. An issue here is that we assume that the first few characters all arrive - // in the same TCP packet, which is true if the sender is a program, but not necessarily - // if the sender is a person typing (although in that case the line-oriented TTY input seems - // to make it work too). If this becomes a problem we need to buffer the first few chars... - if (conn->conn_mode == cmInit) { - - // If the connection starts with the Arduino or ARM reset sequence we perform a RESET - if ((len == 2 && strncmp(data, "0 ", 2) == 0) || - (len == 2 && strncmp(data, "?\n", 2) == 0) || - (len == 3 && strncmp(data, "?\r\n", 3) == 0)) { - os_printf("MCU Reset=%d ISP=%d\n", mcu_reset_pin, mcu_isp_pin); - os_delay_us(2*1000L); // time for os_printf to happen - // send reset to arduino/ARM - if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 0); - os_delay_us(100L); - if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 0); - os_delay_us(100L); - if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 1); - os_delay_us(100L); - if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); - os_delay_us(1000L); - conn->conn_mode = cmAVR; - - - // If the connection starts with a telnet negotiation we will do telnet - } else if (len >= 3 && strncmp(data, (char[]){IAC, WILL, ComPortOpt}, 3) == 0) { - conn->conn_mode = cmTelnet; - conn->telnet_state = TN_normal; - // note that the three negotiation chars will be gobbled-up by telnetUnwrap - os_printf("telnet mode\n"); - - // looks like a plain-vanilla connection! - } else { - conn->conn_mode = cmTransparent; - } - - // Process return data on TCP client connections - } else if (conn->conn_mode == cmTcpClient) { - } - - // write the buffer to the uart - if (conn->conn_mode == cmTelnet) { - conn->telnet_state = telnetUnwrap((uint8_t *)data, len, conn->telnet_state); - } else { - uart0_tx_buffer(data, len); - } - - serledFlash(50); // short blink on serial LED + serbridgeConnData *conn = serbridgeFindConnData(arg); + //os_printf("Receive callback on conn %p\n", conn); + if (conn == NULL) return; + + // at the start of a connection we're in cmInit mode and we wait for the first few characters + // to arrive in order to decide what type of connection this is.. The following if statements + // do this dispatch. An issue here is that we assume that the first few characters all arrive + // in the same TCP packet, which is true if the sender is a program, but not necessarily + // if the sender is a person typing (although in that case the line-oriented TTY input seems + // to make it work too). If this becomes a problem we need to buffer the first few chars... + if (conn->conn_mode == cmInit) { + + // If the connection starts with the Arduino or ARM reset sequence we perform a RESET + if ((len == 2 && strncmp(data, "0 ", 2) == 0) || + (len == 2 && strncmp(data, "?\n", 2) == 0) || + (len == 3 && strncmp(data, "?\r\n", 3) == 0)) { + os_printf("MCU Reset=gpio%d ISP=gpio%d\n", mcu_reset_pin, mcu_isp_pin); + os_delay_us(2*1000L); // time for os_printf to happen + // send reset to arduino/ARM + if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 0); + os_delay_us(100L); + if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 0); + os_delay_us(100L); + if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 1); + os_delay_us(100L); + if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); + os_delay_us(1000L); + conn->conn_mode = cmAVR; + slip_disabled++; // disable SLIP so it doesn't interfere with flashing + + // If the connection starts with a telnet negotiation we will do telnet + } else if (len >= 3 && strncmp(data, (char[]){IAC, WILL, ComPortOpt}, 3) == 0) { + conn->conn_mode = cmTelnet; + conn->telnet_state = TN_normal; + // note that the three negotiation chars will be gobbled-up by telnetUnwrap + os_printf("telnet mode\n"); + + // looks like a plain-vanilla connection! + } else { + conn->conn_mode = cmTransparent; + } + + // Process return data on TCP client connections + } else if (conn->conn_mode == cmTcpClient) { + } + + // write the buffer to the uart + if (conn->conn_mode == cmTelnet) { + conn->telnet_state = telnetUnwrap((uint8_t *)data, len, conn->telnet_state); + } else { + uart0_tx_buffer(data, len); + } + + serledFlash(50); // short blink on serial LED } //===== UART -> TCP @@ -203,18 +209,19 @@ static char txbuffer[MAX_CONN][MAX_TXBUFFER]; // Send all data in conn->txbuffer // returns result from espconn_sent if data in buffer or ESPCONN_OK (0) // Use only internally from espbuffsend and serbridgeSentCb -static sint8 ICACHE_FLASH_ATTR sendtxbuffer(serbridgeConnData *conn) { - sint8 result = ESPCONN_OK; - if (conn->txbufferlen != 0) { - //os_printf("%d TX %d\n", system_get_time(), conn->txbufferlen); - conn->readytosend = false; - result = espconn_sent(conn->conn, (uint8_t*)conn->txbuffer, conn->txbufferlen); - conn->txbufferlen = 0; - if (result != ESPCONN_OK) { - os_printf("sendtxbuffer: espconn_sent error %d on conn %p\n", result, conn); - } - } - return result; +static sint8 ICACHE_FLASH_ATTR +sendtxbuffer(serbridgeConnData *conn) { + sint8 result = ESPCONN_OK; + if (conn->txbufferlen != 0) { + //os_printf("%d TX %d\n", system_get_time(), conn->txbufferlen); + conn->readytosend = false; + result = espconn_sent(conn->conn, (uint8_t*)conn->txbuffer, conn->txbufferlen); + conn->txbufferlen = 0; + if (result != ESPCONN_OK) { + os_printf("sendtxbuffer: espconn_sent error %d on conn %p\n", result, conn); + } + } + return result; } // espbuffsend adds data to the send buffer. If the previous send was completed it calls @@ -222,173 +229,44 @@ static sint8 ICACHE_FLASH_ATTR sendtxbuffer(serbridgeConnData *conn) { // Returns ESPCONN_OK (0) for success, -128 if buffer is full or error from espconn_sent // Use espbuffsend instead of espconn_sent as it solves the problem that espconn_sent must // only be called *after* receiving an espconn_sent_callback for the previous packet. -static sint8 ICACHE_FLASH_ATTR espbuffsend(serbridgeConnData *conn, const char *data, uint16 len) { - if (conn->txbufferlen + len > MAX_TXBUFFER) { - os_printf("espbuffsend: txbuffer full on conn %p\n", conn); - return -128; - } - os_memcpy(conn->txbuffer + conn->txbufferlen, data, len); - conn->txbufferlen += len; - if (conn->readytosend) { - return sendtxbuffer(conn); - } else { - //os_printf("%d QU %d\n", system_get_time(), conn->txbufferlen); - } - return ESPCONN_OK; +static sint8 ICACHE_FLASH_ATTR +espbuffsend(serbridgeConnData *conn, const char *data, uint16 len) { + if (conn->txbufferlen + len > MAX_TXBUFFER) { + os_printf("espbuffsend: txbuffer full on conn %p\n", conn); + return -128; + } + os_memcpy(conn->txbuffer + conn->txbufferlen, data, len); + conn->txbufferlen += len; + if (conn->readytosend) { + return sendtxbuffer(conn); + } else { + //os_printf("%d QU %d\n", system_get_time(), conn->txbufferlen); + } + return ESPCONN_OK; +} + +void ICACHE_FLASH_ATTR +console_process(char *buf, short len) { + // push buffer into web-console + for (short i=0; ireadytosend = true; - sendtxbuffer(conn); // send possible new data in txbuffer -} - -// TCP client connection state machine -// This processes commands from the attached uC to open outboud TCP connections -enum { - TC_idle, // in-between commands - TC_newline, // newline seen - TC_start, // start character (~) seen - TC_cmd, // command start (@) seen - TC_cmdChar, // command character seen - TC_cmdLine, // accumulating command - TC_tdchan, // saw data channel character - TC_tdlen1, // saw first data length character - TC_tdata0, // accumulate data, zero-terminated - TC_tdataN, // accumulate data, length-terminated -}; -static uint8_t tcState = TC_newline; -static uint8_t tcChan; // channel for current command (index into tcConn) - -#define CMD_MAX 256 -static char tcCmdBuf[CMD_MAX]; -static short tcCmdBufLen = 0; -static char tcCmdChar; -static short tcLen; - -// scan a buffer for tcp client commands -static int ICACHE_FLASH_ATTR -tcpClientProcess(char *buf, int len) -{ - char *in=buf, *out=buf; - for (short i=0; i= '0' && c <= '9') { - tcChan = c-'0'; - tcState = TC_tdchan; - continue; - } - *out++ = '~'; // make up for '~' we skipped - break; - case TC_cmd: // saw control char (@), expect channel char - if (c >= '0' && c <= '9') { - tcChan = c-'0'; - tcState = TC_cmdChar; - continue; - } else { - *out++ = '~'; // make up for '~' we skipped - *out++ = '@'; // make up for '@' we skipped - break; - } - case TC_cmdChar: // saw channel number, expect command char - tcCmdChar = c; // save command character - tcCmdBufLen = 0; // empty the command buffer - tcState = TC_cmdLine; - continue; - case TC_cmdLine: // accumulating command in buffer - if (c != '\n') { - if (tcCmdBufLen < CMD_MAX) tcCmdBuf[tcCmdBufLen++] = c; - } else { - tcpClientCommand(tcChan, tcCmdChar, tcCmdBuf); - tcState = TC_newline; - } - continue; - case TC_tdchan: // saw channel number, getting first length char - if (c >= '0' && c <= '9') { - tcLen = c-'0'; - } else if (c >= 'A' && c <= 'F') { - tcLen = c-'A'+10; - } else { - *out++ = '~'; // make up for '~' we skipped - *out++ = '0'+tcChan; - break; - } - tcState = TC_tdlen1; - continue; - case TC_tdlen1: // saw first length char, get second - tcLen *= 16; - if (c >= '0' && c <= '9') { - tcLen += c-'0'; - } else if (c >= 'A' && c <= 'F') { - tcLen += c-'A'+10; - } else { - *out++ = '~'; // make up for '~' we skipped - *out++ = '0'+tcChan; - break; - } - tcState = tcLen == 0 ? TC_tdata0 : TC_tdataN; - continue; - case TC_tdata0: // saw data length, getting data characters zero-terminated - if (c != 0) { - tcpClientSendChar(tcChan, c); - } else { - tcpClientSendPush(tcChan); - tcState = TC_idle; - } - continue; - case TC_tdataN: // saw data length, getting data characters length-terminated - tcpClientSendChar(tcChan, c); - tcLen--; - if (tcLen == 0) { - tcpClientSendPush(tcChan); - tcState = TC_idle; - } - continue; - } - *out++ = c; - } - //if (tcState != TC_idle) os_printf("tcState=%d\n", tcState); - return out-buf; -} - -// callback with a buffer of characters that have arrived on the uart -void ICACHE_FLASH_ATTR -serbridgeUartCb(char *buf, int length) { - // push the buffer into the microcontroller console - for (int i=0; i 0) { - for (int i=0; ireadytosend = true; + sendtxbuffer(conn); // send possible new data in txbuffer } //===== Connect / disconnect @@ -396,101 +274,107 @@ serbridgeUartCb(char *buf, int length) { // Error callback (it's really poorly named, it's not a "connection reconnected" callback, // it's really a "connection broken, please reconnect" callback) static void ICACHE_FLASH_ATTR serbridgeReconCb(void *arg, sint8 err) { - serbridgeConnData *sbConn = serbridgeFindConnData(arg); - if (sbConn == NULL) return; - // Close the connection - espconn_disconnect(sbConn->conn); - // free connection slot - sbConn->conn = NULL; + serbridgeConnData *sbConn = serbridgeFindConnData(arg); + if (sbConn == NULL) return; + if (sbConn->conn_mode == cmAVR) { + if (slip_disabled > 0) slip_disabled--; + } + // Close the connection + espconn_disconnect(sbConn->conn); + // free connection slot + sbConn->conn = NULL; } // Disconnection callback static void ICACHE_FLASH_ATTR serbridgeDisconCb(void *arg) { - serbridgeConnData *sbConn = serbridgeFindConnData(arg); - if (sbConn == NULL) return; - // send reset to arduino/ARM - if (sbConn->conn_mode == cmAVR && mcu_reset_pin >= 0) { - GPIO_OUTPUT_SET(mcu_reset_pin, 0); - os_delay_us(100L); - GPIO_OUTPUT_SET(mcu_reset_pin, 1); - } - // free connection slot - sbConn->conn = NULL; + serbridgeConnData *sbConn = serbridgeFindConnData(arg); + if (sbConn == NULL) return; + // send reset to arduino/ARM + if (sbConn->conn_mode == cmAVR) { + if (slip_disabled > 0) slip_disabled--; + if (mcu_reset_pin >= 0) { + GPIO_OUTPUT_SET(mcu_reset_pin, 0); + os_delay_us(100L); + GPIO_OUTPUT_SET(mcu_reset_pin, 1); + } + } + // free connection slot + sbConn->conn = NULL; } // New connection callback, use one of the connection descriptors, if we have one left. static void ICACHE_FLASH_ATTR serbridgeConnectCb(void *arg) { - struct espconn *conn = arg; - // Find empty conndata in pool - int i; - for (i=0; ireverse = connData+i; - connData[i].conn = conn; - connData[i].txbufferlen = 0; - connData[i].readytosend = true; - connData[i].telnet_state = 0; - connData[i].conn_mode = cmInit; - - espconn_regist_recvcb(conn, serbridgeRecvCb); - espconn_regist_reconcb(conn, serbridgeReconCb); - espconn_regist_disconcb(conn, serbridgeDisconCb); - espconn_regist_sentcb(conn, serbridgeSentCb); - - espconn_set_opt(conn, ESPCONN_REUSEADDR|ESPCONN_NODELAY); + struct espconn *conn = arg; + // Find empty conndata in pool + int i; + for (i=0; ireverse = connData+i; + connData[i].conn = conn; + connData[i].txbufferlen = 0; + connData[i].readytosend = true; + connData[i].telnet_state = 0; + connData[i].conn_mode = cmInit; + + espconn_regist_recvcb(conn, serbridgeRecvCb); + espconn_regist_reconcb(conn, serbridgeReconCb); + espconn_regist_disconcb(conn, serbridgeDisconCb); + espconn_regist_sentcb(conn, serbridgeSentCb); + + espconn_set_opt(conn, ESPCONN_REUSEADDR|ESPCONN_NODELAY); } //===== Initialization void ICACHE_FLASH_ATTR serbridgeInitPins() { - mcu_reset_pin = flashConfig.reset_pin; - mcu_isp_pin = flashConfig.isp_pin; - os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", - mcu_reset_pin, mcu_isp_pin, flashConfig.swap_uart); - - if (flashConfig.swap_uart) { - PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); - PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); - PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTCK_U); - PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTDO_U); - system_uart_swap(); - } else { - PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); - PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, 0); - system_uart_de_swap(); - } - - // set both pins to 1 before turning them on so we don't cause a reset - if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); - if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 1); - // switch pin mux to make these pins GPIO pins - if (mcu_reset_pin >= 0) makeGpio(mcu_reset_pin); - if (mcu_isp_pin >= 0) makeGpio(mcu_isp_pin); + mcu_reset_pin = flashConfig.reset_pin; + mcu_isp_pin = flashConfig.isp_pin; + os_printf("Serbridge pins: reset=%d isp=%d swap=%d\n", + mcu_reset_pin, mcu_isp_pin, flashConfig.swap_uart); + + if (flashConfig.swap_uart) { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTCK_U, 4); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_MTDO_U, 4); + PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTCK_U); + PIN_PULLUP_DIS(PERIPHS_IO_MUX_MTDO_U); + system_uart_swap(); + } else { + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0TXD_U, 0); + PIN_FUNC_SELECT(PERIPHS_IO_MUX_U0RXD_U, 0); + system_uart_de_swap(); + } + + // set both pins to 1 before turning them on so we don't cause a reset + if (mcu_isp_pin >= 0) GPIO_OUTPUT_SET(mcu_isp_pin, 1); + if (mcu_reset_pin >= 0) GPIO_OUTPUT_SET(mcu_reset_pin, 1); + // switch pin mux to make these pins GPIO pins + if (mcu_reset_pin >= 0) makeGpio(mcu_reset_pin); + if (mcu_isp_pin >= 0) makeGpio(mcu_isp_pin); } // Start transparent serial bridge TCP server on specified port (typ. 23) void ICACHE_FLASH_ATTR serbridgeInit(int port) { - serbridgeInitPins(); - - int i; - for (i = 0; i < MAX_CONN; i++) { - connData[i].conn = NULL; - connData[i].txbuffer = txbuffer[i]; - } - serbridgeConn.type = ESPCONN_TCP; - serbridgeConn.state = ESPCONN_NONE; - serbridgeTcp.local_port = port; - serbridgeConn.proto.tcp = &serbridgeTcp; - - espconn_regist_connectcb(&serbridgeConn, serbridgeConnectCb); - espconn_accept(&serbridgeConn); - espconn_tcp_set_max_con_allow(&serbridgeConn, MAX_CONN); - espconn_regist_time(&serbridgeConn, SER_BRIDGE_TIMEOUT, 0); + serbridgeInitPins(); + + int i; + for (i = 0; i < MAX_CONN; i++) { + connData[i].conn = NULL; + connData[i].txbuffer = txbuffer[i]; + } + serbridgeConn.type = ESPCONN_TCP; + serbridgeConn.state = ESPCONN_NONE; + serbridgeTcp.local_port = port; + serbridgeConn.proto.tcp = &serbridgeTcp; + + espconn_regist_connectcb(&serbridgeConn, serbridgeConnectCb); + espconn_accept(&serbridgeConn); + espconn_tcp_set_max_con_allow(&serbridgeConn, MAX_CONN); + espconn_regist_time(&serbridgeConn, SER_BRIDGE_TIMEOUT, 0); } diff --git a/serial/serbridge.h b/serial/serbridge.h index 33c96a2..ce11906 100644 --- a/serial/serbridge.h +++ b/serial/serbridge.h @@ -32,7 +32,7 @@ typedef struct serbridgeConnData { void ICACHE_FLASH_ATTR serbridgeInit(int port); void ICACHE_FLASH_ATTR serbridgeInitPins(void); -void ICACHE_FLASH_ATTR serbridgeUartCb(char *buf, int len); +void ICACHE_FLASH_ATTR serbridgeUartCb(char *buf, short len); void ICACHE_FLASH_ATTR serbridgeReset(); #endif /* __SER_BRIDGE_H__ */ diff --git a/serial/slip.c b/serial/slip.c new file mode 100644 index 0000000..0c76103 --- /dev/null +++ b/serial/slip.c @@ -0,0 +1,136 @@ +// Copyright 2015 by Thorsten von Eicken, see LICENSE.txt + +#include "esp8266.h" +#include "uart.h" +#include "crc16.h" +#include "serbridge.h" +#include "serled.h" +#include "console.h" +#include "cmd.h" + +uint8_t slip_disabled; // disable slip to allow flashing of attached MCU + +extern void ICACHE_FLASH_ATTR console_process(char *buf, short len); + +// This SLIP parser does not conform to RFC 1055 https://tools.ietf.org/html/rfc1055, +// instead, it implements the framing implemented in https://github.com/tuanpmt/esp_bridge +// It accumulates each packet into a static buffer and calls cmd_parse() when the end +// of a packet is reached. It expects cmd_parse() to copy anything it needs from the +// buffer elsewhere as the buffer is immediately reused. +// One special feature is that if the first two characters of a packet are both printable or +// \n or \r then the parser assumes it's dealing with console debug output and calls +// slip_console(c) for each character and does not accumulate chars in the buffer until the +// next SLIP_END marker is seen. This allows random console debug output to come in between +// packets as long as each packet starts *and* ends with SLIP_END (which is an official +// variation on the SLIP protocol). + +static bool slip_escaped; // true when prev char received is escape +static bool slip_inpkt; // true when we're after SLIP_START and before SLIP_END +#define SLIP_MAX 1024 // max length of SLIP packet +static char slip_buf[SLIP_MAX]; // buffer for current SLIP packet +static short slip_len; // accumulated length in slip_buf + +// SLIP process a packet or a bunch of debug console chars +static void ICACHE_FLASH_ATTR +slip_process() { + if (slip_len < 1) return; + + if (!slip_inpkt) { + // debug console stuff + console_process(slip_buf, slip_len); + } else { + // proper SLIP packet, invoke command processor after checking CRC + //os_printf("SLIP: rcv %d\n", slip_len); + if (slip_len > 2) { + uint16_t crc = crc16_data((uint8_t*)slip_buf, slip_len-2, 0); + uint16_t rcv = ((uint16_t)slip_buf[slip_len-2]) | ((uint16_t)slip_buf[slip_len-1] << 8); + if (crc == rcv) { + CMD_parse_packet((uint8_t*)slip_buf, slip_len-2); + } else { + os_printf("SLIP: bad CRC, crc=%x rcv=%x\n", crc, rcv); + for (short i=0; i= ' ' && slip_buf[i] <= '~') + os_printf("%c", slip_buf[i]); + else + os_printf("\\%02X", slip_buf[i]); + } + os_printf("\n"); + } + } + } +} + +#if 0 +// determine whether a character is printable or not (or \r \n) +static bool ICACHE_FLASH_ATTR +slip_printable(char c) { + return (c >= ' ' && c <= '~') || c == '\n' || c == '\r'; +} +#endif + +static void ICACHE_FLASH_ATTR +slip_reset() { + slip_inpkt = false; + slip_escaped = false; + slip_len = 0; +} + +// SLIP parse a single character +static void ICACHE_FLASH_ATTR +slip_parse_char(char c) { + if (!slip_inpkt) { + if (c == SLIP_START) { + if (slip_len > 0) console_process(slip_buf, slip_len); + slip_reset(); + slip_inpkt = true; + return; + } + } else if (slip_escaped) { + // prev char was SLIP_REPL + c = SLIP_ESC(c); + slip_escaped = false; + } else { + switch (c) { + case SLIP_REPL: + slip_escaped = true; + return; + case SLIP_END: + // end of packet, process it and get ready for next one + if (slip_len > 0) slip_process(); + slip_reset(); + return; + case SLIP_START: + os_printf("SLIP: got SLIP_START while in packet?\n"); + slip_reset(); + return; + } + } + if (slip_len < SLIP_MAX) slip_buf[slip_len++] = c; +} + +// callback with a buffer of characters that have arrived on the uart +void ICACHE_FLASH_ATTR +serbridgeUartCb(char *buf, short length) { + if (slip_disabled > 0) { + //os_printf("SLIP: disabled got %d\n", length); + console_process(buf, length); + for (short i=0; i 0) { + slip_process(); + slip_reset(); + } + + serledFlash(50); // short blink on serial LED +} diff --git a/serial/uart.h b/serial/uart.h index 2d1d432..3553155 100644 --- a/serial/uart.h +++ b/serial/uart.h @@ -4,7 +4,7 @@ #include "uart_hw.h" // Receive callback function signature -typedef void (*UartRecv_cb)(char *buf, int len); +typedef void (*UartRecv_cb)(char *buf, short len); // Initialize UARTs to the provided baud rates (115200 recommended). This also makes the os_printf // calls use uart1 for output (for debugging purposes) diff --git a/user/cgiwifi.c b/user/cgiwifi.c index 0d96e96..42ddf24 100644 --- a/user/cgiwifi.c +++ b/user/cgiwifi.c @@ -40,6 +40,8 @@ static char *wifiReasons[] = { static char *wifiMode[] = { 0, "STA", "AP", "AP+STA" }; static char *wifiPhy[] = { 0, "11b", "11g", "11n" }; +void (*wifiStatusCb)(uint8_t); // callback when wifi status changes + static char* ICACHE_FLASH_ATTR wifiGetReason(void) { if (wifiReason <= 24) return wifiReasons[wifiReason]; if (wifiReason >= 200 && wifiReason <= 201) return wifiReasons[wifiReason-200+24]; @@ -86,6 +88,7 @@ static void ICACHE_FLASH_ATTR wifiHandleEventCb(System_Event_t *evt) { default: break; } + if (wifiStatusCb) (*wifiStatusCb)(wifiState); } // ===== wifi scanning @@ -478,7 +481,7 @@ int ICACHE_FLASH_ATTR printWifiInfo(char *buff) { char *mode = wifiMode[op]; char *status = "unknown"; int st = wifi_station_get_connect_status(); - if (st > 0 && st < sizeof(connStatuses)) status = connStatuses[st]; + if (st >= 0 && st < sizeof(connStatuses)) status = connStatuses[st]; int p = wifi_get_phy_mode(); char *phy = wifiPhy[p&3]; char *warn = wifiWarn[op]; diff --git a/user/cgiwifi.h b/user/cgiwifi.h index 557c31a..c78a793 100644 --- a/user/cgiwifi.h +++ b/user/cgiwifi.h @@ -14,4 +14,6 @@ int cgiWiFiConnStatus(HttpdConnData *connData); int cgiWiFiSpecial(HttpdConnData *connData); void wifiInit(void); +extern void (*wifiStatusCb)(uint8_t); // callback when wifi status changes + #endif From 1261344e2ec3c2306413f8ddb80fb2ba41383ab6 Mon Sep 17 00:00:00 2001 From: Benjamin Runnels Date: Mon, 24 Aug 2015 13:20:13 -0500 Subject: [PATCH 24/24] Added gitignores --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 7b7ee90..8a769b0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ html_compressed/ esp-link.tgz tve-patch/ yui +.localhistory/ +esp-link.sdf +espfs/mkespfsimage/mman-win32/libmman.a +espfs/mkespfsimage/mman-win32/mman.o