wirtz
/
esp-link
mirror of https://github.com/jeelabs/esp-link.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Added windows build support

Browse Source
pull/36/head
Benjamin Runnels 9 years ago
parent b48614c373
commit 5cec16c3fd
37 changed files with 5285 additions and 37 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 5
      .gitignore
  2. 5
      Makefile
  3. 22
      esp-link.sln
  4. 175
      esp-link.vcxproj
  5. 11
      espfs/espfs.c
  6. 14
      espfs/heatshrink/LICENSE
  7. 49
      espfs/heatshrink/Makefile
  8. 49
      espfs/heatshrink/README.md
  9. 52
      espfs/heatshrink/dec_sm.dot
  10. 51
      espfs/heatshrink/enc_sm.dot
  11. 591
      espfs/heatshrink/greatest.h
  12. 446
      espfs/heatshrink/heatshrink.c
  13. 20
      espfs/heatshrink/heatshrink_common.h
  14. 24
      espfs/heatshrink/heatshrink_config.h
  15. 382
      espfs/heatshrink/heatshrink_decoder.c
  16. 101
      espfs/heatshrink/heatshrink_decoder.h
  17. 650
      espfs/heatshrink/heatshrink_encoder.c
  18. 109
      espfs/heatshrink/heatshrink_encoder.h
  19. 999
      espfs/heatshrink/test_heatshrink_dynamic.c
  20. 521
      espfs/heatshrink/test_heatshrink_dynamic_theft.c
  21. 167
      espfs/heatshrink/test_heatshrink_static.c
  22. 2
      espfs/heatshrink_decoder.c
  23. 32
      espfs/mkespfsimage/Makefile
  24. 24
      espfs/mkespfsimage/Makefile.linux
  25. 33
      espfs/mkespfsimage/Makefile.windows
  26. 4
      espfs/mkespfsimage/build-linux.sh
  27. 7
      espfs/mkespfsimage/build-win32.sh
  28. 33
      espfs/mkespfsimage/espfsformat.h
  29. 2
      espfs/mkespfsimage/heatshrink_encoder.c
  30. 31
      espfs/mkespfsimage/main.c
  31. 48
      espfs/mkespfsimage/mman-win32/Makefile
  32. 11
      espfs/mkespfsimage/mman-win32/config.mak
  33. 157
      espfs/mkespfsimage/mman-win32/configure
  34. 180
      espfs/mkespfsimage/mman-win32/mman.c
  35. 55
      espfs/mkespfsimage/mman-win32/mman.h
  36. 235
      espfs/mkespfsimage/mman-win32/test.c
  37. 7
      espmake.cmd

5
.gitignore vendored
Unescape View File

@ -10,3 +10,8 @@ html_compressed/
esp-link.tgz esp-link.tgz
tve-patch/ tve-patch/
yui yui
espfs/mkespfsimage/mman-win32/mman.o
esp-link.opensdf
esp-link.sdf
espfs/mkespfsimage/mman-win32/libmman.a
.localhistory/

5
Makefile
Unescape View File

@ -289,6 +289,9 @@ $(BUILD_DIR):
wiflash: all wiflash: all
./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin ./wiflash $(ESP_HOSTNAME) $(FW_BASE)/user1.bin $(FW_BASE)/user2.bin
baseflash: all
$(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) write_flash 0x01000 $(FW_BASE)/user1.bin
flash: all flash: all
$(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \ $(Q) $(ESPTOOL) --port $(ESPPORT) --baud $(ESPBAUD) -fs $(ET_FS) -ff $(ET_FF) write_flash \
0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \ 0x00000 "$(SDK_BASE)/bin/boot_v1.4(b1).bin" 0x01000 $(FW_BASE)/user1.bin \
@ -296,7 +299,7 @@ flash: all
yui/$(YUI-COMPRESSOR): yui/$(YUI-COMPRESSOR):
$(Q) mkdir -p yui $(Q) mkdir -p yui
cd yui; wget https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) cd yui; wget --no-check-certificate https://github.com/yui/yuicompressor/releases/download/v2.4.8/$(YUI-COMPRESSOR) -O $(YUI-COMPRESSOR)
ifeq ("$(COMPRESS_W_YUI)","yes") ifeq ("$(COMPRESS_W_YUI)","yes")
$(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR) $(BUILD_BASE)/espfs_img.o: yui/$(YUI-COMPRESSOR)

22
esp-link.sln
Unescape View File

@ -0,0 +1,22 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2013
VisualStudioVersion = 12.0.31101.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "esp-link", "esp-link.vcxproj", "{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|ARM = Debug|ARM
Release|ARM = Release|ARM
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.ActiveCfg = Debug|ARM
{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Debug|ARM.Build.0 = Debug|ARM
{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.ActiveCfg = Release|ARM
{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}.Release|ARM.Build.0 = Release|ARM
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

175
esp-link.vcxproj
Unescape View File

@ -0,0 +1,175 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|ARM">
<Configuration>Debug</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|ARM">
<Configuration>Release</Configuration>
<Platform>ARM</Platform>
</ProjectConfiguration>
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{A92F0CAA-F89B-4F78-AD2A-A042429BD87F}</ProjectGuid>
<Keyword>MakeFileProj</Keyword>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup>
<ConfigurationType>Makefile</ConfigurationType>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'" Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<PropertyGroup>
<NMakeOutput />
<NMakePreprocessorDefinitions>__ets__;_STDINT_H;ICACHE_FLASH;__MINGW32__;__WIN32__</NMakePreprocessorDefinitions>
<NMakeIncludeSearchPath>.\mqtt\include;.\serial;.\user;.\espfs;.\httpd;.\include;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\x86_64-w64-mingw32;c:\tools\mingw64\x86_64-w64-mingw32\include\c++\backward;c:\tools\mingw64\x86_64-w64-mingw32\include\c++;c:\tools\mingw64\x86_64-w64-mingw32\include;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include-fixed;c:\tools\mingw64\lib\gcc\x86_64-w64-mingw32\4.8.3\include;c:\Espressif\sdk\include</NMakeIncludeSearchPath>
<ExecutablePath />
<ReferencePath />
<LibraryPath />
<LibraryWPath />
<ExcludePath />
<NMakeOutput />
<OutDir>bin</OutDir>
<IntDir>build</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">
<NMakeBuildCommandLine>espmake flash</NMakeBuildCommandLine>
<NMakeReBuildCommandLine>espmake clean all</NMakeReBuildCommandLine>
<NMakeCleanCommandLine>espmake clean</NMakeCleanCommandLine>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">
<NMakeBuildCommandLine>espmake baseflash</NMakeBuildCommandLine>
<NMakeReBuildCommandLine>espmake clean all</NMakeReBuildCommandLine>
<NMakeCleanCommandLine>espmake clean</NMakeCleanCommandLine>
</PropertyGroup>
<ItemDefinitionGroup>
<BuildLog>
<Path />
</BuildLog>
</ItemDefinitionGroup>
<ItemGroup>
<None Include="espfs\mkespfsimage\build-linux.sh" />
<None Include="espfs\mkespfsimage\build-win32.sh" />
<None Include="espfs\mkespfsimage\heatshrink_encoder.o" />
<None Include="espfs\mkespfsimage\main.o" />
<None Include="espfs\mkespfsimage\Makefile" />
<None Include="espfs\mkespfsimage\Makefile.linux" />
<None Include="espfs\mkespfsimage\Makefile.windows" />
<None Include="espfs\mkespfsimage\mman-win32\config.mak" />
<None Include="espfs\mkespfsimage\mman-win32\configure" />
<None Include="espfs\mkespfsimage\mman-win32\Makefile" />
<None Include="html-old\home.js" />
<None Include="espmake.cmd" />
<None Include="Makefile" />
<None Include="wiflash" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="espfs\espfs.c" />
<ClCompile Include="espfs\espfstest\main.c" />
<ClCompile Include="espfs\heatshrink_decoder.c" />
<ClCompile Include="espfs\mkespfsimage\heatshrink_encoder.c" />
<ClCompile Include="espfs\mkespfsimage\main.c" />
<ClCompile Include="espfs\mkespfsimage\mman-win32\mman.c" />
<ClCompile Include="espfs\mkespfsimage\mman-win32\test.c" />
<ClCompile Include="httpd\auth.c" />
<ClCompile Include="httpd\base64.c" />
<ClCompile Include="httpd\httpd.c" />
<ClCompile Include="httpd\httpdespfs.c" />
<ClCompile Include="mqtt\driver\uart.c" />
<ClCompile Include="mqtt\modules\config.c" />
<ClCompile Include="mqtt\modules\wifi.c" />
<ClCompile Include="mqtt\mqtt.c" />
<ClCompile Include="mqtt\mqtt\mqtt.c" />
<ClCompile Include="mqtt\mqtt\mqtt_msg.c" />
<ClCompile Include="mqtt\mqtt\proto.c" />
<ClCompile Include="mqtt\mqtt\queue.c" />
<ClCompile Include="mqtt\mqtt\ringbuf.c" />
<ClCompile Include="mqtt\mqtt\utils.c" />
<ClCompile Include="mqtt\mqtt_msg.c" />
<ClCompile Include="mqtt\proto.c" />
<ClCompile Include="mqtt\queue.c" />
<ClCompile Include="mqtt\ringbuf.c" />
<ClCompile Include="mqtt\user\user_main.c" />
<ClCompile Include="mqtt\utils.c" />
<ClCompile Include="serial\debugpin.c" />
<ClCompile Include="serial\serslip.c" />
<ClCompile Include="serial\console.c" />
<ClCompile Include="serial\crc16.c" />
<ClCompile Include="serial\serbridge.c" />
<ClCompile Include="serial\serled.c" />
<ClCompile Include="serial\softuart.c" />
<ClCompile Include="serial\tcpclient.c" />
<ClCompile Include="serial\uart.c" />
<ClCompile Include="user\cgi.c" />
<ClCompile Include="user\cgiflash.c" />
<ClCompile Include="user\cgipins.c" />
<ClCompile Include="user\cgitcp.c" />
<ClCompile Include="user\cgiwifi.c" />
<ClCompile Include="user\config.c" />
<ClCompile Include="user\log.c" />
<ClCompile Include="user\status.c" />
<ClCompile Include="user\user_main.c" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="espfs\espfs.h" />
<ClInclude Include="espfs\espfsformat.h" />
<ClInclude Include="espfs\heatshrink_config_custom.h" />
<ClInclude Include="espfs\mkespfsimage\espfsformat.h" />
<ClInclude Include="espfs\mkespfsimage\mman-win32\mman.h" />
<ClInclude Include="httpd\auth.h" />
<ClInclude Include="httpd\base64.h" />
<ClInclude Include="httpd\httpd.h" />
<ClInclude Include="httpd\httpdespfs.h" />
<ClInclude Include="include\esp8266.h" />
<ClInclude Include="include\espmissingincludes.h" />
<ClInclude Include="include\uart_hw.h" />
<ClInclude Include="include\user_config.h" />
<ClInclude Include="mqtt\include\debug.h" />
<ClInclude Include="mqtt\include\driver\uart.h" />
<ClInclude Include="mqtt\include\driver\uart_register.h" />
<ClInclude Include="mqtt\include\mqtt.h" />
<ClInclude Include="mqtt\include\mqtt_msg.h" />
<ClInclude Include="mqtt\include\proto.h" />
<ClInclude Include="mqtt\include\queue.h" />
<ClInclude Include="mqtt\include\ringbuf.h" />
<ClInclude Include="mqtt\include\typedef.h" />
<ClInclude Include="mqtt\include\user_config.h" />
<ClInclude Include="mqtt\include\utils.h" />
<ClInclude Include="mqtt\modules\include\config.h" />
<ClInclude Include="mqtt\modules\include\wifi.h" />
<ClInclude Include="mqtt\mqtt\include\debug.h" />
<ClInclude Include="mqtt\mqtt\include\mqtt.h" />
<ClInclude Include="mqtt\mqtt\include\mqtt_msg.h" />
<ClInclude Include="mqtt\mqtt\include\proto.h" />
<ClInclude Include="mqtt\mqtt\include\queue.h" />
<ClInclude Include="mqtt\mqtt\include\ringbuf.h" />
<ClInclude Include="mqtt\mqtt\include\typedef.h" />
<ClInclude Include="mqtt\mqtt\include\utils.h" />
<ClInclude Include="serial\debugpin.h" />
<ClInclude Include="serial\serslip.h" />
<ClInclude Include="serial\console.h" />
<ClInclude Include="serial\crc16.h" />
<ClInclude Include="serial\serbridge.h" />
<ClInclude Include="serial\serled.h" />
<ClInclude Include="serial\softuart.h" />
<ClInclude Include="serial\tcpclient.h" />
<ClInclude Include="serial\uart.h" />
<ClInclude Include="user\cgi.h" />
<ClInclude Include="user\cgiflash.h" />
<ClInclude Include="user\cgipins.h" />
<ClInclude Include="user\cgitcp.h" />
<ClInclude Include="user\cgiwifi.h" />
<ClInclude Include="user\config.h" />
<ClInclude Include="user\log.h" />
<ClInclude Include="user\status.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>

11
espfs/espfs.c
Unescape View File

@ -42,11 +42,12 @@ It's written for use with httpd, but doesn't need to be used as such.
#ifdef ESPFS_HEATSHRINK #ifdef ESPFS_HEATSHRINK
#include "heatshrink_config_custom.h" #include "heatshrink_config_custom.h"
#include "heatshrink_decoder.h" #include "heatshrink/heatshrink_decoder.h"
#endif #endif
static char* espFsData = NULL; static char* espFsData = NULL;
struct EspFsFile { struct EspFsFile {
EspFsHeader *header; EspFsHeader *header;
char decompressor; char decompressor;
@ -146,7 +147,7 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
return NULL; return NULL;
} }
if (h.flags&FLAG_LASTFILE) { if (h.flags&FLAG_LASTFILE) {
//os_printf("End of image.\n"); os_printf("End of image.\n");
return NULL; return NULL;
} }
//Grab the name of the file. //Grab the name of the file.
@ -158,7 +159,7 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
//Yay, this is the file we need! //Yay, this is the file we need!
p+=h.nameLen; //Skip to content. p+=h.nameLen; //Skip to content.
r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem r=(EspFsFile *)os_malloc(sizeof(EspFsFile)); //Alloc file desc mem
//os_printf("Alloc %p[%d]\n", r, sizeof(EspFsFile)); // os_printf("Alloc %p\n", r);
if (r==NULL) return NULL; if (r==NULL) return NULL;
r->header=(EspFsHeader *)hpos; r->header=(EspFsHeader *)hpos;
r->decompressor=h.compression; r->decompressor=h.compression;
@ -175,7 +176,7 @@ EspFsFile ICACHE_FLASH_ATTR *espFsOpen(char *fileName) {
//Decoder params are stored in 1st byte. //Decoder params are stored in 1st byte.
memcpyAligned(&parm, r->posComp, 1); memcpyAligned(&parm, r->posComp, 1);
r->posComp++; r->posComp++;
//os_printf("Heatshrink compressed file; decode parms = %x\n", parm); os_printf("Heatshrink compressed file; decode parms = %x\n", parm);
dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf); dec=heatshrink_decoder_alloc(16, (parm>>4)&0xf, parm&0xf);
r->decompData=dec; r->decompData=dec;
#endif #endif
@ -265,7 +266,7 @@ void ICACHE_FLASH_ATTR espFsClose(EspFsFile *fh) {
// os_printf("Freed %p\n", dec); // os_printf("Freed %p\n", dec);
} }
#endif #endif
//os_printf("Freed %p\n", fh); // os_printf("Freed %p\n", fh);
os_free(fh); os_free(fh);
} }

14
espfs/heatshrink/LICENSE
Unescape View File

@ -0,0 +1,14 @@
Copyright (c) 2013, Scott Vokes <scott.vokes@atomicobject.com>
All rights reserved.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

49
espfs/heatshrink/Makefile
Unescape View File

@ -0,0 +1,49 @@
PROJECT = heatshrink
#OPTIMIZE = -O0
#OPTIMIZE = -Os
OPTIMIZE = -O3
WARN = -Wall -Wextra -pedantic #-Werror
CFLAGS += -std=c99 -g ${WARN} ${OPTIMIZE}
CFLAGS += -Wmissing-prototypes
CFLAGS += -Wstrict-prototypes
CFLAGS += -Wmissing-declarations
# If libtheft is available, build additional property-based tests.
# Uncomment these to use it in test_heatshrink_dynamic.
#CFLAGS += -DHEATSHRINK_HAS_THEFT
#LDFLAGS += -ltheft
all:
@echo "For tests, make test_heatshrink_dynamic (default) or change the"
@echo "config.h to disable static memory and build test_heatshrink_static."
@echo "For the standalone command-line tool, make heatshrink."
${PROJECT}: heatshrink.c
OBJS= heatshrink_encoder.o \
heatshrink_decoder.o \
heatshrink: ${OBJS}
test_heatshrink_dynamic: ${OBJS} test_heatshrink_dynamic_theft.o
test_heatshrink_static: ${OBJS}
*.o: Makefile heatshrink_config.h
heatshrink_decoder.o: heatshrink_decoder.h heatshrink_common.h
heatshrink_encoder.o: heatshrink_encoder.h heatshrink_common.h
tags: TAGS
TAGS:
etags *.[ch]
diagrams: dec_sm.png enc_sm.png
dec_sm.png: dec_sm.dot
dot -o $@ -Tpng $<
enc_sm.png: enc_sm.dot
dot -o $@ -Tpng $<
clean:
rm -f ${PROJECT} test_heatshrink_{dynamic,static} *.o *.core {dec,enc}_sm.png TAGS

49
espfs/heatshrink/README.md
Unescape View File

@ -0,0 +1,49 @@
# heatshrink
A data compression/decompression library for embedded/real-time systems.
## Key Features:
- **Low memory usage (as low as 50 bytes)**
It is useful for some cases with less than 50 bytes, and useful
for many general cases with < 300 bytes.
- **Incremental, bounded CPU use**
You can chew on input data in arbitrarily tiny bites.
This is a useful property in hard real-time environments.
- **Can use either static or dynamic memory allocation**
The library doesn't impose any constraints on memory management.
- **ISC license**
You can use it freely, even for commercial purposes.
## Getting Started:
There is a standalone command-line program, `heatshrink`, but the
encoder and decoder can also be used as libraries, independent of each
other. To do so, copy `heatshrink_common.h`, `heatshrink_config.h`, and
either `heatshrink_encoder.c` or `heatshrink_decoder.c` (and their
respective header) into your project.
Dynamic allocation is used by default, but in an embedded context, you
probably want to statically allocate the encoder/decoder. Set
`HEATSHRINK_DYNAMIC_ALLOC` to 0 in `heatshrink_config.h`.
## More Information and Benchmarks:
heatshrink is based on [LZSS], since it's particularly suitable for
compression in small amounts of memory. It can use an optional, small
[index] to make compression significantly faster, but otherwise can run
in under 100 bytes of memory. The index currently adds 2^(window size+1)
bytes to memory usage for compression, and temporarily allocates 512
bytes on the stack during index construction.
For more information, see the [blog post] for an overview, and the
`heatshrink_encoder.h` / `heatshrink_decoder.h` header files for API
documentation.
[blog post]: http://spin.atomicobject.com/2013/03/14/heatshrink-embedded-data-compression/
[index]: http://spin.atomicobject.com/2014/01/13/lightweight-indexing-for-embedded-systems/
[LZSS]: http://en.wikipedia.org/wiki/Lempel-Ziv-Storer-Szymanski
## Build Status
[![Build Status](https://travis-ci.org/atomicobject/heatshrink.png)](http://travis-ci.org/atomicobject/heatshrink)

52
espfs/heatshrink/dec_sm.dot
Unescape View File

@ -0,0 +1,52 @@
digraph {
graph [label="Decoder state machine", labelloc="t"]
Start [style="invis", shape="point"]
empty
input_available
yield_literal
backref_index_msb
backref_index_lsb
backref_count_msb
backref_count_lsb
yield_backref
check_for_more_input
done [peripheries=2]
empty->input_available [label="sink()", color="blue", weight=10]
Start->empty
input_available->yield_literal [label="pop 1-bit"]
input_available->backref_index_msb [label="pop 0-bit", weight=10]
input_available->backref_index_lsb [label="pop 0-bit, index <8 bits", weight=10]
yield_literal->yield_literal [label="sink()", color="blue"]
yield_literal->yield_literal [label="poll()", color="red"]
yield_literal->check_for_more_input [label="poll(), done", color="red"]
backref_index_msb->backref_index_msb [label="sink()", color="blue"]
backref_index_msb->backref_index_lsb [label="pop index, upper bits", weight=10]
backref_index_msb->done [label="finish()", color="blue"]
backref_index_lsb->backref_index_lsb [label="sink()", color="blue"]
backref_index_lsb->backref_count_msb [label="pop index, lower bits", weight=10]
backref_index_lsb->backref_count_lsb [label="pop index, count <=8 bits", weight=10]
backref_index_lsb->done [label="finish()", color="blue"]
backref_count_msb->backref_count_msb [label="sink()", color="blue"]
backref_count_msb->backref_count_lsb [label="pop count, upper bits", weight=10]
backref_count_msb->done [label="finish()", color="blue"]
backref_count_lsb->backref_count_lsb [label="sink()", color="blue"]
backref_count_lsb->yield_backref [label="pop count, lower bits", weight=10]
backref_count_lsb->done [label="finish()", color="blue"]
yield_backref->yield_backref [label="sink()", color="blue"]
yield_backref->yield_backref [label="poll()", color="red"]
yield_backref->check_for_more_input [label="poll(), done",
color="red", weight=10]
check_for_more_input->empty [label="no"]
check_for_more_input->input_available [label="yes"]
empty->done [label="finish()", color="blue"]
}

51
espfs/heatshrink/enc_sm.dot
Unescape View File

@ -0,0 +1,51 @@
digraph {
graph [label="Encoder state machine", labelloc="t"]
start [style="invis", shape="point"]
not_full
filled
search
yield_tag_bit
yield_literal
yield_br_length
yield_br_index
save_backlog
flush_bits
done [peripheries=2]
start->not_full [label="start"]
not_full->not_full [label="sink(), not full", color="blue"]
not_full->filled [label="sink(), buffer is full", color="blue"]
not_full->filled [label="finish(), set is_finished", color="blue"]
filled->search [label="indexing (if any)"]
search->search [label="step"]
search->yield_tag_bit [label="literal"]
search->yield_tag_bit [label="match found"]
search->save_backlog [label="input exhausted"]
yield_tag_bit->yield_tag_bit [label="poll(), full buf", color="red"]
yield_tag_bit->yield_literal [label="poll(), literal", color="red"]
yield_tag_bit->yield_br_index [label="poll(), no literal", color="red"]
yield_tag_bit->flush_bits [label="finishing, no literal"]
yield_literal->yield_literal [label="poll(), full buf", color="red"]
yield_literal->search [label="poll(), no match", color="red"]
yield_literal->yield_tag_bit [label="poll(), match", color="red"]
yield_literal->flush_bits [label="poll(), final literal", color="red"]
yield_br_index->yield_br_index [label="poll(), full buf", color="red"]
yield_br_index->yield_br_length [label="poll()", color="red"]
yield_br_length->yield_br_length [label="poll(), full buf", color="red"]
yield_br_length->search [label="done"]
save_backlog->flush_bits [label="finishing, no literal"]
save_backlog->yield_tag_bit [label="finishing, literal"]
save_backlog->not_full [label="expect more input"]
flush_bits->flush_bits [label="poll(), full buf", color="red"]
flush_bits->done [label="poll(), flushed", color="red"]
flush_bits->done [label="no more output"]
}

591
espfs/heatshrink/greatest.h
Unescape View File

@ -0,0 +1,591 @@
/*
* Copyright (c) 2011 Scott Vokes <vokes.s@gmail.com>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
#ifndef GREATEST_H
#define GREATEST_H
#define GREATEST_VERSION_MAJOR 0
#define GREATEST_VERSION_MINOR 9
#define GREATEST_VERSION_PATCH 3
/* A unit testing system for C, contained in 1 file.
* It doesn't use dynamic allocation or depend on anything
* beyond ANSI C89. */
/*********************************************************************
* Minimal test runner template
*********************************************************************/
#if 0
#include "greatest.h"
TEST foo_should_foo() {
PASS();
}
static void setup_cb(void *data) {
printf("setup callback for each test case\n");
}
static void teardown_cb(void *data) {
printf("teardown callback for each test case\n");
}
SUITE(suite) {
/* Optional setup/teardown callbacks which will be run before/after
* every test case in the suite.
* Cleared when the suite finishes. */
SET_SETUP(setup_cb, voidp_to_callback_data);
SET_TEARDOWN(teardown_cb, voidp_to_callback_data);
RUN_TEST(foo_should_foo);
}
/* Add all the definitions that need to be in the test runner's main file. */
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */
RUN_SUITE(suite);
GREATEST_MAIN_END(); /* display results */
}
#endif
/*********************************************************************/
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
/***********
* Options *
***********/
/* Default column width for non-verbose output. */
#ifndef GREATEST_DEFAULT_WIDTH
#define GREATEST_DEFAULT_WIDTH 72
#endif
/* FILE *, for test logging. */
#ifndef GREATEST_STDOUT
#define GREATEST_STDOUT stdout
#endif
/* Remove GREATEST_ prefix from most commonly used symbols? */
#ifndef GREATEST_USE_ABBREVS
#define GREATEST_USE_ABBREVS 1
#endif
/*********
* Types *
*********/
/* Info for the current running suite. */
typedef struct greatest_suite_info {
unsigned int tests_run;
unsigned int passed;
unsigned int failed;
unsigned int skipped;
/* timers, pre/post running suite and individual tests */
clock_t pre_suite;
clock_t post_suite;
clock_t pre_test;
clock_t post_test;
} greatest_suite_info;
/* Type for a suite function. */
typedef void (greatest_suite_cb)(void);
/* Types for setup/teardown callbacks. If non-NULL, these will be run
* and passed the pointer to their additional data. */
typedef void (greatest_setup_cb)(void *udata);
typedef void (greatest_teardown_cb)(void *udata);
typedef enum {
GREATEST_FLAG_VERBOSE = 0x01,
GREATEST_FLAG_FIRST_FAIL = 0x02,
GREATEST_FLAG_LIST_ONLY = 0x04
} GREATEST_FLAG;
typedef struct greatest_run_info {
unsigned int flags;
unsigned int tests_run; /* total test count */
/* Overall pass/fail/skip counts. */
unsigned int passed;
unsigned int failed;
unsigned int skipped;
/* currently running test suite */
greatest_suite_info suite;
/* info to print about the most recent failure */
const char *fail_file;
unsigned int fail_line;
const char *msg;
/* current setup/teardown hooks and userdata */
greatest_setup_cb *setup;
void *setup_udata;
greatest_teardown_cb *teardown;
void *teardown_udata;
/* formatting info for ".....s...F"-style output */
unsigned int col;
unsigned int width;
/* only run a specific suite or test */
char *suite_filter;
char *test_filter;
/* overall timers */
clock_t begin;
clock_t end;
} greatest_run_info;
/* Global var for the current testing context.
* Initialized by GREATEST_MAIN_DEFS(). */
extern greatest_run_info greatest_info;
/**********************
* Exported functions *
**********************/
void greatest_do_pass(const char *name);
void greatest_do_fail(const char *name);
void greatest_do_skip(const char *name);
int greatest_pre_test(const char *name);
void greatest_post_test(const char *name, int res);
void greatest_usage(const char *name);
void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata);
void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, void *udata);
/**********
* Macros *
**********/
/* Define a suite. */
#define GREATEST_SUITE(NAME) void NAME(void)
/* Start defining a test function.
* The arguments are not included, to allow parametric testing. */
#define GREATEST_TEST static int
/* Run a suite. */
#define GREATEST_RUN_SUITE(S_NAME) greatest_run_suite(S_NAME, #S_NAME)
/* Run a test in the current suite. */
#define GREATEST_RUN_TEST(TEST) \
do { \
if (greatest_pre_test(#TEST) == 1) { \
int res = TEST(); \
greatest_post_test(#TEST, res); \
} else if (GREATEST_LIST_ONLY()) { \
fprintf(GREATEST_STDOUT, " %s\n", #TEST); \
} \
} while (0)
/* Run a test in the current suite with one void* argument,
* which can be a pointer to a struct with multiple arguments. */
#define GREATEST_RUN_TEST1(TEST, ENV) \
do { \
if (greatest_pre_test(#TEST) == 1) { \
int res = TEST(ENV); \
greatest_post_test(#TEST, res); \
} else if (GREATEST_LIST_ONLY()) { \
fprintf(GREATEST_STDOUT, " %s\n", #TEST); \
} \
} while (0)
/* If __VA_ARGS__ (C99) is supported, allow parametric testing
* without needing to manually manage the argument struct. */
#if __STDC_VERSION__ >= 19901L
#define GREATEST_RUN_TESTp(TEST, ...) \
do { \
if (greatest_pre_test(#TEST) == 1) { \
int res = TEST(__VA_ARGS__); \
greatest_post_test(#TEST, res); \
} else if (GREATEST_LIST_ONLY()) { \
fprintf(GREATEST_STDOUT, " %s\n", #TEST); \
} \
} while (0)
#endif
/* Check if the test runner is in verbose mode. */
#define GREATEST_IS_VERBOSE() (greatest_info.flags & GREATEST_FLAG_VERBOSE)
#define GREATEST_LIST_ONLY() (greatest_info.flags & GREATEST_FLAG_LIST_ONLY)
#define GREATEST_FIRST_FAIL() (greatest_info.flags & GREATEST_FLAG_FIRST_FAIL)
#define GREATEST_FAILURE_ABORT() (greatest_info.suite.failed > 0 && GREATEST_FIRST_FAIL())
/* Message-less forms. */
#define GREATEST_PASS() GREATEST_PASSm(NULL)
#define GREATEST_FAIL() GREATEST_FAILm(NULL)
#define GREATEST_SKIP() GREATEST_SKIPm(NULL)
#define GREATEST_ASSERT(COND) GREATEST_ASSERTm(#COND, COND)
#define GREATEST_ASSERT_FALSE(COND) GREATEST_ASSERT_FALSEm(#COND, COND)
#define GREATEST_ASSERT_EQ(EXP, GOT) GREATEST_ASSERT_EQm(#EXP " != " #GOT, EXP, GOT)
#define GREATEST_ASSERT_STR_EQ(EXP, GOT) GREATEST_ASSERT_STR_EQm(#EXP " != " #GOT, EXP, GOT)
/* The following forms take an additional message argument first,
* to be displayed by the test runner. */
/* Fail if a condition is not true, with message. */
#define GREATEST_ASSERTm(MSG, COND) \
do { \
greatest_info.msg = MSG; \
greatest_info.fail_file = __FILE__; \
greatest_info.fail_line = __LINE__; \
if (!(COND)) return -1; \
greatest_info.msg = NULL; \
} while (0)
#define GREATEST_ASSERT_FALSEm(MSG, COND) \
do { \
greatest_info.msg = MSG; \
greatest_info.fail_file = __FILE__; \
greatest_info.fail_line = __LINE__; \
if ((COND)) return -1; \
greatest_info.msg = NULL; \
} while (0)
#define GREATEST_ASSERT_EQm(MSG, EXP, GOT) \
do { \
greatest_info.msg = MSG; \
greatest_info.fail_file = __FILE__; \
greatest_info.fail_line = __LINE__; \
if ((EXP) != (GOT)) return -1; \
greatest_info.msg = NULL; \
} while (0)
#define GREATEST_ASSERT_STR_EQm(MSG, EXP, GOT) \
do { \
const char *exp_s = (EXP); \
const char *got_s = (GOT); \
greatest_info.msg = MSG; \
greatest_info.fail_file = __FILE__; \
greatest_info.fail_line = __LINE__; \
if (0 != strcmp(exp_s, got_s)) { \
fprintf(GREATEST_STDOUT, \
"Expected:\n####\n%s\n####\n", exp_s); \
fprintf(GREATEST_STDOUT, \
"Got:\n####\n%s\n####\n", got_s); \
return -1; \
} \
greatest_info.msg = NULL; \
} while (0)
#define GREATEST_PASSm(MSG) \
do { \
greatest_info.msg = MSG; \
return 0; \
} while (0)
#define GREATEST_FAILm(MSG) \
do { \
greatest_info.fail_file = __FILE__; \
greatest_info.fail_line = __LINE__; \
greatest_info.msg = MSG; \
return -1; \
} while (0)
#define GREATEST_SKIPm(MSG) \
do { \
greatest_info.msg = MSG; \
return 1; \
} while (0)
#define GREATEST_SET_TIME(NAME) \
NAME = clock(); \
if (NAME == (clock_t) -1) { \
fprintf(GREATEST_STDOUT, \
"clock error: %s\n", #NAME); \
exit(EXIT_FAILURE); \
}
#define GREATEST_CLOCK_DIFF(C1, C2) \
fprintf(GREATEST_STDOUT, " (%lu ticks, %.3f sec)", \
(long unsigned int) (C2) - (C1), \
(double)((C2) - (C1)) / (1.0 * (double)CLOCKS_PER_SEC)) \
/* Include several function definitions in the main test file. */
#define GREATEST_MAIN_DEFS() \
\
/* Is FILTER a subset of NAME? */ \
static int greatest_name_match(const char *name, \
const char *filter) { \
size_t offset = 0; \
size_t filter_len = strlen(filter); \
while (name[offset] != '\0') { \
if (name[offset] == filter[0]) { \
if (0 == strncmp(&name[offset], filter, filter_len)) { \
return 1; \
} \
} \
offset++; \
} \
\
return 0; \
} \
\
int greatest_pre_test(const char *name) { \
if (!GREATEST_LIST_ONLY() \
&& (!GREATEST_FIRST_FAIL() || greatest_info.suite.failed == 0) \
&& (greatest_info.test_filter == NULL || \
greatest_name_match(name, greatest_info.test_filter))) { \
GREATEST_SET_TIME(greatest_info.suite.pre_test); \
if (greatest_info.setup) { \
greatest_info.setup(greatest_info.setup_udata); \
} \
return 1; /* test should be run */ \
} else { \
return 0; /* skipped */ \
} \
} \
\
void greatest_post_test(const char *name, int res) { \
GREATEST_SET_TIME(greatest_info.suite.post_test); \
if (greatest_info.teardown) { \
void *udata = greatest_info.teardown_udata; \
greatest_info.teardown(udata); \
} \
\
if (res < 0) { \
greatest_do_fail(name); \
} else if (res > 0) { \
greatest_do_skip(name); \
} else if (res == 0) { \
greatest_do_pass(name); \
} \
greatest_info.suite.tests_run++; \
greatest_info.col++; \
if (GREATEST_IS_VERBOSE()) { \
GREATEST_CLOCK_DIFF(greatest_info.suite.pre_test, \
greatest_info.suite.post_test); \
fprintf(GREATEST_STDOUT, "\n"); \
} else if (greatest_info.col % greatest_info.width == 0) { \
fprintf(GREATEST_STDOUT, "\n"); \
greatest_info.col = 0; \
} \
if (GREATEST_STDOUT == stdout) fflush(stdout); \
} \
\
static void greatest_run_suite(greatest_suite_cb *suite_cb, \
const char *suite_name) { \
if (greatest_info.suite_filter && \
!greatest_name_match(suite_name, greatest_info.suite_filter)) \
return; \
if (GREATEST_FIRST_FAIL() && greatest_info.failed > 0) return; \
greatest_info.suite.tests_run = 0; \
greatest_info.suite.failed = 0; \
greatest_info.suite.passed = 0; \
greatest_info.suite.skipped = 0; \
greatest_info.suite.pre_suite = 0; \
greatest_info.suite.post_suite = 0; \
greatest_info.suite.pre_test = 0; \
greatest_info.suite.post_test = 0; \
greatest_info.col = 0; \
fprintf(GREATEST_STDOUT, "\n* Suite %s:\n", suite_name); \
GREATEST_SET_TIME(greatest_info.suite.pre_suite); \
suite_cb(); \
GREATEST_SET_TIME(greatest_info.suite.post_suite); \
if (greatest_info.suite.tests_run > 0) { \
fprintf(GREATEST_STDOUT, \
"\n%u tests - %u pass, %u fail, %u skipped", \
greatest_info.suite.tests_run, \
greatest_info.suite.passed, \
greatest_info.suite.failed, \
greatest_info.suite.skipped); \
GREATEST_CLOCK_DIFF(greatest_info.suite.pre_suite, \
greatest_info.suite.post_suite); \
fprintf(GREATEST_STDOUT, "\n"); \
} \
greatest_info.setup = NULL; \
greatest_info.setup_udata = NULL; \
greatest_info.teardown = NULL; \
greatest_info.teardown_udata = NULL; \
greatest_info.passed += greatest_info.suite.passed; \
greatest_info.failed += greatest_info.suite.failed; \
greatest_info.skipped += greatest_info.suite.skipped; \
greatest_info.tests_run += greatest_info.suite.tests_run; \
} \
\
void greatest_do_pass(const char *name) { \
if (GREATEST_IS_VERBOSE()) { \
fprintf(GREATEST_STDOUT, "PASS %s: %s", \
name, greatest_info.msg ? greatest_info.msg : ""); \
} else { \
fprintf(GREATEST_STDOUT, "."); \
} \
greatest_info.suite.passed++; \
} \
\
void greatest_do_fail(const char *name) { \
if (GREATEST_IS_VERBOSE()) { \
fprintf(GREATEST_STDOUT, \
"FAIL %s: %s (%s:%u)", \
name, greatest_info.msg ? greatest_info.msg : "", \
greatest_info.fail_file, greatest_info.fail_line); \
} else { \
fprintf(GREATEST_STDOUT, "F"); \
/* add linebreak if in line of '.'s */ \
if (greatest_info.col % greatest_info.width != 0) \
fprintf(GREATEST_STDOUT, "\n"); \
greatest_info.col = 0; \
fprintf(GREATEST_STDOUT, "FAIL %s: %s (%s:%u)\n", \
name, \
greatest_info.msg ? greatest_info.msg : "", \
greatest_info.fail_file, greatest_info.fail_line); \
} \
greatest_info.suite.failed++; \
} \
\
void greatest_do_skip(const char *name) { \
if (GREATEST_IS_VERBOSE()) { \
fprintf(GREATEST_STDOUT, "SKIP %s: %s", \
name, \
greatest_info.msg ? \
greatest_info.msg : "" ); \
} else { \
fprintf(GREATEST_STDOUT, "s"); \
} \
greatest_info.suite.skipped++; \
} \
\
void greatest_usage(const char *name) { \
fprintf(GREATEST_STDOUT, \
"Usage: %s [-hlfv] [-s SUITE] [-t TEST]\n" \
" -h print this Help\n" \
" -l List suites and their tests, then exit\n" \
" -f Stop runner after first failure\n" \
" -v Verbose output\n" \
" -s SUITE only run suite named SUITE\n" \
" -t TEST only run test named TEST\n", \
name); \
} \
\
void GREATEST_SET_SETUP_CB(greatest_setup_cb *cb, void *udata) { \
greatest_info.setup = cb; \
greatest_info.setup_udata = udata; \
} \
\
void GREATEST_SET_TEARDOWN_CB(greatest_teardown_cb *cb, \
void *udata) { \
greatest_info.teardown = cb; \
greatest_info.teardown_udata = udata; \
} \
\
greatest_run_info greatest_info
/* Handle command-line arguments, etc. */
#define GREATEST_MAIN_BEGIN() \
do { \
int i = 0; \
memset(&greatest_info, 0, sizeof(greatest_info)); \
if (greatest_info.width == 0) { \
greatest_info.width = GREATEST_DEFAULT_WIDTH; \
} \
for (i = 1; i < argc; i++) { \
if (0 == strcmp("-t", argv[i])) { \
if (argc <= i + 1) { \
greatest_usage(argv[0]); \
exit(EXIT_FAILURE); \
} \
greatest_info.test_filter = argv[i+1]; \
i++; \
} else if (0 == strcmp("-s", argv[i])) { \
if (argc <= i + 1) { \
greatest_usage(argv[0]); \
exit(EXIT_FAILURE); \
} \
greatest_info.suite_filter = argv[i+1]; \
i++; \
} else if (0 == strcmp("-f", argv[i])) { \
greatest_info.flags |= GREATEST_FLAG_FIRST_FAIL; \
} else if (0 == strcmp("-v", argv[i])) { \
greatest_info.flags |= GREATEST_FLAG_VERBOSE; \
} else if (0 == strcmp("-l", argv[i])) { \
greatest_info.flags |= GREATEST_FLAG_LIST_ONLY; \
} else if (0 == strcmp("-h", argv[i])) { \
greatest_usage(argv[0]); \
exit(EXIT_SUCCESS); \
} else { \
fprintf(GREATEST_STDOUT, \
"Unknown argument '%s'\n", argv[i]); \
greatest_usage(argv[0]); \
exit(EXIT_FAILURE); \
} \
} \
} while (0); \
GREATEST_SET_TIME(greatest_info.begin)
#define GREATEST_MAIN_END() \
do { \
if (!GREATEST_LIST_ONLY()) { \
GREATEST_SET_TIME(greatest_info.end); \
fprintf(GREATEST_STDOUT, \
"\nTotal: %u tests", greatest_info.tests_run); \
GREATEST_CLOCK_DIFF(greatest_info.begin, \
greatest_info.end); \
fprintf(GREATEST_STDOUT, "\n"); \
fprintf(GREATEST_STDOUT, \
"Pass: %u, fail: %u, skip: %u.\n", \
greatest_info.passed, \
greatest_info.failed, greatest_info.skipped); \
} \
return (greatest_info.failed > 0 \
? EXIT_FAILURE : EXIT_SUCCESS); \
} while (0)
/* Make abbreviations without the GREATEST_ prefix for the
* most commonly used symbols. */
#if GREATEST_USE_ABBREVS
#define TEST GREATEST_TEST
#define SUITE GREATEST_SUITE
#define RUN_TEST GREATEST_RUN_TEST
#define RUN_TEST1 GREATEST_RUN_TEST1
#define RUN_SUITE GREATEST_RUN_SUITE
#define ASSERT GREATEST_ASSERT
#define ASSERTm GREATEST_ASSERTm
#define ASSERT_FALSE GREATEST_ASSERT_FALSE
#define ASSERT_EQ GREATEST_ASSERT_EQ
#define ASSERT_STR_EQ GREATEST_ASSERT_STR_EQ
#define ASSERT_FALSEm GREATEST_ASSERT_FALSEm
#define ASSERT_EQm GREATEST_ASSERT_EQm
#define ASSERT_STR_EQm GREATEST_ASSERT_STR_EQm
#define PASS GREATEST_PASS
#define FAIL GREATEST_FAIL
#define SKIP GREATEST_SKIP
#define PASSm GREATEST_PASSm
#define FAILm GREATEST_FAILm
#define SKIPm GREATEST_SKIPm
#define SET_SETUP GREATEST_SET_SETUP_CB
#define SET_TEARDOWN GREATEST_SET_TEARDOWN_CB
#if __STDC_VERSION__ >= 19901L
#endif /* C99 */
#define RUN_TESTp GREATEST_RUN_TESTp
#endif /* USE_ABBREVS */
#endif

446
espfs/heatshrink/heatshrink.c
Unescape View File

@ -0,0 +1,446 @@
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <stdint.h>
#include <assert.h>
#include <string.h>
#include <err.h>
#include <fcntl.h>
#include "heatshrink_encoder.h"
#include "heatshrink_decoder.h"
#define DEF_WINDOW_SZ2 11
#define DEF_LOOKAHEAD_SZ2 4
#define DEF_DECODER_INPUT_BUFFER_SIZE 256
#define DEF_BUFFER_SIZE (64 * 1024)
#if 0
#define LOG(...) fprintf(stderr, __VA_ARGS__)
#else
#define LOG(...) /* NO-OP */
#endif
static const int version_major = HEATSHRINK_VERSION_MAJOR;
static const int version_minor = HEATSHRINK_VERSION_MINOR;
static const int version_patch = HEATSHRINK_VERSION_PATCH;
static const char author[] = HEATSHRINK_AUTHOR;
static const char url[] = HEATSHRINK_URL;
static void usage(void) {
fprintf(stderr, "heatshrink version %u.%u.%u by %s\n",
version_major, version_minor, version_patch, author);
fprintf(stderr, "Home page: %s\n\n", url);
fprintf(stderr,
"Usage:\n"
" heatshrink [-h] [-e|-d] [-v] [-w SIZE] [-l BITS] [IN_FILE] [OUT_FILE]\n"
"\n"
"heatshrink compresses or uncompresses byte streams using LZSS, and is\n"
"designed especially for embedded, low-memory, and/or hard real-time\n"
"systems.\n"
"\n"
" -h print help\n"
" -e encode (compress, default)\n"
" -d decode (uncompress)\n"
" -v verbose (print input & output sizes, compression ratio, etc.)\n"
"\n"
" -w SIZE Base-2 log of LZSS sliding window size\n"
"\n"
" A larger value allows searches a larger history of the data for repeated\n"
" patterns, potentially compressing more effectively, but will use\n"
" more memory and processing time.\n"
" Recommended default: -w 8 (embedded systems), -w 10 (elsewhere)\n"
" \n"
" -l BITS Number of bits used for back-reference lengths\n"
"\n"
" A larger value allows longer substitutions, but since all\n"
" back-references must use -w + -l bits, larger -w or -l can be\n"
" counterproductive if most patterns are small and/or local.\n"
" Recommended default: -l 4\n"
"\n"
" If IN_FILE or OUT_FILE are unspecified, they will default to\n"
" \"-\" for standard input and standard output, respectively.\n");
exit(1);
}
typedef enum { IO_READ, IO_WRITE, } IO_mode;
typedef enum { OP_ENC, OP_DEC, } Operation;
typedef struct {
int fd; /* file descriptor */
IO_mode mode;
size_t fill; /* fill index */
size_t read; /* read index */
size_t size;
size_t total;
uint8_t buf[];
} io_handle;
typedef struct {
uint8_t window_sz2;
uint8_t lookahead_sz2;
size_t decoder_input_buffer_size;
size_t buffer_size;
uint8_t verbose;
Operation cmd;
char *in_fname;
char *out_fname;
io_handle *in;
io_handle *out;
} config;
static void die(char *msg) {
fprintf(stderr, "%s\n", msg);
exit(EXIT_FAILURE);
}
static void report(config *cfg);
/* Open an IO handle. Returns NULL on error. */
static io_handle *handle_open(char *fname, IO_mode m, size_t buf_sz) {
io_handle *io = NULL;
io = malloc(sizeof(*io) + buf_sz);
if (io == NULL) { return NULL; }
memset(io, 0, sizeof(*io) + buf_sz);
io->fd = -1;
io->size = buf_sz;
io->mode = m;
if (m == IO_READ) {
if (0 == strcmp("-", fname)) {
io->fd = STDIN_FILENO;
} else {
io->fd = open(fname, O_RDONLY);
}
} else if (m == IO_WRITE) {
if (0 == strcmp("-", fname)) {
io->fd = STDOUT_FILENO;
} else {
io->fd = open(fname, O_WRONLY | O_CREAT | O_TRUNC /*| O_EXCL*/, 0644);
}
}
if (io->fd == -1) { /* failed to open */
free(io);
err(1, "open");
return NULL;
}
return io;
}
/* Read SIZE bytes from an IO handle and return a pointer to the content.
* BUF contains at least size_t bytes. Returns 0 on EOF, -1 on error. */
static ssize_t handle_read(io_handle *io, size_t size, uint8_t **buf) {
LOG("@ read %zd\n", size);
if (buf == NULL) { return -1; }
if (size > io->size) {
printf("size %zd, io->size %zd\n", size, io->size);
return -1;
}
if (io->mode != IO_READ) { return -1; }
size_t rem = io->fill - io->read;
if (rem >= size) {
*buf = &io->buf[io->read];
return size;
} else { /* read and replenish */
if (io->fd == -1) { /* already closed, return what we've got */
*buf = &io->buf[io->read];
return rem;
}
memmove(io->buf, &io->buf[io->read], rem);
io->fill -= io->read;
io->read = 0;
ssize_t read_sz = read(io->fd, &io->buf[io->fill], io->size - io->fill);
if (read_sz < 0) { err(1, "read"); }
io->total += read_sz;
if (read_sz == 0) { /* EOF */
if (close(io->fd) < 0) { err(1, "close"); }
io->fd = -1;
}
io->fill += read_sz;
*buf = io->buf;
return io->fill > size ? size : io->fill;
}
}
/* Drop the oldest SIZE bytes from the buffer. Returns <0 on error. */
static int handle_drop(io_handle *io, size_t size) {
LOG("@ drop %zd\n", size);
if (io->read + size <= io->fill) {
io->read += size;
} else {
return -1;
}
if (io->read == io->fill) {
io->read = 0;
io->fill = 0;
}
return 0;
}
/* Sink SIZE bytes from INPUT into the io handle. Returns the number of
* bytes written, or -1 on error. */
static ssize_t handle_sink(io_handle *io, size_t size, uint8_t *input) {
LOG("@ sink %zd\n", size);
if (size > io->size) { return -1; }
if (io->mode != IO_WRITE) { return -1; }
if (io->fill + size > io->size) {
ssize_t written = write(io->fd, io->buf, io->fill);
LOG("@ flushing %zd, wrote %zd\n", io->fill, written);
io->total += written;
if (written == -1) { err(1, "write"); }
memmove(io->buf, &io->buf[written], io->fill - written);
io->fill -= written;
}
memcpy(&io->buf[io->fill], input, size);
io->fill += size;
return size;
}
static void handle_close(io_handle *io) {
if (io->fd != -1) {
if (io->mode == IO_WRITE) {
ssize_t written = write(io->fd, io->buf, io->fill);
io->total += written;
LOG("@ close: flushing %zd, wrote %zd\n", io->fill, written);
if (written == -1) { err(1, "write"); }
}
close(io->fd);
io->fd = -1;
}
}
static void close_and_report(config *cfg) {
handle_close(cfg->in);
handle_close(cfg->out);
if (cfg->verbose) { report(cfg); }
free(cfg->in);
free(cfg->out);
}
static int encoder_sink_read(config *cfg, heatshrink_encoder *hse,
uint8_t *data, size_t data_sz) {
size_t out_sz = 4096;
uint8_t out_buf[out_sz];
memset(out_buf, 0, out_sz);
size_t sink_sz = 0;
size_t poll_sz = 0;
HSE_sink_res sres;
HSE_poll_res pres;
HSE_finish_res fres;
io_handle *out = cfg->out;
size_t sunk = 0;
do {
if (data_sz > 0) {
sres = heatshrink_encoder_sink(hse, &data[sunk], data_sz - sunk, &sink_sz);
if (sres < 0) { die("sink"); }
sunk += sink_sz;
}
do {
pres = heatshrink_encoder_poll(hse, out_buf, out_sz, &poll_sz);
if (pres < 0) { die("poll"); }
if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink");
} while (pres == HSER_POLL_MORE);
if (poll_sz == 0 && data_sz == 0) {
fres = heatshrink_encoder_finish(hse);
if (fres < 0) { die("finish"); }
if (fres == HSER_FINISH_DONE) { return 1; }
}
} while (sunk < data_sz);
return 0;
}
static int encode(config *cfg) {
uint8_t window_sz2 = cfg->window_sz2;
size_t window_sz = 1 << window_sz2;
heatshrink_encoder *hse = heatshrink_encoder_alloc(window_sz2, cfg->lookahead_sz2);
if (hse == NULL) { die("failed to init encoder: bad settings"); }
ssize_t read_sz = 0;
io_handle *in = cfg->in;
/* Process input until end of stream */
while (1) {
uint8_t *input = NULL;
read_sz = handle_read(in, window_sz, &input);
if (input == NULL) {
printf("handle read failure\n");
die("read");
}
if (read_sz < 0) { die("read"); }
/* Pass read to encoder and check if input is fully processed. */
if (encoder_sink_read(cfg, hse, input, read_sz)) break;
if (handle_drop(in, read_sz) < 0) { die("drop"); }
};
if (read_sz == -1) { err(1, "read"); }
heatshrink_encoder_free(hse);
close_and_report(cfg);
return 0;
}
static int decoder_sink_read(config *cfg, heatshrink_decoder *hsd,
uint8_t *data, size_t data_sz) {
io_handle *out = cfg->out;
size_t sink_sz = 0;
size_t poll_sz = 0;
size_t out_sz = 4096;
uint8_t out_buf[out_sz];
memset(out_buf, 0, out_sz);
HSD_sink_res sres;
HSD_poll_res pres;
HSD_finish_res fres;
size_t sunk = 0;
do {
if (data_sz > 0) {
sres = heatshrink_decoder_sink(hsd, &data[sunk], data_sz - sunk, &sink_sz);
if (sres < 0) { die("sink"); }
sunk += sink_sz;
}
do {
pres = heatshrink_decoder_poll(hsd, out_buf, out_sz, &poll_sz);
if (pres < 0) { die("poll"); }
if (handle_sink(out, poll_sz, out_buf) < 0) die("handle_sink");
} while (pres == HSDR_POLL_MORE);
if (data_sz == 0 && poll_sz == 0) {
fres = heatshrink_decoder_finish(hsd);
if (fres < 0) { die("finish"); }
if (fres == HSDR_FINISH_DONE) { return 1; }
}
} while (sunk < data_sz);
return 0;
}
static int decode(config *cfg) {
uint8_t window_sz2 = cfg->window_sz2;
size_t window_sz = 1 << window_sz2;
size_t ibs = cfg->decoder_input_buffer_size;
heatshrink_decoder *hsd = heatshrink_decoder_alloc(ibs,
window_sz2, cfg->lookahead_sz2);
if (hsd == NULL) { die("failed to init decoder"); }
ssize_t read_sz = 0;
io_handle *in = cfg->in;
HSD_finish_res fres;
/* Process input until end of stream */
while (1) {
uint8_t *input = NULL;
read_sz = handle_read(in, window_sz, &input);
if (input == NULL) {
printf("handle read failure\n");
die("read");
}
if (read_sz == 0) {
fres = heatshrink_decoder_finish(hsd);
if (fres < 0) { die("finish"); }
if (fres == HSDR_FINISH_DONE) break;
} else if (read_sz < 0) {
die("read");
} else {
if (decoder_sink_read(cfg, hsd, input, read_sz)) { break; }
if (handle_drop(in, read_sz) < 0) { die("drop"); }
}
}
if (read_sz == -1) { err(1, "read"); }
heatshrink_decoder_free(hsd);
close_and_report(cfg);
return 0;
}
static void report(config *cfg) {
size_t inb = cfg->in->total;
size_t outb = cfg->out->total;
fprintf(cfg->out->fd == STDOUT_FILENO ? stderr : stdout,
"%s %0.2f %%\t %zd -> %zd (-w %u -l %u)\n",
cfg->in_fname, 100.0 - (100.0 * outb) / inb, inb, outb,
cfg->window_sz2, cfg->lookahead_sz2);
}
static void proc_args(config *cfg, int argc, char **argv) {
cfg->window_sz2 = DEF_WINDOW_SZ2;
cfg->lookahead_sz2 = DEF_LOOKAHEAD_SZ2;
cfg->buffer_size = DEF_BUFFER_SIZE;
cfg->decoder_input_buffer_size = DEF_DECODER_INPUT_BUFFER_SIZE;
cfg->cmd = OP_ENC;
cfg->verbose = 0;
cfg->in_fname = "-";
cfg->out_fname = "-";
int a = 0;
while ((a = getopt(argc, argv, "hedi:w:l:v")) != -1) {
switch (a) {
case 'h': /* help */
usage();
case 'e': /* encode */
cfg->cmd = OP_ENC; break;
case 'd': /* decode */
cfg->cmd = OP_DEC; break;
case 'i': /* input buffer size */
cfg->decoder_input_buffer_size = atoi(optarg);
break;
case 'w': /* window bits */
cfg->window_sz2 = atoi(optarg);
break;
case 'l': /* lookahead bits */
cfg->lookahead_sz2 = atoi(optarg);
break;
case 'v': /* verbosity++ */
cfg->verbose++;
break;
case '?': /* unknown argument */
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc > 0) {
cfg->in_fname = argv[0];
argc--;
argv++;
}
if (argc > 0) { cfg->out_fname = argv[0]; }
}
int main(int argc, char **argv) {
config cfg;
memset(&cfg, 0, sizeof(cfg));
proc_args(&cfg, argc, argv);
if (0 == strcmp(cfg.in_fname, cfg.out_fname)
&& (0 != strcmp("-", cfg.in_fname))) {
printf("Refusing to overwrite file '%s' with itself.\n", cfg.in_fname);
exit(1);
}
cfg.in = handle_open(cfg.in_fname, IO_READ, cfg.buffer_size);
if (cfg.in == NULL) { die("Failed to open input file for read"); }
cfg.out = handle_open(cfg.out_fname, IO_WRITE, cfg.buffer_size);
if (cfg.out == NULL) { die("Failed to open output file for write"); }
if (cfg.cmd == OP_ENC) {
return encode(&cfg);
} else if (cfg.cmd == OP_DEC) {
return decode(&cfg);
} else {
usage();
}
}

20
espfs/heatshrink/heatshrink_common.h
Unescape View File

@ -0,0 +1,20 @@
#ifndef HEATSHRINK_H
#define HEATSHRINK_H
#define HEATSHRINK_AUTHOR "Scott Vokes <scott.vokes@atomicobject.com>"
#define HEATSHRINK_URL "https://github.com/atomicobject/heatshrink"
/* Version 0.3.1 */
#define HEATSHRINK_VERSION_MAJOR 0
#define HEATSHRINK_VERSION_MINOR 3
#define HEATSHRINK_VERSION_PATCH 1
#define HEATSHRINK_MIN_WINDOW_BITS 4
#define HEATSHRINK_MAX_WINDOW_BITS 15
#define HEATSHRINK_MIN_LOOKAHEAD_BITS 2
#define HEATSHRINK_LITERAL_MARKER 0x01
#define HEATSHRINK_BACKREF_MARKER 0x00
#endif

24
espfs/heatshrink/heatshrink_config.h
Unescape View File

@ -0,0 +1,24 @@
#ifndef HEATSHRINK_CONFIG_H
#define HEATSHRINK_CONFIG_H
/* Should functionality assuming dynamic allocation be used? */
#define HEATSHRINK_DYNAMIC_ALLOC 1
#if HEATSHRINK_DYNAMIC_ALLOC
/* Optional replacement of malloc/free */
#define HEATSHRINK_MALLOC(SZ) malloc(SZ)
#define HEATSHRINK_FREE(P, SZ) free(P)
#else
/* Required parameters for static configuration */
#define HEATSHRINK_STATIC_INPUT_BUFFER_SIZE 32
#define HEATSHRINK_STATIC_WINDOW_BITS 8
#define HEATSHRINK_STATIC_LOOKAHEAD_BITS 4
#endif
/* Turn on logging for debugging. */
#define HEATSHRINK_DEBUGGING_LOGS 0
/* Use indexing for faster compression. (This requires additional space.) */
#define HEATSHRINK_USE_INDEX 1
#endif

382
espfs/heatshrink/heatshrink_decoder.c
Unescape View File

@ -0,0 +1,382 @@
#include <stdlib.h>
#include <string.h>
#include "heatshrink_decoder.h"
/* States for the polling state machine. */
typedef enum {
HSDS_EMPTY, /* no input to process */
HSDS_INPUT_AVAILABLE, /* new input, completely unprocessed */
HSDS_YIELD_LITERAL, /* ready to yield literal byte */
HSDS_BACKREF_INDEX_MSB, /* most significant byte of index */
HSDS_BACKREF_INDEX_LSB, /* least significant byte of index */
HSDS_BACKREF_COUNT_MSB, /* most significant byte of count */
HSDS_BACKREF_COUNT_LSB, /* least significant byte of count */
HSDS_YIELD_BACKREF, /* ready to yield back-reference */
HSDS_CHECK_FOR_MORE_INPUT, /* check if input is exhausted */
} HSD_state;
#if HEATSHRINK_DEBUGGING_LOGS
#include <stdio.h>
#include <ctype.h>
#include <assert.h>
#define LOG(...) fprintf(stderr, __VA_ARGS__)
#define ASSERT(X) assert(X)
static const char *state_names[] = {
"empty",
"input_available",
"yield_literal",
"backref_index",
"backref_count",
"yield_backref",
"check_for_more_input",
};
#else
#define LOG(...) /* no-op */
#define ASSERT(X) /* no-op */
#endif
typedef struct {
uint8_t *buf; /* output buffer */
size_t buf_size; /* buffer size */
size_t *output_size; /* bytes pushed to buffer, so far */
} output_info;
#define NO_BITS ((uint32_t)-1)
/* Forward references. */
static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count);
static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte);
#if HEATSHRINK_DYNAMIC_ALLOC
heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size,
uint8_t window_sz2,
uint8_t lookahead_sz2) {
if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) ||
(window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) ||
(input_buffer_size == 0) ||
(lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) ||
(lookahead_sz2 > window_sz2)) {
return NULL;
}
size_t buffers_sz = (1 << window_sz2) + input_buffer_size;
size_t sz = sizeof(heatshrink_decoder) + buffers_sz;
heatshrink_decoder *hsd = HEATSHRINK_MALLOC(sz);
if (hsd == NULL) { return NULL; }
hsd->input_buffer_size = input_buffer_size;
hsd->window_sz2 = window_sz2;
hsd->lookahead_sz2 = lookahead_sz2;
heatshrink_decoder_reset(hsd);
LOG("-- allocated decoder with buffer size of %zu (%zu + %u + %u)\n",
sz, sizeof(heatshrink_decoder), (1 << window_sz2), input_buffer_size);
return hsd;
}
void heatshrink_decoder_free(heatshrink_decoder *hsd) {
size_t buffers_sz = (1 << hsd->window_sz2) + hsd->input_buffer_size;
size_t sz = sizeof(heatshrink_decoder) + buffers_sz;
HEATSHRINK_FREE(hsd, sz);
(void)sz; /* may not be used by free */
}
#endif
void heatshrink_decoder_reset(heatshrink_decoder *hsd) {
size_t buf_sz = 1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd);
size_t input_sz = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd);
memset(hsd->buffers, 0, buf_sz + input_sz);
hsd->state = HSDS_EMPTY;
hsd->input_size = 0;
hsd->input_index = 0;
hsd->bit_index = 0x00;
hsd->current_byte = 0x00;
hsd->output_count = 0;
hsd->output_index = 0;
hsd->head_index = 0;
hsd->bit_accumulator = 0x00000000;
}
/* Copy SIZE bytes into the decoder's input buffer, if it will fit. */
HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd,
uint8_t *in_buf, size_t size, size_t *input_size) {
if ((hsd == NULL) || (in_buf == NULL) || (input_size == NULL)) {
return HSDR_SINK_ERROR_NULL;
}
size_t rem = HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd) - hsd->input_size;
if (rem == 0) {
*input_size = 0;
return HSDR_SINK_FULL;
}
size = rem < size ? rem : size;
LOG("-- sinking %zd bytes\n", size);
/* copy into input buffer (at head of buffers) */
memcpy(&hsd->buffers[hsd->input_size], in_buf, size);
hsd->input_size += size;
if (hsd->state == HSDS_EMPTY) {
hsd->state = HSDS_INPUT_AVAILABLE;
hsd->input_index = 0;
}
*input_size = size;
return HSDR_SINK_OK;
}
/*****************
* Decompression *
*****************/
#define BACKREF_COUNT_BITS(HSD) (HEATSHRINK_DECODER_LOOKAHEAD_BITS(HSD))
#define BACKREF_INDEX_BITS(HSD) (HEATSHRINK_DECODER_WINDOW_BITS(HSD))
// States
static HSD_state st_input_available(heatshrink_decoder *hsd);
static HSD_state st_yield_literal(heatshrink_decoder *hsd,
output_info *oi);
static HSD_state st_backref_index_msb(heatshrink_decoder *hsd);
static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd);
static HSD_state st_backref_count_msb(heatshrink_decoder *hsd);
static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd);
static HSD_state st_yield_backref(heatshrink_decoder *hsd,
output_info *oi);
static HSD_state st_check_for_input(heatshrink_decoder *hsd);
HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size) {
if ((hsd == NULL) || (out_buf == NULL) || (output_size == NULL)) {
return HSDR_POLL_ERROR_NULL;
}
*output_size = 0;
output_info oi;
oi.buf = out_buf;
oi.buf_size = out_buf_size;
oi.output_size = output_size;
while (1) {
LOG("-- poll, state is %d (%s), input_size %d\n",
hsd->state, state_names[hsd->state], hsd->input_size);
uint8_t in_state = hsd->state;
switch (in_state) {
case HSDS_EMPTY:
return HSDR_POLL_EMPTY;
case HSDS_INPUT_AVAILABLE:
hsd->state = st_input_available(hsd);
break;
case HSDS_YIELD_LITERAL:
hsd->state = st_yield_literal(hsd, &oi);
break;
case HSDS_BACKREF_INDEX_MSB:
hsd->state = st_backref_index_msb(hsd);
break;
case HSDS_BACKREF_INDEX_LSB:
hsd->state = st_backref_index_lsb(hsd);
break;
case HSDS_BACKREF_COUNT_MSB:
hsd->state = st_backref_count_msb(hsd);
break;
case HSDS_BACKREF_COUNT_LSB:
hsd->state = st_backref_count_lsb(hsd);
break;
case HSDS_YIELD_BACKREF:
hsd->state = st_yield_backref(hsd, &oi);
break;
case HSDS_CHECK_FOR_MORE_INPUT:
hsd->state = st_check_for_input(hsd);
break;
default:
return HSDR_POLL_ERROR_UNKNOWN;
}
/* If the current state cannot advance, check if input or output
* buffer are exhausted. */
if (hsd->state == in_state) {
if (*output_size == out_buf_size) { return HSDR_POLL_MORE; }
return HSDR_POLL_EMPTY;
}
}
}
static HSD_state st_input_available(heatshrink_decoder *hsd) {
uint32_t bits = get_bits(hsd, 1); // get tag bit
if (bits) {
return HSDS_YIELD_LITERAL;
} else if (HEATSHRINK_DECODER_WINDOW_BITS(hsd) > 8) {
return HSDS_BACKREF_INDEX_MSB;
} else {
hsd->output_index = 0;
return HSDS_BACKREF_INDEX_LSB;
}
}
static HSD_state st_yield_literal(heatshrink_decoder *hsd,
output_info *oi) {
/* Emit a repeated section from the window buffer, and add it (again)
* to the window buffer. (Note that the repetition can include
* itself.)*/
if (*oi->output_size < oi->buf_size) {
uint32_t byte = get_bits(hsd, 8);
if (byte == NO_BITS) { return HSDS_YIELD_LITERAL; } /* out of input */
uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)];
uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1;
uint8_t c = byte & 0xFF;
LOG("-- emitting literal byte 0x%02x ('%c')\n", c, isprint(c) ? c : '.');
buf[hsd->head_index++ & mask] = c;
push_byte(hsd, oi, c);
return HSDS_CHECK_FOR_MORE_INPUT;
} else {
return HSDS_YIELD_LITERAL;
}
}
static HSD_state st_backref_index_msb(heatshrink_decoder *hsd) {
uint8_t bit_ct = BACKREF_INDEX_BITS(hsd);
ASSERT(bit_ct > 8);
uint32_t bits = get_bits(hsd, bit_ct - 8);
LOG("-- backref index (msb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_MSB; }
hsd->output_index = bits << 8;
return HSDS_BACKREF_INDEX_LSB;
}
static HSD_state st_backref_index_lsb(heatshrink_decoder *hsd) {
uint8_t bit_ct = BACKREF_INDEX_BITS(hsd);
uint32_t bits = get_bits(hsd, bit_ct < 8 ? bit_ct : 8);
LOG("-- backref index (lsb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_INDEX_LSB; }
hsd->output_index |= bits;
hsd->output_index++;
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
hsd->output_count = 0;
return (br_bit_ct > 8) ? HSDS_BACKREF_COUNT_MSB : HSDS_BACKREF_COUNT_LSB;
}
static HSD_state st_backref_count_msb(heatshrink_decoder *hsd) {
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
ASSERT(br_bit_ct > 8);
uint32_t bits = get_bits(hsd, br_bit_ct - 8);
LOG("-- backref count (msb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_MSB; }
hsd->output_count = bits << 8;
return HSDS_BACKREF_COUNT_LSB;
}
static HSD_state st_backref_count_lsb(heatshrink_decoder *hsd) {
uint8_t br_bit_ct = BACKREF_COUNT_BITS(hsd);
uint32_t bits = get_bits(hsd, br_bit_ct < 8 ? br_bit_ct : 8);
LOG("-- backref count (lsb), got 0x%04x (+1)\n", bits);
if (bits == NO_BITS) { return HSDS_BACKREF_COUNT_LSB; }
hsd->output_count |= bits;
hsd->output_count++;
return HSDS_YIELD_BACKREF;
}
static HSD_state st_yield_backref(heatshrink_decoder *hsd,
output_info *oi) {
size_t count = oi->buf_size - *oi->output_size;
if (count > 0) {
if (hsd->output_count < count) count = hsd->output_count;
uint8_t *buf = &hsd->buffers[HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(hsd)];
uint16_t mask = (1 << HEATSHRINK_DECODER_WINDOW_BITS(hsd)) - 1;
uint16_t neg_offset = hsd->output_index;
LOG("-- emitting %zu bytes from -%u bytes back\n", count, neg_offset);
ASSERT(neg_offset < mask + 1);
ASSERT(count <= 1 << BACKREF_COUNT_BITS(hsd));
for (size_t i=0; i<count; i++) {
uint8_t c = buf[(hsd->head_index - neg_offset) & mask];
push_byte(hsd, oi, c);
buf[hsd->head_index & mask] = c;
hsd->head_index++;
LOG(" -- ++ 0x%02x\n", c);
}
hsd->output_count -= count;
if (hsd->output_count == 0) { return HSDS_CHECK_FOR_MORE_INPUT; }
}
return HSDS_YIELD_BACKREF;
}
static HSD_state st_check_for_input(heatshrink_decoder *hsd) {
return (hsd->input_size == 0) ? HSDS_EMPTY : HSDS_INPUT_AVAILABLE;
}
/* Get the next COUNT bits from the input buffer, saving incremental progress.
* Returns NO_BITS on end of input, or if more than 31 bits are requested. */
static uint32_t get_bits(heatshrink_decoder *hsd, uint8_t count) {
if (count > 31) { return NO_BITS; }
LOG("-- popping %u bit(s)\n", count);
/* If we aren't able to get COUNT bits, suspend immediately, because we
* don't track how many bits of COUNT we've accumulated before suspend. */
if (hsd->input_size == 0) {
if (hsd->bit_index < (1 << (count - 1))) { return NO_BITS; }
}
for (int i = 0; i < count; i++) {
if (hsd->bit_index == 0x00) {
if (hsd->input_size == 0) {
LOG(" -- out of bits, suspending w/ accumulator of %u (0x%02x)\n",
hsd->bit_accumulator, hsd->bit_accumulator);
return NO_BITS;
}
hsd->current_byte = hsd->buffers[hsd->input_index++];
LOG(" -- pulled byte 0x%02x\n", hsd->current_byte);
if (hsd->input_index == hsd->input_size) {
hsd->input_index = 0; /* input is exhausted */
hsd->input_size = 0;
}
hsd->bit_index = 0x80;
}
hsd->bit_accumulator <<= 1;
if (hsd->current_byte & hsd->bit_index) {
hsd->bit_accumulator |= 0x01;
if (0) {
LOG(" -- got 1, accumulator 0x%04x, bit_index 0x%02x\n",
hsd->bit_accumulator, hsd->bit_index);
}
} else {
if (0) {
LOG(" -- got 0, accumulator 0x%04x, bit_index 0x%02x\n",
hsd->bit_accumulator, hsd->bit_index);
}
}
hsd->bit_index >>= 1;
}
uint32_t res = 0;
res = hsd->bit_accumulator;
hsd->bit_accumulator = 0x00000000;
if (count > 1) { LOG(" -- accumulated %08x\n", res); }
return res;
}
HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd) {
if (hsd == NULL) { return HSDR_FINISH_ERROR_NULL; }
switch (hsd->state) {
case HSDS_EMPTY:
return HSDR_FINISH_DONE;
/* If we want to finish with no input, but are in these states, it's
* because the 0-bit padding to the last byte looks like a backref
* marker bit followed by all 0s for index and count bits. */
case HSDS_BACKREF_INDEX_LSB:
case HSDS_BACKREF_INDEX_MSB:
case HSDS_BACKREF_COUNT_LSB:
case HSDS_BACKREF_COUNT_MSB:
return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE;
/* If the output stream is padded with 0xFFs (possibly due to being in
* flash memory), also explicitly check the input size rather than
* uselessly returning MORE but yielding 0 bytes when polling. */
case HSDS_YIELD_LITERAL:
return hsd->input_size == 0 ? HSDR_FINISH_DONE : HSDR_FINISH_MORE;
default:
return HSDR_FINISH_MORE;
}
}
static void push_byte(heatshrink_decoder *hsd, output_info *oi, uint8_t byte) {
LOG(" -- pushing byte: 0x%02x ('%c')\n", byte, isprint(byte) ? byte : '.');
oi->buf[(*oi->output_size)++] = byte;
(void)hsd;
}

101
espfs/heatshrink/heatshrink_decoder.h
Unescape View File

@ -0,0 +1,101 @@
#ifndef HEATSHRINK_DECODER_H
#define HEATSHRINK_DECODER_H
#include <stdint.h>
#include <stddef.h>
#include "heatshrink_common.h"
#include "heatshrink_config.h"
typedef enum {
HSDR_SINK_OK, /* data sunk, ready to poll */
HSDR_SINK_FULL, /* out of space in internal buffer */
HSDR_SINK_ERROR_NULL=-1, /* NULL argument */
} HSD_sink_res;
typedef enum {
HSDR_POLL_EMPTY, /* input exhausted */
HSDR_POLL_MORE, /* more data remaining, call again w/ fresh output buffer */
HSDR_POLL_ERROR_NULL=-1, /* NULL arguments */
HSDR_POLL_ERROR_UNKNOWN=-2,
} HSD_poll_res;
typedef enum {
HSDR_FINISH_DONE, /* output is done */
HSDR_FINISH_MORE, /* more output remains */
HSDR_FINISH_ERROR_NULL=-1, /* NULL arguments */
} HSD_finish_res;
#if HEATSHRINK_DYNAMIC_ALLOC
#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(BUF) \
((BUF)->input_buffer_size)
#define HEATSHRINK_DECODER_WINDOW_BITS(BUF) \
((BUF)->window_sz2)
#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \
((BUF)->lookahead_sz2)
#else
#define HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_) \
HEATSHRINK_STATIC_INPUT_BUFFER_SIZE
#define HEATSHRINK_DECODER_WINDOW_BITS(_) \
(HEATSHRINK_STATIC_WINDOW_BITS)
#define HEATSHRINK_DECODER_LOOKAHEAD_BITS(BUF) \
(HEATSHRINK_STATIC_LOOKAHEAD_BITS)
#endif
typedef struct {
uint16_t input_size; /* bytes in input buffer */
uint16_t input_index; /* offset to next unprocessed input byte */
uint16_t output_count; /* how many bytes to output */
uint16_t output_index; /* index for bytes to output */
uint16_t head_index; /* head of window buffer */
uint16_t bit_accumulator;
uint8_t state; /* current state machine node */
uint8_t current_byte; /* current byte of input */
uint8_t bit_index; /* current bit index */
#if HEATSHRINK_DYNAMIC_ALLOC
/* Fields that are only used if dynamically allocated. */
uint8_t window_sz2; /* window buffer bits */
uint8_t lookahead_sz2; /* lookahead bits */
uint16_t input_buffer_size; /* input buffer size */
/* Input buffer, then expansion window buffer */
uint8_t buffers[];
#else
/* Input buffer, then expansion window buffer */
uint8_t buffers[(1 << HEATSHRINK_DECODER_WINDOW_BITS(_))
+ HEATSHRINK_DECODER_INPUT_BUFFER_SIZE(_)];
#endif
} heatshrink_decoder;
#if HEATSHRINK_DYNAMIC_ALLOC
/* Allocate a decoder with an input buffer of INPUT_BUFFER_SIZE bytes,
* an expansion buffer size of 2^WINDOW_SZ2, and a lookahead
* size of 2^lookahead_sz2. (The window buffer and lookahead sizes
* must match the settings used when the data was compressed.)
* Returns NULL on error. */
heatshrink_decoder *heatshrink_decoder_alloc(uint16_t input_buffer_size,
uint8_t expansion_buffer_sz2, uint8_t lookahead_sz2);
/* Free a decoder. */
void heatshrink_decoder_free(heatshrink_decoder *hsd);
#endif
/* Reset a decoder. */
void heatshrink_decoder_reset(heatshrink_decoder *hsd);
/* Sink at most SIZE bytes from IN_BUF into the decoder. *INPUT_SIZE is set to
* indicate how many bytes were actually sunk (in case a buffer was filled). */
HSD_sink_res heatshrink_decoder_sink(heatshrink_decoder *hsd,
uint8_t *in_buf, size_t size, size_t *input_size);
/* Poll for output from the decoder, copying at most OUT_BUF_SIZE bytes into
* OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */
HSD_poll_res heatshrink_decoder_poll(heatshrink_decoder *hsd,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size);
/* Notify the dencoder that the input stream is finished.
* If the return value is HSDR_FINISH_MORE, there is still more output, so
* call heatshrink_decoder_poll and repeat. */
HSD_finish_res heatshrink_decoder_finish(heatshrink_decoder *hsd);
#endif

650
espfs/heatshrink/heatshrink_encoder.c
Unescape View File

@ -0,0 +1,650 @@
#include <stdlib.h>
#include <string.h>
#include <stdbool.h>
#include "heatshrink_encoder.h"
typedef enum {
HSES_NOT_FULL, /* input buffer not full enough */
HSES_FILLED, /* buffer is full */
HSES_SEARCH, /* searching for patterns */
HSES_YIELD_TAG_BIT, /* yield tag bit */
HSES_YIELD_LITERAL, /* emit literal byte */
HSES_YIELD_BR_INDEX, /* yielding backref index */
HSES_YIELD_BR_LENGTH, /* yielding backref length */
HSES_SAVE_BACKLOG, /* copying buffer to backlog */
HSES_FLUSH_BITS, /* flush bit buffer */
HSES_DONE, /* done */
} HSE_state;
#if HEATSHRINK_DEBUGGING_LOGS
#include <stdio.h>
#include <ctype.h>
#include <assert.h>
#define LOG(...) fprintf(stderr, __VA_ARGS__)
#define ASSERT(X) assert(X)
static const char *state_names[] = {
"not_full",
"filled",
"search",
"yield_tag_bit",
"yield_literal",
"yield_br_index",
"yield_br_length",
"save_backlog",
"flush_bits",
"done",
};
#else
#define LOG(...) /* no-op */
#define ASSERT(X) /* no-op */
#endif
// Encoder flags
enum {
FLAG_IS_FINISHING = 0x01,
FLAG_HAS_LITERAL = 0x02,
FLAG_ON_FINAL_LITERAL = 0x04,
FLAG_BACKLOG_IS_PARTIAL = 0x08,
FLAG_BACKLOG_IS_FILLED = 0x10,
};
typedef struct {
uint8_t *buf; /* output buffer */
size_t buf_size; /* buffer size */
size_t *output_size; /* bytes pushed to buffer, so far */
} output_info;
#define MATCH_NOT_FOUND ((uint16_t)-1)
static uint16_t get_input_offset(heatshrink_encoder *hse);
static uint16_t get_input_buffer_size(heatshrink_encoder *hse);
static uint16_t get_lookahead_size(heatshrink_encoder *hse);
static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag);
static int can_take_byte(output_info *oi);
static int is_finishing(heatshrink_encoder *hse);
static int backlog_is_partial(heatshrink_encoder *hse);
static int backlog_is_filled(heatshrink_encoder *hse);
static int on_final_literal(heatshrink_encoder *hse);
static void save_backlog(heatshrink_encoder *hse);
static int has_literal(heatshrink_encoder *hse);
/* Push COUNT (max 8) bits to the output buffer, which has room. */
static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits,
output_info *oi);
static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi);
static void push_literal_byte(heatshrink_encoder *hse, output_info *oi);
#if HEATSHRINK_DYNAMIC_ALLOC
heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2,
uint8_t lookahead_sz2) {
if ((window_sz2 < HEATSHRINK_MIN_WINDOW_BITS) ||
(window_sz2 > HEATSHRINK_MAX_WINDOW_BITS) ||
(lookahead_sz2 < HEATSHRINK_MIN_LOOKAHEAD_BITS) ||
(lookahead_sz2 > window_sz2)) {
return NULL;
}
/* Note: 2 * the window size is used because the buffer needs to fit
* (1 << window_sz2) bytes for the current input, and an additional
* (1 << window_sz2) bytes for the previous buffer of input, which
* will be scanned for useful backreferences. */
size_t buf_sz = (2 << window_sz2);
heatshrink_encoder *hse = HEATSHRINK_MALLOC(sizeof(*hse) + buf_sz);
if (hse == NULL) { return NULL; }
hse->window_sz2 = window_sz2;
hse->lookahead_sz2 = lookahead_sz2;
heatshrink_encoder_reset(hse);
#if HEATSHRINK_USE_INDEX
size_t index_sz = buf_sz*sizeof(uint16_t);
hse->search_index = HEATSHRINK_MALLOC(index_sz + sizeof(struct hs_index));
if (hse->search_index == NULL) {
HEATSHRINK_FREE(hse, sizeof(*hse) + buf_sz);
return NULL;
}
hse->search_index->size = index_sz;
#endif
LOG("-- allocated encoder with buffer size of %zu (%u byte input size)\n",
buf_sz, get_input_buffer_size(hse));
return hse;
}
void heatshrink_encoder_free(heatshrink_encoder *hse) {
size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
#if HEATSHRINK_USE_INDEX
size_t index_sz = sizeof(struct hs_index) + hse->search_index->size;
HEATSHRINK_FREE(hse->search_index, index_sz);
(void)index_sz;
#endif
HEATSHRINK_FREE(hse, sizeof(heatshrink_encoder) + buf_sz);
(void)buf_sz;
}
#endif
void heatshrink_encoder_reset(heatshrink_encoder *hse) {
size_t buf_sz = (2 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
memset(hse->buffer, 0, buf_sz);
hse->input_size = 0;
hse->state = HSES_NOT_FULL;
hse->match_scan_index = 0;
hse->flags = 0;
hse->bit_index = 0x80;
hse->current_byte = 0x00;
hse->match_length = 0;
hse->outgoing_bits = 0x0000;
hse->outgoing_bits_count = 0;
#ifdef LOOP_DETECT
hse->loop_detect = (uint32_t)-1;
#endif
}
HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse,
uint8_t *in_buf, size_t size, size_t *input_size) {
if ((hse == NULL) || (in_buf == NULL) || (input_size == NULL)) {
return HSER_SINK_ERROR_NULL;
}
/* Sinking more content after saying the content is done, tsk tsk */
if (is_finishing(hse)) { return HSER_SINK_ERROR_MISUSE; }
/* Sinking more content before processing is done */
if (hse->state != HSES_NOT_FULL) { return HSER_SINK_ERROR_MISUSE; }
uint16_t write_offset = get_input_offset(hse) + hse->input_size;
uint16_t ibs = get_input_buffer_size(hse);
uint16_t rem = ibs - hse->input_size;
uint16_t cp_sz = rem < size ? rem : size;
memcpy(&hse->buffer[write_offset], in_buf, cp_sz);
*input_size = cp_sz;
hse->input_size += cp_sz;
LOG("-- sunk %u bytes (of %zu) into encoder at %d, input buffer now has %u\n",
cp_sz, size, write_offset, hse->input_size);
if (cp_sz == rem) {
LOG("-- internal buffer is now full\n");
hse->state = HSES_FILLED;
}
return HSER_SINK_OK;
}
/***************
* Compression *
***************/
static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start,
uint16_t end, const uint16_t maxlen, uint16_t *match_length);
static void do_indexing(heatshrink_encoder *hse);
static HSE_state st_step_search(heatshrink_encoder *hse);
static HSE_state st_yield_tag_bit(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_literal(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_br_index(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_yield_br_length(heatshrink_encoder *hse,
output_info *oi);
static HSE_state st_save_backlog(heatshrink_encoder *hse);
static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse,
output_info *oi);
HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size) {
if ((hse == NULL) || (out_buf == NULL) || (output_size == NULL)) {
return HSER_POLL_ERROR_NULL;
}
if (out_buf_size == 0) {
LOG("-- MISUSE: output buffer size is 0\n");
return HSER_POLL_ERROR_MISUSE;
}
*output_size = 0;
output_info oi;
oi.buf = out_buf;
oi.buf_size = out_buf_size;
oi.output_size = output_size;
while (1) {
LOG("-- polling, state %u (%s), flags 0x%02x\n",
hse->state, state_names[hse->state], hse->flags);
uint8_t in_state = hse->state;
switch (in_state) {
case HSES_NOT_FULL:
return HSER_POLL_EMPTY;
case HSES_FILLED:
do_indexing(hse);
hse->state = HSES_SEARCH;
break;
case HSES_SEARCH:
hse->state = st_step_search(hse);
break;
case HSES_YIELD_TAG_BIT:
hse->state = st_yield_tag_bit(hse, &oi);
break;
case HSES_YIELD_LITERAL:
hse->state = st_yield_literal(hse, &oi);
break;
case HSES_YIELD_BR_INDEX:
hse->state = st_yield_br_index(hse, &oi);
break;
case HSES_YIELD_BR_LENGTH:
hse->state = st_yield_br_length(hse, &oi);
break;
case HSES_SAVE_BACKLOG:
hse->state = st_save_backlog(hse);
break;
case HSES_FLUSH_BITS:
hse->state = st_flush_bit_buffer(hse, &oi);
case HSES_DONE:
return HSER_POLL_EMPTY;
default:
LOG("-- bad state %s\n", state_names[hse->state]);
return HSER_POLL_ERROR_MISUSE;
}
if (hse->state == in_state) {
/* Check if output buffer is exhausted. */
if (*output_size == out_buf_size) return HSER_POLL_MORE;
}
}
}
HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse) {
if (hse == NULL) { return HSER_FINISH_ERROR_NULL; }
LOG("-- setting is_finishing flag\n");
hse->flags |= FLAG_IS_FINISHING;
if (hse->state == HSES_NOT_FULL) { hse->state = HSES_FILLED; }
return hse->state == HSES_DONE ? HSER_FINISH_DONE : HSER_FINISH_MORE;
}
static HSE_state st_step_search(heatshrink_encoder *hse) {
uint16_t window_length = get_input_buffer_size(hse);
uint16_t lookahead_sz = get_lookahead_size(hse);
uint16_t msi = hse->match_scan_index;
LOG("## step_search, scan @ +%d (%d/%d), input size %d\n",
msi, hse->input_size + msi, 2*window_length, hse->input_size);
bool fin = is_finishing(hse);
if (msi >= hse->input_size - (fin ? 0 : lookahead_sz)) {
/* Current search buffer is exhausted, copy it into the
* backlog and await more input. */
LOG("-- end of search @ %d, saving backlog\n", msi);
return HSES_SAVE_BACKLOG;
}
uint16_t input_offset = get_input_offset(hse);
uint16_t end = input_offset + msi;
uint16_t start = 0;
if (backlog_is_filled(hse)) { /* last WINDOW_LENGTH bytes */
start = end - window_length + 1;
} else if (backlog_is_partial(hse)) { /* clamp to available data */
start = end - window_length + 1;
if (start < lookahead_sz) { start = lookahead_sz; }
} else { /* only scan available input */
start = input_offset;
}
uint16_t max_possible = lookahead_sz;
if (hse->input_size - msi < lookahead_sz) {
max_possible = hse->input_size - msi;
}
uint16_t match_length = 0;
uint16_t match_pos = find_longest_match(hse,
start, end, max_possible, &match_length);
if (match_pos == MATCH_NOT_FOUND) {
LOG("ss Match not found\n");
hse->match_scan_index++;
hse->flags |= FLAG_HAS_LITERAL;
hse->match_length = 0;
return HSES_YIELD_TAG_BIT;
} else {
LOG("ss Found match of %d bytes at %d\n", match_length, match_pos);
hse->match_pos = match_pos;
hse->match_length = match_length;
ASSERT(match_pos < 1 << hse->window_sz2 /*window_length*/);
return HSES_YIELD_TAG_BIT;
}
}
static HSE_state st_yield_tag_bit(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
if (hse->match_length == 0) {
add_tag_bit(hse, oi, HEATSHRINK_LITERAL_MARKER);
return HSES_YIELD_LITERAL;
} else {
add_tag_bit(hse, oi, HEATSHRINK_BACKREF_MARKER);
hse->outgoing_bits = hse->match_pos - 1;
hse->outgoing_bits_count = HEATSHRINK_ENCODER_WINDOW_BITS(hse);
return HSES_YIELD_BR_INDEX;
}
} else {
return HSES_YIELD_TAG_BIT; /* output is full, continue */
}
}
static HSE_state st_yield_literal(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
push_literal_byte(hse, oi);
hse->flags &= ~FLAG_HAS_LITERAL;
if (on_final_literal(hse)) { return HSES_FLUSH_BITS; }
return hse->match_length > 0 ? HSES_YIELD_TAG_BIT : HSES_SEARCH;
} else {
return HSES_YIELD_LITERAL;
}
}
static HSE_state st_yield_br_index(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
LOG("-- yielding backref index %u\n", hse->match_pos);
if (push_outgoing_bits(hse, oi) > 0) {
return HSES_YIELD_BR_INDEX; /* continue */
} else {
hse->outgoing_bits = hse->match_length - 1;
hse->outgoing_bits_count = HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse);
return HSES_YIELD_BR_LENGTH; /* done */
}
} else {
return HSES_YIELD_BR_INDEX; /* continue */
}
}
static HSE_state st_yield_br_length(heatshrink_encoder *hse,
output_info *oi) {
if (can_take_byte(oi)) {
LOG("-- yielding backref length %u\n", hse->match_length);
if (push_outgoing_bits(hse, oi) > 0) {
return HSES_YIELD_BR_LENGTH;
} else {
hse->match_scan_index += hse->match_length;
hse->match_length = 0;
return HSES_SEARCH;
}
} else {
return HSES_YIELD_BR_LENGTH;
}
}
static HSE_state st_save_backlog(heatshrink_encoder *hse) {
if (is_finishing(hse)) {
/* copy remaining literal (if necessary) */
if (has_literal(hse)) {
hse->flags |= FLAG_ON_FINAL_LITERAL;
return HSES_YIELD_TAG_BIT;
} else {
return HSES_FLUSH_BITS;
}
} else {
LOG("-- saving backlog\n");
save_backlog(hse);
return HSES_NOT_FULL;
}
}
static HSE_state st_flush_bit_buffer(heatshrink_encoder *hse,
output_info *oi) {
if (hse->bit_index == 0x80) {
LOG("-- done!\n");
return HSES_DONE;
} else if (can_take_byte(oi)) {
LOG("-- flushing remaining byte (bit_index == 0x%02x)\n", hse->bit_index);
oi->buf[(*oi->output_size)++] = hse->current_byte;
LOG("-- done!\n");
return HSES_DONE;
} else {
return HSES_FLUSH_BITS;
}
}
static void add_tag_bit(heatshrink_encoder *hse, output_info *oi, uint8_t tag) {
LOG("-- adding tag bit: %d\n", tag);
push_bits(hse, 1, tag, oi);
}
static uint16_t get_input_offset(heatshrink_encoder *hse) {
return get_input_buffer_size(hse);
}
static uint16_t get_input_buffer_size(heatshrink_encoder *hse) {
return (1 << HEATSHRINK_ENCODER_WINDOW_BITS(hse));
(void)hse;
}
static uint16_t get_lookahead_size(heatshrink_encoder *hse) {
return (1 << HEATSHRINK_ENCODER_LOOKAHEAD_BITS(hse));
(void)hse;
}
static void do_indexing(heatshrink_encoder *hse) {
#if HEATSHRINK_USE_INDEX
/* Build an index array I that contains flattened linked lists
* for the previous instances of every byte in the buffer.
*
* For example, if buf[200] == 'x', then index[200] will either
* be an offset i such that buf[i] == 'x', or a negative offset
* to indicate end-of-list. This significantly speeds up matching,
* while only using sizeof(uint16_t)*sizeof(buffer) bytes of RAM.
*
* Future optimization options:
* 1. Since any negative value represents end-of-list, the other
* 15 bits could be used to improve the index dynamically.
*
* 2. Likewise, the last lookahead_sz bytes of the index will
* not be usable, so temporary data could be stored there to
* dynamically improve the index.
* */
struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse);
uint16_t last[256];
memset(last, 0xFF, sizeof(last));
uint8_t * const data = hse->buffer;
int16_t * const index = hsi->index;
const uint16_t input_offset = get_input_offset(hse);
const uint16_t end = input_offset + hse->input_size;
for (uint16_t i=0; i<end; i++) {
uint8_t v = data[i];
uint16_t lv = last[v];
index[i] = lv;
last[v] = i;
}
#else
(void)hse;
#endif
}
static int is_finishing(heatshrink_encoder *hse) {
return hse->flags & FLAG_IS_FINISHING;
}
static int backlog_is_partial(heatshrink_encoder *hse) {
return hse->flags & FLAG_BACKLOG_IS_PARTIAL;
}
static int backlog_is_filled(heatshrink_encoder *hse) {
return hse->flags & FLAG_BACKLOG_IS_FILLED;
}
static int on_final_literal(heatshrink_encoder *hse) {
return hse->flags & FLAG_ON_FINAL_LITERAL;
}
static int has_literal(heatshrink_encoder *hse) {
return (hse->flags & FLAG_HAS_LITERAL);
}
static int can_take_byte(output_info *oi) {
return *oi->output_size < oi->buf_size;
}
/* Return the longest match for the bytes at buf[end:end+maxlen] between
* buf[start] and buf[end-1]. If no match is found, return -1. */
static uint16_t find_longest_match(heatshrink_encoder *hse, uint16_t start,
uint16_t end, const uint16_t maxlen, uint16_t *match_length) {
LOG("-- scanning for match of buf[%u:%u] between buf[%u:%u] (max %u bytes)\n",
end, end + maxlen, start, end + maxlen - 1, maxlen);
uint8_t *buf = hse->buffer;
uint16_t match_maxlen = 0;
uint16_t match_index = MATCH_NOT_FOUND;
const uint16_t break_even_point = 3;
uint16_t len = 0;
uint8_t * const needlepoint = &buf[end];
#if HEATSHRINK_USE_INDEX
struct hs_index *hsi = HEATSHRINK_ENCODER_INDEX(hse);
int16_t pos = hsi->index[end];
while (pos >= start) {
uint8_t * const pospoint = &buf[pos];
len = 0;
/* Only check matches that will potentially beat the current maxlen.
* This is redundant with the index if match_maxlen is 0, but the
* added branch overhead to check if it == 0 seems to be worse. */
if (pospoint[match_maxlen] != needlepoint[match_maxlen]) {
pos = hsi->index[pos];
continue;
}
for (len = 1; len < maxlen; len++) {
if (pospoint[len] != needlepoint[len]) break;
}
if (len > match_maxlen) {
match_maxlen = len;
match_index = pos;
if (len == maxlen) { break; } /* won't find better */
}
pos = hsi->index[pos];
}
#else
for (int16_t pos=end - 1; pos >= start; pos--) {
uint8_t * const pospoint = &buf[pos];
if ((pospoint[match_maxlen] == needlepoint[match_maxlen])
&& (*pospoint == *needlepoint)) {
for (len=1; len<maxlen; len++) {
if (0) {
LOG(" --> cmp buf[%d] == 0x%02x against %02x (start %u)\n",
pos + len, pospoint[len], needlepoint[len], start);
}
if (pospoint[len] != needlepoint[len]) { break; }
}
if (len > match_maxlen) {
match_maxlen = len;
match_index = pos;
if (len == maxlen) { break; } /* don't keep searching */
}
}
}
#endif
if (match_maxlen >= break_even_point) {
LOG("-- best match: %u bytes at -%u\n",
match_maxlen, end - match_index);
*match_length = match_maxlen;
return end - match_index;
}
LOG("-- none found\n");
return MATCH_NOT_FOUND;
}
static uint8_t push_outgoing_bits(heatshrink_encoder *hse, output_info *oi) {
uint8_t count = 0;
uint8_t bits = 0;
if (hse->outgoing_bits_count > 8) {
count = 8;
bits = hse->outgoing_bits >> (hse->outgoing_bits_count - 8);
} else {
count = hse->outgoing_bits_count;
bits = hse->outgoing_bits;
}
if (count > 0) {
LOG("-- pushing %d outgoing bits: 0x%02x\n", count, bits);
push_bits(hse, count, bits, oi);
hse->outgoing_bits_count -= count;
}
return count;
}
/* Push COUNT (max 8) bits to the output buffer, which has room.
* Bytes are set from the lowest bits, up. */
static void push_bits(heatshrink_encoder *hse, uint8_t count, uint8_t bits,
output_info *oi) {
ASSERT(count <= 8);
LOG("++ push_bits: %d bits, input of 0x%02x\n", count, bits);
/* If adding a whole byte and at the start of a new output byte,
* just push it through whole and skip the bit IO loop. */
if (count == 8 && hse->bit_index == 0x80) {
oi->buf[(*oi->output_size)++] = bits;
} else {
for (int i=count - 1; i>=0; i--) {
bool bit = bits & (1 << i);
if (bit) { hse->current_byte |= hse->bit_index; }
if (0) {
LOG(" -- setting bit %d at bit index 0x%02x, byte => 0x%02x\n",
bit ? 1 : 0, hse->bit_index, hse->current_byte);
}
hse->bit_index >>= 1;
if (hse->bit_index == 0x00) {
hse->bit_index = 0x80;
LOG(" > pushing byte 0x%02x\n", hse->current_byte);
oi->buf[(*oi->output_size)++] = hse->current_byte;
hse->current_byte = 0x00;
}
}
}
}
static void push_literal_byte(heatshrink_encoder *hse, output_info *oi) {
uint16_t processed_offset = hse->match_scan_index - 1;
uint16_t input_offset = get_input_offset(hse) + processed_offset;
uint8_t c = hse->buffer[input_offset];
LOG("-- yielded literal byte 0x%02x ('%c') from +%d\n",
c, isprint(c) ? c : '.', input_offset);
push_bits(hse, 8, c, oi);
}
static void save_backlog(heatshrink_encoder *hse) {
size_t input_buf_sz = get_input_buffer_size(hse);
uint16_t msi = hse->match_scan_index;
/* Copy processed data to beginning of buffer, so it can be
* used for future matches. Don't bother checking whether the
* input is less than the maximum size, because if it isn't,
* we're done anyway. */
uint16_t rem = input_buf_sz - msi; // unprocessed bytes
uint16_t shift_sz = input_buf_sz + rem;
memmove(&hse->buffer[0],
&hse->buffer[input_buf_sz - rem],
shift_sz);
if (backlog_is_partial(hse)) {
/* The whole backlog is filled in now, so include it in scans. */
hse->flags |= FLAG_BACKLOG_IS_FILLED;
} else {
/* Include backlog, except for the first lookahead_sz bytes, which
* are still undefined. */
hse->flags |= FLAG_BACKLOG_IS_PARTIAL;
}
hse->match_scan_index = 0;
hse->input_size -= input_buf_sz - rem;
}

109
espfs/heatshrink/heatshrink_encoder.h
Unescape View File

@ -0,0 +1,109 @@
#ifndef HEATSHRINK_ENCODER_H
#define HEATSHRINK_ENCODER_H
#include <stdint.h>
#include <stddef.h>
#include "heatshrink_common.h"
#include "heatshrink_config.h"
typedef enum {
HSER_SINK_OK, /* data sunk into input buffer */
HSER_SINK_ERROR_NULL=-1, /* NULL argument */
HSER_SINK_ERROR_MISUSE=-2, /* API misuse */
} HSE_sink_res;
typedef enum {
HSER_POLL_EMPTY, /* input exhausted */
HSER_POLL_MORE, /* poll again for more output */
HSER_POLL_ERROR_NULL=-1, /* NULL argument */
HSER_POLL_ERROR_MISUSE=-2, /* API misuse */
} HSE_poll_res;
typedef enum {
HSER_FINISH_DONE, /* encoding is complete */
HSER_FINISH_MORE, /* more output remaining; use poll */
HSER_FINISH_ERROR_NULL=-1, /* NULL argument */
} HSE_finish_res;
#if HEATSHRINK_DYNAMIC_ALLOC
#define HEATSHRINK_ENCODER_WINDOW_BITS(HSE) \
((HSE)->window_sz2)
#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(HSE) \
((HSE)->lookahead_sz2)
#define HEATSHRINK_ENCODER_INDEX(HSE) \
((HSE)->search_index)
struct hs_index {
uint16_t size;
int16_t index[];
};
#else
#define HEATSHRINK_ENCODER_WINDOW_BITS(_) \
(HEATSHRINK_STATIC_WINDOW_BITS)
#define HEATSHRINK_ENCODER_LOOKAHEAD_BITS(_) \
(HEATSHRINK_STATIC_LOOKAHEAD_BITS)
#define HEATSHRINK_ENCODER_INDEX(HSE) \
(&(HSE)->search_index)
struct hs_index {
uint16_t size;
int16_t index[2 << HEATSHRINK_STATIC_WINDOW_BITS];
};
#endif
typedef struct {
uint16_t input_size; /* bytes in input buffer */
uint16_t match_scan_index;
uint16_t match_length;
uint16_t match_pos;
uint16_t outgoing_bits; /* enqueued outgoing bits */
uint8_t outgoing_bits_count;
uint8_t flags;
uint8_t state; /* current state machine node */
uint8_t current_byte; /* current byte of output */
uint8_t bit_index; /* current bit index */
#if HEATSHRINK_DYNAMIC_ALLOC
uint8_t window_sz2; /* 2^n size of window */
uint8_t lookahead_sz2; /* 2^n size of lookahead */
#if HEATSHRINK_USE_INDEX
struct hs_index *search_index;
#endif
/* input buffer and / sliding window for expansion */
uint8_t buffer[];
#else
#if HEATSHRINK_USE_INDEX
struct hs_index search_index;
#endif
/* input buffer and / sliding window for expansion */
uint8_t buffer[2 << HEATSHRINK_ENCODER_WINDOW_BITS(_)];
#endif
} heatshrink_encoder;
#if HEATSHRINK_DYNAMIC_ALLOC
/* Allocate a new encoder struct and its buffers.
* Returns NULL on error. */
heatshrink_encoder *heatshrink_encoder_alloc(uint8_t window_sz2,
uint8_t lookahead_sz2);
/* Free an encoder. */
void heatshrink_encoder_free(heatshrink_encoder *hse);
#endif
/* Reset an encoder. */
void heatshrink_encoder_reset(heatshrink_encoder *hse);
/* Sink up to SIZE bytes from IN_BUF into the encoder.
* INPUT_SIZE is set to the number of bytes actually sunk (in case a
* buffer was filled.). */
HSE_sink_res heatshrink_encoder_sink(heatshrink_encoder *hse,
uint8_t *in_buf, size_t size, size_t *input_size);
/* Poll for output from the encoder, copying at most OUT_BUF_SIZE bytes into
* OUT_BUF (setting *OUTPUT_SIZE to the actual amount copied). */
HSE_poll_res heatshrink_encoder_poll(heatshrink_encoder *hse,
uint8_t *out_buf, size_t out_buf_size, size_t *output_size);
/* Notify the encoder that the input stream is finished.
* If the return value is HSER_FINISH_MORE, there is still more output, so
* call heatshrink_encoder_poll and repeat. */
HSE_finish_res heatshrink_encoder_finish(heatshrink_encoder *hse);
#endif

999
espfs/heatshrink/test_heatshrink_dynamic.c
Unescape View File

@ -0,0 +1,999 @@
#include <stdint.h>
#include <ctype.h>
#include <assert.h>
#include "heatshrink_encoder.h"
#include "heatshrink_decoder.h"
#include "greatest.h"
#if !HEATSHRINK_DYNAMIC_ALLOC
#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for dynamic allocation test suite.
#endif
SUITE(encoding);
SUITE(decoding);
SUITE(integration);
#ifdef HEATSHRINK_HAS_THEFT
SUITE(properties);
#endif
static void dump_buf(char *name, uint8_t *buf, uint16_t count) {
for (int i=0; i<count; i++) {
uint8_t c = (uint8_t)buf[i];
printf("%s %d: 0x%02x ('%c')\n", name, i, c, isprint(c) ? c : '.');
}
}
TEST encoder_alloc_should_reject_invalid_arguments(void) {
ASSERT_EQ(NULL, heatshrink_encoder_alloc(
HEATSHRINK_MIN_WINDOW_BITS - 1, 8));
ASSERT_EQ(NULL, heatshrink_encoder_alloc(
HEATSHRINK_MAX_WINDOW_BITS + 1, 8));
ASSERT_EQ(NULL, heatshrink_encoder_alloc(8, HEATSHRINK_MIN_LOOKAHEAD_BITS - 1));
ASSERT_EQ(NULL, heatshrink_encoder_alloc(8, 9));
PASS();
}
TEST encoder_sink_should_reject_nulls(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
uint8_t input[] = {'f', 'o', 'o'};
size_t input_size = 0;
ASSERT(hse);
ASSERT_EQ(HSER_SINK_ERROR_NULL, heatshrink_encoder_sink(NULL, input, 3, &input_size));
ASSERT_EQ(HSER_SINK_ERROR_NULL, heatshrink_encoder_sink(hse, NULL, 3, &input_size));
ASSERT_EQ(HSER_SINK_ERROR_NULL, heatshrink_encoder_sink(hse, input, 3, NULL));
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_poll_should_reject_nulls(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
uint8_t output[256];
size_t output_size = 0;
ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(NULL,
output, 256, &output_size));
ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(hse,
NULL, 256, &output_size));
ASSERT_EQ(HSER_POLL_ERROR_NULL, heatshrink_encoder_poll(hse,
output, 256, NULL));
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_finish_should_reject_nulls(void) {
ASSERT_EQ(HSER_FINISH_ERROR_NULL, heatshrink_encoder_finish(NULL));
PASS();
}
TEST encoder_sink_should_accept_input_when_it_will_fit(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
ASSERT(hse);
uint8_t input[256];
size_t bytes_copied = 0;
memset(input, '*', 256);
ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse,
input, 256, &bytes_copied));
ASSERT_EQ(256, bytes_copied);
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_sink_should_accept_partial_input_when_some_will_fit(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
ASSERT(hse);
uint8_t input[512];
size_t bytes_copied = 0;
memset(input, '*', 512);
ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse,
input, 512, &bytes_copied));
ASSERT_EQ(256, bytes_copied);
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_poll_should_indicate_when_no_input_is_provided(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
uint8_t output[512];
size_t output_size = 0;
HSE_poll_res res = heatshrink_encoder_poll(hse,
output, 512, &output_size);
ASSERT_EQ(HSER_POLL_EMPTY, res);
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_should_emit_data_without_repetitions_as_literal_sequence(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
ASSERT(hse);
uint8_t input[5];
uint8_t output[1024];
size_t copied = 0;
uint8_t expected[] = { 0x80, 0x40, 0x60, 0x50, 0x38, 0x20 };
for (int i=0; i<5; i++) { input[i] = i; }
memset(output, 0, 1024);
ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse, input, 5, &copied));
ASSERT_EQ(5, copied);
/* Should get no output yet, since encoder doesn't know input is complete. */
copied = 0;
HSE_poll_res pres = heatshrink_encoder_poll(hse, output, 1024, &copied);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
ASSERT_EQ(0, copied);
/* Mark input stream as done, to force small input to be processed. */
HSE_finish_res fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_MORE, fres);
pres = heatshrink_encoder_poll(hse, output, 1024, &copied);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
for (size_t i=0; i<sizeof(expected); i++) {
ASSERT_EQ(expected[i], output[i]);
}
ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse));
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_should_emit_series_of_same_byte_as_literal_then_backref(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
ASSERT(hse);
uint8_t input[5];
uint8_t output[1024];
size_t copied = 0;
uint8_t expected[] = {0xb0, 0x80, 0x01, 0x80};
for (int i=0; i<5; i++) { input[i] = 'a'; } /* "aaaaa" */
memset(output, 0, 1024);
ASSERT_EQ(HSER_SINK_OK, heatshrink_encoder_sink(hse, input, 5, &copied));
ASSERT_EQ(5, copied);
/* Should get no output yet, since encoder doesn't know input is complete. */
copied = 0;
HSE_poll_res pres = heatshrink_encoder_poll(hse, output, 1024, &copied);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
ASSERT_EQ(0, copied);
/* Mark input stream as done, to force small input to be processed. */
HSE_finish_res fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_MORE, fres);
pres = heatshrink_encoder_poll(hse, output, 1024, &copied);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
ASSERT_EQ(4, copied);
if (0) dump_buf("output", output, copied);
for (size_t i=0; i<copied; i++) ASSERT_EQ(expected[i], output[i]);
ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse));
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_poll_should_detect_repeated_substring(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3);
uint8_t input[] = {'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd'};
uint8_t output[1024];
uint8_t expected[] = {0xb0, 0xd8, 0xac, 0x76, 0x40, 0x1b };
size_t copied = 0;
memset(output, 0, 1024);
HSE_sink_res sres = heatshrink_encoder_sink(hse,
input, sizeof(input), &copied);
ASSERT_EQ(HSER_SINK_OK, sres);
ASSERT_EQ(sizeof(input), copied);
HSE_finish_res fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_MORE, fres);
ASSERT_EQ(HSER_POLL_EMPTY, heatshrink_encoder_poll(hse, output, 1024, &copied));
fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_DONE, fres);
if (0) dump_buf("output", output, copied);
ASSERT_EQ(sizeof(expected), copied);
for (size_t i=0; i<sizeof(expected); i++) ASSERT_EQ(expected[i], output[i]);
heatshrink_encoder_free(hse);
PASS();
}
TEST encoder_poll_should_detect_repeated_substring_and_preserve_trailing_literal(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3);
uint8_t input[] = {'a', 'b', 'c', 'd', 'a', 'b', 'c', 'd', 'e'};
uint8_t output[1024];
uint8_t expected[] = {0xb0, 0xd8, 0xac, 0x76, 0x40, 0x1b, 0xb2, 0x80 };
size_t copied = 0;
memset(output, 0, 1024);
HSE_sink_res sres = heatshrink_encoder_sink(hse,
input, sizeof(input), &copied);
ASSERT_EQ(HSER_SINK_OK, sres);
ASSERT_EQ(sizeof(input), copied);
HSE_finish_res fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_MORE, fres);
ASSERT_EQ(HSER_POLL_EMPTY, heatshrink_encoder_poll(hse, output, 1024, &copied));
fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_DONE, fres);
if (0) dump_buf("output", output, copied);
ASSERT_EQ(sizeof(expected), copied);
for (size_t i=0; i<sizeof(expected); i++) ASSERT_EQ(expected[i], output[i]);
heatshrink_encoder_free(hse);
PASS();
}
SUITE(encoding) {
RUN_TEST(encoder_alloc_should_reject_invalid_arguments);
RUN_TEST(encoder_sink_should_reject_nulls);
RUN_TEST(encoder_sink_should_accept_input_when_it_will_fit);
RUN_TEST(encoder_sink_should_accept_partial_input_when_some_will_fit);
RUN_TEST(encoder_poll_should_reject_nulls);
RUN_TEST(encoder_poll_should_indicate_when_no_input_is_provided);
RUN_TEST(encoder_finish_should_reject_nulls);
RUN_TEST(encoder_should_emit_data_without_repetitions_as_literal_sequence);
RUN_TEST(encoder_should_emit_series_of_same_byte_as_literal_then_backref);
RUN_TEST(encoder_poll_should_detect_repeated_substring);
RUN_TEST(encoder_poll_should_detect_repeated_substring_and_preserve_trailing_literal);
}
TEST decoder_alloc_should_reject_excessively_small_window(void) {
ASSERT_EQ(NULL, heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS - 1, 4));
PASS();
}
TEST decoder_alloc_should_reject_zero_byte_input_buffer(void) {
ASSERT_EQ(NULL, heatshrink_decoder_alloc(0,
HEATSHRINK_MIN_WINDOW_BITS, 4));
PASS();
}
TEST decoder_sink_should_reject_null_hsd_pointer(void) {
uint8_t input[] = {0,1,2,3,4,5};
size_t count = 0;
ASSERT_EQ(HSDR_SINK_ERROR_NULL, heatshrink_decoder_sink(NULL, input, 6, &count));
PASS();
}
TEST decoder_sink_should_reject_null_input_pointer(void) {
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
size_t count = 0;
ASSERT_EQ(HSDR_SINK_ERROR_NULL, heatshrink_decoder_sink(hsd, NULL, 6, &count));
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_sink_should_reject_null_count_pointer(void) {
uint8_t input[] = {0,1,2,3,4,5};
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
ASSERT_EQ(HSDR_SINK_ERROR_NULL, heatshrink_decoder_sink(hsd, input, 6, NULL));
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_sink_should_reject_excessively_large_input(void) {
uint8_t input[] = {0,1,2,3,4,5};
heatshrink_decoder *hsd = heatshrink_decoder_alloc(1,
HEATSHRINK_MIN_WINDOW_BITS, 4);
size_t count = 0;
// Sink as much as will fit
HSD_sink_res res = heatshrink_decoder_sink(hsd, input, 6, &count);
ASSERT_EQ(HSDR_SINK_OK, res);
ASSERT_EQ(1, count);
// And now, no more should fit.
res = heatshrink_decoder_sink(hsd, &input[count], sizeof(input) - count, &count);
ASSERT_EQ(HSDR_SINK_FULL, res);
ASSERT_EQ(0, count);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_sink_should_sink_data_when_preconditions_hold(void) {
uint8_t input[] = {0,1,2,3,4,5};
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
size_t count = 0;
HSD_sink_res res = heatshrink_decoder_sink(hsd, input, 6, &count);
ASSERT_EQ(HSDR_SINK_OK, res);
ASSERT_EQ(hsd->input_size, 6);
ASSERT_EQ(hsd->input_index, 0);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_return_empty_if_empty(void) {
uint8_t output[256];
size_t out_sz = 0;
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, &out_sz);
ASSERT_EQ(HSDR_POLL_EMPTY, res);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_reject_null_hsd(void) {
uint8_t output[256];
size_t out_sz = 0;
HSD_poll_res res = heatshrink_decoder_poll(NULL, output, 256, &out_sz);
ASSERT_EQ(HSDR_POLL_ERROR_NULL, res);
PASS();
}
TEST decoder_poll_should_reject_null_output_buffer(void) {
size_t out_sz = 0;
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
HSD_poll_res res = heatshrink_decoder_poll(hsd, NULL, 256, &out_sz);
ASSERT_EQ(HSDR_POLL_ERROR_NULL, res);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_reject_null_output_size_pointer(void) {
uint8_t output[256];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256,
HEATSHRINK_MIN_WINDOW_BITS, 4);
HSD_poll_res res = heatshrink_decoder_poll(hsd, output, 256, NULL);
ASSERT_EQ(HSDR_POLL_ERROR_NULL, res);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_expand_short_literal(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0 }; //"foo"
uint8_t output[4];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 3);
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
ASSERT_EQ(3, out_sz);
ASSERT_EQ('f', output[0]);
ASSERT_EQ('o', output[1]);
ASSERT_EQ('o', output[2]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_expand_short_literal_and_backref(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo"
uint8_t output[6];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
memset(output, 0, sizeof(*output));
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
(void)heatshrink_decoder_poll(hsd, output, 6, &out_sz);
if (0) dump_buf("output", output, out_sz);
ASSERT_EQ(6, out_sz);
ASSERT_EQ('f', output[0]);
ASSERT_EQ('o', output[1]);
ASSERT_EQ('o', output[2]);
ASSERT_EQ('f', output[3]);
ASSERT_EQ('o', output[4]);
ASSERT_EQ('o', output[5]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_expand_short_self_overlapping_backref(void) {
/* "aaaaa" == (literal, 1), ('a'), (backref, 1 back, 4 bytes) */
uint8_t input[] = {0xb0, 0x80, 0x01, 0x80};
uint8_t output[6];
uint8_t expected[] = {'a', 'a', 'a', 'a', 'a'};
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 7);
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
(void)heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz);
if (0) dump_buf("output", output, out_sz);
ASSERT_EQ(sizeof(expected), out_sz);
for (size_t i=0; i<sizeof(expected); i++) ASSERT_EQ(expected[i], output[i]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_suspend_if_out_of_space_in_output_buffer_during_literal_expansion(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80};
uint8_t output[1];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 1, &out_sz);
ASSERT_EQ(HSDR_POLL_MORE, pres);
ASSERT_EQ(1, out_sz);
ASSERT_EQ('f', output[0]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_suspend_if_out_of_space_in_output_buffer_during_backref_expansion(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo"
uint8_t output[4];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
memset(output, 0, sizeof(*output));
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, 6, &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 4, &out_sz);
ASSERT_EQ(HSDR_POLL_MORE, pres);
ASSERT_EQ(4, out_sz);
ASSERT_EQ('f', output[0]);
ASSERT_EQ('o', output[1]);
ASSERT_EQ('o', output[2]);
ASSERT_EQ('f', output[3]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_poll_should_expand_short_literal_and_backref_when_fed_input_byte_by_byte(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo"
uint8_t output[7];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
memset(output, 0, sizeof(*output));
size_t count = 0;
HSD_sink_res sres;
for (int i=0; i<6; i++) {
sres = heatshrink_decoder_sink(hsd, &input[i], 1, &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
}
heatshrink_decoder_finish(hsd);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, 7, &out_sz);
ASSERT_EQ(6, out_sz);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
ASSERT_EQ('f', output[0]);
ASSERT_EQ('o', output[1]);
ASSERT_EQ('o', output[2]);
ASSERT_EQ('f', output[3]);
ASSERT_EQ('o', output[4]);
ASSERT_EQ('o', output[5]);
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_finish_should_reject_null_input(void) {
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
HSD_finish_res exp = HSDR_FINISH_ERROR_NULL;
ASSERT_EQ(exp, heatshrink_decoder_finish(NULL));
heatshrink_decoder_free(hsd);
PASS();
}
TEST decoder_finish_should_note_when_done(void) {
uint8_t input[] = {0xb3, 0x5b, 0xed, 0xe0, 0x40, 0x80}; //"foofoo"
uint8_t output[7];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 7, 7);
memset(output, 0, sizeof(*output));
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, sizeof(input), &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
ASSERT_EQ(6, out_sz);
ASSERT_EQ('f', output[0]);
ASSERT_EQ('o', output[1]);
ASSERT_EQ('o', output[2]);
ASSERT_EQ('f', output[3]);
ASSERT_EQ('o', output[4]);
ASSERT_EQ('o', output[5]);
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
ASSERT_EQ(HSDR_FINISH_DONE, fres);
heatshrink_decoder_free(hsd);
PASS();
}
TEST gen(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 7);
uint8_t input[] = {'a', 'a', 'a', 'a', 'a'};
uint8_t output[1024];
size_t copied = 0;
memset(output, 0, 1024);
HSE_sink_res sres = heatshrink_encoder_sink(hse,
input, sizeof(input), &copied);
ASSERT_EQ(HSER_SINK_OK, sres);
ASSERT_EQ(sizeof(input), copied);
HSE_finish_res fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_MORE, fres);
ASSERT_EQ(HSER_POLL_EMPTY, heatshrink_encoder_poll(hse, output, 1024, &copied));
fres = heatshrink_encoder_finish(hse);
ASSERT_EQ(HSER_FINISH_DONE, fres);
if (0) {
printf("{");
for (size_t i=0; i<copied; i++) printf("0x%02x, ", output[i]);
printf("}\n");
}
heatshrink_encoder_free(hse);
PASS();
}
TEST decoder_should_not_get_stuck_with_finish_yielding_MORE_but_0_bytes_output_from_poll(void) {
uint8_t input[512];
memset(input, 0xff, 256);
uint8_t output[1024];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 4);
ASSERT(hsd);
/* Confirm that no byte of trailing context can lead to
* heatshrink_decoder_finish erroneously returning HSDR_FINISH_MORE
* when heatshrink_decoder_poll will yield 0 bytes.
*
* Before 0.3.1, a final byte of 0xFF could potentially cause
* this to happen, if at exactly the byte boundary. */
for (uint16_t byte = 0; byte < 256; byte++) {
for (int i = 1; i < 512; i++) {
input[i] = byte;
heatshrink_decoder_reset(hsd);
memset(output, 0, sizeof(*output));
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, input, i, &count);
ASSERT_EQ(HSDR_SINK_OK, sres);
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
ASSERT_EQ(HSDR_FINISH_DONE, fres);
input[i] = 0xff;
}
}
heatshrink_decoder_free(hsd);
PASS();
}
SUITE(decoding) {
RUN_TEST(decoder_alloc_should_reject_excessively_small_window);
RUN_TEST(decoder_alloc_should_reject_zero_byte_input_buffer);
RUN_TEST(decoder_sink_should_reject_null_hsd_pointer);
RUN_TEST(decoder_sink_should_reject_null_input_pointer);
RUN_TEST(decoder_sink_should_reject_null_count_pointer);
RUN_TEST(decoder_sink_should_reject_excessively_large_input);
RUN_TEST(decoder_sink_should_sink_data_when_preconditions_hold);
RUN_TEST(gen);
RUN_TEST(decoder_poll_should_return_empty_if_empty);
RUN_TEST(decoder_poll_should_reject_null_hsd);
RUN_TEST(decoder_poll_should_reject_null_output_buffer);
RUN_TEST(decoder_poll_should_reject_null_output_size_pointer);
RUN_TEST(decoder_poll_should_expand_short_literal);
RUN_TEST(decoder_poll_should_expand_short_literal_and_backref);
RUN_TEST(decoder_poll_should_expand_short_self_overlapping_backref);
RUN_TEST(decoder_poll_should_suspend_if_out_of_space_in_output_buffer_during_literal_expansion);
RUN_TEST(decoder_poll_should_suspend_if_out_of_space_in_output_buffer_during_backref_expansion);
RUN_TEST(decoder_poll_should_expand_short_literal_and_backref_when_fed_input_byte_by_byte);
RUN_TEST(decoder_finish_should_reject_null_input);
RUN_TEST(decoder_finish_should_note_when_done);
// Regressions
RUN_TEST(decoder_should_not_get_stuck_with_finish_yielding_MORE_but_0_bytes_output_from_poll);
}
typedef struct {
uint8_t log_lvl;
uint8_t window_sz2;
uint8_t lookahead_sz2;
size_t decoder_input_buffer_size;
} cfg_info;
static int compress_and_expand_and_check(uint8_t *input, uint32_t input_size, cfg_info *cfg) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(cfg->window_sz2,
cfg->lookahead_sz2);
heatshrink_decoder *hsd = heatshrink_decoder_alloc(cfg->decoder_input_buffer_size,
cfg->window_sz2, cfg->lookahead_sz2);
size_t comp_sz = input_size + (input_size/2) + 4;
size_t decomp_sz = input_size + (input_size/2) + 4;
uint8_t *comp = malloc(comp_sz);
uint8_t *decomp = malloc(decomp_sz);
if (comp == NULL) FAILm("malloc fail");
if (decomp == NULL) FAILm("malloc fail");
memset(comp, 0, comp_sz);
memset(decomp, 0, decomp_sz);
size_t count = 0;
if (cfg->log_lvl > 1) {
printf("\n^^ COMPRESSING\n");
dump_buf("input", input, input_size);
}
size_t sunk = 0;
size_t polled = 0;
while (sunk < input_size) {
ASSERT(heatshrink_encoder_sink(hse, &input[sunk], input_size - sunk, &count) >= 0);
sunk += count;
if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count);
if (sunk == input_size) {
ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse));
}
HSE_poll_res pres;
do { /* "turn the crank" */
pres = heatshrink_encoder_poll(hse, &comp[polled], comp_sz - polled, &count);
ASSERT(pres >= 0);
polled += count;
if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count);
} while (pres == HSER_POLL_MORE);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
if (polled >= comp_sz) FAILm("compression should never expand that much");
if (sunk == input_size) {
ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(hse));
}
}
if (cfg->log_lvl > 0) printf("in: %u compressed: %zu ", input_size, polled);
size_t compressed_size = polled;
sunk = 0;
polled = 0;
if (cfg->log_lvl > 1) {
printf("\n^^ DECOMPRESSING\n");
dump_buf("comp", comp, compressed_size);
}
while (sunk < compressed_size) {
ASSERT(heatshrink_decoder_sink(hsd, &comp[sunk], compressed_size - sunk, &count) >= 0);
sunk += count;
if (cfg->log_lvl > 1) printf("^^ sunk %zd\n", count);
if (sunk == compressed_size) {
ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(hsd));
}
HSD_poll_res pres;
do {
pres = heatshrink_decoder_poll(hsd, &decomp[polled],
decomp_sz - polled, &count);
ASSERT(pres >= 0);
ASSERT(count > 0);
polled += count;
if (cfg->log_lvl > 1) printf("^^ polled %zd\n", count);
} while (pres == HSDR_POLL_MORE);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
if (sunk == compressed_size) {
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
ASSERT_EQ(HSDR_FINISH_DONE, fres);
}
if (polled > input_size) {
printf("\nExpected %zd, got %zu\n", (size_t)input_size, polled);
FAILm("Decompressed data is larger than original input");
}
}
if (cfg->log_lvl > 0) printf("decompressed: %zu\n", polled);
if (polled != input_size) {
FAILm("Decompressed length does not match original input length");
}
if (cfg->log_lvl > 1) dump_buf("decomp", decomp, polled);
for (uint32_t i=0; i<input_size; i++) {
if (input[i] != decomp[i]) {
printf("*** mismatch at %d\n", i);
if (0) {
for (uint32_t j=0; j<=/*i*/ input_size; j++) {
printf("in[%d] == 0x%02x ('%c') => out[%d] == 0x%02x ('%c') %c\n",
j, input[j], isprint(input[j]) ? input[j] : '.',
j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.',
input[j] == decomp[j] ? ' ' : 'X');
}
}
}
ASSERT_EQ(input[i], decomp[i]);
}
free(comp);
free(decomp);
heatshrink_encoder_free(hse);
heatshrink_decoder_free(hsd);
PASS();
}
TEST data_without_duplication_should_match(void) {
uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 256;
return compress_and_expand_and_check(input, sizeof(input), &cfg);
}
TEST data_with_simple_repetition_should_compress_and_decompress_properly(void) {
uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b',
'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b',
'c', 'd', 'e', 'f', 'g', 'h'};
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 256;
return compress_and_expand_and_check(input, sizeof(input), &cfg);
}
TEST data_without_duplication_should_match_with_absurdly_tiny_buffers(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3);
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3);
uint8_t input[] = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i',
'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
uint8_t comp[60];
uint8_t decomp[60];
size_t count = 0;
int log = 0;
if (log) dump_buf("input", input, sizeof(input));
for (uint32_t i=0; i<sizeof(input); i++) {
ASSERT(heatshrink_encoder_sink(hse, &input[i], 1, &count) >= 0);
}
ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse));
size_t packed_count = 0;
do {
ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0);
packed_count += count;
} while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE);
if (log) dump_buf("comp", comp, packed_count);
for (uint32_t i=0; i<packed_count; i++) {
HSD_sink_res sres = heatshrink_decoder_sink(hsd, &comp[i], 1, &count);
//printf("sres is %d\n", sres);
ASSERT(sres >= 0);
}
for (uint32_t i=0; i<sizeof(input); i++) {
ASSERT(heatshrink_decoder_poll(hsd, &decomp[i], 1, &count) >= 0);
}
if (log) dump_buf("decomp", decomp, sizeof(input));
for (uint32_t i=0; i<sizeof(input); i++) ASSERT_EQ(input[i], decomp[i]);
heatshrink_encoder_free(hse);
heatshrink_decoder_free(hsd);
PASS();
}
TEST data_with_simple_repetition_should_match_with_absurdly_tiny_buffers(void) {
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 3);
heatshrink_decoder *hsd = heatshrink_decoder_alloc(256, 8, 3);
uint8_t input[] = {'a', 'b', 'c', 'a', 'b', 'c', 'd', 'a', 'b',
'c', 'd', 'e', 'a', 'b', 'c', 'd', 'e', 'f',
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'a', 'b',
'c', 'd', 'e', 'f', 'g', 'h'};
uint8_t comp[60];
uint8_t decomp[60];
size_t count = 0;
int log = 0;
if (log) dump_buf("input", input, sizeof(input));
for (uint32_t i=0; i<sizeof(input); i++) {
ASSERT(heatshrink_encoder_sink(hse, &input[i], 1, &count) >= 0);
}
ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(hse));
size_t packed_count = 0;
do {
ASSERT(heatshrink_encoder_poll(hse, &comp[packed_count], 1, &count) >= 0);
packed_count += count;
} while (heatshrink_encoder_finish(hse) == HSER_FINISH_MORE);
if (log) dump_buf("comp", comp, packed_count);
for (uint32_t i=0; i<packed_count; i++) {
HSD_sink_res sres = heatshrink_decoder_sink(hsd, &comp[i], 1, &count);
//printf("sres is %d\n", sres);
ASSERT(sres >= 0);
}
for (uint32_t i=0; i<sizeof(input); i++) {
ASSERT(heatshrink_decoder_poll(hsd, &decomp[i], 1, &count) >= 0);
}
if (log) dump_buf("decomp", decomp, sizeof(input));
for (uint32_t i=0; i<sizeof(input); i++) ASSERT_EQ(input[i], decomp[i]);
heatshrink_encoder_free(hse);
heatshrink_decoder_free(hsd);
PASS();
}
static void fill_with_pseudorandom_letters(uint8_t *buf, uint32_t size, uint32_t seed) {
uint64_t rn = 9223372036854775783; /* prime under 2^64 */
for (uint32_t i=0; i<size; i++) {
rn = rn*seed + seed;
buf[i] = (rn % 26) + 'a';
}
}
TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed, cfg_info *cfg) {
uint8_t input[size];
if (cfg->log_lvl > 0) {
printf("\n-- size %u, seed %u, input buf %zu\n",
size, seed, cfg->decoder_input_buffer_size);
}
fill_with_pseudorandom_letters(input, size, seed);
return compress_and_expand_and_check(input, size, cfg);
}
TEST small_input_buffer_should_not_impact_decoder_correctness(void) {
int size = 5;
uint8_t input[size];
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 5;
for (uint16_t i=0; i<size; i++) input[i] = 'a' + (i % 26);
if (compress_and_expand_and_check(input, size, &cfg) != 0) return -1;
PASS();
}
TEST regression_backreference_counters_should_not_roll_over(void) {
/* Searching was scanning the entire context buffer, not just
* the maximum range addressable by the backref index.*/
uint32_t size = 337;
uint32_t seed = 3;
uint8_t input[size];
fill_with_pseudorandom_letters(input, size, seed);
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 64; // 1
return compress_and_expand_and_check(input, size, &cfg);
}
TEST regression_index_fail(void) {
/* Failured when indexed, cause unknown.
*
* This has something to do with bad data at the very last
* byte being indexed, due to spillover. */
uint32_t size = 507;
uint32_t seed = 3;
uint8_t input[size];
fill_with_pseudorandom_letters(input, size, seed);
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 64;
return compress_and_expand_and_check(input, size, &cfg);
}
TEST sixty_four_k(void) {
/* Regression: An input buffer of 64k should not cause an
* overflow that leads to an infinite loop. */
uint32_t size = 64 * 1024;
uint32_t seed = 1;
uint8_t input[size];
fill_with_pseudorandom_letters(input, size, seed);
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = 3;
cfg.decoder_input_buffer_size = 64;
return compress_and_expand_and_check(input, size, &cfg);
}
SUITE(integration) {
RUN_TEST(data_without_duplication_should_match);
RUN_TEST(data_with_simple_repetition_should_compress_and_decompress_properly);
RUN_TEST(data_without_duplication_should_match_with_absurdly_tiny_buffers);
RUN_TEST(data_with_simple_repetition_should_match_with_absurdly_tiny_buffers);
// Regressions from fuzzing
RUN_TEST(small_input_buffer_should_not_impact_decoder_correctness);
RUN_TEST(regression_backreference_counters_should_not_roll_over);
RUN_TEST(regression_index_fail);
RUN_TEST(sixty_four_k);
#if __STDC_VERSION__ >= 19901L
printf("\n\nFuzzing (single-byte sizes):\n");
for (uint8_t lsize=3; lsize < 8; lsize++) {
for (uint32_t size=1; size < 128*1024L; size <<= 1) {
if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size);
for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */
if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs);
for (uint32_t seed=1; seed<=10; seed++) {
if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed);
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 8;
cfg.lookahead_sz2 = lsize;
cfg.decoder_input_buffer_size = ibs;
RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg);
}
}
}
}
printf("\nFuzzing (multi-byte sizes):\n");
for (uint8_t lsize=6; lsize < 9; lsize++) {
for (uint32_t size=1; size < 128*1024L; size <<= 1) {
if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size);
for (uint16_t ibs=32; ibs<=8192; ibs <<= 1) { /* input buffer size */
if (GREATEST_IS_VERBOSE()) printf(" -- input buffer %u\n", ibs);
for (uint32_t seed=1; seed<=10; seed++) {
if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed);
cfg_info cfg;
cfg.log_lvl = 0;
cfg.window_sz2 = 11;
cfg.lookahead_sz2 = lsize;
cfg.decoder_input_buffer_size = ibs;
RUN_TESTp(pseudorandom_data_should_match, size, seed, &cfg);
}
}
}
}
#endif
}
/* Add all the definitions that need to be in the test runner's main file. */
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */
RUN_SUITE(encoding);
RUN_SUITE(decoding);
RUN_SUITE(integration);
#ifdef HEATSHRINK_HAS_THEFT
RUN_SUITE(properties);
#endif
GREATEST_MAIN_END(); /* display results */
}

521
espfs/heatshrink/test_heatshrink_dynamic_theft.c
Unescape View File

@ -0,0 +1,521 @@
#include "heatshrink_config.h"
#ifdef HEATSHRINK_HAS_THEFT
#include <stdint.h>
#include <ctype.h>
#include <assert.h>
#include <string.h>
#include <sys/time.h>
#include "heatshrink_encoder.h"
#include "heatshrink_decoder.h"
#include "greatest.h"
#include "theft.h"
#include "greatest_theft.h"
#if !HEATSHRINK_DYNAMIC_ALLOC
#error Must set HEATSHRINK_DYNAMIC_ALLOC to 1 for this test suite.
#endif
SUITE(properties);
typedef struct {
int limit;
int fails;
int dots;
} test_env;
typedef struct {
size_t size;
uint8_t buf[];
} rbuf;
static void *rbuf_alloc_cb(struct theft *t, theft_hash seed, void *env) {
test_env *te = (test_env *)env;
//printf("seed is 0x%016llx\n", seed);
size_t sz = (size_t)(seed % te->limit) + 1;
rbuf *r = malloc(sizeof(rbuf) + sz);
if (r == NULL) { return THEFT_ERROR; }
r->size = sz;
for (size_t i = 0; i < sz; i += sizeof(theft_hash)) {
theft_hash s = theft_random(t);
for (uint8_t b = 0; b < sizeof(theft_hash); b++) {
if (i + b >= sz) { break; }
r->buf[i + b] = (uint8_t) (s >> (8*b)) & 0xff;
}
}
return r;
}
static void rbuf_free_cb(void *instance, void *env) {
free(instance);
(void)env;
}
static uint64_t rbuf_hash_cb(void *instance, void *env) {
rbuf *r = (rbuf *)instance;
(void)env;
return theft_hash_onepass(r->buf, r->size);
}
/* Make a copy of a buffer, keeping NEW_SZ bytes starting at OFFSET. */
static void *copy_rbuf_subset(rbuf *cur, size_t new_sz, size_t byte_offset) {
if (new_sz == 0) { return THEFT_DEAD_END; }
rbuf *nr = malloc(sizeof(rbuf) + new_sz);
if (nr == NULL) { return THEFT_ERROR; }
nr->size = new_sz;
memcpy(nr->buf, &cur->buf[byte_offset], new_sz);
/* printf("%zu -> %zu\n", cur->size, new_sz); */
return nr;
}
/* Make a copy of a buffer, but only PORTION, starting OFFSET in
* (e.g. the third quarter is (0.25 at +0.75). Rounds to ints. */
static void *copy_rbuf_percent(rbuf *cur, float portion, float offset) {
size_t new_sz = cur->size * portion;
size_t byte_offset = (size_t)(cur->size * offset);
return copy_rbuf_subset(cur, new_sz, byte_offset);
}
/* How to shrink a random buffer to a simpler one. */
static void *rbuf_shrink_cb(void *instance, uint32_t tactic, void *env) {
rbuf *cur = (rbuf *)instance;
if (tactic == 0) { /* first half */
return copy_rbuf_percent(cur, 0.5, 0);
} else if (tactic == 1) { /* second half */
return copy_rbuf_percent(cur, 0.5, 0.5);
} else if (tactic <= 18) { /* drop 1-16 bytes at start */
const int last_tactic = 1;
const size_t drop = tactic - last_tactic;
if (cur->size < drop) { return THEFT_DEAD_END; }
return copy_rbuf_subset(cur, cur->size - drop, drop);
} else if (tactic <= 34) { /* drop 1-16 bytes at end */
const int last_tactic = 18;
const size_t drop = tactic - last_tactic;
if (cur->size < drop) { return THEFT_DEAD_END; }
return copy_rbuf_subset(cur, cur->size - drop, 0);
} else if (tactic == 35) {
/* Divide every byte by 2, saturating at 0 */
rbuf *cp = copy_rbuf_percent(cur, 1, 0);
if (cp == NULL) { return THEFT_ERROR; }
for (size_t i = 0; i < cp->size; i++) { cp->buf[i] /= 2; }
return cp;
} else if (tactic == 36) {
/* subtract 1 from every byte, saturating at 0 */
rbuf *cp = copy_rbuf_percent(cur, 1, 0);
if (cp == NULL) { return THEFT_ERROR; }
for (size_t i = 0; i < cp->size; i++) {
if (cp->buf[i] > 0) { cp->buf[i]--; }
}
return cp;
} else {
(void)env;
return THEFT_NO_MORE_TACTICS;
}
return THEFT_NO_MORE_TACTICS;
}
static void rbuf_print_cb(FILE *f, void *instance, void *env) {
rbuf *r = (rbuf *)instance;
(void)env;
fprintf(f, "buf[%zd]:\n ", r->size);
uint8_t bytes = 0;
for (size_t i = 0; i < r->size; i++) {
fprintf(f, "%02x", r->buf[i]);
bytes++;
if (bytes == 16) {
fprintf(f, "\n ");
bytes = 0;
}
}
fprintf(f, "\n");
}
static struct theft_type_info rbuf_info = {
.alloc = rbuf_alloc_cb,
.free = rbuf_free_cb,
.hash = rbuf_hash_cb,
.shrink = rbuf_shrink_cb,
.print = rbuf_print_cb,
};
static theft_progress_callback_res
progress_cb(struct theft_trial_info *info, void *env) {
test_env *te = (test_env *)env;
if ((info->trial & 0xff) == 0) {
printf(".");
fflush(stdout);
te->dots++;
if (te->dots == 64) {
printf("\n");
te->dots = 0;
}
}
if (info->status == THEFT_TRIAL_FAIL) {
te->fails++;
rbuf *cur = info->args[0];
if (cur->size < 5) { return THEFT_PROGRESS_HALT; }
}
if (te->fails > 10) {
return THEFT_PROGRESS_HALT;
}
return THEFT_PROGRESS_CONTINUE;
}
/* For an arbitrary input buffer, it should never get stuck in a
* state where the data has been sunk but no data can be polled. */
static theft_trial_res prop_should_not_get_stuck(void *input) {
/* Make a buffer large enough for the output: 4 KB of input with
* each 16 bits becoming up to 16 bytes will fit in a 64 KB buffer.
* (4 KB of input comes from `env.limit = 1 << 12;` below.) */
uint8_t output[64 * 1024];
heatshrink_decoder *hsd = heatshrink_decoder_alloc((64 * 1024L) - 1, 12, 4);
if (hsd == NULL) { return THEFT_TRIAL_ERROR; }
rbuf *r = (rbuf *)input;
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, r->buf, r->size, &count);
if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; }
size_t out_sz = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, output, sizeof(output), &out_sz);
if (pres != HSDR_POLL_EMPTY) { return THEFT_TRIAL_FAIL; }
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
heatshrink_decoder_free(hsd);
if (fres != HSDR_FINISH_DONE) { return THEFT_TRIAL_FAIL; }
return THEFT_TRIAL_PASS;
}
static bool get_time_seed(theft_seed *seed)
{
struct timeval tv;
if (-1 == gettimeofday(&tv, NULL)) { return false; }
*seed = (theft_seed)((tv.tv_sec << 32) | tv.tv_usec);
/* printf("seed is 0x%016llx\n", *seed); */
return true;
}
TEST decoder_fuzzing_should_not_detect_stuck_state(void) {
// Get a random number seed based on the time
theft_seed seed;
if (!get_time_seed(&seed)) { FAIL(); }
/* Pass the max buffer size for this property (4 KB) in a closure */
test_env env = { .limit = 1 << 12 };
theft_seed always_seeds = { 0xe87bb1f61032a061 };
struct theft *t = theft_init(0);
struct theft_cfg cfg = {
.name = __func__,
.fun = prop_should_not_get_stuck,
.type_info = { &rbuf_info },
.seed = seed,
.trials = 100000,
.progress_cb = progress_cb,
.env = &env,
.always_seeds = &always_seeds,
.always_seed_count = 1,
};
theft_run_res res = theft_run(t, &cfg);
theft_free(t);
printf("\n");
GREATEST_ASSERT_EQm("should_not_get_stuck", THEFT_RUN_PASS, res);
PASS();
}
static theft_trial_res prop_encoded_and_decoded_data_should_match(void *input) {
uint8_t e_output[64 * 1024];
uint8_t d_output[64 * 1024];
heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4);
if (hse == NULL) { return THEFT_TRIAL_ERROR; }
heatshrink_decoder *hsd = heatshrink_decoder_alloc(4096, 12, 4);
if (hsd == NULL) { return THEFT_TRIAL_ERROR; }
rbuf *r = (rbuf *)input;
size_t e_input_size = 0;
HSE_sink_res esres = heatshrink_encoder_sink(hse,
r->buf, r->size, &e_input_size);
if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; }
if (e_input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
HSE_finish_res efres = heatshrink_encoder_finish(hse);
if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
size_t e_output_size = 0;
HSE_poll_res epres = heatshrink_encoder_poll(hse,
e_output, sizeof(e_output), &e_output_size);
if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
size_t count = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd, e_output, e_output_size, &count);
if (sres != HSDR_SINK_OK) { return THEFT_TRIAL_ERROR; }
size_t d_output_size = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd, d_output,
sizeof(d_output), &d_output_size);
if (pres != HSDR_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
if (d_output_size != r->size) {
printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL;
}
if (0 != memcmp(d_output, r->buf, d_output_size)) {
return THEFT_TRIAL_FAIL;
}
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
if (fres != HSDR_FINISH_DONE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
heatshrink_encoder_free(hse);
heatshrink_decoder_free(hsd);
return THEFT_TRIAL_PASS;
}
TEST encoded_and_decoded_data_should_match(void) {
test_env env = { .limit = 1 << 11 };
theft_seed seed;
if (!get_time_seed(&seed)) { FAIL(); }
struct theft *t = theft_init(0);
struct theft_cfg cfg = {
.name = __func__,
.fun = prop_encoded_and_decoded_data_should_match,
.type_info = { &rbuf_info },
.seed = seed,
.trials = 1000000,
.env = &env,
.progress_cb = progress_cb,
};
theft_run_res res = theft_run(t, &cfg);
theft_free(t);
printf("\n");
ASSERT_EQ(THEFT_RUN_PASS, res);
PASS();
}
static size_t ceil_nine_eighths(size_t sz) {
return sz + sz/8 + (sz & 0x07 ? 1 : 0);
}
static theft_trial_res
prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void *input) {
uint8_t output[32 * 1024];
heatshrink_encoder *hse = heatshrink_encoder_alloc(12, 4);
if (hse == NULL) { return THEFT_TRIAL_ERROR; }
rbuf *r = (rbuf *)input;
size_t input_size = 0;
HSE_sink_res esres = heatshrink_encoder_sink(hse,
r->buf, r->size, &input_size);
if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; }
/* Assumes data fits in one sink, failure here means buffer must be larger. */
if (input_size != r->size) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
HSE_finish_res efres = heatshrink_encoder_finish(hse);
if (efres != HSER_FINISH_MORE) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
size_t output_size = 0;
HSE_poll_res epres = heatshrink_encoder_poll(hse,
output, sizeof(output), &output_size);
if (epres != HSER_POLL_EMPTY) { printf("FAIL %d\n", __LINE__); return THEFT_TRIAL_FAIL; }
size_t ceil_9_8s = ceil_nine_eighths(r->size);
if (output_size > ceil_9_8s) {
return THEFT_TRIAL_FAIL;
}
heatshrink_encoder_free(hse);
return THEFT_TRIAL_PASS;
}
TEST encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst(void) {
test_env env = { .limit = 1 << 11 };
theft_seed seed;
if (!get_time_seed(&seed)) { FAIL(); }
struct theft *t = theft_init(0);
struct theft_cfg cfg = {
.name = __func__,
.fun = prop_encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst,
.type_info = { &rbuf_info },
.seed = seed,
.trials = 10000,
.env = &env,
.progress_cb = progress_cb,
};
theft_run_res res = theft_run(t, &cfg);
theft_free(t);
printf("\n");
ASSERT_EQ(THEFT_RUN_PASS, res);
PASS();
}
static theft_trial_res
prop_encoder_should_always_make_progress(void *instance) {
uint8_t output[64 * 1024];
heatshrink_encoder *hse = heatshrink_encoder_alloc(8, 4);
if (hse == NULL) { return THEFT_TRIAL_ERROR; }
rbuf *r = (rbuf *)instance;
size_t sunk = 0;
int no_progress = 0;
while (1) {
if (sunk < r->size) {
size_t input_size = 0;
HSE_sink_res esres = heatshrink_encoder_sink(hse,
&r->buf[sunk], r->size - sunk, &input_size);
if (esres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; }
sunk += input_size;
} else {
HSE_finish_res efres = heatshrink_encoder_finish(hse);
if (efres == HSER_FINISH_DONE) {
break;
} else if (efres != HSER_FINISH_MORE) {
printf("FAIL %d\n", __LINE__);
return THEFT_TRIAL_FAIL;
}
}
size_t output_size = 0;
HSE_poll_res epres = heatshrink_encoder_poll(hse,
output, sizeof(output), &output_size);
if (epres < 0) { return THEFT_TRIAL_ERROR; }
if (output_size == 0 && sunk == r->size) {
no_progress++;
if (no_progress > 2) {
return THEFT_TRIAL_FAIL;
}
} else {
no_progress = 0;
}
}
heatshrink_encoder_free(hse);
return THEFT_TRIAL_PASS;
}
TEST encoder_should_always_make_progress(void) {
test_env env = { .limit = 1 << 15 };
theft_seed seed;
if (!get_time_seed(&seed)) { FAIL(); }
struct theft *t = theft_init(0);
struct theft_cfg cfg = {
.name = __func__,
.fun = prop_encoder_should_always_make_progress,
.type_info = { &rbuf_info },
.seed = seed,
.trials = 10000,
.env = &env,
.progress_cb = progress_cb,
};
theft_run_res res = theft_run(t, &cfg);
theft_free(t);
printf("\n");
ASSERT_EQ(THEFT_RUN_PASS, res);
PASS();
}
static theft_trial_res
prop_decoder_should_always_make_progress(void *instance) {
uint8_t output[64 * 1024];
heatshrink_decoder *hsd = heatshrink_decoder_alloc(512, 8, 4);
if (hsd == NULL) { return THEFT_TRIAL_ERROR; }
rbuf *r = (rbuf *)instance;
size_t sunk = 0;
int no_progress = 0;
while (1) {
if (sunk < r->size) {
size_t input_size = 0;
HSD_sink_res sres = heatshrink_decoder_sink(hsd,
&r->buf[sunk], r->size - sunk, &input_size);
if (sres != HSER_SINK_OK) { return THEFT_TRIAL_ERROR; }
sunk += input_size;
} else {
HSD_finish_res fres = heatshrink_decoder_finish(hsd);
if (fres == HSDR_FINISH_DONE) {
break;
} else if (fres != HSDR_FINISH_MORE) {
printf("FAIL %d\n", __LINE__);
return THEFT_TRIAL_FAIL;
}
}
size_t output_size = 0;
HSD_poll_res pres = heatshrink_decoder_poll(hsd,
output, sizeof(output), &output_size);
if (pres < 0) { return THEFT_TRIAL_ERROR; }
if (output_size == 0 && sunk == r->size) {
no_progress++;
if (no_progress > 2) {
return THEFT_TRIAL_FAIL;
}
} else {
no_progress = 0;
}
}
heatshrink_decoder_free(hsd);
return THEFT_TRIAL_PASS;
}
TEST decoder_should_always_make_progress(void) {
test_env env = { .limit = 1 << 15 };
theft_seed seed;
if (!get_time_seed(&seed)) { FAIL(); }
struct theft *t = theft_init(0);
struct theft_cfg cfg = {
.name = __func__,
.fun = prop_decoder_should_always_make_progress,
.type_info = { &rbuf_info },
.seed = seed,
.trials = 10000,
.env = &env,
.progress_cb = progress_cb,
};
theft_run_res res = theft_run(t, &cfg);
theft_free(t);
printf("\n");
ASSERT_EQ(THEFT_RUN_PASS, res);
PASS();
}
SUITE(properties) {
RUN_TEST(decoder_fuzzing_should_not_detect_stuck_state);
RUN_TEST(encoded_and_decoded_data_should_match);
RUN_TEST(encoding_data_should_never_increase_it_by_more_than_an_eighth_at_worst);
RUN_TEST(encoder_should_always_make_progress);
RUN_TEST(decoder_should_always_make_progress);
}
#else
struct because_iso_c_requires_at_least_one_declaration;
#endif

167
espfs/heatshrink/test_heatshrink_static.c
Unescape View File

@ -0,0 +1,167 @@
#include <stdint.h>
#include <ctype.h>
#include "heatshrink_encoder.h"
#include "heatshrink_decoder.h"
#include "greatest.h"
#if HEATSHRINK_DYNAMIC_ALLOC
#error HEATSHRINK_DYNAMIC_ALLOC must be false for static allocation test suite.
#endif
SUITE(integration);
/* The majority of the tests are in test_heatshrink_dynamic, because that allows
* instantiating encoders/decoders with different settings at run-time. */
static heatshrink_encoder hse;
static heatshrink_decoder hsd;
static void fill_with_pseudorandom_letters(uint8_t *buf, uint16_t size, uint32_t seed) {
uint64_t rn = 9223372036854775783; /* prime under 2^64 */
for (int i=0; i<size; i++) {
rn = rn*seed + seed;
buf[i] = (rn % 26) + 'a';
}
}
static void dump_buf(char *name, uint8_t *buf, uint16_t count) {
for (int i=0; i<count; i++) {
uint8_t c = (uint8_t)buf[i];
printf("%s %d: 0x%02x ('%c')\n", name, i, c, isprint(c) ? c : '.');
}
}
static int compress_and_expand_and_check(uint8_t *input, uint32_t input_size, int log_lvl) {
heatshrink_encoder_reset(&hse);
heatshrink_decoder_reset(&hsd);
size_t comp_sz = input_size + (input_size/2) + 4;
size_t decomp_sz = input_size + (input_size/2) + 4;
uint8_t *comp = malloc(comp_sz);
uint8_t *decomp = malloc(decomp_sz);
if (comp == NULL) FAILm("malloc fail");
if (decomp == NULL) FAILm("malloc fail");
memset(comp, 0, comp_sz);
memset(decomp, 0, decomp_sz);
size_t count = 0;
if (log_lvl > 1) {
printf("\n^^ COMPRESSING\n");
dump_buf("input", input, input_size);
}
uint32_t sunk = 0;
uint32_t polled = 0;
while (sunk < input_size) {
ASSERT(heatshrink_encoder_sink(&hse, &input[sunk], input_size - sunk, &count) >= 0);
sunk += count;
if (log_lvl > 1) printf("^^ sunk %zd\n", count);
if (sunk == input_size) {
ASSERT_EQ(HSER_FINISH_MORE, heatshrink_encoder_finish(&hse));
}
HSE_poll_res pres;
do { /* "turn the crank" */
pres = heatshrink_encoder_poll(&hse, &comp[polled], comp_sz - polled, &count);
ASSERT(pres >= 0);
polled += count;
if (log_lvl > 1) printf("^^ polled %zd\n", count);
} while (pres == HSER_POLL_MORE);
ASSERT_EQ(HSER_POLL_EMPTY, pres);
if (polled >= comp_sz) FAILm("compression should never expand that much");
if (sunk == input_size) {
ASSERT_EQ(HSER_FINISH_DONE, heatshrink_encoder_finish(&hse));
}
}
if (log_lvl > 0) printf("in: %u compressed: %u ", input_size, polled);
uint32_t compressed_size = polled;
sunk = 0;
polled = 0;
if (log_lvl > 1) {
printf("\n^^ DECOMPRESSING\n");
dump_buf("comp", comp, compressed_size);
}
while (sunk < compressed_size) {
ASSERT(heatshrink_decoder_sink(&hsd, &comp[sunk], compressed_size - sunk, &count) >= 0);
sunk += count;
if (log_lvl > 1) printf("^^ sunk %zd\n", count);
if (sunk == compressed_size) {
ASSERT_EQ(HSDR_FINISH_MORE, heatshrink_decoder_finish(&hsd));
}
HSD_poll_res pres;
do {
pres = heatshrink_decoder_poll(&hsd, &decomp[polled],
decomp_sz - polled, &count);
ASSERT(pres >= 0);
polled += count;
if (log_lvl > 1) printf("^^ polled %zd\n", count);
} while (pres == HSDR_POLL_MORE);
ASSERT_EQ(HSDR_POLL_EMPTY, pres);
if (sunk == compressed_size) {
HSD_finish_res fres = heatshrink_decoder_finish(&hsd);
ASSERT_EQ(HSDR_FINISH_DONE, fres);
}
if (polled > input_size) {
FAILm("Decompressed data is larger than original input");
}
}
if (log_lvl > 0) printf("decompressed: %u\n", polled);
if (polled != input_size) {
FAILm("Decompressed length does not match original input length");
}
if (log_lvl > 1) dump_buf("decomp", decomp, polled);
for (size_t i=0; i<input_size; i++) {
if (input[i] != decomp[i]) {
printf("*** mismatch at %zd\n", i);
if (0) {
for (size_t j=0; j<=/*i*/ input_size; j++) {
printf("in[%zd] == 0x%02x ('%c') => out[%zd] == 0x%02x ('%c')\n",
j, input[j], isprint(input[j]) ? input[j] : '.',
j, decomp[j], isprint(decomp[j]) ? decomp[j] : '.');
}
}
}
ASSERT_EQ(input[i], decomp[i]);
}
free(comp);
free(decomp);
PASS();
}
TEST pseudorandom_data_should_match(uint32_t size, uint32_t seed) {
uint8_t input[size];
fill_with_pseudorandom_letters(input, size, seed);
return compress_and_expand_and_check(input, size, 0);
}
SUITE(integration) {
#if __STDC_VERSION__ >= 19901L
for (uint32_t size=1; size < 64*1024; size <<= 1) {
if (GREATEST_IS_VERBOSE()) printf(" -- size %u\n", size);
for (uint32_t seed=1; seed<=100; seed++) {
if (GREATEST_IS_VERBOSE()) printf(" -- seed %u\n", seed);
RUN_TESTp(pseudorandom_data_should_match, size, seed);
}
}
#endif
}
/* Add all the definitions that need to be in the test runner's main file. */
GREATEST_MAIN_DEFS();
int main(int argc, char **argv) {
GREATEST_MAIN_BEGIN(); /* command-line arguments, initialization. */
printf("INPUT_BUFFER_SIZE: %u\n", HEATSHRINK_STATIC_INPUT_BUFFER_SIZE);
printf("WINDOW_BITS: %u\n", HEATSHRINK_STATIC_WINDOW_BITS);
printf("LOOKAHEAD_BITS: %u\n", HEATSHRINK_STATIC_LOOKAHEAD_BITS);
printf("sizeof(heatshrink_encoder): %zd\n", sizeof(heatshrink_encoder));
printf("sizeof(heatshrink_decoder): %zd\n", sizeof(heatshrink_decoder));
RUN_SUITE(integration);
GREATEST_MAIN_END(); /* display results */
}

2
espfs/heatshrink_decoder.c
Unescape View File

@ -13,7 +13,7 @@
#endif #endif
#include "heatshrink_config_custom.h" #include "heatshrink_config_custom.h"
#include "../lib/heatshrink/heatshrink_decoder.c" #include "heatshrink/heatshrink_decoder.c"
#endif #endif

32
espfs/mkespfsimage/Makefile
Unescape View File

@ -1,24 +1,36 @@
GZIP_COMPRESSION ?= no GZIP_COMPRESSION ?= no
USE_HEATSHRINK ?= yes USE_HEATSHRINK ?= yes
CFLAGS=-I../../lib/heatshrink -I.. -std=gnu99 TARGET = mkespfsimage.exe
CC = gcc
LD = $(CC)
CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99
LDFLAGS=-Lmman-win32 -lmman
ifeq ("$(GZIP_COMPRESSION)","yes") ifeq ("$(GZIP_COMPRESSION)","yes")
CFLAGS += -DESPFS_GZIP CFLAGS += -DESPFS_GZIP
LDFLAGS += -lz
endif endif
ifeq ("$(USE_HEATSHRINK)","yes") ifeq ("$(USE_HEATSHRINK)","yes")
CFLAGS += -DESPFS_HEATSHRINK CFLAGS += -DESPFS_HEATSHRINK
endif endif
OBJS=main.o heatshrink_encoder.o OBJECTS = main.o heatshrink_encoder.o
TARGET=mkespfsimage
$(TARGET): $(OBJS) all: libmman $(TARGET)
ifeq ("$(GZIP_COMPRESSION)","yes")
$(CC) -o $@ $^ -lz libmman:
else $(Q) make -C mman-win32
$(CC) -o $@ $^
endif $(TARGET): $(OBJECTS)
$(LD) -o $@ $^ $(LDFLAGS)
%.o: %.c
$(CC) $(CFLAGS) -o $@ $^
clean: clean:
rm -f $(TARGET) $(OBJS) rm -f $(OBJECTS) $(TARGET)
.PHONY: all clean

24
espfs/mkespfsimage/Makefile.linux
Unescape View File

@ -0,0 +1,24 @@
GZIP_COMPRESSION ?= no
USE_HEATSHRINK ?= yes
CFLAGS=-I../../heatshrink -I../../include -I.. -std=gnu99
ifeq ("$(GZIP_COMPRESSION)","yes")
CFLAGS += -DESPFS_GZIP
endif
ifeq ("$(USE_HEATSHRINK)","yes")
CFLAGS += -DESPFS_HEATSHRINK
endif
OBJS=main.o heatshrink_encoder.o
TARGET=mkespfsimage
$(TARGET): $(OBJS)
ifeq ("$(GZIP_COMPRESSION)","yes")
$(CC) -o $@ $^ -lz
else
$(CC) -o $@ $^
endif
clean:
rm -f $(TARGET) $(OBJS)

33
espfs/mkespfsimage/Makefile.windows
Unescape View File

@ -0,0 +1,33 @@
GZIP_COMPRESSION ?= no
USE_HEATSHRINK ?= yes
TARGET = mkespfsimage.exe
CC = gcc
LD = $(CC)
CFLAGS=-c -I../../heatshrink -I.. -Imman-win32 -std=gnu99
LDFLAGS=-Lmman-win32 -lmman
ifeq ("$(GZIP_COMPRESSION)","yes")
CFLAGS += -DESPFS_GZIP
LDFLAGS += -lz
endif
ifeq ("$(USE_HEATSHRINK)","yes")
CFLAGS += -DESPFS_HEATSHRINK
endif
OBJECTS = main.o heatshrink_encoder.o
all: $(TARGET)
$(TARGET): $(OBJECTS)
$(LD) -o $@ $^ $(LDFLAGS)
%.o: %.c
$(CC) $(CFLAGS) -o $@ $^
clean:
rm -f $(OBJECTS) $(TARGET)
.PHONY: all clean

4
espfs/mkespfsimage/build-linux.sh
Unescape View File

@ -0,0 +1,4 @@
#!/bin/sh
make -f Makefile.linux clean
make -f Makefile.linux USE_HEATSHRINK="yes" GZIP_COMPRESSION="no"

7
espfs/mkespfsimage/build-win32.sh
Unescape View File

@ -0,0 +1,7 @@
#!/bin/sh
cd mman-win32
./configure && make
cd ..
make -f Makefile.windows clean
make -f Makefile.windows USE_HEATSHRINK="yes" GZIP_COMPRESSION="no"

33
espfs/mkespfsimage/espfsformat.h
Unescape View File

@ -0,0 +1,33 @@
#ifndef ESPROFSFORMAT_H
#define ESPROFSFORMAT_H
/*
Stupid cpio-like tool to make read-only 'filesystems' that live on the flash SPI chip of the module.
Can (will) use lzf compression (when I come around to it) to make shit quicker. Aligns names, files,
headers on 4-byte boundaries so the SPI abstraction hardware in the ESP8266 doesn't crap on itself
when trying to do a <4byte or unaligned read.
*/
/*
The idea 'borrows' from cpio: it's basically a concatenation of {header, filename, file} data.
Header, filename and file data is 32-bit aligned. The last file is indicated by data-less header
with the FLAG_LASTFILE flag set.
*/
#define FLAG_LASTFILE (1<<0)
#define FLAG_GZIP (1<<1)
#define COMPRESS_NONE 0
#define COMPRESS_HEATSHRINK 1
#define ESPFS_MAGIC 0x73665345
typedef struct {
int32_t magic;
int8_t flags;
int8_t compression;
int16_t nameLen;
int32_t fileLenComp;
int32_t fileLenDecomp;
} __attribute__((packed)) EspFsHeader;
#endif

2
espfs/mkespfsimage/heatshrink_encoder.c
Unescape View File

@ -1,4 +1,4 @@
//Stupid wraparound include to make sure object file doesn't end up in heatshrink dir //Stupid wraparound include to make sure object file doesn't end up in heatshrink dir
#ifdef ESPFS_HEATSHRINK #ifdef ESPFS_HEATSHRINK
#include "../lib/heatshrink/heatshrink_encoder.c" #include "../heatshrink/heatshrink_encoder.c"
#endif #endif

31
espfs/mkespfsimage/main.c
Unescape View File

@ -5,17 +5,26 @@
#include <fcntl.h> #include <fcntl.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h>
#include "espfs.h"
#ifdef __MINGW32__
#include <mman.h>
#else
#include <sys/mman.h> #include <sys/mman.h>
#endif
#ifdef __WIN32__
#include <winsock2.h>
#else
#include <arpa/inet.h> #include <arpa/inet.h>
#endif
#include <string.h> #include <string.h>
#include "espfs.h"
#include "espfsformat.h" #include "espfsformat.h"
//Heatshrink //Heatshrink
#ifdef ESPFS_HEATSHRINK #ifdef ESPFS_HEATSHRINK
#include "heatshrink_common.h" #include "../heatshrink/heatshrink_common.h"
#include "heatshrink_config.h" #include "../heatshrink/heatshrink_config.h"
#include "heatshrink_encoder.h" #include "../heatshrink/heatshrink_encoder.h"
#endif #endif
//Gzip //Gzip
@ -175,7 +184,7 @@ int parseGzipExtensions(char *input) {
} }
#endif #endif
int handleFile(int f, char *name, int compression, int level, char **compName, off_t *csizePtr) { int handleFile(int f, char *name, int compression, int level, char **compName) {
char *fdat, *cdat; char *fdat, *cdat;
off_t size, csize; off_t size, csize;
EspFsHeader h; EspFsHeader h;
@ -257,7 +266,6 @@ int handleFile(int f, char *name, int compression, int level, char **compName, o
*compName = "unknown"; *compName = "unknown";
} }
} }
*csizePtr = csize;
return (csize*100)/size; return (csize*100)/size;
} }
@ -310,7 +318,7 @@ int main(int argc, char **argv) {
#ifdef ESPFS_GZIP #ifdef ESPFS_GZIP
if (gzipExtensions == NULL) { if (gzipExtensions == NULL) {
parseGzipExtensions(strdup("html,css,js,ico")); parseGzipExtensions(strdup("html,css,js"));
} }
#endif #endif
@ -334,6 +342,10 @@ int main(int argc, char **argv) {
exit(0); exit(0);
} }
#ifdef __WIN32__
setmode(fileno(stdout), _O_BINARY);
#endif
while(fgets(fileName, sizeof(fileName), stdin)) { while(fgets(fileName, sizeof(fileName), stdin)) {
//Kill off '\n' at the end //Kill off '\n' at the end
fileName[strlen(fileName)-1]=0; fileName[strlen(fileName)-1]=0;
@ -347,9 +359,8 @@ int main(int argc, char **argv) {
f=open(fileName, O_RDONLY); f=open(fileName, O_RDONLY);
if (f>0) { if (f>0) {
char *compName = "unknown"; char *compName = "unknown";
off_t csize; rate=handleFile(f, realName, compType, compLvl, &compName);
rate=handleFile(f, realName, compType, compLvl, &compName, &csize); fprintf(stderr, "%s (%d%%, %s)\n", realName, rate, compName);
fprintf(stderr, "%-16s (%3d%%, %s, %4u bytes)\n", realName, rate, compName, (uint32_t)csize);
close(f); close(f);
} else { } else {
perror(fileName); perror(fileName);

48
espfs/mkespfsimage/mman-win32/Makefile
Unescape View File

@ -0,0 +1,48 @@
#
# mman-win32 (mingw32) Makefile
#
include config.mak
ifeq ($(BUILD_STATIC),yes)
TARGETS+=libmman.a
INSTALL+=static-install
endif
ifeq ($(BUILD_MSVC),yes)
SHFLAGS+=-Wl,--output-def,libmman.def
INSTALL+=lib-install
endif
all: $(TARGETS)
mman.o: mman.c mman.h
$(CC) -o mman.o -c mman.c -Wall -O3 -fomit-frame-pointer
libmman.a: mman.o
$(AR) cru libmman.a mman.o
$(RANLIB) libmman.a
static-install:
mkdir -p $(DESTDIR)$(libdir)
cp libmman.a $(DESTDIR)$(libdir)
mkdir -p $(DESTDIR)$(incdir)
cp mman.h $(DESTDIR)$(incdir)
lib-install:
mkdir -p $(DESTDIR)$(libdir)
cp libmman.lib $(DESTDIR)$(libdir)
install: $(INSTALL)
test.exe: test.c mman.c mman.h
$(CC) -o test.exe test.c -L. -lmman
test: $(TARGETS) test.exe
test.exe
clean::
rm -f mman.o libmman.a libmman.def libmman.lib test.exe *.dat
distclean: clean
rm -f config.mak
.PHONY: clean distclean install test

11
espfs/mkespfsimage/mman-win32/config.mak
Unescape View File

@ -0,0 +1,11 @@
# Automatically generated by configure
PREFIX=/mingw
libdir=/mingw/lib
incdir=/mingw/include/sys
AR=ar
CC=gcc
RANLIB=ranlib
STRIP=strip
BUILD_STATIC=yes
BUILD_MSVC=
LIBCMD=echo ignoring lib

157
espfs/mkespfsimage/mman-win32/configure vendored
Unescape View File

@ -0,0 +1,157 @@
#!/bin/sh
# mmap-win32 configure script
#
# Parts copied from FFmpeg's configure
#
set_all(){
value=$1
shift
for var in $*; do
eval $var=$value
done
}
enable(){
set_all yes $*
}
disable(){
set_all no $*
}
enabled(){
eval test "x\$$1" = "xyes"
}
disabled(){
eval test "x\$$1" = "xno"
}
show_help(){
echo "Usage: configure [options]"
echo "Options: [defaults in brackets after descriptions]"
echo "All \"enable\" options have \"disable\" counterparts"
echo
echo " --help print this message"
echo " --prefix=PREFIX install in PREFIX [$PREFIX]"
echo " --libdir=DIR install libs in DIR [$PREFIX/lib]"
echo " --incdir=DIR install includes in DIR [$PREFIX/include]"
echo " --enable-static build static libraries [yes]"
echo " --enable-msvc create msvc-compatible import lib [auto]"
echo
echo " --cc=CC use C compiler CC [$cc_default]"
echo " --cross-prefix=PREFIX use PREFIX for compilation tools [$cross_prefix]"
exit 1
}
die_unknown(){
echo "Unknown option \"$1\"."
echo "See $0 --help for available options."
exit 1
}
PREFIX="/mingw"
libdir="${PREFIX}/lib"
incdir="${PREFIX}/include/sys"
ar="ar"
cc_default="gcc"
ranlib="ranlib"
strip="strip"
DEFAULT="msvc
"
DEFAULT_YES="static
stripping
"
CMDLINE_SELECT="$DEFAULT
$DEFAULT_NO
$DEFAULT_YES
"
enable $DEFAULT_YES
disable $DEFAULT_NO
for opt do
optval="${opt#*=}"
case "$opt" in
--help)
show_help
;;
--prefix=*)
PREFIX="$optval"
;;
--libdir=*)
libdir="$optval"
;;
--incdir=*)
incdir="$optval"
;;
--cc=*)
cc="$optval"
;;
--cross-prefix=*)
cross_prefix="$optval"
;;
--enable-?*|--disable-?*)
eval `echo "$opt" | sed 's/--/action=/;s/-/ option=/;s/-/_/g'`
echo "$CMDLINE_SELECT" | grep -q "^ *$option\$" || die_unknown $opt
$action $option
;;
*)
die_unknown $opt
;;
esac
done
ar="${cross_prefix}${ar}"
cc_default="${cross_prefix}${cc_default}"
ranlib="${cross_prefix}${ranlib}"
strip="${cross_prefix}${strip}"
if ! test -z $cc; then
cc_default="${cc}"
fi
cc="${cc_default}"
disabled static && {
echo "At least one library type must be set.";
exit 1;
}
if enabled msvc; then
lib /? > /dev/null 2>&1 /dev/null || {
echo "MSVC's lib command not found."
echo "Make sure MSVC is installed and its bin folder is in your \$PATH."
exit 1
}
fi
if ! enabled stripping; then
strip="echo ignoring strip"
fi
enabled msvc && libcmd="lib" || libcmd="echo ignoring lib"
echo "# Automatically generated by configure" > config.mak
echo "PREFIX=$PREFIX" >> config.mak
echo "libdir=$libdir" >> config.mak
echo "incdir=$incdir" >> config.mak
echo "AR=$ar" >> config.mak
echo "CC=$cc" >> config.mak
echo "RANLIB=$ranlib" >> config.mak
echo "STRIP=$strip" >> config.mak
echo "BUILD_STATIC=$static" >> config.mak
echo "BUILD_MSVC=$msvc" >> config.mak
echo "LIBCMD=$libcmd" >> config.mak
echo "prefix: $PREFIX"
echo "libdir: $libdir"
echo "incdir: $incdir"
echo "ar: $ar"
echo "cc: $cc"
echo "ranlib: $ranlib"
echo "strip: $strip"
echo "static: $static"

180
espfs/mkespfsimage/mman-win32/mman.c
Unescape View File

@ -0,0 +1,180 @@
#include <windows.h>
#include <errno.h>
#include <io.h>
#include "mman.h"
#ifndef FILE_MAP_EXECUTE
#define FILE_MAP_EXECUTE 0x0020
#endif /* FILE_MAP_EXECUTE */
static int __map_mman_error(const DWORD err, const int deferr)
{
if (err == 0)
return 0;
//TODO: implement
return err;
}
static DWORD __map_mmap_prot_page(const int prot)
{
DWORD protect = 0;
if (prot == PROT_NONE)
return protect;
if ((prot & PROT_EXEC) != 0)
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
}
else
{
protect = ((prot & PROT_WRITE) != 0) ?
PAGE_READWRITE : PAGE_READONLY;
}
return protect;
}
static DWORD __map_mmap_prot_file(const int prot)
{
DWORD desiredAccess = 0;
if (prot == PROT_NONE)
return desiredAccess;
if ((prot & PROT_READ) != 0)
desiredAccess |= FILE_MAP_READ;
if ((prot & PROT_WRITE) != 0)
desiredAccess |= FILE_MAP_WRITE;
if ((prot & PROT_EXEC) != 0)
desiredAccess |= FILE_MAP_EXECUTE;
return desiredAccess;
}
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
{
HANDLE fm, h;
void * map = MAP_FAILED;
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4293)
#endif
const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
const DWORD protect = __map_mmap_prot_page(prot);
const DWORD desiredAccess = __map_mmap_prot_file(prot);
const off_t maxSize = off + (off_t)len;
const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
(DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);
#ifdef _MSC_VER
#pragma warning(pop)
#endif
errno = 0;
if (len == 0
/* Unsupported flag combinations */
|| (flags & MAP_FIXED) != 0
/* Usupported protection combinations */
|| prot == PROT_EXEC)
{
errno = EINVAL;
return MAP_FAILED;
}
h = ((flags & MAP_ANONYMOUS) == 0) ?
(HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;
if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
{
errno = EBADF;
return MAP_FAILED;
}
fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);
if (fm == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);
CloseHandle(fm);
if (map == NULL)
{
errno = __map_mman_error(GetLastError(), EPERM);
return MAP_FAILED;
}
return map;
}
int munmap(void *addr, size_t len)
{
if (UnmapViewOfFile(addr))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mprotect(void *addr, size_t len, int prot)
{
DWORD newProtect = __map_mmap_prot_page(prot);
DWORD oldProtect = 0;
if (VirtualProtect(addr, len, newProtect, &oldProtect))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int msync(void *addr, size_t len, int flags)
{
if (FlushViewOfFile(addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int mlock(const void *addr, size_t len)
{
if (VirtualLock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}
int munlock(const void *addr, size_t len)
{
if (VirtualUnlock((LPVOID)addr, len))
return 0;
errno = __map_mman_error(GetLastError(), EPERM);
return -1;
}

55
espfs/mkespfsimage/mman-win32/mman.h
Unescape View File

@ -0,0 +1,55 @@
/*
* sys/mman.h
* mman-win32
*/
#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_
#ifndef _WIN32_WINNT // Allow use of features specific to Windows XP or later.
#define _WIN32_WINNT 0x0501 // Change this to the appropriate value to target other versions of Windows.
#endif
/* All the headers include this file. */
#ifndef _MSC_VER
#include <_mingw.h>
#endif
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
#define PROT_NONE 0
#define PROT_READ 1
#define PROT_WRITE 2
#define PROT_EXEC 4
#define MAP_FILE 0
#define MAP_SHARED 1
#define MAP_PRIVATE 2
#define MAP_TYPE 0xf
#define MAP_FIXED 0x10
#define MAP_ANONYMOUS 0x20
#define MAP_ANON MAP_ANONYMOUS
#define MAP_FAILED ((void *)-1)
/* Flags for msync. */
#define MS_ASYNC 1
#define MS_SYNC 2
#define MS_INVALIDATE 4
void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
int munmap(void *addr, size_t len);
int mprotect(void *addr, size_t len, int prot);
int msync(void *addr, size_t len, int flags);
int mlock(const void *addr, size_t len);
int munlock(const void *addr, size_t len);
#ifdef __cplusplus
};
#endif
#endif /* _SYS_MMAN_H_ */

235
espfs/mkespfsimage/mman-win32/test.c
Unescape View File

@ -0,0 +1,235 @@
#include "mman.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#ifndef NULL
#define NULL (void*)0
#endif
const char* map_file_name = "map_file.dat";
int test_anon_map_readwrite()
{
void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map == MAP_FAILED)
{
printf("mmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno);
return -1;
}
*((unsigned char*)map) = 1;
int result = munmap(map, 1024);
if (result != 0)
printf("munmap (MAP_ANONYMOUS, PROT_READ | PROT_WRITE) returned unexpected error: %d\n", errno);
return result;
}
int test_anon_map_readonly()
{
void* map = mmap(NULL, 1024, PROT_READ,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map == MAP_FAILED)
{
printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno);
return -1;
}
*((unsigned char*)map) = 1;
int result = munmap(map, 1024);
if (result != 0)
printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno);
return result;
}
int test_anon_map_writeonly()
{
void* map = mmap(NULL, 1024, PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map == MAP_FAILED)
{
printf("mmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno);
return -1;
}
*((unsigned char*)map) = 1;
int result = munmap(map, 1024);
if (result != 0)
printf("munmap (MAP_ANONYMOUS, PROT_WRITE) returned unexpected error: %d\n", errno);
return result;
}
int test_anon_map_readonly_nowrite()
{
void* map = mmap(NULL, 1024, PROT_READ,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (map == MAP_FAILED)
{
printf("mmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno);
return -1;
}
if (*((unsigned char*)map) != 0)
printf("test_anon_map_readonly_nowrite (MAP_ANONYMOUS, PROT_READ) returned unexpected value: %d\n",
(int)*((unsigned char*)map));
int result = munmap(map, 1024);
if (result != 0)
printf("munmap (MAP_ANONYMOUS, PROT_READ) returned unexpected error: %d\n", errno);
return result;
}
int test_file_map_readwrite()
{
mode_t mode = S_IRUSR | S_IWUSR;
int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode);
void* map = mmap(NULL, 1024, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0);
if (map == MAP_FAILED)
{
printf("mmap returned unexpected error: %d\n", errno);
return -1;
}
*((unsigned char*)map) = 1;
int result = munmap(map, 1024);
if (result != 0)
printf("munmap returned unexpected error: %d\n", errno);
close(o);
/*TODO: get file info and content and compare it with the sources conditions */
unlink(map_file_name);
return result;
}
int test_file_map_mlock_munlock()
{
const size_t map_size = 1024;
int result = 0;
mode_t mode = S_IRUSR | S_IWUSR;
int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode);
if (o == -1)
{
printf("unable to create file %s: %d\n", map_file_name, errno);
return -1;
}
void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0);
if (map == MAP_FAILED)
{
printf("mmap returned unexpected error: %d\n", errno);
result = -1;
goto done_close;
}
if (mlock(map, map_size) != 0)
{
printf("mlock returned unexpected error: %d\n", errno);
result = -1;
goto done_munmap;
}
*((unsigned char*)map) = 1;
if (munlock(map, map_size) != 0)
{
printf("munlock returned unexpected error: %d\n", errno);
result = -1;
}
done_munmap:
result = munmap(map, map_size);
if (result != 0)
printf("munmap returned unexpected error: %d\n", errno);
done_close:
close(o);
unlink(map_file_name);
done:
return result;
}
int test_file_map_msync()
{
const size_t map_size = 1024;
int result = 0;
mode_t mode = S_IRUSR | S_IWUSR;
int o = open(map_file_name, O_TRUNC | O_BINARY | O_RDWR | O_CREAT, mode);
if (o == -1)
{
printf("unable to create file %s: %d\n", map_file_name, errno);
return -1;
}
void* map = mmap(NULL, map_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, o, 0);
if (map == MAP_FAILED)
{
printf("mmap returned unexpected error: %d\n", errno);
result = -1;
goto done_close;
}
*((unsigned char*)map) = 1;
if (msync(map, map_size, MS_SYNC) != 0)
{
printf("msync returned unexpected error: %d\n", errno);
result = -1;
}
result = munmap(map, map_size);
if (result != 0)
printf("munmap returned unexpected error: %d\n", errno);
done_close:
close(o);
unlink(map_file_name);
done:
return result;
}
#define EXEC_TEST(name) \
if (name() != 0) { result = -1; printf( #name ": fail\n"); } \
else { printf(#name ": pass\n"); }
int main()
{
int result = 0;
EXEC_TEST(test_anon_map_readwrite);
//NOTE: this test must cause an access violation exception
//EXEC_TEST(test_anon_map_readonly);
EXEC_TEST(test_anon_map_readonly_nowrite);
EXEC_TEST(test_anon_map_writeonly);
EXEC_TEST(test_file_map_readwrite);
EXEC_TEST(test_file_map_mlock_munlock);
EXEC_TEST(test_file_map_msync);
//TODO: EXEC_TEST(test_file_map_mprotect);
return result;
}

7
espmake.cmd
Unescape View File

@ -0,0 +1,7 @@
@echo off
REM remove automatic created obj folder
rd obj /S /Q >nul 2>&1
PATH=%PATH%;C:\Espressif\xtensa-lx106-elf\bin;C:\MinGW\bin;C:\MinGW\msys\1.0\bin;C:\espressif\git-bin;C:\espressif\java-bin;C:\Python27
make -f Makefile %1 %2 %3 %4 %5
Write Preview
Loading…
Cancel
Save