From edd22ba8c65f76446aab5a58a84bee71bad7943b Mon Sep 17 00:00:00 2001
From: Kevin <68612569+diyelectromusic@users.noreply.github.com>
Date: Fri, 7 Apr 2023 20:00:48 +0100
Subject: [PATCH] Support for subdirectories in the SysEx voices directory
 (#473)

* Fix for issue #424 create a build script to check out the correct versions of sub libraries.

* Initial attempt at supporting loading SysEx files from subdirectories which seems to speed up loading significantly.

* Reinstate headerless SysEx file loading after subdirectory change

* Limit nesting of subdirectories to open

* Fix bank number in UI to match the numbers from the bank files.

* Update to fix bank numbers.  Bank file numbers now start from 00001 and show as 1-indexed in the UI.  But internally and via MIDI are still 0-indexed as per MIDI spec.
---
 src/sysexfileloader.cpp | 201 +++++++++++++++++++++++++---------------
 src/sysexfileloader.h   |   4 +
 submod.sh               |  13 +++
 3 files changed, 142 insertions(+), 76 deletions(-)
 create mode 100755 submod.sh

diff --git a/src/sysexfileloader.cpp b/src/sysexfileloader.cpp
index 4eee065..d72bc7d 100644
--- a/src/sysexfileloader.cpp
+++ b/src/sysexfileloader.cpp
@@ -84,6 +84,7 @@ CSysExFileLoader::~CSysExFileLoader (void)
 void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 {
 	m_nNumHighestBank = 0;
+	m_nBanksLoaded = 0;
 
     DIR *pDirectory = opendir (m_DirName.c_str ());
 	if (!pDirectory)
@@ -96,107 +97,155 @@ void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 	dirent *pEntry;
 	while ((pEntry = readdir (pDirectory)) != nullptr)
 	{
-		unsigned nBank;
-		size_t nLen = strlen (pEntry->d_name);
+		LoadBank(m_DirName.c_str (), pEntry->d_name, bHeaderlessSysExVoices, 0);
+	}
+	LOGDBG ("%u Banks loaded. Highest Bank loaded: #%u", m_nBanksLoaded, m_nNumHighestBank);
 
-		if (   nLen < 5						// "[NNNN]N[_name].syx"
-		    || strcasecmp (&pEntry->d_name[nLen-4], ".syx") != 0
-		    || sscanf (pEntry->d_name, "%u", &nBank) != 1)
-		{
-			LOGWARN ("%s: Invalid filename format", pEntry->d_name);
+	closedir (pDirectory);
+}
 
-			continue;
-		}
+void CSysExFileLoader::LoadBank (const char * sDirName, const char * sBankName, bool bHeaderlessSysExVoices, unsigned nSubDirCount)
+{
+	unsigned nBank;
+	size_t nLen = strlen (sBankName);
+	
+	if (   nLen < 5						// "[NNNN]N[_name].syx"
+		|| strcasecmp (&sBankName[nLen-4], ".syx") != 0
+		|| sscanf (sBankName, "%u", &nBank) != 1)
+	{
+		// See if this is a subdirectory...
+		std::string Dirname (sDirName);
+		Dirname += "/";
+		Dirname += sBankName;
 
-		if (nBank > MaxVoiceBankID)
+		DIR *pDirectory = opendir (Dirname.c_str ());
+		if (pDirectory)
 		{
-			LOGWARN ("Bank #%u is not supported", nBank);
+			if (nSubDirCount >= MaxSubDirs)
+			{
+				LOGWARN ("Too many nested subdirectories: %s", sBankName);
+				return;
+			}
+	
+			LOGDBG ("Processing subdirectory %s", sBankName);
 
-			continue;
+			dirent *pEntry;
+			while ((pEntry = readdir (pDirectory)) != nullptr)
+			{
+				LoadBank(Dirname.c_str (), pEntry->d_name, bHeaderlessSysExVoices, nSubDirCount+1);
+			}
+			closedir (pDirectory);
 		}
-
-		if (m_pVoiceBank[nBank])
+		else
 		{
-			LOGWARN ("Bank #%u already loaded", nBank);
-
-			continue;
+			LOGWARN ("%s: Invalid filename format", sBankName);
 		}
 
-		m_pVoiceBank[nBank] = new TVoiceBank;
-		assert (m_pVoiceBank[nBank]);
-		assert (sizeof(TVoiceBank) == VoiceSysExHdrSize + VoiceSysExSize);
+		return;
+	}
+	
+	// File and UI handling requires banks to be 1..indexed.
+	// Internally (and via MIDI) we need 0..indexed.
+	// Any mention of a BankID internally is assumed to be 0..indexed.
+	unsigned nBankIdx = nBank - 1;
+
+	// BankIdx goes from 0 to MaxVoiceBankID inclusive
+	if (nBankIdx > MaxVoiceBankID)
+	{
+		LOGWARN ("Bank #%u is not supported", nBank);
+
+		return;
+	}
 
-		std::string Filename (m_DirName);
-		Filename += "/";
-		Filename += pEntry->d_name;
+	if (m_pVoiceBank[nBankIdx])
+	{
+		LOGWARN ("Bank #%u already loaded", nBank);
+
+		return;
+	}
 
-		FILE *pFile = fopen (Filename.c_str (), "rb");
-		if (pFile)
+	m_pVoiceBank[nBankIdx] = new TVoiceBank;
+	assert (m_pVoiceBank[nBankIdx]);
+	assert (sizeof(TVoiceBank) == VoiceSysExHdrSize + VoiceSysExSize);
+
+	std::string Filename (sDirName);
+	Filename += "/";
+	Filename += sBankName;
+
+	FILE *pFile = fopen (Filename.c_str (), "rb");
+	if (pFile)
+	{
+		bool bBankLoaded = false;
+		if (   fread (m_pVoiceBank[nBankIdx], VoiceSysExHdrSize+VoiceSysExSize, 1, pFile) == 1
+			&& m_pVoiceBank[nBankIdx]->StatusStart == 0xF0
+			&& m_pVoiceBank[nBankIdx]->CompanyID   == 0x43
+			&& m_pVoiceBank[nBankIdx]->Format      == 0x09
+			&& m_pVoiceBank[nBankIdx]->StatusEnd   == 0xF7)
 		{
-			bool bBankLoaded = false;
-			if (   fread (m_pVoiceBank[nBank], VoiceSysExHdrSize+VoiceSysExSize, 1, pFile) == 1
-			    && m_pVoiceBank[nBank]->StatusStart == 0xF0
-			    && m_pVoiceBank[nBank]->CompanyID   == 0x43
-			    && m_pVoiceBank[nBank]->Format      == 0x09
-			    && m_pVoiceBank[nBank]->StatusEnd   == 0xF7)
+			if (m_nBanksLoaded % 100 == 0)
 			{
-				LOGDBG ("Bank #%u successfully loaded", nBank);
+				LOGDBG ("Banks successfully loaded #%u", m_nBanksLoaded);
+			}
+			//LOGDBG ("Bank #%u successfully loaded", nBank);
 
-				m_BankFileName[nBank] = pEntry->d_name;
+			m_BankFileName[nBankIdx] = sBankName;
+			if (nBank > m_nNumHighestBank)
+			{
+				// This is the bank ID of the highest loaded bank
+				m_nNumHighestBank = nBank;
+			}
+			m_nBanksLoaded++;
+			bBankLoaded = true;
+		}
+		else if (bHeaderlessSysExVoices)
+		{
+			// Config says to accept headerless SysEx Voice Banks
+			// so reset file pointer and try again.
+			fseek (pFile, 0, SEEK_SET);
+			if (fread (m_pVoiceBank[nBankIdx]->Voice, VoiceSysExSize, 1, pFile) == 1)
+			{
+				if (m_nBanksLoaded % 100 == 0)
+				{
+					LOGDBG ("Banks successfully loaded #%u", m_nBanksLoaded);
+				}
+				//LOGDBG ("Bank #%u successfully loaded (headerless)", nBank);
+
+				// Add in the missing header items.
+				// Naturally it isn't possible to validate these!
+				m_pVoiceBank[nBankIdx]->StatusStart = 0xF0;
+				m_pVoiceBank[nBankIdx]->CompanyID   = 0x43;
+				m_pVoiceBank[nBankIdx]->Format      = 0x09;
+				m_pVoiceBank[nBankIdx]->ByteCountMS = 0x20;
+				m_pVoiceBank[nBankIdx]->ByteCountLS = 0x00;
+				m_pVoiceBank[nBankIdx]->Checksum    = 0x00;
+				m_pVoiceBank[nBankIdx]->StatusEnd   = 0xF7;
+
+				m_BankFileName[nBankIdx] = sBankName;
 				if (nBank > m_nNumHighestBank)
 				{
 					// This is the bank ID of the highest loaded bank
 					m_nNumHighestBank = nBank;
 				}
 				bBankLoaded = true;
+				m_nBanksLoaded++;
 			}
-			else if (bHeaderlessSysExVoices)
-			{
-				// Config says to accept headerless SysEx Voice Banks
-				// so reset file pointer and try again.
-				fseek (pFile, 0, SEEK_SET);
-				if (fread (m_pVoiceBank[nBank]->Voice, VoiceSysExSize, 1, pFile) == 1)
-				{
-					LOGDBG ("Bank #%u successfully loaded (headerless)", nBank);
-
-					// Add in the missing header items.
-					// Naturally it isn't possible to validate these!
-					m_pVoiceBank[nBank]->StatusStart = 0xF0;
-					m_pVoiceBank[nBank]->CompanyID   = 0x43;
-					m_pVoiceBank[nBank]->Format      = 0x09;
-					m_pVoiceBank[nBank]->ByteCountMS = 0x20;
-					m_pVoiceBank[nBank]->ByteCountLS = 0x00;
-					m_pVoiceBank[nBank]->Checksum    = 0x00;
-					m_pVoiceBank[nBank]->StatusEnd   = 0xF7;
-
-					m_BankFileName[nBank] = pEntry->d_name;
-					if (nBank > m_nNumHighestBank)
-					{
-						// This is the bank ID of the highest loaded bank
-						m_nNumHighestBank = nBank;
-					}
-					bBankLoaded = true;
-				}
-			}
-
-			if (!bBankLoaded)
-			{
-				LOGWARN ("%s: Invalid size or format", Filename.c_str ());
-
-				delete m_pVoiceBank[nBank];
-				m_pVoiceBank[nBank] = nullptr;
-			}
-
-			fclose (pFile);
 		}
-		else
+
+		if (!bBankLoaded)
 		{
-			delete m_pVoiceBank[nBank];
-			m_pVoiceBank[nBank] = nullptr;
+			LOGWARN ("%s: Invalid size or format", Filename.c_str ());
+
+			delete m_pVoiceBank[nBankIdx];
+			m_pVoiceBank[nBankIdx] = nullptr;
 		}
-	}
 
-	closedir (pDirectory);
+		fclose (pFile);
+	}
+	else
+	{
+		delete m_pVoiceBank[nBankIdx];
+		m_pVoiceBank[nBankIdx] = nullptr;
+	}
 }
 
 std::string CSysExFileLoader::GetBankName (unsigned nBankID)
diff --git a/src/sysexfileloader.h b/src/sysexfileloader.h
index d3821da..4918db6 100644
--- a/src/sysexfileloader.h
+++ b/src/sysexfileloader.h
@@ -35,6 +35,7 @@ public:
 	static const size_t SizeSingleVoice = 156;
 	static const unsigned VoiceSysExHdrSize = 8; // Additional (optional) Header/Footer bytes for bank of 32 voices
 	static const unsigned VoiceSysExSize = 4096; // Bank of 32 voices as per DX7 MIDI Spec
+	static const unsigned MaxSubDirs = 3; // Number of nested subdirectories supported.
 
 	struct TVoiceBank
 	{
@@ -75,11 +76,14 @@ private:
 	std::string m_DirName;
 	
 	unsigned m_nNumHighestBank;
+	unsigned m_nBanksLoaded;
 
 	TVoiceBank *m_pVoiceBank[MaxVoiceBankID+1];
 	std::string m_BankFileName[MaxVoiceBankID+1];
 
 	static uint8_t s_DefaultVoice[SizeSingleVoice];
+	
+	void LoadBank (const char * sDirName, const char * sBankName, bool bHeaderlessSysExVoices, unsigned nSubDirCount);
 };
 
 #endif
diff --git a/submod.sh b/submod.sh
new file mode 100755
index 0000000..e941660
--- /dev/null
+++ b/submod.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+git submodule update --init --recursive
+cd circle-stdlib/
+git checkout e318f89 # Needed to support Circle develop?
+cd -
+cd circle-stdlib/libs/circle
+git checkout ec09d7e # develop
+cd -
+cd circle-stdlib/libs/circle-newlib
+git checkout 48bf91d # needed for circle ec09d7e
+cd -
+