Support for subdirectories in the SysEx voices directory (#473)

* Fix for issue #424 create a build script to check out the correct versions of sub libraries.

* Initial attempt at supporting loading SysEx files from subdirectories which seems to speed up loading significantly.

* Reinstate headerless SysEx file loading after subdirectory change

* Limit nesting of subdirectories to open

* Fix bank number in UI to match the numbers from the bank files.

* Update to fix bank numbers.  Bank file numbers now start from 00001 and show as 1-indexed in the UI.  But internally and via MIDI are still 0-indexed as per MIDI spec.

src/sysexfileloader.cpp

@@ -84,6 +84,7 @@ CSysExFileLoader::~CSysExFileLoader (void)
 void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 {
 	m_nNumHighestBank = 0;
+	m_nBanksLoaded = 0;
 
     DIR *pDirectory = opendir (m_DirName.c_str ());
 	if (!pDirectory)
@@ -96,58 +97,104 @@ void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 	dirent *pEntry;
 	while ((pEntry = readdir (pDirectory)) != nullptr)
 	{
+		LoadBank(m_DirName.c_str (), pEntry->d_name, bHeaderlessSysExVoices, 0);
+	}
+	LOGDBG ("%u Banks loaded. Highest Bank loaded: #%u", m_nBanksLoaded, m_nNumHighestBank);
+
+	closedir (pDirectory);
+}
+
+void CSysExFileLoader::LoadBank (const char * sDirName, const char * sBankName, bool bHeaderlessSysExVoices, unsigned nSubDirCount)
+{
 	unsigned nBank;
-		size_t nLen = strlen (pEntry->d_name);
+	size_t nLen = strlen (sBankName);
 	
 	if (   nLen < 5						// "[NNNN]N[_name].syx"
-		    || strcasecmp (&pEntry->d_name[nLen-4], ".syx") != 0
+		|| strcasecmp (&sBankName[nLen-4], ".syx") != 0
-		    || sscanf (pEntry->d_name, "%u", &nBank) != 1)
+		|| sscanf (sBankName, "%u", &nBank) != 1)
 	{
-			LOGWARN ("%s: Invalid filename format", pEntry->d_name);
+		// See if this is a subdirectory...
+		std::string Dirname (sDirName);
+		Dirname += "/";
+		Dirname += sBankName;
 
-			continue;
+		DIR *pDirectory = opendir (Dirname.c_str ());
+		if (pDirectory)
+		{
+			if (nSubDirCount >= MaxSubDirs)
+			{
+				LOGWARN ("Too many nested subdirectories: %s", sBankName);
+				return;
 			}
 	
-		if (nBank > MaxVoiceBankID)
+			LOGDBG ("Processing subdirectory %s", sBankName);
+
+			dirent *pEntry;
+			while ((pEntry = readdir (pDirectory)) != nullptr)
+			{
+				LoadBank(Dirname.c_str (), pEntry->d_name, bHeaderlessSysExVoices, nSubDirCount+1);
+			}
+			closedir (pDirectory);
+		}
+		else
+		{
+			LOGWARN ("%s: Invalid filename format", sBankName);
+		}
+
+		return;
+	}
+	
+	// File and UI handling requires banks to be 1..indexed.
+	// Internally (and via MIDI) we need 0..indexed.
+	// Any mention of a BankID internally is assumed to be 0..indexed.
+	unsigned nBankIdx = nBank - 1;
+
+	// BankIdx goes from 0 to MaxVoiceBankID inclusive
+	if (nBankIdx > MaxVoiceBankID)
 	{
 		LOGWARN ("Bank #%u is not supported", nBank);
 
-			continue;
+		return;
 	}
 
-		if (m_pVoiceBank[nBank])
+	if (m_pVoiceBank[nBankIdx])
 	{
 		LOGWARN ("Bank #%u already loaded", nBank);
 
-			continue;
+		return;
 	}
 
-		m_pVoiceBank[nBank] = new TVoiceBank;
+	m_pVoiceBank[nBankIdx] = new TVoiceBank;
-		assert (m_pVoiceBank[nBank]);
+	assert (m_pVoiceBank[nBankIdx]);
 	assert (sizeof(TVoiceBank) == VoiceSysExHdrSize + VoiceSysExSize);
 
-		std::string Filename (m_DirName);
+	std::string Filename (sDirName);
 	Filename += "/";
-		Filename += pEntry->d_name;
+	Filename += sBankName;
 
 	FILE *pFile = fopen (Filename.c_str (), "rb");
 	if (pFile)
 	{
 		bool bBankLoaded = false;
-			if (   fread (m_pVoiceBank[nBank], VoiceSysExHdrSize+VoiceSysExSize, 1, pFile) == 1
+		if (   fread (m_pVoiceBank[nBankIdx], VoiceSysExHdrSize+VoiceSysExSize, 1, pFile) == 1
-			    && m_pVoiceBank[nBank]->StatusStart == 0xF0
+			&& m_pVoiceBank[nBankIdx]->StatusStart == 0xF0
-			    && m_pVoiceBank[nBank]->CompanyID   == 0x43
+			&& m_pVoiceBank[nBankIdx]->CompanyID   == 0x43
-			    && m_pVoiceBank[nBank]->Format      == 0x09
+			&& m_pVoiceBank[nBankIdx]->Format      == 0x09
-			    && m_pVoiceBank[nBank]->StatusEnd   == 0xF7)
+			&& m_pVoiceBank[nBankIdx]->StatusEnd   == 0xF7)
+		{
+			if (m_nBanksLoaded % 100 == 0)
 			{
-				LOGDBG ("Bank #%u successfully loaded", nBank);
+				LOGDBG ("Banks successfully loaded #%u", m_nBanksLoaded);
+			}
+			//LOGDBG ("Bank #%u successfully loaded", nBank);
 
-				m_BankFileName[nBank] = pEntry->d_name;
+			m_BankFileName[nBankIdx] = sBankName;
 			if (nBank > m_nNumHighestBank)
 			{
 				// This is the bank ID of the highest loaded bank
 				m_nNumHighestBank = nBank;
 			}
+			m_nBanksLoaded++;
 			bBankLoaded = true;
 		}
 		else if (bHeaderlessSysExVoices)
@@ -155,27 +202,32 @@ void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 			// Config says to accept headerless SysEx Voice Banks
 			// so reset file pointer and try again.
 			fseek (pFile, 0, SEEK_SET);
-				if (fread (m_pVoiceBank[nBank]->Voice, VoiceSysExSize, 1, pFile) == 1)
+			if (fread (m_pVoiceBank[nBankIdx]->Voice, VoiceSysExSize, 1, pFile) == 1)
+			{
+				if (m_nBanksLoaded % 100 == 0)
 				{
-					LOGDBG ("Bank #%u successfully loaded (headerless)", nBank);
+					LOGDBG ("Banks successfully loaded #%u", m_nBanksLoaded);
+				}
+				//LOGDBG ("Bank #%u successfully loaded (headerless)", nBank);
 
 				// Add in the missing header items.
 				// Naturally it isn't possible to validate these!
-					m_pVoiceBank[nBank]->StatusStart = 0xF0;
+				m_pVoiceBank[nBankIdx]->StatusStart = 0xF0;
-					m_pVoiceBank[nBank]->CompanyID   = 0x43;
+				m_pVoiceBank[nBankIdx]->CompanyID   = 0x43;
-					m_pVoiceBank[nBank]->Format      = 0x09;
+				m_pVoiceBank[nBankIdx]->Format      = 0x09;
-					m_pVoiceBank[nBank]->ByteCountMS = 0x20;
+				m_pVoiceBank[nBankIdx]->ByteCountMS = 0x20;
-					m_pVoiceBank[nBank]->ByteCountLS = 0x00;
+				m_pVoiceBank[nBankIdx]->ByteCountLS = 0x00;
-					m_pVoiceBank[nBank]->Checksum    = 0x00;
+				m_pVoiceBank[nBankIdx]->Checksum    = 0x00;
-					m_pVoiceBank[nBank]->StatusEnd   = 0xF7;
+				m_pVoiceBank[nBankIdx]->StatusEnd   = 0xF7;
-
+
-					m_BankFileName[nBank] = pEntry->d_name;
+				m_BankFileName[nBankIdx] = sBankName;
 				if (nBank > m_nNumHighestBank)
 				{
 					// This is the bank ID of the highest loaded bank
 					m_nNumHighestBank = nBank;
 				}
 				bBankLoaded = true;
+				m_nBanksLoaded++;
 			}
 		}
 
@@ -183,20 +235,17 @@ void CSysExFileLoader::Load (bool bHeaderlessSysExVoices)
 		{
 			LOGWARN ("%s: Invalid size or format", Filename.c_str ());
 
-				delete m_pVoiceBank[nBank];
+			delete m_pVoiceBank[nBankIdx];
-				m_pVoiceBank[nBank] = nullptr;
+			m_pVoiceBank[nBankIdx] = nullptr;
 		}
 
 		fclose (pFile);
 	}
 	else
 	{
-			delete m_pVoiceBank[nBank];
+		delete m_pVoiceBank[nBankIdx];
-			m_pVoiceBank[nBank] = nullptr;
+		m_pVoiceBank[nBankIdx] = nullptr;
-		}
 	}
-
-	closedir (pDirectory);
 }
 
 std::string CSysExFileLoader::GetBankName (unsigned nBankID)

src/sysexfileloader.h

@@ -35,6 +35,7 @@ public:
 	static const size_t SizeSingleVoice = 156;
 	static const unsigned VoiceSysExHdrSize = 8; // Additional (optional) Header/Footer bytes for bank of 32 voices
 	static const unsigned VoiceSysExSize = 4096; // Bank of 32 voices as per DX7 MIDI Spec
+	static const unsigned MaxSubDirs = 3; // Number of nested subdirectories supported.
 
 	struct TVoiceBank
 	{
@@ -75,11 +76,14 @@ private:
 	std::string m_DirName;
 	
 	unsigned m_nNumHighestBank;
+	unsigned m_nBanksLoaded;
 
 	TVoiceBank *m_pVoiceBank[MaxVoiceBankID+1];
 	std::string m_BankFileName[MaxVoiceBankID+1];
 
 	static uint8_t s_DefaultVoice[SizeSingleVoice];
+	
+	void LoadBank (const char * sDirName, const char * sBankName, bool bHeaderlessSysExVoices, unsigned nSubDirCount);
 };
 
 #endif

submod.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -ex
+git submodule update --init --recursive
+cd circle-stdlib/
+git checkout e318f89 # Needed to support Circle develop?
+cd -
+cd circle-stdlib/libs/circle
+git checkout ec09d7e # develop
+cd -
+cd circle-stdlib/libs/circle-newlib
+git checkout 48bf91d # needed for circle ec09d7e
+cd -
+