Add USB Host mode and pressure sensitivity.

This patch wires up pressure sensitivity from the touchscreen. It also
adds simple USB Host mode for MIDI keyboards (tested on M-Audio KeyRig 49
and Akai MPK mini). Finally, it cleans up a bit.

The USB listening thread should be moved out of the activity and into a
service (there can be problems on multiple plug and unplug). But this
should be good enough to play with.

android/AndroidManifest.xml

@@ -4,7 +4,10 @@
     package="com.google.synthesizer"
     android:versionCode="2"
     android:versionName="0.9">
-  <uses-sdk android:minSdkVersion="9" /> <!-- 8 = Froyo 2.2 -->
+  <uses-sdk android:minSdkVersion="9" 
+      android:targetSdkVersion="14"
+      /> <!-- 9 = Gingerbread -->
+  <uses-feature android:name="android.hardware.usb.host" />
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
   <application android:icon="@drawable/icon" android:label="@string/app_name">
     <activity
@@ -15,6 +18,11 @@
         <action android:name="android.intent.action.MAIN" />
         <category android:name="android.intent.category.LAUNCHER" />
       </intent-filter>
+      <intent-filter>
+        <action android:name="android.hardware.usb.action.USB_DEVICE_ATTACHED"/>
+      </intent-filter>
+      <meta-data android:name="android.hardware.usb.action.USB_DEVICE_ATTACHED"
+          android:resource="@xml/device_filter" />
     </activity>
     <activity
         android:name=".android.ui.MainActivity"

android/project.properties

@@ -11,4 +11,4 @@
 #proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
 
 # Project target.
-target=android-10
+target=android-14

android/res/xml/device_filter.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<resources>
+  <usb-device class="1" subclass="3" />
+</resources>

android/src/com/google/synthesizer/android/AndroidGlue.java

@@ -16,6 +16,11 @@ public class AndroidGlue extends MessageOutputProcessor {
    */
   public native void start();
 
+  /**
+   *  Shut down the OpenSL ES engine and audio synthesizer.
+   */
+  public native void shutdown();
+  
   /**
    * Start or pause the actual sound generation.
    * 

android/src/com/google/synthesizer/android/ui/PianoActivity2.java

@@ -19,15 +19,27 @@ package com.google.synthesizer.android.ui;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.HashMap;
 
 import android.app.Activity;
+import android.content.Context;
+import android.hardware.usb.UsbConstants;
+import android.hardware.usb.UsbDevice;
+import android.hardware.usb.UsbDeviceConnection;
+import android.hardware.usb.UsbEndpoint;
+import android.hardware.usb.UsbInterface;
+import android.hardware.usb.UsbManager;
+import android.os.Build;
 import android.os.Bundle;
 import android.util.Log;
 import android.view.View;
+import android.view.WindowManager;
 import android.widget.AdapterView;
 import android.widget.AdapterView.OnItemSelectedListener;
 import android.widget.ArrayAdapter;
 import android.widget.Spinner;
+import android.widget.TextView;
+import android.widget.ToggleButton;
 
 import com.google.synthesizer.R;
 import com.google.synthesizer.android.AndroidGlue;
@@ -43,12 +55,13 @@ public class PianoActivity2 extends Activity {
   @Override
   public void onCreate(Bundle savedInstanceState) {
     super.onCreate(savedInstanceState);
+    getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN |
+            WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
     setContentView(R.layout.piano);
 
     piano_ = (PianoView)findViewById(R.id.piano);
     volumeKnob_ = (KnobView)findViewById(R.id.volumeKnob);
     presetSpinner_ = (Spinner)findViewById(R.id.presetSpinner);
-    // TODO: wire these up (preset spinner should send patch selection)
 
     androidGlue_ = new AndroidGlue();
     androidGlue_.start();
@@ -77,9 +90,17 @@ public class PianoActivity2 extends Activity {
       }
     });
     piano_.bindTo(androidGlue_);
-    
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ICE_CREAM_SANDWICH) {
+      tryConnectUsb();
+    }
   }
 
+  @Override
+  protected void onDestroy() {
+    androidGlue_.shutdown();
+    super.onDestroy();
+  }
+  
   @Override
   protected void onPause() {
     androidGlue_.setPlayState(false);
@@ -92,6 +113,64 @@ public class PianoActivity2 extends Activity {
     super.onResume();
   }
 
+  private UsbEndpoint getInputEndpoint(UsbInterface usbIf) {
+    int nEndpoints = usbIf.getEndpointCount();
+    for (int i = 0; i < nEndpoints; i++) {
+      UsbEndpoint endpoint = usbIf.getEndpoint(i);
+      if (endpoint.getDirection() == UsbConstants.USB_DIR_IN &&
+              endpoint.getType() == UsbConstants.USB_ENDPOINT_XFER_BULK) {
+        return endpoint;
+      }
+    }
+    return null;
+  }
+  
+  private void startUsbThread(final UsbDeviceConnection connection, final UsbEndpoint endpoint) {
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        byte[] buf = new byte[endpoint.getMaxPacketSize()];
+        while (true) {
+          int nBytes = connection.bulkTransfer(endpoint, buf, buf.length, 10000);
+          for (int i = 0; i < nBytes; i += 4) {
+            int codeIndexNumber = buf[i] & 0xf;
+            int payloadBytes = 0;
+            if (codeIndexNumber == 8 || codeIndexNumber == 9) {
+              // TODO: pitchbend, control, etc
+              payloadBytes = 3;
+            } else if (codeIndexNumber == 12) {
+              payloadBytes = 2;
+            }
+            if (payloadBytes > 0) {
+              byte[] newBuf = new byte[payloadBytes];
+              System.arraycopy(buf, i + 1, newBuf, 0, payloadBytes);
+              androidGlue_.onMessage(newBuf);
+            }
+          }
+        }
+      }
+    });
+    thread.start();
+  }
+  private void tryConnectUsb() {
+    UsbManager usbManager = (UsbManager) getSystemService(Context.USB_SERVICE);
+    HashMap<String, UsbDevice> deviceList = usbManager.getDeviceList();
+    TextView label = (TextView)findViewById(R.id.volumeLabel);
+    if (!deviceList.isEmpty()) {
+      UsbDevice device = deviceList.values().iterator().next();
+      //label.setText("ic:" + device.getInterfaceCount());
+      UsbInterface usbIf = device.getInterface(1);
+      UsbDeviceConnection connection = usbManager.openDevice(device);
+      if (connection != null) {
+        connection.claimInterface(usbIf, true);
+        UsbEndpoint endpoint = getInputEndpoint(usbIf);
+        //label.setText(endpoint.toString());
+        startUsbThread(connection, endpoint);
+      } else {
+        label.setText("error opening device");
+      }
+    }
+  }
+  
   private AndroidGlue androidGlue_;
   private PianoView piano_;
   private KnobView volumeKnob_;

android/src/com/google/synthesizer/android/widgets/ChordGridView.java

@@ -73,7 +73,7 @@ public class ChordGridView extends View {
    */
   private void notifyNoteDown(double logFrequency, int finger, boolean retriggerIfOn) {
     if (pianoViewListener_ != null) {
-      pianoViewListener_.noteDown(logFrequency, finger, retriggerIfOn);
+      pianoViewListener_.noteDown(logFrequency, finger, retriggerIfOn, 1.0f);
     }
   }
 
@@ -282,7 +282,8 @@ public class ChordGridView extends View {
    */
   public void bindTo(final MultiChannelSynthesizer synth, final int channel) {
     this.setPianoViewListener(new PianoViewListener() {
-      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn) {
+      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn,
+              float pressure) {
         synth.getChannel(channel).setPitch(logFrequency, finger);
         synth.getChannel(channel).turnOn(retriggerIfOn, finger);
       }

android/src/com/google/synthesizer/android/widgets/piano/PianoView.java

@@ -113,9 +113,10 @@ public class PianoView extends View {
    * @param logFrequency - the log frequency of the new note.
    * @param retriggerIfOn - true if this is a new touch, rather than just moving.
    */
-  private void notifyNoteDown(double logFrequency, int finger, boolean retriggerIfOn) {
+  private void notifyNoteDown(double logFrequency, int finger, boolean retriggerIfOn,
+          float pressure) {
     if (pianoViewListener_ != null) {
-      pianoViewListener_.noteDown(logFrequency, finger, retriggerIfOn);
+      pianoViewListener_.noteDown(logFrequency, finger, retriggerIfOn, pressure);
     }
   }
 
@@ -147,7 +148,7 @@ public class PianoView extends View {
    * Called to handle touch down events.
    * Returns true iff we need to redraw.
    */
-  protected boolean onTouchDown(int finger, int x, int y) {
+  protected boolean onTouchDown(int finger, int x, int y, float pressure) {
     // Look through keys from top to bottom, and set the first one found as down, the rest as up.
     PianoKey keyDown = null;
     boolean redraw = false;
@@ -165,7 +166,7 @@ public class PianoView extends View {
       }
     }
     if (keyDown instanceof NotePianoKey) {
-      notifyNoteDown(((NotePianoKey)keyDown).getLogFrequency(), finger, true);
+      notifyNoteDown(((NotePianoKey)keyDown).getLogFrequency(), finger, true, pressure);
     }
     return redraw;
   }
@@ -173,7 +174,7 @@ public class PianoView extends View {
   /**
    * Called to handle touch move events.
    */
-  protected boolean onTouchMove(int finger, int x, int y) {
+  protected boolean onTouchMove(int finger, int x, int y, float pressure) {
     // Look through keys from top to bottom, and set the first one found as moved, the rest as up.
     PianoKey keyDown = null;
     boolean redraw = false;
@@ -193,8 +194,11 @@ public class PianoView extends View {
       }
     }
     if (keyDown instanceof NotePianoKey) {
+      if (!usePressure_) {
+        pressure = 0.5f;
+      }
       if (!wasPressed) {
-        notifyNoteDown(((NotePianoKey)keyDown).getLogFrequency(), finger, false);
+        notifyNoteDown(((NotePianoKey)keyDown).getLogFrequency(), finger, false, pressure);
       }
     } else {
       notifyNoteUp(finger);
@@ -229,17 +233,18 @@ public class PianoView extends View {
       if (pointerId < FINGERS) {
         int x = (int)event.getX();
         int y = (int)event.getY();
-        redraw |= onTouchDown(pointerId, x, y);
+        float pressure = event.getPressure();
+        redraw |= onTouchDown(pointerId, x, y, pressure);
       }
     } else if (actionCode == MotionEvent.ACTION_POINTER_DOWN) {
-      int pointerId = action >> MotionEvent.ACTION_POINTER_ID_SHIFT;
+      int pointerIndex = (action & MotionEvent.ACTION_POINTER_INDEX_MASK)
-      if (pointerId < FINGERS) {
+          >> MotionEvent.ACTION_POINTER_INDEX_SHIFT;
-        int pointerIndex = event.findPointerIndex(pointerId);
+      int pointerId = event.getPointerId(pointerIndex);
-        if (pointerIndex >= 0) {
+      if (pointerId < FINGERS && pointerId >= 0) {
-          int x = (int)event.getX(pointerIndex);
+        int x = (int)event.getX(pointerIndex);
-          int y = (int)event.getY(pointerIndex);
+        int y = (int)event.getY(pointerIndex);
-          redraw |= onTouchDown(pointerId, x, y);
+        float pressure = event.getPressure(pointerIndex);
-        }
+        redraw |= onTouchDown(pointerId, x, y, pressure);
       }
     } else if (actionCode == MotionEvent.ACTION_MOVE) {
       for (int pointerIndex = 0; pointerIndex < event.getPointerCount(); ++pointerIndex) {
@@ -250,7 +255,8 @@ public class PianoView extends View {
         if (pointerIndex >= 0) {
           int x = (int)event.getX(pointerIndex);
           int y = (int)event.getY(pointerIndex);
-          redraw |= onTouchMove(pointerId, x, y);
+          float pressure = event.getPressure(pointerIndex);
+          redraw |= onTouchMove(pointerId, x, y, pressure);
         }
       }
     } else if (actionCode == MotionEvent.ACTION_UP) {
@@ -272,14 +278,15 @@ public class PianoView extends View {
         }
       }
     } else if (actionCode == MotionEvent.ACTION_POINTER_UP) {
-      int pointerId = action >> MotionEvent.ACTION_POINTER_ID_SHIFT;
+      int pointerIndex = (action & MotionEvent.ACTION_POINTER_INDEX_MASK) >> MotionEvent.ACTION_POINTER_INDEX_SHIFT;
+      int pointerId = event.getPointerId(pointerIndex);
       if (pointerId < FINGERS) {
         redraw |= onTouchUp(pointerId);
       }
-      // Clean up any other pointers that have disappeared.
+      // Clean up any other pointers that have disappeared. Note: this is probably not necessary.
       for (pointerId = 0; pointerId < FINGERS; ++pointerId) {
         boolean found = false;
-        for (int pointerIndex = 0; pointerIndex < event.getPointerCount(); ++pointerIndex) {
+        for (pointerIndex = 0; pointerIndex < event.getPointerCount(); ++pointerIndex) {
           if (pointerId == event.getPointerId(pointerIndex)) {
             found = true;
             break;
@@ -346,7 +353,8 @@ public class PianoView extends View {
    */
   public void bindTo(final MultiChannelSynthesizer synth, final int channel) {
     this.setPianoViewListener(new PianoViewListener() {
-      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn) {
+      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn,
+              float pressure) {
         synth.getChannel(channel).setPitch(logFrequency, finger);
         synth.getChannel(channel).turnOn(retriggerIfOn, finger);
       }
@@ -357,25 +365,26 @@ public class PianoView extends View {
   }
 
   /**
-   * Connects the PianoView to an AndroidGlue. This should probably be a MidiListener instead,
+   * Connects the PianoView to an MidiListener.
-   * though...
    */
-  public void bindTo(final MidiListener midiSink) {
+  public void bindTo(final MidiListener midiListener) {
     this.setPianoViewListener(new PianoViewListener() {
       {
         fingerMap_ = new HashMap<Integer, Integer>();
       }
-      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn) {
+      public void noteDown(double logFrequency, int finger, boolean retriggerIfOn,
+              float pressure) {
         noteUp(finger);
         int midiNote = Note.getKeyforLog12TET(logFrequency);
         fingerMap_.put(finger, midiNote);
-        midiSink.onNoteOn(0, midiNote, 64);
+        int midiPressure = Math.max(1, Math.min(127, (int)(127 * pressure)));
+        midiListener.onNoteOn(0, midiNote, midiPressure);
       }
       public void noteUp(int finger) {
         if (fingerMap_.containsKey(finger)) {
           int midiNote = fingerMap_.get(finger);
           fingerMap_.remove(finger);
-          midiSink.onNoteOff(0, midiNote, 64);
+          midiListener.onNoteOff(0, midiNote, 64);
         }
       }
       private Map<Integer, Integer> fingerMap_;
@@ -402,4 +411,7 @@ public class PianoView extends View {
 
   // The number of simultaneous fingers supported by this control.
   protected static final int FINGERS = 5;
+  
+  // Whether to use pressure (doesn't work well on all hardware)
+  private boolean usePressure_ = true;
 }

android/src/com/google/synthesizer/android/widgets/piano/PianoViewListener.java

@@ -23,7 +23,7 @@ public interface PianoViewListener {
    * @param logFrequency - the log frequency of the note pressed.
    * @param retriggerIfOn - true if this is a new touch, rather than just moving.
    */
-  void noteDown(double logFrequency, int finger, boolean retriggerIfOn);
+  void noteDown(double logFrequency, int finger, boolean retriggerIfOn, float pressure);
   
   /**
    * The note was released.

cpp/src/android_glue.cc

@@ -1,3 +1,19 @@
+/*
+* Copyright 2012 Google Inc.
+* 
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+* 
+*      http://www.apache.org/licenses/LICENSE-2.0
+* 
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
 #include <assert.h>
 #include <stddef.h>
 #include <jni.h>
@@ -16,7 +32,7 @@ RingBuffer *ring_buffer;
 SynthUnit *synth_unit;
 
 const int N_BUFFERS = 2;
-const int BUFFER_SIZE = 64;
+const int BUFFER_SIZE = 384;
 
 int16_t buffer[BUFFER_SIZE * N_BUFFERS];
 int cur_buffer = 0;
@@ -32,23 +48,22 @@ static SLObjectItf outputMixObject = NULL;
 // buffer queue player interfaces
 static SLObjectItf bqPlayerObject = NULL;
 static SLPlayItf bq_player_play;
-static SLVolumeItf bq_player_volume;
 static SLAndroidSimpleBufferQueueItf bq_player_buffer_queue;
 static SLBufferQueueItf buffer_queue_itf;
 
 extern "C" void BqPlayerCallback(SLAndroidSimpleBufferQueueItf queueItf,
-    void *data) {
+  void *data) {
   if (count >= 1000) return;
   int16_t *buf_ptr = buffer + BUFFER_SIZE * cur_buffer;
   synth_unit->GetSamples(BUFFER_SIZE, buf_ptr);
   SLresult result = (*queueItf)->Enqueue(bq_player_buffer_queue,
-      buf_ptr, BUFFER_SIZE * 2);
+    buf_ptr, BUFFER_SIZE * 2);
   assert(SL_RESULT_SUCCESS == result);
   cur_buffer = (cur_buffer + 1) % N_BUFFERS;
 }
 
 void CreateEngine() {
-  SLresult result;
+SLresult result;
   result = slCreateEngine(&engineObject, 0, NULL, 0, NULL, NULL);
   assert(SL_RESULT_SUCCESS == result);
 
@@ -64,7 +79,8 @@ void CreateEngine() {
   assert(SL_RESULT_SUCCESS == result);
   result = (*outputMixObject)->Realize(outputMixObject, SL_BOOLEAN_FALSE);
   assert(SL_RESULT_SUCCESS == result);
-}
+  LOGI("engine started");
+  }
 
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_synthesizer_android_AndroidGlue_start(JNIEnv *env,
@@ -73,7 +89,7 @@ Java_com_google_synthesizer_android_AndroidGlue_start(JNIEnv *env,
   SLDataLocator_AndroidSimpleBufferQueue loc_bufq =
     {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, N_BUFFERS};
   SLDataFormat_PCM format_pcm = {
-    SL_DATAFORMAT_PCM, 1, SL_SAMPLINGRATE_48,
+    SL_DATAFORMAT_PCM, 1, SL_SAMPLINGRATE_44_1,
     SL_PCMSAMPLEFORMAT_FIXED_16, SL_PCMSAMPLEFORMAT_FIXED_16,
     SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN
       // TODO: compute real endianness
@@ -97,14 +113,11 @@ Java_com_google_synthesizer_android_AndroidGlue_start(JNIEnv *env,
       &bq_player_buffer_queue);
   assert(SL_RESULT_SUCCESS == result);
 
-  result = (*bqPlayerObject)->GetInterface(bqPlayerObject, SL_IID_VOLUME,
-      &bq_player_volume);
-  assert(SL_RESULT_SUCCESS == result);
   result = (*bq_player_buffer_queue)->RegisterCallback(bq_player_buffer_queue,
-      &BqPlayerCallback, NULL);
+        &BqPlayerCallback, NULL);
   assert(SL_RESULT_SUCCESS == result);
 
-  double sample_rate = 48000.0;
+  double sample_rate = 44100.0;
   Freqlut::init(sample_rate);
   Sin::init();
   ring_buffer = new RingBuffer();
@@ -118,6 +131,31 @@ Java_com_google_synthesizer_android_AndroidGlue_start(JNIEnv *env,
   assert(SL_RESULT_SUCCESS == result);
 }
 
+extern "C" JNIEXPORT void JNICALL
+Java_com_google_synthesizer_android_AndroidGlue_shutdown(JNIEnv *env,
+    jobject thiz) {
+  LOGI("shutting down engine");
+  if (bqPlayerObject != NULL) {
+    (*bqPlayerObject)->Destroy(bqPlayerObject);
+    bqPlayerObject = NULL;
+    bq_player_play = NULL;
+    bq_player_buffer_queue = NULL;
+  }
+  if (outputMixObject != NULL) {
+    (*outputMixObject)->Destroy(outputMixObject);
+    outputMixObject = NULL;
+  }
+  if (engineObject != NULL) {
+    (*engineObject)->Destroy(engineObject);
+    engineObject = NULL;
+    engineEngine = NULL;
+  }
+  delete ring_buffer;
+  ring_buffer = NULL;
+  delete synth_unit;
+  synth_unit = NULL;
+}
+
 extern "C" JNIEXPORT void JNICALL
 Java_com_google_synthesizer_android_AndroidGlue_sendMidi(JNIEnv *env,
     jobject thiz, jbyteArray jb) {