Arduino Nano 33 BLE Sense TFlite inference hangs when using bluetooth

Question

I'm running into an issue where when I try to set up a BLE service my tflite inference seems to hang/crash. I used https://experiments.withgoogle.com/tiny-motion-trainer to train a model to recognize a few gestures with my nano. The code I'm using is mostly sample code generated by the site. Here's the code that correctly outputs gestures to the Serial monitor:

/**
* TinyML Trainer Example. 
* Version: v005
*
* This is an example generated by Tiny Motion Trainer 
* It is pre-filled with your trained model and your capture settings 
* 
* Required Libraries:
* - TensorFlow Lite for Microcontrollers (tested with v2.4-beta)
* - Arduino_LSM9DS1
*
* Usage:
* - Make sure the Arduino BLE 33 board is installed (tools->board->Board Manager) and connected
* - Make sure you have the required libraries installed
* - Build and upload the sketch
* - Keep the Arduino connected via USB and open the Serial Monitor (Tools->Serial Monitor)
* - The sketch will log out the label with the highest score once detected.
*
* This is meant as an example for you to develop your own code, and is not production ready.
*
**/
//==============================================================================
// Includes
//==============================================================================
#include <Arduino_LSM9DS1.h>
#include <ArduinoBLE.h>
#include <TensorFlowLite.h>
#include <tensorflow/lite/micro/all_ops_resolver.h>
#include <tensorflow/lite/micro/micro_error_reporter.h>
#include <tensorflow/lite/micro/micro_interpreter.h>
#include <tensorflow/lite/schema/schema_generated.h>
#include <tensorflow/lite/version.h>
//==============================================================================
// Your custom data / settings
// - Editing these is not recommended
//==============================================================================
// This is the model you trained in Tiny Motion Trainer, converted to 
// a C style byte array.
#include "model.h"
// Values from Tiny Motion Trainer
#define MOTION_THRESHOLD 0.2
#define CAPTURE_DELAY 200 // This is now in milliseconds
#define NUM_SAMPLES 20
// Array to map gesture index to a name
const char *GESTURES[] = {
    "gesture1", "gesture2", "gesture3"
};
//==============================================================================
// Capture variables
//==============================================================================
#define NUM_GESTURES (sizeof(GESTURES) / sizeof(GESTURES[0]))
bool isCapturing = false;
// Num samples read from the IMU sensors
// "Full" by default to start in idle
int numSamplesRead = 0;
//==============================================================================
// TensorFlow variables
//==============================================================================
// Global variables used for TensorFlow Lite (Micro)
tflite::MicroErrorReporter tflErrorReporter;
// Auto resolve all the TensorFlow Lite for MicroInterpreters ops, for reduced memory-footprint change this to only 
// include the op's you need.
tflite::AllOpsResolver tflOpsResolver;
// Setup model
const tflite::Model* tflModel = nullptr;
tflite::MicroInterpreter* tflInterpreter = nullptr;
TfLiteTensor* tflInputTensor = nullptr;
TfLiteTensor* tflOutputTensor = nullptr;
// Create a static memory buffer for TensorFlow Lite for MicroInterpreters, the size may need to
// be adjusted based on the model you are using
constexpr int tensorArenaSize = 8 * 1024;
byte tensorArena[tensorArenaSize];
//==============================================================================
// Setup / Loop
//==============================================================================
// Variables to hold IMU data
float aX, aY, aZ, gX, gY, gZ;
void setup() {
  pinMode(LED_BUILTIN, OUTPUT);
Serial.begin(9600);
// Wait for serial monitor to connect
  while (!Serial);
// Initialize IMU sensors
  if (!IMU.begin()) {
    Serial.println("Failed to initialize IMU!");
    while (1);
  }
// Print out the samples rates of the IMUs
  Serial.print("Accelerometer sample rate: ");
  Serial.print(IMU.accelerationSampleRate());
  Serial.println(" Hz");
  Serial.print("Gyroscope sample rate: ");
  Serial.print(IMU.gyroscopeSampleRate());
  Serial.println(" Hz");
Serial.println();
// Get the TFL representation of the model byte array
  tflModel = tflite::GetModel(model);
  if (tflModel->version() != TFLITE_SCHEMA_VERSION) {
    Serial.println("Model schema mismatch!");
    while (1);
  }
// Create an interpreter to run the model
  tflInterpreter = new tflite::MicroInterpreter(tflModel, tflOpsResolver, tensorArena, tensorArenaSize, &tflErrorReporter);
// Allocate memory for the model's input and output tensors
  tflInterpreter->AllocateTensors();
// Get pointers for the model's input and output tensors
  tflInputTensor = tflInterpreter->input(0);
  tflOutputTensor = tflInterpreter->output(0);
}
void loop() {
  // Wait for motion above the threshold setting
 if (!isCapturing) {
    if (IMU.accelerationAvailable() && IMU.gyroscopeAvailable()) {
  IMU.readAcceleration(aX, aY, aZ);
  IMU.readGyroscope(gX, gY, gZ);

  // Sum absolute values
  float average = fabs(aX / 4.0) + fabs(aY / 4.0) + fabs(aZ / 4.0) + fabs(gX / 2000.0) + fabs(gY / 2000.0) + fabs(gZ / 2000.0);
  average /= 6.;

  // Above the threshold?
  if (average &gt;= MOTION_THRESHOLD) {
    isCapturing = true;
    numSamplesRead = 0;
    //break;
  }
}

}
if (isCapturing) {
// Check if both acceleration and gyroscope data is available
if (IMU.accelerationAvailable() &amp;&amp; IMU.gyroscopeAvailable()) {

  // read the acceleration and gyroscope data
  IMU.readAcceleration(aX, aY, aZ);
  IMU.readGyroscope(gX, gY, gZ);

  // Normalize the IMU data between -1 to 1 and store in the model's
  // input tensor. Accelerometer data ranges between -4 and 4,
  // gyroscope data ranges between -2000 and 2000
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 0] = aX / 4.0;
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 1] = aY / 4.0;
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 2] = aZ / 4.0;
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 3] = gX / 2000.0;
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 4] = gY / 2000.0;
  tflInputTensor-&gt;data.f[numSamplesRead * 6 + 5] = gZ / 2000.0;

  numSamplesRead++;

  // Do we have the samples we need?
  if (numSamplesRead == NUM_SAMPLES) {

    // Stop capturing
    isCapturing = false;

    // Run inference
    Serial.println(&quot;invoking&quot;);
    TfLiteStatus invokeStatus = tflInterpreter-&gt;Invoke();
    Serial.println(&quot;invoked&quot;);
    if (invokeStatus != kTfLiteOk) {
      Serial.println(&quot;Error: Invoke failed!&quot;);
      while (1);
      return;
    }

    // Loop through the output tensor values from the model
    int maxIndex = 0;
    float maxValue = 0;
    for (int i = 0; i &lt; NUM_GESTURES; i++) {
      float _value = tflOutputTensor-&gt;data.f[i];
      if(_value &gt; maxValue){
        maxValue = _value;
        maxIndex = i;
      }
      Serial.print(GESTURES[i]);
      Serial.print(&quot;: &quot;);
      Serial.println(tflOutputTensor-&gt;data.f[i], 6);
    }

    Serial.print(&quot;Winner: &quot;);
    Serial.print(GESTURES[maxIndex]);

    Serial.println();

    // Add delay to not double trigger
    delay(CAPTURE_DELAY);
  }
}

}
}

When I add the declaration BLEService service("4798e0f2-0000-4d68-af64-8a8f5258404e"); before the setup function, this code no longer works. I see

Accelerometer sample rate: 119.00 Hz
Gyroscope sample rate: 119.00 Hz
invoking

in my Serial monitor and then nothing. It seems like just declaring this service is somehow causing the inference code to crash. Does anyone have any idea why that would be happening? I'd really appreciate any help!

Arduino Nano 33 BLE Sense TFlite inference hangs when using bluetooth

0 Answers0