Diff: main.cpp

Revision:

0:e2daaf858e13

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/main.cpp	Sat Jun 15 19:43:44 2013 +0000
@@ -0,0 +1,552 @@
+/* BlinkTalk - Bob Stone June 2013
+ * Hands-free control of a speech synthesis system by blinking.
+ *
+ * This project implements a proof of concept speech system actuated by a single input
+ * - this could be a button, but for a more fun application I am going to use blinks as
+ * detected by an EEG headset, with a row & column time-sweep keyboard, inspired by
+ * (but not the same as) the speech system used by Prof Stephen Hawking, mainly because the
+ * Emic2 voice board includes the DECTalk 'PerfectPaul' voice familiar to all as Hawking.
+ *
+ * In Hawking's actual system, an infrared sensor mounted on his glasses detects a movement
+ * of his cheek muscle, selecting letters from a keyboard  sweeping rows and columns, with a
+ * predictive text system handling word completion and suggested follow-on words.  In
+ * our version we will detect a blink using a Neurosky Mindwave Mobile headset and use it to
+ * stop and select a cursor sweeping a keyboard grid.
+ *
+ * Hardware:
+ *      Neurosky Mindwave Mobile headset - sends serial data packets over BlueTooth
+ *      BlueSMiRF Silver Mate - receives data over BlueTooth from headset and relays over serial to mbed
+ *      Parallax / Grand Idea Studio Emic2 speech synthesis - speaks text sent over serial from the mbed
+ *      MikroElektronika TFT Proto - 320x240 TFT display with HX8374 controller driven by SPI
+ *      Sparkfun level shifter - translates Emic2 5V serial to 3.3V serial for mbed
+ *
+ * Connections:
+ *  mbed        BlueSMiRF   LevelShift  Emic2   Speaker TFT-PROTO
+ *   GND         GND         GND/GND     GND             GND
+ *   VOUT(3.3V)  VCC         LV                          3.3V
+ *   VU (5V)                 HV          5V
+ *   p9 (TX)     RX-I
+ *   p10(RX)     TX-O
+ *   p11(MOSI)                                           SDI
+ *   p12(MISO)                                           SDO
+ *   p13(SCLK)                                           SCL
+ *   p14                                                 CS
+ *   p15                                                 RST
+ *   p27(RX)                 LV:RXO
+ *   p28(TX)                 LV:TXI
+ *                           HV:RXI      SOUT
+ *                           HV:TXO      SN
+ *                                       SP+     +
+ *                                       SP-     -
+ *      
+ * Borrows from http://developer.neurosky.com/docs/doku.php?id=mindwave_mobile_and_arduino
+ * Display library from Peter Drescher: http://mbed.org/cookbook/SPI-driven-QVGA-TFT
+ */
+#include "mbed.h"
+#include "SPI_TFT.h"
+#include "Arial12x12.h"
+#include "Arial28x28.h"
+#include <string>
+
+//Peripherals
+Serial blueSmirf(p9, p10);                      //bluetooth comms (TX, RX)
+Serial voice(p28,p27);                          //Emic2 text to speech voice synth
+SPI_TFT screen(p11, p12, p13, p14, p15, "TFT"); //display
+
+//control variables
+int quality=0;
+bool connected=false;
+bool started=false;
+int row=-1;
+int col=-1;
+Timer initialDelay;
+
+//Keyboard - rows to display
+#define ROWS 5
+#define COLS 10
+//keys or text to display
+string keys[ROWS][COLS] = {
+    {"1","2","3","4","5","6","7","8","9","0"},
+    {"Q","W","E","R","T","Y","U","I","O","P"},
+    {"A","S","D","F","G","H","J","K","L","!"},
+    {"Z","X","C","V","B","N","M",",",".","?"},
+    {"Space","\"SAY!\"","Delete"}
+};
+//positions to draw a line at the end of a column
+int keysColPos[ROWS][COLS] = {
+    {36,67,98,129,160,191,222,253,284,315},
+    {36,67,98,129,160,191,222,253,284,315},
+    {36,67,98,129,160,191,222,253,284,315},
+    {36,67,98,129,160,191,222,253,284,315},
+    {100,210,315}
+};
+//number of elements in each row
+int keysRowSize[ROWS] = {10, 10, 10, 10, 3};
+
+//text being typed
+string typingBuffer("");
+
+//*****************************
+//User routines to process data
+//*****************************
+
+/** Say some text out loud
+ *
+ * @param text The text to say out loud
+ */
+void say(string text)
+{
+    voice.printf("S%s\n", text);//send command to speak to the Emic2
+    while(!voice.readable())    //wait for Emic2 to return ':'
+        ;                       //do nothing whilst it's still speaking
+    voice.getc();               //pop the ':' response from the stream
+}
+
+/** Maps a value from one scale to another
+ *
+ * @param value Value we're trying to scale
+ * @param min,max The range that value came from
+ * @param newMin,newMax The new range we're scaling value into
+ * @returns value mapped into new scale
+ */
+int map(int value, int min, int max, int newMin, int newMax)
+{
+    if (min==max)
+        return newMax;
+    else
+        return newMin + (newMax-newMin) * (value-min) / (max-min);
+}
+
+/** Returns a 16-bit RGB565 colour from three 8-bit component values.
+ *
+ * @param red,green,blue primary colour channel values expressed as 0-255 each
+ * @returns 16-bit RGB565 colour constructed as RRRRRGGGGGGBBBBB
+ */
+int RGBColour(int red, int green, int blue)
+{
+    //take most-significant parts of red, green and blue and bit-shift into RGB565 positions
+    return ((red & 0xf8) << 8) | ((green & 0xfc) << 3) | ((blue & 0xf8) >> 3);
+}
+
+/** Returns a colour mapped on a gradient from one colour to another.
+ *
+ * @param value Value we're trying to pick a colour for
+ * @param min,max Scale that value belongs in
+ * @param minColour,maxColour start and end colours of the gradient we're choosing from (16-bit RGB565)
+ * @returns colour that's as far along the gradient from minColour to maxColour as value is between min and max (16-bit RGB565)
+ */
+int getMappedColour(int value, int min, int max, int minColour, int maxColour)
+{
+    // TFT screen colours are 16-bit RGB565 i.e. RRRRRGGGGGGBBBBB
+    int minRed = (minColour & 0xf800) >> 11; //bitmask for 5 bits red
+    int maxRed = (maxColour & 0xf800) >> 11;
+    int minGreen = (minColour & 0x7e0) >> 5; //bitmask for 6 bits green
+    int maxGreen = (maxColour & 0x7e0) >> 5;
+    int minBlue = minColour & 0x1f; // bitmask for 5 bits blue
+    int maxBlue = maxColour & 0x1f;
+    int valRed = map(value, min, max, minRed, maxRed);
+    int valGreen = map(value, min, max, minGreen, maxGreen);
+    int valBlue = map(value, min, max, minBlue, maxBlue);
+    int valColour = ((valRed & 0x1F) << 11) | ((valGreen & 0x3F) << 5) | (valBlue & 0x1F);
+    return valColour;
+}
+
+/** Displays a bar graph showing 'value' on a scale 'min' to 'max', where coords (x0,y0) are at 'min' and (x1,y1) are at 'max'.
+ *
+ * @param x0,y0 coordinates of the 'min' end of the bargraph
+ * @param x1,y1 coordinates of the 'max' end of the bargraph
+ * @param isHorizontal If true, bar graph will be drawn with horizontal bars
+ * @param value Value of the bar, with bars drawn from min up to value, remaining 'backColour' from there to max
+ * @param min,max Scale of the bar graph that value should be found within
+ * @param minColour,maxColour colours at the min and max ends of the bar, drawn in a gradient between the two (16-bit RGB565)
+ * @param backColour background colour of the bar graph (16-bit RGB565)
+ */
+void displayBarGraph(int x0, int y0, int x1, int y1, bool isHorizontal, int value, int min, int max, int minColour, int maxColour, int backColour)
+{
+    int valColour;
+    if (isHorizontal) {
+        if (x1>x0) {
+            for (int i = x0; i < x1; i+=5) {
+                if (map(i, x0, x1, min, max) > value)
+                    valColour = backColour;
+                else
+                    valColour = getMappedColour(i, x0, x1, minColour, maxColour);
+                screen.fillrect(i, y0, i+3, y1, valColour);
+            }
+        } else {
+            for (int i = x1; i < x0; i+=5) {
+                if (map(i, x0, x1, min, max) > value)
+                    valColour = backColour;
+                else
+                    valColour = getMappedColour(i, x0, x1, minColour, maxColour);
+                screen.fillrect(i-3, y0, i, y1, valColour);
+            }
+        }
+    } else {
+        if (y1>y0) {
+            for (int i = y0; i < y1; i+=5) {
+                if (map(i, y0, y1, min, max) > value)
+                    valColour = backColour;
+                else
+                    valColour = getMappedColour(i, y0, y1, minColour, maxColour);
+                screen.fillrect(x0, i, x1, i+3, valColour);
+            }
+        } else {
+            for (int i = y1; i < y0; i+=5) {
+                if (map(i, y0, y1, min, max) > value)
+                    valColour = backColour;
+                else
+                    valColour = getMappedColour(i, y0, y1, minColour, maxColour);
+                screen.fillrect(x0, i-3, x1, i, valColour);
+            }
+        }
+    }
+}
+
+/** Draw a keyboard based on the supplied array of cells
+ *
+ * @param cells the cells to draw
+ * @param rowHighlight off if -1, else highlights the numbered row
+ * @param colHighlight off if -1, else highlights the cell in row/col
+ */
+void drawKeyboard(int rowSize[ROWS], string cells[ROWS][COLS], int colPos[ROWS][COLS], int rowHighlight, int colHighlight)
+{
+    int lineColour = RGBColour(0x20,0xFF,0xE0);
+    screen.foreground(RGBColour(0xE0,0xC0,0x10));
+    screen.set_font((unsigned char*) Arial28x28);
+    for (int i=0; i<ROWS; i++) {
+        int yPos=111+i*32;
+        for (int j=0; j< rowSize[i]; j++) {
+            if (j>0)
+                screen.locate(colPos[i][j-1]+5,84+i*32);
+            else
+                screen.locate(10,84+i*32);
+            screen.printf("%s",cells[i][j]);
+            screen.line(colPos[i][j],yPos-31,colPos[i][j],yPos, lineColour);
+        }
+        screen.line(5, yPos, 315, yPos, lineColour);
+    }
+    screen.line(5, 79, 315, 79, lineColour);
+    screen.line(5, 79, 5, 239, lineColour);
+    if (rowHighlight >= 0) {
+        if (colHighlight >= 0) { //highlight a cell
+            if (colHighlight==0)
+                screen.rect(5,79+rowHighlight*32,colPos[rowHighlight][0],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
+            else
+                screen.rect(colPos[rowHighlight][colHighlight-1],79+rowHighlight*32,colPos[rowHighlight][colHighlight],111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
+        } else { // highlight whole row
+            screen.rect(5,79+rowHighlight*32,315,111+rowHighlight*32,RGBColour(0xFF,0x40,0x40));
+        }
+    }
+}
+
+void drawText()
+{
+    screen.rect(5,35,315,74,RGBColour(0x20,0xFF,0xE0));
+    screen.locate(10,40);
+    screen.foreground(RGBColour(0xE0,0xC0,0x10));
+    screen.set_font((unsigned char*) Arial28x28);
+    screen.printf("%s_  ", typingBuffer);
+}
+
+/** This will be called if you blink.
+ */
+void blinked(void)
+{
+    //draw blink indicator
+    if (quality == 0) {
+        screen.fillrect(313, 13, 317, 17, White);
+    }
+    //select row or cell
+    if (col == -1)
+        col=-2;
+    else {
+        string s = keys[row][col];
+        if (s.compare("Space") == 0)
+            s=" ";
+        if (s.compare("Delete") == 0) {
+            s="";
+            if (typingBuffer.length() > 0) {
+                typingBuffer = typingBuffer.substr(0, typingBuffer.length() -1);
+            }
+        } else if (s.compare("\"SAY!\"") == 0) {
+            say(typingBuffer);
+            screen.fillrect(5,35,315,74,Black);
+            typingBuffer="";
+        } else {
+            typingBuffer.append(s);
+        }
+        row = -1;
+        col = -1;
+        drawText();
+    }
+}
+
+/** This will be called when processed eSense data comes in, about once a second.
+ *
+ * @param poorQuality will be 0 if connections are good, 200 if connections are useless, and somewhere in between if connection dodgy.
+ * @param attention processed percentage denoting focus and attention.  0 to 100
+ * @param meditation processed percentage denoting calmness and serenity.  0 to 100
+ * @param timeSinceLastPacket time since last packet processed, in milliseconds.
+ */
+void eSenseData(int poorQuality, int attention, int meditation, int timeSinceLastPacket)
+{
+    //quality indicator
+    quality=poorQuality;
+    if (poorQuality == 200)
+        screen.fillrect(313, 3, 317, 7, Red);
+    else if (poorQuality == 0)
+        screen.fillrect(313, 3, 317, 7, Green);
+    else
+        screen.fillrect(313, 3, 317, 7, Yellow);
+
+    //minimal eSense bars up at the top of the screen
+    screen.set_font((unsigned char*) Arial12x12);
+    if (attention > 0) {
+        displayBarGraph(200, 5, 310, 15, true, attention, 0, 100, RGBColour(0x10,0x00,0x00), RGBColour(0xFF,0x00,0x00), 0x00);
+        screen.locate(135, 6);
+        screen.foreground(Red);
+        screen.printf("Att: %d ",attention);
+    }
+    if (meditation > 0) {
+        displayBarGraph(200, 18, 310, 28, true, meditation, 0, 100, RGBColour(0x00,0x10,0x00), RGBColour(0x00,0xFF,0x00), 0x00);
+        screen.locate(128, 19);
+        screen.foreground(Green);
+        screen.printf("Med: %d ",meditation);
+    }
+    //clear blink indicator
+    screen.fillrect(313, 13, 317, 17, Black);
+    //Safe to start yet?
+    if (initialDelay.read() == 0)
+        initialDelay.start();
+    else if (initialDelay.read() > 5) {
+        started=true;
+        initialDelay.stop();
+    }
+}
+
+/** This will be called when processed meter reading data arrives, about once a second.
+ * This is a breakdown of frequencies in the wave data into 8 named bands, these are:
+ *   0: Delta        (0.5-2.75 Hz)
+ *   1: Theta        (3.5-6.75 Hz)
+ *   2: Low-Alpha    (7.5-9.25 Hz)
+ *   3: High-Alpha   (10-11.75 Hz)
+ *   4: Low-Beta     (13-16.75 Hz)
+ *   5: High-Beta    (18-29.75 Hz)
+ *   6: Low-Gamma    (31-39.75 Hz)
+ *   7: High-Gamma   (41-49.75 Hz)
+ *
+ * @param meter array of meter data for different frequency bands
+ * @param meterMin array of minimum recorded samples of each band
+ * @param meterMax arrat if naximum recorded samples of each band
+ */
+void meterData(int meter[8], int meterMin[8], int meterMax[8])
+{
+    //first good signal?
+    if (!connected) {
+        connected=true;
+        screen.fillrect(0,0,319,30,Black); //clear the Waiting to connect msg
+        initialDelay.reset();
+        initialDelay.start();
+    }
+    //minimal meter bars up at the top of the screen
+    for (int j=0; j<8; j++) {
+        displayBarGraph(5 + j * 13, 30, 16 + j * 13, 3, false, meter[j], meterMin[j], meterMax[j], RGBColour(0, j*2, 0x10), RGBColour(0, j*32, 0xFF), 0x00);
+    }
+    //Hijack this routine for menu movement
+    if (started) {
+        if (col==-1) {
+            row++;
+            if (row>=ROWS)
+                row=0;
+        } else if (col==-2) {
+            col=0;
+        } else {
+            col++;
+            if (col>=keysRowSize[row])
+                col=-1;
+        }
+        drawKeyboard(keysRowSize, keys, keysColPos, row, col);
+    }
+}
+
+/** This will be called when wave data arrives.
+ * There will be a lot of these, 512 a second, so if you're planning to do anything
+ * here, don't let it take long.  Best not to printf this out as it will just choke.
+ *
+ * param wave Raw wave data point
+ */
+void waveData(int wave)
+{
+}
+
+//*****************
+//End User routines
+//*****************
+
+//System routines to obtain and parse data
+
+/** Simplify serial comms
+ */
+unsigned char ReadOneByte()
+{
+    int ByteRead;
+
+    while(!blueSmirf.readable());
+    ByteRead = blueSmirf.getc();
+
+    return ByteRead;
+}
+
+/** Main loop, sets up and keeps listening for serial
+ */
+int main()
+{
+    //Video setup
+    screen.claim(stdout);        // send stdout to the TFT display
+    screen.background(Black);    // set background to black
+    screen.foreground(White);    // set chars to white
+    screen.cls();                // clear the screen
+    screen.set_orientation(1);
+    screen.set_font((unsigned char*) Arial12x12);
+    screen.locate(5,5);
+    screen.printf("Waiting to connect...");
+
+    drawText();
+    drawKeyboard(keysRowSize, keys, keysColPos, -1, -1);
+
+    //Voice setup
+    voice.baud(9600);
+    voice.printf("\n");
+    while (!voice.readable())
+        ;
+    wait(0.01);
+    voice.getc();
+    say("Welcome to Blink talk.");
+
+    Timer t; //packet timer
+    t.start();
+    Timer blinkTimer; //used for detecting blinks
+    int time;
+    int generatedChecksum = 0;
+    int checksum = 0;
+    int payloadLength = 0;
+    int payloadData[64] = {0};
+    int poorQuality = 0;
+    int attention = 0;
+    int meditation = 0;
+    int wave = 0;
+    int meter[8] = {0};
+    int meterMin[8];
+    int meterMax[8];
+    for (int j = 0; j < 8; j++) {
+        meterMin[j]=99999999;
+        meterMax[j]=-99999999;
+    }
+    bool eSensePacket = false;
+    bool meterPacket = false;
+    bool wavePacket = false;
+
+    blueSmirf.baud(57600);
+    blinkTimer.reset();
+
+    while(1) {
+        // Look for sync bytes
+        if(ReadOneByte() == 170) {
+            if(ReadOneByte() == 170) {
+                //Synchronised to start of packet
+                payloadLength = ReadOneByte();
+                if(payloadLength > 169) //Payload length can not be greater than 169
+                    return;
+
+                generatedChecksum = 0;
+                for(int i = 0; i < payloadLength; i++) {
+                    payloadData[i] = ReadOneByte();            //Read payload into memory
+                    generatedChecksum += payloadData[i];
+                }
+
+                checksum = ReadOneByte();                      //Read checksum byte from stream
+                generatedChecksum = 255 - (generatedChecksum & 0xFF);   //Take one's compliment of generated checksum
+
+                if(checksum == generatedChecksum) {
+                    //Packet seems OK
+                    poorQuality = 200;
+                    attention = 0;
+                    meditation = 0;
+                    wave = 0;
+                    for(int i = 0; i < payloadLength; i++) {    // Parse the payload
+                        switch (payloadData[i]) {
+                            case 2: //quality
+                                i++;
+                                poorQuality = payloadData[i];
+                                eSensePacket = true;
+                                break;
+                            case 4: //attention
+                                i++;
+                                attention = payloadData[i];
+                                eSensePacket = true;
+                                break;
+                            case 5: //meditation
+                                i++;
+                                meditation = payloadData[i];
+                                eSensePacket = true;
+                                break;
+                            case 0x80: //wave
+                                wave = payloadData[i+2] * 256 + payloadData[i+3];
+                                //We also want to try to detect blinks via analysing wave data
+                                time = blinkTimer.read_ms();
+                                if (wave > 32767) wave -= 65535; //cope with negatives
+                                if (wave>200 && time == 0) {
+                                    blinkTimer.start();
+                                } else if (wave<-90 && time > 10 && time < 350) {
+                                    blinkTimer.stop();
+                                    blinkTimer.reset();
+                                    blinked();
+                                } else if (time>500) {
+                                    blinkTimer.stop();
+                                    blinkTimer.reset();
+                                }
+                                i = i + 3;
+                                wavePacket = true;
+                                break;
+                            case 0x83: //meter readings for different frequency bands
+                                for (int j=0; j<8; j++) {
+                                    //documentation is inconsistent about whether these values are big-endian or little-endian,
+                                    //and claims both in different places.  But wave data is big-endian so assuming that here.
+                                    meter[j] = payloadData[i+j*3+2]*65536 + payloadData[i+j*3+3]*256 + payloadData[i+j*3+4];
+                                    if (quality==0) {
+                                        if (meter[j]<meterMin[j])
+                                            meterMin[j]=meter[j];
+                                        if (meter[j]>meterMax[j])
+                                            meterMax[j]=meter[j];
+                                    }
+                                }
+                                meterPacket = true;
+                                i = i + 25;
+                                break;
+                            default:
+                                break;
+                        } // switch
+                    } // for loop
+
+                    //Call routines to process data
+                    if(eSensePacket) {
+                        eSenseData(poorQuality, attention, meditation, t.read_ms());
+                        eSensePacket = false;
+                    }
+                    if (meterPacket) {
+                        meterData(meter, meterMin, meterMax);
+                        t.reset();
+                        meterPacket=false;
+                    }
+                    if (wavePacket) {
+                        waveData(wave);
+                        wavePacket=false;
+                    }
+                } else {
+                    // Checksum Error
+                }  // end if else for checksum
+            } // end if read 0xAA byte
+        } // end if read 0xAA byte
+    } //end while
+}
\ No newline at end of file