Voice Controlled Car
Page last updated 13 Dec 2018, by 
Samuel Gedaly.
0
replies
Samuel Gedaly.
0
replies
Our project consists of an mbed powered wifi controlled car, an iphone app for recording voice and a central server that receives, processes and acts upon a voice command.
Car
The Car contains and mbed, a Huzzah ESP8266 Wi-Fi to create a local server, and two dc-motors controlled through an H-Bridge. The car will receive instructions from the Node.js server to control speed, direction, and rotation.
Wiring
Wiring for H-Bridge
Wiring for Huzzah ESP8266 Wi-Fi
Code
#include "mbed.h"
#include "Motor.h"
#include <string>
Serial pc(USBTX, USBRX);
Serial esp(p28, p27); // tx, rx
DigitalOut reset(p26);
DigitalOut led1(LED1);
DigitalOut led4(LED4);
Timer t;
int count,ended,timeout;
char buf[2024];
char snd[1024];
char ssid[32] = "University House Midtown"; // enter WiFi router ssid inside the quotes
char pwd [32] = "UniHouseATL"; // enter WiFi router password inside the quotes
void SendCMD(),getreply(),ESPconfig(),ESPsetbaudrate(),process_params(string);
Motor lm(p24, p11, p12);
Motor rm(p23, p6, p5);
string complete = "";
bool processed = false;
int run;
int dance;
int rotate;
float lvel = 0, rvel = 0;
void dev_recv()
{
led1 = !led1;
char temp;
while(esp.readable()) {
temp = esp.getc();
complete += temp;
// pc.putc(temp);
}
size_t found = complete.find("abcdef");
if (found != string::npos){
string token = complete.substr(found + 6, complete.length());
size_t found2 = token.find("abcdef");
if (found2 != string::npos){
token = token.substr(0, found2);
complete = "";
found = string::npos;
found2 = string::npos;
//pc.printf("\nNo Exception....\n");
if (processed) {
processed = false;
} else {
pc.printf("\nNo Exception....\n");
processed = true;
process_params(token);
}
//for (int i = 0; i < token.length(); i++) {
// pc.putc(token[i]);
// }
// pc.putc('\n');
}
}
}
void pc_recv()
{
led4 = !led4;
while(pc.readable()) {
esp.putc(pc.getc());
}
}
void process_params(string querystring) {
string str = querystring.substr(2,1) + ", " + querystring.substr(6,3) + ", " + querystring.substr(12,1) + "\n";
for (int i = 0; i < str.length(); i++) {
pc.putc(str[i]);
}
run = atoi(querystring.substr(2,1).c_str());
rotate = atoi(querystring.substr(6,3).c_str());
dance = atoi(querystring.substr(12,1).c_str());
if (run == 1) {
lvel = 0.5f;
rvel = 0.5f;
} else if (run == 0) {
lvel = 0.0f;
rvel = 0.0f;
} else {
lvel = -0.5f;
rvel = -0.5f;
}
if (rotate != 0) {
if (rotate >= 90) {
lvel = -0.5f;
rvel = 0.5f;
lm.speed(lvel);
rm.speed(rvel);
pc.printf("%f", ((float)(rotate)*0.25)/90.0);
wait(((float)(rotate)*0.25)/90.0);
//wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
} else {
lvel = 0.5f;
rvel = -0.5f;
lm.speed(lvel);
rm.speed(rvel);
pc.printf("%f", ((float)(180 - rotate)*0.5)/90.0);
wait(((float)(180 - rotate)*0.5)/90.0);
//wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
}
}
if (dance == 1) {
lvel = -0.5f;
rvel = 0.5f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.3);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.3);
lvel = 0.5f;
rvel = -0.5f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 0.5f;
rvel = 0.5f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = -1.0f;
rvel = -1.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = -1.0f;
rvel = 1.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 0.0f;
rvel = 0.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
lvel = 1.0f;
rvel = -1.0f;
lm.speed(lvel);
rm.speed(rvel);
wait(0.5);
}
}
int main()
{
reset=0; //hardware reset for 8266
pc.baud(9600); // set what you want here depending on your terminal program speed
pc.printf("\f\n\r-------------ESP8266 Hardware Reset-------------\n\r");
wait(0.5);
reset=1;
timeout=2;
getreply();
esp.baud(9600); // change this to the new ESP8266 baudrate if it is changed at any time.
//ESPsetbaudrate(); //** include this routine to set a different ESP8266 baudrate **
ESPconfig(); //** include Config to set the ESP8266 configuration ***
pc.attach(&pc_recv, Serial::RxIrq);
esp.attach(&dev_recv, Serial::RxIrq);
// continuosly get AP list and IP
while(1) {
lm.speed(lvel);
rm.speed(rvel);
sleep();
}
}
// Sets new ESP8266 baurate, change the esp.baud(xxxxx) to match your new setting once this has been executed
void ESPsetbaudrate()
{
strcpy(snd, "AT+CIOBAUD=115200\r\n"); // change the numeric value to the required baudrate
SendCMD();
}
// +++++++++++++++++++++++++++++++++ This is for ESP8266 config only, run this once to set up the ESP8266 +++++++++++++++
void ESPconfig()
{
wait(5);
pc.printf("\f---------- Starting ESP Config ----------\r\n\n");
strcpy(snd,".\r\n.\r\n");
SendCMD();
wait(1);
pc.printf("---------- Reset & get Firmware ----------\r\n");
strcpy(snd,"node.restart()\r\n");
SendCMD();
timeout=5;
getreply();
pc.printf(buf);
wait(2);
pc.printf("\n---------- Get Version ----------\r\n");
strcpy(snd,"print(node.info())\r\n");
SendCMD();
timeout=4;
getreply();
pc.printf(buf);
wait(3);
// set CWMODE to 1=Station,2=AP,3=BOTH, default mode 1 (Station)
pc.printf("\n---------- Setting Mode ----------\r\n");
strcpy(snd, "wifi.setmode(wifi.STATION)\r\n");
SendCMD();
timeout=4;
getreply();
pc.printf(buf);
wait(2);
pc.printf("\n---------- Listing Access Points ----------\r\n");
strcpy(snd, "function listap(t)\r\n");
SendCMD();
wait(1);
strcpy(snd, "for k,v in pairs(t) do\r\n");
SendCMD();
wait(1);
strcpy(snd, "print(k..\" : \"..v)\r\n");
SendCMD();
wait(1);
strcpy(snd, "end\r\n");
SendCMD();
wait(1);
strcpy(snd, "end\r\n");
SendCMD();
wait(1);
strcpy(snd, "wifi.sta.getap(listap)\r\n");
SendCMD();
wait(1);
timeout=15;
getreply();
pc.printf(buf);
wait(2);
pc.printf("\n---------- Connecting to AP ----------\r\n");
pc.printf("ssid = %s pwd = %s\r\n",ssid,pwd);
strcpy(snd, "wifi.sta.config(\"");
strcat(snd, ssid);
strcat(snd, "\",\"");
strcat(snd, pwd);
strcat(snd, "\")\r\n");
SendCMD();
timeout=10;
getreply();
pc.printf(buf);
wait(5);
pc.printf("\n---------- Get IP's ----------\r\n");
strcpy(snd, "print(wifi.sta.getip())\r\n");
SendCMD();
timeout=3;
getreply();
pc.printf(buf);
wait(1);
pc.printf("\n---------- Get Connection Status ----------\r\n");
strcpy(snd, "print(wifi.sta.status())\r\n");
SendCMD();
timeout=5;
getreply();
pc.printf(buf);
pc.printf("\n\n\n If you get a valid (non zero) IP, ESP8266 has been set up.\r\n");
pc.printf(" Run this if you want to reconfig the ESP8266 at any time.\r\n");
pc.printf(" It saves the SSID and password settings internally\r\n");
wait(10);
pc.printf("\n---------- Setting up http server ----------\r\n");
strcpy(snd, "srv=net.createServer(net.TCP)\r\n");
SendCMD();
wait(1);
strcpy(snd, "srv:listen(80,function(conn)\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:on(\"receive\",function(conn,payload)\r\n");
SendCMD();
wait(1);
strcpy(snd, "print(payload)\r\n");
SendCMD();
wait(1);
SendCMD();
wait(1);
strcpy(snd, "conn:send(\"<!DOCTYPE html>\")\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:send(\"<html>\")\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:send(\"<h1> Hi Max Bentata!, NodeMcu.</h1>\")\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:send(\"<h2> test</h2>\")\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:send(\"</html>\")\r\n");
SendCMD();
wait(1);
strcpy(snd, "end)\r\n");
SendCMD();
wait(1);
strcpy(snd, "conn:on(\"sent\",function(conn) conn:close() end)\r\n");
SendCMD();
wait(1);
strcpy(snd, "end)\r\n");
SendCMD();
wait(1);
timeout=17;
getreply();
pc.printf("\r\nDONE");
}
void SendCMD()
{
esp.printf("%s", snd);
}
void getreply()
{
memset(buf, '\0', sizeof(buf));
t.start();
ended=0;
count=0;
while(!ended) {
if(esp.readable()) {
buf[count] = esp.getc();
count++;
}
if(t.read() > timeout) {
ended = 1;
t.stop();
t.reset();
}
}
}
string split() {
string buf2(buf);
string token = buf2.substr(buf2.find("abcdef"), buf2.length());
for (int i = 0; i < token.length(); i++) {
pc.putc(token[i]);
}
//token = token.substr(0, buf2.find("abcdef"));
pc.printf("the other one \n");
//for (int i = 0; i < token.length(); i++) {
// pc.putc(token[i]);
// }
return token;
}
Node.js server
The Node.js server will receive an audio (.wav) file from the Iphone app and then send that file to Google Cloud Speech-to-text API, which will return a text representation of the audio file. Then, the server will create an instruction out of that text and send it to the server in the car through an http POST request.
Code
const express = require('express');
const bodyParser = require('body-parser');
const port = process.env.PORT || 3000;
const app = express();
var multer = require('multer')
const speech = require('@google-cloud/speech');
const fs = require('fs');
const mm = require('music-metadata');
const util = require('util');
var upload = multer({ dest: 'uploads/' });
let axios = require('axios');
app.post('/postAudio', upload.single('file'), function (req, res) {
const client = new speech.SpeechClient();
const audioBytes = fs.readFileSync(req.file.path);
mm.parseBuffer(audioBytes, 'audio/mpeg', { fileSize: audioBytes.fileSize })
.then(metadata => {
console.log(util.inspect(metadata, { showHidden: false, depth: null }));
});
const audio = {
content: audioBytes,
};
const config = {
encoding: 'LINEAR16',
sampleRateHertz: 44100,
languageCode: 'en-US',
};
const request = {
audio: audio,
config: config,
};
// Detects speech in the audio file
client
.recognize(request)
.then(data => {
const response = data[0];
console.log(data);
const transcription = response.results
.map(result => result.alternatives[0].transcript)
.join('\n');
console.log(`Transcription: ${transcription}`);
process_data(transcription);
})
.catch(err => {
console.error('ERROR:', err);
});
res.send(
{
message: 'Hello from server!!',
});
});
app.listen(port, function () {
console.log('Server started!');
});
let process_data = async (sentence) => {
console.log(sentence);
if (sentence.includes("advance") || sentence.includes("straight")) {
try {
let response = await axios.get('http://10.180.247.189?abcdefr=1&d=000&x=0abcdef');
} catch (e) {
console.log(e);
}
}
else if (sentence.includes("stop")) {
try {
let response = await axios.get('http://10.180.247.189?abcdefr=0&d=000&x=0abcdef');
} catch (e) {
console.log(e);
}
}
else if (sentence.includes("turn") || sentence.includes("degrees") || sentence.includes("°")) {
try {
var number = sentence.match(/\d+/g).map(Number);
} catch (e) {
var number = null
}
let angle = 0;
let url;
if (sentence.includes("right")) {
if (number == null) {
url = 'http://10.180.247.189?abcdefr=0&d=001&x=0abcdef';
} else {
angle = 90 - number;
console.log(angle);
url = 'http://10.180.247.189?abcdefr=0&d=0' + ((angle != 0) ? angle.toString() : "01") + '&x=0abcdef';
console.log(url)
}
} else if (sentence.includes("left")) {
if (number == null) {
url = 'http://10.180.247.189?abcdefr=0&d=179&x=0abcdef';
} else {
angle = 180 - (90 - number);
url = 'http://10.180.247.189?abcdefr=0&d=' + ((angle >= 100) ? angle.toString() : ("0" + angle.toString())) + '&x=0abcdef';
console.log(url);
}
}
try {
let response = await axios.get(url);
} catch (e) {
// console.log(e);
}
}
else if (sentence.includes("dance") || sentence.includes("crazy")) {
try {
let response = await axios.get('http://10.180.247.189?abcdefr=0&d=000&x=1abcdef');
} catch (e) {
console.log(e);
}
}
else if (sentence.includes("back") || sentence.includes("backwards")) {
try {
let response = await axios.get('http://10.180.247.189?abcdefr=2&d=000&x=0abcdef');
} catch (e) {
console.log(e);
}
}
}
Iphone App
The Iphone app was created using react native. It is a simple app that let's the user record an audio instruction and it automatically sends it to the Node.js server to be processed.
Code
import React from 'react';
import { StyleSheet, Text, View, TouchableOpacity } from 'react-native';
import { SimpleLineIcons } from '@expo/vector-icons';
import { Audio, FileSystem, Permissions } from 'expo';
export default class App extends React.Component {
constructor(props) {
super(props)
this.recording = null;
this.sound = null;
this.state = {
page: 'first'
}
}
startRecording = async () => {
this.setState({
page: 'second',
});
const response = await Permissions.askAsync(Permissions.AUDIO_RECORDING);
await Audio.setAudioModeAsync({
allowsRecordingIOS: true,
interruptionModeIOS: Audio.INTERRUPTION_MODE_IOS_DO_NOT_MIX,
playsInSilentModeIOS: true,
shouldDuckAndroid: true,
interruptionModeAndroid: Audio.INTERRUPTION_MODE_ANDROID_DO_NOT_MIX,
playThroughEarpieceAndroid: true
});
try {
const recording = new Audio.Recording();
this.recording = recording
await this.recording.prepareToRecordAsync({
android: {
extension: '.3gp',
outputFormat: Expo.Audio.RECORDING_OPTION_ANDROID_OUTPUT_FORMAT_THREE_GPP,
audioEncoder: Expo.Audio.RECORDING_OPTION_ANDROID_AUDIO_ENCODER_AMR_NB,
sampleRate: 44100,
numberOfChannels: 1,
bitRate: 128000,
},
ios: {
extension: '.wav',
//audioQuality: Expo.Audio.RECORDING_OPTION_IOS_AUDIO_QUALITY_MIN,
sampleRate: 44100,
numberOfChannels: 1,
bitRate: 128000,
linearPCMBitDepth: 16,
// linearPCMIsBigEndian: false,
// linearPCMIsFloat: false,
},
});
await this.recording.startAsync();
// You are now recording!
} catch (error) {
alert(error)
}
}
stopRecording = async () => {
this.setState({
page: 'first',
});
try {
await this.recording.stopAndUnloadAsync();
const info = await FileSystem.getInfoAsync(this.recording.getURI());
console.log("hello")
console.log(info.uri)
console.log(`FILE INFO: ${JSON.stringify(info)}`);
const path = info.uri
var name = info.uri.split('AV/')[1]
const formData = new FormData()
formData.append('file', {
uri: path,
name: 'file',
type: 'audio/caf',
})
////////
// await Audio.setAudioModeAsync({
// allowsRecordingIOS: false,
// interruptionModeIOS: Audio.INTERRUPTION_MODE_IOS_DO_NOT_MIX,
// playsInSilentModeIOS: true,
// playsInSilentLockedModeIOS: true,
// shouldDuckAndroid: true,
// interruptionModeAndroid: Audio.INTERRUPTION_MODE_ANDROID_DO_NOT_MIX,
// playThroughEarpieceAndroid: true
// });
// const { sound, status } = await this.recording.createNewLoadedSound(
// {
// isLooping: true,
// isMuted: false,
// volume: 1.0,
// rate: 1.0,
// shouldCorrectPitch: true,
// },
// );
// this.sound = sound;
// //this.sound.playAsync();
/////////
////////
////////
var uploadUrl = 'http://10.180.242.89:3000/postAudio/';
const res = await fetch(uploadUrl, {
method: 'POST',
headers: {
'Content-Type': 'multipart/form-data',
},
body: formData,
})
const json = await res.json()
console.log(json)
} catch (error) {
alert(error)
}
}
render() {
if (this.state.page == 'first') {
return (
<View style={styles.container}>
<Text style={styles.txt}>Voice Controlled Car</Text>
<TouchableOpacity onPress={this.startRecording}>
<SimpleLineIcons name='control-start' color='#3cb371' size={50} />
</TouchableOpacity>
</View>
);
} else if (this.state.page == 'second') {
return (
<View style={styles.container}>
<Text style={styles.txt}>Voice Controlled Car</Text>
<TouchableOpacity onPress={this.stopRecording}>
<SimpleLineIcons name='control-pause' color='#3cb371' size={50} />
</TouchableOpacity>
</View>
);
}
}
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: '#fff',
alignItems: 'center',
justifyContent: 'center',
},
txt: {
color: '#3cb371',
fontSize: 30,
paddingBottom: 100,
},
txt2: {
color: '#3cb371',
fontSize: 12,
},
});
Please log in to post comments.