Project Status | Finished |
Project Type | Personal / Family |
Project Duration | 2 months |
Software Used | Visual Studio, Ubuntu |
Languages Used | Node.js |
Ever since Natural-language processing and Smart Home devices became the juggernaut they are today with several big tech companies competing in an arms race I wanted to join in on making my home smarter.
My two main goals were:
The aim was to not acquire several new third-party devices, but to use what was already available to me. Amazon Alexa became my personal assistant of choice for this task due its widespread availability and advanced developer tools that are already in a mature state.
A quick look around the market revealed there was no off-the-shelf solution to fully control a Samsung TV either - everything either relied on HDMI-CEC or infrared blasting. The former required additional expensive devices and the latter is a hopelessly outdated and dirty solution.
They both also have in common that they can only replace the remote, but I wanted to go even further by manipulating the entire TV including its smart apps.
So I went ahead and made my own Alexa skills that not only replace the remote, but beyond that can also set the volume directly to any value in one step, launch any smart app installed on the device, query the webbrowser for information, search videos on YouTube, play Spotify content using the free edition, watch series and movies on Netflix and Amazon Prime all using voice inputs.
This was achieved entirely by reverse engineering the network protocols used by my Samsung Smart TV and its various companion apps to enable me to transmit my own commands to it.
A Raspberry Pi server at home is listening for instructions sent from AWS Lambda which is hosting the skill code written in Node.js. Once received the Pi routes the commands into my local network and to the TV and other appliances. WoWLAN (Wake on Wireless LAN) packages can be dispatched to wake devices on demand as well.
The Raspberry Pi server at home is responsible for routing the tasks received from AWS Lambda to my appliances. To this end it is running a simple Node.js WebSocket server listening on a port for incoming connections.
To be able to control my TV I had to reverse engineer the Samsung smartphone companion app. This helped me to understand what abilities and protocols the TV supported and how to take advantage of them to achieve what I wanted.
server.js is running on the Pi which receives incoming commands from AWS Lambda and translates them into actions for either my Samsung Smart TV or thermostats connected to a FRITZ!Box router:
'use strict'; var WebSocket = require('ws'); var wol = require('wake_on_lan'); var request = require('request'); var fritz = require('fritzapi'); var upnpClient = require('upnp-device-client'); var tvConfig = { friendlyName: "RaspberryAlexaTVControl", friendlyName64: "", macAddress: "40:16:3B:EA:C7:DC", ipAddress: "192.168.178.2", apiTimeout: 1000, } var fritzOptions = { url: "http://192.168.178.1", strictSSL: false //Workaround for DEPTH_ZERO_SELF_SIGNED_CERT SSL error } var tvAppList = { netflix: "11101200001", amazon: "3201512006785", spotify: "3201606009684", sky: "3201411000562", youtube: "111299001912", maxdome: "3201506003123", ard: "3201412000679", zdf: "3201705012365", prosieben: "3201608010221", maxx: "3201608010226", sixx: "3201608010224", n24: "111477001150", ntv: "3201508004843", clipfish: "3201507004027", netzkino: "111299001605", disneyChannel: "111477001366", sat1: "3201608010222", kabelEins: "3201608010223", browser: "org.tizen.browser" } var tvSocket; function wake(done) { wol.wake(tvConfig.macAddress, function (error) { if (error) { done(1); } else { done(0); } }); }; function sendRemoteKey(key, done) { var cmd = { method: "ms.remote.control", params: { Cmd: "Click", DataOfCmd: key, Option: "false", TypeOfRemote: "SendRemoteKey", to: "host" } }; tvSocket.send(JSON.stringify(cmd)); done(0); }; function isApiActive(done) { request.get({ url: 'http://' + tvConfig.ipAddress + ':8001/api/v2/', timeout: tvConfig.apiTimeout }, function (err, res, body) { if (!err && res.statusCode === 200) { console.log('TV API is active'); done(true); } else { console.log('No response from TV'); done(false); } }); }; function socketCmd(skipWait, actionCmd) { if (actionCmd == "WOL") return; if (!skipWait) { isApiActive(function (done) { if (!done) { //TV is still off after 1 second, try again socketCmd(false, actionCmd); } else { console.log("its on, connect to socket"); socketCmd(true, actionCmd); } }); } else { if (actionCmd.startsWith("upnpCmd=")) { //UPNP Command let actualCmd = actionCmd.substr(actionCmd.indexOf('=') + 1); console.log("connect to upnp API now with cmd: " + actualCmd); //Instanciate a client with a device description URL var client = new upnpClient('http://' + tvConfig.ipAddress + ':9197/dmr'); if (actualCmd.startsWith("adjustVolume=")) { let adjustValue = actualCmd.substr(actualCmd.indexOf('=') + 1); client.callAction('RenderingControl', 'GetVolume', { InstanceID: '0', Channel: 'Master' }, function (err, volume) { if (err) throw err; let newVolume = parseInt(volume.CurrentVolume) + parseInt(adjustValue); if (newVolume < 0) newVolume = 0; if (newVolume > 100) newVolume = 100; console.log("Current Volume: " + volume.CurrentVolume + " ; New Volume: " + newVolume); client.callAction('RenderingControl', 'SetVolume', { InstanceID: '0', Channel: 'Master', DesiredVolume: newVolume }, function (err) { if (err) throw err; console.log("New Volume set"); }); }); } else if (actualCmd.startsWith("setVolume=")) { let setValue = actualCmd.substr(actualCmd.indexOf('=') + 1); let newVolume = parseInt(setValue); if (newVolume < 0) newVolume = 0; if (newVolume > 100) newVolume = 100; console.log("New Volume: " + newVolume); client.callAction('RenderingControl', 'SetVolume', { InstanceID: '0', Channel: 'Master', DesiredVolume: newVolume }, function (err) { if (err) throw err; console.log("New Volume set"); }); } else if (actualCmd.startsWith("setMute=")) { let setMute = actualCmd.substr(actualCmd.indexOf('=') + 1); let newMute = parseInt(setMute); if (newMute != 0 && newMute != 1) return; console.log("New Mute Status: " + newMute); client.callAction('RenderingControl', 'SetMute', { InstanceID: '0', Channel: 'Master', DesiredMute: newMute }, function (err) { if (err) throw err; console.log("New Mute status set"); }); } } else { //Socket Command console.log("connect to API socket now with cmd: " + actionCmd); tvSocket = new WebSocket('http://' + tvConfig.ipAddress + ':8001/api/v2/channels/samsung.remote.control?name=' + tvConfig.friendlyName64, function (error) { console.log(new Error(error).toString()); }); tvSocket.on('error', function (e) { console.log('Error in WebSocket communication'); tvSocket.close(); }); tvSocket.on('message', function (data, flags) { data = JSON.parse(data); if (flags) console.log("incoming message: " + data.toString() + " flags: " + flags.toString()); else console.log("incoming message: ", data); if (data.event == "ms.channel.clientConnect") { //Initial connect console.log("attributes: ", data.data[1].attributes); } else if (data.event == "ms.channel.connect") { //Connected to socket as client if (actionCmd.startsWith("[REPEAT=")) { let repeatCount = actionCmd.substr(actionCmd.indexOf("=") + 1); let delayPerPress = repeatCount.substr(repeatCount.indexOf(";") + 1); let actionJSON = delayPerPress.substr(delayPerPress.indexOf("]") + 1); let repeatCountNum = parseInt(repeatCount.substr(0, repeatCount.indexOf(";"))); let delayPerPressNum = parseInt(delayPerPress.substr(0, delayPerPress.indexOf("]"))); console.log("Execute command " + actionJSON + " " + repeatCountNum.toString() + " times! The delay between each call is " + delayPerPressNum.toString() + " ms!"); let count = 0; let timer = setInterval(() => { if (count == repeatCountNum) { clearInterval(timer); setTimeout(() => { tvSocket.close(); //delayed so cmds can successfully arrive console.log("All Cmds send. Shut down TV socket!"); }, 3000); return; } count++; tvSocket.send(actionJSON); }, delayPerPressNum); //800 ms for volume keys, 1000 ms for channel numbers } else { console.log("Just execute cmd: " + actionCmd); tvSocket.send(actionCmd); setTimeout(() => { tvSocket.close(); //delayed so cmd can successfully arrive console.log("Cmd send. Shut down TV socket!"); }, 5000); } } else if (data.event == "ed.installedApp.get") { //return of installed apps list console.log("installed app return, close socket"); tvSocket.close(); data.data.data.forEach(app => { //log app info console.log(app); }); } }); } } } function setNewPowerState(on) { return new Promise((resolve, reject) => { console.log('try to set TV to: ' + on); if (on) { //turn on console.log('attempting wake'); wake(function (err) { if (err) { reject(new Error(err)); return; } else { //Command has been successfully transmitted to TV console.log('wake request sent successfully'); resolve("turned on"); return; } }); } else { //turn off console.log('sending power key'); sendRemoteKey('KEY_POWER', function (err) { if (err) { reject(new Error(err)); return; } else { //TV is turning off console.log('successfully powered off tv'); resolve("turned off"); return; } }); } }); } function establishConnectionToTV(cmd) { return new Promise((resolve, reject) => { console.log('try to establish TV connection. check status'); isApiActive(function (done) { var skipWait; var powerPromise; if (!done) { if (cmd.includes("ms.remote.control") || cmd.startsWith("upnpCmd=")) { console.log("Remote control command received, but the TV is off!"); resolve("tvOff"); return; } console.log("api is not available, turn on tv with wol"); powerPromise = setNewPowerState(true); skipWait = false; } else { console.log("api is already available"); skipWait = true; } if (!skipWait) { Promise.all([powerPromise]) .then(() => { console.log("All promises resolved. TV is turning on"); resolve("turningOn"); }); } else { console.log("api check result: TV was already on"); resolve("alreadyOn"); } }); }); } function api2Temp(param) { if (param == 254) return 'on'; else if (param == 253) return 'off'; else { //Accuracy: 0.5°C return (parseFloat(param) - 16) / 2 + 8; } } function main() { //Convert friendly name to base64 so the TV can display it when showing the pairing screen tvConfig.friendlyName64 = new Buffer(tvConfig.friendlyName, 'utf8').toString('base64'); //Temperature check var cachedSid; var cachedHealth; var cachedState; var cachedTemp; var cachedTempTarget; var thermostat_kitchen_aim = "119600650576"; //Query thermostat info every minute setInterval(() => { fritz.getSessionID("XXXX", "XXXX", fritzOptions).then(function (sid) { cachedSid = sid; fritz.getDevice(sid, thermostat_kitchen_aim, fritzOptions).then(function (device) { let present = device.present; if (present == '0') { cachedHealth = "UNREACHABLE"; console.log("[WARNING] Kitchen thermostat is disconnected!!"); } else { let currTemp = parseFloat(device.temperature.celsius) / 10; let setTemp = api2Temp(device.hkr.tsoll); cachedHealth = "OK"; cachedTemp = currTemp; if (setTemp == 'off') { cachedState = "OFF"; console.log("Kitchen thermostat is turned off"); console.log("Curr Temp: " + currTemp + " ; Set Temp: " + setTemp); } else if (setTemp == 'on') { cachedState = "ON"; cachedTempTarget = currTemp; console.log("Kitchen thermostat is turned on, no set temp"); console.log("Curr Temp: " + currTemp + " ; Set Temp: " + setTemp); } else { cachedState = "ON"; cachedTempTarget = setTemp; console.log("Kitchen thermostat is turned on"); console.log("Curr Temp: " + currTemp + " ; Set Temp: " + setTemp); } } }); }); }, 60000); //WebSocket Server for AWS Lambda const wss = new WebSocket.Server({ port: 9090 }); console.log("Started Websocket server on port 9090!"); wss.on('connection', function (ws, req) { const ip = req.connection.remoteAddress; console.log("[SERVER] New connection established from: " + ip); ws.on('message', function (msg) { //Response from AWS Lambda to initial request console.log('[SERVER] Received Msg: %s', msg); if (msg.startsWith("[TVCMD]")) { //Samsung TV Command let cmd = msg.substr(msg.indexOf('=') + 1); establishConnectionToTV(cmd).then((successMessage) => { console.log("api check result: " + successMessage); setTimeout(() => { ws.close(); //delay so command END (see below) can arrive first }, 3000); if (successMessage == "turningOn") { ws.send("[END]=Delay"); socketCmd(false, cmd); } else if (successMessage == "tvOff") { ws.send("[END]=tvOff"); } else { ws.send("[END]=Done"); socketCmd(true, cmd); } }); } else if (msg.startsWith("[FRITZCMD]")) { //Command for thermostats let cmd = msg.substr(msg.indexOf('=') + 1); let cmdJSON = JSON.parse(cmd); if (cmdJSON.cmd == "turnOn") { fritz.setTempTarget(cachedSid, cmdJSON.id, 'ON', fritzOptions).then(function (setTemp) { let callback; if (setTemp === "ON") { console.log("turned on successfully"); callback = { result: "success" }; } else { callback = { result: "error" }; } ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); //delay so command END can arrive first }, 3000); }); } else if (cmdJSON.cmd == "turnOff") { fritz.setTempTarget(cachedSid, cmdJSON.id, 'OFF', fritzOptions).then(function (setTemp) { let callback; if (setTemp === "OFF") { console.log("turned off successfully"); callback = { result: "success" }; } else { callback = { result: "error" }; } ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); }, 3000); }); } else if (cmdJSON.cmd == "setTemp") { fritz.setTempTarget(cachedSid, cmdJSON.id, cmdJSON.target, fritzOptions).then(function (setTemp) { let callback; if (setTemp == cmdJSON.target) { console.log("set temp successfully"); callback = { result: "success", target: cmdJSON.target }; } else { callback = { result: "error" }; } ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); }, 3000); }); } else if (cmdJSON.cmd == "adjustTemp") { fritz.getSessionID("XXX", "XXX", fritzOptions).then(function (sid) { //New session cause we need the current temperature value NOW console.log(sid); fritz.getTempTarget(sid, cmdJSON.id, fritzOptions).then(function (temp) { if (isNaN(temp)) { //when temp target is set to On or Off can not compute the delta properly, return an error let callback = { result: "error" }; ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); }, 3000); return; } var newTarget = temp + cmdJSON.delta; fritz.setTempTarget(sid, cmdJSON.id, newTarget, fritzOptions).then(function (setTemp) { let callback; if (setTemp == newTarget) { console.log("adjusted temp successfully"); callback = { result: "success", target: newTarget }; } else { callback = { result: "error" }; } ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); }, 3000); }); }); }); } else if (cmdJSON.cmd == "getStats") { let callback; if (cachedHealth == "UNREACHABLE") { callback = { result: "error", connectivity: cachedHealth }; } else { callback = { result: "success", connectivity: cachedHealth, powerState: cachedState, temperature: cachedTemp, targetSetpoint: cachedTempTarget }; } ws.send("[END]=" + JSON.stringify(callback)); setTimeout(() => { ws.close(); }, 3000); } } }); ws.send("[REQUEST]"); //Send command request to AWS Lambda }); } main();
In order to launch Netflix and Amazon shows and movies as well as Spotify content on the Samsung TV it was necessary to obtain the internal id of the title in question.
If the user, let's say, requests a certain song to play the skill code will then look it up by querying the Spotify API and, if found, retrieve the unique content identifier. At last the id gets embedded in a command to the Smart TV which will open the desired app and turn the id over as a launch parameter. The app then handles opening/playing the desired content.
To allow a natural conversation with the skill it was paramount to create an in-depth interaction model first:
The skill, supporting both German and English, allows the user to formulate his commands in various fashions with the help of many sample utterances. I did not want the skill to be too rigid.
This snippet mainly shows off an API call to a third-party content aggregator for Netflix titles since it doesn't have its own API. The skill code running on AWS Lambda queries the API to retrieve the unique id of a series or movie the user asks to play after saying its full name to an Alexa-enabled device:
function launchAppWithContent(appName, contentType, contentURI) { var tvCmd = { method: "ms.channel.emit", params: { event: "ed.apps.launch", data: { appId: appList[appName] }, to: "host" } }; console.log("Launch app: " + appName + " with type: " + contentType + " and play title: " + contentURI); if (appName == "browser") { //Browser search (use Google) tvCmd.params.data.action_type = "NATIVE_LAUNCH"; let searchParam = "https://www.google.de/search?q="; searchParam = searchParam.concat(contentURI.replace(" ", "+").replace(/ /g, "+")); //replace spaces and underscores to have a clean link searchParam = searchParam.concat("&oe=utf8&hl=de"); tvCmd.params.data.metaTag = searchParam; transmitCommandToPi(JSON.stringify(tvCmd)).then((successMsg, error) => { //Sends command to Raspberry Pi server at home which relays it to the TV if (error) { alexaRequest.response.speak(langPhrases["errorRaspberryPi"]); alexaRequest.emit(':responseReady'); } else { if (successMsg == "Delay") { alexaRequest.response.speak(util.format(langPhrases["launchBrowserSearchDelayed"], contentURI)); alexaRequest.emit(':responseReady'); } else { alexaRequest.response.speak(util.format(langPhrases["launchBrowserSearch"], contentURI)); alexaRequest.emit(':responseReady'); } } }); } else { //Smart Apps tvCmd.params.data.action_type = "DEEP_LINK"; tvCmd.params.data.checkUpdate = false; if (appName == "netflix" || appName == "amazon" || appName == "youtube") { //netflix/amazon/youtube have special deepLink tag, the rest is webapp tvCmd.params.data.deepLink = appName; } else { tvCmd.params.data.deepLink = "webapp"; } if (appName == "netflix") { //Query JustWatch.com API for Netflix title var header = { 'User-Agent': 'Alexa Samsung Skill' }; var type; if (contentType == "film") type = "movie"; else type = "show"; var options = { uri: 'https://api.justwatch.com/titles/de_DE/popular', method: 'POST', headers: header, json: { "query": contentURI, "content_types": [type], //"show", "movie" "providers": ["nfx"], //nfx for Netflix "page": 0, "page_size": 1 //only return a single page worth of results } }; console.log("Access JustWatch API for Netflix"); request(options, function (error, response, body) { if (!error && response.statusCode == 200) { let contentID = ""; body.items.find(item => { if (stringSimilarity.compareTwoStrings(item.title, contentURI) > 0.8 || stringSimilarity.compareTwoStrings(item.original_title, contentURI) > 0.8) { //Ensure the result is similar to the title the user requested //console.log("found title: " + item); item.offers.find(offer => { if (offer.provider_id == 8) { //8 Netflix, 9 Amazon Prime contentID = offer.urls.standard_web.substr(offer.urls.standard_web.lastIndexOf('/') + 1); return true; } return false; }); return true; } return false; }); if (contentID == "") { alexaRequest.response.speak(util.format(langPhrases["contentNotFound"], appName)); alexaRequest.emit(':responseReady'); } else { tvCmd.params.data.metaTag = "m=" + contentID + "&&source_type_payload=groupIndex%3D1%26tileIndex%3D0%26action%3Dplayback%26movieId%3D" + contentID; transmitCommandToPi(JSON.stringify(tvCmd)).then((successMsg, error) => { //Sends command to Raspberry Pi server at home which relays it to the TV if (error) { alexaRequest.response.speak(langPhrases["errorRaspberryPi"]); alexaRequest.emit(':responseReady'); } else { if (successMsg == "Delay") { alexaRequest.response.speak(util.format(langPhrases["launchAppContentDelayed"], contentType, contentURI, appName)); alexaRequest.emit(':responseReady'); } else { alexaRequest.response.speak(util.format(langPhrases["launchAppContent"], contentType, contentURI, appName)); alexaRequest.emit(':responseReady'); } } }); } } }); } } }