Integrating Streamdata.io with node.js

Simple integration of Streamdata.io with node.js

Integrating Streamdata.io with node.js or any runtime environment is simple.

We already demonstrated such integration with Android, iOS or various Javascript environments like Angular.js, Riot.js in previous posts.

These are just a few examples, you’ll find more by checking out our blog.

But let’s focus on how to leverage your Node.js runtime environment with Streamdata.io.

Prerequisites

If you’re using Node.js you’re probably familiar with node package manager: nmp. npm comes along with Node.js.

As a prerequisite, we assume that you have Node.js and npm installed on your system. If this is not the case, you can find it here. You’ll need version 0.11.5 or above of Node to run the sample.

In order to use the streamdata.io service, you need a valid App Token. Register on our Portal to create an account and get a valid token.

The streamdata.io request URL looks like this:

https://streamdata.motwin.net/https://my.api.com/service?X-Sd-Token=[YOURTOKEN]

Step by step Node.js tutorial

Step 1: node dependencies

Streamdata.io uses HTTP/Web standards to stream data from your favorite API to your application.

We do it using the standard HTML5 Server-Sent Events to easily integrate with browsers and any HTTP environment. This suits also perfectly with Node.js. Node.js has various implementations of the HTML5 EventSource object. You’ll find them on the https://www.npmjs.com/ site.

In this tutorial we will use: npm evensource implementation.

We stream data efficiently, meaning rather than streaming the whole Json data provided by your favorite API, we calculate only the changes you need and send those changes in the form of a patch.
We do it using the standard JSON-Patch format (here is the rfc). Again, there are multiple implementations of this standard for Node.js which can be found on the https://www.npmjs.com/ site.

Here is the second and last dependency you’ll need to integrate Streamdata.io with Node.js: npm fast json patch implementation.

Lets write the package.json file we need for our tiny example.

It should look like this:

{
  "name": "streamdataio-nodejs",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "start": "node index.js",
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "Apache 2 License",
  "dependencies": {
    "eventsource": "^0.1.6",
    "fast-json-patch": "^0.5.4"
  }
}

Step 2: connect to Streamdata.io

Here is the canvas of a basic application connecting to your API through Streamdata.io proxy.

Let’s edit a server.js file as follow:

// add EventSource dependency
var EventSource = require('eventsource');

// add json patch dependency
var JsonPatch   = require('fast-json-patch');

function server() {
  // define variables
  var eventSource = null;

  function connect() {

    // building the URL to get your API streamed as follow:

    // targetUrl is the JSON API you wish to stream
    // you can use this example API which simulates updating stocks prices from a financial market
    var targetUrl = 'http://stockmarket.streamdata.io/prices';

    // appToken is the way Streamdata.io authenticates you as a valid user.
    // you MUST provide a valid token for your request to go through.
    var appToken = '[YOUR_STREAMDATAIO_APP_TOKEN]';

    // finally the url you will request is composed as follow
    // the call to your target API is made through https://streamdata.motwin.net/ proxy
    var url = 'https://streamdata.motwin.net/' + targetUrl + '?X-Sd-Token=' + appToken;

    // simply use the eventsource API to get connected
    var eventSource = new EventSource(url);
  }

  connect();
}

console.log('starting');
server();

Step 3: add callbacks to get the data stream

Let’s add callbacks to our a server.js file as follows:

// add EventSource dependency
var EventSource = require('eventsource');
// add json patch dependency
var JsonPatch   = require('fast-json-patch');

function server() {
  // define variables
  var eventSource = null;

  // initialize data 
  var data = [];

  function connect() {

    // building the URL to get your API streamed as follow:

    // targetUrl is the JSON API you wish to stream
    // you can use this example API which simulates updating stocks prices from a financial market
    var targetUrl = 'http://stockmarket.streamdata.io/prices';

    // appToken is the way Streamdata.io authenticates you as a valid user.
    // you MUST provide a valid token for your request to go through.
    var appToken = '[YOUR_STREAMDATAIO_APP_TOKEN]';

    // finally the url you will request is composed as follow
    // the call to your target API is made through https://streamdata.motwin.net/ proxy
    var url = 'https://streamdata.motwin.net/' + targetUrl + '?X-Sd-Token=' + appToken;

    // simply use the eventsource API to get connected
    var eventSource = new EventSource(url);


    // add callbacks to react to EventSource events :

    // the standard 'open' callback will be called when connection is established with the server
    eventSource.addEventListener('open', function() {
    	console.log("connected!");
    });

    // the standard 'error' callback will be called when an error occurs with the evenSource
    // for example with an invalid token provided
    eventSource.addEventListener('error', function(e) {
      console.log('ERROR!' + e);
      eventSource.close();
    });

    // the streamdata.io specific 'data' event will be called when a fresh Json data set 
    // is pushed by Streamdata.io coming from the API
    eventSource.addEventListener('data', function(e) {
      console.log(e.data);
      // memorize the fresh data set
      data = JSON.parse(e.data);
    });

    // the streamdata.io specific 'patch' event will be called when a fresh Json patch 
    // is pushed by streamdata.io from the API. This patch has to be applied to the 
    // latest data set provided.
    eventSource.addEventListener('patch', function(e) {
      // display the patch
      console.log(e.data);
      // apply the patch to data using json patch API
      JsonPatch.apply(data, JSON.parse(e.data));
      // do whatever you wish with the update data
      // console.log(data);
    });
  }

  connect();
}

console.log('starting');
server();

Once connected, you can use the two Streamdata.io specific Events to play with data through event listener callbacks as described above.

Each time a ‘data’ Event is received, the whole data set is provided. It will happen the first time you receive data.

Each time a ‘patch’ Event is received, a patch applicable to the latest data set is provided.

Step 4: run it

First download dependencies by running npm install :

>npm install
fast-json-patch@0.5.4 node_modules/fast-json-patch
eventsource@0.1.6 node_modules/eventsource
└── original@0.0.8 (url-parse@0.2.3)

NOTE : Be sure you’ve copied your token into the server.js file.

Then run the sample application:

>node server.js

starting
connected!
data: 
[{"title":"Value 0","price":34,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 1","price":12,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 2","price":82,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 3","price":64,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 4","price":42,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 5","price":41,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 6","price":94,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 7","price":27,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 8","price":42,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 9","price":87,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 10","price":79,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 11","price":76,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 12","price":75,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 13","price":82,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"},{"title":"Value 14","price":93,"param1":"value1","param2":"value2","param3":"value3","param4":"value4","param5":"value5","param6":"value6","param7":"value7","param8":"value8"}]
patch: 
[{"op":"replace","path":"/0/price","value":22},{"op":"replace","path":"/1/price","value":52},{"op":"replace","path":"/3/price","value":28},{"op":"replace","path":"/4/price","value":4},{"op":"replace","path":"/7/price","value":70},{"op":"replace","path":"/8/price","value":81},{"op":"replace","path":"/12/price","value":30},{"op":"replace","path":"/13/price","value":40}]
patch: 
[{"op":"replace","path":"/0/price","value":79},{"op":"replace","path":"/1/price","value":38},{"op":"replace","path":"/2/price","value":32},{"op":"replace","path":"/4/price","value":20},{"op":"replace","path":"/7/price","value":27},{"op":"replace","path":"/8/price","value":24},{"op":"replace","path":"/12/price","value":52},{"op":"replace","path":"/13/price","value":61},{"op":"replace","path":"/14/price","value":36}]

Conclusion

In this article, we have learned how to use the Streamdata.io with an Node.js event source to stream the content of a frequently updated API, and handle those updates through callbacks in a simple Node.js application.

Hope this helps you figure out the simplicity of using Stremadata.io with Node.

You can download the full source code of this tiny example on Github here.

Enjoy !

Share it :
0020

Give it a try!

Try streaming any JSON REST API within 30 sec
curl -v "https://proxy.streamdata.io/http://mysite.com/myJsonRestService?param1=[]&param2=[]"

Leave a Reply

Your email address will not be published. Required fields are marked *