function createContext(ip, port, checkpointDirectory, outputPath) {
// If you do not see this printed, that means the StreamingContext has been loaded
// from the new checkpoint
print("Creating new context");
// Create the context with a 1 second batch size
var conf = new SparkConf().setAppName("Javascript Recoverable Network Word Count");
var ssc = new StreamingContext(conf, new Duration(1000));
ssc.checkpoint(checkpointDirectory);
// Create a socket stream on target ip:port and count the
// words in input stream of \n delimited text (eg. generated by 'nc')
var lines = ssc.socketTextStream(ip, port);
var words = lines.flatMap(function (x) {
return x.split(/\s+/);
});
var wordCounts = words.mapToPair(
function (s, Tuple) {
return new Tuple(s, 1);
}, [Tuple]).reduceByKey(function (i1, i2) {
return i1 + i2;
});
wordCounts.foreachRDDWithTime(function (rdd, time) {
var counts = "Counts sat time " + time + " " + rdd.collect();
print(counts);
return null;
});
return ssc;
}
var hostname = ((typeof args !== "undefined") && (args.length > 1)) ? args[1] : "localhost";
var port = ((typeof args !== "undefined") && (args.length > 2)) ? 0 + args[2] : 9999;
// Create the context with a 1 second batch size
var StorageLevel = require('eclairjs/storage/StorageLevel');
var conf = new SparkConf().setAppName("Javascript Network Word Count");
var ssc = new StreamingContext(conf, new Duration(2000));
// Create a JavaReceiverInputDStream on target ip:port and count the
// words in input stream of \n delimited text (eg. generated by 'nc')
// Note that no duplication in storage level only for running locally.
// Replication necessary in distributed scenario for fault tolerance.
var lines = ssc.socketTextStream(
hostname, port, StorageLevel.MEMORY_AND_DISK_2);
// "localhost", 9999, StorageLevel.MEMORY_AND_DISK_2);
var words = lines.flatMap(function(x) {
return x.split(/\s+/);
});
var wordCounts = words.mapToPair( function(s, Tuple) {
return new Tuple(s, 1);
}, [Tuple]).reduceByKey(function(i1,i2) {
return i1 + i2;
});
wordCounts.print();
ssc.start();
ssc.awaitTermination();