Ejemplos de webhdfs createClient en JavaScript

Ejemplo n.º 1

Archivo: memory.js Proyecto: harrisiirak/webhdfs-proxy-memory

  before(function (done) {
    client = WebHDFS.createClient({
      user: '******',
      host: 'localhost',
      port: 45000,
      path: '/webhdfs/v1'
    });

    proxyServer = WebHDFSProxy.createServer(opts, handler, done);
  });

Ejemplo n.º 2

Archivo: hdfs.js Proyecto: ripple/rippled-historical-database

var config = require('../config')
var WebHDFS = require('webhdfs')
var hdfs = WebHDFS.createClient(config.get('hdfs'))
var Logger = require('./logger');
var moment = require('moment')
var EPOCH_OFFSET = 946684800

var log = new Logger({
  scope: 'hdfs',
  level: config.get('logLevel') || 3,
  file: config.get('logFile')
})

var ledgerFields = [
  'new_fields',
  'hbase_key',
  'ledger_hash',
  'ledger_index',
  'executed_time',
  'transaction_hash',
  'hash',
  'parent_hash',
  'accepted',
  'closed',
  'account_hash',
  'close_time_resolution',
  'rowkey',
  'close_time_human',
  'total_coins',
  'totalCoins',
  'seqNum',

Ejemplo n.º 3

Archivo: app.js Proyecto: xushvai/Doopbox

var breadcrumbs = require('express-breadcrumbs');
var swig = require('swig');
var bodyParser = require('body-parser')
var mysql      = require('mysql');
var fileUpload = require('express-fileupload');
var mkdirp = require('mkdirp');
var fs = require('fs');

var app = express();

var svrHost = 'master.hadoop';
var svrPort = 50070;

var hdfs = webhdfs.createClient({
	user: '',
	host: svrHost,
	port: svrPort
});

// config 
app.use(bodyParser.urlencoded());
app.use(bodyParser.json());
app.use('/assets', express.static(__dirname + '/assets'));
app.use(fileUpload());

app.use(breadcrumbs.init());
//app.use(breadcrumbs.setHome());
app.use('/admin', breadcrumbs.setHome({
  name: 'Dashboard',
  url: '/admin'
}));

Ejemplo n.º 4

Archivo: github-fetcher.js Proyecto: gordongovan/github-fetcher

    console.error(err)
  });
  remoteFileStream.on('finish', function onFinish () {
    console.log("Written " + buffer.length + " commits to " + fn)
  });

  fs.createReadStream(fn).pipe(remoteFileStream);

  commitBuffer = [];
}

// HDFS bit ///////////////////////////////////////////////////////////////////

var hdfs = WebHDFS.createClient({
  user: '******',
  host: 'hadoop-name-node',
  port: 50070
});


// Run Bit ////////////////////////////////////////////////////////////////////

function getEvents(){
  makeRequest("https://api.github.com/events?")
    .then(function(success){
      var pushEvents = success.body.filter(function(event){return event.type==="PushEvent"})
      var commits = flatMap(pushEvents,function(pe){return pe.payload.commits.map(function(c){return c.url})})

      console.log("Collected " + pushEvents.length + " push events with " + commits.length + " commits");

      pushEvents.forEach(addEvent);

Ejemplo n.º 5

Archivo: hdfs-upload.js Proyecto: Kundjanasith/All_graph

var WebHDFS = require('webhdfs');
var hdfs = WebHDFS.createClient();
var fs = require('fs')

var localFilePath = "stupidfile.txt";
var remoteFilePath = "/user/cloudera/doesthiswork.txt";

var localFileStream = fs.createReadStream(localFilePath);
var remoteFileStream = hdfs.createWriteStream(remoteFilePath);

localFileStream.pipe(remoteFileStream);

console.log("opening stream to HDFS");

remoteFileStream.on('error', function onError (err) {
  // Do something with the error
console.log("it failed");
console.log(err);
});

remoteFileStream.on('finish', function onFinish () {
  // Upload is done
console.log("it is done!");
});